upstream/upstream-jitsi-videobridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

snapshot of https://github.com/jitsi/jitsi-videobridge.git @ fcbf741c646bc237d2ed85a1e86910532532b831 (2026-09-15)
Some checks failed
Java CI with Maven / Java 17 (push) Failing after 7m2s
Details
Java CI with Maven / Java 21 (push) Failing after 5m0s
Details

Browse Source
This commit is contained in:
upstream-mirror
2026-09-15 16:12:38 +05:00
commit e1257b7a05
647 changed files with 110946 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
jitsi-media-transform/README.md Normal file
View File

@@ -0,0 +1,21 @@
# Jitsi Media Transform
Jitsi Media Transform contains classes for processing and transforming RTP and RTCP packets
# Code style
We use [ktlint](https://pinterest.github.io/ktlint/) for linting and autoformatting. It runs as part of
`mvn verify` (the build fails on style violations), and the version used is pinned by `ktlint.version` in the
root `pom.xml`, so no separate installation is needed.
To run only the check, or to autoformat, run these in this module's directory:
```
mvn exec:exec@ktlint-check
mvn exec:exec@ktlint-format
```
If you also want the `ktlint` command-line tool (for editor integration or a git hook), install a version matching
`ktlint.version`, e.g. on macOS with Homebrew:
```
brew install ktlint
```
It can then be run directly with `ktlint` (or `ktlint -F` to autoformat), install a pre-commit hook with
`ktlint installGitPreCommitHook`, and configure IntelliJ IDEA with `ktlint applyToIDEAProject`.

37
jitsi-media-transform/checkstyle.xml Normal file
View File

@@ -0,0 +1,37 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker">
<module name="LineLength">
<property name="fileExtensions" value="java"/>
<property name="max" value="120"/>
<property name="ignorePattern" value="http://|https://"/>
</module>
<module name="TreeWalker">
<module name="LeftCurly">
<property name="option" value="nl"/>
<property name="tokens" value="LITERAL_CATCH"/>
<property name="tokens" value="LITERAL_ELSE"/>
<property name="tokens" value="LITERAL_FINALLY"/>
<property name="tokens" value="LITERAL_IF"/>
<property name="tokens" value="LITERAL_TRY"/>
</module>
<!-- Skips empty blocks {}, to not skip them: value="alone" -->
<module name="RightCurly">
<property name="option" value="alone_or_singleline"/>
</module>
<module name="WhitespaceAfter">
<!-- Drops TYPECAST from default tokens -->
<property name="tokens" value="COMMA , SEMI , LITERAL_IF , LITERAL_ELSE , LITERAL_WHILE , LITERAL_DO , LITERAL_FOR , DO_WHILE"/>
</module>
</module>
<module name="FileTabCharacter">
<property name="eachLine" value="true"/>
</module>
</module>

408
jitsi-media-transform/pom.xml Normal file
View File

@@ -0,0 +1,408 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jitsi</groupId>
<artifactId>jvb-parent</artifactId>
<version>2.3-SNAPSHOT</version>
</parent>
<groupId>org.jitsi</groupId>
<artifactId>jitsi-media-transform</artifactId>
<dependencies>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-srtp</artifactId>
<version>1.1-23-gaf3cd06</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rtp</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-utils</artifactId>
<version>${jitsi.utils.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-config</artifactId>
<version>${jicoco.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-metaconfig</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<version>${kotlin.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bctls-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
<groupId>org.pcap4j</groupId>
<artifactId>pcap4j-core</artifactId>
<version>1.8.2</version>
</dependency>
<dependency>
<groupId>org.pcap4j</groupId>
<artifactId>pcap4j-packetfactory-static</artifactId>
<version>1.8.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.spotbugs/spotbugs-annotations -->
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
<version>${spotbugs.version}</version>
</dependency>
<!-- test -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-test-kotlin</artifactId>
<version>${jicoco.version}</version>
</dependency>
<dependency>
<groupId>io.pkts</groupId>
<artifactId>pkts-core</artifactId>
<version>3.0.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<version>${junit.platform.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.kotest</groupId>
<artifactId>kotest-runner-junit5-jvm</artifactId>
<version>${kotest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.kotest</groupId>
<artifactId>kotest-assertions-core-jvm</artifactId>
<version>${kotest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.kotest</groupId>
<artifactId>kotest-property-jvm</artifactId>
<version>${kotest.version}</version>
<scope>test</scope>
</dependency>
<!-- Specify this dependency here because kotest needs it, and otherwise an older
version will be pulled in -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.mockk</groupId>
<artifactId>mockk-jvm</artifactId>
<version>${mockk.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<!--<sourceDirectory>${project.basedir}/src/main/kotlin</sourceDirectory>-->
<plugins>
<plugin>
<artifactId>kotlin-maven-plugin</artifactId>
<groupId>org.jetbrains.kotlin</groupId>
<version>${kotlin.version}</version>
<executions>
<execution>
<id>compile</id>
<goals> <goal>compile</goal> </goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/main/kotlin</sourceDir>
<sourceDir>${project.basedir}/src/main/java</sourceDir>
</sourceDirs>
</configuration>
</execution>
<execution>
<id>test-compile</id>
<goals> <goal>test-compile</goal> </goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/test/kotlin</sourceDir>
<sourceDir>${project.basedir}/src/test/java</sourceDir>
</sourceDirs>
</configuration>
</execution>
</executions>
<configuration>
<jvmTarget>17</jvmTarget>
</configuration>
</plugin>
<plugin>
<!-- To try and get both java and kotlin sources in the sources jar -->
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.basedir}/src/main/kotlin</source>
<source>${project.basedir}/src/main/java</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>${maven-source-plugin.version}</version>
<executions>
<execution>
<id>attach-sources</id>
<phase>verify</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<executions>
<!-- Replacing default-compile as it is treated specially by maven -->
<execution>
<id>default-compile</id>
<phase>none</phase>
</execution>
<!-- Replacing default-testCompile as it is treated specially by maven -->
<execution>
<id>default-testCompile</id>
<phase>none</phase>
</execution>
<execution>
<id>java-compile</id>
<phase>compile</phase>
<goals> <goal>compile</goal> </goals>
</execution>
<execution>
<id>java-test-compile</id>
<phase>test-compile</phase>
<goals> <goal>testCompile</goal> </goals>
</execution>
</executions>
<configuration>
<source>17</source>
<target>17</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<systemPropertyVariables>
<kotest.framework.classpath.scanning.autoscan.disable>true</kotest.framework.classpath.scanning.autoscan.disable>
</systemPropertyVariables>
<excludes>
<!-- init() currently throws an exception -->
<exclude>DtlsStackTest.*</exclude>
</excludes>
</configuration>
</plugin>
<!-- ktlint: check runs in the verify phase; auto-format with "mvn exec:exec@ktlint-format" -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.6.3</version>
<dependencies>
<dependency>
<groupId>com.pinterest.ktlint</groupId>
<artifactId>ktlint-cli</artifactId>
<version>${ktlint.version}</version>
<classifier>all</classifier>
</dependency>
</dependencies>
<executions>
<execution>
<id>ktlint-check</id>
<phase>verify</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>java</executable>
<includePluginDependencies>true</includePluginDependencies>
<arguments>
<argument>-classpath</argument>
<classpath>
<dependency>com.pinterest.ktlint:ktlint-cli</dependency>
</classpath>
<argument>com.pinterest.ktlint.Main</argument>
<argument>--relative</argument>
<argument>src/main/kotlin/**/*.kt</argument>
<argument>src/test/kotlin/**/*.kt</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>ktlint-format</id>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>java</executable>
<includePluginDependencies>true</includePluginDependencies>
<arguments>
<argument>-classpath</argument>
<classpath>
<dependency>com.pinterest.ktlint:ktlint-cli</dependency>
</classpath>
<argument>com.pinterest.ktlint.Main</argument>
<argument>--format</argument>
<argument>--relative</argument>
<argument>src/main/kotlin/**/*.kt</argument>
<argument>src/test/kotlin/**/*.kt</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<configLocation>checkstyle.xml</configLocation>
</configuration>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>${spotbugs-maven-plugin.version}</version>
<configuration>
<excludeFilterFile>${project.basedir}/spotbugs-exclude.xml</excludeFilterFile>
<includeTests>true</includeTests>
</configuration>
<executions>
<execution>
<phase>verify</phase>
<goals><goal>check</goal></goals>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs</artifactId>
<version>${spotbugs.version}</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>jitsi-maven-repository-releases</id>
<layout>default</layout>
<name>Jitsi Maven Repository (Releases)</name>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<url>https://github.com/jitsi/jitsi-maven-repository/raw/master/releases/</url>
</repository>
<repository>
<id>jitsi-maven-repository-snapshots</id>
<layout>default</layout>
<name>Jitsi Maven Repository (Snapshots)</name>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
<url>https://github.com/jitsi/jitsi-maven-repository/raw/master/snapshots/</url>
</repository>
</repositories>
<profiles>
<profile>
<id>coverage</id>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>${jacoco.version}</version>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<!-- attached to Maven test phase -->
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

84
jitsi-media-transform/spotbugs-exclude.xml Normal file
View File

@@ -0,0 +1,84 @@
<?xml version="1.0" encoding="UTF-8"?>
<FindBugsFilter>
<Match>
<Or>
<!-- Our Packet hierarchy intentionally uses clone() without calling super.clone(). -->
<Bug pattern="CN_IDIOM_NO_SUPER_CALL"/>
<!-- Serialization -->
<Bug pattern="SE_BAD_FIELD"/>
<Bug pattern="SE_BAD_FIELD_STORE"/>
<!-- We expose arrays in multiple places on purpose. -->
<Bug pattern="EI_EXPOSE_REP"/>
<Bug pattern="EI_EXPOSE_REP2"/>
<!-- We throw from constructors on illegal arguments, i.e. programming errors. -->
<Bug pattern="CT_CONSTRUCTOR_THROW"/>
<!-- We don't expect to run on 32-bit platforms. -->
<Bug pattern="AT_NONATOMIC_64BIT_PRIMITIVE"/>
</Or>
</Match>
<!-- Kotlin-specific -->
<Match>
<Or>
<!-- False positives with lateinit -->
<Bug pattern="NP_NONNULL_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR"/>
<!-- False positives with lateinit -->
<Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE"/>
<!-- False positives with lateinit -->
<Bug pattern="NP_NULL_PARAM_DEREF"/>
<!-- False positives with lateinit -->
<Bug pattern="NP_NONNULL_RETURN_VIOLATION"/>
<!-- False positives with method references -->
<Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE"/>
<!-- False positives with kotlin's "when" -->
<Bug pattern="SF_SWITCH_NO_DEFAULT"/>
<Bug pattern="SF_SWITCH_FALLTHROUGH"/>
<!-- False positives with lazy vals -->
<Bug pattern="NP_NULL_ON_SOME_PATH"/>
<!-- False positives with lazy vals -->
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
<!-- False positives with lazy vals -->
<Bug pattern="BC_IMPOSSIBLE_INSTANCEOF"/>
<!-- False positives with Kotlin inlines -->
<Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT"/>
<!-- False positives for the synthetic bridge methods Kotlin generates on anonymous
subclasses of generic collections (e.g. an anonymous LRUCache overriding removeEldestEntry). -->
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS"/>
<!-- False positives for an unknown reason -->
<Bug pattern="BC_BAD_CAST_TO_ABSTRACT_COLLECTION"/>
<Bug pattern="RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"/>
<Bug pattern="UR_UNINIT_READ"/>
<Bug pattern="UC_USELESS_OBJECT"/>
<Bug pattern="AT_STALE_THREAD_WRITE_OF_PRIMITIVE"/>
<Bug pattern="MS_EXPOSE_REP"/>
</Or>
<Source name="~.*\.kt" />
</Match>
<!-- Tests -->
<Match>
<Or>
<!-- Tests have a valid reason to ignore return values (expecting exceptions, etc). -->
<Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT"/>
<!-- Same for exceptions. -->
<Bug pattern="DE_MIGHT_IGNORE"/>
<!-- In tests it is convenient to use Random() randomly. -->
<Bug pattern="DMI_RANDOM_USED_ONLY_ONCE"/>
<!-- Looks like a false positive. -->
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS"/>
<!-- Confused by kotest beforeSpec patterns. -->
<Bug pattern="NP_NONNULL_PARAM_VIOLATION" />
<!-- False positives for an unknown reason. -->
<Bug pattern="SA_LOCAL_SELF_ASSIGNMENT"/>
</Or>
<Or>
<Source name="~.*\.*Test.kt" />
<Source name="~.*\.*Test.java" />
<Class name="~.*\.module_tests\..*" />
</Or>
</Match>
</FindBugsFilter>

114
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/DePacketizer.java Normal file
View File

@@ -0,0 +1,114 @@
/*
* Copyright @ 2015 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.codec.video.h264;
import org.jitsi.utils.logging2.*;
import static org.jitsi_modified.impl.neomedia.codec.video.h264.H264.*;
/**
* Implements <tt>Codec</tt> to represent a depacketizer of H.264 RTP packets
* into NAL units.
*
* @author Lyubomir Marinov
* @author Damian Minkov
*/
public class DePacketizer
{
/**
* The <tt>Logger</tt> used by the <tt>DePacketizer</tt> class and its
* instances for logging output.
*/
private static final Logger logger = new LoggerImpl(DePacketizer.class.toString());
/**
* Returns true if the buffer contains a H264 key frame at offset
* <tt>offset</tt>.
*
* @param buf the byte buffer to check
* @param off the offset in the byte buffer where the actuall data starts
* @param len the length of the data in the byte buffer
* @return true if the buffer contains a H264 key frame at offset
* <tt>offset</tt>.
*/
public static boolean isKeyFrame(byte[] buf, int off, int len)
{
if (buf == null || buf.length < off + Math.max(len, 1))
{
return false;
}
int nalType = buf[off] & kTypeMask;
// Single NAL Unit Packet
if (nalType == kFuA)
{
// Fragmented NAL units (FU-A).
if (parseFuaNaluForKeyFrame(buf, off, len))
{
return true;
}
}
else
{
if (parseSingleNaluForKeyFrame(buf, off, len))
{
return true;
}
}
return false;
}
/**
* Checks if a fragment of a NAL unit from a specific FU-A RTP packet
* payload is keyframe or not.
*/
private static boolean parseFuaNaluForKeyFrame(byte[] buf, int off, int len) {
if (len < kFuAHeaderSize)
{
return false;
}
return ((buf[off + 1] & kTypeMask) == kIdr);
}
/**
* Checks if a fragment of a NAL unit from a specific FU-A RTP packet
* payload is keyframe or not.
*/
private static boolean parseSingleNaluForKeyFrame(byte[] buf, int off, int len)
{
int naluStart = off + kNalHeaderSize;
int naluLength = len - kNalHeaderSize;
int nalType = buf[off] & kTypeMask;
if (nalType == kStapA)
{
// Skip the StapA header (StapA nal type + length).
if (len <= kStapAHeaderSize)
{
logger.error("StapA header truncated.");
return false;
}
if (!verifyStapANaluLengths(buf, naluStart, naluLength))
{
logger.error("StapA packet with incorrect NALU packet lengths.");
return false;
}
nalType = buf[off + kStapAHeaderSize] & kTypeMask;
}
return (nalType == kIdr || nalType == kSps ||
nalType == kPps || nalType == kSei);
}
}

83
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264.java Normal file
View File

@@ -0,0 +1,83 @@
/*
* Copyright @ 2015 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.codec.video.h264;
/**
* Common utilities and constants for H264.
*/
public class H264
{
/**
* The bytes to prefix any NAL unit to be output by this
* <tt>DePacketizer</tt> and given to a H.264 decoder. Includes
* start_code_prefix_one_3bytes. According to "B.1 Byte stream NAL unit
* syntax and semantics" of "ITU-T Rec. H.264 Advanced video coding for
* generic audiovisual services", zero_byte "shall be present" when "the
* nal_unit_type within the nal_unit() is equal to 7 (sequence parameter
* set) or 8 (picture parameter set)" or "the byte stream NAL unit syntax
* structure contains the first NAL unit of an access unit in decoding
* order".
*/
static final byte[] NAL_PREFIX = {0, 0, 0, 1};
/**
* Constants used to detect H264 keyframes in rtp packet
*/
static final byte kTypeMask = 0x1F; // Nalu
static final byte kIdr = 5;
static final byte kSei = 6;
static final byte kSps = 7;
static final byte kPps = 8;
static final byte kStapA = 24;
static final byte kFuA = 28; // Header sizes
static final int kNalHeaderSize = 1;
static final int kFuAHeaderSize = 2;
static final int kLengthFieldSize = 2;
static final int kStapAHeaderSize = kNalHeaderSize + kLengthFieldSize;
static final int kNalUSize = 2;
/**
* Check if Single-Time Aggregation Packet (STAP-A) NAL unit is correctly formed.
*
* @param data STAP-A payload
* @param offset Starting position of NAL unit
* @param lengthRemaining Bytes left in STAP-A
* @return True if STAP-A NAL Unit is correct
*/
static boolean verifyStapANaluLengths(byte[] data, int offset, int lengthRemaining)
{
if (data.length < offset + lengthRemaining)
{
return false;
}
while (lengthRemaining != 0)
{
if (lengthRemaining < kNalUSize)
{
return false;
}
int naluSize = kNalUSize + getUint16(data, offset);
offset += naluSize;
lengthRemaining -= naluSize;
}
return true;
}
static int getUint16(byte[] data, int offset)
{
return ((data[offset] & 0xff) << 8) | (data[offset + 1] & 0xff);
}
}

595
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp8/DePacketizer.java Normal file
View File

@@ -0,0 +1,595 @@
/*
* Copyright @ 2015 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.codec.video.vp8;
import org.jitsi.utils.*;
import org.jitsi.rtp.extensions.ByteKt;
/**
* A depacketizer from VP8.
* See {@link "http://tools.ietf.org/html/rfc7741"}
*
* @author Boris Grozev
* @author George Politis
* @author Jonathan Lennox
*/
public class DePacketizer
{
/**
* Returns true if the buffer contains a VP8 key frame at offset
* <tt>offset</tt>.
*
* @param buf the byte buffer to check
* @param off the offset in the byte buffer where the actual data starts
* @param len the length of the data in the byte buffer
* @return true if the buffer contains a VP8 key frame at offset
* <tt>offset</tt>.
*/
public static boolean isKeyFrame(byte[] buf, int off, int len)
{
// Check if this is the start of a VP8 partition in the payload
// descriptor.
if (!DePacketizer.VP8PayloadDescriptor.isValid(buf, off, len))
{
return false;
}
if (!DePacketizer.VP8PayloadDescriptor.isStartOfFrame(buf, off))
{
return false;
}
int szVP8PayloadDescriptor = DePacketizer
.VP8PayloadDescriptor.getSize(buf, off, len);
return DePacketizer.VP8PayloadHeader.isKeyFrame(
buf, off + szVP8PayloadDescriptor);
}
/**
* A class that represents the VP8 Payload Descriptor structure defined
* in {@link "https://tools.ietf.org/html/rfc7741"}
*/
public static class VP8PayloadDescriptor
{
/**
* I bit from the X byte of the Payload Descriptor.
*/
private static final byte I_BIT = (byte) 0x80;
/**
* K bit from the X byte of the Payload Descriptor.
*/
private static final byte K_BIT = (byte) 0x10;
/**
* L bit from the X byte of the Payload Descriptor.
*/
private static final byte L_BIT = (byte) 0x40;
/**
* M bit from the I byte of the Payload Descriptor.
*/
private static final byte M_BIT = (byte) 0x80;
/**
* Maximum length of a VP8 Payload Descriptor.
*/
public static final int MAX_LENGTH = 6;
/**
* S bit from the first byte of the Payload Descriptor.
*/
private static final byte S_BIT = (byte) 0x10;
/**
* T bit from the X byte of the Payload Descriptor.
*/
private static final byte T_BIT = (byte) 0x20;
/**
* X bit from the first byte of the Payload Descriptor.
*/
private static final byte X_BIT = (byte) 0x80;
/**
* N bit from the first byte of the Payload Descriptor.
*/
private static final byte N_BIT = (byte) 0x20;
/**
* Gets whether a VP8 payload has a temporal layer index (TID).
*
* @param buf the byte buffer that holds the VP8 packet.
* @param off the offset in the byte buffer where the VP8 payload starts.
* @param len the length of the VP8 payload.
* @return true if the VP8 payload has a temporal layer index, false if not.
*/
public static boolean hasTemporalLayerIndex(byte[] buf, int off, int len)
{
return getTemporalLayerIndex(buf, off, len) != -1;
}
/**
* Gets the temporal layer index (TID), if that's set.
*
* @param buf the byte buffer that holds the VP8 packet.
* @param off the offset in the byte buffer where the VP8 packet starts.
* @param len the length of the VP8 packet.
* @return the temporal layer index (TID), if that's set, -1 otherwise.
*/
public static int getTemporalLayerIndex(byte[] buf, int off, int len)
{
if (buf == null || buf.length < off + len || len < 2)
{
return -1;
}
if ((buf[off] & X_BIT) == 0 || (buf[off + 1] & T_BIT) == 0)
{
return -1;
}
int sz = getSize(buf, off, len);
if (buf.length < off + sz || sz < 1)
{
return -1;
}
return (buf[off + sz - 1] & 0xc0) >> 6;
}
/**
* Sets the temporal layer index (TID), if there's room for it.
*
* @param buf the byte buffer that holds the VP8 packet.
* @param off the offset in the byte buffer where the VP8 packet starts.
* @param len the length of the VP8 packet.
* @param tid the temporal layer value to
* @return true if the layer was set successfully, false otherwise.
*/
public static boolean setTemporalLayerIndex(byte[] buf, int off, int len, int tid)
{
if (tid < 0 || tid > 3)
{
throw new IllegalArgumentException("Bad tid value " + tid);
}
if (buf == null || buf.length < off + len || len < 2)
{
return false;
}
if ((buf[off] & X_BIT) == 0 || (buf[off + 1] & T_BIT) == 0)
{
return false;
}
int sz = getSize(buf, off, len);
if (buf.length < off + sz || sz < 1)
{
return false;
}
buf[off + sz - 1] = (byte)((tid << 6) | buf[off + sz - 1] & 0x3f);
return true;
}
/**
* Returns a simple Payload Descriptor, with PartID = 0, the 'start
* of partition' bit set according to <tt>startOfPartition</tt>, and
* all other bits set to 0.
*
* @param startOfPartition whether to 'start of partition' bit should be
* set
* @return a simple Payload Descriptor, with PartID = 0, the 'start
* of partition' bit set according to <tt>startOfPartition</tt>, and
* all other bits set to 0.
*/
public static byte[] create(boolean startOfPartition)
{
byte[] pd = new byte[1];
pd[0] = startOfPartition ? (byte) 0x10 : 0;
return pd;
}
/**
* The size in bytes of the Payload Descriptor at offset
* <tt>offset</tt> in <tt>input</tt>. The size is between 1 and 6.
*
* @param baf the <tt>ByteArrayBuffer</tt> that holds the VP8 payload
* descriptor.
* @return The size in bytes of the Payload Descriptor at offset
* <tt>offset</tt> in <tt>input</tt>, or -1 if the input is not a valid
* VP8 Payload Descriptor. The size is between 1 and 6.
*/
public static int getSize(ByteArrayBuffer baf)
{
if (baf == null)
{
return -1;
}
return getSize(baf.getBuffer(), baf.getOffset(), baf.getLength());
}
/**
* The size in bytes of the Payload Descriptor at offset
* <tt>offset</tt> in <tt>input</tt>. The size is between 1 and 6.
*
* @param input input
* @param offset offset
* @param length length
* @return The size in bytes of the Payload Descriptor at offset
* <tt>offset</tt> in <tt>input</tt>, or -1 if the input is not a valid
* VP8 Payload Descriptor. The size is between 1 and 6.
*/
public static int getSize(byte[] input, int offset, int length)
{
if (!isValid(input, offset, length))
return -1;
if ((input[offset] & X_BIT) == 0)
return 1;
int size = 2;
if ((input[offset + 1] & I_BIT) != 0)
{
size++;
if ((input[offset + 2] & M_BIT) != 0)
size++;
}
if ((input[offset + 1] & L_BIT) != 0)
size++;
if ((input[offset + 1] & (T_BIT | K_BIT)) != 0)
size++;
return size;
}
/**
* Determines whether the VP8 payload specified in the buffer that is
* passed as an argument has a picture ID or not.
*
* @param buf the byte buffer that contains the VP8 payload.
* @param off the offset in the byte buffer where the VP8 payload
* starts.
* @param len the length of the VP8 payload in the byte buffer.
* @return true if the VP8 payload contains a picture ID, false
* otherwise.
*/
public static boolean hasPictureId(byte[] buf, int off, int len)
{
return isValid(buf, off, len)
&& (buf[off] & X_BIT) != 0 && (buf[off + 1] & I_BIT) != 0;
}
/**
* Determines whether the VP8 payload specified in the buffer that is
* passed as an argument has an extended picture ID or not.
*
* @param buf the byte buffer that contains the VP8 payload.
* @param off the offset in the byte buffer where the VP8 payload
* starts.
* @param len the length of the VP8 payload in the byte buffer.
* @return true if the VP8 payload contains an extended picture ID,
* false otherwise.
*/
public static boolean hasExtendedPictureId(byte[] buf, int off, int len)
{
return hasPictureId(buf, off, len) && (buf[off + 2] & M_BIT) != 0;
}
/**
* Determines whether the VP8 payload specified in the buffer that is
* passed as an argument has a TL0PICIDX or not.
*
* @param buf the byte buffer that contains the VP8 payload.
* @param off the offset in the byte buffer where the VP8 payload
* starts.
* @param len the length of the VP8 payload in the byte buffer.
* @return true if the VP8 payload contains a TL0PICIDX,
* false otherwise.
*/
public static boolean hasTL0PICIDX(byte[] buf, int off, int len)
{
return getLByteOffset(buf, off, len) >= 0;
}
/**
* Gets the value of the PictureID field of a VP8 Payload Descriptor.
*
* @param input
* @param offset
* @return the value of the PictureID field of a VP8 Payload Descriptor,
* or -1 if the fields is not present.
*/
public static int getPictureId(byte[] input, int offset)
{
if (input == null
|| !hasPictureId(input, offset, input.length - offset))
{
return -1;
}
boolean isLong = (input[offset + 2] & M_BIT) != 0;
if (isLong)
return (input[offset + 2] & 0x7f) << 8
| (input[offset + 3] & 0xff);
else
return input[offset + 2] & 0x7f;
}
/**
* Sets the extended picture ID for the VP8 payload specified in the
* buffer that is passed as an argument.
*
* @param buf the byte buffer that contains the VP8 payload.
* @param off the offset in the byte buffer where the VP8 payload
* starts.
* @param len the length of the VP8 payload in the byte buffer.
* @return true if the operation succeeded, false otherwise.
*/
public static boolean setExtendedPictureId(
byte[] buf, int off, int len, int val)
{
if (!hasExtendedPictureId(buf, off, len))
{
return false;
}
buf[off + 2] = (byte) (0x80 | (val >> 8) & 0x7F);
buf[off + 3] = (byte) (val & 0xFF);
return true;
}
/**
* Sets the TL0PICIDX field for the VP8 payload specified in the
* buffer that is passed as an argument.
*
* @param buf the byte buffer that contains the VP8 payload.
* @param off the offset in the byte buffer where the VP8 payload
* starts.
* @param len the length of the VP8 payload in the byte buffer.
* @return true if the operation succeeded, false otherwise.
*/
public static boolean setTL0PICIDX(byte[] buf, int off, int len,
int val)
{
int offL = getLByteOffset(buf, off, len);
if (offL < 0)
{
return false;
}
buf[off + offL] = (byte) val;
return true;
}
/**
* Checks whether the arguments specify a valid buffer.
*
* @param buf
* @param off
* @param len
* @return true if the arguments specify a valid buffer, false
* otherwise.
*/
public static boolean isValid(byte[] buf, int off, int len)
{
return buf != null && buf.length >= off + len && off > -1
&& len > 0;
}
/**
* Checks whether the '<tt>start of partition</tt>' bit is set in the
* VP8 Payload Descriptor at offset <tt>offset</tt> in <tt>input</tt>.
*
* @param input input
* @param offset offset
* @return <tt>true</tt> if the '<tt>start of partition</tt>' bit is set,
* <tt>false</tt> otherwise.
*/
public static boolean isStartOfPartition(byte[] input, int offset)
{
return (input[offset] & S_BIT) != 0;
}
/**
* Sets the '<tt>start of partition</tt>' bit in the
* VP8 Payload Descriptor at offset <tt>offset</tt> in <tt>input</tt>.
*
* @param input input
* @param offset offset
* @param start whether it is start or not.
*/
public static void setStartOfPartition(byte[] input, int offset, boolean start)
{
input[offset] = ByteKt.putBitWithMask(input[offset], S_BIT, start);
}
/**
* Returns <tt>true</tt> if both the '<tt>start of partition</tt>' bit
* is set and the <tt>PID</tt> fields has value 0 in the VP8 Payload
* Descriptor at offset <tt>offset</tt> in <tt>input</tt>.
*
* @param input
* @param offset
* @return <tt>true</tt> if both the '<tt>start of partition</tt>' bit
* is set and the <tt>PID</tt> fields has value 0 in the VP8 Payload
* Descriptor at offset <tt>offset</tt> in <tt>input</tt>.
*/
public static boolean isStartOfFrame(byte[] input, int offset)
{
return isStartOfPartition(input, offset)
&& getPartitionId(input, offset) == 0;
}
/**
* Returns the value of the <tt>PID</tt> (partition ID) field of the
* VP8 Payload Descriptor at offset <tt>offset</tt> in <tt>input</tt>.
*
* @param input
* @param offset
* @return the value of the <tt>PID</tt> (partition ID) field of the
* VP8 Payload Descriptor at offset <tt>offset</tt> in <tt>input</tt>.
*/
public static int getPartitionId(byte[] input, int offset)
{
return input[offset] & 0x07;
}
/**
* Gets a boolean that indicates whether or not the non-reference bit is
* set.
*
* @param buf the byte buffer that holds the VP8 payload descriptor.
* @param off the offset in the byte buffer where the payload descriptor
* starts.
* @param len the length of the payload descriptor in the byte buffer.
* @return true if the non-reference bit is NOT set, false otherwise.
*/
public static boolean isReference(byte[] buf, int off, int len)
{
return (buf[off] & N_BIT) == 0;
}
/**
* Gets the TL0PICIDX from the payload descriptor.
*
* @param buf the byte buffer that holds the VP8 payload descriptor.
* @param off the offset in the byte buffer where the payload descriptor
* starts.
* @param len the length of the payload descriptor in the byte buffer.
* @return the TL0PICIDX from the payload descriptor, or -1 if the packet
* does not have one.
*/
public static int getTL0PICIDX(byte[] buf, int off, int len)
{
int offL = getLByteOffset(buf, off, len);
if (offL < 0)
{
return -1;
}
return buf[off + offL] & 0xff;
}
/**
* Return the offset of the {@code L} byte relative to the start of
* the payload descriptor, or -1 if the PR has no {@code L} byte.
*
* @param buf the byte buffer that holds the VP8 payload descriptor.
* @param off the offset in the byte buffer where the payload descriptor
* starts.
* @param len the length of the payload descriptor in the byte buffer.
* @return
*/
private static int getLByteOffset(byte[] buf, int off, int len)
{
if (!isValid(buf, off, len)
|| (buf[off] & X_BIT) == 0 || (buf[off + 1] & L_BIT) == 0)
{
return -1;
}
// We could use hasPictureId(), but it would unnecessarily repeat
// the isValid() check.
int offL = 2;
if ((buf[off + 1] & I_BIT) != 0)
{
offL++;
if ((buf[off + 2] & M_BIT) != 0)
{
offL++;
}
}
return offL;
}
}
/**
* A class that represents the VP8 Payload Header structure described
* in {@link "http://tools.ietf.org/html/rfc7741"}
*/
public static class VP8PayloadHeader
{
/**
* S bit of the Payload Header.
*/
private static final byte S_BIT = (byte) 0x01;
/**
* Returns true if the <tt>P</tt> (inverse key frame flag) field of the
* VP8 Payload Header at offset <tt>offset</tt> in <tt>input</tt> is 0.
*
* @return true if the <tt>P</tt> (inverse key frame flag) field of the
* VP8 Payload Header at offset <tt>offset</tt> in <tt>input</tt> is 0,
* false otherwise.
*/
public static boolean isKeyFrame(byte[] input, int offset)
{
// When set to 0 the current frame is a key frame. When set to 1
// the current frame is an interframe. Defined in [RFC6386]
return (input[offset] & S_BIT) == 0;
}
/* Sets the <tt>P</tt> (inverse key frame flag) field of the
* VP8 Payload header at offset <tt>offset</tt> in <tt>input</tt>.
* Sets P to 0 if keyFrame is false, or to 1 if keyFrame is true.
*/
public static void setKeyFrame(byte[] input, int offset, boolean keyFrame)
{
input[offset] = ByteKt.putBitWithMask(input[offset], S_BIT, !keyFrame);
}
}
/**
* A class that represents a keyframe header structure (see RFC 6386,
* paragraph 9.1).
*
* @author George Politis
*/
public static class VP8KeyframeHeader
{
// From RFC 6386, the keyframe header has this format.
//
// Start code byte 0 0x9d
// Start code byte 1 0x01
// Start code byte 2 0x2a
//
// 16 bits : (2 bits Horizontal Scale << 14) | Width (14 bits)
// 16 bits : (2 bits Vertical Scale << 14) | Height (14 bits)
/**
* @return the height of this instance.
*/
public static int getHeight(byte[] buf, int off)
{
return (((buf[off + 6] & 0xff) << 8) | buf[off + 5] & 0xff) & 0x3fff;
}
}
}

810
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp9/DePacketizer.java Normal file
View File

@@ -0,0 +1,810 @@
/*
* Copyright @ 2015 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.codec.video.vp9;
import org.jitsi.rtp.extensions.*;
/**
* A depacketizer from VP9.
* See {@link "https://tools.ietf.org/html/draft-ietf-payload-vp9-10"}
*
* @author George Politis
*/
public class DePacketizer
{
/*
* VP9 Payload header, flexible mode (F=1):
* 0 1 2 3 4 5 6 7
* +-+-+-+-+-+-+-+-+
* |I|P|L|F|B|E|V|Z| (REQUIRED)
* +-+-+-+-+-+-+-+-+
* I: |M| PICTURE ID | (REQUIRED)
* +-+-+-+-+-+-+-+-+
* M: | EXTENDED PID | (RECOMMENDED)
* +-+-+-+-+-+-+-+-+
* L: | TID |U| SID |D| (CONDITIONALLY RECOMMENDED)
* +-+-+-+-+-+-+-+-+ -\
* P,F: | P_DIFF |N| (CONDITIONALLY REQUIRED) - up to 3 times
* +-+-+-+-+-+-+-+-+ -/
* V: | SS |
* | .. |
* +-+-+-+-+-+-+-+-+
*
* Non-flexible mode (F=0):
* 0 1 2 3 4 5 6 7
* +-+-+-+-+-+-+-+-+
* |I|P|L|F|B|E|V|Z| (REQUIRED)
* +-+-+-+-+-+-+-+-+
* I: |M| PICTURE ID | (RECOMMENDED)
* +-+-+-+-+-+-+-+-+
* M: | EXTENDED PID | (RECOMMENDED)
* +-+-+-+-+-+-+-+-+
* L: | TID |U| SID |D| (CONDITIONALLY RECOMMENDED)
* +-+-+-+-+-+-+-+-+
* | TL0PICIDX | (CONDITIONALLY REQUIRED)
* +-+-+-+-+-+-+-+-+
* V: | SS |
* | .. |
* +-+-+-+-+-+-+-+-+
*/
/**
* I bit from the first byte of the Payload Descriptor:
* Picture ID present.
*/
private static final byte I_BIT = (byte) (1 << 7);
/**
* P bit from the first byte of the Payload Descriptor:
* Inter-picture predicted frame.
*/
private static final byte P_BIT = (byte) (1 << 6);
/**
* L bit from the first byte of the Payload Descriptor:
* Layer indices present.
*/
private static final byte L_BIT = (byte) (1 << 5);
/**
* F bit from the first byte of the Payload Descriptor:
* flexible mode.
*/
private static final byte F_BIT = (byte) (1 << 4);
/**
* B bit from the first byte of the Payload Descriptor:
* Start of frame.
*/
private static final byte B_BIT = (byte) (1 << 3);
/**
* E bit from the first byte of the Payload Descriptor:
* End of frame.
*/
private static final byte E_BIT = (byte) (1 << 2);
/**
* V bit from the first byte of the Payload Descriptor:
* Scalability structure present.
*/
private static final byte V_BIT = (byte) (1 << 1);
/**
* Z bit from first byte of the Payload Descriptor:
* Not an upper-level reference frame
*/
private static final byte Z_BIT = (byte)1;
/**
* M bit from the first picture ID byte of the Payload Descriptor:
* extended picture ID. Only present if I is true in the first byte.
*/
private static final byte M_BIT = (byte) (1 << 7);
/**
* U bit from the layer indices byte of the Payload Descriptor:
* switching up point. Only present if L is true in the first byte.
*/
private static final byte U_BIT = (byte) (1 << 4);
/**
* D bit from the layer indices byte of the Payload Descriptor:
* inter-layer dependency. Only present if L is true in the first byte.
*/
private static final byte D_BIT = (byte) (1);
/**
* N bit from the picture difference byte of the Payload Descriptor:
* more picture diffs present. Only present if P and F are true in the first byte.
*/
private static final byte N_BIT = (byte) (1);
/**
* A class that represents the VP9 Payload Descriptor structure defined
* in {@link "https://tools.ietf.org/html/draft-ietf-payload-vp9-10"}
*/
public static class VP9PayloadDescriptor
{
/**
* Returns <tt>true</tt> if the arguments specify a valid non-empty
* VP9 packet.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the arguments specify a valid non-empty
* buffer.
*/
private static boolean isValid(byte[] buf, int off, int len)
{
return
(buf != null && buf.length >= off + len && off > -1 && len > 0);
}
/**
* Returns <tt>true</tt> if the B bit from the first byte of the payload
* descriptor has value 0. (Note this is start of frame, not start of picture!)
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the B bit from the first byte of the
* payload descriptor has value 0, false otherwise.
*
*/
public static boolean isStartOfFrame(byte[] buf, int off, int len)
{
// Check if this is the start of a VP9 layer frame in the payload
// descriptor.
return isValid(buf, off, len) && (buf[off] & B_BIT) != 0;
}
/**
* Sets the B bit (start of frame) in the first byte of the payload
* descriptor.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
* @param start the new value of the start of frame bit.
*/
public static void setStartOfFrame(byte[] buf, int off, int len, boolean start)
{
// Check if this is the start of a VP9 layer frame in the payload
// descriptor.
if (!isValid(buf, off, len))
{
throw new IllegalStateException("Can't set startOfFrame for invalid VP9 packet");
}
buf[off] = ByteKt.putBitWithMask(buf[off], B_BIT, start);
}
/**
* Returns <tt>true</tt> if the E bit from the first byte of the payload
* descriptor has value 0. (Note this is end of frame, not end of picture!)
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the E bit from the first byte of the
* payload descriptor has value 0, false otherwise.
*/
public static boolean isEndOfFrame(byte[] buf, int off, int len)
{
// Check if this is the end of a VP9 layer frame in the payload
// descriptor.
return isValid(buf, off, len) && (buf[off] & E_BIT) != 0;
}
/**
* Sets the E bit (end of frame) in the first byte of the payload
* descriptor.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
* @param end the new value of the end of frame bit.
*/
public static void setEndOfFrame(byte[] buf, int off, int len, boolean end)
{
if (!isValid(buf, off, len))
{
throw new IllegalStateException("Can't set endOfFrame for invalid VP9 packet");
}
buf[off] = ByteKt.putBitWithMask(buf[off], E_BIT, end);
}
/**
* Returns <tt>true</tt> if the packet is encoded in flexible mode
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet is encoded in flexible mode,
* false otherwise
*/
public static boolean isFlexibleMode(byte[] buf, int off, int len)
{
return isValid(buf, off, len) && (buf[off] & F_BIT) != 0;
}
/**
* Returns <tt>true</tt> if the packet contains a scalability structure
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet contains a scalability structure,
* false otherwise
*/
public static boolean hasScalabilityStructure(byte[] buf, int off, int len)
{
return isValid(buf, off, len) && (buf[off] & V_BIT) != 0;
}
/**
* Returns <tt>true</tt> if the packet might be an upper-level reference.
*
* (Note this is inverted from the sense of the Z bit in the payload header.)
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet contains a scalability structure,
* false otherwise
*/
public static boolean isUpperLevelReference(byte[] buf, int off, int len)
{
return isValid(buf, off, len) && (buf[off] & Z_BIT) == 0;
}
/**
* Sets the Z bit (upper layer reference) in the first byte of the payload
* descriptor.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
* @param ref whether the frame might be an upper-layer reference.
* (Note this is inverted from the sense of the Z bit in the payload header.)
*/
public static void setUpperLevelReference(byte[] buf, int off, int len, boolean ref)
{
if (!isValid(buf, off, len))
{
throw new IllegalStateException("Can't set upperLevelReference for invalid VP9 packet");
}
buf[off] = ByteKt.putBitWithMask(buf[off], Z_BIT, !ref);
}
/**
* Returns <tt>true</tt> if the packet has a picture ID.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet has a picture ID, <tt>false</tt>
* if not.
*/
public static boolean hasPictureId(byte[] buf, int off, int len)
{
return isValid(buf, off, len) && (buf[off] & I_BIT) != 0 && len > 1;
}
/**
* Returns <tt>true</tt> if the packet has an extended (15-bit) picture ID.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet has a picture ID, <tt>false</tt>
* if not.
*/
public static boolean hasExtendedPictureId(byte[] buf, int off, int len)
{
return hasPictureId(buf, off, len) &&
(buf[off+1] & M_BIT) != 0 && len > 2;
}
/**
* Gets the value of the PictureID field of a VP9 Payload Descriptor.
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet has a picture ID, <tt>false</tt>
* if not.
*/
public static int getPictureId(byte[] buf, int off, int len)
{
if (!hasPictureId(buf, off, len))
{
return -1;
}
if (hasExtendedPictureId(buf, off, len))
{
return (buf[off + 1] & 0x7f) << 8
| (buf[off + 2] & 0xff);
}
else
{
return buf[off + 1] & 0x7f;
}
}
/**
* Sets the extended picture ID for the VP9 payload specified in the
* buffer that is passed as an argument.
*
* @param buf the byte buffer that contains the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload
* starts.
* @param len the length of the VP9 payload in the byte buffer.
* @return true if the operation succeeded, false otherwise.
*/
public static boolean setExtendedPictureId(
byte[] buf, int off, int len, int val)
{
if (!hasExtendedPictureId(buf, off, len))
{
return false;
}
buf[off + 1] = (byte) (0x80 | (val >> 8) & 0x7F);
buf[off + 2] = (byte) (val & 0xFF);
return true;
}
/**
* Returns <tt>true</tt> if the packet contains an inter-picture
* predicted frame.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet contains an inter-picture
* predicted frame, <tt>false</tt> if not.
*/
public static boolean isInterPicturePredicted(byte[] buf, int off, int len)
{
return isValid(buf, off, len) && (buf[off] & P_BIT) != 0;
}
/**
* Sets the P bit (inter-picture predicted) in the first byte of the payload
* descriptor.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
* @param pred the new value of the inter-picture predicted bit.
*/
public static void setInterPicturePredicted(byte[] buf, int off, int len, boolean pred)
{
if (!isValid(buf, off, len))
{
throw new IllegalStateException("Can't set interPicturePredicted for invalid VP9 packet");
}
buf[off] = ByteKt.putBitWithMask(buf[off], P_BIT, pred);
}
/**
* Returns <tt>true</tt> if the packet is part of a keyframe.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet is part of a keyframe, false if not.
*/
public static boolean isKeyFrame(byte[] buf, int off, int len)
{
if (!isValid(buf, off, len))
{
return false;
}
if ((buf[off] & P_BIT) != 0)
{
return false;
}
if (!hasLayerIndices(buf, off, len))
{
/* P bit without layer indices is a key frame. */
return true;
}
return (getSpatialLayerIndex(buf, off, len) == 0 ||
!usesInterLayerDependency(buf, off, len));
}
/**
* Query whether the packet has layer indices values.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return the temporal layer index (TID), if that's set, -1 otherwise.
*/
public static boolean hasLayerIndices(byte[] buf, int off, int len)
{
return (isValid(buf, off, len) && (buf[off] & L_BIT) != 0);
}
private static int getLayerIndexOffset(byte[] buf, int off, int len)
{
if (!hasLayerIndices(buf, off, len))
{
return -1;
}
int loff = off + 1;
if ((buf[off] & I_BIT) != 0)
{
loff += 1;
if ((buf[off + 1] & M_BIT) != 0)
{
// extended pid.
loff += 1;
}
}
if (loff >= off+len)
{
return -1;
}
return loff;
}
/**
* Determines whether the VP9 payload specified in the buffer that is
* passed as an argument has a TL0PICIDX or not.
*
* @param buf the byte buffer that contains the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload
* starts.
* @param len the length of the VP9 payload in the byte buffer.
* @return true if the VP9 payload contains a TL0PICIDX,
* false otherwise.
*/
public static boolean hasTL0PICIDX(byte[] buf, int off, int len)
{
int loff = getLayerIndexOffset(buf, off, len);
if (loff < 0 || loff + 1 >= off+len)
{
return false;
}
return !isFlexibleMode(buf, off, len);
}
/**
* Gets the TL0PICIDX from the payload descriptor.
*
* @param buf the byte buffer that holds the VP9 payload descriptor.
* @param off the offset in the byte buffer where the payload descriptor
* starts.
* @param len the length of the payload descriptor in the byte buffer.
* @return the TL0PICIDX from the payload descriptor, or -1 if the packet
* does not have one.
*/
public static int getTL0PICIDX(byte[] buf, int off, int len)
{
int loff = getLayerIndexOffset(buf, off, len);
if (!hasTL0PICIDX(buf, off, len))
return -1;
return buf[loff+1] & 0xff;
}
/**
* Sets the TL0PICIDX field for the VP9 payload specified in the
* buffer that is passed as an argument.
*
* @param buf the byte buffer that contains the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload
* starts.
* @param len the length of the VP9 payload in the byte buffer.
* @return true if the operation succeeded, false otherwise.
*/
public static boolean setTL0PICIDX(byte[] buf, int off, int len,
int val)
{
if (!hasTL0PICIDX(buf, off, len))
{
return false;
}
int loff = getLayerIndexOffset(buf, off, len);
buf[loff+1] = (byte) val;
return true;
}
/**
* Gets the temporal layer index (TID), if that's set.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return the temporal layer index (TID), if that's set, -1 otherwise.
*/
public static int getTemporalLayerIndex(byte[] buf, int off, int len)
{
int loff = getLayerIndexOffset(buf, off, len);
if (loff < 0)
{
return -1;
}
return (buf[loff] & 0xE0) >> 5;
}
/**
* Gets the spatial layer index (SID), if that's set.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return the spatial layer index (SID), if that's set, -1 otherwise.
*/
public static int getSpatialLayerIndex(byte[] buf, int off, int len)
{
int loff = getLayerIndexOffset(buf, off, len);
if (loff < 0)
{
return -1;
}
return (buf[loff] & 0xE) >> 1;
}
/**
* Gets whether this frame is a temporal switching-up point, where it is
* valid to switch the stream to a higher temporal layer.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet is a switching-up point, <tt>false</tt>
* if not.
*/
public static boolean isSwitchingUpPoint(byte[] buf, int off, int len)
{
int loff = getLayerIndexOffset(buf, off, len);
if (loff < 0)
{
return false;
}
return (buf[loff] & U_BIT) != 0;
}
/**
* Gets whether this frame uses inter-layer dependency, where the frame
* depends on lower spatial-layer frames of the same picture.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
*
* @return <tt>true</tt> if the packet uses inter-layer dependency, <tt>false</tt>
* if not.
*/
public static boolean usesInterLayerDependency(byte[] buf, int off, int len)
{
int loff = getLayerIndexOffset(buf, off, len);
if (loff < 0)
{
return false;
}
return (buf[loff] & D_BIT) != 0;
}
/**
* Sets the packet's layer indices, if there's room for it in the packet.
*
* @param buf the byte buffer that holds the VP9 payload.
* @param off the offset in the byte buffer where the VP9 payload starts.
* @param len the length of the VP9 payload.
* @param sid the spatial layer of the packet
* @param tid the temporal layer of the packet
* @param isSwitchingUpPoint whether the packet is a switching-up point
* @param usesInterLayerDependency whether the packet uses inter-layer dependency
*
* @return Whether the layer indices were successfully set
*/
public static boolean setLayerIndices(byte[] buf, int off, int len,
int sid, int tid, boolean isSwitchingUpPoint, boolean usesInterLayerDependency)
{
int loff = getLayerIndexOffset(buf, off, len);
if (loff < 0)
{
return false;
}
if (sid < 0 || sid > 8)
{
throw new IllegalArgumentException("Invalid spatial ID " + sid);
}
if (tid < 0 || tid > 8)
{
throw new IllegalArgumentException("Invalid spatial ID " + sid);
}
buf[loff] = (byte)((tid << 5) |
(isSwitchingUpPoint ? U_BIT : 0) |
(sid << 1) |
(usesInterLayerDependency ? D_BIT : 0));
return true;
}
/**
* The size in bytes of the Payload Descriptor at off
* <tt>off</tt> in <tt>buf</tt>. The size is at least 1.
*
* @param buf buf
* @param off off
* @param len len
* @return The size in bytes of the Payload Descriptor at off
* <tt>off</tt> in <tt>buf</tt>, or -1 if the buffer is not a valid
* VP9 Payload Descriptor. The size is at least 1.
*/
public static int getSize(byte[] buf, int off, int len)
{
if (!isValid(buf, off, len))
return -1;
int pos = off + 1;
if ((buf[off] & I_BIT) != 0)
{
boolean extended = (buf[pos] & M_BIT) != 0;
pos++;
if (extended)
{
pos++;
}
}
if ((buf[off] & L_BIT) != 0)
{
pos++;
if ((buf[off] & F_BIT) == 0)
{
pos++;
}
}
if ((buf[off] & F_BIT) != 0 && (buf[off] & P_BIT) != 0)
{
for (int i = 0; i < 3; i++) {
int hasNext = buf[pos] & N_BIT;
pos++;
if (hasNext == 0)
{
break;
}
}
}
if ((buf[off] & V_BIT) != 0)
{
/* SS present */
int n_s = (buf[pos] & 0xE0) >> 5;
boolean resPresent = ((buf[pos] & (1 << 4)) != 0);
boolean pgPresent = ((buf[pos] & (1 << 3)) != 0);
pos++;
if (resPresent)
{
pos += 4 * (n_s + 1);
}
if (pgPresent)
{
int n_g = (buf[pos] & 0xff);
int i;
pos++;
for (i = 0; i < n_g; i++)
{
int r = (buf[pos] & 0x0C) >> 2;
pos += r + 1;
}
}
}
return pos - off;
}
/**
* The offset of the scalability structure within the buffer, if
* the buffer contains a VP9 packet with a scalability structure; otherwise -1.
*
* @param buf buf
* @param off off
* @param len len
* @return The size in bytes of the Payload Descriptor at off
* <tt>off</tt> in <tt>buf</tt>, or -1 if the buffer is not a valid
* VP9 Payload Descriptor. The size is at least 1.
*/
public static int getScalabilityStructureOffset(byte[] buf, int off, int len)
{
if (!hasScalabilityStructure(buf, off, len))
{
return -1;
}
int pos = off + 1;
if ((buf[off] & I_BIT) != 0)
{
boolean extended = (buf[pos] & M_BIT) != 0;
pos++;
if (extended)
{
pos++;
}
}
if ((buf[off] & L_BIT) != 0)
{
pos++;
if ((buf[off] & F_BIT) == 0)
{
pos++;
}
}
if ((buf[off] & F_BIT) != 0 && (buf[off] & P_BIT) != 0)
{
do {
pos++;
} while ((buf[pos] & N_BIT) != 0);
}
return pos;
}
}
}

532
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java Normal file
View File

@@ -0,0 +1,532 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging.DiagnosticContext;
import org.jitsi.utils.logging.TimeSeriesLogger;
/**
* A rate control implementation based on additive increases of bitrate when no
* over-use is detected and multiplicative decreases when over-uses are
* detected. When we think the available bandwidth has changes or is unknown, we
* will switch to a "slow-start mode" where we increase multiplicatively.
*
* webrtc/modules/remote_bitrate_estimator/aimd_rate_control.cc
* webrtc/modules/remote_bitrate_estimator/aimd_rate_control.h
*
* @author Lyubomir Marinov
*/
class AimdRateControl
{
/**
* The <tt>TimeSeriesLogger</tt> used by the
* <tt>RemoteBitrateEstimatorAbsSendTime</tt> class and its instances for
* logging output.
*/
private static final TimeSeriesLogger logger
= TimeSeriesLogger.getTimeSeriesLogger(AimdRateControl.class);
private static final int kDefaultRttMs = 200;
private static final long kInitializationTimeMs = 5000;
private static final long kFirstIncomingEstimateExpirationMs = 2 * kInitializationTimeMs;
private static final long kLogIntervalMs = 1000;
private static final long kMaxFeedbackIntervalMs = 1000;
private static final long kMinFeedbackIntervalMs = 200;
private static final int kRtcpSize = 80;
private static final double kWithinIncomingBitrateHysteresis = 1.05;
private final DiagnosticContext diagnosticContext;
private float avgMaxBitrateKbps;
private float beta;
private boolean bitrateIsInitialized;
private long currentBitrateBps;
private final RateControlInput currentInput
= new RateControlInput(BandwidthUsage.kBwNormal, 0L, 0D);
private boolean inExperiment;
private long minConfiguredBitrateBps;
private RateControlRegion rateControlRegion;
private RateControlState rateControlState;
private long rtt;
private long timeFirstIncomingEstimate;
private long timeLastBitrateChange;
private long timeOfLastLog;
private boolean updated;
private float varMaxBitrateKbps;
public AimdRateControl(@NotNull DiagnosticContext diagnosticContext)
{
reset();
this.diagnosticContext = diagnosticContext;
}
private long additiveRateIncrease(
long nowMs,
long lastMs,
long responseTimeMs)
{
if (responseTimeMs <= 0)
throw new IllegalArgumentException("responseTimeMs");
double beta = 0.0;
if (lastMs > 0)
{
beta
= Math.min((nowMs - lastMs) / (double) responseTimeMs, 1.0);
if (inExperiment)
beta /= 2.0;
}
double bitsPerFrame = (double) currentBitrateBps / 30.0;
double packetsPerFrame = Math.ceil(bitsPerFrame / (8.0 * 1200.0));
double avgPacketSizeBits = bitsPerFrame / packetsPerFrame;
return (long) Math.max(1000.0, beta * avgPacketSizeBits);
}
private long changeBitrate(
long currentBitrateBps,
long incomingBitrateBps,
long nowMs)
{
if (!updated)
return this.currentBitrateBps;
// An over-use should always trigger us to reduce the bitrate, even though
// we have not yet established our first estimate. By acting on the over-use,
// we will end up with a valid estimate.
if (!bitrateIsInitialized
&& currentInput.bwState != BandwidthUsage.kBwOverusing)
{
return this.currentBitrateBps;
}
updated = false;
changeState(currentInput, nowMs);
// Calculated here because it's used in multiple places.
float incomingBitrateKbps = incomingBitrateBps / 1000.0F;
// Calculate the max bit rate std dev given the normalized variance and
// the current incoming bit rate.
float stdMaxBitRate
= (float) Math.sqrt(varMaxBitrateKbps * avgMaxBitrateKbps);
switch (rateControlState)
{
case kRcHold:
break;
case kRcIncrease:
{
if (avgMaxBitrateKbps >= 0F
&& incomingBitrateKbps
> avgMaxBitrateKbps + 3F * stdMaxBitRate)
{
changeRegion(RateControlRegion.kRcMaxUnknown, nowMs);
avgMaxBitrateKbps = -1F;
}
if (rateControlRegion == RateControlRegion.kRcNearMax)
{
// Approximate the over-use estimator delay to 100 ms.
long responseTime = rtt + 100;
long additiveIncreaseBps
= additiveRateIncrease(
nowMs,
timeLastBitrateChange,
responseTime);
currentBitrateBps += additiveIncreaseBps;
}
else // kRcMaxUnknown || kRcAboveMax
{
long multiplicativeIncreaseBps
= multiplicativeRateIncrease(
nowMs,
timeLastBitrateChange,
currentBitrateBps);
currentBitrateBps += multiplicativeIncreaseBps;
}
timeLastBitrateChange = nowMs;
break;
}
case kRcDecrease:
{
bitrateIsInitialized = true;
if (incomingBitrateBps < minConfiguredBitrateBps)
{
currentBitrateBps = minConfiguredBitrateBps;
}
else
{
// Set bit rate to something slightly lower than max to get rid
// of any self-induced delay.
currentBitrateBps = (long) (beta * incomingBitrateBps + 0.5);
if (currentBitrateBps > this.currentBitrateBps)
{
// Avoid increasing the rate when over-using.
if (rateControlRegion != RateControlRegion.kRcMaxUnknown)
{
currentBitrateBps
= (long) (beta * avgMaxBitrateKbps * 1000F + 0.5F);
}
currentBitrateBps
= Math.min(currentBitrateBps, this.currentBitrateBps);
}
changeRegion(RateControlRegion.kRcNearMax, nowMs);
if (incomingBitrateKbps
< avgMaxBitrateKbps - 3F * stdMaxBitRate)
{
avgMaxBitrateKbps = -1F;
}
updateMaxBitRateEstimate(incomingBitrateKbps);
}
// Stay on hold until the pipes are cleared.
changeState(RateControlState.kRcHold, nowMs);
timeLastBitrateChange = nowMs;
break;
}
default:
throw new IllegalStateException("rateControlState");
}
if ((incomingBitrateBps > 100000L || currentBitrateBps > 150000L)
&& currentBitrateBps > 1.5 * incomingBitrateBps)
{
// Allow changing the bit rate if we are operating at very low rates
// Don't change the bit rate if the send side is too far off
currentBitrateBps = this.currentBitrateBps;
timeLastBitrateChange = nowMs;
}
return currentBitrateBps;
}
private void changeRegion(RateControlRegion region, long nowMs)
{
if (rateControlRegion == region)
{
return;
}
rateControlRegion = region;
if (logger.isTraceEnabled())
{
logger.trace(diagnosticContext
.makeTimeSeriesPoint("aimd_region", nowMs)
.addField("aimd_id", hashCode())
.addField("region", region));
}
}
private void changeState(RateControlInput input, long nowMs)
{
switch (currentInput.bwState)
{
case kBwNormal:
if (rateControlState == RateControlState.kRcHold)
{
timeLastBitrateChange = nowMs;
changeState(RateControlState.kRcIncrease, nowMs);
}
break;
case kBwOverusing:
if (rateControlState != RateControlState.kRcDecrease)
{
changeState(RateControlState.kRcDecrease, nowMs);
}
break;
case kBwUnderusing:
changeState(RateControlState.kRcHold, nowMs);
break;
default:
throw new IllegalStateException("currentInput.bwState");
}
}
private void changeState(RateControlState newState, long nowMs)
{
if (rateControlState == newState)
{
return;
}
rateControlState = newState;
if (logger.isTraceEnabled())
{
logger.trace(diagnosticContext
.makeTimeSeriesPoint("aimd_state", nowMs)
.addField("aimd_id", hashCode())
.addField("state", rateControlState));
}
}
public long getFeedBackInterval()
{
// Estimate how often we can send RTCP if we allocate up to 5% of
// bandwidth to feedback.
long interval
= (long)
(kRtcpSize * 8.0 * 1000.0 / (0.05 * currentBitrateBps) + 0.5);
return
Math.min(
Math.max(interval, kMinFeedbackIntervalMs),
kMaxFeedbackIntervalMs);
}
public long getLatestEstimate()
{
return currentBitrateBps;
}
/**
* Returns <tt>true</tt> if the bitrate estimate hasn't been changed for
* more than an RTT, or if the <tt>incomingBitrate</tt> is more than 5%
* above the current estimate. Should be used to decide if we should reduce
* the rate further when over-using.
*
* @param timeNow
* @param incomingBitrateBps
* @return
*/
public boolean isTimeToReduceFurther(long timeNow, long incomingBitrateBps)
{
long bitrateReductionInterval = Math.max(Math.min(rtt, 200L), 10L);
if (timeNow - timeLastBitrateChange >= bitrateReductionInterval)
return true;
if (isValidEstimate())
{
long threshold
= (long) (kWithinIncomingBitrateHysteresis * incomingBitrateBps);
long bitrateDifference = getLatestEstimate() - incomingBitrateBps;
return bitrateDifference > threshold;
}
return false;
}
/**
* Returns <tt>true</tt> if there is a valid estimate of the incoming
* bitrate, <tt>false</tt> otherwise.
*
* @return
*/
public boolean isValidEstimate()
{
return bitrateIsInitialized;
}
private long multiplicativeRateIncrease(
long nowMs,
long lastMs,
long currentBitrateBps)
{
double alpha = 1.08;
if (lastMs > -1)
{
long timeSinceLastUpdateMs = Math.min(nowMs - lastMs, 1000);
alpha = Math.pow(alpha, timeSinceLastUpdateMs / 1000.0);
}
return (long) Math.max(currentBitrateBps * (alpha - 1.0), 1000.0);
}
public void reset()
{
reset(RemoteBitrateEstimatorAbsSendTime.kDefaultMinBitrateBps);
}
private void reset(long minBitrateBps)
{
minConfiguredBitrateBps = minBitrateBps;
currentBitrateBps = /* maxConfiguredBitrateBps */ 30000000L;
avgMaxBitrateKbps = -1F;
varMaxBitrateKbps = 0.4F;
rateControlState = RateControlState.kRcHold;
rateControlRegion = RateControlRegion.kRcMaxUnknown;
timeLastBitrateChange = -1L;
currentInput.bwState = BandwidthUsage.kBwNormal;
currentInput.incomingBitRate = 0L;
currentInput.noiseVar = 1D;
updated = false;
timeFirstIncomingEstimate = -1L;
bitrateIsInitialized = false;
beta = 0.85F;
rtt = kDefaultRttMs;
timeOfLastLog = -1L;
inExperiment = false;
}
public void setEstimate(long bitrateBps, long nowMs)
{
updated = true;
bitrateIsInitialized = true;
currentBitrateBps = changeBitrate(bitrateBps, bitrateBps, nowMs);
}
public void setMinBitrate(long minBitrateBps)
{
minConfiguredBitrateBps = minBitrateBps;
currentBitrateBps = Math.max(minBitrateBps, currentBitrateBps);
}
public void setRtt(long nowMs, long rtt)
{
if (logger.isTraceEnabled())
{
logger.trace(diagnosticContext
.makeTimeSeriesPoint("aimd_rtt", nowMs)
.addField("aimd_id", hashCode())
.addField("rtt", rtt));
}
this.rtt = rtt;
}
public void update(RateControlInput input, long nowMs)
{
if (input == null)
throw new NullPointerException("input");
// Set the initial bit rate value to what we're receiving the first half
// second.
if (!bitrateIsInitialized)
{
if (timeFirstIncomingEstimate < 0L)
{
if (input.incomingBitRate > 0L)
timeFirstIncomingEstimate = nowMs;
}
else
{
long timeSinceFirstIncomingEstimate = nowMs - timeFirstIncomingEstimate;
if (timeSinceFirstIncomingEstimate > kFirstIncomingEstimateExpirationMs)
{
if (input.incomingBitRate > 0L)
{
timeFirstIncomingEstimate = nowMs;
}
else
{
timeFirstIncomingEstimate = -1L;
}
}
else if (timeSinceFirstIncomingEstimate > kInitializationTimeMs
&& input.incomingBitRate > 0L)
{
currentBitrateBps = input.incomingBitRate;
bitrateIsInitialized = true;
}
}
}
if (updated && currentInput.bwState == BandwidthUsage.kBwOverusing)
{
// Only update delay factor and incoming bit rate. We always want to
// react on an over-use.
currentInput.noiseVar = input.noiseVar;
currentInput.incomingBitRate = input.incomingBitRate;
}
else
{
updated = true;
currentInput.copy(input);
}
}
public long updateBandwidthEstimate(long nowMs)
{
currentBitrateBps
= changeBitrate(
currentBitrateBps,
currentInput.incomingBitRate,
nowMs);
if (logger.isTraceEnabled() && isValidEstimate())
{
logger.trace(diagnosticContext
.makeTimeSeriesPoint("aimd_estimate", nowMs)
.addField("aimd_id", hashCode())
.addField("estimate_bps", currentBitrateBps)
.addField("incoming_bps", currentInput.incomingBitRate));
}
if (nowMs - timeOfLastLog > kLogIntervalMs)
timeOfLastLog = nowMs;
return currentBitrateBps;
}
private void updateMaxBitRateEstimate(float incomingBitrateKbps)
{
float alpha = 0.05F;
if (avgMaxBitrateKbps == -1F)
{
avgMaxBitrateKbps = incomingBitrateKbps;
}
else
{
avgMaxBitrateKbps
= (1 - alpha) * avgMaxBitrateKbps + alpha * incomingBitrateKbps;
}
// Estimate the max bit rate variance and normalize the variance with
// the average max bit rate.
float norm = Math.max(avgMaxBitrateKbps, 1F);
varMaxBitrateKbps
= (1 - alpha) * varMaxBitrateKbps
+ alpha
* (avgMaxBitrateKbps - incomingBitrateKbps)
* (avgMaxBitrateKbps - incomingBitrateKbps)
/ norm;
// 0.4 ~= 14 kbit/s at 500 kbit/s
if (varMaxBitrateKbps < 0.4F)
varMaxBitrateKbps = 0.4F;
// 2.5f ~= 35 kbit/s at 500 kbit/s
if (varMaxBitrateKbps > 2.5f)
varMaxBitrateKbps = 2.5f;
}
}

35
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
/**
* webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h
*
* @author Lyubomir Marinov
*/
public enum BandwidthUsage
{
kBwNormal(0), kBwUnderusing(-1), kBwOverusing(1);
private int value;
BandwidthUsage(int value){
this.value = value;
}
public int getValue() {
return value;
}
}

374
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/InterArrival.java Normal file
View File

@@ -0,0 +1,374 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import org.jitsi.utils.TimestampUtils;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.utils.logging2.Logger;
/**
* Helper class to compute the inter-arrival time delta and the size delta
* between two timestamp groups. A timestamp is a 32 bit unsigned number with a
* client defined rate.
*
* webrtc/modules/remote_bitrate_estimator/inter_arrival.cc
* webrtc/modules/remote_bitrate_estimator/inter_arrival.h
*
* @author Lyubomir Marinov
* @author Julian Chukwu
* @author George Politis
*/
class InterArrival
{
private static final int kBurstDeltaThresholdMs = 5;
private final Logger logger;
private final TimeSeriesLogger timeSeriesLogger;
/**
* webrtc/modules/include/module_common_types.h
*
* @param timestamp1
* @param timestamp2
* @return
*/
private static long latestTimestamp(long timestamp1, long timestamp2)
{
return TimestampUtils.latestTimestamp(timestamp1, timestamp2);
}
private boolean burstGrouping;
private TimestampGroup currentTimestampGroup = new TimestampGroup();
private final long kTimestampGroupLengthTicks;
private final DiagnosticContext diagnosticContext;
private TimestampGroup prevTimestampGroup = new TimestampGroup();
private double timestampToMsCoeff;
private int numConsecutiveReorderedPackets;
// After this many packet groups received out of order InterArrival will
// reset, assuming that clocks have made a jump.
private static final int kReorderedResetThreshold = 3;
private static final int kArrivalTimeOffsetThresholdMs = 3000;
/**
* A timestamp group is defined as all packets with a timestamp which are at
* most {@code timestampGroupLengthTicks} older than the first timestamp in
* that group.
*
* @param timestampGroupLengthTicks
* @param timestampToMsCoeff
* @param enableBurstGrouping
*/
public InterArrival(
long timestampGroupLengthTicks,
double timestampToMsCoeff,
boolean enableBurstGrouping,
@NotNull DiagnosticContext diagnosticContext,
@NotNull Logger parentLogger)
{
kTimestampGroupLengthTicks = timestampGroupLengthTicks;
this.timestampToMsCoeff = timestampToMsCoeff;
burstGrouping = enableBurstGrouping;
numConsecutiveReorderedPackets = 0;
this.diagnosticContext = diagnosticContext;
this.logger = parentLogger.createChildLogger(getClass().getName());
this.timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(getClass());
}
private boolean belongsToBurst(long arrivalTimeMs, long timestamp)
{
if (!burstGrouping)
return false;
if (currentTimestampGroup.completeTimeMs < 0)
{
throw new IllegalStateException(
"currentTimestampGroup.completeTimeMs");
}
long arrivalTimeDeltaMs
= arrivalTimeMs - currentTimestampGroup.completeTimeMs;
long timestampDiff = TimestampUtils
.subtractAsUnsignedInt32(timestamp, currentTimestampGroup.timestamp);
long tsDeltaMs = (long) (timestampToMsCoeff * timestampDiff + 0.5);
if (tsDeltaMs == 0)
return true;
long propagationDeltaMs = arrivalTimeDeltaMs - tsDeltaMs;
return
propagationDeltaMs < 0
&& arrivalTimeDeltaMs <= kBurstDeltaThresholdMs;
}
/**
* Returns {@code true} if a delta was computed, or {@code false} if the
* current group is still incomplete or if only one group has been
* completed.
*
* @param timestamp is the timestamp.
* @param arrivalTimeMs is the local time at which the packet arrived.
* @param packetSize is the size of the packet.
* @param deltas {@code timestampDelta} is the computed timestamp delta,
* {@code arrivalTimeDeltaMs} is the computed arrival-time delta,
* {@code packetSizeDelta} is the computed size delta.
* @return
*
* @Note: We have two {@code computeDeltas}.
* One with a valid {@code systemTimeMs} according to webrtc
* implementation as of June 12,2017 and a previous one
* with a default systemTimeMs (-1L). the later may be removed or
* deprecated.
*/
public boolean computeDeltas(
long timestamp,
long arrivalTimeMs,
int packetSize,
long[] deltas){
return computeDeltas(timestamp, arrivalTimeMs, packetSize, deltas, -1L);
}
public boolean computeDeltas(
long timestamp,
long arrivalTimeMs,
int packetSize,
long[] deltas,
long systemTimeMs)
{
if (deltas == null)
throw new NullPointerException("deltas");
if (deltas.length != 3)
throw new IllegalArgumentException("deltas.length");
boolean calculatedDeltas = false;
if (currentTimestampGroup.isFirstPacket())
{
// We don't have enough data to update the filter, so we store it
// until we have two frames of data to process.
currentTimestampGroup.timestamp = timestamp;
currentTimestampGroup.firstTimestamp = timestamp;
}
else if (!isPacketInOrder(timestamp))
{
return false;
}
else if (isNewTimestampGroup(arrivalTimeMs, timestamp))
{
// First packet of a later frame, the previous frame sample is
// ready.
if (prevTimestampGroup.completeTimeMs >= 0)
{
/* long timestampDelta */ deltas[0]
= TimestampUtils.subtractAsUnsignedInt32(
currentTimestampGroup.timestamp,
prevTimestampGroup.timestamp);
long arrivalTimeDeltaMs
= deltas[1]
= currentTimestampGroup.completeTimeMs
- prevTimestampGroup.completeTimeMs;
// Check system time differences to see if we have an unproportional jump
// in arrival time. In that case reset the inter-arrival computations.
long systemTimeDeltaMs =
currentTimestampGroup.lastSystemTimeMs -
prevTimestampGroup.lastSystemTimeMs;
if (prevTimestampGroup.lastSystemTimeMs != -1L &&
currentTimestampGroup.lastSystemTimeMs != -1L &&
arrivalTimeDeltaMs - systemTimeDeltaMs >=
kArrivalTimeOffsetThresholdMs)
{
logger.warn( "The arrival time clock offset has changed (diff = "
+ (arrivalTimeDeltaMs - systemTimeDeltaMs)
+ " ms), resetting.");
Reset();
return false;
}
if (arrivalTimeDeltaMs < 0)
{
++numConsecutiveReorderedPackets;
if (numConsecutiveReorderedPackets >= kReorderedResetThreshold)
{
// The group of packets has been reordered since receiving
// its local arrival timestamp.
logger.warn(
"Packets are being reordered on the path from the "
+ "socket to the bandwidth estimator. Ignoring "
+ "this packet for bandwidth estimation.");
Reset();
}
return false;
}
else
{
numConsecutiveReorderedPackets = 0;
}
/* int packetSizeDelta */ deltas[2]
= (int)
(currentTimestampGroup.size - prevTimestampGroup.size);
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("computed_deltas")
.addField("inter_arrival", hashCode())
.addField("arrival_time_ms", arrivalTimeMs)
.addField("timestamp_delta", deltas[0])
.addField("arrival_time_ms_delta", deltas[1])
.addField("payload_size_delta", deltas[2]));
}
calculatedDeltas = true;
}
prevTimestampGroup.copy(currentTimestampGroup);
// The new timestamp is now the current frame.
currentTimestampGroup.firstTimestamp = timestamp;
currentTimestampGroup.timestamp = timestamp;
currentTimestampGroup.size = 0;
}
else
{
currentTimestampGroup.timestamp = latestTimestamp(
currentTimestampGroup.timestamp, timestamp);
}
// Accumulate the frame size.
currentTimestampGroup.size += packetSize;
currentTimestampGroup.completeTimeMs = arrivalTimeMs;
currentTimestampGroup.lastSystemTimeMs = systemTimeMs;
return calculatedDeltas;
}
/**
* Returns {@code true} if the last packet was the end of the current batch
* and the packet with {@code timestamp} is the first of a new batch.
*
* @param arrivalTimeMs
* @param timestamp
* @return
*/
private boolean isNewTimestampGroup(long arrivalTimeMs, long timestamp)
{
if (currentTimestampGroup.isFirstPacket())
{
return false;
}
else if (belongsToBurst(arrivalTimeMs, timestamp))
{
return false;
}
else
{
long timestampDiff
= TimestampUtils.subtractAsUnsignedInt32(
timestamp, currentTimestampGroup.firstTimestamp);
return timestampDiff > kTimestampGroupLengthTicks;
}
}
/**
* Returns {@code true} if the packet with timestamp {@code timestamp}
* arrived in order.
*
* @param timestamp
* @return
*/
private boolean isPacketInOrder(long timestamp)
{
if (currentTimestampGroup.isFirstPacket())
{
return true;
}
else
{
// Assume that a diff which is bigger than half the timestamp
// interval (32 bits) must be due to reordering. This code is almost
// identical to that in isNewerTimestamp() in module_common_types.h.
long timestampDiff
= TimestampUtils.subtractAsUnsignedInt32(
timestamp, currentTimestampGroup.firstTimestamp);
long tsDeltaMs = (long) (timestampToMsCoeff * timestampDiff + 0.5);
boolean inOrder = timestampDiff < 0x80000000L;
if (!inOrder && timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("reordered_packet")
.addField("inter_arrival", hashCode())
.addField("ts_delta_ms", tsDeltaMs));
}
return inOrder;
}
}
private static class TimestampGroup
{
public long completeTimeMs = -1L;
public long size = 0L;
public long firstTimestamp = 0L;
public long timestamp = 0L;
public long lastSystemTimeMs = -1L;
/**
* Assigns the values of the fields of <tt>source</tt> to the respective
* fields of this {@code TimestampGroup}.
*
* @param source the {@code TimestampGroup} the values of the fields of
* which are to be assigned to the respective fields of this
* {@code TimestampGroup}
*/
public void copy(TimestampGroup source)
{
completeTimeMs = source.completeTimeMs;
firstTimestamp = source.firstTimestamp;
size = source.size;
timestamp = source.timestamp;
}
public boolean isFirstPacket()
{
return completeTimeMs == -1L;
}
}
public void Reset() {
numConsecutiveReorderedPackets = 0;
currentTimestampGroup = new TimestampGroup();
prevTimestampGroup = new TimestampGroup();
}
}

44
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OverUseDetectorOptions.java Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.config.OveruseDetectorConfig;
/**
* Bandwidth over-use detector options. These are used to drive experimentation
* with bandwidth estimation parameters.
*
* webrtc/common_types.h
*
* @author Lyubomir Marinov
*/
class OverUseDetectorOptions
{
public double initialAvgNoise = 0.0D;
public final double[][] initialE
= new double[][] { { 100, 0 }, { 0, 1e-1 } };
public double initialOffset = 0.0D;
public final double[] initialProcessNoise = new double[] { 1e-10, 1e-2 };
public double initialSlope = 8.0D / 512.0D;
public double initialThreshold = OveruseDetectorConfig.Companion.getInitialThreshold();
public double initialVarNoise = 50.0D;
}

203
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java Normal file
View File

@@ -0,0 +1,203 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import edu.umd.cs.findbugs.annotations.*;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging.*;
import org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.config.OveruseDetectorConfig;
import java.util.*;
/**
* webrtc/modules/remote_bitrate_estimator/overuse_detector.cc
* webrtc/modules/remote_bitrate_estimator/overuse_detector.h
*
* @author Lyubomir Marinov
*/
@SuppressFBWarnings(
value = {"FE_FLOATING_POINT_EQUALITY", "DB_DUPLICATE_BRANCHES"},
justification = "We only compare timeOverUsing with the special value -1")
class OveruseDetector
{
private final TimeSeriesLogger timeSeriesLogger;
private static final double kMaxAdaptOffsetMs = 15.0;
private BandwidthUsage hypothesis = BandwidthUsage.kBwNormal;
private static final double kDown = 0D;
private static final double kUp = 0D;
private long lastUpdateMs = -1L;
private int overuseCounter;
private static final double overusingTimeThreshold = OveruseDetectorConfig.Companion.getOverusingTimeThreshold();
private double prevOffset;
private double threshold;
private double timeOverUsing = -1D;
private final DiagnosticContext diagnosticContext;
public OveruseDetector(
OverUseDetectorOptions options,
@NotNull DiagnosticContext diagnosticContext)
{
Objects.requireNonNull(options, "options");
threshold = options.initialThreshold;
timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(getClass());
this.diagnosticContext = diagnosticContext;
}
/**
* Update the detection state based on the estimated inter-arrival time
* delta offset. {@code timestampDelta} is the delta between the last
* timestamp which the estimated offset is based on and the last timestamp
* on which the last offset was based on, representing the time between
* detector updates. {@code numOfDeltas} is the number of deltas the offset
* estimate is based on. Returns the state after the detection update.
*
* @param offset
* @param tsDelta
* @param numOfDeltas
* @param nowMs
* @return
*/
public BandwidthUsage detect(
double offset,
double tsDelta,
int numOfDeltas,
long nowMs)
{
if (numOfDeltas < 2)
return BandwidthUsage.kBwNormal;
double prev_offset = this.prevOffset;
this.prevOffset = offset;
double T = Math.min(numOfDeltas, 60) * offset;
boolean newHypothesis = false;
if (T > threshold)
{
if (timeOverUsing == -1)
{
// Initialize the timer. Assume that we've been
// over-using half of the time since the previous
// sample.
timeOverUsing = tsDelta / 2;
}
else
{
// Increment timer
timeOverUsing += tsDelta;
}
overuseCounter++;
if (timeOverUsing > overusingTimeThreshold && overuseCounter > 1)
{
if (offset >= prev_offset)
{
timeOverUsing = 0;
overuseCounter = 0;
hypothesis = BandwidthUsage.kBwOverusing;
newHypothesis = true;
}
}
}
else if (T < -threshold)
{
timeOverUsing = -1;
overuseCounter = 0;
hypothesis = BandwidthUsage.kBwUnderusing;
newHypothesis = true;
}
else
{
timeOverUsing = -1;
overuseCounter = 0;
hypothesis = BandwidthUsage.kBwNormal;
newHypothesis = true;
}
if (newHypothesis && timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("utilization_hypothesis", nowMs)
.addField("detector", hashCode())
.addField("offset", offset)
.addField("prev_offset", prev_offset)
.addField("T", T)
.addField("threshold", threshold)
.addField("hypothesis", hypothesis.getValue()));
}
updateThreshold(T, nowMs);
return hypothesis;
}
/**
* Returns the current detector state.
*
* @return
*/
public BandwidthUsage getState()
{
return hypothesis;
}
private void updateThreshold(double modifiedOffset, long nowMs)
{
if (lastUpdateMs == -1)
lastUpdateMs = nowMs;
if (Math.abs(modifiedOffset) > threshold + kMaxAdaptOffsetMs)
{
// Avoid adapting the threshold to big latency spikes, caused e.g.,
// by a sudden capacity drop.
lastUpdateMs = nowMs;
return;
}
double k = Math.abs(modifiedOffset) < threshold ? kDown : kUp;
threshold
+= k
* (Math.abs(modifiedOffset) - threshold)
* (nowMs - lastUpdateMs);
final double kMinThreshold = 6;
final double kMaxThreshold = 600;
threshold = Math.min(Math.max(threshold, kMinThreshold), kMaxThreshold);
lastUpdateMs = nowMs;
}
public double getThreshold()
{
return threshold;
}
}

330
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseEstimator.java Normal file
View File

@@ -0,0 +1,330 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.utils.logging2.Logger;
import java.util.*;
/**
* webrtc/modules/remote_bitrate_estimator/overuse_estimator.cc
* webrtc/modules/remote_bitrate_estimator/overuse_estimator.h
*
* @author Lyubomir Marinov
*/
class OveruseEstimator
{
/**
* The <tt>Logger</tt> used by
* <tt>OveruseEstimator</tt> instances for
* logging output.
*/
private final Logger logger;
/**
* The <tt>TimeSeriesLogger</tt> used for time series.
*/
private final TimeSeriesLogger timeSeriesLogger;
private static final int kDeltaCounterMax = 1000;
private static final int kMinFramePeriodHistoryLength = 60;
/**
* Creates and returns a deep copy of a <tt>double</tt> two-dimensional
* matrix.
*
* @param matrix the <tt>double</tt> two-dimensional matrix to create and
* return a deep copy of
* @return a deep copy of <tt>matrix</tt>
*/
private static double[][] clone(double[][] matrix)
{
int length = matrix.length;
double[][] clone;
clone = new double[length][];
for (int i = 0; i < length; i++)
clone[i] = matrix[i].clone();
return clone;
}
private double avgNoise;
private final double[][] E;
private DiagnosticContext diagnosticContext;
/**
* Reduces the effects of allocations and garbage collection of the method
* {@code update}.
*/
private final double[] Eh = new double[2];
/**
* Reduces the effects of allocations and garbage collection of the method
* {@code update}.
*/
private final double[] h = new double[2];
/**
* Reduces the effects of allocations and garbage collection of the method
* {@code update}.
*/
private final double[][] IKh = new double[][] { { 0, 0 }, { 0, 0 } };
/**
* Reduces the effects of allocations and garbage collection of the method
* {@code update}.
*/
private final double[] K = new double[2];
private int numOfDeltas;
private double offset;
private double prevOffset;
private final double[] processNoise;
private double slope;
/**
* Store the tsDelta history into a {@code double[]} used as a circular buffer
* Original c++ code uses std::list<double> but in Java this translate
* to {@code List<Double>} which causes a lot of autoboxing
*/
private final double[] tsDeltaHist = new double[kMinFramePeriodHistoryLength];
/**
* Index to insert next value into {@link #tsDeltaHist}
*/
private int tsDeltaHistInsIdx;
private double varNoise;
public OveruseEstimator(
OverUseDetectorOptions options,
@NotNull DiagnosticContext diagnosticContext,
@NotNull Logger parentLogger)
{
logger = parentLogger.createChildLogger(getClass().getName());
timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(getClass());
this.diagnosticContext = diagnosticContext;
slope = options.initialSlope;
offset = options.initialOffset;
prevOffset = offset;
avgNoise = options.initialAvgNoise;
varNoise = options.initialVarNoise;
E = clone(options.initialE);
processNoise = options.initialProcessNoise.clone();
/**
* Initialize {@link tsDeltaHist} with {@code Double.MAX_VALUE}
* to simplify {@link updateMinFramePeriod}
*/
Arrays.fill(tsDeltaHist, Double.MAX_VALUE);
}
/**
* Returns the number of deltas which the current over-use estimator state
* is based on.
*
* @return
*/
public int getNumOfDeltas()
{
return numOfDeltas;
}
/**
* Returns the estimated inter-arrival time delta offset in ms.
*
* @return
*/
public double getOffset()
{
return offset;
}
/**
* Returns the estimated noise/jitter variance in ms^2.
*
* @return
*/
public double getVarNoise()
{
return varNoise;
}
/**
* Update the estimator with a new sample. The deltas should represent
* deltas between timestamp groups as defined by the InterArrival class.
* {@code currentHypothesis} should be the hypothesis of the over-use
* detector at this time.
*
* @param tDelta
* @param tsDelta
* @param sizeDelta
* @param currentHypothesis
*/
public void update(
long tDelta,
double tsDelta,
int sizeDelta,
BandwidthUsage currentHypothesis, long systemTimeMs)
{
double minFramePeriod = updateMinFramePeriod(tsDelta);
double tTsDelta = tDelta - tsDelta;
++numOfDeltas;
if (numOfDeltas > kDeltaCounterMax)
numOfDeltas = kDeltaCounterMax;
// Update the Kalman filter
E[0][0] += processNoise[0];
E[1][1] += processNoise[1];
if ((currentHypothesis == BandwidthUsage.kBwOverusing
&& offset < prevOffset)
|| (currentHypothesis == BandwidthUsage.kBwUnderusing
&& offset > prevOffset))
{
E[1][1] += 10D * processNoise[1];
}
double[] h = this.h;
double[] Eh = this.Eh;
h[0] = sizeDelta;
h[1] = 1D;
Eh[0] = E[0][0] * h[0] + E[0][1] * h[1];
Eh[1] = E[1][0] * h[0] + E[1][1] * h[1];
double residual = tTsDelta - slope * h[0] - offset;
boolean inStableState = (currentHypothesis == BandwidthUsage.kBwNormal);
// We try to filter out very late frames. For instance periodic key
// frames doesn't fit the Gaussian model well.
double maxResidual = 3D * Math.sqrt(varNoise);
double residualForUpdateNoiseEstimate
= (Math.abs(residual) < maxResidual)
? residual
: (residual < 0 ? -maxResidual : maxResidual);
updateNoiseEstimate(
residualForUpdateNoiseEstimate,
minFramePeriod,
inStableState);
double denom = varNoise + h[0]*Eh[0] + h[1]*Eh[1];
double[] K = this.K;
double[][] IKh = this.IKh;
K[0] = Eh[0] / denom;
K[1] = Eh[1] / denom;
IKh[0][0] = 1D - K[0]*h[0];
IKh[0][1] = -K[0]*h[1];
IKh[1][0] = -K[1]*h[0];
IKh[1][1] = 1D - K[1]*h[1];
double e00 = E[0][0];
double e01 = E[0][1];
// Update state.
E[0][0] = e00 * IKh[0][0] + E[1][0] * IKh[0][1];
E[0][1] = e01 * IKh[0][0] + E[1][1] * IKh[0][1];
E[1][0] = e00 * IKh[1][0] + E[1][0] * IKh[1][1];
E[1][1] = e01 * IKh[1][0] + E[1][1] * IKh[1][1];
// Covariance matrix, must be positive semi-definite.
boolean positiveSemiDefinite
= E[0][0] + E[1][1] >= 0
&& E[0][0] * E[1][1] - E[0][1] * E[1][0] >= 0
&& E[0][0] >= 0;
if (!positiveSemiDefinite)
throw new IllegalStateException("positiveSemiDefinite");
slope = slope + K[0] * residual;
prevOffset = offset;
offset = offset + K[1] * residual;
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("delay_variation_estimation", systemTimeMs)
.addField("estimator", hashCode())
.addField("time_delta", tDelta)
.addField("ts_delta", tsDelta)
.addField("tts_delta", tTsDelta)
.addField("offset", offset)
.addField("hypothesis", currentHypothesis.getValue()));
}
}
private double updateMinFramePeriod(double tsDelta)
{
/**
* Change from C++ version:
* We use {@link tsDeltaHist} as a circular buffer initialized
* with {@code Double.MAX_VALUE}, so we insert new {@link tsDelta}
* value at {@link tsDeltaHistInsIdx} and we search for the
* minimum value in {@link tsDeltaHist}
*/
tsDeltaHist[tsDeltaHistInsIdx] = tsDelta;
tsDeltaHistInsIdx = (tsDeltaHistInsIdx + 1) % tsDeltaHist.length;
double min = tsDelta;
for (double d : tsDeltaHist)
{
if (d < min)
min = d;
}
return min;
}
private void updateNoiseEstimate(
double residual,
double tsDelta,
boolean stableState)
{
if (!stableState)
return;
// Faster filter during startup to faster adapt to the jitter level of
// the network alpha is tuned for 30 frames per second, but is scaled
// according to tsDelta.
double alpha = 0.01D;
if (numOfDeltas > 10 * 30)
alpha = 0.002D;
// Only update the noise estimate if we're not over-using beta is a
// function of alpha and the time delta since the previous update.
double beta = Math.pow(1 - alpha, tsDelta * 30D / 1000D);
avgNoise = beta * avgNoise + (1 - beta) * residual;
varNoise
= beta * varNoise
+ (1 - beta) * (avgNoise - residual) * (avgNoise - residual);
if (varNoise < 1)
varNoise = 1;
}
}

58
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlInput.java Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
/**
* webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h
*
* @author Lyubomir Marinov
*/
@SuppressFBWarnings("PA_PUBLIC_PRIMITIVE_ATTRIBUTE")
class RateControlInput
{
public BandwidthUsage bwState;
public long incomingBitRate;
public double noiseVar;
public RateControlInput(
BandwidthUsage bwState,
long incomingBitRate,
double noiseVar)
{
this.bwState = bwState;
this.incomingBitRate = incomingBitRate;
this.noiseVar = noiseVar;
}
/**
* Assigns the values of the fields of <tt>source</tt> to the respective
* fields of this <tt>RateControlInput</tt>.
*
* @param source the <tt>RateControlInput</tt> the values of the fields of
* which are to be assigned to the respective fields of this
* <tt>RateControlInput</tt>
*/
public void copy(RateControlInput source)
{
bwState = source.bwState;
incomingBitRate = source.incomingBitRate;
noiseVar = source.noiseVar;
}
}

28
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlRegion.java Normal file
View File

@@ -0,0 +1,28 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
/**
* webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h
*
* @author Lyubomir Marinov
*/
enum RateControlRegion
{
kRcNearMax,
kRcAboveMax,
kRcMaxUnknown
}

28
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlState.java Normal file
View File

@@ -0,0 +1,28 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
/**
* webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h
*
* @author Lyubomir Marinov
*/
enum RateControlState
{
kRcHold,
kRcIncrease,
kRcDecrease
}

463
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java Normal file
View File

@@ -0,0 +1,463 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.util.*;
import org.jitsi.utils.logging.DiagnosticContext;
import org.jitsi.utils.logging.TimeSeriesLogger;
import org.jitsi.utils.logging2.*;
import java.time.*;
/**
* webrtc.org abs_send_time implementation as of June 26, 2017.
* commit ID: 23fbd2aa2c81d065b84d17b09b747e75672e1159
*
* @author Julian Chukwu
* @author George Politis
*/
public class RemoteBitrateEstimatorAbsSendTime
{
/**
* webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h
*/
static final private int kBitrateWindowMs = 1000;
static final int kDefaultMinBitrateBps = 30000;
static final private int kStreamTimeOutMs = 2000;
/**
* The {@link TimeSeriesLogger} to be used by this instance to print time
* series.
*/
private static final TimeSeriesLogger timeSeriesLogger
= TimeSeriesLogger.getTimeSeriesLogger(
RemoteBitrateEstimatorAbsSendTime.class);
/**
* Defines the number of digits in the AST representation (24 bits, 6.18
* fixed point) after the radix.
*/
private final static int kAbsSendTimeFraction = 18;
/**
* Defines the upshift (left bit-shift) to apply to AST (24 bits, 6.18 fixed
* point) to make it inter-arrival compatible (expanded AST, 32 bits, 6.26
* fixed point).
*/
private final static int kAbsSendTimeInterArrivalUpshift = 8;
/**
* This is used in the {@link InterArrival} computations. In this estimator
* a timestamp group is defined as all packets with a timestamp which are at
* most 5ms older than the first timestamp in that group.
*/
private final static int kTimestampGroupLengthMs = 5;
/**
* Defines the number of digits in the expanded AST representation (32 bits,
* 6.26 fixed point) after the radix.
*/
private final static int kInterArrivalShift
= kAbsSendTimeFraction + kAbsSendTimeInterArrivalUpshift;
/**
* Converts the {@link #kTimestampGroupLengthMs} into "ticks" for use with
* the {@link InterArrival}.
*/
private static final long kTimestampGroupLengthTicks
= (kTimestampGroupLengthMs << kInterArrivalShift) / 1000;
/**
* Defines the expanded AST (32 bits) to millis conversion rate. Units are
* ms per timestamp
*/
private static final double
kTimestampToMs = (double) 1000 / (1 << kInterArrivalShift);
/**
* Reduces the effects of allocations and garbage collection of the method
* {@code incomingPacket}.
*/
private final long[] deltas = new long[3];
/**
* Reduces the effects of allocations and garbage collection of the method
* {@link #incomingPacketInfo(long, long, long, int)}} by promoting the
* {@code RateControlInput} instance from a local variable to a field and
* reusing the same instance across method invocations. (Consequently, the
* default values used to initialize the field are of no importance because
* they will be overwritten before they are actually used.)
*/
private final RateControlInput input
= new RateControlInput(BandwidthUsage.kBwNormal, 0L, 0D);
private long lastPacketTimeMs;
/**
* The time (in millis) when we saw the first packet. Useful to determine
* the probing period.
*/
private long firstPacketTimeMs;
/**
* Keeps track of the last time (in millis) that we updated the bitrate
* estimate.
*/
private long lastUpdateMs;
/**
* The rate control implementation based on additive increases of bitrate
* when no over-use is detected and multiplicative decreases when over-uses
* are detected.
*/
private final AimdRateControl remoteRate;
/**
* Holds the {@link InterArrival}, {@link OveruseEstimator} and
* {@link OveruseDetector} instances of this RBE.
*/
private Detector detector;
/**
* Keeps track of how much data we're receiving.
*/
private BitrateTracker incomingBitrate;
/**
* Determines whether or not the incoming bitrate is initialized or not.
*/
private boolean incomingBitrateInitialized;
/**
* The {@link DiagnosticContext} of this instance.
*/
private final DiagnosticContext diagnosticContext;
/**
* The instance logger.
*/
private final Logger logger;
/**
* Ctor.
*
* @param diagnosticContext the {@link DiagnosticContext} of this instance.
*/
public RemoteBitrateEstimatorAbsSendTime(
@NotNull DiagnosticContext diagnosticContext,
@NotNull Logger parentLogger)
{
this.logger = parentLogger.createChildLogger(getClass().getName());
this.diagnosticContext = diagnosticContext;
this.remoteRate = new AimdRateControl(diagnosticContext);
this.incomingBitrate = new BitrateTracker(Duration.ofMillis(kBitrateWindowMs));
this.incomingBitrateInitialized = false;
this.firstPacketTimeMs = -1;
this.lastPacketTimeMs = -1;
this.lastUpdateMs = -1;
}
/**
* Reset back to the state immediately after construction.
*/
public synchronized void reset()
{
remoteRate.reset();
this.incomingBitrate = new BitrateTracker(Duration.ofMillis(kBitrateWindowMs));
this.incomingBitrateInitialized = false;
this.firstPacketTimeMs = -1;
this.lastPacketTimeMs = -1;
this.lastUpdateMs = -1;
this.detector = null;
}
/**
* Notifies this instance of an incoming packet.
*
* @param nowMs the current time when this method is called
* @param arrivalTimeMs the arrival time of the packet in millis.
* @param sendTimeMs the send time of the packet in millis
* @param payloadSize the payload size of the packet.
*/
public synchronized void incomingPacketInfo(
long nowMs,
long sendTimeMs,
long arrivalTimeMs,
int payloadSize)
{
long sendTime24bits = convertMsTo24Bits(sendTimeMs);
// Shift up send time to use the full 32 bits that inter_arrival
// works with, so wrapping works properly.
long timestamp = sendTime24bits << kAbsSendTimeInterArrivalUpshift;
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("in_pkt", nowMs)
.addField("rbe_id", hashCode())
.addField("recv_ts_ms", arrivalTimeMs)
.addField("timestamp", timestamp)
.addField("pkt_sz_bytes", payloadSize));
}
// should be broken out from here.
// Check if incoming bitrate estimate is valid, and if it
// needs to be reset.
long incomingBitrate_ = incomingBitrate.getRateBps(arrivalTimeMs);
if (incomingBitrate_ != 0)
{
incomingBitrateInitialized = true;
}
else if (incomingBitrateInitialized)
{
// Incoming bitrate had a previous valid value, but now not
// enough data point are left within the current window.
// Reset incoming bitrate estimator so that the window
// size will only contain new data points.
incomingBitrate = new BitrateTracker(Duration.ofMillis(kBitrateWindowMs));
incomingBitrateInitialized = false;
}
incomingBitrate.update(DataSizeKt.getBytes(payloadSize), arrivalTimeMs);
if (firstPacketTimeMs == -1)
{
firstPacketTimeMs = nowMs;
}
boolean updateEstimate = false;
checkTimeouts(nowMs);
lastPacketTimeMs = nowMs;
long[] deltas = this.deltas;
/* long timestampDelta */ deltas[0] = 0;
/* long timeDelta */ deltas[1] = 0;
/* int sizeDelta */ deltas[2] = 0;
if (detector == null)
{
detector = new Detector(new OverUseDetectorOptions(), true, logger);
}
if (detector.interArrival.computeDeltas(
timestamp, arrivalTimeMs, payloadSize, deltas, nowMs))
{
double tsDeltaMs = deltas[0] * kTimestampToMs;
detector.estimator.update(
/* timeDelta */ deltas[1],
/* timestampDelta */ tsDeltaMs,
/* sizeDelta */ (int) deltas[2],
detector.detector.getState(), nowMs);
detector.detector.detect(
detector.estimator.getOffset(), tsDeltaMs,
detector.estimator.getNumOfDeltas(), arrivalTimeMs);
}
// Check if it's time for a periodic update or if we
// should update because of an over-use.
if (lastUpdateMs == -1
|| nowMs - lastUpdateMs > remoteRate.getFeedBackInterval())
{
updateEstimate = true;
}
else if (
detector.detector.getState() == BandwidthUsage.kBwOverusing)
{
long incomingRate_ = incomingBitrate.getRateBps(arrivalTimeMs);
if (incomingRate_ > 0 && remoteRate
.isTimeToReduceFurther(nowMs, incomingBitrate_))
{
updateEstimate = true;
}
}
if (updateEstimate)
{
// The first overuse should immediately trigger a new estimate.
// We also have to update the estimate immediately if we are
// overusing and the target bitrate is too high compared to
// what we are receiving.
input.bwState = detector.detector.getState();
input.incomingBitRate = incomingBitrate.getRateBps(arrivalTimeMs);
input.noiseVar = detector.estimator.getVarNoise();
remoteRate.update(input, nowMs);
remoteRate.updateBandwidthEstimate(nowMs);
updateEstimate = remoteRate.isValidEstimate();
}
if (updateEstimate)
{
lastUpdateMs = nowMs;
}
}
/**
* Timeouts SSRCs that have not received any data for
* kTimestampGroupLengthMs millis.
*
* @param nowMs the current time in millis.
*/
private synchronized void checkTimeouts(long nowMs)
{
if (nowMs - lastPacketTimeMs > kStreamTimeOutMs)
{
detector = null;
// We deliberately don't reset the first_packet_time_ms_
// here for now since we only probe for bandwidth in the
// beginning of a call right now.
}
}
/**
* Called when an RTP sender has a new round-trip time estimate.
*/
public synchronized void onRttUpdate(long nowMs, long avgRttMs)
{
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("new_rtt", nowMs)
.addField("avg_ms", avgRttMs));
}
remoteRate.setRtt(nowMs, avgRttMs);
}
public synchronized long getLatestEstimate()
{
long bitrateBps;
if (!remoteRate.isValidEstimate())
{
return -1;
}
if (detector == null)
{
bitrateBps = 0;
}
else
{
bitrateBps = remoteRate.getLatestEstimate();
}
return bitrateBps;
}
/**
* Sets the minimum bitrate for this instance.
*
* @param minBitrateBps the minimum bitrate in bps.
*/
public synchronized void setMinBitrate(int minBitrateBps)
{
// Called from both the configuration thread and the network thread.
// Shouldn't be called from the network thread in the future.
remoteRate.setMinBitrate(minBitrateBps);
}
/**
* Get various statistics about the estimation process. [Local addition, not in original C++.]
*/
public synchronized @Nullable Statistics getStatistics()
{
if (detector == null)
{
return null;
}
return new Statistics(
detector.estimator.getOffset(),
detector.detector.getThreshold(),
detector.detector.getState()
);
}
/**
* Holds the {@link InterArrival}, {@link OveruseEstimator} and
* {@link OveruseDetector} instances that estimate the remote bitrate of a
* stream.
*/
private class Detector
{
/**
* Computes the send-time and recv-time deltas to feed to the estimator.
*/
private final InterArrival interArrival;
/**
* The Kalman filter implementation that estimates the jitter.
*/
private final OveruseEstimator estimator;
/**
* The overuse detector that compares the jitter to an adaptive threshold.
*/
private final OveruseDetector detector;
/**
* Ctor.
*
* @param options the over-use detector options.
* @param enableBurstGrouping true to activate burst detection, false
* otherwise
*/
Detector(OverUseDetectorOptions options,
boolean enableBurstGrouping,
Logger parentLogger)
{
this.interArrival = new InterArrival(
kTimestampGroupLengthTicks, kTimestampToMs,
enableBurstGrouping, diagnosticContext, parentLogger);
this.estimator = new OveruseEstimator(options, diagnosticContext, logger);
this.detector = new OveruseDetector(options, diagnosticContext);
}
}
/**
* Converts rtp timestamps to 24bit timestamp equivalence
* @param timeMs is the RTP timestamp e.g System.currentTimeMillis().
* @return time stamp representation in 24 bit representation.
*/
public static long convertMsTo24Bits(long timeMs)
{
return (((timeMs << kAbsSendTimeFraction) + 500) / 1000) & 0x00FFFFFF;
}
/**
* Various statistics about the estimation process. [Local addition, not in original C++.]
*/
public static class Statistics {
public double offset;
public double threshold;
public BandwidthUsage hypothesis;
public Statistics(double o, double t, BandwidthUsage h)
{
offset = o;
threshold = t;
hypothesis = h;
}
}
}

38
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateObserver.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator;
import java.util.*;
/**
* <tt>RemoteBitrateObserver</tt> is used to signal changes in bitrate estimates
* for the incoming streams.
*
* webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h
*
* @author Lyubomir Marinov
*/
public interface RemoteBitrateObserver
{
/**
* Called when a receive channel group has a new bitrate estimate for the
* incoming streams.
*
* @param ssrcs
* @param bitrate
*/
void onReceiveBitrateChanged(Collection<Long> ssrcs, long bitrate);
}

882
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/SendSideBandwidthEstimation.java Normal file
View File

@@ -0,0 +1,882 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation;
import org.jetbrains.annotations.NotNull;
import org.jitsi.utils.logging.DiagnosticContext;
import org.jitsi.utils.logging.TimeSeriesLogger;
import org.jitsi.utils.logging2.Logger;
import org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation.config.SendSideBandwidthEstimationConfig;
import java.time.Duration;
import java.util.*;
/**
* Implements the send-side bandwidth estimation described in
* https://tools.ietf.org/html/draft-ietf-rmcat-gcc-01
* Heavily based on code from webrtc.org (send_side_bandwidth_estimation.cc,
* commit ID 7ad9e661f8a035d49d049ccdb87c77ae8ecdfa35).
*
* @author Boris Grozev
*/
public class SendSideBandwidthEstimation
{
/**
* send_side_bandwidth_estimation.cc
*/
private static final int kBweIncreaseIntervalMs = 1000;
/**
* send_side_bandwidth_estimation.cc
*/
private static final long kBweDecreaseIntervalMs = 300;
/**
* send_side_bandwidth_estimation.cc
*/
private static final int kDefaultMinBitrateBps = 10000;
/**
* send_side_bandwidth_estimation.cc
*/
private static final int kDefaultMaxBitrateBps = 1000000000;
/**
* send_side_bandwidth_estimation.cc
*/
private static final int kStartPhaseMs = 2000;
/**
* send_side_bandwidth_estimation.cc
*/
private static final int kLimitNumPackets = 20;
/**
* send_side_bandwidth_estimation.cc
*
* Expecting that RTCP feedback is sent uniformly within [0.5, 1.5]s
* intervals.
*/
private static final long kFeedbackIntervalMs = 1500;
/**
* send_side_bandwidth_estimation.cc
*/
private static final double kPacketReportTimeoutIntervals = 1.2;
/**
* send_side_bandwidth_estimation.cc
*/
private static final long kFeedbackTimeoutIntervals = 3;
/**
* send_side_bandwidth_estimation.cc
*/
private static final long kTimeoutIntervalMs = 1000;
/**
* The random number generator for all instances of this class.
*/
private static final Random kRandom = new Random();
/**
* The <tt>Logger</tt> used by this instance for logging output.
*/
private final Logger logger;
/**
* The {@link TimeSeriesLogger} to be used by this instance to print time
* series.
*/
private static final TimeSeriesLogger timeSeriesLogger
= TimeSeriesLogger.getTimeSeriesLogger(
SendSideBandwidthEstimation.class);
/**
* send_side_bandwidth_estimation.h
*/
private final double low_loss_threshold_;
/**
* send_side_bandwidth_estimation.h
*/
private final double high_loss_threshold_;
/**
* send_side_bandwidth_estimation.h
*/
private final int bitrate_threshold_bps_;
/**
* send_side_bandwidth_estimation.h
*/
private long first_report_time_ms_ = -1;
/**
* send_side_bandwidth_estimation.h
*/
private int lost_packets_since_last_loss_update_Q8_ = 0;
/**
* send_side_bandwidth_estimation.h
*/
private int expected_packets_since_last_loss_update_ = 0;
/**
* send_side_bandwidth_estimation.h
*/
private boolean has_decreased_since_last_fraction_loss_ = false;
/**
* send_side_bandwidth_estimation.h
*
* uint8_t last_fraction_loss_;
*/
private int last_fraction_loss_ = 0;
/**
* send_side_bandwidth_estimation.h
*/
private long last_feedback_ms_ = -1;
/**
* send_side_bandwidth_estimation.h
*/
private long last_packet_report_ms_ = -1;
/**
* send_side_bandwidth_estimation.h
*/
private long last_timeout_ms_ = -1;
/**
* send_side_bandwidth_estimation.h
*/
private final boolean in_timeout_experiment_;
/**
* send_side_bandwidth_estimation.h
*/
private int min_bitrate_configured_ = kDefaultMinBitrateBps;
/**
* send_side_bandwidth_estimation.h
*/
private int max_bitrate_configured_ = kDefaultMaxBitrateBps;
/**
* send_side_bandwidth_estimation.h
*/
private long time_last_decrease_ms_= 0;
/**
* send_side_bandwidth_estimation.h
*/
private long bwe_incoming_ = 0;
/**
* send_side_bandwidth_estimation.h
*/
private long bitrate_;
/**
* send_side_bandwidth_estimation.h
*/
private Deque<BitrateHistoryRecord> min_bitrate_history_ = new LinkedList<>();
/**
* The {@link DiagnosticContext} of this instance.
*/
private final DiagnosticContext diagnosticContext;
/**
* The most recent RTT calculation we've received for our connection with
* the remote endpoint in milliseconds.
*/
private long rttMs;
/**
* The instance that holds stats for this instance.
*/
private final Statistics statistics = new Statistics();
public SendSideBandwidthEstimation(DiagnosticContext diagnosticContext, long startBitrate,
@NotNull Logger parentLogger)
{
logger = parentLogger.createChildLogger(getClass().getName());
this.diagnosticContext = diagnosticContext;
double lossExperimentProbability = SendSideBandwidthEstimationConfig.lossExperimentProbability();
if (kRandom.nextFloat() < lossExperimentProbability)
{
low_loss_threshold_ = SendSideBandwidthEstimationConfig.experimentalLowLossThreshold();
high_loss_threshold_ = SendSideBandwidthEstimationConfig.experimentalHighLossThreshold();
bitrate_threshold_bps_ = (int) SendSideBandwidthEstimationConfig.experimentalBitrateThresholdBps();
}
else
{
low_loss_threshold_ = SendSideBandwidthEstimationConfig.defaultLowLossThreshold();
high_loss_threshold_ = SendSideBandwidthEstimationConfig.defaultHighLossThreshold();
bitrate_threshold_bps_ = (int) SendSideBandwidthEstimationConfig.defaultBitrateThresholdBps();
}
double timeoutExperimentProbability = SendSideBandwidthEstimationConfig.timeoutExperimentProbability();
in_timeout_experiment_
= kRandom.nextFloat() < timeoutExperimentProbability;
setBitrate(startBitrate);
}
public synchronized void reset(long startBitrate)
{
first_report_time_ms_ = -1;
lost_packets_since_last_loss_update_Q8_ = 0;
expected_packets_since_last_loss_update_ = 0;
has_decreased_since_last_fraction_loss_ = false;
last_fraction_loss_ = 0;
last_feedback_ms_ = -1;
last_packet_report_ms_ = -1;
last_timeout_ms_ = -1;
time_last_decrease_ms_= 0;
bwe_incoming_ = 0;
min_bitrate_history_.clear();
setBitrate(startBitrate);
statistics.reset();
}
/**
* bool SendSideBandwidthEstimation::IsInStartPhase(int64_t now_ms)
*/
private synchronized boolean isInStartPhase(long now)
{
return first_report_time_ms_ == -1 ||
now - first_report_time_ms_ < kStartPhaseMs;
}
/**
* int SendSideBandwidthEstimation::CapBitrateToThresholds
*/
private synchronized long capBitrateToThresholds(long bitrate)
{
if (bwe_incoming_ > 0 && bitrate > bwe_incoming_)
{
bitrate = bwe_incoming_;
}
if (bitrate > max_bitrate_configured_)
{
bitrate = max_bitrate_configured_;
}
if (bitrate < min_bitrate_configured_)
{
bitrate = min_bitrate_configured_;
}
return bitrate;
}
/**
* void SendSideBandwidthEstimation::UpdateEstimate(int64_t now_ms)
*/
synchronized public void updateEstimate(long now)
{
long bitrate = bitrate_;
// We trust the REMB during the first 2 seconds if we haven't had any
// packet loss reported, to allow startup bitrate probing.
if (last_fraction_loss_ == 0 && isInStartPhase(now) &&
bwe_incoming_ > bitrate)
{
setBitrate(capBitrateToThresholds(bwe_incoming_));
min_bitrate_history_.clear();
min_bitrate_history_.addLast(new BitrateHistoryRecord(now, bitrate_));
return;
}
updateMinHistory(now);
if (last_packet_report_ms_ == -1)
{
// No feedback received.
bitrate_ = capBitrateToThresholds(bitrate_);
return;
}
long time_since_packet_report_ms = now - last_packet_report_ms_;
long time_since_feedback_ms = now - last_feedback_ms_;
if (time_since_packet_report_ms
< kPacketReportTimeoutIntervals * kFeedbackIntervalMs)
{
// We only care about loss above a given bitrate threshold.
float loss = last_fraction_loss_ / 256.0f;
// We only make decisions based on loss when the bitrate is above a
// threshold. This is a crude way of handling loss which is
// uncorrelated to congestion.
if (bitrate_ < bitrate_threshold_bps_ || loss <= low_loss_threshold_)
{
// Loss < 2%: Increase rate by 8% of the min bitrate in the last
// kBweIncreaseIntervalMs.
// Note that by remembering the bitrate over the last second one can
// rampup up one second faster than if only allowed to start ramping
// at 8% per second rate now. E.g.:
// If sending a constant 100kbps it can rampup immediatly to 108kbps
// whenever a receiver report is received with lower packet loss.
// If instead one would do: bitrate_ *= 1.08^(delta time), it would
// take over one second since the lower packet loss to achieve 108kbps.
bitrate = (long) (min_bitrate_history_.getFirst().bitrate * 1.08 + 0.5);
// Add 1 kbps extra, just to make sure that we do not get stuck
// (gives a little extra increase at low rates, negligible at higher
// rates).
bitrate += 1000;
statistics.update(now, false, LossRegion.LossFree);
}
else if (bitrate_ > bitrate_threshold_bps_)
{
if (loss <= high_loss_threshold_)
{
// Loss between 2% - 10%: Do nothing.
statistics.update(now, false, LossRegion.LossLimited);
}
else
{
// Loss > 10%: Limit the rate decreases to once a kBweDecreaseIntervalMs +
// rtt.
if (!has_decreased_since_last_fraction_loss_ &&
(now - time_last_decrease_ms_) >=
(kBweDecreaseIntervalMs + getRttMs()))
{
time_last_decrease_ms_ = now;
// Reduce rate:
// newRate = rate * (1 - 0.5*lossRate);
// where packetLoss = 256*lossRate;
bitrate = (long) (
(bitrate * (512 - last_fraction_loss_)) / 512.0);
has_decreased_since_last_fraction_loss_ = true;
statistics.update(now, false, LossRegion.LossDegraded);
}
}
}
}
else
{
statistics.update(now, true, null);
if (time_since_feedback_ms >
kFeedbackTimeoutIntervals * kFeedbackIntervalMs
&& (last_timeout_ms_ == -1
|| now - last_timeout_ms_ > kTimeoutIntervalMs))
{
if (in_timeout_experiment_)
{
bitrate_ *= 0.8;
// Reset accumulators since we've already acted on missing
// feedback and shouldn't to act again on these old lost
// packets.
lost_packets_since_last_loss_update_Q8_ = 0;
expected_packets_since_last_loss_update_ = 0;
last_timeout_ms_ = now;
}
}
}
setBitrate(capBitrateToThresholds(bitrate));
}
/**
* Report that one packet has arrived.
*/
public void reportPacketArrived(long now, boolean previouslyReportedLost)
{
if (previouslyReportedLost)
{
lostPacketArrived();
}
else
{
updateReceiverBlock(0, 1, now);
}
}
/**
* Report that one packet was lost.
*/
public void reportPacketLost(long now)
{
updateReceiverBlock(256, 1, now);
}
/**
* void SendSideBandwidthEstimation::UpdateReceiverBlock
*/
synchronized private void updateReceiverBlock(long fraction_lost, long number_of_packets, long now)
{
last_feedback_ms_ = now;
if (first_report_time_ms_ == -1)
{
first_report_time_ms_ = now;
}
// Check sequence number diff and weight loss report
if (number_of_packets > 0)
{
// Calculate number of lost packets.
long num_lost_packets_Q8 = fraction_lost * number_of_packets;
// Accumulate reports.
lost_packets_since_last_loss_update_Q8_ += num_lost_packets_Q8;
expected_packets_since_last_loss_update_ += number_of_packets;
// Don't generate a loss rate until it can be based on enough packets.
if (expected_packets_since_last_loss_update_ < kLimitNumPackets)
return;
has_decreased_since_last_fraction_loss_ = false;
last_fraction_loss_ =
lost_packets_since_last_loss_update_Q8_ /
expected_packets_since_last_loss_update_;
// Reset accumulators.
lost_packets_since_last_loss_update_Q8_ = 0;
expected_packets_since_last_loss_update_ = 0;
last_packet_report_ms_ = now;
}
}
synchronized private void lostPacketArrived()
{
/* This algorithm can only cancel out losses that have occurred
since the last loss update, unfortunately. */
if (lost_packets_since_last_loss_update_Q8_ >= 256)
{
lost_packets_since_last_loss_update_Q8_ -= 256;
}
}
private synchronized void updateMinHistory(long now_ms)
{
// Remove old data points from history.
// Since history precision is in ms, add one so it is able to increase
// bitrate if it is off by as little as 0.5ms.
while (!min_bitrate_history_.isEmpty() &&
now_ms - min_bitrate_history_.getFirst().timestampMs + 1 >
kBweIncreaseIntervalMs)
{
min_bitrate_history_.removeFirst();
}
// Typical minimum sliding-window algorithm: Pop values higher than current
// bitrate before pushing it.
while (!min_bitrate_history_.isEmpty() &&
bitrate_ <= min_bitrate_history_.getLast().bitrate)
{
min_bitrate_history_.removeLast();
}
min_bitrate_history_.addLast(new BitrateHistoryRecord(now_ms, bitrate_));
}
/**
* {@link SendSideBandwidthEstimation#updateReceiverEstimate}
* This is the entry/update point for the estimated bitrate in the
* REMBPacket or a Delay Based Controller estimated bitrate when the
* Delay based controller and the loss based controller lives on the
* send side. see internet draft on "Congestion Control for RTCWEB"
*/
public synchronized void updateReceiverEstimate(long bandwidth)
{
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("bwe_incoming")
.addField("bitrate_bps", bandwidth));
}
bwe_incoming_ = bandwidth;
setBitrate(capBitrateToThresholds(bitrate_));
}
/**
* void SendSideBandwidthEstimation::SetMinMaxBitrate
*/
synchronized public void setMinMaxBitrate(int min_bitrate, int max_bitrate)
{
min_bitrate_configured_ = Math.max(min_bitrate, kDefaultMinBitrateBps);
if (max_bitrate > 0)
{
max_bitrate_configured_ =
Math.max(min_bitrate_configured_, max_bitrate);
}
else
{
max_bitrate_configured_ = kDefaultMaxBitrateBps;
}
}
/**
* Sets the value of {@link #bitrate_}.
* @param newValue the value to set
*/
private synchronized void setBitrate(long newValue)
{
bitrate_ = newValue;
}
/**
* @return the latest estimate.
*/
public long getLatestEstimate()
{
return bitrate_;
}
/**
* @return the latest values of the Receiver Estimated Maximum Bandwidth.
*/
public long getLatestREMB()
{
return bwe_incoming_;
}
/**
* @return the latest effective fraction loss calculated.
* The value is between 0 and 256 (corresponding
* to 0% and 100% respectively).
*/
public int getLatestFractionLoss()
{
return last_fraction_loss_;
}
/**
* @return the statistics specific to this bandwidth estimator.
*/
public Statistics getStatistics()
{
return statistics;
}
public synchronized void onRttUpdate(Duration newRtt)
{
this.rttMs = newRtt.toMillis();
}
/**
* Returns the last calculated RTT to the endpoint in milliseconds.
* @return the last calculated RTT to the endpoint in milliseconds.
*/
private synchronized long getRttMs()
{
if (rttMs < 0)
{
logger.warn("RTT not calculated, using the default of 100ms.");
rttMs = 100;
}
else if (rttMs > 1000)
{
logger.warn(
"RTT suspiciously high (" + rttMs + "ms), capping to 1000ms.");
rttMs = 1000;
}
return rttMs;
}
private static final class BitrateHistoryRecord
{
final long timestampMs;
final long bitrate;
BitrateHistoryRecord(long now, long br)
{
timestampMs = now;
bitrate = br;
}
}
/**
* This class records statistics information about how much time we spend
* in different loss-states (loss-free, loss-limited and loss-degraded).
*/
public class Statistics
{
/**
* The current state {@link LossRegion}.
*/
private LossRegion currentState = null;
/**
* Keeps the time (in millis) of the last transition (including a loop).
*/
private long lastTransitionTimestampMs = -1;
/**
* The cumulative duration (in millis) of the current state
* {@link #currentState} after having looped
* {@link #currentStateConsecutiveVisits} times.
*/
private long currentStateCumulativeDurationMs;
/**
* The number of loops over the current state {@link #currentState}.
*/
private int currentStateConsecutiveVisits;
/**
* The bitrate when we entered the current state {@link #currentState}.
*/
private long currentStateStartBitrateBps;
/**
* Computes the min/max/avg/sd of the bitrate while in
* {@link #currentState}.
*/
private LongSummaryStatistics currentStateBitrateStatistics
= new LongSummaryStatistics();
/**
* Computes the min/max/avg/sd of the loss while in
* {@link #currentState}.
*/
private IntSummaryStatistics currentStateLossStatistics
= new IntSummaryStatistics();
/**
* True when the fields of this class have changed from their default
* value. The purpose is to avoid creating new IntSummaryStatistics and
* LongSummaryStatistics when it's not needed.
*/
private boolean isDirty = false;
/**
* Computes the sum of the duration of the different states.
*/
private LongSummaryStatistics
lossFreeMsStats = new LongSummaryStatistics(),
lossDegradedMsStats = new LongSummaryStatistics(),
lossLimitedMsStats = new LongSummaryStatistics();
public void update(long nowMs)
{
synchronized (SendSideBandwidthEstimation.this)
{
long time_since_packet_report_ms
= nowMs - last_packet_report_ms_;
boolean currentStateHasTimedOut = time_since_packet_report_ms
< kPacketReportTimeoutIntervals * kFeedbackIntervalMs;
update(nowMs, currentStateHasTimedOut, null);
}
}
/**
* Records a state transition and updates the statistics information.
*
* @param nowMs the time (in millis) of the transition.
* @param currentStateHasTimedOut true if the current state has timed
* out, i.e. we haven't received receiver reports "in a while".
* @param nextState the that the bwe is transitioning to.
*/
void update(
long nowMs, boolean currentStateHasTimedOut, LossRegion nextState)
{
synchronized (SendSideBandwidthEstimation.this)
{
if (lastTransitionTimestampMs > -1 && !currentStateHasTimedOut)
{
isDirty = true;
currentStateCumulativeDurationMs
+= nowMs - lastTransitionTimestampMs;
}
lastTransitionTimestampMs = nowMs;
if (!currentStateHasTimedOut)
{
isDirty = true;
// If the current state has not timed out, then update the
// stats that we gather.
currentStateLossStatistics.accept(last_fraction_loss_);
currentStateConsecutiveVisits++; // we start counting from 0.
if (this.currentState == nextState)
{
currentStateBitrateStatistics.accept(bitrate_);
return;
}
}
if (this.currentState != null)
{
// This is not a loop, we're transitioning to another state.
// Record how much time we've spent on this state, how many
// times we've looped through it and what was the impact on
// the bitrate.
switch (this.currentState)
{
case LossDegraded:
lossDegradedMsStats.accept(
currentStateCumulativeDurationMs);
break;
case LossFree:
lossFreeMsStats.accept(currentStateCumulativeDurationMs);
break;
case LossLimited:
lossLimitedMsStats.accept(
currentStateCumulativeDurationMs);
break;
}
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("loss_estimate")
.addField("state", currentState.name())
.addField("max_loss",
currentStateLossStatistics.getMax() / 256.0f)
.addField("min_loss",
currentStateLossStatistics.getMin() / 256.0f)
.addField("avg_loss",
currentStateLossStatistics.getAverage()/256.0f)
.addField("max_bps",
currentStateBitrateStatistics.getMax())
.addField("min_bps",
currentStateBitrateStatistics.getMin())
.addField("avg_bps",
currentStateBitrateStatistics.getAverage())
.addField("duration_ms",
currentStateCumulativeDurationMs)
.addField("consecutive_visits",
currentStateConsecutiveVisits)
.addField("bitrate_threshold",
bitrate_threshold_bps_)
.addField("low_loss_threshold",
low_loss_threshold_)
.addField("high_loss_threshold",
high_loss_threshold_)
.addField("delta_bps",
bitrate_ - currentStateStartBitrateBps)
.addField("bitrate",
bitrate_)
.addField("bwe_incoming",
bwe_incoming_)
.addField("rtt_ms",
rttMs));
}
}
currentState = nextState;
currentStateStartBitrateBps = bitrate_;
if (isDirty)
{
currentStateLossStatistics = new IntSummaryStatistics();
currentStateBitrateStatistics = new LongSummaryStatistics();
currentStateConsecutiveVisits = 0;
currentStateCumulativeDurationMs = 0;
isDirty = false;
}
currentStateBitrateStatistics.accept(bitrate_);
}
}
/** Return to state immediately after construction. */
void reset()
{
synchronized (SendSideBandwidthEstimation.this)
{
currentState = null;
lastTransitionTimestampMs = -1;
lossFreeMsStats = new LongSummaryStatistics();
lossDegradedMsStats = new LongSummaryStatistics();
lossLimitedMsStats = new LongSummaryStatistics();
if (isDirty)
{
currentStateLossStatistics = new IntSummaryStatistics();
currentStateBitrateStatistics = new LongSummaryStatistics();
currentStateConsecutiveVisits = 0;
currentStateCumulativeDurationMs = 0;
isDirty = false;
}
}
}
/**
* @return the number of millis spent in the loss-limited state.
*/
public long getLossLimitedMs()
{
synchronized (SendSideBandwidthEstimation.this)
{
return lossLimitedMsStats.getSum();
}
}
/**
* @return the number of millis spent in the loss-degraded state.
*/
public long getLossDegradedMs()
{
synchronized (SendSideBandwidthEstimation.this)
{
return lossDegradedMsStats.getSum();
}
}
/**
* @return the number of millis spent in the loss-free state.
*/
public long getLossFreeMs()
{
synchronized (SendSideBandwidthEstimation.this)
{
return lossFreeMsStats.getSum();
}
}
}
/**
* Represents the loss-based controller states.
*/
private enum LossRegion
{
/**
* Loss is between 2% and 10%.
*/
LossLimited,
/**
* Loss is above 10%.
*/
LossDegraded,
/**
* Loss is bellow 2%.
*/
LossFree
}
}

53
jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/transform/PacketTransformer.java Normal file
View File

@@ -0,0 +1,53 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi_modified.impl.neomedia.transform;
import org.jitsi.rtp.*;
/**
* Encapsulate the concept of packet transformation. Given an array of packets,
* <tt>PacketTransformer</tt> can either "transform" each one of them, or
* "reverse transform" (e.g. restore) each one of them.
*
* @author Bing SU (nova.su@gmail.com)
* @author Boris Grozev
*/
public interface PacketTransformer
{
/**
* Closes this <tt>PacketTransformer</tt> i.e. releases the resources
* allocated by it and prepares it for garbage collection.
*/
public void close();
/**
* Reverse-transforms each packet in an array of packets. Null values
* must be ignored.
*
* @param pkts the transformed packets to be restored.
* @return the restored packets.
*/
public Packet[] reverseTransform(Packet[] pkts);
/**
* Transforms each packet in an array of packets. Null values must be
* ignored.
*
* @param pkts the packets to be transformed
* @return the transformed packets
*/
public Packet[] transform(Packet[] pkts);
}

29
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/AudioLevelListener.kt Normal file
View File

@@ -0,0 +1,29 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
interface AudioLevelListener {
/**
* Process the audio level from a received audio RTP packet.
*
* @param sourceSsrc The SSRC identifying the source of the audio stream.
* @param level The audio level.
* @return A boolean set to {@code true} if this packet should be
* discarded without being forwarded to other endpoints.
*/
fun onLevelReceived(sourceSsrc: Long, level: Long): Boolean
}

27
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/DebugStateMode.kt Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright @ 2025 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
enum class DebugStateMode {
/** Include full details, mostly for manually debugging */
FULL,
/** Include all information needed for stats/metric collection (e.g. rtcstats) */
STATS,
/** Include only the most important information, e.g. listing conferences and endpoints. */
SHORT
}

27
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Event.kt Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import org.jitsi.utils.MediaType
interface Event
class SetMediaSourcesEvent(
val mediaSourceDescs: Array<MediaSourceDesc>,
val signaledMediaSourceDescs: Array<MediaSourceDesc>
) : Event
class SetLocalSsrcEvent(val mediaType: MediaType, val ssrc: Long) : Event

20
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/EventHandler.kt Normal file
View File

@@ -0,0 +1,20 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
interface EventHandler {
fun handleEvent(event: Event)
}

21
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Features.kt Normal file
View File

@@ -0,0 +1,21 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
enum class Features {
TRANSCEIVER_PCAP_DUMP
}

222
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt Normal file
View File

@@ -0,0 +1,222 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import org.jitsi.nlj.rtp.VideoRtpPacket
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bps
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.utils.ArrayUtils
import java.util.Collections
import java.util.NavigableMap
import java.util.TreeMap
/**
* Represents a collection of [RtpLayerDesc]s that encode the same
* media source. This specific implementation provides webrtc simulcast stream
* suspension detection.
*
* We take the definition of "Media Source" from RFC 7656. It takes a single
* logical source of media, which might be represented by multiple encoded streams.
*
* @author George Politis
*/
class MediaSourceDesc
@JvmOverloads constructor(
/**
* The [RtpEncodingDesc]s that this [MediaSourceDesc]
* possesses, ordered by their subjective quality from low to high.
*/
val rtpEncodings: Array<RtpEncodingDesc>,
/**
* A string which identifies the owner of this source (e.g. the endpoint
* which is the sender of the source).
*/
val owner: String,
/**
* A string which identifies this source.
*/
val sourceName: String,
/**
* The {@link VideoType} signaled for this media source (defaulting to {@code CAMERA} if nothing has been signaled).
*/
var videoType: VideoType = VideoType.CAMERA,
/**
* Whether this is a synthetic source, i.e. one generated by the bridge rather than sent by an endpoint.
*/
val synthetic: Boolean = false,
) {
/**
* Current single-list view of all the encodings' layers.
*/
private lateinit var layers: List<RtpLayerDesc>
/**
* Allow the lookup of a layer by the encoding id of a received packet.
*/
private val layersById: MutableMap<Long, RtpLayerDesc> = HashMap()
/**
* Allow the lookup of a layer by index.
*/
private val layersByIndex: NavigableMap<Int, RtpLayerDesc> = TreeMap()
/**
* Get a view of the source's RTP layers, in quality order.
*/
val rtpLayers: List<RtpLayerDesc>
@Synchronized
get() = layers
/**
* Update the layer cache. Should be synchronized on [this].
*/
private fun updateLayerCache() {
layersById.clear()
layersByIndex.clear()
val tempLayers = ArrayList<RtpLayerDesc>()
for (encoding in rtpEncodings) {
for (layer in encoding.layers) {
layersById[encoding.encodingId(layer)] = layer
layersByIndex[layer.index] = layer
tempLayers.add(layer)
}
}
layers = Collections.unmodifiableList(tempLayers)
}
init {
updateLayerCache()
}
/**
* Gets the last "stable" bitrate (in bps) of the encoding of the specified
* index. The "stable" bitrate is measured on every new frame and with a
* 5000ms window.
*
* If the bitrate for the specified index is 0, return bitrate of the highest-
* index layer less than the index with a non-zero bitrate.
*
* @return the last "stable" bitrate (bps) of the encoding with a non-zero rate
* at or below the specified index.
*/
@Synchronized
fun getBitrate(nowMs: Long, idx: Int): Bandwidth {
for (entry in layersByIndex.headMap(idx, true).descendingMap()) {
val bitrate = entry.value.getBitrate(nowMs)
if (bitrate.bps > 0) {
return bitrate
}
}
return 0.bps
}
@Synchronized
fun hasRtpLayers(): Boolean = layers.isNotEmpty()
@Synchronized
fun numRtpLayers(): Int = layersByIndex.size
val primarySSRC: Long
get() = rtpEncodings[0].primarySSRC
@Synchronized
fun getRtpLayerByQualityIdx(idx: Int): RtpLayerDesc? = layersByIndex[idx]
@Synchronized
fun findRtpLayerDescs(videoRtpPacket: VideoRtpPacket): Collection<RtpLayerDesc> {
if (ArrayUtils.isNullOrEmpty(rtpEncodings)) {
return emptyList()
}
return videoRtpPacket.getEncodingIds().mapNotNull { layersById[it] }
}
@Synchronized
fun findRtpEncodingDesc(ssrc: Long): RtpEncodingDesc? = rtpEncodings.find { it.hasSsrc(ssrc) }
@Synchronized
fun getEncodingLayers(ssrc: Long): Array<RtpLayerDesc> {
val enc = findRtpEncodingDesc(ssrc) ?: return emptyArray()
return Array(enc.layers.size) { i ->
enc.layers[i].copy()
}
}
@Synchronized
fun setEncodingLayers(layers: Array<RtpLayerDesc>, ssrc: Long) {
val enc = findRtpEncodingDesc(ssrc) ?: return
enc.layers = layers
updateLayerCache()
}
/**
* Clone an existing media source desc, inheriting layer descs' statistics.
*/
@Synchronized
fun copy() = MediaSourceDesc(
Array(this.rtpEncodings.size) { i -> this.rtpEncodings[i].copy() },
this.owner,
this.sourceName,
this.videoType,
this.synthetic
)
override fun toString(): String = "MediaSourceDesc[name=$sourceName owner=$owner, videoType=$videoType, " +
"synthetic=$synthetic, encodings=${rtpEncodings.joinToString(",")}]"
/**
* Checks whether the given SSRC matches this source's [primarySSRC].
* This is mostly useful only for determining quickly whether two source
* descriptions describe the same source; other functions (probably [hasSsrc]) should be used
* to match received media packets.
*
* @param ssrc the SSRC to match.
* @return `true` if the specified `ssrc` is the primary SSRC
* for this source.
*/
fun matches(ssrc: Long) = rtpEncodings.getOrNull(0)?.primarySSRC == ssrc
/**
* Checks whether any encoding of this source has this [ssrc]
*/
@Synchronized
fun hasSsrc(ssrc: Long) = rtpEncodings.any { it.hasSsrc(ssrc) }
}
/**
* Clone an array of media source descriptions.
*/
fun Array<MediaSourceDesc>.copy() = Array(this.size) { i -> this[i].copy() }
fun Array<MediaSourceDesc>.findRtpLayerDescs(packet: VideoRtpPacket): Collection<RtpLayerDesc> = this.flatMap {
it.findRtpLayerDescs(packet)
}
fun Array<MediaSourceDesc>.findRtpSourceByPrimary(ssrc: Long): MediaSourceDesc? = this.find { it.matches(ssrc) }
fun Array<MediaSourceDesc>.findRtpSource(ssrc: Long): MediaSourceDesc? = this.find { it.hasSsrc(ssrc) }
fun Array<MediaSourceDesc>.findRtpSource(packet: RtpPacket): MediaSourceDesc? = findRtpSource(packet.ssrc)
fun Array<MediaSourceDesc>.findRtpEncodingId(packet: VideoRtpPacket): Int? {
for (source in this) {
source.findRtpEncodingDesc(packet.ssrc)?.let {
return it.eid
}
}
return null
}

75
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
/**
* Maintains an array of [MediaSourceDesc]. The set method preserves the existing sources that match one of the new
* sources, because [MediaSourceDesc] holds some local state (rate statistics) that we would like to keep. Ideally
* this state should be out of the source descriptor, in which case this class would be obsolete.
*
* @author Boris Grozev
*/
class MediaSources {
private var sources: Array<MediaSourceDesc> = arrayOf()
fun setMediaSources(newSources: Array<MediaSourceDesc>): Boolean {
val oldSources = sources
if (oldSources.isEmpty() || newSources.isEmpty()) {
sources = newSources
return oldSources.size != newSources.size
}
var cntMatched = 0
val mergedSources: Array<MediaSourceDesc> = Array(newSources.size) { i ->
val newPrimarySSRC = newSources[i].primarySSRC
for (j in 0 until oldSources.size) {
if (oldSources[j].matches(newPrimarySSRC)) {
cntMatched++
// NOTE: we deliberately do not update the old source instance
// with the encodings of the new one. Values set on the
// source are more likely to be correct than ones generated
// from signaling about the source. (Revisit this if we
// encounter a scenario where this isn't true...)
return@Array oldSources[j]
}
}
newSources[i]
}
sources = mergedSources
return oldSources.size != newSources.size || cntMatched != oldSources.size
}
fun getMediaSources(): Array<MediaSourceDesc> = sources
fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
sources.forEach { source ->
val sourceNode = JsonNodeFactory.instance.objectNode().apply {
put("owner", source.owner)
put("video_type", source.videoType.toString())
source.rtpEncodings.forEach {
set<ObjectNode>("rtp_encoding_${it.primarySSRC}", it.debugState())
}
}
set<ObjectNode>(source.sourceName, sourceNode)
}
}
}

81
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketHandler.kt Normal file
View File

@@ -0,0 +1,81 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import org.jitsi.nlj.stats.NodeStatsBlock
import org.jitsi.nlj.transform.NodeStatsProducer
import org.jitsi.nlj.util.addMbps
interface PacketHandler {
/**
* Process the given packets
*/
fun processPacket(packetInfo: PacketInfo)
}
/**
* A [PacketHandler] which tracks statistics about the packets it handles
*/
abstract class StatsKeepingPacketHandler : PacketHandler, NodeStatsProducer {
private val statistics = Statistics()
final override fun processPacket(packetInfo: PacketInfo) {
val now = System.currentTimeMillis()
statistics.packetsReceived++
statistics.bytesReceived += packetInfo.packet.length
if (statistics.firstPacketReceivedTime == 0L) {
statistics.firstPacketReceivedTime = now
}
statistics.lastPacketReceivedTime = now
doProcessPacket(packetInfo)
}
/**
* We override [processPacket] above and prevent subclasses from doing so
* to make sure this class sees all packets that pass through it and it
* can track stats correctly. This method should be implemented by
* subclasses as a stand-in for [processPacket], as it is called once
* all the stat-tracking has been done.
*/
protected abstract fun doProcessPacket(packetInfo: PacketInfo)
override fun getNodeStats(): NodeStatsBlock = statistics.getNodeStats()
/**
* Statistics common to [PacketHandler]s which are interesting to track
*/
data class Statistics(
var firstPacketReceivedTime: Long = 0,
var lastPacketReceivedTime: Long = 0,
var bytesReceived: Long = 0,
var packetsReceived: Long = 0
) : NodeStatsProducer {
override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("Packet handler stats").apply {
addNumber(RECEIVED_PACKETS, packetsReceived)
addNumber(RECEIVED_BYTES, bytesReceived)
addNumber(RECEIVED_DURATION_MS, lastPacketReceivedTime - firstPacketReceivedTime)
addMbps(RECEIVED_BITRATE_MBPS, RECEIVED_BYTES, RECEIVED_DURATION_MS)
}
companion object {
private const val RECEIVED_PACKETS = "received_packets"
private const val RECEIVED_BYTES = "received_bytes"
private const val RECEIVED_DURATION_MS = "received_duration_ms"
private const val RECEIVED_BITRATE_MBPS = "received_bitrate_mbps"
}
}
}

298
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt Normal file
View File

@@ -0,0 +1,298 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.nlj.format.PayloadType
import org.jitsi.rtp.Packet
import org.jitsi.utils.logging2.createLogger
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.Collections
@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE")
class EventTimeline(
/**
* We want this thread safe, because while PacketInfo objects are only handled by a single thread at a time,
* [StatsKeepingNode] may add an "exit" event after the packets has been added to another queue potentially handled
* by a different thread. This is not critical as it only affects the timeline and the result is just some "exit"
* events missing from the trace logs.
*/
timelineArg: MutableList<Pair<String, Duration>> = mutableListOf(),
private val clock: Clock = Clock.systemUTC()
) : Iterable<Pair<String, Duration>> {
private val timeline = Collections.synchronizedList(timelineArg)
/**
* The [referenceTime] refers to the first timestamp we have
* in the timeline. In the timeline this is used as time "0" and
* all other times are represented as deltas from this 0.
*/
var referenceTime: Instant? = null
val size: Int
get() = timeline.size
fun addEvent(desc: String) {
val now = clock.instant()
if (referenceTime == null) {
referenceTime = now
}
timeline.add(desc to Duration.between(referenceTime, now))
}
fun clone(): EventTimeline {
val clone = EventTimeline(timeline.toMutableList(), clock)
clone.referenceTime = referenceTime
return clone
}
override fun iterator(): Iterator<Pair<String, Duration>> = timeline.iterator()
/**
* Return the total time between this packet's first event and last event
* or -1 if there is no reference time
*/
fun totalDelay(): Duration {
return referenceTime?.let {
return timeline.last().second
} ?: Duration.ofMillis(-1)
}
override fun toString(): String = with(StringBuffer()) {
referenceTime?.let {
append("Reference time: $referenceTime; ")
synchronized(timeline) {
append(timeline.joinToString(separator = "; "))
}
} ?: run {
append("[No timeline]")
}
toString()
}
}
/**
* The origin of a packet in the system; used to track outgoing packets in [OutgoingStatisticsTracker]
* to measure the bitrates of each type of data.
*
* Currently only used for RTP, so RTCP, SCTP, and Datachannel will all be either Routed or Misc.
*/
enum class PacketOrigin {
Routed,
Retransmission,
Probing,
Padding,
Synthesized,
Misc
/* TODO: Add RTCP, SCTP, and datachannel if needed */
}
/**
* [PacketInfo] is a wrapper around a [Packet] instance to be passed through
* a pipeline. Since the [Packet] can change as it moves through the pipeline
* (as it is parsed into different types), the wrapping [PacketInfo] stays consistent
* and allows for metadata to be passed along with a packet.
*/
@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE")
open class PacketInfo @JvmOverloads constructor(
var packet: Packet,
/** The original length of the packet, i.e. before decryption. Stays unchanged even if the packet is updated. */
val originalLength: Int = packet.length,
val timeline: EventTimeline? = if (enableTimeline) EventTimeline() else null
) {
/**
* An explicit tag for when this packet was originally received (assuming it
* was an incoming packet and not one created by jvb itself).
*/
var receivedTime: Instant? = null
set(value) {
field = value
if (timeline != null && timeline.referenceTime == null) {
timeline.referenceTime = value
}
}
/** Whether the packet originally had cryptex RTP header extensions. */
var originalHadCryptex: Boolean = false
/**
* Whether this packet has been recognized to contain only shouldDiscard.
*/
var shouldDiscard: Boolean = false
/**
* The ID of the endpoint associated with this packet (i.e. the source endpoint).
*/
var endpointId: String? = null
/**
* Whether this packet indicates a point in which its stream's layering changed, in
* a way that indicates that bitrate allocation may need to be recomputed.
*/
var layeringChanged = false
var payloadType: PayloadType? = null
/**
* The RFC 6464 audio level (0-127, i.e. -dBov: 0 is loudest, 127 is silence) reported by the sender in the
* ssrc-audio-level RTP header extension, or null if the packet had no such extension. Populated on the incoming
* path by [org.jitsi.nlj.transform.node.incoming.AudioLevelReader].
*/
var audioLevel: Int? = null
/**
* The RFC 6464 Voice Activity Detection flag reported by the sender in the ssrc-audio-level RTP header extension,
* or null if the packet had no such extension. Populated alongside [audioLevel].
*/
var vad: Boolean? = null
/**
* The payload verification string for the packet, or 'null' if payload verification is disabled. Calculating the
* it is expensive, thus we only do it when the flag is enabled.
*/
var payloadVerification = if (enablePayloadVerification) packet.payloadVerification else null
/**
* Information about whether this packet is used for probing by the transport-cc engine.
* The type is internal to that object.
*/
var probingInfo: Any? = null
/**
* The origin of the packet, used for tracking the sources of media being routed.
*/
var packetOrigin: PacketOrigin = PacketOrigin.Misc
/**
* Re-calculates the expected payload verification string. This should be called any time that the code
* intentionally modifies the packet in a way that could change the verification string (for example, re-creates
* it with a new type (parsing), or intentionally modifies the payload (SRTP)).
*/
fun resetPayloadVerification() {
payloadVerification = if (enablePayloadVerification) packet.payloadVerification else null
}
/**
* Get the contained packet cast to [ExpectedPacketType]
*/
@Suppress("UNCHECKED_CAST")
fun <ExpectedPacketType : Packet> packetAs(): ExpectedPacketType = packet as ExpectedPacketType
/**
* Create a deep clone of this PacketInfo (both the contained packet and the metadata map
* will be copied for the cloned PacketInfo).
*/
fun clone(): PacketInfo {
val clone = PacketInfo(packet.clone(), originalLength, timeline?.clone())
clone.receivedTime = receivedTime
clone.originalHadCryptex = originalHadCryptex
clone.shouldDiscard = shouldDiscard
clone.endpointId = endpointId
clone.payloadType = payloadType
clone.audioLevel = audioLevel
clone.vad = vad
clone.layeringChanged = layeringChanged
clone.payloadVerification = payloadVerification
clone.probingInfo = probingInfo
clone.packetOrigin = packetOrigin
@Suppress("UNCHECKED_CAST") // ArrayList.clone() really does return ArrayList, not Object.
clone.onSentActions = onSentActions?.clone() as ArrayList<(PacketInfo) -> Unit>?
return clone
}
fun addEvent(desc: String) = timeline?.addEvent(desc)
/**
* The list of pending actions, or [null] if none.
*/
private var onSentActions: ArrayList<(PacketInfo) -> Unit>? = null
/**
* Add an action to be performed when the packet is sent (i.e. when this packet's
* [sent] method is called).
*
* If this [PacketInfo] object is cloned, the action will be called for every
* cloned instance. If packet is dropped (i.e. [sent] is never called), the
* action will not be called.
*/
fun onSent(action: (PacketInfo) -> Unit) {
synchronized(this) {
if (onSentActions == null) {
onSentActions = ArrayList(1)
}
onSentActions!!.add(action)
}
}
/**
* Invoke any actions previously registered with this [PacketInfo]'s [onSent]
* method. This should be called just before, or after, this packet is sent.
*/
fun sent() {
var actions: List<(PacketInfo) -> Unit> = Collections.emptyList()
synchronized(this) {
onSentActions?.let {
actions = it
onSentActions = null
} ?: run { return@sent }
}
for (action in actions) {
action.invoke(this)
}
}
companion object {
private val enableTimeline: Boolean by config {
"jmt.debug.packet-timeline.enabled".from(JitsiConfig.newConfig)
}
private val enablePayloadVerificationDefault: Boolean by config {
"jmt.debug.payload-verification.enabled".from(JitsiConfig.newConfig)
}
/**
* If this is enabled all [Node]s will verify that the payload didn't unexpectedly change. This is expensive.
*/
@field:Suppress("ktlint:standard:property-naming")
var enablePayloadVerification = enablePayloadVerificationDefault
init {
if (enableTimeline) {
createLogger().info("Packet timeline is enabled.")
}
if (enablePayloadVerification) {
createLogger().info("Payload verification is enabled.")
}
}
}
}
/**
* This is a specialization of [org.jitsi.nlj.util.forEachAs] method which makes it easier
* to operate on lists of [PacketInfo] when the caller wants to treat the contained [Packet]
* as a specific packet type. This method iterates over the iterable of [PacketInfo]s and calls
* the given lambda with the [PacketInfo] instance and the contained [Packet] instance, cast
* as [ExpectedPacketType]. This will throw if the cast attempt is unsuccessful.
*/
@Suppress("UNCHECKED_CAST")
inline fun <ExpectedPacketType> Iterable<PacketInfo>.forEachAs(action: (PacketInfo, ExpectedPacketType) -> Unit) {
for (element in this) action(element, element.packet as ExpectedPacketType)
}

203
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt Normal file
View File

@@ -0,0 +1,203 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.rtp.SsrcAssociationType
import org.jitsi.nlj.rtp.VideoRtpPacket
import org.jitsi.nlj.stats.NodeStatsBlock
/**
* Keeps track of information specific to an RTP encoded stream
* (and its associated secondary sources).
*
* @author Jonathan Lennox
*/
class RtpEncodingDesc
@JvmOverloads
constructor(
/**
* The primary SSRC for this encoding.
*/
val primarySSRC: Long,
/**
* The [RtpLayerDesc]s describing the encoding's layers.
*/
initialLayers: Array<RtpLayerDesc>,
/**
* The ID of this encoding.
*/
val eid: Int = requireNotNull(initialLayers.getOrNull(0)?.eid) {
"initialLayers may not be empty if no explicit EID is provided"
}
) {
constructor(primarySSRC: Long, eid: Int) : this(primarySSRC, arrayOf(), eid)
/**
* The ssrcs associated with this encoding (for example, RTX or FLEXFEC)
* Maps ssrc -> type [SsrcAssociationType] (rtx, etc.)
*/
private val secondarySsrcs: MutableMap<Long, SsrcAssociationType> = HashMap()
fun addSecondarySsrc(ssrc: Long, type: SsrcAssociationType) {
secondarySsrcs[ssrc] = type
}
/**
* All SSRCs (primary and secondary) associated with this encoding.
*/
val ssrcs: Collection<Long>
get() = HashSet<Long>().also { set ->
set.add(primarySSRC)
set.addAll(secondarySsrcs.keys)
}
private fun validateLayerEids(layers: Array<RtpLayerDesc>) {
for (layer in layers) {
require(layer.eid == eid) { "Cannot add layer with EID ${layer.eid} to encoding with EID $eid" }
}
}
init {
validateLayerEids(initialLayers)
}
private var nominalHeight = initialLayers.getNominalHeight()
internal var layers = initialLayers
set(newLayers) {
validateLayerEids(newLayers)
/* Check if the new layer set is a single spatial layer that doesn't specify a height - if so, we
* want to apply the nominal height to them.
*/
val useNominalHeight = nominalHeight != RtpLayerDesc.NO_HEIGHT &&
newLayers.all { it.sid == 0 } &&
newLayers.all { it.height == RtpLayerDesc.NO_HEIGHT }
/* Copy the rate statistics objects from the old layers to the new layers
* with matching layer IDs.
*/
/* Note: because layer arrays are sorted by ID we could avoid creating this
* intermediate map object, and do this in a single pass in O(1) space.
* The number of layers is small enough that this more complicated code
* is probably unnecessary, though.
*/
val oldLayerMap = field.associateBy { it.layerId }
for (newLayer in newLayers) {
oldLayerMap[newLayer.layerId]?.let {
newLayer.inheritFrom(it)
}
if (useNominalHeight) {
newLayer.height = nominalHeight
}
}
if (!useNominalHeight) {
val newNominalHeight = newLayers.getNominalHeight()
if (newNominalHeight != RtpLayerDesc.NO_HEIGHT) {
nominalHeight = newNominalHeight
}
}
field = newLayers
}
/**
* @return the "id" of a layer within this source, across all encodings. This is a server-side id and should
* not be confused with any encoding id defined in the client (such as the
* rid). This server-side id is used in the layer lookup table that is
* maintained in [MediaSourceDesc].
*/
fun encodingId(layer: RtpLayerDesc): Long = calcEncodingId(primarySSRC, layer.layerId)
/**
* Get the secondary ssrc for this encoding that corresponds to the given
* type
* @param type the type of the secondary ssrc (e.g. RTX)
* @return the ssrc for the encoding that corresponds to the given type,
* if it exists; otherwise -1
*/
fun getSecondarySsrc(type: SsrcAssociationType): Long {
for ((key, value) in secondarySsrcs) {
if (value == type) {
return key
}
}
return -1
}
/**
* Clone an existing encoding desc, inheriting layer descs' statistics,
* modifying only specific values.
*/
fun copy(
primarySSRC: Long = this.primarySSRC,
layers: Array<RtpLayerDesc> = Array(this.layers.size) { i -> this.layers[i].copy() }
) = RtpEncodingDesc(primarySSRC, layers, eid).also {
this.secondarySsrcs.forEach { (ssrc, type) -> it.addSecondarySsrc(ssrc, type) }
}
/**
* {@inheritDoc}
*/
override fun toString(): String = "primary_ssrc=$primarySSRC,secondary_ssrcs=$secondarySsrcs," +
"layers=\n ${layers.joinToString(separator = "\n ")}"
/**
* Gets a boolean indicating whether the SSRC specified in the
* arguments is used by this encoding.
*
* @param ssrc the SSRC to match.
*/
fun hasSsrc(ssrc: Long): Boolean = if (primarySSRC == ssrc) {
true
} else {
secondarySsrcs.containsKey(ssrc)
}
/**
* Extracts a [NodeStatsBlock] from an [RtpEncodingDesc].
*/
fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("rtx_ssrc", getSecondarySsrc(SsrcAssociationType.RTX))
put("fec_ssrc", getSecondarySsrc(SsrcAssociationType.FEC))
put("eid", eid)
put("nominal_height", nominalHeight)
for (layer in layers) {
set<ObjectNode>(layer.indexString(), layer.debugState())
}
}
companion object {
fun calcEncodingId(ssrc: Long, layerId: Int) = ssrc or (layerId.toLong() shl 32)
}
}
fun VideoRtpPacket.getEncodingIds(): Collection<Long> = this.layerIds.map { RtpEncodingDesc.calcEncodingId(ssrc, it) }
/**
* Get the "nominal" height of a set of layers - if they all indicate the same spatial layer and same height.
*/
private fun Array<RtpLayerDesc>.getNominalHeight(): Int {
if (isEmpty()) {
return RtpLayerDesc.NO_HEIGHT
}
val firstHeight = first().height
if (!(all { it.sid == 0 } || all { it.sid == -1 })) {
return RtpLayerDesc.NO_HEIGHT
}
if (any { it.height != firstHeight }) {
return RtpLayerDesc.NO_HEIGHT
}
return firstHeight
}

280
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt Normal file
View File

@@ -0,0 +1,280 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.transform.node.incoming.BitrateCalculator
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.BitrateTracker
import org.jitsi.nlj.util.DataSize
/**
* Keeps track of its subjective quality index,
* its last stable bitrate and other useful things for adaptivity/routing.
*
* @author George Politis
*/
abstract class RtpLayerDesc(
/**
* The index of this instance's encoding in the source encoding array.
*/
val eid: Int,
/**
* The temporal layer ID of this instance.
*/
val tid: Int,
/**
* The spatial layer ID of this instance.
*/
val sid: Int,
/**
* The max "height" of the bitstream that this instance represents. The actual
* height may be less due to bad network or system load. [NO_HEIGHT] for unknown.
*
* In order to handle portrait-mode video correctly, this should actually be the smaller of
* the bitstream's height or width.
*
* Where possible we sniff the actual height from the RTP packets.
*/
var height: Int,
/**
* The max frame rate (in fps) of the bitstream that this instance
* represents. The actual frame rate may be less due to bad network or
* system load. [NO_FRAME_RATE] for unknown.
*/
var frameRate: Double,
) {
abstract fun copy(height: Int = this.height, tid: Int = this.tid, inherit: Boolean = true): RtpLayerDesc
/**
* The [BitrateTracker] instance used to calculate the receiving bitrate of this RTP layer.
*/
protected var bitrateTracker = BitrateCalculator.createBitrateTracker()
/**
* A second [BitrateTracker] with a longer window, for sources whose bitrate is bursty. It is only created once
* [enableSmoothedBitrate] is called, because it costs memory per layer and an update per packet per layer, and
* only screen sharing sources use it. See [BitrateCalculator.createSmoothedBitrateTracker].
*
* Volatile because it is enabled from the threads which run bandwidth allocation (one per receiver, so possibly
* several at once), but read and updated from the thread which reads packets.
*/
@Volatile
private var smoothedBitrateTracker: BitrateTracker? = null
/**
* The [clock] time at which [smoothedBitrateTracker] was created. Only meaningful while the tracker is non-null.
* Writes to it are made visible by the subsequent volatile write to [smoothedBitrateTracker].
*/
private var smoothedBitrateTrackerCreatedMs: Long = -1
/**
* Start tracking a smoothed bitrate for this layer. Idempotent. Until the tracker created here has a full window
* of history, [getSmoothedBitrate] falls back to [getBitrate], so that a source is not priced at close to zero
* while the longer window fills up.
*/
fun enableSmoothedBitrate(nowMs: Long) {
if (smoothedBitrateTracker == null) {
synchronized(this) {
if (smoothedBitrateTracker == null) {
smoothedBitrateTrackerCreatedMs = nowMs
smoothedBitrateTracker = BitrateCalculator.createSmoothedBitrateTracker()
}
}
}
}
/**
* Stop tracking a smoothed bitrate for this layer, dropping the tracker and its history. Called when the source
* stops being bursty (e.g. a screen share reverting to camera), so that the layer stops paying for the tracker,
* and so that a later re-enable does not price the source from the previous mode's traffic.
*/
fun disableSmoothedBitrate() {
if (smoothedBitrateTracker != null) {
synchronized(this) {
smoothedBitrateTracker = null
}
}
}
/**
* The rate of this layer alone, either over the estimator-matched window or over the longer window used for
* bursty sources. The smoothed rate falls back to the other one while it is not being tracked, and while its
* tracker does not yet have a full window of history (since it always divides by the full window, a young tracker
* would under-report the rate, in the extreme pricing the source at zero at the moment it is enabled).
*/
protected fun layerRate(nowMs: Long, smoothed: Boolean): Bandwidth {
val smoothedTracker = if (smoothed) smoothedBitrateTracker else null
return if (smoothedTracker != null && nowMs - smoothedBitrateTrackerCreatedMs >= smoothedWindowMs) {
smoothedTracker.getRateOverFullWindow(nowMs)
} else {
bitrateTracker.getRate(nowMs)
}
}
var targetBitrate: Bandwidth? = null
/**
* @return the "id" of this layer within this encoding. This is a server-side id and should
* not be confused with any encoding id defined in the client (such as the
* rid).
*/
abstract val layerId: Int
/**
* A local index of this track.
*/
abstract val index: Int
/**
* Inherit a [BitrateTracker] object
*/
internal fun inheritStatistics(tracker: BitrateTracker) {
bitrateTracker = tracker
}
/**
* Inherit another layer description's [BitrateTracker] object.
*/
internal open fun inheritFrom(other: RtpLayerDesc) {
inheritStatistics(other.bitrateTracker)
// Only take the other layer's smoothed tracker if it has one: layer arrays are replaced whenever the
// scalability structure is re-signaled, and dropping an established tracker would restart its window.
other.smoothedBitrateTracker?.let {
smoothedBitrateTrackerCreatedMs = other.smoothedBitrateTrackerCreatedMs
smoothedBitrateTracker = it
}
targetBitrate = other.targetBitrate
}
/**
*
* @param packetSize
* @param nowMs
* @return true if this caused the rate being tracked to transition from zero to nonzero
*/
fun updateBitrate(packetSize: DataSize, nowMs: Long): Boolean {
val wasInactive = hasZeroBitrate(nowMs)
// Update rate stats (this should run after padding termination).
bitrateTracker.update(packetSize, nowMs)
smoothedBitrateTracker?.update(packetSize, nowMs)
return wasInactive && packetSize.bits > 0L
}
/**
* Gets the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies.
*
* @param nowMs
* @return the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies.
*/
abstract fun getBitrate(nowMs: Long): Bandwidth
/**
* Like [getBitrate], but measured over the longer window of [smoothedBitrateTracker]. Used for sources whose
* bitrate is bursty, where the instantaneous rate over-states the bandwidth the source needs.
*/
abstract fun getSmoothedBitrate(nowMs: Long): Bandwidth
/**
* Recursively checks this layer and its dependencies to see if the bitrate is zero.
* Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead
* of the hash table is usually more than the cost of any double-visits.
*/
abstract fun hasZeroBitrate(nowMs: Long): Boolean
open fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("frameRate", frameRate)
put("height", height)
put("index", index)
put("bitrate_bps", getBitrate(System.currentTimeMillis()).bps)
put("smoothed_bitrate_bps", getSmoothedBitrate(System.currentTimeMillis()).bps)
put("target_bitrate", targetBitrate?.bps ?: 0)
put("indexString", indexString())
}
abstract fun indexString(): String
companion object {
/** The length of the smoothed trackers' window, cached (it is fixed by configuration). */
private val smoothedWindowMs: Long by lazy { BitrateCalculator.smoothedWindowSize.toMillis() }
/**
* The index value that is used to represent that forwarding is suspended.
*/
const val SUSPENDED_INDEX = -1
/**
* The encoding ID value that is used to represent that forwarding is suspended.
*/
const val SUSPENDED_ENCODING_ID = -1
/**
* A value used to designate the absence of height information.
*/
const val NO_HEIGHT = -1
/**
* A value used to designate the absence of frame rate information.
*/
const val NO_FRAME_RATE = -1.0
/**
* Calculate the "index" of a layer based on its encoding, spatial, and temporal ID.
* This is a server-side id and should not be confused with any encoding id defined
* in the client (such as the rid) or the encodingId. This is used by the videobridge's
* adaptive source projection for filtering.
*/
@JvmStatic
fun getIndex(eid: Int, sid: Int, tid: Int): Int {
val e = if (eid < 0) 0 else eid
val s = if (sid < 0) 0 else sid
val t = if (tid < 0) 0 else tid
return (e shl 6) or (s shl 3) or t
}
/**
* Get an encoding ID from a layer index. If the index is [SUSPENDED_INDEX],
* [SUSPENDED_ENCODING_ID] will be returned.
*/
@JvmStatic
fun getEidFromIndex(index: Int) = index shr 6
/**
* Get a spatial ID from a layer index. If the index is [SUSPENDED_INDEX],
* the value is unspecified.
*/
@JvmStatic
fun getSidFromIndex(index: Int) = (index and 0x38) shr 3
/**
* Get a temporal ID from a layer index. If the index is [SUSPENDED_INDEX],
* the value is unspecified.
*/
@JvmStatic
fun getTidFromIndex(index: Int) = index and 0x7
/**
* Get a string description of a layer index.
*/
fun indexString(index: Int): String = if (index == SUSPENDED_INDEX) {
"SUSP"
} else {
"E${getEidFromIndex(index)}S${getSidFromIndex(index)}T${getTidFromIndex(index)}"
}
}
}

79
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt Normal file
View File

@@ -0,0 +1,79 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.rtp.LossListener
import org.jitsi.nlj.srtp.SrtpTransformers
import org.jitsi.nlj.stats.EndpointConnectionStats
import org.jitsi.nlj.stats.RtpReceiverStats
import org.jitsi.nlj.util.Bandwidth
abstract class RtpReceiver :
StatsKeepingPacketHandler(),
EventHandler,
Stoppable,
EndpointConnectionStats.EndpointConnectionStatsListener {
/**
* The handler which will be invoked for each RTP/RTCP packet received
* by this receiver (after it has gone through the receiver's
* input chain).
*/
abstract var packetHandler: PacketHandler?
/**
* Enqueue an incoming packet to be processed
*/
abstract fun enqueuePacket(p: PacketInfo)
abstract fun debugState(mode: DebugStateMode): ObjectNode
/**
* Set the SRTP transformers to be used for RTP/RTCP encryption and decryption
*/
abstract fun setSrtpTransformers(srtpTransformers: SrtpTransformers)
abstract fun getStats(): RtpReceiverStats
abstract fun tearDown()
abstract fun isReceivingAudio(): Boolean
abstract fun isReceivingVideo(): Boolean
abstract fun addLossListener(lossListener: LossListener)
abstract fun setFeature(feature: Features, enabled: Boolean)
abstract fun isFeatureEnabled(feature: Features): Boolean
/**
* Forcibly mute or unmute the incoming audio stream
*/
abstract fun forceMuteAudio(shouldMute: Boolean)
abstract fun forceMuteVideo(shouldMute: Boolean)
}
interface RtpReceiverEventHandler {
/**
* We received an audio level indication from the remote endpoint.
*/
fun audioLevelReceived(sourceSsrc: Long, level: Long): Boolean = false
/**
* The estimation of the available send bandwidth changed.
*/
fun bandwidthEstimationChanged(newValue: Bandwidth) {}
}

355
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt Normal file
View File

@@ -0,0 +1,355 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import org.jitsi.nlj.rtcp.CompoundRtcpParser
import org.jitsi.nlj.rtcp.RembHandler
import org.jitsi.nlj.rtcp.RtcpEventNotifier
import org.jitsi.nlj.rtcp.RtcpRrGenerator
import org.jitsi.nlj.rtp.AudioRtpPacket
import org.jitsi.nlj.rtp.LossListener
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.rtp.VideoRtpPacket
import org.jitsi.nlj.srtp.SrtpTransformers
import org.jitsi.nlj.stats.RtpReceiverStats
import org.jitsi.nlj.transform.NodeDebugStateVisitor
import org.jitsi.nlj.transform.NodeEventVisitor
import org.jitsi.nlj.transform.NodeTeardownVisitor
import org.jitsi.nlj.transform.node.ConsumerNode
import org.jitsi.nlj.transform.node.Node
import org.jitsi.nlj.transform.node.PacketLossConfig
import org.jitsi.nlj.transform.node.PacketLossNode
import org.jitsi.nlj.transform.node.PacketStreamStatsNode
import org.jitsi.nlj.transform.node.RtpParser
import org.jitsi.nlj.transform.node.SrtcpDecryptNode
import org.jitsi.nlj.transform.node.SrtpDecryptNode
import org.jitsi.nlj.transform.node.ToggleablePcapWriter
import org.jitsi.nlj.transform.node.incoming.AudioLevelReader
import org.jitsi.nlj.transform.node.incoming.BitrateCalculator
import org.jitsi.nlj.transform.node.incoming.DiscardableDiscarder
import org.jitsi.nlj.transform.node.incoming.DuplicateTermination
import org.jitsi.nlj.transform.node.incoming.IncomingStatisticsTracker
import org.jitsi.nlj.transform.node.incoming.PaddingTermination
import org.jitsi.nlj.transform.node.incoming.RemoteBandwidthEstimator
import org.jitsi.nlj.transform.node.incoming.RetransmissionRequesterNode
import org.jitsi.nlj.transform.node.incoming.RtcpTermination
import org.jitsi.nlj.transform.node.incoming.RtxHandler
import org.jitsi.nlj.transform.node.incoming.TccGeneratorNode
import org.jitsi.nlj.transform.node.incoming.VideoBitrateCalculator
import org.jitsi.nlj.transform.node.incoming.VideoMuteNode
import org.jitsi.nlj.transform.node.incoming.VideoParser
import org.jitsi.nlj.transform.node.incoming.VideoQualityLayerLookup
import org.jitsi.nlj.transform.node.incoming.VlaReaderNode
import org.jitsi.nlj.transform.packetPath
import org.jitsi.nlj.transform.pipeline
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.BufferPool
import org.jitsi.nlj.util.PacketInfoQueue
import org.jitsi.nlj.util.PacketPredicate
import org.jitsi.nlj.util.ReadOnlyStreamInformationStore
import org.jitsi.rtp.Packet
import org.jitsi.rtp.extensions.looksLikeRtcp
import org.jitsi.rtp.extensions.looksLikeRtp
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.queue.CountingErrorHandler
import java.util.concurrent.ExecutorService
import java.util.concurrent.ScheduledExecutorService
class RtpReceiverImpl @JvmOverloads constructor(
val id: String,
/**
* A function to be used when the receiver wants to send RTCP packets to the
* participant it's receiving data from (NACK packets, for example)
*/
private val rtcpSender: (RtcpPacket) -> Unit = {},
rtcpEventNotifier: RtcpEventNotifier,
/**
* The executor this class will use for its primary work (i.e. critical path
* packet processing). This [RtpReceiver] will execute a blocking queue read
* on this executor.
*/
private val executor: ExecutorService,
/**
* A [ScheduledExecutorService] which can be used for less important
* background tasks, or tasks that need to execute at some fixed delay/rate
*/
private val backgroundExecutor: ScheduledExecutorService,
streamInformationStore: ReadOnlyStreamInformationStore,
private val eventHandler: RtpReceiverEventHandler,
parentLogger: Logger,
diagnosticContext: DiagnosticContext = DiagnosticContext()
) : RtpReceiver() {
private val logger = createChildLogger(parentLogger)
private var running: Boolean = true
private val inputTreeRoot: Node
private val incomingPacketQueue = PacketInfoQueue(
"rtp-receiver-incoming-packet-queue",
executor,
this::handleIncomingPacket,
queueSize
)
private val srtpDecryptWrapper = SrtpDecryptNode()
private val srtcpDecryptWrapper = SrtcpDecryptNode()
private val tccGenerator = TccGeneratorNode(rtcpSender, streamInformationStore, logger)
private val remoteBandwidthEstimator = RemoteBandwidthEstimator(streamInformationStore, logger, diagnosticContext)
private val audioLevelReader = AudioLevelReader(streamInformationStore).apply {
audioLevelListener = object : AudioLevelListener {
override fun onLevelReceived(sourceSsrc: Long, level: Long): Boolean =
eventHandler.audioLevelReceived(sourceSsrc, level)
}
}
private val videoMuteNode = VideoMuteNode()
private val silenceDiscarder = DiscardableDiscarder("Silence discarder", false)
private val paddingOnlyDiscarder = DiscardableDiscarder("Padding-only discarder", true)
private val statsTracker = IncomingStatisticsTracker(streamInformationStore)
private val packetStreamStats = PacketStreamStatsNode(diagnosticContext, "receive")
private val rtcpRrGenerator = RtcpRrGenerator(backgroundExecutor, rtcpSender, statsTracker) {
remoteBandwidthEstimator.createRemb()?.let {
listOf(it)
} ?: emptyList()
}
private val rtcpTermination = RtcpTermination(rtcpEventNotifier, logger)
private val retransmissionRequester = RetransmissionRequesterNode(rtcpSender, backgroundExecutor, logger)
private val rembHandler = RembHandler(streamInformationStore, logger).apply {
addListener(object : TransportCcEngine.BandwidthListener {
override fun bandwidthEstimationChanged(newValue: Bandwidth) {
eventHandler.bandwidthEstimationChanged(newValue)
}
})
}
private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-rx")
private val videoBitrateCalculator = VideoBitrateCalculator(parentLogger)
private val audioBitrateCalculator = BitrateCalculator("Audio bitrate calculator")
private val videoParser = VideoParser(streamInformationStore, logger, diagnosticContext)
override fun isReceivingAudio() = audioBitrateCalculator.active
override fun isReceivingVideo() = videoBitrateCalculator.active
override fun addLossListener(lossListener: LossListener) {
tccGenerator.addLossListener(lossListener)
}
companion object {
var queueErrorCounter = CountingErrorHandler()
private const val PACKET_QUEUE_ENTRY_EVENT = "Entered RTP receiver incoming queue"
private const val PACKET_QUEUE_EXIT_EVENT = "Exited RTP receiver incoming queue"
private val queueSize: Int by config("jmt.transceiver.recv.queue-size".from(JitsiConfig.newConfig))
/**
* Configuration for the packet loss to introduce in the receive pipeline (for debugging/testing purposes).
*/
private val packetLossConfig = PacketLossConfig("jmt.debug.packet-loss.incoming")
}
/**
* [packetHandler] will be invoked with RTP packets that have made
* it through the entire receive pipeline. Some external entity should
* assign it to a [PacketHandler] with appropriate logic.
*/
override var packetHandler: PacketHandler? = null
/**
* The [packetHandler] can be re-assigned at any time, but it should maintain
* its place in the receive pipeline. To support both keeping it in the same
* place and allowing it to be re-assigned, we wrap it with this.
*/
private val packetHandlerWrapper = object : ConsumerNode("Input pipeline termination node") {
override fun consume(packetInfo: PacketInfo) {
// When there's no handler set we're effectively dropping packets, so their buffers
// should be returned.
packetHandler?.processPacket(packetInfo) ?: packetDiscarded(packetInfo)
}
override val aggregationKey: String = name
override fun trace(f: () -> Unit) = f.invoke()
}
init {
logger.cdebug { "using executor ${executor.hashCode()}" }
if (packetLossConfig.enabled) {
logger.warn("Will simulate packet loss: $packetLossConfig")
}
rtcpEventNotifier.addRtcpEventListener(rtcpRrGenerator)
rtcpEventNotifier.addRtcpEventListener(rembHandler)
incomingPacketQueue.setErrorHandler(queueErrorCounter)
inputTreeRoot = pipeline {
node(packetStreamStats)
demux("SRTP/SRTCP") {
packetPath {
name = "SRTP"
predicate = PacketPredicate(Packet::looksLikeRtp)
path = pipeline {
node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled })
node(RtpParser(streamInformationStore, logger))
// TODO: temporarily putting the audioLevelReader node here such that we can determine whether
// or not a packet should be discarded before doing SRTP. audioLevelReader has been moved here
// (instead of introducing a different class to read audio levels) to avoid parsing the RTP
// header extensions twice (which is expensive). In the future we will parse and cache the
// header extensions to make this lookup more efficient, at which time we could move
// audioLevelReader back to where it was (in the audio path) and add a new node here which would
// check for different discard conditions (i.e. checking the audio level for silence)
node(audioLevelReader.preDecryptNode)
node(videoMuteNode)
node(srtpDecryptWrapper)
node(tccGenerator)
node(remoteBandwidthEstimator)
// This reads audio levels from packets that use cryptex. TODO: should it go in the Audio path?
node(audioLevelReader.postDecryptNode)
node(toggleablePcapWriter.newObserverNode(outbound = false, suffix = "rx_rtp"))
node(statsTracker)
node(PaddingTermination(logger))
demux("Media Type") {
packetPath {
name = "Audio"
predicate = PacketPredicate { it is AudioRtpPacket }
path = pipeline {
node(silenceDiscarder)
node(audioBitrateCalculator)
node(packetHandlerWrapper)
}
}
packetPath {
name = "Video"
predicate = PacketPredicate { it is VideoRtpPacket }
path = pipeline {
node(RtxHandler(streamInformationStore, logger))
node(DuplicateTermination())
node(retransmissionRequester)
node(paddingOnlyDiscarder)
node(videoParser)
node(VideoQualityLayerLookup(logger))
node(videoBitrateCalculator)
node(VlaReaderNode(streamInformationStore, logger))
node(packetHandlerWrapper)
}
}
}
}
}
packetPath {
name = "SRTCP"
predicate = PacketPredicate(Packet::looksLikeRtcp)
path = pipeline {
node(srtcpDecryptWrapper)
node(toggleablePcapWriter.newObserverNode(outbound = false, suffix = "rx_rtcp"))
node(CompoundRtcpParser(logger))
node(rtcpTermination)
node(packetHandlerWrapper)
}
}
}
}
}
private fun handleIncomingPacket(packet: PacketInfo): Boolean = if (running) {
packet.addEvent(PACKET_QUEUE_EXIT_EVENT)
processPacket(packet)
true
} else {
BufferPool.returnBuffer(packet.packet.buffer)
false
}
override fun doProcessPacket(packetInfo: PacketInfo) = inputTreeRoot.processPacket(packetInfo)
override fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
NodeDebugStateVisitor(this, mode).visit(inputTreeRoot)
}
override fun enqueuePacket(p: PacketInfo) {
// logger.cinfo { "Receiver $id enqueing data" }
if (running) {
p.addEvent(PACKET_QUEUE_ENTRY_EVENT)
incomingPacketQueue.add(p)
} else {
BufferPool.returnBuffer(p.packet.buffer)
}
}
override fun setSrtpTransformers(srtpTransformers: SrtpTransformers) {
srtpDecryptWrapper.transformer = srtpTransformers.srtpDecryptTransformer
srtcpDecryptWrapper.transformer = srtpTransformers.srtcpDecryptTransformer
}
override fun handleEvent(event: Event) {
NodeEventVisitor(event).visit(inputTreeRoot)
}
override fun getStats(): RtpReceiverStats = RtpReceiverStats(
incomingStats = statsTracker.getSnapshot(),
packetStreamStats = packetStreamStats.snapshot(),
videoParserStats = videoParser.getStats()
)
override fun forceMuteAudio(shouldMute: Boolean) {
audioLevelReader.forceMute = shouldMute
}
override fun forceMuteVideo(shouldMute: Boolean) {
videoMuteNode.forceMute = shouldMute
}
override fun setFeature(feature: Features, enabled: Boolean) {
when (feature) {
Features.TRANSCEIVER_PCAP_DUMP -> {
if (enabled) {
toggleablePcapWriter.enable()
} else {
toggleablePcapWriter.disable()
}
}
}
}
override fun isFeatureEnabled(feature: Features): Boolean = when (feature) {
Features.TRANSCEIVER_PCAP_DUMP -> toggleablePcapWriter.isEnabled()
}
override fun stop() {
running = false
rtcpRrGenerator.running = false
retransmissionRequester.stop()
}
override fun tearDown() {
logger.debug("Tearing down")
NodeTeardownVisitor().visit(inputTreeRoot)
incomingPacketQueue.close()
toggleablePcapWriter.disable()
}
override fun onRttUpdate(newRttMs: Double) {
remoteBandwidthEstimator.onRttUpdate(newRttMs)
}
}

59
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.rtp.LossListener
import org.jitsi.nlj.rtp.RtpExtensionType
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.srtp.SrtpTransformers
import org.jitsi.nlj.stats.EndpointConnectionStats
import org.jitsi.nlj.stats.PacketStreamStats
import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsSnapshot
/**
* Not an 'RtpSender' in the sense that it sends only RTP (and not
* RTCP) but in the sense of a webrtc 'RTCRTPSender' which handles
* all RTP and RTP control packets.
*/
abstract class RtpSender :
StatsKeepingPacketHandler(),
EventHandler,
Stoppable,
EndpointConnectionStats.EndpointConnectionStatsListener {
abstract fun sendProbing(mediaSsrcs: Collection<Long>, numBytes: Int): Int
abstract fun onOutgoingPacket(handler: PacketHandler)
abstract fun setSrtpTransformers(srtpTransformers: SrtpTransformers)
abstract fun getStreamStats(): OutgoingStatisticsSnapshot
abstract fun getPacketStreamStats(): PacketStreamStats.Snapshot
abstract fun addBandwidthListener(listener: TransportCcEngine.BandwidthListener)
abstract fun removeBandwidthListener(listener: TransportCcEngine.BandwidthListener)
abstract fun getTransportCcEngineStats(): TransportCcEngine.StatisticsSnapshot
abstract fun requestKeyframe(requesterID: String?, mediaSsrc: Long? = null)
abstract fun addLossListener(lossListener: LossListener)
abstract fun setFeature(feature: Features, enabled: Boolean)
abstract fun isFeatureEnabled(feature: Features): Boolean
abstract fun tearDown()
abstract fun addRtpExtensionToRetain(extensionType: RtpExtensionType)
/**
* An optional function to be executed for each RTP packet, as the first step of the send pipeline.
*/
var preProcesor: ((PacketInfo) -> PacketInfo?)? = null
abstract fun debugState(mode: DebugStateMode): ObjectNode
}

384
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt Normal file
View File

@@ -0,0 +1,384 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import org.jitsi.nlj.rtcp.KeyframeRequester
import org.jitsi.nlj.rtcp.NackHandler
import org.jitsi.nlj.rtcp.RtcpEventNotifier
import org.jitsi.nlj.rtcp.RtcpSrUpdater
import org.jitsi.nlj.rtp.ClassicTransportCcEngine
import org.jitsi.nlj.rtp.LossListener
import org.jitsi.nlj.rtp.RtpExtensionType
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorConfig
import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorEngine
import org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator
import org.jitsi.nlj.rtp.bandwidthestimation2.GoogCcTransportCcEngine
import org.jitsi.nlj.srtp.SrtpTransformers
import org.jitsi.nlj.transform.NodeDebugStateVisitor
import org.jitsi.nlj.transform.NodeEventVisitor
import org.jitsi.nlj.transform.NodeTeardownVisitor
import org.jitsi.nlj.transform.node.AudioRedHandler
import org.jitsi.nlj.transform.node.ConsumerNode
import org.jitsi.nlj.transform.node.Node
import org.jitsi.nlj.transform.node.ObserverNode
import org.jitsi.nlj.transform.node.PacketCacher
import org.jitsi.nlj.transform.node.PacketLossConfig
import org.jitsi.nlj.transform.node.PacketLossNode
import org.jitsi.nlj.transform.node.PacketStreamStatsNode
import org.jitsi.nlj.transform.node.PluggableTransformerNode
import org.jitsi.nlj.transform.node.SrtcpEncryptNode
import org.jitsi.nlj.transform.node.SrtpEncryptNode
import org.jitsi.nlj.transform.node.ToggleablePcapWriter
import org.jitsi.nlj.transform.node.outgoing.AbsSendTime
import org.jitsi.nlj.transform.node.outgoing.HeaderExtEncoder
import org.jitsi.nlj.transform.node.outgoing.HeaderExtStripper
import org.jitsi.nlj.transform.node.outgoing.MidStamper
import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsTracker
import org.jitsi.nlj.transform.node.outgoing.ProbingDataSender
import org.jitsi.nlj.transform.node.outgoing.RetransmissionSender
import org.jitsi.nlj.transform.node.outgoing.SentRtcpStats
import org.jitsi.nlj.transform.node.outgoing.TccSeqNumTagger
import org.jitsi.nlj.transform.pipeline
import org.jitsi.nlj.util.BufferPool
import org.jitsi.nlj.util.PacketInfoQueue
import org.jitsi.nlj.util.StreamInformationStore
import org.jitsi.nlj.util.appendAll
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.utils.MediaType
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.queue.CountingErrorHandler
import java.time.Duration
import java.util.concurrent.ExecutorService
import java.util.concurrent.ScheduledExecutorService
class RtpSenderImpl(
val id: String,
private val rtcpEventNotifier: RtcpEventNotifier,
/**
* The executor this class will use for its primary work (i.e. critical path
* packet processing). This [RtpSender] will execute a blocking queue read
* on this executor.
*/
val executor: ExecutorService,
/**
* A [ScheduledExecutorService] which can be used for less important
* background tasks, or tasks that need to execute at some fixed delay/rate
*/
val backgroundExecutor: ScheduledExecutorService,
private val streamInformationStore: StreamInformationStore,
parentLogger: Logger,
diagnosticContext: DiagnosticContext = DiagnosticContext(),
/**
* Resolves the sdes:mid to stamp on an outgoing packet given its (rewritten) SSRC, or null to not stamp a mid.
* Used under SSRC rewriting for mid-based demuxing on the receiving client.
*/
private val getMidBySsrc: (Long) -> String? = { null }
) : RtpSender() {
private val logger = createChildLogger(parentLogger)
private val outgoingRtpRoot: Node
private val outgoingRtxRoot: Node
private val outgoingRtcpRoot: Node
private val incomingPacketQueue = PacketInfoQueue(
"rtp-sender-incoming-packet-queue",
executor,
this::handlePacket,
queueSize
)
var running = true
private var localVideoSsrc: Long? = null
private var localAudioSsrc: Long? = null
// TODO(brian): this is changed to a handler instead of a queue because we want to use
// a PacketQueue, and the handler for a PacketQueue must be set at the time of creation.
// since we want the handler to be another entity (something in jvb) we just use
// a generic handler here and then the bridge can put it into its PacketQueue and have
// its handler (likely in another thread) grab the packet and send it out
private var outgoingPacketHandler: PacketHandler? = null
private val transportCcEngine: TransportCcEngine =
when (BandwidthEstimatorConfig.engine) {
BandwidthEstimatorEngine.GoogleCc ->
ClassicTransportCcEngine(GoogleCcEstimator(diagnosticContext, logger), logger)
BandwidthEstimatorEngine.GoogleCc2 ->
GoogCcTransportCcEngine(diagnosticContext, logger, backgroundExecutor, { dataSize, probingData ->
return@GoogCcTransportCcEngine probingDataSender.sendProbing(
null,
dataSize.bytes.toInt(),
probingData
)
})
}.also { it.start() }
private val srtpEncryptWrapper = SrtpEncryptNode()
private val srtcpEncryptWrapper = SrtcpEncryptNode()
private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-tx")
private val outgoingPacketCache = PacketCacher()
private val headerExtensionStripper = HeaderExtStripper(streamInformationStore)
private val absSendTime = AbsSendTime(streamInformationStore)
private val statsTracker = OutgoingStatisticsTracker(diagnosticContext)
private val packetStreamStats = PacketStreamStatsNode(diagnosticContext, "send")
private val rtcpSrUpdater = RtcpSrUpdater(statsTracker)
private val keyframeRequester = KeyframeRequester(streamInformationStore, logger)
private val probingDataSender: ProbingDataSender
private val nackHandler: NackHandler
private val outputPipelineTerminationNode = object : ConsumerNode("Output pipeline termination node") {
override fun consume(packetInfo: PacketInfo) {
// While there's no handler set we're effectively dropping packets, so their buffers
// should be returned.
outgoingPacketHandler?.processPacket(packetInfo) ?: packetDiscarded(packetInfo)
}
override val aggregationKey = name
override fun trace(f: () -> Unit) = f.invoke()
}
init {
logger.cdebug { "Sender $id using executor ${executor.hashCode()}" }
if (packetLossConfig.enabled) {
logger.warn("Will simulate packet loss: $packetLossConfig")
}
incomingPacketQueue.setErrorHandler(queueErrorCounter)
outgoingRtpRoot = pipeline {
node(PluggableTransformerNode("RTP pre-processor") { preProcesor })
node(AudioRedHandler(streamInformationStore, logger))
node(headerExtensionStripper)
node(outgoingPacketCache)
node(absSendTime)
// Placed right after absSendTime, which is where the RTX pipeline joins the main pipeline, so that both
// media packets and their retransmissions get the mid stamped.
node(MidStamper(streamInformationStore, getMidBySsrc))
node(statsTracker)
node(TccSeqNumTagger(transportCcEngine, streamInformationStore))
node(HeaderExtEncoder(streamInformationStore, logger))
node(toggleablePcapWriter.newObserverNode(outbound = true, suffix = "tx_rtp"))
node(srtpEncryptWrapper)
node(packetStreamStats.createNewNode())
node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled })
node(outputPipelineTerminationNode)
}
outgoingRtxRoot = pipeline {
node(RetransmissionSender(streamInformationStore, logger))
// We want RTX packets to hook into the main RTP pipeline starting at AbsSendTime
node(absSendTime)
}
nackHandler = NackHandler(outgoingPacketCache.getPacketCache(), outgoingRtxRoot, logger)
rtcpEventNotifier.addRtcpEventListener(nackHandler)
rtcpEventNotifier.addRtcpEventListener(transportCcEngine)
// TODO: are we setting outgoing rtcp sequence numbers correctly? just add a simple node here to rewrite them
outgoingRtcpRoot = pipeline {
node(PluggableTransformerNode("RTCP pre-processor") { preProcesor })
node(keyframeRequester)
node(SentRtcpStats())
// TODO(brian): not sure this is a great idea. it works as a catch-call but can also be error-prone
// (i found i was accidentally clobbering the sender ssrc for SRs which caused issues). I think
// it'd be better to notify everything creating RTCP the bridge SSRCs and then everything should be
// responsible for setting it themselves
simpleNode("RTCP sender ssrc setter") { packetInfo ->
val senderSsrc = localVideoSsrc ?: return@simpleNode packetInfo
val rtcpPacket = packetInfo.packetAs<RtcpPacket>()
if (rtcpPacket.senderSsrc == 0L) {
rtcpPacket.senderSsrc = senderSsrc
}
packetInfo
}
node(rtcpSrUpdater)
node(toggleablePcapWriter.newObserverNode(outbound = true, suffix = "tx_rtcp"))
node(object : ObserverNode("RTCP sent notifier") {
override fun observe(packetInfo: PacketInfo) {
val packet = packetInfo.packet
if (packet is RtcpPacket) {
rtcpEventNotifier.notifyRtcpSent(packet)
}
}
override fun trace(f: () -> Unit) {}
})
node(srtcpEncryptWrapper)
node(packetStreamStats.createNewNode())
node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled })
node(outputPipelineTerminationNode)
}
probingDataSender = ProbingDataSender(
packetCache = outgoingPacketCache.getPacketCache(),
rtxDataSender = outgoingRtxRoot,
garbageDataSender = absSendTime,
diagnosticContext = diagnosticContext,
streamInformationStore = streamInformationStore,
parentLogger = logger
)
}
override fun onRttUpdate(newRttMs: Double) {
nackHandler.onRttUpdate(newRttMs)
keyframeRequester.onRttUpdate(newRttMs)
transportCcEngine.onRttUpdate(Duration.ofNanos((newRttMs * 1e6).toLong()))
}
/**
* Insert packets into the incoming packet queue
*/
override fun doProcessPacket(packetInfo: PacketInfo) {
if (running) {
packetInfo.addEvent(PACKET_QUEUE_ENTRY_EVENT)
incomingPacketQueue.add(packetInfo)
} else {
BufferPool.returnBuffer(packetInfo.packet.buffer)
}
}
override fun sendProbing(mediaSsrcs: Collection<Long>, numBytes: Int): Int =
probingDataSender.sendProbing(mediaSsrcs, numBytes, null)
override fun onOutgoingPacket(handler: PacketHandler) {
outgoingPacketHandler = handler
}
override fun setSrtpTransformers(srtpTransformers: SrtpTransformers) {
srtpEncryptWrapper.transformer = srtpTransformers.srtpEncryptTransformer
srtcpEncryptWrapper.transformer = srtpTransformers.srtcpEncryptTransformer
}
override fun requestKeyframe(requesterID: String?, mediaSsrc: Long?) {
keyframeRequester.requestKeyframe(requesterID, mediaSsrc)
}
override fun addLossListener(lossListener: LossListener) {
transportCcEngine.addLossListener(lossListener)
}
override fun setFeature(feature: Features, enabled: Boolean) {
when (feature) {
Features.TRANSCEIVER_PCAP_DUMP -> {
if (enabled) {
toggleablePcapWriter.enable()
} else {
toggleablePcapWriter.disable()
}
}
}
}
override fun isFeatureEnabled(feature: Features): Boolean = when (feature) {
Features.TRANSCEIVER_PCAP_DUMP -> toggleablePcapWriter.isEnabled()
}
/**
* Handles packets that have gone through the incoming queue and sends them
* through the sender pipeline
*/
private fun handlePacket(packetInfo: PacketInfo): Boolean = if (running) {
packetInfo.addEvent(PACKET_QUEUE_EXIT_EVENT)
val root = when (packetInfo.packet) {
is RtcpPacket -> outgoingRtcpRoot
else -> outgoingRtpRoot
}
root.processPacket(packetInfo)
true
} else {
BufferPool.returnBuffer(packetInfo.packet.buffer)
false
}
override fun getStreamStats() = statsTracker.getSnapshot()
override fun getPacketStreamStats() = packetStreamStats.snapshot()
override fun getTransportCcEngineStats() = transportCcEngine.getStatistics()
override fun addBandwidthListener(listener: TransportCcEngine.BandwidthListener) =
transportCcEngine.addBandwidthListener(listener)
override fun removeBandwidthListener(listener: TransportCcEngine.BandwidthListener) =
transportCcEngine.removeBandwidthListener(listener)
override fun handleEvent(event: Event) {
when (event) {
is SetLocalSsrcEvent -> {
when (event.mediaType) {
MediaType.VIDEO -> localVideoSsrc = event.ssrc
MediaType.AUDIO -> localAudioSsrc = event.ssrc
else -> {}
}
}
}
NodeEventVisitor(event).reverseVisit(outputPipelineTerminationNode)
probingDataSender.handleEvent(event)
}
override fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
if (mode == DebugStateMode.FULL) {
appendAll(super.getNodeStats().toJson())
set<ObjectNode>("packet_queue", incomingPacketQueue.debugState)
put("running", running)
}
appendAll(nackHandler.getNodeStats().toJson())
set<ObjectNode>("probing_data_sender", probingDataSender.debugState(mode))
put("local_video_ssrc", localVideoSsrc?.toString() ?: "null")
put("local_audio_ssrc", localAudioSsrc?.toString() ?: "null")
set<ObjectNode>("transport_cc_engine", transportCcEngine.getStatistics().toJson())
NodeDebugStateVisitor(this, mode).reverseVisit(outputPipelineTerminationNode)
}
override fun stop() {
running = false
}
override fun tearDown() {
logger.debug("Tearing down")
NodeTeardownVisitor().reverseVisit(outputPipelineTerminationNode)
incomingPacketQueue.close()
toggleablePcapWriter.disable()
}
override fun addRtpExtensionToRetain(extensionType: RtpExtensionType) {
headerExtensionStripper.addRtpExtensionToRetain(extensionType)
}
companion object {
var queueErrorCounter = CountingErrorHandler()
private const val PACKET_QUEUE_ENTRY_EVENT = "Entered RTP sender incoming queue"
private const val PACKET_QUEUE_EXIT_EVENT = "Exited RTP sender incoming queue"
private val queueSize: Int by config("jmt.transceiver.send.queue-size".from(JitsiConfig.newConfig))
/**
* Configuration for the packet loss to introduce in the send pipeline (for debugging/testing purposes).
*/
private val packetLossConfig = PacketLossConfig("jmt.debug.packet-loss.outgoing")
}
}

20
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Stoppable.kt Normal file
View File

@@ -0,0 +1,20 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
interface Stoppable {
fun stop()
}

397
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt Normal file
View File

@@ -0,0 +1,397 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.format.PayloadType
import org.jitsi.nlj.rtcp.RtcpEventNotifier
import org.jitsi.nlj.rtp.RtpExtension
import org.jitsi.nlj.rtp.RtpExtensionType
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.srtp.SrtpTransformers
import org.jitsi.nlj.srtp.SrtpUtil
import org.jitsi.nlj.srtp.TlsRole
import org.jitsi.nlj.stats.EndpointConnectionStats
import org.jitsi.nlj.stats.PacketIOActivity
import org.jitsi.nlj.stats.TransceiverStats
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.LocalSsrcAssociation
import org.jitsi.nlj.util.ReadOnlyStreamInformationStore
import org.jitsi.nlj.util.SsrcAssociation
import org.jitsi.nlj.util.StreamInformationStoreImpl
import org.jitsi.utils.MediaType
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.cinfo
import org.jitsi.utils.logging2.createChildLogger
import java.time.Clock
import java.util.concurrent.ExecutorService
import java.util.concurrent.ScheduledExecutorService
// This is an API class, so its usages will largely be outside of this library
/**
* Handles all packets (incoming and outgoing) for a particular stream.
* (TODO: 'stream' defined as what, exactly, here?)
* Handles the DTLS negotiation
*
* Incoming packets should be written via [handleIncomingPacket]. Outgoing
* packets are put in [outgoingQueue] (and should be read by something)
* TODO: maybe we want to have this 'push' the outgoing packets somewhere
* else instead (then we could have all senders push to a single queue and
* have the one thread just read from the queue and send, rather than that thread
* having to read from a bunch of individual queues)
*/
@Suppress("unused")
class Transceiver(
private val id: String,
receiverExecutor: ExecutorService,
senderExecutor: ExecutorService,
/**
* A [ScheduledExecutorService] which can be used for less important
* background tasks, or tasks that need to execute at some fixed delay/rate
*/
backgroundExecutor: ScheduledExecutorService,
diagnosticContext: DiagnosticContext,
parentLogger: Logger,
/**
* The handler for events coming out of this [Transceiver]. Note that these events are fired synchronously,
* potentially in one of the threads in the CPU pool, and it is up to the user of the library to handle the
* transition to another thread if necessary.
*/
private val eventHandler: TransceiverEventHandler,
private val clock: Clock = Clock.systemUTC(),
/**
* Resolves the sdes:mid to stamp on an outgoing packet given its (rewritten) SSRC, or null to not stamp a mid.
*/
getMidBySsrc: (Long) -> String? = { null }
) : Stoppable {
private val logger = createChildLogger(parentLogger)
val packetIOActivity = PacketIOActivity()
private val endpointConnectionStats = EndpointConnectionStats(logger)
private val streamInformationStore = StreamInformationStoreImpl()
val readOnlyStreamInformationStore: ReadOnlyStreamInformationStore = streamInformationStore
/**
* A central place to subscribe to be notified on the reception or transmission of RTCP packets for
* this transceiver. This is intended to be used by internal entities: mainly logic for things like generating
* SRs and RRs and calculating RTT. Since it is used for both send and receive, it is held here and passed to
* the sender and receive so each can push or subscribe to updates.
*/
val rtcpEventNotifier = RtcpEventNotifier()
private var mediaSources = MediaSources()
var srtpTransformers: SrtpTransformers? = null
/** Whether the srtpTransformers were created inside this object, or passed in externally.
* For external transformers it's the external owner's responsibility to close them.
*/
var internalTransformers = false
/**
* Whether this [Transceiver] is receiving audio from the remote endpoint.
*/
fun isReceivingAudio(): Boolean = rtpReceiver.isReceivingAudio()
/**
* Whether this [Transceiver] is receiving video from the remote endpoint.
*/
fun isReceivingVideo(): Boolean = rtpReceiver.isReceivingVideo()
val rtpSender: RtpSender = RtpSenderImpl(
id,
rtcpEventNotifier,
senderExecutor,
backgroundExecutor,
streamInformationStore,
logger,
diagnosticContext,
getMidBySsrc
)
private val rtpReceiver: RtpReceiver =
RtpReceiverImpl(
id,
{ rtcpPacket ->
if (rtcpPacket.length >= 1500) {
logger.warn(
"Sending large locally-generated RTCP packet of size ${rtcpPacket.length}, " +
"first packet of type ${rtcpPacket.packetType} rc ${rtcpPacket.reportCount}."
)
}
rtpSender.processPacket(PacketInfo(rtcpPacket))
},
rtcpEventNotifier,
receiverExecutor,
backgroundExecutor,
streamInformationStore,
eventHandler,
logger,
diagnosticContext
)
init {
rtpSender.addBandwidthListener(
object : TransportCcEngine.BandwidthListener {
override fun bandwidthEstimationChanged(newValue: Bandwidth) {
eventHandler.bandwidthEstimationChanged(newValue)
}
}
)
rtpReceiver.addLossListener(endpointConnectionStats.incomingLossTracker)
rtpSender.addLossListener(endpointConnectionStats.outgoingLossTracker)
rtcpEventNotifier.addRtcpEventListener(endpointConnectionStats)
endpointConnectionStats.addListener(rtpSender)
endpointConnectionStats.addListener(rtpReceiver)
}
/**
* Handle an incoming [PacketInfo] (that is, a packet received by the endpoint
* this transceiver is associated with) to be processed by the receiver pipeline.
*/
fun handleIncomingPacket(p: PacketInfo) {
packetIOActivity.lastRtpPacketReceivedInstant = clock.instant()
rtpReceiver.enqueuePacket(p)
}
/**
* Send packets to the endpoint this transceiver is associated with by
* passing them out the sender's outgoing pipeline
*/
fun sendPacket(packetInfo: PacketInfo) {
packetIOActivity.lastRtpPacketSentInstant = clock.instant()
rtpSender.processPacket(packetInfo)
}
fun sendProbing(mediaSsrcs: Collection<Long>, numBytes: Int): Int = rtpSender.sendProbing(mediaSsrcs, numBytes)
/**
* Set a handler to be invoked when incoming RTP packets have finished
* being processed.
*/
fun setIncomingPacketHandler(rtpHandler: PacketHandler) {
rtpReceiver.packetHandler = rtpHandler
}
/**
* Set a handler to be invoked when outgoing packets have finished
* being processed (and are ready to be sent)
*/
fun setOutgoingPacketHandler(outgoingPacketHandler: PacketHandler) {
rtpSender.onOutgoingPacket(outgoingPacketHandler)
}
fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) {
logger.cdebug { "${hashCode()} adding receive ssrc $ssrc of type $mediaType" }
streamInformationStore.addReceiveSsrc(ssrc, mediaType)
}
fun removeReceiveSsrc(ssrc: Long) {
logger.cinfo { "Transceiver ${hashCode()} removing receive ssrc $ssrc" }
streamInformationStore.removeReceiveSsrc(ssrc)
}
/**
* Set the 'local' bridge SSRC to [ssrc] for [mediaType]
*/
fun setLocalSsrc(mediaType: MediaType, ssrc: Long) {
val localSsrcSetEvent = SetLocalSsrcEvent(mediaType, ssrc)
rtpSender.handleEvent(localSsrcSetEvent)
rtpReceiver.handleEvent(localSsrcSetEvent)
}
fun addRtpExtensionToRetain(extensionType: RtpExtensionType) {
rtpSender.addRtpExtensionToRetain(extensionType)
}
fun receivesSsrc(ssrc: Long): Boolean = streamInformationStore.receiveSsrcs.contains(ssrc)
val receiveSsrcs: Set<Long>
get() = streamInformationStore.receiveSsrcs
fun setMediaSources(mediaSources: Array<MediaSourceDesc>): Boolean {
logger.cdebug { "$id setting media sources: ${mediaSources.joinToString()}" }
val ret = this.mediaSources.setMediaSources(mediaSources)
val mergedMediaSources = this.mediaSources.getMediaSources()
val signaledMediaSources = mediaSources.copy()
rtpReceiver.handleEvent(SetMediaSourcesEvent(mergedMediaSources, signaledMediaSources))
return ret
}
// TODO(brian): we should only expose an immutable version of this, but Array doesn't have that. Go in
// and change all the storage of the media sources to use a list
fun getMediaSources(): Array<MediaSourceDesc> = mediaSources.getMediaSources()
@JvmOverloads
fun requestKeyFrame(requesterID: String? = null, mediaSsrc: Long? = null) =
rtpSender.requestKeyframe(requesterID, mediaSsrc)
fun addPayloadType(payloadType: PayloadType) {
logger.cdebug { "Payload type added: $payloadType" }
streamInformationStore.addRtpPayloadType(payloadType)
}
fun clearPayloadTypes() {
logger.cinfo { "All payload types being cleared" }
streamInformationStore.clearRtpPayloadTypes()
}
fun addRtpExtension(rtpExtension: RtpExtension) {
logger.cdebug { "Adding RTP extension: $rtpExtension" }
streamInformationStore.addRtpExtensionMapping(rtpExtension)
}
fun clearRtpExtensions() {
logger.cinfo { "Clearing all RTP extensions" }
// TODO: ignoring this for now, since we'll have conflicts from each channel calling it
// val rtpExtensionClearEvent = RtpExtensionClearEvent()
// rtpReceiver.handleEvent(rtpExtensionClearEvent)
// rtpSender.handleEvent(rtpExtensionClearEvent)
// rtpExtensions.clear()
}
fun setExtmapAllowMixed(allow: Boolean) {
streamInformationStore.setExtmapAllowMixed(allow)
}
// TODO(brian): we may want to handle local and remote ssrc associations differently, as different parts of the
// code care about one or the other, but currently there is no issue treating them the same.
fun addSsrcAssociation(ssrcAssociation: SsrcAssociation) {
logger.cdebug {
val location = if (ssrcAssociation is LocalSsrcAssociation) "local" else "remote"
"Adding $location SSRC association: $ssrcAssociation"
}
streamInformationStore.addSsrcAssociation(ssrcAssociation)
}
fun setSrtpInformation(
chosenSrtpProtectionProfile: Int,
tlsRole: TlsRole,
keyingMaterial: ByteArray,
cryptex: Boolean
) {
val srtpProfileInfo =
SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile)
logger.cdebug {
"Transceiver $id creating transformers with:\n" +
"profile info:\n$srtpProfileInfo\n" +
"tls role: $tlsRole\n" +
"cryptex: $cryptex"
}
srtpTransformers = SrtpUtil.initializeTransformer(
srtpProfileInfo,
keyingMaterial,
tlsRole,
cryptex,
logger
).also { setSrtpInformationInternal(it, true) }
}
private fun setSrtpInformationInternal(srtpTransformers: SrtpTransformers, internal: Boolean) {
rtpReceiver.setSrtpTransformers(srtpTransformers)
rtpSender.setSrtpTransformers(srtpTransformers)
internalTransformers = internal
}
fun setSrtpInformation(srtpTransformers: SrtpTransformers) = setSrtpInformationInternal(srtpTransformers, false)
/**
* Forcibly mute or unmute the incoming audio stream
*/
fun forceMuteAudio(shouldMute: Boolean) {
when (shouldMute) {
true -> logger.info("Muting incoming audio")
false -> logger.info("Unmuting incoming audio")
}
rtpReceiver.forceMuteAudio(shouldMute)
}
fun forceMuteVideo(shouldMute: Boolean) {
when (shouldMute) {
true -> logger.info("Muting incoming video")
false -> logger.info("Unmuting incoming video")
}
rtpReceiver.forceMuteVideo(shouldMute)
}
fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
set<ObjectNode>("stream_information_store", streamInformationStore.debugState(mode))
set<ObjectNode>("media_sources", mediaSources.debugState())
set<ObjectNode>("endpoint_connection_stats", endpointConnectionStats.getSnapshot().toJson())
set<ObjectNode>("receiver", rtpReceiver.debugState(mode))
set<ObjectNode>("sender", rtpSender.debugState(mode))
}
/**
* Get various media and network stats
*/
fun getTransceiverStats(): TransceiverStats = TransceiverStats(
endpointConnectionStats.getSnapshot(),
rtpReceiver.getStats(),
rtpSender.getStreamStats(),
rtpSender.getPacketStreamStats(),
rtpSender.getTransportCcEngineStats()
)
fun addEndpointConnectionStatsListener(listener: EndpointConnectionStats.EndpointConnectionStatsListener) =
endpointConnectionStats.addListener(listener)
fun removeEndpointConnectionStatsListener(listener: EndpointConnectionStats.EndpointConnectionStatsListener) =
endpointConnectionStats.removeListener(listener)
override fun stop() {
rtpReceiver.stop()
rtpSender.stop()
if (internalTransformers) {
srtpTransformers?.close()
}
}
fun teardown() {
logger.debug("Tearing down")
rtpReceiver.tearDown()
rtpSender.tearDown()
}
fun setFeature(feature: Features, enabled: Boolean) {
rtpReceiver.setFeature(feature, enabled)
rtpSender.setFeature(feature, enabled)
}
fun isFeatureEnabled(feature: Features): Boolean {
// As of now, the only feature we have (pcap) is always enabled on both
// the RTP sender and RTP receiver at the same time, so returning
// the state of one of them is sufficient. If that were to change
// in the future we'd have to rethink this API
return rtpReceiver.isFeatureEnabled(feature)
}
companion object {
init {
// Node.plugins.add(BufferTracePlugin)
// Node.PLUGINS_ENABLED = true
}
}
}
/**
* Interface for handling events coming from a [Transceiver]
* The intention is to extend if needed (e.g. merge with EndpointConnectionStats or a potential RtpSenderEventHandler).
*/
interface TransceiverEventHandler : RtpReceiverEventHandler

37
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright @ 2021-Present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj
enum class VideoType {
CAMERA,
DESKTOP,
DESKTOP_HIGH_FPS,
DISABLED,
// NONE was used in the context where an Endpoint has always one media source description. It used to cover both
// lack of the actual source and the source being temporarily disabled. With the support for multiple sources per
// endpoint DISABLED means a source is turned off. Lack of a MediaSourceDesc is equivalent to NONE.
@Deprecated("Use DISABLED instead", ReplaceWith("DISABLED"))
NONE;
fun isEnabled() = this != NONE && this != DISABLED
/**
* Whether this is a screen sharing type. Note that among other things screen sharing sources have bursty bitrates,
* because the encoder may leave long gaps between frames while the content is static.
*/
fun isScreenshare() = this == DESKTOP || this == DESKTOP_HIGH_FPS
}

94
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt Normal file
View File

@@ -0,0 +1,94 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.codec.vp8
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi_modified.impl.neomedia.codec.video.vp8.DePacketizer
import java.nio.ByteBuffer
class Vp8Utils {
companion object {
private const val VP8_PAYLOAD_HEADER_LEN = 3
// TODO(brian): should move these elsewhere probably
private const val MIN_HD_HEIGHT = 540
private const val MIN_SD_HEIGHT = 360
private const val HD_LAYER_ID = 2
private const val SD_LAYER_ID = 1
private const val LD_LAYER_ID = 0
private const val SUSPENDED_LAYER_ID = -1
fun isKeyFrame(vp8Payload: ByteBuffer): Boolean =
DePacketizer.isKeyFrame(vp8Payload.array(), vp8Payload.arrayOffset(), vp8Payload.limit())
fun getSpatialLayerIndexFromKeyFrame(vp8Payload: ByteBuffer): Int {
// Copied from VP8QualityFilter#getSpatialLayerIndexFromKeyframe
val payloadDescriptorLen =
DePacketizer.VP8PayloadDescriptor.getSize(
vp8Payload.array(),
vp8Payload.arrayOffset(),
vp8Payload.limit()
)
val height = DePacketizer.VP8KeyframeHeader.getHeight(
vp8Payload.array(),
vp8Payload.arrayOffset() + payloadDescriptorLen + VP8_PAYLOAD_HEADER_LEN
)
return when {
height >= MIN_HD_HEIGHT -> HD_LAYER_ID
height >= MIN_SD_HEIGHT -> SD_LAYER_ID
height > -1 -> LD_LAYER_ID
else -> -1
}
}
fun getHeightFromKeyFrame(vp8Packet: RtpPacket): Int {
val payloadDescriptorLen =
DePacketizer.VP8PayloadDescriptor.getSize(
vp8Packet.buffer,
vp8Packet.payloadOffset,
vp8Packet.payloadLength
)
return DePacketizer.VP8KeyframeHeader.getHeight(
vp8Packet.buffer,
vp8Packet.payloadOffset + payloadDescriptorLen + VP8_PAYLOAD_HEADER_LEN
)
}
fun getSpatialLayerIndexFromKeyFrame(vp8Packet: RtpPacket): Int {
// Copied from VP8QUalityFilter#getSpatialLayerIndexFromKeyframe
val height = getHeightFromKeyFrame(vp8Packet)
return when {
height >= MIN_HD_HEIGHT -> HD_LAYER_ID
height >= MIN_SD_HEIGHT -> SD_LAYER_ID
height > -1 -> LD_LAYER_ID
else -> -1
}
}
fun getTemporalLayerIdOfFrame(vp8Payload: ByteBuffer) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex(
vp8Payload.array(),
vp8Payload.arrayOffset(),
vp8Payload.limit()
)
fun getTemporalLayerIdOfFrame(vp8Packet: RtpPacket) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex(
vp8Packet.buffer,
vp8Packet.payloadOffset,
vp8Packet.payloadLength
)
}
}

90
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt Normal file
View File

@@ -0,0 +1,90 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.codec.vpx
class VpxUtils {
companion object {
/**
* The bitmask for the TL0PICIDX field.
*/
const val TL0PICIDX_MASK = 0xff
/**
* The bitmask for the extended picture id field.
*/
const val EXTENDED_PICTURE_ID_MASK = 0x7fff
/**
* Returns the delta between two VP8/VP9 extended picture IDs, taking into account
* rollover. This will return the 'shortest' delta between the two
* picture IDs in the form of the number you'd add to b to get a. e.g.:
* getExtendedPictureIdDelta(1, 10) -> -9 (10 + -9 = 1)
* getExtendedPictureIdDelta(1, 32760) -> 9 (32760 + 9 = 1)
* @return the delta between two extended picture IDs (modulo 2^15).
*/
@JvmStatic
fun getExtendedPictureIdDelta(a: Int, b: Int): Int {
val diff = a - b
return when {
diff < -(1 shl 14) -> diff + (1 shl 15)
diff > (1 shl 14) -> diff - (1 shl 15)
else -> diff
}
}
/**
* Apply a delta to a given extended picture ID and return the result (taking
* rollover into account)
* @param start the starting extended picture ID
* @param delta the delta to be applied
* @return the extended picture ID resulting from doing "start + delta"
*/
@JvmStatic
fun applyExtendedPictureIdDelta(start: Int, delta: Int): Int = (start + delta) and EXTENDED_PICTURE_ID_MASK
/**
* Returns the delta between two VP8/VP9 Tl0PicIdx values, taking into account
* rollover. This will return the 'shortest' delta between the two
* picture IDs in the form of the number you'd add to b to get a. e.g.:
* getTl0PicIdxDelta(1, 10) -> -9 (10 + -9 = 1)
* getTl0PicIdxDelta(1, 250) -> 7 (250 + 7 = 1)
*
* If either value is -1 (meaning tl0picidx not found) return 0.
* @return the delta between two extended picture IDs (modulo 2^8).
*/
@JvmStatic
fun getTl0PicIdxDelta(a: Int, b: Int): Int {
if (a < 0 || b < 0) return 0
val diff = a - b
return when {
diff < -(1 shl 7) -> diff + (1 shl 8)
diff > (1 shl 7) -> diff - (1 shl 8)
else -> diff
}
}
/**
* Apply a delta to a given Tl0PidIcx and return the result (taking
* rollover into account)
* @param start the starting Tl0PicIdx
* @param delta the delta to be applied
* @return the Tl0PicIdx resulting from doing "start + delta"
*/
@JvmStatic
fun applyTl0PicIdxDelta(start: Int, delta: Int): Int = (start + delta) and TL0PICIDX_MASK
}
}

34
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/CertificateInfo.kt Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import java.security.KeyPair
/**
* Store various information about a generated Certificate, including:
* The [KeyPair] used to build and sign it, as well as the fingerprint
* hash function and the calculated fingerprint we use (which is transmitted
* over the signaling channel so the certificate can be verified). We also
* store the time at which this certificate was created so we can refresh it
* appropriately.
*/
data class CertificateInfo(
val keyPair: KeyPair,
val certificate: org.bouncycastle.tls.Certificate,
val localFingerprintHashFunction: String,
val localFingerprint: String,
val creationTimestampMs: Long
)

58
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsClient.kt Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.tls.Certificate
import org.bouncycastle.tls.DTLSClientProtocol
import org.bouncycastle.tls.DTLSTransport
import org.bouncycastle.tls.DatagramTransport
import org.jitsi.nlj.srtp.TlsRole
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.cerror
import org.jitsi.utils.logging2.createChildLogger
class DtlsClient(
private val datagramTransport: DatagramTransport,
certificateInfo: CertificateInfo,
private val handshakeCompleteHandler: (Int, TlsRole, ByteArray) -> Unit = { _, _, _ -> },
verifyAndValidateRemoteCertificate: (Certificate?) -> Unit = {},
parentLogger: Logger,
private val dtlsClientProtocol: DTLSClientProtocol = DTLSClientProtocol()
) : DtlsRole {
private val logger = createChildLogger(parentLogger)
private val tlsClient: TlsClientImpl = TlsClientImpl(certificateInfo, verifyAndValidateRemoteCertificate, logger)
override fun start(): DTLSTransport = connect()
fun connect(): DTLSTransport {
try {
return dtlsClientProtocol.connect(this.tlsClient, datagramTransport).also {
logger.cdebug { "DTLS handshake finished" }
handshakeCompleteHandler(
tlsClient.chosenSrtpProtectionProfile,
TlsRole.CLIENT,
tlsClient.srtpKeyingMaterial
)
}
} catch (e: Exception) {
logger.cerror { "Error during DTLS connection: $e" }
throw e
}
}
}

76
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt Normal file
View File

@@ -0,0 +1,76 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder
import org.bouncycastle.tls.CipherSuite
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.ConfigException
import org.jitsi.metaconfig.config
import java.time.Duration
class DtlsConfig private constructor() {
val handshakeTimeout: Duration by config {
"jmt.dtls.handshake-timeout".from(JitsiConfig.newConfig)
}
val cipherSuites: List<Int> by config {
"jmt.dtls.cipher-suites".from(JitsiConfig.newConfig).convertFrom<List<String>> { list ->
val ciphers = list.map { it.toBcCipherSuite() }
if (ciphers.isEmpty()) {
throw ConfigException.UnableToRetrieve.ConditionNotMet("cipher-suites must not be empty")
}
ciphers
}
}
val localFingerprintHashFunction: String by config {
"jmt.dtls.local-fingerprint-hash-function".from(JitsiConfig.newConfig).transformedBy {
validateHashFunction(it)
}
}
val acceptedFingerprintHashFunctions: List<String> by config {
"jmt.dtls.accepted-fingerprint-hash-functions".from(JitsiConfig.newConfig).convertFrom<List<String>> { list ->
if (list.isEmpty()) {
throw ConfigException.UnableToRetrieve.ConditionNotMet(
"accepted-fingerprint-hash-functions must not be empty"
)
}
list.map { validateHashFunction(it) }
}
}
companion object {
val config = DtlsConfig()
}
}
private fun validateHashFunction(func: String): String {
val ucFunc = func.uppercase()
DefaultDigestAlgorithmIdentifierFinder().find(ucFunc)
?: throw ConfigException.UnableToRetrieve.WrongType("Unknown hash function $func")
if (ucFunc == "MD5" || ucFunc == "MD2") {
throw ConfigException.UnableToRetrieve.WrongType("Forbidden hash function $func")
}
return func.lowercase()
}
private fun String.toBcCipherSuite(): Int = try {
CipherSuite::class.java.getDeclaredField(this).getInt(null)
} catch (e: Exception) {
throw ConfigException.UnableToRetrieve.ConditionNotMet("Value is not a valid BouncyCastle cipher suite name: $this")
}

28
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsProtocolPacket.kt Normal file
View File

@@ -0,0 +1,28 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.jitsi.rtp.Packet
class DtlsProtocolPacket(
buf: ByteArray,
offset: Int,
length: Int
) : Packet(buf, offset, length) {
override fun clone(): DtlsProtocolPacket = DtlsProtocolPacket(cloneBuffer(0), 0, length)
}

32
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsRole.kt Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.tls.DTLSTransport
interface DtlsRole {
/**
* 'Starts' a DTLS connection. Blocks until the connection
* has been successfully established (an exception is thrown
* upon failure). Returns the resulting [DTLSTransport].
*
* The definition of 'start' varies between roles. For a client
* this means 'connect' to a remote server, for a server it means
* 'accept' and wait for an incoming connection.
*/
fun start(): DTLSTransport
}

58
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt Normal file
View File

@@ -0,0 +1,58 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.tls.Certificate
import org.bouncycastle.tls.DTLSServerProtocol
import org.bouncycastle.tls.DTLSTransport
import org.bouncycastle.tls.DatagramTransport
import org.jitsi.nlj.srtp.TlsRole
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.cerror
import org.jitsi.utils.logging2.createChildLogger
class DtlsServer(
private val datagramTransport: DatagramTransport,
certificateInfo: CertificateInfo,
private val handshakeCompleteHandler: (Int, TlsRole, ByteArray) -> Unit = { _, _, _ -> },
verifyAndValidateRemoteCertificate: (Certificate?) -> Unit = {},
parentLogger: Logger,
private val dtlsServerProtocol: DTLSServerProtocol = DTLSServerProtocol()
) : DtlsRole {
private val logger = createChildLogger(parentLogger)
private val tlsServer: TlsServerImpl = TlsServerImpl(certificateInfo, verifyAndValidateRemoteCertificate, logger)
override fun start(): DTLSTransport = accept()
fun accept(): DTLSTransport {
try {
return dtlsServerProtocol.accept(tlsServer, datagramTransport).also {
logger.cdebug { "DTLS handshake finished" }
handshakeCompleteHandler(
tlsServer.chosenSrtpProtectionProfile,
TlsRole.SERVER,
tlsServer.srtpKeyingMaterial
)
}
} catch (t: Throwable) {
logger.cerror { "Error during DTLS connection: $t" }
throw t
}
}
}

355
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt Normal file
View File

@@ -0,0 +1,355 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.bouncycastle.tls.Certificate
import org.bouncycastle.tls.DTLSTransport
import org.bouncycastle.tls.DatagramTransport
import org.jitsi.nlj.srtp.TlsRole
import org.jitsi.nlj.util.BufferPool
import org.jitsi.utils.concurrent.ArrayBlockingQueueWithShutdown
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import java.nio.ByteBuffer
import java.time.Duration
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
import kotlin.math.min
/**
* A DTLS stack implementation, which can act as either the DTLS server or DTLS client, and can be used
* for negotiating the DTLS connection and sending and receiving data over that connection.
*
* This class does not directly communicate with the network in any way. Raw DTLS packets must be fed into
* this stack via the [processIncomingProtocolData] method. [incomingDataHandler] will be invoked with the
* decrypted, application data sent over the connection.
*
* Data can be sent through the DTLS connection via [sendApplicationData]; it will be encrypted by the stack and
* then sent out via the [outgoingDataHandler], which must be set to have the data go anywhere interesting :)
*
* [eventHandler] will be invoked when any events occur.
*
* After wiring up all the handlers, to start a connection the stack must be told which role to fullfil: client
* or server. This can be done by calling either the [actAsClient] or [actAsServer] methods. Once the role has
* been set, [start] can be called to start the negotiation.
*/
class DtlsStack(
parentLogger: Logger
) {
private val logger = createChildLogger(parentLogger)
private val roleSet = CountDownLatch(1)
/**
* The certificate info for this particular [DtlsStack] instance. We save it in a local val because the global one
* might be refreshed.
*/
private val certificateInfo = DtlsStack.certificateInfo
val localFingerprintHashFunction: String
get() = certificateInfo.localFingerprintHashFunction
val localFingerprint: String
get() = certificateInfo.localFingerprint
/**
* The remote fingerprints sent to us over the signaling path.
*/
var remoteFingerprints: Map<String, List<String>> = mapOf()
/**
* A handler which will be invoked when DTLS application data is received
*/
var incomingDataHandler: IncomingDataHandler? = null
/**
* The method [DtlsStack] will invoke when it wants to send DTLS data out onto the network.
*/
var outgoingDataHandler: OutgoingDataHandler? = null
/**
* Handle to be invoked when events occur
*/
var eventHandler: EventHandler? = null
private val incomingProtocolData = ArrayBlockingQueueWithShutdown<ByteBuffer>(QUEUE_SIZE, true)
private var numPacketDropsQueueFull = 0
/**
* The [DtlsRole] 'plugin' that will determine how this stack operates (as a client
* or a server). A call to [actAsClient] or [actAsServer] must be made to fill out
* this role and successfully call [start]
*/
var role: DtlsRole? = null
private set
/**
* A buffer we'll use to receive data from [dtlsTransport].
*/
private val dtlsAppDataBuf = ByteArray(1500)
/**
* The negotiated DTLS transport. This is used to read and write DTLS application data.
*/
private var dtlsTransport: DTLSTransport? = null
/**
* The [DatagramTransport] implementation we use for this stack.
*/
private val datagramTransport: DatagramTransport = DatagramTransportImpl(logger)
/**
* Set the role to server, unless it already is one.
*
* The role is re-applied whenever the peer signals its `setup` attribute again, which it does on every
* transport update and not only when something about DTLS changed. Building a new [DtlsRole] each time would
* replace the one that has already negotiated the connection with one that never will, so repeating the role
* we already have is a no-op.
*
* @return true if the role changed.
*/
fun actAsServer(): Boolean {
if (role is DtlsServer) {
return false
}
warnIfRoleChangesAfterStart("server")
role = DtlsServer(
datagramTransport,
certificateInfo,
{ chosenSrtpProfile, tlsRole, keyingMaterial ->
eventHandler?.handshakeComplete(chosenSrtpProfile, tlsRole, keyingMaterial)
},
this::verifyAndValidateRemoteCertificate,
logger
)
roleSet.countDown()
return true
}
/** Set the role to client, unless it already is one. See [actAsServer]. */
fun actAsClient(): Boolean {
if (role is DtlsClient) {
return false
}
warnIfRoleChangesAfterStart("client")
role = DtlsClient(
datagramTransport,
certificateInfo,
{ chosenSrtpProfile, tlsRole, keyingMaterial ->
eventHandler?.handshakeComplete(chosenSrtpProfile, tlsRole, keyingMaterial)
},
this::verifyAndValidateRemoteCertificate,
logger
)
roleSet.countDown()
return true
}
/**
* The stack has been started with the role it has now, so replacing that role leaves it with one that was
* never started while [dtlsTransport] keeps running the old one. Nothing in the stack does this today, and
* the peer is not supposed to change its `setup` after the handshake, so log loudly if it happens.
*/
private fun warnIfRoleChangesAfterStart(newRole: String) {
if (dtlsTransport != null) {
logger.warn("Changing the DTLS role to $newRole after the stack was started with ${role?.javaClass}.")
}
}
/**
* 'start' this stack, in whatever role it has been told to operate (client or server). If a role
* has not yet been yet (via [actAsServer] or [actAsClient]), then it will block until the role
* has been set.
*/
fun start() {
roleSet.await()
dtlsTransport = role?.start()
// There is a bit of a race here: It's technically possible the
// far side could finish the handshake and send a message before
// this side assigns dtlsTransport here. If so, that message
// would be passed to #processIncomingProtocolData and put in
// incomingProtocolData, but, since dtlsTransport won't be set
// yet, we won't 'receive' it yet. Check for any incoming packets
// here, to handle this case.
processIncomingProtocolData()
}
fun close() {
datagramTransport.close()
incomingProtocolData.shutdown()
incomingProtocolData.forEach {
BufferPool.returnBuffer(it.array())
}
incomingProtocolData.clear()
}
/**
* Checks that a specific [Certificate] matches the remote fingerprints sent to us over the signaling path.
*/
private fun verifyAndValidateRemoteCertificate(remoteCertificate: Certificate?) {
remoteCertificate?.let {
DtlsUtils.verifyAndValidateCertificate(it, remoteFingerprints)
// The above throws an exception if the checks fail.
logger.cdebug { "Fingerprints verified." }
} ?: run {
throw DtlsUtils.DtlsException("Remote certificate was null")
}
}
fun sendApplicationData(data: ByteArray, off: Int, len: Int) {
dtlsTransport?.send(data, off, len)
}
private fun processIncomingProtocolData() {
var bytesReceived: Int
do {
val bufCopy2: ByteArray? = synchronized(dtlsAppDataBuf) {
bytesReceived = dtlsTransport?.receive(dtlsAppDataBuf, 0, 1500, 1) ?: -1
if (bytesReceived > 0) {
// Copy again to copy out of dtlsAppDataBuf, which we re-use.
BufferPool.getBuffer(bytesReceived).apply {
System.arraycopy(dtlsAppDataBuf, 0, this, 0, bytesReceived)
}
} else {
null
}
}
if (bufCopy2 != null) {
incomingDataHandler?.dataReceived(bufCopy2, 0, bytesReceived)
}
} while (bytesReceived > 0)
}
/**
* We get 'pushed' the data from a lower transport layer, but bouncycastle wants to 'pull' the data
* itself. To mimic this, we put the received data into a queue, and then 'pull' it through ourselves by
* calling 'receive' on the negotiated [DTLSTransport].
*
* Note: the data we get here may be a DTLS protocol packet and therefore won't generate anything
* to be received by [dtlsTransport] or a DTLS app packet (data sent over DTLS after the handshake has
* completed) and will result in data being received through [dtlsTransport]. It's possible, though,
* that the handshake has finished and the far end has sent application data but we've not yet set
* [dtlsTransport], so we "miss" it. We don't lose this data, but it will sit inside of [incomingProtocolData]
* until the next packet comes through. This means that we must make a copy of the buffer we receive, as we
* won't necessarily be done with it by the time this method completes.
*/
fun processIncomingProtocolData(data: ByteArray, off: Int, len: Int) {
val bufCopy = BufferPool.getBuffer(len).apply {
System.arraycopy(data, off, this, 0, len)
}
if (!incomingProtocolData.offer(ByteBuffer.wrap(bufCopy, 0, len))) {
BufferPool.returnBuffer(bufCopy)
if (!incomingProtocolData.isShutdown) {
logger.warn("DTLS stack queue full, dropping packet")
numPacketDropsQueueFull++
}
}
processIncomingProtocolData()
}
fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("localFingerprintHashFunction", certificateInfo.localFingerprint)
put("remoteFingerprints", remoteFingerprints.map { (hash, fp) -> "$hash: $fp" }.joinToString())
put("role", (role?.javaClass ?: "null").toString())
put("num_packet_drops_queue_full", numPacketDropsQueueFull)
}
companion object {
private const val QUEUE_SIZE = 50
/**
* Because generating the certificateInfo can be expensive, we generate a single
* one to be used everywhere which expires in 24 hours (when we'll generate
* another one).
*/
private val syncRoot: Any = Any()
private var certificateInfo: CertificateInfo = DtlsUtils.generateCertificateInfo()
get() = synchronized(syncRoot) {
val expirationPeriodMs = Duration.ofDays(1).toMillis()
if (field.creationTimestampMs + expirationPeriodMs < System.currentTimeMillis()) {
// TODO: avoid creating our own thread
thread { field = DtlsUtils.generateCertificateInfo() }
}
return field
}
}
/**
* An implementation of [DatagramTransport] which 'receives' from the queue in [DtlsStack] and sends out
* via [DtlsStack]'s [outgoingDataHandler]
*/
inner class DatagramTransportImpl(parentLogger: Logger) : DatagramTransport {
private val logger = createChildLogger(parentLogger)
override fun receive(buf: ByteArray, off: Int, len: Int, waitMillis: Int): Int {
val data = try {
incomingProtocolData.poll(waitMillis.toLong(), TimeUnit.MILLISECONDS) ?: return -1
} catch (ie: InterruptedException) {
return -1
}
val length = min(len, data.limit())
if (length < data.limit()) {
logger.warn(
"Passed buffer size ($len) was too small to hold incoming data size (${data.limit()}); " +
"data was truncated"
)
}
System.arraycopy(data.array(), data.arrayOffset(), buf, off, length)
BufferPool.returnBuffer(data.array())
return length
}
override fun send(buf: ByteArray, off: Int, len: Int) {
outgoingDataHandler?.sendData(buf, off, len)
}
/**
* Receive limit computation copied from [org.bouncycastle.tls.UDPTransport]
*/
override fun getReceiveLimit(): Int = 1500 - 20 - 8
/**
* Send limit computation copied from [org.bouncycastle.tls.UDPTransport]
*/
override fun getSendLimit(): Int = 1500 - 84 - 8
override fun close() {}
}
interface IncomingDataHandler {
/**
* Notify the handler that data has been received. The handler takes ownership of the passed
* buffer, and it should be returned to the buffer pool when done with it.
*/
fun dataReceived(data: ByteArray, off: Int, len: Int)
}
interface OutgoingDataHandler {
fun sendData(data: ByteArray, off: Int, len: Int)
}
interface EventHandler {
fun handshakeComplete(chosenSrtpProtectionProfile: Int, tlsRole: TlsRole, keyingMaterial: ByteArray)
}
}

323
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt Normal file
View File

@@ -0,0 +1,323 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.asn1.ASN1Encoding
import org.bouncycastle.asn1.x500.X500Name
import org.bouncycastle.asn1.x500.X500NameBuilder
import org.bouncycastle.asn1.x500.style.BCStyle
import org.bouncycastle.asn1.x509.Certificate
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder
import org.bouncycastle.jce.ECNamedCurveTable
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder
import org.bouncycastle.operator.bc.BcDefaultDigestProvider
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder
import org.bouncycastle.tls.AlertDescription
import org.bouncycastle.tls.TlsContext
import org.bouncycastle.tls.TlsUtils
import org.bouncycastle.tls.crypto.TlsSecret
import org.bouncycastle.tls.crypto.impl.bc.BcTlsCertificate
import org.bouncycastle.tls.crypto.impl.bc.BcTlsCrypto
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.cerror
import org.jitsi.utils.logging2.cinfo
import java.math.BigInteger
import java.security.KeyPair
import java.security.KeyPairGenerator
import java.security.SecureRandom
import java.security.Security
import java.time.Duration
import java.util.Date
import java.util.NoSuchElementException
private val SECURE_RANDOM = SecureRandom()
val BC_TLS_CRYPTO = BcTlsCrypto(SECURE_RANDOM)
/**
* Various helper utilities for DTLS
*
* https://tools.ietf.org/html/draft-ietf-rtcweb-security-arch-18
*/
class DtlsUtils {
companion object {
init {
Security.addProvider(BouncyCastleProvider())
}
fun generateCertificateInfo(): CertificateInfo {
val cn = generateCN("TODO-APP-NAME", "TODO-APP-VERSION")
val keyPair = generateEcKeyPair()
val x509certificate = generateCertificate(cn, keyPair)
val localFingerprintHashFunction = DtlsConfig.config.localFingerprintHashFunction
val localFingerprint = x509certificate.getFingerprint(localFingerprintHashFunction)
val certificate = org.bouncycastle.tls.Certificate(
arrayOf(BcTlsCertificate(BC_TLS_CRYPTO, x509certificate))
)
return CertificateInfo(
keyPair,
certificate,
localFingerprintHashFunction,
localFingerprint,
System.currentTimeMillis()
)
}
/**
* A helper which finds an SRTP protection profile present in both
* [ours] and [theirs]. Throws [DtlsException] if no common profile is found.
*/
fun chooseSrtpProtectionProfile(ours: Iterable<Int>, theirs: Iterable<Int>): Int = try {
ours.first(theirs::contains)
} catch (e: NoSuchElementException) {
throw DtlsException(
"No common SRTP protection profile found. Ours: ${ours.joinToString()} " +
"Theirs: ${theirs.joinToString()}"
)
}
/**
* Generate an x509 certificate valid from 1 day ago until 7 days from now.
*
* TODO: make the algorithm dynamic (passed in) to support older dtls versions/clients
*/
private fun generateCertificate(subject: X500Name, keyPair: KeyPair): Certificate {
val now = System.currentTimeMillis()
val startDate = Date(now - Duration.ofDays(1).toMillis())
val expiryDate = Date(now + Duration.ofDays(7).toMillis())
val serialNumber = BigInteger.valueOf(now)
val certBuilder = JcaX509v3CertificateBuilder(
subject,
serialNumber,
startDate,
expiryDate,
subject,
keyPair.public
)
val signer = JcaContentSignerBuilder("SHA256withECDSA").build(keyPair.private)
return certBuilder.build(signer).toASN1Structure()
}
/**
* Generate an eliptic-curve keypair using the secp256r1 named curve:
* "All Implementations MUST implement DTLS 1.2 with the
* TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 cipher suite and the P-256
* curve"
*
* --https://tools.ietf.org/html/draft-ietf-rtcweb-security-arch-18#section-6.5
*
* NOTE(brian): I used 'secp256r1' specifically because it's what I saw in wireshark traces from chrome
*/
private fun generateEcKeyPair(): KeyPair {
val keyGen = KeyPairGenerator.getInstance("EC", "BC")
val ecCurveSpec = ECNamedCurveTable.getParameterSpec("secp256r1")
keyGen.initialize(ecCurveSpec)
return keyGen.generateKeyPair()
}
/**
* Generate an [X500Name] using the given [appName] and [appVersion]
*/
private fun generateCN(appName: String, appVersion: String): X500Name {
val builder = X500NameBuilder(BCStyle.INSTANCE)
val rdn = "$appName $appVersion"
builder.addRDN(BCStyle.CN, rdn)
return builder.build()
}
/**
* Verifies and validates a specific certificate against the fingerprints
* presented by the remote endpoint via the signaling path.
*
* @param certificateInfo the certificate to be verified and validated against
* the fingerprints presented by the remote endpoint via the signaling path
* @throws [DtlsException] if [certificateInfo] fails validation
*/
fun verifyAndValidateCertificate(
certificateInfo: org.bouncycastle.tls.Certificate,
remoteFingerprints: Map<String, List<String>>
) {
if (certificateInfo.certificateList.isEmpty()) {
throw DtlsException("No remote fingerprints.")
}
for (currCertificate in certificateInfo.certificateList) {
val x509Cert = Certificate.getInstance(currCertificate.encoded)
verifyAndValidateCertificate(x509Cert, remoteFingerprints)
}
}
/**
* Verifies and validates a specific certificate against the fingerprints
* presented by the remote endpoint via the signaling path.
*
* @param certificate the certificate to be verified and validated against
* the fingerprints presented by the remote endpoint via the signaling path.
* @throws DtlsException if the specified [certificate] failed to verify
* and validate against the fingerprints presented by the remote endpoint
* via the signaling path.
*/
private fun verifyAndValidateCertificate(
certificate: Certificate,
remoteFingerprints: Map<String, List<String>>
) {
/* RFC 8122:
* An endpoint MUST select the set of fingerprints that use its most
* preferred hash function (out of those offered by the peer) and verify
* that each certificate used matches one fingerprint out of that set.
*/
DtlsConfig.config.acceptedFingerprintHashFunctions.forEach { hashFunction ->
val fingerprints = remoteFingerprints[hashFunction] ?: return@forEach
val certificateFingerprint = certificate.getFingerprint(hashFunction)
if (fingerprints.none { it == certificateFingerprint }) {
throw DtlsException(
"None of the fingerprints ${fingerprints.joinToString()} match the $hashFunction-hashed " +
"certificate $certificateFingerprint"
)
}
return@verifyAndValidateCertificate
}
/* If we got here none of our accepted fingerprint functions were listed. */
throw DtlsException(
"No fingerprint declared over the signaling path with any of the accepted hash functions: " +
DtlsConfig.config.acceptedFingerprintHashFunctions.joinToString()
)
}
/**
* Computes the fingerprint of a [org.bouncycastle.asn1.x509.Certificate] using [hashFunction] and returns it
* as a [String]
*/
private fun Certificate.getFingerprint(hashFunction: String): String {
val digAlgId = DefaultDigestAlgorithmIdentifierFinder().find(hashFunction.uppercase())
val digest = BcDefaultDigestProvider.INSTANCE.get(digAlgId)
val input: ByteArray = getEncoded(ASN1Encoding.DER)
val output = ByteArray(digest.digestSize)
digest.update(input, 0, input.size)
digest.doFinal(output, 0)
return output.toFingerprint()
}
private val HEX_CHARS = "0123456789ABCDEF".toCharArray()
/**
* Helper function to convert a [ByteArray] to a colon-delimited hex string
*/
private fun ByteArray.toFingerprint(): String {
val buf = StringBuffer()
for (i in 0 until size) {
val octet = get(i).toInt()
val firstIndex = (octet and 0xF0).ushr(4)
val secondIndex = octet and 0x0F
buf.append(HEX_CHARS[firstIndex])
buf.append(HEX_CHARS[secondIndex])
if (i < size - 1) {
buf.append(":")
}
}
return buf.toString()
}
/*
* Copied from TlsContext#exportKeyingMaterial and modified to work with
* an externally provided masterSecret value.
*/
fun exportKeyingMaterial(
context: TlsContext,
asciiLabel: String,
context_value: ByteArray?,
length: Int,
masterSecret: TlsSecret
): ByteArray {
if (context_value != null && !TlsUtils.isValidUint16(context_value.size)) {
throw IllegalArgumentException("'context_value' must have a length less than 2^16 (or be null)")
}
val sp = context.securityParameters
val cr = sp.clientRandom
val sr = sp.serverRandom
var seedLength = cr.size + sr.size
if (context_value != null) {
seedLength += (2 + context_value.size)
}
val seed = ByteArray(seedLength)
var seedPos = 0
System.arraycopy(cr, 0, seed, seedPos, cr.size)
seedPos += cr.size
System.arraycopy(sr, 0, seed, seedPos, sr.size)
seedPos += sr.size
if (context_value != null) {
TlsUtils.writeUint16(context_value.size, seed, seedPos)
seedPos += 2
System.arraycopy(context_value, 0, seed, seedPos, context_value.size)
seedPos += context_value.size
}
if (seedPos != seedLength) {
throw IllegalStateException("error in calculation of seed for export")
}
return TlsUtils.PRF(sp, masterSecret, asciiLabel, seed, length).extract()
}
}
class DtlsException(msg: String) : Exception(msg)
}
@Suppress("NOTHING_TO_INLINE") // Avoid adding to the trace in the log file
inline fun Logger.notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) {
when (alertDescription) {
AlertDescription.close_notify -> cdebug { "close_notify raised, connection closing" }
else -> {
val stack = with(StringBuffer()) {
val e = Exception()
for (el in e.stackTrace) {
appendLine(el.toString())
}
toString()
}
cinfo {
"Alert raised: level=$alertLevel, description=$alertDescription, message=$message " +
"cause=$cause $stack"
}
}
}
}
@Suppress("NOTHING_TO_INLINE") // Avoid adding to the trace in the log file
inline fun Logger.notifyAlertReceived(alertLevel: Short, alertDescription: Short) {
when (alertDescription) {
AlertDescription.close_notify -> cinfo { "close_notify received, connection closing" }
else -> cerror {
"Alert received: level=$alertLevel, description=$alertDescription " +
"(${AlertDescription.getName(alertDescription)})"
}
}
}

170
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt Normal file
View File

@@ -0,0 +1,170 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.crypto.util.PrivateKeyFactory
import org.bouncycastle.tls.Certificate
import org.bouncycastle.tls.CertificateRequest
import org.bouncycastle.tls.DefaultTlsClient
import org.bouncycastle.tls.ExporterLabel
import org.bouncycastle.tls.ExtensionType
import org.bouncycastle.tls.HashAlgorithm
import org.bouncycastle.tls.ProtocolVersion
import org.bouncycastle.tls.SignatureAlgorithm
import org.bouncycastle.tls.SignatureAndHashAlgorithm
import org.bouncycastle.tls.TlsAuthentication
import org.bouncycastle.tls.TlsCredentials
import org.bouncycastle.tls.TlsSRTPUtils
import org.bouncycastle.tls.TlsServerCertificate
import org.bouncycastle.tls.TlsSession
import org.bouncycastle.tls.TlsUtils
import org.bouncycastle.tls.UseSRTPData
import org.bouncycastle.tls.crypto.TlsCryptoParameters
import org.bouncycastle.tls.crypto.impl.bc.BcDefaultTlsCredentialedSigner
import org.bouncycastle.tls.crypto.impl.bc.BcTlsCrypto
import org.jitsi.nlj.srtp.SrtpConfig
import org.jitsi.nlj.srtp.SrtpUtil
import org.jitsi.rtp.extensions.toHex
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.cinfo
import org.jitsi.utils.logging2.createChildLogger
import java.nio.ByteBuffer
import java.util.Hashtable
/**
* Implementation of [DefaultTlsClient].
*/
class TlsClientImpl(
private val certificateInfo: CertificateInfo,
/**
* The function to call when the server certificateInfo is available.
*/
private val notifyServerCertificate: (Certificate?) -> Unit,
parentLogger: Logger
) : DefaultTlsClient(BC_TLS_CRYPTO) {
private val logger = createChildLogger(parentLogger)
private var session: TlsSession? = null
private var clientCredentials: TlsCredentials? = null
/**
* Only set after a handshake has completed
*/
lateinit var srtpKeyingMaterial: ByteArray
private set
var chosenSrtpProtectionProfile: Int = 0
override fun getSessionToResume(): TlsSession? = session
override fun getAuthentication(): TlsAuthentication {
return object : TlsAuthentication {
override fun getClientCredentials(certificateRequest: CertificateRequest): TlsCredentials {
// NOTE: can't set clientCredentials when it is declared because 'context' won't be set yet
if (clientCredentials == null) {
clientCredentials = BcDefaultTlsCredentialedSigner(
TlsCryptoParameters(context),
(context.crypto as BcTlsCrypto),
PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded),
certificateInfo.certificate,
if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) {
SignatureAndHashAlgorithm(
HashAlgorithm.sha256,
SignatureAlgorithm.ecdsa
)
} else {
null
}
)
}
return clientCredentials!!
}
override fun notifyServerCertificate(serverCertificate: TlsServerCertificate) {
this@TlsClientImpl.notifyServerCertificate(serverCertificate.certificate)
}
}
}
override fun getClientExtensions(): Hashtable<*, *> {
var clientExtensions = super.getClientExtensions()
if (TlsSRTPUtils.getUseSRTPExtension(clientExtensions) == null) {
if (clientExtensions == null) {
clientExtensions = Hashtable<Int, ByteArray>()
}
TlsSRTPUtils.addUseSRTPExtension(
clientExtensions,
UseSRTPData(SrtpConfig.protectionProfiles.toIntArray(), TlsUtils.EMPTY_BYTES)
)
}
clientExtensions.put(ExtensionType.renegotiation_info, byteArrayOf(0))
return clientExtensions
}
override fun processServerExtensions(serverExtensions: Hashtable<*, *>?) {
// TODO: a few cases we should be throwing alerts for in here. see old TlsClientImpl
val useSRTPData = TlsSRTPUtils.getUseSRTPExtension(serverExtensions)
val protectionProfiles = useSRTPData.protectionProfiles
chosenSrtpProtectionProfile =
DtlsUtils.chooseSrtpProtectionProfile(SrtpConfig.protectionProfiles, protectionProfiles.asIterable())
}
override fun getCipherSuites() = DtlsConfig.config.cipherSuites.toIntArray()
override fun getHandshakeTimeoutMillis(): Int = DtlsConfig.config.handshakeTimeout.toMillis().toInt()
override fun notifyHandshakeComplete() {
super.notifyHandshakeComplete()
logger.cinfo { "Negotiated DTLS version ${context.securityParameters.negotiatedVersion}" }
context.resumableSession?.let { newSession ->
session?.let { existingSession ->
if (existingSession.sessionID?.contentEquals(newSession.sessionID) == true) {
logger.cdebug {
val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex()
"Resumed DTLS session $newSessionIdHex"
}
}
} ?: run {
logger.cdebug {
val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex()
"Established DTLS session $newSessionIdHex"
}
this.session = newSession
}
}
val srtpProfileInformation =
SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile)
srtpKeyingMaterial = context.exportKeyingMaterial(
ExporterLabel.dtls_srtp,
null,
2 * (srtpProfileInformation.cipherKeyLength + srtpProfileInformation.cipherSaltLength)
)
}
override fun getSupportedVersions(): Array<ProtocolVersion> = arrayOf(ProtocolVersion.DTLSv12)
override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) =
logger.notifyAlertRaised(alertLevel, alertDescription, message, cause)
override fun notifyAlertReceived(alertLevel: Short, alertDescription: Short) =
logger.notifyAlertReceived(alertLevel, alertDescription)
}

168
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt Normal file
View File

@@ -0,0 +1,168 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.dtls
import org.bouncycastle.crypto.util.PrivateKeyFactory
import org.bouncycastle.tls.Certificate
import org.bouncycastle.tls.CertificateRequest
import org.bouncycastle.tls.ClientCertificateType
import org.bouncycastle.tls.DefaultTlsServer
import org.bouncycastle.tls.ExporterLabel
import org.bouncycastle.tls.HashAlgorithm
import org.bouncycastle.tls.ProtocolVersion
import org.bouncycastle.tls.SignatureAlgorithm
import org.bouncycastle.tls.SignatureAndHashAlgorithm
import org.bouncycastle.tls.TlsCredentialedDecryptor
import org.bouncycastle.tls.TlsCredentialedSigner
import org.bouncycastle.tls.TlsSRTPUtils
import org.bouncycastle.tls.TlsSession
import org.bouncycastle.tls.TlsUtils
import org.bouncycastle.tls.UseSRTPData
import org.bouncycastle.tls.crypto.TlsCryptoParameters
import org.bouncycastle.tls.crypto.impl.bc.BcDefaultTlsCredentialedDecryptor
import org.bouncycastle.tls.crypto.impl.bc.BcDefaultTlsCredentialedSigner
import org.bouncycastle.tls.crypto.impl.bc.BcTlsCrypto
import org.jitsi.nlj.srtp.SrtpConfig
import org.jitsi.nlj.srtp.SrtpUtil
import org.jitsi.rtp.extensions.toHex
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cinfo
import org.jitsi.utils.logging2.createChildLogger
import java.nio.ByteBuffer
import java.util.Hashtable
import java.util.Vector
class TlsServerImpl(
private val certificateInfo: CertificateInfo,
/**
* The function to call when the client certificateInfo is available.
*/
private val notifyClientCertificateReceived: (Certificate?) -> Unit,
parentLogger: Logger
) : DefaultTlsServer(BC_TLS_CRYPTO) {
private val logger = createChildLogger(parentLogger)
private var session: TlsSession? = null
/**
* Only set after a handshake has completed
*/
lateinit var srtpKeyingMaterial: ByteArray
private set
var chosenSrtpProtectionProfile: Int = 0
override fun getSessionToResume(sessionID: ByteArray?): TlsSession? {
return session
// TODO: do we need to map multiple sessions (per sessionID?)
// return super.getSessionToResume(sessionID)
}
override fun getServerExtensions(): Hashtable<*, *> {
val extensions = super.getServerExtensions()
?: Hashtable<Int, ByteArray>()
return extensions.also {
if (TlsSRTPUtils.getUseSRTPExtension(it) == null) {
TlsSRTPUtils.addUseSRTPExtension(
it,
UseSRTPData(intArrayOf(chosenSrtpProtectionProfile), TlsUtils.EMPTY_BYTES)
)
}
}
}
override fun processClientExtensions(clientExtensions: Hashtable<*, *>?) {
super.processClientExtensions(clientExtensions)
val useSRTPData = TlsSRTPUtils.getUseSRTPExtension(clientExtensions)
val protectionProfiles = useSRTPData.protectionProfiles
chosenSrtpProtectionProfile =
DtlsUtils.chooseSrtpProtectionProfile(SrtpConfig.protectionProfiles, protectionProfiles.asIterable())
}
override fun getCipherSuites() = DtlsConfig.config.cipherSuites.toIntArray()
override fun getRSAEncryptionCredentials(): TlsCredentialedDecryptor = BcDefaultTlsCredentialedDecryptor(
(context.crypto as BcTlsCrypto),
certificateInfo.certificate,
PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded)
)
override fun getECDSASignerCredentials(): TlsCredentialedSigner = BcDefaultTlsCredentialedSigner(
TlsCryptoParameters(context),
(context.crypto as BcTlsCrypto),
PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded),
certificateInfo.certificate,
SignatureAndHashAlgorithm(HashAlgorithm.sha256, SignatureAlgorithm.ecdsa)
)
override fun getCertificateRequest(): CertificateRequest {
val signatureAlgorithms = Vector<SignatureAndHashAlgorithm>(1)
signatureAlgorithms.add(SignatureAndHashAlgorithm(HashAlgorithm.sha256, SignatureAlgorithm.ecdsa))
return CertificateRequest(shortArrayOf(ClientCertificateType.ecdsa_sign), signatureAlgorithms, null)
}
override fun getHandshakeTimeoutMillis(): Int = DtlsConfig.config.handshakeTimeout.toMillis().toInt()
override fun notifyHandshakeComplete() {
super.notifyHandshakeComplete()
logger.cinfo { "Negotiated DTLS version ${context.securityParameters.negotiatedVersion}" }
context.resumableSession?.let { newSession ->
val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex()
session?.let { existingSession ->
if (existingSession.sessionID?.contentEquals(newSession.sessionID) == true) {
logger.cinfo { "Resumed DTLS session $newSessionIdHex" }
}
} ?: run {
logger.cinfo { "Established DTLS session $newSessionIdHex" }
this.session = newSession
}
}
val srtpProfileInformation =
SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile)
if (!context.securityParameters.isExtendedMasterSecret) {
context.session?.exportSessionParameters()?.masterSecret?.let {
srtpKeyingMaterial = DtlsUtils.exportKeyingMaterial(
context,
ExporterLabel.dtls_srtp,
null,
2 * (srtpProfileInformation.cipherKeyLength + srtpProfileInformation.cipherSaltLength),
it
)
}
} else {
srtpKeyingMaterial = context.exportKeyingMaterial(
ExporterLabel.dtls_srtp,
null,
2 * (srtpProfileInformation.cipherKeyLength + srtpProfileInformation.cipherSaltLength)
)
}
}
override fun notifyClientCertificate(clientCertificate: Certificate?) {
notifyClientCertificateReceived(clientCertificate)
}
override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) =
logger.notifyAlertRaised(alertLevel, alertDescription, message, cause)
override fun notifyAlertReceived(alertLevel: Short, alertDescription: Short) =
logger.notifyAlertReceived(alertLevel, alertDescription)
override fun getSupportedVersions(): Array<ProtocolVersion> = arrayOf(ProtocolVersion.DTLSv12)
}

218
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt Normal file
View File

@@ -0,0 +1,218 @@
/*
* Copyright @ 2019-Present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.format
import org.jitsi.utils.MediaType
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArraySet
typealias PayloadTypeParams = Map<String, String>
typealias RtcpFeedbackSet = Set<String>
fun RtcpFeedbackSet.supportsPli(): Boolean = this.contains("nack pli")
fun RtcpFeedbackSet.supportsFir(): Boolean = this.contains("ccm fir")
fun RtcpFeedbackSet.supportsRemb(): Boolean = this.contains("goog-remb")
fun RtcpFeedbackSet.supportsTcc(): Boolean = this.contains("transport-cc")
/**
* Represents an RTP payload type.
*
* @author Boris Grozev
* @author Brian Baldino
*/
abstract class PayloadType(
/**
* The 7-bit RTP payload type number.
*/
val pt: Byte,
/**
* The encoding name.
*/
val encoding: PayloadTypeEncoding,
/**
* The media type (audio or video).
*/
val mediaType: MediaType,
/**
* The RTP clock rate.
*/
val clockRate: Int,
/**
* Additional parameters associated with the payload type (e.g. the "apt" used for RTX).
*/
val parameters: PayloadTypeParams = ConcurrentHashMap(),
/**
* The rtcp feedback messages associated with the payload type (e.g. nack, nack pli, transport-cc, goog-remb, ccm fir, etc).
*/
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) {
val rtcpFeedbackSet = CopyOnWriteArraySet(rtcpFeedbackSet)
override fun toString(): String = with(StringBuffer()) {
append(pt).append(" -> ").append(encodingName())
.append(" (").append(clockRate)
.append(channelsString())
.append("): ").append(parameters)
toString()
}
open fun encodingName() = encoding.name.lowercase()
open fun channelsString() = ""
}
enum class PayloadTypeEncoding {
OTHER,
VP8,
VP9,
AV1,
H264,
RED,
RTX,
OPUS,
TELEPHONE_EVENT;
companion object {
private const val TEL_EVENT_TEXT = "telephone-event"
/**
* [valueOf] does not allow for case-insensitivity and can't be overridden, so this
* method should be used when creating an instance of this enum from a string
*/
fun createFrom(value: String): PayloadTypeEncoding {
return try {
if (value.lowercase() == TEL_EVENT_TEXT) {
return TELEPHONE_EVENT
}
valueOf(value.uppercase())
} catch (e: IllegalArgumentException) {
return OTHER
}
}
}
override fun toString(): String = with(StringBuffer()) {
if (super.equals(TELEPHONE_EVENT)) {
append(TEL_EVENT_TEXT)
} else {
append(super.toString())
}
toString()
}
}
abstract class VideoPayloadType(
pt: Byte,
encoding: PayloadTypeEncoding,
clockRate: Int = 90000,
parameters: PayloadTypeParams = ConcurrentHashMap(),
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) : PayloadType(pt, encoding, MediaType.VIDEO, clockRate, parameters, rtcpFeedbackSet)
class Vp8PayloadType(
pt: Byte,
parameters: PayloadTypeParams = ConcurrentHashMap(),
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) : VideoPayloadType(pt, PayloadTypeEncoding.VP8, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet)
class Vp9PayloadType(
pt: Byte,
parameters: PayloadTypeParams = ConcurrentHashMap(),
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) : VideoPayloadType(pt, PayloadTypeEncoding.VP9, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet)
class Av1PayloadType(
pt: Byte,
parameters: PayloadTypeParams = ConcurrentHashMap(),
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) : VideoPayloadType(pt, PayloadTypeEncoding.AV1, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet)
class H264PayloadType(
pt: Byte,
parameters: PayloadTypeParams = ConcurrentHashMap(),
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) : VideoPayloadType(pt, PayloadTypeEncoding.H264, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet)
class RtxPayloadType(
pt: Byte,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : VideoPayloadType(pt, PayloadTypeEncoding.RTX, parameters = parameters) {
val associatedPayloadType: Int =
parameters["apt"]?.toInt() ?: error("RtxPayloadType must contain 'apt'")
}
abstract class AudioPayloadType(
pt: Byte,
encoding: PayloadTypeEncoding,
clockRate: Int = 48000,
/**
* The number of channels
*/
val channels: Int = 1,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : PayloadType(pt, encoding, MediaType.AUDIO, clockRate, parameters) {
override fun channelsString(): String = if (channels > 1) "/$channels" else ""
}
class OpusPayloadType(
pt: Byte,
clockRate: Int = 48000,
channels: Int = 2,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : AudioPayloadType(pt, PayloadTypeEncoding.OPUS, clockRate, channels, parameters = parameters)
class TelephoneEventPayloadType(
pt: Byte,
clockRate: Int,
channels: Int = 1,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : AudioPayloadType(pt, PayloadTypeEncoding.TELEPHONE_EVENT, clockRate, channels, parameters)
class AudioRedPayloadType(
pt: Byte,
clockRate: Int = 48000,
channels: Int = 1,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : AudioPayloadType(pt, PayloadTypeEncoding.RED, clockRate, channels, parameters)
class VideoRedPayloadType(
pt: Byte,
clockRate: Int = 90000,
parameters: PayloadTypeParams = ConcurrentHashMap(),
rtcpFeedbackSet: RtcpFeedbackSet = emptySet()
) : VideoPayloadType(pt, PayloadTypeEncoding.RED, clockRate, parameters, rtcpFeedbackSet)
class OtherAudioPayloadType(
pt: Byte,
private val encodingName: String,
clockRate: Int,
channels: Int = 1,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : AudioPayloadType(pt, PayloadTypeEncoding.OTHER, clockRate, channels, parameters) {
override fun encodingName() = encodingName.lowercase()
}
class OtherVideoPayloadType(
pt: Byte,
private val encodingName: String,
clockRate: Int,
parameters: PayloadTypeParams = ConcurrentHashMap()
) : VideoPayloadType(pt, PayloadTypeEncoding.OTHER, clockRate, parameters) {
override fun encodingName() = encodingName.lowercase()
}

310
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt Normal file
View File

@@ -0,0 +1,310 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.nlj.Event
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.SetLocalSsrcEvent
import org.jitsi.nlj.stats.NodeStatsBlock
import org.jitsi.nlj.transform.node.TransformerNode
import org.jitsi.nlj.util.ReadOnlyStreamInformationStore
import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket
import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket
import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacketBuilder
import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket
import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacketBuilder
import org.jitsi.utils.MediaType
import org.jitsi.utils.RateLimit
import org.jitsi.utils.durationOfDoubleSeconds
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.min
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.concurrent.atomic.AtomicInteger
/**
* [KeyframeRequester] handles a few things around keyframes:
* 1) The bridge requesting a keyframe (e.g. in order to switch) via the [KeyframeRequester#requestKeyframe]
* method which will create a new keyframe request and forward it
* 2) PLI/FIR translation. If a PLI or FIR packet is forwarded through here, this class may translate it depending
* on what the client supports
* 3) Aggregation. This class will pace outgoing requests such that we don't spam the sender
*/
class KeyframeRequester @JvmOverloads constructor(
private val streamInformationStore: ReadOnlyStreamInformationStore,
parentLogger: Logger,
private val clock: Clock = Clock.systemDefaultZone()
) : TransformerNode("Keyframe Requester") {
private val logger = createChildLogger(parentLogger)
// Map the tuple of requester and SSRC to a rate limiter
private val perReceiverKeyframeLimiter = mutableMapOf<String, MutableMap<Long, RateLimit>>()
private val perSourceKeyframeLimiter = mutableMapOf<Long, RateLimit>()
private val keyframeLimiterSyncRoot = Any()
private val firCommandSequenceNumber: AtomicInteger = AtomicInteger(0)
private var localSsrc: Long? = null
private var waitInterval = minInterval
// Stats
// Number of PLI/FIRs received and forwarded to the endpoint.
private var numPlisForwarded: Int = 0
private var numFirsForwarded: Int = 0
// Number of PLI/FIRs received but dropped due to throttling.
private var numPlisDropped: Int = 0
private var numFirsDropped: Int = 0
// Number of PLI/FIRs generated as a result of an API request or due to translation between PLI/FIR.
private var numPlisGenerated: Int = 0
private var numFirsGenerated: Int = 0
// Number of calls to requestKeyframe
private var numApiRequests: Int = 0
// Number of calls to requestKeyframe ignored due to throttling
private var numApiRequestsDropped: Int = 0
// Number of requests dropped by each limiter, to show which one is binding.
private var numRequestsDroppedPerReceiverLimit: Int = 0
private var numRequestsDroppedSourceWideLimit: Int = 0
override fun transform(packetInfo: PacketInfo): PacketInfo? {
val pliOrFirPacket = packetInfo.getPliOrFirPacket() ?: return packetInfo
val now = clock.instant()
val sourceSsrc: Long
val canSend: Boolean
val forward: Boolean
when (pliOrFirPacket) {
is RtcpFbPliPacket -> {
sourceSsrc = pliOrFirPacket.mediaSourceSsrc
canSend = canSendKeyframeRequest(packetInfo.endpointId, sourceSsrc, now)
forward = canSend && streamInformationStore.supportsPli
if (forward) numPlisForwarded++
if (!canSend) numPlisDropped++
}
is RtcpFbFirPacket -> {
sourceSsrc = pliOrFirPacket.mediaSenderSsrc
canSend = canSendKeyframeRequest(packetInfo.endpointId, sourceSsrc, now)
// When both are supported, we favor generating a PLI rather than forwarding a FIR
forward = canSend && streamInformationStore.supportsFir && !streamInformationStore.supportsPli
if (forward) {
// When we forward a FIR we need to update the seq num.
pliOrFirPacket.seqNum = firCommandSequenceNumber.incrementAndGet()
// We manage the seq num space, so we should use the same SSRC
localSsrc?.let { pliOrFirPacket.mediaSenderSsrc = it }
numFirsForwarded++
}
if (!canSend) numFirsDropped++
}
// This is not possible, but the compiler doesn't know it.
else -> throw IllegalStateException("Packet is neither PLI nor FIR")
}
if (!forward && canSend) {
doRequestKeyframe(sourceSsrc)
}
return if (forward) packetInfo else null
}
/**
* Returns 'true' when at least one method is supported, AND this requester hasn't sent a request very recently.
*/
private fun canSendKeyframeRequest(requesterID: String?, mediaSsrc: Long, now: Instant): Boolean {
if (!streamInformationStore.supportsPli && !streamInformationStore.supportsFir) {
return false
}
synchronized(keyframeLimiterSyncRoot) {
/* A null requesterID is a dominant speaker switch, or a request relayed from another bridge (relayed
* RTCP carries no endpoint id). There is no receiver to attribute it to, so skip only the per-receiver
* limit; the source-wide limit still applies, since it is what protects the sender's encoder and this
* is the only bridge that sees every requester for the source. */
val perReceiverLimiter = requesterID?.let { requester ->
perReceiverKeyframeLimiter.computeIfAbsent(requester) { mutableMapOf() }
.computeIfAbsent(mediaSsrc) {
RateLimit(
defaultMinInterval = minInterval,
maxRequests = maxRequests,
interval = maxRequestInterval
)
}
}
if (perReceiverLimiter != null && !perReceiverLimiter.wouldAccept(now, waitInterval)) {
numRequestsDroppedPerReceiverLimit++
logger.cdebug {
"Ignoring keyframe request for $mediaSsrc from $requesterID, per-receiver rate limited"
}
return false
}
val perSourceLimiter = perSourceKeyframeLimiter.computeIfAbsent(mediaSsrc) {
RateLimit(
defaultMinInterval = sourceWideMinInterval,
maxRequests = sourceWideMaxRequests,
interval = sourceWideMaxRequestInterval
)
}
if (!perSourceLimiter.wouldAccept(now, sourceWideInterval())) {
numRequestsDroppedSourceWideLimit++
logger.cdebug { "Ignoring keyframe request for $mediaSsrc from $requesterID, per-source rate limited" }
return false
}
/* Both limits accept, so record the request with both only now. A receiver waiting for a keyframe
* re-requests on every packet, so if requests dropped by the source-wide limit counted against its
* per-receiver limit it would exhaust that limit while the source-wide one is closed, and then be unable
* to request again for max-request-interval after the source-wide limit reopens. */
perReceiverLimiter?.record(now)
perSourceLimiter.record(now)
logger.cdebug { "Keyframe requester requesting keyframe for $mediaSsrc, requested by $requesterID" }
return true
}
}
/**
* The minimum interval to enforce between keyframe requests for a source, from any receiver. [waitInterval] is
* derived from the per-receiver min-interval and the RTT, so it is never longer than the per-receiver interval;
* take the larger of the two so that source-wide-min-interval is actually enforced.
*/
private fun sourceWideInterval(): Duration = maxOf(waitInterval, sourceWideMinInterval)
fun requestKeyframe(requesterID: String?, mediaSsrc: Long? = null) {
val ssrc = mediaSsrc ?: streamInformationStore.primaryMediaSsrcs.firstOrNull() ?: run {
numApiRequestsDropped++
logger.cdebug { "No video SSRC found to request keyframe" }
return
}
numApiRequests++
if (!canSendKeyframeRequest(requesterID, ssrc, clock.instant())) {
numApiRequestsDropped++
return
}
doRequestKeyframe(ssrc)
}
private fun doRequestKeyframe(mediaSsrc: Long) {
val pkt = when {
streamInformationStore.supportsPli -> {
numPlisGenerated++
RtcpFbPliPacketBuilder(mediaSourceSsrc = mediaSsrc).build()
}
streamInformationStore.supportsFir -> {
numFirsGenerated++
RtcpFbFirPacketBuilder(
mediaSenderSsrc = mediaSsrc,
firCommandSeqNum = firCommandSequenceNumber.incrementAndGet()
).build()
}
else -> {
logger.warn("Can not send neither PLI nor FIR")
return
}
}
next(PacketInfo(pkt))
}
override fun handleEvent(event: Event) {
when (event) {
is SetLocalSsrcEvent -> {
if (event.mediaType == MediaType.VIDEO) {
localSsrc = event.ssrc
}
}
}
}
override fun trace(f: () -> Unit) = f.invoke()
override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply {
addNumber("wait_interval_ms", waitInterval.toMillis())
addNumber("num_api_requests", numApiRequests)
addNumber("num_api_requests_dropped", numApiRequestsDropped)
addNumber("num_firs_dropped", numFirsDropped)
addNumber("num_firs_generated", numFirsGenerated)
addNumber("num_firs_forwarded", numFirsForwarded)
addNumber("num_plis_dropped", numPlisDropped)
addNumber("num_plis_generated", numPlisGenerated)
addNumber("num_plis_forwarded", numPlisForwarded)
addNumber("num_requests_dropped_per_receiver_limit", numRequestsDroppedPerReceiverLimit)
addNumber("num_requests_dropped_source_wide_limit", numRequestsDroppedSourceWideLimit)
}
override fun statsJson() = super.statsJson().apply {
put("num_api_requests", numApiRequests)
put("num_api_requests_dropped", numApiRequestsDropped)
put("num_firs_dropped", numFirsDropped)
put("num_firs_generated", numFirsGenerated)
put("num_firs_forwarded", numFirsForwarded)
put("num_plis_dropped", numPlisDropped)
put("num_plis_generated", numPlisGenerated)
put("num_plis_forwarded", numPlisForwarded)
put("num_requests_dropped_per_receiver_limit", numRequestsDroppedPerReceiverLimit)
put("num_requests_dropped_source_wide_limit", numRequestsDroppedSourceWideLimit)
}
fun onRttUpdate(newRtt: Double) {
// avg(rtt) + stddev(rtt) would be more accurate than rtt + 10.
waitInterval = min(minInterval, durationOfDoubleSeconds((newRtt + 10) / 1e3))
}
companion object {
private val minInterval: Duration by config {
"jmt.keyframe.min-interval".from(JitsiConfig.newConfig)
}
private val maxRequests: Int by config {
"jmt.keyframe.max-requests".from(JitsiConfig.newConfig)
}
private val maxRequestInterval: Duration by config {
"jmt.keyframe.max-request-interval".from(JitsiConfig.newConfig)
}
private val sourceWideMinInterval: Duration by config {
"jmt.keyframe.source-wide-min-interval".from(JitsiConfig.newConfig)
}
private val sourceWideMaxRequests: Int by config {
"jmt.keyframe.source-wide-max-requests".from(JitsiConfig.newConfig)
}
private val sourceWideMaxRequestInterval: Duration by config {
"jmt.keyframe.source-wide-max-request-interval".from(JitsiConfig.newConfig)
}
}
}
private fun PacketInfo.getPliOrFirPacket(): RtcpFbPacket? = when (val pkt = packet) {
// We intentionally ignore compound RTCP packets in order to avoid unnecessary parsing. We can do this because:
// 1. Compound packets coming from remote endpoint are terminated in RtcpTermination
// 2. Whenever a PLI or FIR is generated in our code, it is not part of a compound packet.
is RtcpFbFirPacket -> pkt
is RtcpFbPliPacket -> pkt
else -> null
}

104
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/NackHandler.kt Normal file
View File

@@ -0,0 +1,104 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.nlj.PacketHandler
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.PacketOrigin
import org.jitsi.nlj.stats.EndpointConnectionStats
import org.jitsi.nlj.stats.NodeStatsBlock
import org.jitsi.nlj.transform.NodeStatsProducer
import org.jitsi.nlj.util.BufferPool
import org.jitsi.nlj.util.PacketCache
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacket
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import java.time.Instant
/**
* When a nack packet is received, the [NackHandler] will try to retrieve the
* nacked packets from the cache and then send them to the RTX output pipeline.
*/
class NackHandler(
private val packetCache: PacketCache,
private val onNackedPacketsReady: PacketHandler,
parentLogger: Logger
) : NodeStatsProducer, RtcpListener, EndpointConnectionStats.EndpointConnectionStatsListener {
private var numNacksReceived = 0
private var numNackedPackets = 0
private var numRetransmittedPackets = 0
private var numPacketsNotResentDueToDelay = 0
private var numCacheMisses = 0
private val logger = createChildLogger(parentLogger)
private var currRtt: Double = -1.0
override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) {
if (packet is RtcpFbNackPacket) {
onNackPacket(packet)
}
}
private fun onNackPacket(nackPacket: RtcpFbNackPacket) {
logger.cdebug { "Nack received for ${nackPacket.mediaSourceSsrc} ${nackPacket.missingSeqNums}" }
val now = System.currentTimeMillis()
numNacksReceived++
val nackedPackets = mutableListOf<RtpPacket>()
val ssrc = nackPacket.mediaSourceSsrc
numNackedPackets += nackPacket.missingSeqNums.size
nackPacket.missingSeqNums.forEach { missingSeqNum ->
packetCache.get(ssrc, missingSeqNum)?.let { container ->
val delay = now - container.timeAdded
val shouldResendPacket =
(currRtt == -1.0) ||
(delay >= Math.min(currRtt * .9, currRtt - 5))
if (shouldResendPacket) {
// The cache returns a null container on failure, never a container with a null packet. Maybe we
// can refactor to make it explicit.
nackedPackets.add(container.item!!)
packetCache.updateTimestamp(ssrc, missingSeqNum, now)
numRetransmittedPackets++
} else {
BufferPool.returnBuffer(container.item!!.buffer)
numPacketsNotResentDueToDelay++
}
} ?: run {
logger.cdebug { "Nack'd packet $ssrc $missingSeqNum wasn't in cache, unable to retransmit" }
numCacheMisses++
}
}
nackedPackets.forEach {
val packetInfo = PacketInfo(it)
packetInfo.packetOrigin = PacketOrigin.Retransmission
onNackedPacketsReady.processPacket(packetInfo)
}
}
override fun onRttUpdate(newRttMs: Double) {
currRtt = newRttMs
}
override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("Nack handler").apply {
addNumber("num_nack_packets_received", numNackedPackets)
addNumber("num_nacked_packets", numNackedPackets)
addNumber("num_retransmitted_packets", numRetransmittedPackets)
addNumber("num_packets_not_retransmitted", numPacketsNotResentDueToDelay)
addNumber("num_cache_misses", numCacheMisses)
}
}

72
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RembHandler.kt Normal file
View File

@@ -0,0 +1,72 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.util.ReadOnlyStreamInformationStore
import org.jitsi.nlj.util.bps
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import java.time.Instant
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.atomic.AtomicInteger
class RembHandler(
val streamInformationStore: ReadOnlyStreamInformationStore,
parentLogger: Logger
) : RtcpListener {
private val logger = createChildLogger(parentLogger)
private var sawSpuriousRemb = false
private val bweUpdateListeners: MutableList<TransportCcEngine.BandwidthListener> =
CopyOnWriteArrayList()
override fun rtcpPacketReceived(packet: RtcpPacket?, receivedTime: Instant?) {
if (packet is RtcpFbRembPacket) {
logger.debug { "Received REMB packet" }
onRembPacket(packet)
}
}
fun addListener(bweUpdateListener: TransportCcEngine.BandwidthListener) {
bweUpdateListeners.add(bweUpdateListener)
}
private fun onRembPacket(rembPacket: RtcpFbRembPacket) {
if (streamInformationStore.supportsTcc) {
if (!sawSpuriousRemb) {
logger.warn {
"Ignoring unexpected REMB, when using TCC (for ${rembPacket.bitrate.bps}). Will " +
"suppress future logs for this endpoint."
}
endpointsWithSpuriousRemb.incrementAndGet()
sawSpuriousRemb = true
}
return
}
logger.debug { "Updating bandwidth to ${rembPacket.bitrate.bps}" }
bweUpdateListeners.forEach { it.bandwidthEstimationChanged(rembPacket.bitrate.bps) }
}
companion object {
private val endpointsWithSpuriousRemb = AtomicInteger()
fun endpointsWithSpuriousRemb() = endpointsWithSpuriousRemb.get()
}
}

260
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RetransmissionRequester.kt Normal file
View File

@@ -0,0 +1,260 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacketBuilder
import org.jitsi.rtp.util.RtpUtils
import org.jitsi.rtp.util.isNextAfter
import org.jitsi.rtp.util.isOlderThan
import org.jitsi.rtp.util.numPacketsTo
import org.jitsi.utils.LRUCache
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.logging2.cwarn
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.SortedSet
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
class RetransmissionRequester(
private val rtcpSender: (RtcpPacket) -> Unit,
private val scheduler: ScheduledExecutorService,
parentLogger: Logger,
private val clock: Clock = Clock.systemUTC()
) {
private val logger = createChildLogger(parentLogger)
/**
* Per-SSRC requesters, keyed on the RTP SSRC. Bounded with an LRU to limit the number of SSRCs tracked; evicted
* requesters are stopped to cancel their scheduled tasks. Access is synchronized on the map.
*/
private val streamPacketRequesters: MutableMap<Long, StreamPacketRequester> =
object : LRUCache<Long, StreamPacketRequester>(MAX_SSRCS, true) {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Long, StreamPacketRequester>): Boolean =
super.removeEldestEntry(eldest).also { if (it) eldest.value.stop() }
}
fun packetReceived(ssrc: Long, seqNum: Int) {
val streamPacketRequester = synchronized(streamPacketRequesters) {
streamPacketRequesters.computeIfAbsent(ssrc) { key ->
StreamPacketRequester(key, scheduler, clock, rtcpSender, logger)
}
}
streamPacketRequester.packetReceived(seqNum)
}
fun stop() {
synchronized(streamPacketRequesters) {
streamPacketRequesters.values.forEach(StreamPacketRequester::stop)
streamPacketRequesters.clear()
}
}
/**
* Manages retransmission requests for all packets for a specific SSRC
*/
class StreamPacketRequester(
val ssrc: Long,
private val scheduler: ScheduledExecutorService,
private val clock: Clock,
private val rtcpSender: (RtcpPacket) -> Unit,
parentLogger: Logger,
private val maxMissingSeqNums: Int = 100
) {
companion object {
val NO_REQUEST_DUE: Instant = Instant.MAX
}
private var running: AtomicBoolean = AtomicBoolean(true)
private val logger =
createChildLogger(parentLogger, mapOf("ssrc" to ssrc.toString()))
private var highestReceivedSeqNum = -1
private val requests: MutableMap<Int, PacketRetransmissionRequest> = HashMap()
private val taskHandleLock = Any()
private var currentTaskHandle: ScheduledFuture<*>? = null
fun packetReceived(seqNum: Int) {
if (highestReceivedSeqNum == -1) {
highestReceivedSeqNum = seqNum
return
}
synchronized(requests) {
when {
seqNum == highestReceivedSeqNum -> {
logger.cdebug { "$ssrc packet $seqNum was received, currently missing ${getMissingSeqNums()}" }
// By definition we've already received highestReceivedSeqNum, so nothing needs to be done.
}
seqNum isOlderThan highestReceivedSeqNum -> {
logger.cdebug { "$ssrc packet $seqNum was received, currently missing ${getMissingSeqNums()}" }
// An older packet, possibly already requested
requests.remove(seqNum)
if (requests.isEmpty()) {
logger.cdebug { "$ssrc no more missing seq nums, cancelling pending work" }
updateWorkDueTime(NO_REQUEST_DUE)
}
}
seqNum isNextAfter highestReceivedSeqNum -> {
highestReceivedSeqNum = seqNum
}
highestReceivedSeqNum numPacketsTo seqNum < maxMissingSeqNums -> {
logger.cdebug {
"$ssrc missing packet detected! Just received " +
"$seqNum, last received was $highestReceivedSeqNum"
}
RtpUtils.sequenceNumbersBetween(highestReceivedSeqNum, seqNum).forEach { missingSeqNum ->
val request = PacketRetransmissionRequest(missingSeqNum)
requests[missingSeqNum] = request
updateWorkDueTime(clock.instant())
}
highestReceivedSeqNum = seqNum
}
else -> { // diff > maxMissingSeqNums
logger.cwarn {
"$ssrc large jump in sequence numbers detected (highest received was " +
"$highestReceivedSeqNum, current is $seqNum, jump of " +
"${highestReceivedSeqNum numPacketsTo seqNum}) , not requesting retransmissions"
}
highestReceivedSeqNum = seqNum
// Reset and clear any pending work to do for this source
requests.clear()
logger.cdebug { "$ssrc large packet gap, resetting and clearing all work" }
updateWorkDueTime(NO_REQUEST_DUE)
}
}
}
}
fun stop() {
running.set(false)
synchronized(taskHandleLock) {
currentTaskHandle?.cancel(false)
}
synchronized(requests) {
requests.clear()
}
}
private fun updateWorkDueTime(newWorkDueTs: Instant) {
logger.cdebug { "$ssrc updating next work due time to $newWorkDueTs" }
synchronized(taskHandleLock) {
if (!running.get()) {
logger.cdebug { "$ssrc is stopped, not rescheduling task" }
}
when (newWorkDueTs) {
NO_REQUEST_DUE -> {
logger.cdebug { "$ssrc no more work to do, cancelling job handle" }
currentTaskHandle?.cancel(false)
}
else -> {
// TODO(brian): only re-schedule if the change is larger than X ms?
// The work is now due either sooner or later than we previously thought, so
// re-schedule the task
currentTaskHandle?.cancel(false)
currentTaskHandle = scheduler.schedule(
::doWork,
Duration.between(clock.instant(), newWorkDueTs).toMillis(),
TimeUnit.MILLISECONDS
)
}
}
}
}
private fun doWork() {
logger.cdebug { "$ssrc doing work at ${clock.instant()}" }
val now = clock.instant()
val missingSeqNums = getMissingSeqNums()
if (missingSeqNums.size >= maxMissingSeqNums) {
logger.warn { "$ssrc sending NACK for ${missingSeqNums.size} missing packets" }
}
val nackPacket = RtcpFbNackPacketBuilder(
mediaSourceSsrc = ssrc,
missingSeqNums = missingSeqNums
).build()
notifyNackSent(now, missingSeqNums)
rtcpSender(nackPacket)
}
private fun notifyNackSent(timestamp: Instant, nackedSeqNums: Collection<Int>) {
synchronized(requests) {
nackedSeqNums.forEach { nackedSeqNum ->
// It's possible that in between sending the nack and calling this method the packet
// was received and is no longer in the requests map
requests[nackedSeqNum]?.let { request ->
request.requested(timestamp)
if (request.numTimesRequested == MAX_REQUESTS) {
logger.cdebug {
"$ssrc generated the last NACK for seq num ${request.seqNum}, " +
"time since the first request = " +
"${Duration.between(request.firstRequestTimestamp, timestamp)}"
}
requests.remove(nackedSeqNum)
}
} ?: run {
logger.cdebug {
"$ssrc packet $nackedSeqNum must have just been received, it was " +
"no longer in the requests map"
}
}
}
val nextDueTime = if (requests.isNotEmpty()) timestamp.plus(REQUEST_INTERVAL) else NO_REQUEST_DUE
logger.cdebug { "$ssrc nack sent at $timestamp, next one will be sent at $nextDueTime" }
updateWorkDueTime(nextDueTime)
}
}
private fun getMissingSeqNums(): SortedSet<Int> = synchronized(requests) { requests.keys.toSortedSet() }
}
/**
* Tracks a request for retransmission of a specific RTP packet.
*/
private class PacketRetransmissionRequest(
val seqNum: Int
) {
var numTimesRequested = 0
private set
var firstRequestTimestamp: Instant = Instant.MIN
private set
fun requested(timestamp: Instant) {
if (firstRequestTimestamp == Instant.MIN) {
firstRequestTimestamp = timestamp
}
numTimesRequested++
}
}
companion object {
private const val MAX_REQUESTS = 10
private val REQUEST_INTERVAL = Duration.ofMillis(150)
/** The maximum number of SSRCs to track retransmission requesters for. */
const val MAX_SSRCS = 1024
}
}

51
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpEventNotifier.kt Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.rtp.rtcp.RtcpPacket
import java.time.Instant
import java.util.concurrent.CopyOnWriteArrayList
/**
* A central place to allow the publishing of when RTCP packets are received or sent. We're
* interested in both of these scenarios for things like SRs, RRs and for RTT calculations
*/
// TODO(brian): maybe post the notifications to another pool, so we don't hold up the caller?
class RtcpEventNotifier {
private val rtcpListeners: MutableList<Pair<RtcpListener, Boolean>> = CopyOnWriteArrayList()
/**
* Add an [RtcpListener]. An [external] listener will not receive notifications that are themselves
* marked as external. This allows notifications to be passed between multiple listeners without
* creating an infinite loop.
*/
fun addRtcpEventListener(listener: RtcpListener, external: Boolean = false) {
rtcpListeners.add(Pair(listener, external))
}
fun notifyRtcpReceived(packet: RtcpPacket, receivedTime: Instant?, external: Boolean = false) {
rtcpListeners.forEach { (listener, listenerIsExternal) ->
if (!external || !listenerIsExternal) listener.rtcpPacketReceived(packet, receivedTime)
}
}
fun notifyRtcpSent(rtcpPacket: RtcpPacket, external: Boolean = false) {
rtcpListeners.forEach { (listener, listenerIsExternal) ->
if (!external || !listenerIsExternal) listener.rtcpPacketSent(rtcpPacket)
}
}
}

34
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpListener.java Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp;
import org.jetbrains.annotations.*;
import org.jitsi.rtp.rtcp.*;
import java.time.*;
/**
* NOTE(brian): Java doesn't see default implementation interface methods when
* written from kotlin, so this is implemented in java (which works for both
* kotlin and java) because I got tired of adding no-op methods for one of
* these methods as most listeners care about one but not the other
*/
public interface RtcpListener {
default void rtcpPacketReceived(RtcpPacket packet, @Nullable Instant receivedTime) {}
default void rtcpPacketSent(RtcpPacket packet) {}
}

45
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright @ 2019 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.nlj.transform.node.PacketParser
import org.jitsi.rtp.rtcp.CompoundRtcpPacket
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.utils.logging2.Logger
class CompoundRtcpParser(parentLogger: Logger) : PacketParser(
"Compound RTCP parser",
parentLogger,
{
CompoundRtcpPacket(it.buffer, it.offset, it.length).also { compoundPacket ->
// Force packets to be evaluated to trigger any parsing errors
compoundPacket.packets
}
}
) {
override fun trace(f: () -> Unit) = f.invoke()
}
class SingleRtcpParser(parentLogger: Logger) : PacketParser(
"Single RTCP parser",
parentLogger,
{
RtcpPacket.parse(it.buffer, it.offset, it.length)
}
) {
override fun trace(f: () -> Unit) = f.invoke()
}

152
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt Normal file
View File

@@ -0,0 +1,152 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.nlj.transform.node.incoming.IncomingSsrcStats
import org.jitsi.nlj.transform.node.incoming.IncomingStatisticsTracker
import org.jitsi.nlj.util.schedule
import org.jitsi.rtp.rtcp.CompoundRtcpPacket
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.RtcpReportBlock
import org.jitsi.rtp.rtcp.RtcpRrPacketBuilder
import org.jitsi.rtp.rtcp.RtcpSrPacket
import org.jitsi.utils.LRUCache
import org.jitsi.utils.MediaType
import org.jitsi.utils.ms
import org.jitsi.utils.times
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.Collections
import java.util.concurrent.ScheduledExecutorService
/**
* Information about a sender that is used in the generation of RTCP report blocks. NOTE that this does NOT correspond
* to the Sender Info block of an SR
* TODO: rename to not be confused with Sender Info in SR?
*/
private data class SenderInfo(
var lastSrCompactedTimestamp: Long = 0,
var lastSrReceivedTime: Instant? = null,
// The media type doesn't affect RTCP RR/SR generation. Initialize with a dummy value.
var statsSnapshot: IncomingSsrcStats.Snapshot = IncomingSsrcStats.Snapshot(mediaType = MediaType.VIDEO)
) {
private fun hasReceivedSr(): Boolean = lastSrReceivedTime != null
fun getDelaySinceLastSr(now: Instant): Long = if (hasReceivedSr()) {
// This value is in 1/65536 of a second, so multiplying by 65536 gives us the value
Duration.between(lastSrReceivedTime, now).times(65536).seconds
} else {
0
}
}
/**
* Retrieves statistics about incoming streams and creates RTCP RR packets. Since RR packets are created based on
* time (and not on a number of incoming packets received, etc.) it does not live within the packet pipelines.
*
* @param additionalPacketSupplier A function which supplies additional RTCP packets (such as REMB) to be sent together
* with RRs.
*/
class RtcpRrGenerator(
private val backgroundExecutor: ScheduledExecutorService,
private val rtcpSender: (RtcpPacket) -> Unit = {},
private val incomingStatisticsTracker: IncomingStatisticsTracker,
private val clock: Clock = Clock.systemUTC(),
private val additionalPacketSupplier: () -> List<RtcpPacket>
) : RtcpListener {
var running: Boolean = true
/**
* Per-SSRC sender info, keyed on the RTCP sender SSRC. Bounded with an LRU to limit the number of SSRCs tracked.
*/
private val senderInfos: MutableMap<Long, SenderInfo> =
Collections.synchronizedMap(LRUCache(MAX_SSRCS, true))
init {
doWork()
}
override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) {
when (packet) {
is RtcpSrPacket -> {
// Note the time we received an SR so that it can be used when creating RtcpReportBlocks
// TODO: we have a concurrency issue here: we could be halfway through updating the senderinfo when
// the doWork context thread runs
val senderInfo = senderInfos.computeIfAbsent(packet.senderSsrc) { SenderInfo() }
senderInfo.lastSrCompactedTimestamp = packet.senderInfo.compactedNtpTimestamp
senderInfo.lastSrReceivedTime = receivedTime
}
}
}
private fun doWork() {
if (running) {
val streamStats = incomingStatisticsTracker.getSnapshotOfActiveSsrcs()
val now = clock.instant()
val reportBlocks = mutableListOf<RtcpReportBlock>()
streamStats.ssrcStats.forEach { (ssrc, statsSnapshot) ->
val senderInfo = senderInfos.computeIfAbsent(ssrc) {
SenderInfo()
}
val fractionLost = statsSnapshot.computeFractionLost(senderInfo.statsSnapshot)
senderInfo.statsSnapshot = statsSnapshot
reportBlocks.add(
RtcpReportBlock(
ssrc,
fractionLost,
statsSnapshot.cumulativePacketsLost,
statsSnapshot.seqNumCycles,
statsSnapshot.maxSeqNum,
statsSnapshot.jitter.toLong(),
senderInfo.lastSrCompactedTimestamp,
senderInfo.getDelaySinceLastSr(now)
)
)
}
val packets = mutableListOf<RtcpPacket>()
if (reportBlocks.isNotEmpty()) {
for (blocks in reportBlocks.chunked(31)) {
packets.add(RtcpRrPacketBuilder(reportBlocks = blocks).build())
}
}
packets.addAll(additionalPacketSupplier())
when (packets.size) {
0 -> {}
1 -> rtcpSender(packets.first())
else -> for (packet in CompoundRtcpPacket.createWithMtu(packets)) {
rtcpSender(packet)
}
}
backgroundExecutor.schedule(this::doWork, reportingInterval)
}
}
companion object {
/**
* The interval at which RRs and REMBs will be sent.
*/
val reportingInterval = 500.ms
/** The maximum number of SSRCs to track sender info for. */
const val MAX_SSRCS = 1024
}
}

44
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpSrUpdater.kt Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright @ 2019 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtcp
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.transform.node.TransformerNode
import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsTracker
import org.jitsi.rtp.rtcp.RtcpSrPacket
/**
* Updates RTCP Sender Reports with the current octet and packet count.
*/
class RtcpSrUpdater(
val statsTracker: OutgoingStatisticsTracker
) : TransformerNode("RtcpSrUpdater") {
override fun transform(packetInfo: PacketInfo): PacketInfo? {
// TODO support compound packets
val rtcpSrPacket = packetInfo.packet as? RtcpSrPacket ?: return packetInfo
// If we haven't sent RTP for this SSRC, drop the SR
val ssrcStats = statsTracker.getSsrcSnapshot(rtcpSrPacket.senderSsrc) ?: return null
rtcpSrPacket.senderInfo.sendersOctetCount = ssrcStats.octetCount.toLong()
rtcpSrPacket.senderInfo.sendersPacketCount = ssrcStats.packetCount.toLong()
return packetInfo
}
override fun trace(f: () -> Unit) = f.invoke()
}

31
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import org.jitsi.rtp.rtp.RtpPacket
open class AudioRtpPacket(
buffer: ByteArray,
offset: Int,
length: Int
) : RtpPacket(buffer, offset, length) {
override fun clone(): AudioRtpPacket = AudioRtpPacket(
cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET),
BYTES_TO_LEAVE_AT_START_OF_PACKET,
length
).also { postClone(it) }
}

340
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ClassicTransportCcEngine.kt Normal file
View File

@@ -0,0 +1,340 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator
import org.jitsi.nlj.util.ArrayCache
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.RtpSequenceIndexTracker
import org.jitsi.nlj.util.bytes
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.PacketReport
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.ReceivedPacketReport
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.UnreceivedPacketReport
import org.jitsi.utils.NEVER
import org.jitsi.utils.formatMilli
import org.jitsi.utils.joinToRangedString
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.concurrent.atomic.LongAdder
/**
* Implements transport-cc functionality.
*
* See https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01
*
* @author Boris Grozev
* @author Julian Chukwu
* @author George Politis
*/
class ClassicTransportCcEngine(
private val bandwidthEstimator: BandwidthEstimator,
parentLogger: Logger,
private val clock: Clock = Clock.systemUTC()
) : TransportCcEngine() {
/**
* The [Logger] used by this instance for logging output.
*/
private val logger: Logger = createChildLogger(parentLogger)
val numPacketsReported = LongAdder()
val numPacketsReportedLost = LongAdder()
val numDuplicateReports = LongAdder()
val numPacketsReportedAfterLost = LongAdder()
val numPacketsUnreported = LongAdder()
val numMissingPacketReports = LongAdder()
/**
* The reference time of the remote clock. This is used to rebase the
* arrival times in the TCC packets to a meaningful time base (that of the
* sender). This is technically not necessary and it's done for convenience.
*/
private var remoteReferenceTime: Instant = NEVER
/**
* Local time to map to the reference time of the remote clock. This is used
* to rebase the arrival times in the TCC packets to a meaningful time base
* (that of the sender). This is technically not necessary and it's done for
* convenience.
*/
private var localReferenceTime: Instant = NEVER
/**
* Holds a key value pair of the packet sequence number and an object made
* up of the packet send time and the packet size.
*/
private val sentPacketDetails = PacketDetailTracker(clock)
private val missingPacketDetailSeqNums = mutableListOf<Int>()
private var lastRtt: Duration? = null
/**
* Called when an RTP sender has a new round-trip time estimate.
*/
override fun onRttUpdate(rtt: Duration) {
val now = clock.instant()
bandwidthEstimator.onRttUpdate(now, rtt)
lastRtt = rtt
}
override fun rtcpPacketReceived(rtcpPacket: RtcpPacket, receivedTime: Instant?) {
if (rtcpPacket is RtcpFbTccPacket) {
tccReceived(rtcpPacket)
}
}
private fun tccReceived(tccPacket: RtcpFbTccPacket) {
val now = clock.instant()
var currArrivalTimestamp = tccPacket.BaseTime()
if (remoteReferenceTime == NEVER) {
remoteReferenceTime = currArrivalTimestamp
localReferenceTime = now
}
for (packetReport in tccPacket) {
val tccSeqNum = packetReport.seqNum
val packetDetail = sentPacketDetails.get(tccSeqNum)
if (packetDetail == null) {
if (packetReport is ReceivedPacketReport) {
currArrivalTimestamp += packetReport.deltaDuration
missingPacketDetailSeqNums.add(tccSeqNum)
numMissingPacketReports.increment()
}
continue
}
when (packetReport) {
is UnreceivedPacketReport -> {
if (packetDetail.state == PacketDetailState.Unreported) {
bandwidthEstimator.processPacketLoss(now, packetDetail.packetSendTime, tccSeqNum)
packetDetail.state = PacketDetailState.ReportedLost
numPacketsReported.increment()
numPacketsReportedLost.increment()
synchronized(this) {
lossListeners.forEach {
it.packetLost(1)
}
}
}
}
is ReceivedPacketReport -> {
currArrivalTimestamp += packetReport.deltaDuration
when (packetDetail.state) {
PacketDetailState.Unreported, PacketDetailState.ReportedLost -> {
val previouslyReportedLost = packetDetail.state == PacketDetailState.ReportedLost
if (previouslyReportedLost) {
numPacketsReportedAfterLost.increment()
numPacketsReportedLost.decrement()
/* Packet has already been counted in numPacketsReported */
} else {
numPacketsReported.increment()
}
val arrivalTimeInLocalClock =
currArrivalTimestamp - Duration.between(localReferenceTime, remoteReferenceTime)
bandwidthEstimator.processPacketArrival(
now,
packetDetail.packetSendTime,
arrivalTimeInLocalClock,
tccSeqNum,
packetDetail.packetLength,
previouslyReportedLost = previouslyReportedLost
)
synchronized(this) {
lossListeners.forEach {
it.packetReceived(previouslyReportedLost)
}
}
packetDetail.state = PacketDetailState.ReportedReceived
}
PacketDetailState.ReportedReceived ->
numDuplicateReports.increment()
}
}
}
}
bandwidthEstimator.feedbackComplete(now)
if (missingPacketDetailSeqNums.isNotEmpty()) {
logger.warn(
"TCC packet contained received sequence numbers: " +
"${tccPacket.iterator().asSequence()
.filterIsInstance<ReceivedPacketReport>()
.map(PacketReport::seqNum)
.joinToRangedString()}. " +
"Couldn't find packet detail for the seq nums: " +
"${missingPacketDetailSeqNums.joinToRangedString()}. " +
if (sentPacketDetails.empty) {
"Sent packet details map was empty."
} else {
"Latest seqNum was ${sentPacketDetails.lastSequence}, size is ${sentPacketDetails.size}."
} +
(
lastRtt?.let {
" Latest RTT is ${it.formatMilli()} ms."
} ?: run {
""
}
)
)
missingPacketDetailSeqNums.clear()
}
}
override fun mediaPacketTagged(packetInfo: PacketInfo, tccSeqNum: Long) {
/* Nothing needs to be done */
}
override fun mediaPacketSent(packetInfo: PacketInfo, tccSeqNum: Long) {
val now = clock.instant()
val seq = (tccSeqNum and 0xFFFF).toInt()
if (!sentPacketDetails.insert(seq, PacketDetail(packetInfo.packet.length.bytes, now))) {
/* Very old seq? Something odd is happening with whatever is
* generating tccSeqNum values.
*/
logger.warn(
"Not inserting very old TCC seq num $seq ($tccSeqNum), latest is " +
"${sentPacketDetails.lastSequence}, size is ${sentPacketDetails.size}"
)
return
}
}
override fun getStatistics(): StatisticsSnapshot = StatisticsSnapshot(
numPacketsReported.sum(),
numPacketsReportedLost.sum(),
numDuplicateReports.sum(),
numPacketsReportedAfterLost.sum(),
numPacketsUnreported.sum(),
numMissingPacketReports.sum(),
bandwidthEstimator.getStats()
)
override fun addBandwidthListener(listener: BandwidthListener) = bandwidthEstimator.addListener(listener)
override fun removeBandwidthListener(listener: BandwidthListener) = bandwidthEstimator.removeListener(listener)
override fun start() { }
override fun stop() { }
/**
* [PacketDetailState] is the state of a [PacketDetail]
*/
private enum class PacketDetailState {
Unreported,
ReportedLost,
ReportedReceived
}
/**
* [PacketDetail] is an object that holds the
* length(size) of the packet in [packetLength]
* and the time stamps of the outgoing packet
* in [packetSendTime]
*/
@ConsistentCopyVisibility
private data class PacketDetail internal constructor(
val packetLength: DataSize,
val packetSendTime: Instant
) {
/**
* [state] represents the state of this packet detail with regards to the
* reception of a TCC feedback from the remote side. [PacketDetail]s start out
* as [unreported]: once we receive a TCC feedback from the remote side referring
* to this packet, the state will transition to either [reportedLost] or [reportedReceived].
*/
var state = PacketDetailState.Unreported
}
class StatisticsSnapshot(
val numPacketsReported: Long,
val numPacketsReportedLost: Long,
val numDuplicateReports: Long,
val numPacketsReportedAfterLost: Long,
val numPacketsUnreported: Long,
val numMissingPacketReports: Long,
val bandwidthEstimatorStats: BandwidthEstimator.StatisticsSnapshot
) : TransportCcEngine.StatisticsSnapshot() {
override fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().also {
it.put("name", ClassicTransportCcEngine::class.java.simpleName)
it.put("numPacketsReported", numPacketsReported)
it.put("numPacketsReportedLost", numPacketsReportedLost)
it.put("numDuplicateReports", numDuplicateReports)
it.put("numPacketsReportedAfterLost", numPacketsReportedAfterLost)
it.put("numPacketsUnreported", numPacketsUnreported)
it.put("numMissingPacketReports", numMissingPacketReports)
it.set<ObjectNode>("bandwidth_estimator_stats", bandwidthEstimatorStats.toJson())
}
}
private inner class PacketDetailTracker(clock: Clock) : ArrayCache<PacketDetail>(
MAX_OUTGOING_PACKETS_HISTORY,
/* We don't want to clone [PacketDetail] objects that get put in the tracker. */
{ it },
clock = clock
) {
override fun discardItem(item: PacketDetail) {
if (item.state == PacketDetailState.Unreported) {
numPacketsUnreported.increment()
}
}
private val rtpSequenceIndexTracker = RtpSequenceIndexTracker()
/**
* Gets a packet with a given RTP sequence number from the cache.
*/
fun get(sequenceNumber: Int): PacketDetail? {
// Note that we use [interpret] because we don't want the ROC to get out of sync because of funny requests
// (TCCs)
val index = rtpSequenceIndexTracker.interpret(sequenceNumber)
return super.getContainer(index, shouldCloneItem = true)?.item
}
fun insert(seq: Int, packetDetail: PacketDetail): Boolean {
val index = rtpSequenceIndexTracker.update(seq)
return super.insertItem(packetDetail, index, packetDetail.packetSendTime.toEpochMilli())
}
val lastSequence: Int
get() = if (lastIndex == -1L) -1 else (lastIndex and 0xFFFF).toInt()
}
companion object {
/**
* The maximum number of received packets and their timestamps to save.
*
* NOTE rtt + minimum amount
* XXX this is an uninformed value.
*/
private const val MAX_OUTGOING_PACKETS_HISTORY = 1000
}
}

65
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/LossListener.kt Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import org.jitsi.utils.secs
import org.jitsi.utils.stats.RateTracker
class LossTracker : LossListener {
private val lostPackets = RateTracker(60.secs, 1.secs)
private val receivedPackets = RateTracker(60.secs, 1.secs)
@Synchronized
override fun packetReceived(previouslyReportedLost: Boolean) {
receivedPackets.update(1)
if (previouslyReportedLost) {
lostPackets.update(-1)
}
}
@Synchronized
override fun packetLost(numLost: Int) {
lostPackets.update(numLost.toLong())
}
@Synchronized
fun getSnapshot(): Snapshot = Snapshot(
lostPackets.getAccumulatedCount(),
receivedPackets.getAccumulatedCount()
)
data class Snapshot(
val packetsLost: Long,
val packetsReceived: Long
) {
fun toJson() = JsonNodeFactory.instance.objectNode().apply {
put("packets_lost", packetsLost)
put("packets_received", packetsReceived)
}
}
}
/**
* An interface to report when a packet is received, or is observed to be lost.
*/
/* TODO? This kind of overlaps with BandwidthEstimator? But it can be used in cases where we
* don't have all the information the BandwidthEstimator API needs. */
interface LossListener {
fun packetReceived(previouslyReportedLost: Boolean)
fun packetLost(numLost: Int)
}

54
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt Normal file
View File

@@ -0,0 +1,54 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import org.jitsi.nlj.util.BufferPool
import org.jitsi.rtp.rtp.RtpHeader
class PaddingVideoPacket private constructor(
buffer: ByteArray,
offset: Int,
length: Int
) : VideoRtpPacket(buffer, offset, length) {
override fun clone(): PaddingVideoPacket = throw NotImplementedError("clone() not supported for padding packets.")
companion object {
/**
* Creating a PaddingVideoPacket by directly grabbing a buffer in its
* ctor is problematic because we cannot clear the buffer we retrieve
* before calling the parent class' constructor. Because the buffer
* may contain invalid data, any attempts to parse it by parent class(es)
* could fail, so we use a helper here instead
*/
fun create(length: Int): PaddingVideoPacket {
val buf = BufferPool.getBuffer(length)
// It's possible we the buffer we pulled from the pool already has
// data in it, and we won't be overwriting it with anything so clear
// out the data
buf.fill(0, 0, length)
return PaddingVideoPacket(buf, 0, length).apply {
// Recalculate the header length now that we've zero'd everything out
// and set the fields
version = RtpHeader.VERSION
headerLength = RtpHeader.getTotalLength(buffer, offset)
paddingSize = payloadLength
}
}
}
}

40
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt Normal file
View File

@@ -0,0 +1,40 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
/**
* A [VideoRtpPacket] which has been parsed and therefore
* contains more information, such as whether or not it's
* a keyframe
*/
abstract class ParsedVideoPacket(
buffer: ByteArray,
offset: Int,
length: Int,
encodingId: Int
) : VideoRtpPacket(buffer, offset, length, encodingId) {
abstract val isKeyframe: Boolean
abstract val isStartOfFrame: Boolean
abstract val isEndOfFrame: Boolean
/** Whether the packet meets the needs of the routing infrastructure.
* If a packet could be parsed more than one way (e.g. it is VP8 or VP9 but also has an AV1 DD)
* this will let us choose which parse to prefer.
*/
abstract fun meetsRoutingNeeds(): Boolean
}

52
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import org.jitsi.rtp.rtp.RedPacketBuilder
import org.jitsi.rtp.rtp.RedPacketParser
import java.lang.IllegalStateException
class RedAudioRtpPacket(
buffer: ByteArray,
offset: Int,
length: Int
) : AudioRtpPacket(buffer, offset, length) {
private var removed = false
fun removeRed() {
if (removed) throw IllegalStateException("RED encapsulation already removed.")
parser.decapsulate(this, parseRedundancy = false)
removed = true
}
fun removeRedAndGetRedundancyPackets(): List<AudioRtpPacket> = if (removed) {
throw IllegalStateException("RED encapsulation already removed.")
} else {
parser.decapsulate(this, parseRedundancy = true).also { removed = true }
}
override fun clone(): RedAudioRtpPacket = RedAudioRtpPacket(
cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET),
BYTES_TO_LEAVE_AT_START_OF_PACKET,
length
).also { postClone(it) }
companion object {
val parser = RedPacketParser(::AudioRtpPacket)
val builder = RedPacketBuilder(::RedAudioRtpPacket)
}
}

259
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt Normal file
View File

@@ -0,0 +1,259 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import org.jitsi.nlj.util.ArrayCache
import org.jitsi.nlj.util.RtpSequenceIndexTracker
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.rtp.util.isNewerThan
/**
* Rewrites sequence numbers for RTP streams by hiding any gaps caused by
* dropped packets. Rewriters are not thread-safe. If multiple threads access a
* rewriter concurrently, it must be synchronized externally.
*
* Port of the class in libjitsi.
*
* @author Maryam Daneshi
* @author George Politis
* @author Boris Grozev
* @author Jonathan Lennox
*/
class ResumableStreamRewriter(val keepHistory: Boolean = false) {
/**
* The sequence number delta between what's been accepted and what's been
* received, mod 2^16.
*/
var seqnumDelta = 0
private set
private var history: StreamRewriteHistory? = null
/**
* The highest sequence number that got accepted, mod 2^16.
*/
var highestSequenceNumberSent = -1
private set
/**
* Rewrites the sequence number of the given RTP packet hiding any gaps caused by drops.
*
* @param accept true if the packet is accepted, false otherwise
*/
fun rewriteRtp(accept: Boolean, rtpPacket: RtpPacket) {
val sequenceNumber = rtpPacket.sequenceNumber
val newSequenceNumber = rewriteSequenceNumber(accept, sequenceNumber)
if (sequenceNumber != newSequenceNumber) {
rtpPacket.sequenceNumber = newSequenceNumber
}
}
/**
* Rewrites the sequence number passed as a parameter, hiding any gaps
* caused by drops.
*
* @param accept true if the packet is accepted, false otherwise
* @param sequenceNumber the sequence number to rewrite
* @return a rewritten sequence number that hides any gaps caused by drops.
*/
fun rewriteSequenceNumber(accept: Boolean, sequenceNumber: Int): Int {
if (keepHistory && !accept && history == null) {
/* Don't instantiate history until it's needed; many streams never discard. */
history = StreamRewriteHistory(highestSequenceNumberSent)
}
history?.let {
return it.rewriteSequenceNumber(accept, sequenceNumber)
}
if (accept) {
// overwrite the sequence number (if needed)
val newSequenceNumber = (sequenceNumber - seqnumDelta) and 0xffff
// init or update the highest sent sequence number (if needed)
if (highestSequenceNumberSent == -1 || newSequenceNumber isNewerThan highestSequenceNumberSent) {
highestSequenceNumberSent = newSequenceNumber
}
return newSequenceNumber
} else {
// update the sequence number delta (if needed)
if (highestSequenceNumberSent != -1) {
val newDelta = (sequenceNumber - highestSequenceNumberSent) and 0xffff
if (newDelta isNewerThan seqnumDelta) {
seqnumDelta = newDelta
}
}
return sequenceNumber
}
}
val gapsLeft: Int
get() = history?.gapsLeft ?: 0
private class RewriteHistoryItem(
var accept: Boolean?,
val newIndex: Long
)
private class StreamRewriteHistory(highestSeqSent: Int) : ArrayCache<RewriteHistoryItem>(
MAX_REWRITE_HISTORY,
/* We don't want to clone objects that get put in the tracker. */
{ it },
/* Caller should have this object synchronized if needed. */
synchronize = false
) {
var firstIndex: Long = -1L
var gapsLeft = 0
private set
private val rtpSequenceIndexTracker = RtpSequenceIndexTracker()
private fun fillBetween(start: Long, end: Long, firstNewIndex: Long) {
if (end <= lastIndex - size + 1) {
return
}
val actualStart = if (start <= lastIndex - size) {
lastIndex - size + 1
} else {
start
}
var newIndex = firstNewIndex
for (i in actualStart..end) {
insertItem(RewriteHistoryItem(null, newIndex), i)
newIndex++
}
}
fun rewriteSequenceNumber(accept: Boolean, sequenceNumber: Int): Int {
val index = rtpSequenceIndexTracker.update(sequenceNumber)
val newIndex: Long
when {
(firstIndex == -1L) -> {
/* First index seen. */
insertItem(RewriteHistoryItem(accept, index), index)
firstIndex = index
newIndex = index
}
(index > lastIndex) -> {
/* New. Roll forward, filling in gap if necessary. */
val newestIndex = lastIndex
val newest = getContainer(newestIndex)
?: throw IllegalStateException("No newest container found")
val newestNewIndex = newest.item!!.newIndex
val indexGap = index - newestIndex
val newGap = indexGap - if (accept) 0 else 1
newIndex = newestNewIndex + newGap
insertItem(RewriteHistoryItem(accept, newIndex), index)
fillBetween(newestIndex + 1, index - 1, newestNewIndex + 1)
}
(index > lastIndex - size && index >= firstIndex) -> {
/* In history. Retrieve. */
val container = getContainer(index)
?: throw IllegalStateException("No container found for index $index in history")
val item = container.item!!
if (item.accept == null) {
item.accept = accept
if (!accept) {
gapsLeft++
}
}
newIndex = item.newIndex
}
(index > lastIndex - size && index < firstIndex) -> {
/* Older than previous oldest, but still in range of the map.
Project map backwards. */
val oldestIndex = Math.max(lastIndex - size + 1, firstIndex)
val oldest = getContainer(oldestIndex)
?: throw IllegalStateException("No oldest container found")
val oldestNewIndex = oldest.item!!.newIndex
val indexGap = index - oldestIndex // Negative
val newGap = indexGap + if (accept) 0 else 1
newIndex = oldestNewIndex + newGap
insertItem(RewriteHistoryItem(accept, newIndex), index)
fillBetween(index + 1, oldestIndex - 1, newIndex + 1)
firstIndex = index
}
else -> {
/* Older than the map. */
val oldestIndex = Math.max(lastIndex - size + 1, firstIndex)
val oldest = getContainer(oldestIndex)
?: throw IllegalStateException("No oldest container found")
val oldestDelta = oldestIndex - oldest.item!!.newIndex
newIndex = index - oldestDelta
}
}
if (accept) return toSequenceNumber(newIndex)
/* Don't care about sequence numbers for non-accepted packets,
so make sure rewriteRtp does nothing. */
return sequenceNumber
}
init {
if (highestSeqSent != -1) {
rewriteSequenceNumber(true, highestSeqSent)
}
}
companion object {
/**
* The maximum number of packets to save history for.
*
* NOTE rtt + minimum amount
* XXX this is an uninformed value.
*/
private const val MAX_REWRITE_HISTORY = 1000
/** Map an index back to a sequence number. */
private fun toSequenceNumber(index: Long): Int = (index and 0xffff).toInt()
}
}
}

123
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt Normal file
View File

@@ -0,0 +1,123 @@
/*
* Copyright @ 2019-Present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
/**
* Represents a signalled RTP header extension.
*
* @author Boris Grozev
*/
data class RtpExtension(
/**
* The ID that was signalled.
*/
val id: Byte,
/**
* The type.
*/
val type: RtpExtensionType
)
/**
* Represents an RTP header extension type.
*
* @author Boris Grozev
*/
enum class RtpExtensionType(val uri: String) {
/**
* The URN identifying the RTP extension that allows mixers to send to
* conference participants the audio levels of all contributing sources.
* Defined in RFC6465.
*/
CSRC_AUDIO_LEVEL("urn:ietf:params:rtp-hdrext:csrc-audio-level"),
/**
* The URN identifying the RTP extension that allows clients to send to
* conference mixers the audio level of their packet payload. Defined in
* RFC6464.
*/
SSRC_AUDIO_LEVEL("urn:ietf:params:rtp-hdrext:ssrc-audio-level"),
/**
* The URN identifying the abs-send-time RTP extension.
* Defined at
* {@link "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}
*/
ABS_SEND_TIME("http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"),
/**
* The URN which identifies the framemarking RTP extension defined at
* {@link "https://tools.ietf.org/html/draft-ietf-avtext-framemarking-03"}
*/
FRAME_MARKING("http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"),
/**
* The URN which identifies the Original Header Block RTP extension defined
* in {@link "https://tools.ietf.org/html/draft-ietf-perc-double-02"}.
*/
ORIGINAL_HEADER_BLOCK("urn:ietf:params:rtp-hdrext:ohb"),
/**
* The URN which identifies the Transport-Wide Congestion Control RTP
* extension.
*/
TRANSPORT_CC("http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"),
/**
* The URN which identifies the rtp-stream-id extensions
* in {@link "https://tools.ietf.org/html/draft-ietf-mmusic-rid-10"}.
*/
RTP_STREAM_ID("urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"),
/**
* The URN which identifies the transmission time-offset extensions
* in {@link "https://tools.ietf.org/html/rfc5450"}.
*/
TOF("urn:ietf:params:rtp-hdrext:toffset"),
/**
* The URN which identifies the RTP Header Extension for Video Content Type.
*/
VIDEO_CONTENT_TYPE("http://www.webrtc.org/experiments/rtp-hdrext/video-content-type"),
/**
* The URN which identifies the RTP Header Extension for Video Orientation.
*/
VIDEO_ORIENTATION("urn:3gpp:video-orientation"),
/**
* The URN which identifies the AV1 Dependency Descriptor RTP Header Extension
*/
AV1_DEPENDENCY_DESCRIPTOR(
"https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension"
),
/**
* Video Layers Allocation
* https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/video-layers-allocation00
*/
VLA("http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00"),
/**
* The URN which identifies the sdes:mid (media identification) RTP header extension defined in RFC 8843/9143.
*/
MID("urn:ietf:params:rtp-hdrext:sdes:mid");
companion object {
private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri)
fun createFromUri(uri: String): RtpExtensionType? = uriMap.getOrDefault(uri, null)
}
}

62
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtxPacket.kt Normal file
View File

@@ -0,0 +1,62 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import org.jitsi.nlj.util.shiftPayloadRight
import org.jitsi.rtp.extensions.bytearray.putShort
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.rtp.util.getShortAsInt
/**
* https://tools.ietf.org/html/rfc4588#section-4
*
* 0 1 2 3
* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
* | RTP Header |
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
* | OSN | |
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |
* | Original RTP Packet Payload |
* | |
* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/
class RtxPacket {
companion object {
fun getOriginalSequenceNumber(rtxPacket: RtpPacket): Int =
rtxPacket.buffer.getShortAsInt(rtxPacket.offset + rtxPacket.headerLength)
/**
* Removes the original sequence number by shifting the header 2
* bytes to the right
*/
fun removeOriginalSequenceNumber(rtxPacket: RtpPacket) = rtxPacket.apply {
// Remove the original sequence number by moving the RTP header 2 bytes to the right
// Note this changes the buffer underlying the RtpPacket -- this is safe (but fragile)
// because we leave header values (which are cached) unchanged.
System.arraycopy(buffer, offset, buffer, offset + 2, headerLength)
offset += 2
length -= 2
}
fun addOriginalSequenceNumber(rtpPacket: RtpPacket) = rtpPacket.apply {
// TODO: possible optimization to try to shift the header left instead
shiftPayloadRight(2)
buffer.putShort(offset + headerLength, sequenceNumber.toShort())
}
}
}

23
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/SsrcAssociationType.kt Normal file
View File

@@ -0,0 +1,23 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
enum class SsrcAssociationType {
RTX,
SIM,
FEC
}

83
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt Normal file
View File

@@ -0,0 +1,83 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.rtcp.RtcpListener
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.rtp.rtcp.RtcpPacket
import java.time.Duration
import java.time.Instant
abstract class TransportCcEngine : RtcpListener {
protected val lossListeners = mutableListOf<LossListener>()
/**
* Called when an RTP sender has a new round-trip time estimate.
*/
abstract fun onRttUpdate(rtt: Duration)
abstract override fun rtcpPacketReceived(rtcpPacket: RtcpPacket, receivedTime: Instant?)
/** This is called when a tccSeqNum is first assigned to a packet, i.e. the soonest the packet can meaningfully
* be described to the engine. */
abstract fun mediaPacketTagged(packetInfo: PacketInfo, tccSeqNum: Long)
abstract fun mediaPacketSent(packetInfo: PacketInfo, tccSeqNum: Long)
abstract fun getStatistics(): StatisticsSnapshot
/**
* Adds a loss listener to be notified about packet arrival and loss reports.
* @param listener
*/
@Synchronized
fun addLossListener(listener: LossListener) {
lossListeners.add(listener)
}
/**
* Removes a loss listener.
* @param listener
*/
@Synchronized
fun removeLossListener(listener: LossListener) {
lossListeners.remove(listener)
}
abstract fun addBandwidthListener(listener: BandwidthListener)
abstract fun removeBandwidthListener(listener: BandwidthListener)
abstract fun start()
abstract fun stop()
abstract class StatisticsSnapshot {
abstract fun toJson(): ObjectNode
/** Cumulative count of transport-cc feedback reports that could not be matched against
* the send-time history. A sustained flood of these means the estimator has lost both
* its loss and delay signals (unmatched reports are dropped entirely) and is the
* earliest observable marker of a feedback-starvation saturation lock. */
open val unmatchedFeedback: Long get() = 0
}
interface BandwidthListener {
fun bandwidthEstimationChanged(newValue: Bandwidth)
}
}

44
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp
import org.jitsi.nlj.RtpLayerDesc
import org.jitsi.rtp.rtp.RtpPacket
/**
* A packet which we know contains video, but we've not yet
* parsed (i.e. we don't know information gained from
* parsing codec-specific data).
*/
open class VideoRtpPacket @JvmOverloads constructor(
buffer: ByteArray,
offset: Int,
length: Int,
/** The encoding ID of this packet. */
var encodingId: Int = RtpLayerDesc.SUSPENDED_ENCODING_ID
) : RtpPacket(buffer, offset, length) {
open val layerIds: Collection<Int> = listOf(0)
override fun toString(): String = super.toString() + ", EncID=$encodingId"
override fun clone(): VideoRtpPacket = VideoRtpPacket(
cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET),
BYTES_TO_LEAVE_AT_START_OF_PACKET,
length,
encodingId = encodingId
).also { postClone(it) }
}

353
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimator.kt Normal file
View File

@@ -0,0 +1,353 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bps
import org.jitsi.utils.NEVER
import org.jitsi.utils.formatMilli
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.LinkedList
/**
* An abstract interface to a bandwidth estimation algorithm.
*
* The invoker of the algorithm will periodically call [processPacketArrival]
* and/or [processPacketLoss] as it learns information about packets
* that have traversed the network.
*
* All bandwidths/bitrates are in bits per second.
*/
abstract class BandwidthEstimator(
protected val diagnosticContext: DiagnosticContext
) {
/**
* The [TimeSeriesLogger] to be used by this instance to print time
* series.
*/
protected val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass)
/** The name of the algorithm implemented by this [BandwidthEstimator]. */
abstract val algorithmName: String
/** The initial bandwidth estimate. */
abstract var initBw: Bandwidth
/** The minimum bandwidth the estimator will return. */
abstract var minBw: Bandwidth
/** The maximum bandwidth the estimator will return. */
abstract var maxBw: Bandwidth
/**
* Inform the bandwidth estimator about a packet that has arrived at its
* destination.
*
* This function will be called at most once for any value of [seq] (up to cycles);
* however, it may be called after a call to [processPacketLoss] for the
* same [seq] value, if a packet is delayed.
*
* It is possible (e.g., if feedback was lost) that neither
* [processPacketArrival] nor [processPacketLoss] is called for a given [seq].
*
* The clocks reported by [now], [sendTime], and [recvTime] do not
* necessarily have any relationship to each other, but must be consistent
* within themselves across all calls to functions of this [BandwidthEstimator].
*
* All arrival and loss reports based on a single feedback message should have the
* same [now] value. [feedbackComplete] should be called once all feedback reports
* based on a single feedback message have been processed.
*
* @param[now] The current time, when this function is called.
* @param[sendTime] The time the packet was sent, if known, or null.
* @param[recvTime] The time the packet was received, if known, or null.
* @param[seq] A 16-bit sequence number of packets processed by this
* [BandwidthEstimator].
* @param[size] The size of the packet.
* @param[ecn] The ECN markings with which the packet was received.
* @param[previouslyReportedLost] Whether [processPacketLoss] was previously
* called for this packet.
*/
fun processPacketArrival(
now: Instant,
sendTime: Instant?,
recvTime: Instant?,
seq: Int,
size: DataSize,
ecn: Byte = 0,
previouslyReportedLost: Boolean = false
) {
if (timeSeriesLogger.isTraceEnabled) {
val point = diagnosticContext.makeTimeSeriesPoint("bwe_packet_arrival", now)
if (sendTime != null) {
point.addField("sendTime", sendTime.formatMilli())
}
if (recvTime != null) {
point.addField("recvTime", recvTime.formatMilli())
}
point.addField("seq", seq)
point.addField("size", size.bytes)
if (ecn != 0.toByte()) {
point.addField("ecn", ecn)
}
point.addField("previouslyReportedLost", previouslyReportedLost)
timeSeriesLogger.trace(point)
}
doProcessPacketArrival(now, sendTime, recvTime, seq, size, ecn, previouslyReportedLost)
}
/**
* A subclass's implementation of [processPacketArrival].
*
* See that function for parameter details.
*/
protected abstract fun doProcessPacketArrival(
now: Instant,
sendTime: Instant?,
recvTime: Instant?,
seq: Int,
size: DataSize,
ecn: Byte = 0,
previouslyReportedLost: Boolean = false
)
/**
* Inform the bandwidth estimator that a packet was lost.
*
* All arrival and loss reports based on a single feedback message should have the
* same [now] value. [feedbackComplete] should be called once all feedback reports
* based on a single feedback message have been processed.
*
* @param[now] The current time, when this function is called.
* @param[sendTime] The time the packet was sent, if known, or null.
* @param[seq] A 16-bit sequence number of packets processed by this
* [BandwidthEstimator].
*/
fun processPacketLoss(now: Instant, sendTime: Instant?, seq: Int) {
if (timeSeriesLogger.isTraceEnabled) {
val point = diagnosticContext.makeTimeSeriesPoint("bwe_packet_loss", now)
if (sendTime != null) {
point.addField("sendTime", sendTime.formatMilli())
}
point.addField("seq", seq)
timeSeriesLogger.trace(point)
}
doProcessPacketLoss(now, sendTime, seq)
}
/**
* A subclass's implementation of [processPacketLoss].
*
* See that function for parameter details.
*/
protected abstract fun doProcessPacketLoss(now: Instant, sendTime: Instant?, seq: Int)
/**
* Inform the bandwidth estimator that a block of feedback is complete.
*
* @param[now] The current time, when this function is called. This should match
* the value passed to [processPacketArrival] and [processPacketLoss].
*/
fun feedbackComplete(now: Instant) {
if (timeSeriesLogger.isTraceEnabled) {
val point = diagnosticContext.makeTimeSeriesPoint("bwe_feedback_complete", now)
timeSeriesLogger.trace(point)
}
doFeedbackComplete(now)
}
/**
* A subclass's implementation of [feedbackComplete].
*
* See that function for parameter details.
*/
protected abstract fun doFeedbackComplete(now: Instant)
/**
* Inform the bandwidth estimator about a new round-trip time value
*/
fun onRttUpdate(now: Instant, newRtt: Duration) {
if (timeSeriesLogger.isTraceEnabled) {
val point = diagnosticContext.makeTimeSeriesPoint("bwe_rtt", now)
point.addField("rtt", newRtt.formatMilli())
timeSeriesLogger.trace(point)
}
doRttUpdate(now, newRtt)
}
/**
* A subclass's implementation of [onRttUpdate].
*
* See that function for parameter details.
*/
protected abstract fun doRttUpdate(now: Instant, newRtt: Duration)
/** Get the estimator's current estimate of the available bandwidth.
*
* @param[now] The current time, when this function is called.
*/
abstract fun getCurrentBw(now: Instant): Bandwidth
/** Get the current statistics related to this estimator.
*
* @param[now] The current time, when this function is called.
*/
abstract fun getStats(now: Instant = Clock.systemUTC().instant()): StatisticsSnapshot
/** Reset the estimator to its initial state. */
abstract fun reset()
private val listeners = LinkedList<TransportCcEngine.BandwidthListener>()
private var curBandwidth = (-1).bps
private var lastBweLogTime = NEVER
private val minBweLogInterval = Duration.ofMillis(500)
/**
* Notifies registered listeners that the estimate of the available
* bandwidth has changed.
*/
@Synchronized
protected fun reportBandwidthEstimate(now: Instant, newValue: Bandwidth) {
if (timeSeriesLogger.isTraceEnabled) {
if (newValue != curBandwidth ||
lastBweLogTime == NEVER ||
Duration.between(lastBweLogTime, now) >= minBweLogInterval
) {
val point = diagnosticContext.makeTimeSeriesPoint("bwe_estimate", now)
point.addField("bw", newValue.bps)
timeSeriesLogger.trace(point)
lastBweLogTime = now
}
}
if (newValue == curBandwidth) {
return
}
for (listener in listeners) {
listener.bandwidthEstimationChanged(newValue)
}
curBandwidth = newValue
}
/**
* Adds a listener to be notified about changes to the bandwidth estimation.
* @param listener
*/
@Synchronized
fun addListener(listener: TransportCcEngine.BandwidthListener) {
listeners.add(listener)
}
/**
* Removes a listener.
* @param listener
*/
@Synchronized
fun removeListener(listener: TransportCcEngine.BandwidthListener) {
listeners.remove(listener)
}
/**
* Holds a snapshot of stats specific to the bandwidth estimator.
*/
class StatisticsSnapshot(algorithmName: String, currentEstimate: Bandwidth) {
private val stats = mutableMapOf<String, Any>()
var algorithmName: String by stats
var currentEstimate: Bandwidth by stats
init {
addString("algorithmName", algorithmName)
addBandwidth("currentEstimate", currentEstimate)
}
fun getValue(name: String): Any? = stats[name]
/**
* Gets the value of a stat with a given name, if this [StatisticsSnapshot] has it and it is a [Number].
* Otherwise returns 'null'.
*/
fun getNumber(name: String): Number? = stats[name] as? Number
/**
* Promotes integer values to [Long] and floating point values to [Double]. Returns a
* [Long], [Double], or null.
*/
private fun promote(n: Number): Number? = when (n) {
is Byte, is Short, is Int, is Long -> n.toLong()
is Float, is Double -> n.toDouble()
else -> null
}
/**
* Adds a stat with a number value. Integral values are promoted to [Long], while floating point values are
* promoted to [Double].
*/
fun addNumber(name: String, value: Number) {
promote(value)?.let { stats[name] = it }
}
/**
* Adds a stat with a string value.
*/
fun addString(name: String, value: String) {
stats[name] = value
}
/**
* Adds a stat with a boolean value.
*/
fun addBoolean(name: String, value: Boolean) {
stats[name] = value
}
/**
* Adds a stat with a bandwidth value.
*/
fun addBandwidth(name: String, value: Bandwidth) {
stats[name] = value
}
/**
* Returns a JSON representation of this [StatisticsSnapshot] object.
*/
fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
stats.forEach { (name, value) ->
when (value) {
is Bandwidth -> put(name, value.bps)
is Long -> put(name, value)
is Int -> put(name, value)
is Double -> put(name, value)
is Boolean -> put(name, value)
else -> put(name, value.toString())
}
}
}
}
}

38
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimatorConfig.kt Normal file
View File

@@ -0,0 +1,38 @@
package org.jitsi.nlj.rtp.bandwidthestimation
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.nlj.util.Bandwidth
class BandwidthEstimatorConfig {
companion object {
val engine: BandwidthEstimatorEngine by config {
"jmt.bwe.estimator.engine".from(JitsiConfig.newConfig)
.convertFrom<String> { BandwidthEstimatorEngine.valueOf(it) }
}
val initBw: Bandwidth by config {
"jmt.bwe.estimator.initial-bw".from(JitsiConfig.newConfig)
.convertFrom<String> { Bandwidth.fromString(it) }
}
val minBw: Bandwidth by config {
"jmt.bwe.google-cc.min-bw".from(JitsiConfig.newConfig)
.convertFrom<String> { Bandwidth.fromString(it) }.softDeprecated("use jmt.bwe.estimator.min-bw")
"jmt.bwe.estimator.min-bw".from(JitsiConfig.newConfig)
.convertFrom<String> { Bandwidth.fromString(it) }
}
val maxBw: Bandwidth by config {
"jmt.bwe.google-cc.max-bw".from(JitsiConfig.newConfig)
.convertFrom<String> { Bandwidth.fromString(it) }.softDeprecated("use jmt.bwe.estimator.max-bw")
"jmt.bwe.estimator.max-bw".from(JitsiConfig.newConfig)
.convertFrom<String> { Bandwidth.fromString(it) }
}
}
}
enum class BandwidthEstimatorEngine {
GoogleCc,
GoogleCc2
}

146
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt Normal file
View File

@@ -0,0 +1,146 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bps
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.RemoteBitrateEstimatorAbsSendTime
import org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation.SendSideBandwidthEstimation
import java.time.Duration
import java.time.Instant
import kotlin.properties.Delegates
class GoogleCcEstimator(diagnosticContext: DiagnosticContext, parentLogger: Logger) :
BandwidthEstimator(diagnosticContext) {
override val algorithmName = "Google CC"
/* TODO: Use configuration service to set this default value. */
override var initBw: Bandwidth = BandwidthEstimatorConfig.initBw
/* TODO: observable which sets the components' values if we're in initial state. */
override var minBw: Bandwidth by Delegates.observable(BandwidthEstimatorConfig.minBw) { _, _, newValue ->
bitrateEstimatorAbsSendTime.setMinBitrate(newValue.bps.toInt())
sendSideBandwidthEstimation.setMinMaxBitrate(newValue.bps.toInt(), maxBw.bps.toInt())
}
override var maxBw: Bandwidth by Delegates.observable(BandwidthEstimatorConfig.maxBw) { _, _, newValue ->
sendSideBandwidthEstimation.setMinMaxBitrate(minBw.bps.toInt(), newValue.bps.toInt())
}
private val logger = createChildLogger(parentLogger)
/**
* Implements the delay-based part of Google CC.
*/
private val bitrateEstimatorAbsSendTime = RemoteBitrateEstimatorAbsSendTime(diagnosticContext, logger).also {
it.setMinBitrate(minBw.bps.toInt())
}
/**
* Implements the loss-based part of Google CC.
*/
private val sendSideBandwidthEstimation =
SendSideBandwidthEstimation(diagnosticContext, initBw.bps, logger).also {
it.setMinMaxBitrate(minBw.bps.toInt(), maxBw.bps.toInt())
}
override fun doProcessPacketArrival(
now: Instant,
sendTime: Instant?,
recvTime: Instant?,
seq: Int,
size: DataSize,
ecn: Byte,
previouslyReportedLost: Boolean
) {
if (sendTime != null && recvTime != null) {
bitrateEstimatorAbsSendTime.incomingPacketInfo(
now.toEpochMilli(),
sendTime.toEpochMilli(),
recvTime.toEpochMilli(),
size.bytes.toInt()
)
}
sendSideBandwidthEstimation.updateReceiverEstimate(bitrateEstimatorAbsSendTime.latestEstimate)
sendSideBandwidthEstimation.reportPacketArrived(now.toEpochMilli(), previouslyReportedLost)
}
override fun doProcessPacketLoss(now: Instant, sendTime: Instant?, seq: Int) {
sendSideBandwidthEstimation.reportPacketLost(now.toEpochMilli())
}
override fun doFeedbackComplete(now: Instant) {
/* TODO: rate-limit how often we call updateEstimate? */
sendSideBandwidthEstimation.updateEstimate(now.toEpochMilli())
reportBandwidthEstimate(now, sendSideBandwidthEstimation.latestEstimate.bps)
}
override fun doRttUpdate(now: Instant, newRtt: Duration) {
bitrateEstimatorAbsSendTime.onRttUpdate(now.toEpochMilli(), newRtt.toMillis())
sendSideBandwidthEstimation.onRttUpdate(newRtt)
}
override fun getCurrentBw(now: Instant): Bandwidth = sendSideBandwidthEstimation.latestEstimate.bps
override fun getStats(now: Instant): StatisticsSnapshot = StatisticsSnapshot(
"GoogleCcEstimator",
getCurrentBw(now)
).apply {
bitrateEstimatorAbsSendTime.statistics?.run {
addNumber("delayBasedEstimatorOffset", offset)
addNumber("delayBasedEstimatorThreshold", threshold)
addNumber("delayBasedEstimatorHypothesis", hypothesis.value)
}
addNumber("latestDelayBasedEstimate", sendSideBandwidthEstimation.latestREMB)
addNumber("latestLossFraction", sendSideBandwidthEstimation.latestFractionLoss / 256.0)
with(sendSideBandwidthEstimation.statistics) {
update(now.toEpochMilli())
addNumber("lossDegradedMs", lossDegradedMs)
addNumber("lossFreeMs", lossFreeMs)
addNumber("lossLimitedMs", lossLimitedMs)
}
}
override fun reset() {
initBw = BandwidthEstimatorConfig.initBw
minBw = BandwidthEstimatorConfig.minBw
maxBw = BandwidthEstimatorConfig.maxBw
bitrateEstimatorAbsSendTime.reset()
sendSideBandwidthEstimation.reset(initBw.bps)
sendSideBandwidthEstimation.setMinMaxBitrate(minBw.bps.toInt(), maxBw.bps.toInt())
}
companion object {
/* Default config settings to use when the classic Google CC estimator engine is used. */
val defaultRateTrackerWindowSize: Duration by config {
"jmt.bwe.estimator.GoogleCc.default-window-size".from(JitsiConfig.newConfig)
}
val defaultRateTrackerBucketSize: Duration by config {
"jmt.bwe.estimator.GoogleCc.default-bucket-size".from(JitsiConfig.newConfig)
}
val defaultInitialIgnoreBwePeriod: Duration by config {
"jmt.bwe.estimator.GoogleCc.default-initial-ignore-bwe-period".from(JitsiConfig.newConfig)
}
}
}

62
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimator.kt Normal file
View File

@@ -0,0 +1,62 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import java.time.Instant
/**
* Basic implementation to estimate acknowledged bitrate.
*
* Based on WebRTC modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
class AcknowledgedBitrateEstimator(
val bitrateEstimator: BitrateEstimator
) : AcknowledgedBitrateEstimatorInterface {
private var alrEndedTime: Instant? = null
private var inAlr = false
constructor() : this(BitrateEstimator())
override fun incomingPacketFeedbackVector(packetFeedbackVector: List<PacketResult>) {
check(packetFeedbackVector.isSortedByReceiveTime())
for (packet in packetFeedbackVector) {
if (alrEndedTime != null && packet.sentPacket.sendTime > alrEndedTime) {
bitrateEstimator.expectFastRateChange()
alrEndedTime = null
}
val acknowledgedEstimate = packet.sentPacket.size + packet.sentPacket.priorUnackedData
bitrateEstimator.update(packet.receiveTime, acknowledgedEstimate, inAlr)
}
}
override fun bitrate(): Bandwidth? = bitrateEstimator.bitrate()
override fun peekRate(): Bandwidth? = bitrateEstimator.peekRate()
override fun setAlr(inAlr: Boolean) {
this.inAlr = inAlr
}
override fun setAlrEndedTime(alrEndedTime: Instant) {
this.alrEndedTime = alrEndedTime
}
}
private fun List<PacketResult>.isSortedByReceiveTime(): Boolean = this.asSequence().zipWithNext { a, b ->
a.receiveTime <= b.receiveTime
}.all { it }

82
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorInterface.kt Normal file
View File

@@ -0,0 +1,82 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.utils.secs
import java.time.Duration
import java.time.Instant
/**
* Interface to estimate acknowledged bitrate.
* *
* Based on WebRTC modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_interface.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
interface AcknowledgedBitrateEstimatorInterface {
fun incomingPacketFeedbackVector(packetFeedbackVector: List<PacketResult>)
fun bitrate(): Bandwidth?
fun peekRate(): Bandwidth?
fun setAlr(inAlr: Boolean)
fun setAlrEndedTime(alrEndedTime: Instant)
companion object {
fun create(
settings: RobustThroughputEstimatorSettings = RobustThroughputEstimatorSettings()
): AcknowledgedBitrateEstimatorInterface {
if (settings.enabled) {
return RobustThroughputEstimator(settings)
} else {
return AcknowledgedBitrateEstimator()
}
}
}
}
data class RobustThroughputEstimatorSettings(
// Set `enabled` to true to use the RobustThroughputEstimator, false to use
// the AcknowledgedBitrateEstimator.
val enabled: Boolean = true,
// The estimator keeps the smallest window containing at least
// `window_packets` and at least the packets received during the last
// `min_window_duration` milliseconds.
// (This means that it may store more than `window_packets` at high bitrates,
// and a longer duration than `min_window_duration` at low bitrates.)
// However, if will never store more than kMaxPackets (for performance
// reasons), and never longer than max_window_duration (to avoid very old
// packets influencing the estimate for example when sending is paused).
val windowPackets: Int = 20,
val maxWindowPackets: Int = 500,
val minWindowDuration: Duration = 1.secs,
val maxWindowDuration: Duration = 5.secs,
// The estimator window requires at least `required_packets` packets
// to produce an estimate.
val requiredPackets: Int = 10,
// If audio packets aren't included in allocation (i.e. the
// estimated available bandwidth is divided only among the video
// streams), then `unacked_weight` should be set to 0.
// If audio packets are included in allocation, but not in bandwidth
// estimation (i.e. they don't have transport-wide sequence numbers,
// but we nevertheless divide the estimated available bandwidth among
// both audio and video streams), then `unacked_weight` should be set to 1.
// If all packets have transport-wide sequence numbers, then the value
// of `unacked_weight` doesn't matter.
val unackedWeight: Double = 1.0
)

361
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControl.kt Normal file
View File

@@ -0,0 +1,361 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file uses WebRTC's naming style for enums
@file:Suppress("ktlint:standard:enum-entry-name-case", "ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bps
import org.jitsi.nlj.util.bytes
import org.jitsi.nlj.util.div
import org.jitsi.nlj.util.kbps
import org.jitsi.nlj.util.max
import org.jitsi.nlj.util.per
import org.jitsi.nlj.util.times
import org.jitsi.utils.NEVER
import org.jitsi.utils.coerceIn
import org.jitsi.utils.div
import org.jitsi.utils.durationOfDoubleSeconds
import org.jitsi.utils.max
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import org.jitsi.utils.times
import org.jitsi.utils.toDouble
import java.time.Duration
import java.time.Instant
import kotlin.math.ceil
import kotlin.math.min
import kotlin.math.pow
/**
* A rate control implementation based on additive increases of bitrate when no
* over-use is detected and multiplicative decreases when over-uses are
* detected. When we think the available bandwidth has changes or is unknown, we
* will switch to a "slow-start mode" where we increase multiplicatively.
*
* Based on WebRTC modules/remote_bitrate_estimator/aimd_rate_control.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
class AimdRateControl(private val sendSide: Boolean = false) {
private var minConfiguredBitrate: Bandwidth = kCongestionControllerMinBitrate
private var maxConfiguredBitrate: Bandwidth = 30000.kbps
private var currentBitrate: Bandwidth = maxConfiguredBitrate
private var latestEstimatedThroughput: Bandwidth = currentBitrate
val linkCapacity = LinkCapacityEstimator()
// Omitted:
// val networkEstimate: NetworkStateEstimate? = null
// As far as I can tell it is not ever set in current Chromium?
var rateControlState: RateControlState = RateControlState.kRcHold
private set
private var timeLastBitrateChange: Instant = NEVER
@SuppressFBWarnings("URF_UNREAD_FIELD")
private var timeLastBitrateDecrease: Instant = NEVER
private var timeFirstThroughputEstimate: Instant = NEVER
private var bitrateIsInitialized: Boolean = false
private var beta: Double = kDefaultBackoffFactor
var inAlr: Boolean = false
var rtt: Duration = kDefaultRtt
// TODO: field trials: remove code that checks these
// Allow the delay based estimate to only increase as long as application
// limited region (alr) is not detected.
private val noBitrateIncreaseInAlr: Boolean = false
// Only tested when networkEstimate != null
// private val disableEstimateBoundedIncrease: Boolean = false
// private val useCurrentEstimateAsMinUpperBound = true
private var lastDecrease: Bandwidth? = null
fun setStartBitrate(startBitrate: Bandwidth) {
currentBitrate = startBitrate
latestEstimatedThroughput = startBitrate
bitrateIsInitialized = true
}
fun setMinBitrate(minBitrate: Bandwidth) {
minConfiguredBitrate = minBitrate
currentBitrate = max(minBitrate, currentBitrate)
}
fun validEstimate(): Boolean = bitrateIsInitialized
fun getFeedbackInterval(): Duration {
// Estimate how often we can send RTCP if we allocate up to 5% of bandwidth
// to feedback.
val kRtcpSize = 80.bytes
val rtcpBitrate = currentBitrate * 0.05
val interval = kRtcpSize / rtcpBitrate
val kMinFeedbackInterval = 200.ms
val kMaxFeedbackInterval = 1000.ms
return interval.coerceIn(kMinFeedbackInterval, kMaxFeedbackInterval)
}
fun timeToReduceFurther(atTime: Instant, estimatedThroughput: Bandwidth): Boolean {
val bitrateReductionInterval = rtt.coerceIn(10.ms, 200.ms)
if (Duration.between(timeLastBitrateChange, atTime) >= bitrateReductionInterval) {
return true
}
if (validEstimate()) {
// TODO(terelius/holmer): Investigate consequences of increasing
// the threshold to 0.95 * LatestEstimate().
val threshold = 0.5 * latestEstimate()
return estimatedThroughput < threshold
}
return false
}
fun initialTimeToReduceFurther(atTime: Instant): Boolean = validEstimate() &&
timeToReduceFurther(atTime, latestEstimate() / 2 - 1.bps)
fun latestEstimate() = currentBitrate
fun update(input: RateControlInput, atTime: Instant): Bandwidth {
// Set the initial bit rate value to what we're receiving the first half
// second.
// TODO(bugs.webrtc.org/9379): The comment above doesn't match to the code.
if (!bitrateIsInitialized) {
val kInitializionTime = 5.secs
check(kBitrateWindow <= kInitializionTime)
if (timeFirstThroughputEstimate == NEVER) {
if (input.estimatedThroughput != null) {
timeFirstThroughputEstimate = atTime
}
} else if (Duration.between(timeFirstThroughputEstimate, atTime) > kInitializionTime) {
input.estimatedThroughput?.let {
currentBitrate = it
bitrateIsInitialized = true
}
}
}
changeBitrate(input, atTime)
return currentBitrate
}
fun setEstimate(bitrate: Bandwidth, atTime: Instant) {
bitrateIsInitialized = true
val prevBitrate = currentBitrate
currentBitrate = clampBitrate(bitrate)
timeLastBitrateChange = atTime
if (currentBitrate < prevBitrate) {
timeLastBitrateDecrease = atTime
}
}
fun getNearMaxIncreaseRateBpsPerSecond(): Double {
check(currentBitrate != 0.bps)
val kFrameInterval = 1.secs / 30.0
val frameSize = (currentBitrate * kFrameInterval).toWholeBytes()
val kPacketSize = 1200.bytes
val packetsPerFrame = ceil(frameSize / kPacketSize)
val avgPacketSize = (frameSize / packetsPerFrame).toWholeBytes()
// Approximate the over-use estimator delay to 100 ms.
var responseTime = rtt + 100.ms
responseTime = responseTime * 2
val increaseRateBpsPerSecond = (avgPacketSize.per(responseTime)).bps.toDouble()
val kMinIncreaseRateBpsPerSecond = 4000.0
return increaseRateBpsPerSecond.coerceAtLeast(kMinIncreaseRateBpsPerSecond)
}
fun getExpectedBandwidthPeriod(): Duration {
val kMinPeriod = 2.secs
val kDefaultPeriod = 3.secs
val kMaxPeriod = 50.secs
val increaseRateBpsPerSecond = getNearMaxIncreaseRateBpsPerSecond()
if (lastDecrease == null) {
return kDefaultPeriod
}
val timeToRecoverDecreaseSeconds =
lastDecrease!!.bps / increaseRateBpsPerSecond
val period = durationOfDoubleSeconds(timeToRecoverDecreaseSeconds)
return period.coerceIn(kMinPeriod, kMaxPeriod)
}
private fun changeBitrate(input: RateControlInput, atTime: Instant) {
var newBitrate: Bandwidth? = null
val estimatedThroughput = input.estimatedThroughput ?: latestEstimatedThroughput
input.estimatedThroughput?.let { latestEstimatedThroughput = it }
// An over-use should always trigger us to reduce the bitrate, even though
// we have not yet established our first estimate. By acting on the over-use,
// we will end up with a valid estimate.
if (!bitrateIsInitialized && input.bwState != BandwidthUsage.kBwOverusing) {
return
}
changeState(input, atTime)
when (rateControlState) {
RateControlState.kRcHold -> Unit
RateControlState.kRcIncrease -> {
if (estimatedThroughput > linkCapacity.upperBound()) {
linkCapacity.reset()
}
// We limit the new bitrate based on the troughput to avoid unlimited
// bitrate increases. We allow a bit more lag at very low rates to not too
// easily get stuck if the encoder produces uneven outputs.
var increaseLimit = 1.5 * estimatedThroughput + 10.kbps
if (sendSide && inAlr && noBitrateIncreaseInAlr) {
// TODO this is dead code because we're not using the noBitrateIncreaseInAlr field trial
// Do not increase the delay based estimate in alr since the estimator
// will not be able to get transport feedback necessary to detect if
// the new estimate is correct.
// If we have previously increased above the limit (for instance due to
// probing), we don't allow further changes.
increaseLimit = currentBitrate
}
if (currentBitrate < increaseLimit) {
var increasedBitrate = Bandwidth.MINUS_INFINITY
if (linkCapacity.hasEstimate()) {
// The link_capacity estimate is reset if the measured throughput
// is too far from the estimate. We can therefore assume that our
// target rate is reasonably close to link capacity and use additive
// increase.
val additiveIncrease =
additiveRateIncrease(atTime, timeLastBitrateChange)
increasedBitrate = currentBitrate + additiveIncrease
} else {
// If we don't have an estimate of the link capacity, use faster ramp
// up to discover the capacity.
val multiplicativeIncrease = multiplicativeRateIncrease(
atTime,
timeLastBitrateChange,
currentBitrate
)
increasedBitrate = currentBitrate + multiplicativeIncrease
}
newBitrate = increasedBitrate.coerceAtMost(increaseLimit)
}
timeLastBitrateChange = atTime
}
RateControlState.kRcDecrease -> {
var decreasedBitrate = Bandwidth.INFINITY
// Set bit rate to something slightly lower than the measured throughput
// to get rid of any self-induced delay.
decreasedBitrate = estimatedThroughput * beta
if (decreasedBitrate > 5.kbps) {
decreasedBitrate -= 5.kbps
}
if (decreasedBitrate > currentBitrate) {
// TODO(terelius): The link_capacity estimate may be based on old
// throughput measurements. Relying on them may lead to unnecessary
// BWE drops.
if (linkCapacity.hasEstimate()) {
decreasedBitrate = linkCapacity.estimate!! * beta
}
}
// Avoid increasing the rate when over-using.
if (decreasedBitrate < currentBitrate) {
newBitrate = decreasedBitrate
}
if (bitrateIsInitialized && estimatedThroughput < currentBitrate) {
if (newBitrate == null) {
lastDecrease = Bandwidth.ZERO
} else {
lastDecrease = currentBitrate - newBitrate
}
}
if (estimatedThroughput < linkCapacity.lowerBound()) {
// The current throughput is far from the estimated link capacity. Clear
// the estimate to allow an immediate update in onOveruseDetected.
linkCapacity.reset()
}
bitrateIsInitialized = true
linkCapacity.onOveruseDetected(estimatedThroughput)
// Stay on hold until the pipes are cleared.
rateControlState = RateControlState.kRcHold
timeLastBitrateChange = atTime
timeLastBitrateDecrease = atTime
}
}
currentBitrate = clampBitrate(newBitrate ?: currentBitrate)
}
private fun clampBitrate(newBitrate: Bandwidth): Bandwidth {
/* Skipping some conditions related to networkEstimate != null */
return newBitrate.coerceAtLeast(minConfiguredBitrate)
}
private fun multiplicativeRateIncrease(atTime: Instant, lastTime: Instant, currentBitrate: Bandwidth): Bandwidth {
var alpha = 1.08
if (lastTime != NEVER) {
val timeSinceLastUpdate = Duration.between(lastTime, atTime)
alpha = alpha.pow(min(timeSinceLastUpdate.toDouble(), 1.0))
}
val multiplicativeIncrease =
max(currentBitrate * (alpha - 1.0), 1000.bps)
return multiplicativeIncrease
}
private fun additiveRateIncrease(atTime: Instant, lastTime: Instant): Bandwidth {
val timePeriodSeconds = Duration.between(lastTime, atTime).toDouble()
val dataRateIncreaseBps = getNearMaxIncreaseRateBpsPerSecond() * timePeriodSeconds
return dataRateIncreaseBps.bps
}
private fun changeState(input: RateControlInput, atTime: Instant) {
when (input.bwState) {
BandwidthUsage.kBwNormal ->
if (rateControlState == RateControlState.kRcHold) {
timeLastBitrateChange = atTime
rateControlState = RateControlState.kRcIncrease
}
BandwidthUsage.kBwOverusing ->
if (rateControlState != RateControlState.kRcDecrease) {
rateControlState = RateControlState.kRcDecrease
}
BandwidthUsage.kBwUnderusing ->
rateControlState = RateControlState.kRcHold
}
}
enum class RateControlState {
kRcHold,
kRcIncrease,
kRcDecrease
}
companion object {
private val kDefaultRtt = 200.ms
private const val kDefaultBackoffFactor = 0.85
}
}

101
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetector.kt Normal file
View File

@@ -0,0 +1,101 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
/**
* An Application-limited region detector.
*
* Based on WebRTC modules/congestion_controller/goog_cc/alr_detector.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings.
*/
data class AlrDetectorConfig(
// Sent traffic ratio as a function of network capacity used to determine
// application-limited region. ALR region start when bandwidth usage drops
// below kAlrStartUsageRatio and ends when it raises above
// kAlrEndUsageRatio. NOTE: This is intentionally conservative at the moment
// until BW adjustments of application limited region is fine tuned.
val bandwidthUsageRatio: Double = 0.65,
val startBudgetLevelRatio: Double = 0.80,
val stopBudgetLevelRatio: Double = 0.50
)
/** Application limited region detector is a class that utilizes signals of
elapsed time and bytes sent to estimate whether network traffic is
currently limited by the application's ability to generate traffic.
AlrDetector provides a signal that can be utilized to adjust
estimate bandwidth.
Note: This class is not thread-safe.
*/
class AlrDetector {
fun onBytesSent(bytesSent: Long, sendTimeMs: Long) {
if (lastSendTimeMs == null) {
lastSendTimeMs = sendTimeMs
// Since the duration for sending the bytes is unknwon, return without
// updating alr state.
return
}
val deltaTimeMs = sendTimeMs - lastSendTimeMs!!
lastSendTimeMs = sendTimeMs
alrBudget.useBudget(bytesSent)
alrBudget.increaseBudget(deltaTimeMs)
var stateChanged = false
if (alrBudget.budgetRatio() > conf.startBudgetLevelRatio &&
alrStartedTimeMs == null
) {
alrStartedTimeMs = System.currentTimeMillis()
stateChanged = true
} else if (alrBudget.budgetRatio() < conf.startBudgetLevelRatio &&
alrStartedTimeMs != null
) {
stateChanged = true
alrStartedTimeMs = null
}
if (stateChanged) {
// TODO? Log event
}
}
// Set current estimated bandwidth.
fun setEstimatedBitrate(bitrateBps: Int) {
val targetRateKbps = bitrateBps.toDouble() * conf.bandwidthUsageRatio / 1000
alrBudget.setTargetRateKbps(targetRateKbps.toInt())
}
// Returns time in milliseconds when the current application-limited region
// started or empty result if the sender is currently not application-limited.
fun getApplicationLimitedRegionStartTime(): Long? = alrStartedTimeMs
private val conf = kDefaultProbingScreenshareBweSettings
private var lastSendTimeMs: Long? = null
private val alrBudget = IntervalBudget(0, true)
private var alrStartedTimeMs: Long? = null
companion object {
/* From rtc_base/experiments/alr_experiment.cc */
val kDefaultProbingScreenshareBweSettings = AlrDetectorConfig(
bandwidthUsageRatio = 0.80,
startBudgetLevelRatio = 0.40,
stopBudgetLevelRatio = -0.60
)
}
}

154
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateEstimator.kt Normal file
View File

@@ -0,0 +1,154 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bytes
import org.jitsi.nlj.util.kbps
import org.jitsi.nlj.util.per
import org.jitsi.utils.ms
import org.jitsi.utils.toRoundedEpochMilli
import java.time.Instant
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min
import kotlin.math.roundToInt
/**
* Computes a bayesian estimate of the throughput given acks containing
* the arrival time and payload size. Samples which are far from the current
* estimate or are based on few packets are given a smaller weight, as they
* are considered to be more likely to have been caused by, e.g., delay spikes
* unrelated to congestion.
*
* Based on WebRTC modules/congestion_controller/goog_cc/bitrate_estimator.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
open class BitrateEstimator {
private var sum = 0
private val initialWindowMs = kInitialRateWindowMs
private val noninitialWindowMs = kRateWindowMs
private val uncertaintyScale = 10.0f
private val uncertaintyScaleInAlr = uncertaintyScale
private val smallSampleUncertaintyScale = uncertaintyScale
private val smallSampleThreshold = DataSize.ZERO
private val uncertaintySymmetryCap = Bandwidth.ZERO
private val estimateFloor = Bandwidth.ZERO
private var currentWindowMs = 0L
private var prevTimeMs = -1L
private var bitrateEstimateKbps = -1.0f
private var bitrateEstimateVar = 50.0f
open fun update(atTime: Instant, amount: DataSize, inAlr: Boolean) {
val rateWindowMs = if (bitrateEstimateKbps < 0.0f) initialWindowMs else noninitialWindowMs
val (bitrateSampleKbps, isSmallSample) =
updateWindow(atTime.toRoundedEpochMilli(), amount.bytes.roundToInt(), rateWindowMs)
if (bitrateSampleKbps < 0.0f) {
return
}
if (bitrateEstimateKbps < 0.0f) {
// This is the very first sample we get. Use it to initialize the estimate.
bitrateEstimateKbps = bitrateSampleKbps
return
}
// Optionally use higher uncertainty for very small samples to avoid dropping
// estimate and for samples obtained in ALR.
val scale = if (isSmallSample && bitrateSampleKbps < bitrateEstimateKbps) {
smallSampleUncertaintyScale
} else if (inAlr && bitrateSampleKbps < bitrateEstimateKbps) {
uncertaintyScaleInAlr
} else {
uncertaintyScale
}
// Define the sample uncertainty as a function of how far away it is from the
// current estimate. With low values of uncertaintySymmetryCap we add more
// uncertainty to increases than to decreases. For higher values we approach
// symmetry.
val sampleUncertainty =
scale * abs(bitrateEstimateKbps - bitrateSampleKbps) /
(
bitrateEstimateKbps +
min(bitrateSampleKbps, uncertaintySymmetryCap.kbps.toFloat())
)
val sampleVar = sampleUncertainty * sampleUncertainty
// Update a bayesian estimate of the rate, weighting it lower if the sample
// uncertainty is large.
// The bitrate estimate uncertainty is increased with each update to model
// that the bitrate changes over time.
val predBitrateEstimateVar = bitrateEstimateVar + 5.0f
bitrateEstimateKbps = (
sampleVar * bitrateEstimateKbps +
predBitrateEstimateVar * bitrateSampleKbps
) /
(sampleVar + predBitrateEstimateVar)
bitrateEstimateKbps = max(bitrateEstimateKbps, estimateFloor.kbps.toFloat())
bitrateEstimateVar = sampleVar * predBitrateEstimateVar /
(sampleVar + predBitrateEstimateVar)
}
private fun updateWindow(nowMs: Long, bytes: Int, rateWindowMs: Int): Pair<Float, Boolean> {
// Reset if time moves backwards.
if (nowMs < prevTimeMs) {
prevTimeMs = -1
sum = 0
currentWindowMs = 0
}
if (prevTimeMs >= 0) {
currentWindowMs += nowMs - prevTimeMs
// Reset if nothing has been received for more than a full window.
if (nowMs - prevTimeMs > rateWindowMs) {
sum = 0
currentWindowMs %= rateWindowMs
}
}
prevTimeMs = nowMs
val bitrateSample: Float
val isSmallSample: Boolean
if (currentWindowMs >= rateWindowMs) {
isSmallSample = sum < smallSampleThreshold.bytes
bitrateSample = 8.0f * sum / rateWindowMs.toFloat()
currentWindowMs -= rateWindowMs
sum = 0
} else {
isSmallSample = false
bitrateSample = -1.0f
}
sum += bytes
return Pair(bitrateSample, isSmallSample)
}
open fun bitrate(): Bandwidth? = if (bitrateEstimateKbps < 0.0f) null else bitrateEstimateKbps.kbps
fun peekRate(): Bandwidth? = if (currentWindowMs > 0) sum.bytes.per(currentWindowMs.ms) else null
open fun expectFastRateChange() {
// By setting the bitrate-estimate variance to a higher value we allow the
// bitrate to change fast for the next few samples.
bitrateEstimateVar += 200
}
companion object {
const val kInitialRateWindowMs = 500
const val kRateWindowMs = 150
const val kMinRateWindowMs = 150
const val kMaxRateWindowMs = 1000
}
}

274
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProber.kt Normal file
View File

@@ -0,0 +1,274 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:enum-entry-name-case")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bps
import org.jitsi.nlj.util.bytes
import org.jitsi.nlj.util.div
import org.jitsi.nlj.util.min
import org.jitsi.nlj.util.times
import org.jitsi.utils.isFinite
import org.jitsi.utils.isInfinite
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.min
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import java.time.Duration
import java.time.Instant
/** Bitrate prober,
* based on WebRTC modules/pacing/bitrate_prober.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
private val kProbeClusterTimeout = 5.secs
private val kMaxPendingProbeClusters = 5
data class BitrateProberConfig(
var maxProbeDelay: Duration = 20.ms,
var minPacketSize: DataSize = 200.bytes,
var allowStartProbingImmediately: Boolean = false
)
class BitrateProber(
parentLogger: Logger,
configIn: BitrateProberConfig = BitrateProberConfig()
) {
private val logger = createChildLogger(parentLogger)
fun setEnabled(enable: Boolean) {
if (enable) {
if (probingState == ProbingState.kDisabled) {
probingState = ProbingState.kInactive
logger.info("Bandwidth probing enabled, set to inactive")
}
} else {
probingState = ProbingState.kDisabled
logger.info("Bandwidth probing disabled")
}
}
fun setAllowProbeWithoutMediaPacket(allow: Boolean) {
config.allowStartProbingImmediately = allow
maybeSetActiveState(DataSize.ZERO)
}
// Returns true if the prober is in a probing session, i.e., it currently
// wants packets to be sent out according to the time returned by
// TimeUntilNextProbe().
fun isProbing(): Boolean = probingState == ProbingState.kActive
// Initializes a new probing session if the prober is allowed to probe. Does
// not initialize the prober unless the packet size is large enough to probe
// with.
fun onIncomingPacket(packetSize: DataSize) {
maybeSetActiveState(packetSize)
}
// Create a cluster used to probe.
fun createProbeCluster(clusterConfig: ProbeClusterConfig) {
check(probingState != ProbingState.kDisabled)
check(clusterConfig.minProbeDelta > Duration.ZERO)
while (clusters.isNotEmpty() && (
Duration.between(clusters.first().requestedAt, clusterConfig.atTime) > kProbeClusterTimeout ||
clusters.size > kMaxPendingProbeClusters
)
) {
clusters.removeFirst()
}
val cluster = ProbeCluster(
requestedAt = clusterConfig.atTime,
paceInfo = PacedPacketInfo(
probeClusterMinProbes = clusterConfig.targetProbeCount,
probeClusterMinBytes = (clusterConfig.targetDataRate * clusterConfig.targetDuration).bytes.toInt(),
probeClusterId = clusterConfig.id
),
minProbeDelta = clusterConfig.minProbeDelta
)
check(cluster.paceInfo.probeClusterMinBytes >= 0)
cluster.paceInfo.sendBitrate = clusterConfig.targetDataRate
clusters.addLast(cluster)
maybeSetActiveState(packetSize = DataSize.ZERO)
check(probingState == ProbingState.kActive || probingState == ProbingState.kInactive)
logger.info {
"Probe cluster (bitrate_bps:min bytes:min packets): (" +
"${cluster.paceInfo.sendBitrate}:${cluster.paceInfo.probeClusterMinBytes}:" +
"${cluster.paceInfo.probeClusterMinProbes}, $probingState)"
}
}
// Returns the time at which the next probe should be sent to get accurate
// probing. If probing is not desired at this time, [Instant.MAX]
// will be returned.
// TODO(bugs.webrtc.org/11780): Remove `now` argument when old mode is gone.
fun nextProbeTime(now: Instant): Instant {
// Probing is not active or probing is already complete.
if (probingState != ProbingState.kActive || clusters.isEmpty()) {
return Instant.MAX
}
return nextProbeTime
}
// Information about the current probing cluster.
fun currentCluster(now: Instant): PacedPacketInfo? {
if (clusters.isEmpty() || probingState != ProbingState.kActive) {
return null
}
if (nextProbeTime.isFinite() &&
Duration.between(nextProbeTime, now) > config.maxProbeDelay
) {
logger.warn(
"Probe delay too high (next:$nextProbeTime, now:$now)" +
"discarding probe cluster."
)
clusters.removeFirst()
if (clusters.isEmpty()) {
probingState = ProbingState.kInactive
return null
}
}
val info = clusters.first().paceInfo.copy()
info.probeClusterBytesSent = clusters.first().sentBytes
return info
}
// Returns the minimum number of bytes that the prober recommends for
// the next probe, or zero if not probing. A probe can consist of multiple
// packets that are sent back to back.
fun recommendedMinProbeSize(): DataSize {
if (clusters.isEmpty()) {
return DataSize.ZERO
}
val sendRate = clusters.first().paceInfo.sendBitrate
return sendRate * clusters.first().minProbeDelta
}
// Called to report to the prober that a probe has been sent. In case of
// multiple packets per probe, this call would be made at the end of sending
// the last packet in probe. `size` is the total size of all packets in probe.
fun probeSent(now: Instant, size: DataSize) {
check(probingState == ProbingState.kActive)
check(size != DataSize.ZERO)
if (clusters.isNotEmpty()) {
val cluster = clusters.first()
if (cluster.sentProbes == 0) {
check(cluster.startedAt.isInfinite())
cluster.startedAt = now
}
cluster.sentBytes += size.bytes.toInt()
cluster.sentProbes += 1
nextProbeTime = calculateNextProbeTime(cluster)
if (cluster.sentBytes >= cluster.paceInfo.probeClusterMinBytes &&
cluster.sentProbes >= cluster.paceInfo.probeClusterMinProbes
) {
clusters.removeFirst()
}
if (clusters.isEmpty()) {
probingState = ProbingState.kInactive
}
}
}
private enum class ProbingState {
// Probing will not be triggered in this state at all times.
kDisabled,
// Probing is enabled and ready to trigger on the first packet arrival if
// there is a probe cluster.
kInactive,
// Probe cluster is filled with the set of data rates to be probed and
// probes are being sent.
kActive,
}
// A probe cluster consists of a set of probes. Each probe in turn can be
// divided into a number of packets to accommodate the MTU on the network.
private data class ProbeCluster(
val paceInfo: PacedPacketInfo,
var sentProbes: Int = 0,
var sentBytes: Int = 0,
var minProbeDelta: Duration = Duration.ZERO,
var requestedAt: Instant = Instant.MIN,
var startedAt: Instant = Instant.MIN
)
private fun calculateNextProbeTime(cluster: ProbeCluster): Instant {
check(cluster.paceInfo.sendBitrate >= 0.bps)
check(cluster.startedAt.isFinite())
// Compute the time delta from the cluster start to ensure probe bitrate stays
// close to the target bitrate. Result is in milliseconds.
val sentBytes = cluster.sentBytes.bytes
val sendBitrate = cluster.paceInfo.sendBitrate
val delta = sentBytes / sendBitrate
return cluster.startedAt + delta
}
private fun maybeSetActiveState(packetSize: DataSize) {
if (readyToSetActiveState(packetSize)) {
nextProbeTime = Instant.MIN
probingState = ProbingState.kActive
}
}
private fun readyToSetActiveState(packetSize: DataSize): Boolean {
if (clusters.isEmpty()) {
check(probingState == ProbingState.kDisabled || probingState == ProbingState.kInactive)
return false
}
when (probingState) {
ProbingState.kDisabled, ProbingState.kActive -> {
return false
}
ProbingState.kInactive -> {
if (config.allowStartProbingImmediately) {
return true
}
// If config_.min_packet_size > 0, a "large enough" packet must be
// sent first, before a probe can be generated and sent. Otherwise,
// send the probe asap.
return packetSize >= min(recommendedMinProbeSize(), config.minPacketSize)
}
}
}
private var probingState: ProbingState = ProbingState.kInactive
// Probe bitrate per packet. These are used to compute the delta relative to
// the previous probe packet based on the size and time when that packet was
// sent.
private val clusters = ArrayDeque<ProbeCluster>()
// Time that the next probe should be sent when in kActive state
private var nextProbeTime: Instant = Instant.MAX
private var config = configIn.copy()
}

64
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BweDefines.kt Normal file
View File

@@ -0,0 +1,64 @@
/*
* Copyright @ 2015 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file uses WebRTC's naming style for enums
@file:Suppress("ktlint:standard:enum-entry-name-case")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bps
import org.jitsi.utils.secs
/** Common defines for bandwidth estimation,
* based on WebRTC modules/remote_bitrate_estimator/{include/bwe_defines.h,bwe_defines.cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
val kCongestionControllerMinBitrate = 5000.bps
val kBitrateWindow = 1.secs
/**
* Input to Rate Control.
*/
data class RateControlInput(
var bwState: BandwidthUsage,
var estimatedThroughput: Bandwidth?
) {
/**
* Assigns the values of the fields of [source] to the respective
* fields of this [RateControlInput].
*
* @param source the [RateControlInput] the values of the fields of
* which are to be assigned to the respective fields of this
* [RateControlInput]
*/
fun copy(source: RateControlInput) {
bwState = source.bwState
estimatedThroughput = source.estimatedThroughput
}
}
/**
* This is from WebRTC api/transport/bandwidth_usage.h in the same WebRTC revision.
*/
enum class BandwidthUsage {
kBwNormal,
kBwUnderusing,
kBwOverusing
// kLast not needed in Kotlin
}

86
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackController.kt Normal file
View File

@@ -0,0 +1,86 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.DataSize
/** Congestion window pushback controller,
* based on WebRTC modules/congestion_controller/goog_cc/congestion_window_pushback_controller.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings.
*/
class CongestionWindowPushbackController {
fun updateOutstandingData(outstandingBytes: Long) {
this.outstandingBytes = outstandingBytes
}
fun updatePacingQueue(pacingBytes: Long) {
this.pacingBytes = pacingBytes
}
fun updateTargetBitrate(bitrateBps: Int): Int {
if (currentDataWindow == null || currentDataWindow == DataSize.ZERO) {
return bitrateBps
}
var totalBytes = outstandingBytes
if (addPacing) {
totalBytes += pacingBytes
}
val fillRatio = totalBytes / currentDataWindow!!.bytes.toLong()
if (fillRatio > 1.5) {
encodingRateRatio *= 0.9
} else if (fillRatio > 1.0) {
encodingRateRatio *= 0.95
} else if (fillRatio < 0.1) {
encodingRateRatio = 1.0
} else {
encodingRateRatio *= 1.05
encodingRateRatio = Math.min(encodingRateRatio, 1.0)
}
val adjustedTargetBitrateBps = (bitrateBps * encodingRateRatio).toInt()
// Do not adjust below the minimum pushback bitrate but do obey if the
// original estimate is below it.
return if (adjustedTargetBitrateBps < minPushbackTargetBitrateBps) {
Math.min(bitrateBps, minPushbackTargetBitrateBps)
} else {
adjustedTargetBitrateBps
}
}
fun setDataWindow(dataWindow: DataSize) {
currentDataWindow = dataWindow
}
private val addPacing = false
private val minPushbackTargetBitrateBps: Int = kDefaultMinPushbackTargetBitrateBps
private var currentDataWindow: DataSize? = null
private var outstandingBytes = 0L
private var pacingBytes = 0L
private var encodingRateRatio = 1.0
}
/** From rtc_base/experiments/rate_control_settings.cc */
private const val kDefaultMinPushbackTargetBitrateBps = 30000

250
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBwe.kt Normal file
View File

@@ -0,0 +1,250 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bps
import org.jitsi.utils.NEVER
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import org.jitsi.utils.toDoubleMillis
import org.jitsi.utils.toRoundedEpochMilli
import java.time.Duration
import java.time.Instant
/** Delay-based bandwidth estimation,
* based on WebRTC modules/congestion_controller/goog_cc/delay_based_bwe.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings, and APIs that aren't
* used by Chrome have also been removed.
*/
class DelayBasedBwe(
parentLogger: Logger,
private val diagnosticContext: DiagnosticContext
) {
private val logger = parentLogger.createChildLogger(javaClass.name)
private var interArrivalDelta: InterArrivalDelta? = null
val delayDetector: DelayIncreaseDetectorInterface = TrendlineEstimator(logger, diagnosticContext)
private var lastSeenPacket: Instant = NEVER
/* private var umaRecorded: Boolean = false */
val rateControl = AimdRateControl(sendSide = true)
private var prevBitrate: Bandwidth = Bandwidth.ZERO
private var prevState: BandwidthUsage = BandwidthUsage.kBwNormal
fun incomingPacketFeedbackVector(
msg: TransportPacketsFeedback,
ackedBitrate: Bandwidth?,
probeBitrate: Bandwidth?,
/* networkEstimate: NetworkStateEstimate?, */
inAlr: Boolean
): Result {
val packetFeedbackVector = msg.sortedByReceiveTime()
if (packetFeedbackVector.isEmpty()) {
// TODO(holmer): An empty feedback vector here likely means that
// all acks were too late and that the send time history had
// timed out. We should reduce the rate when this occurs.
logger.warn("Very late feedback received.")
return Result()
}
/* Skipping uma_recorded, it's just for histograms. */
var delayedFeedback = true
var recoveredFromOveruse = false
var prevDetectorState = delayDetector.state()
for (packetFeedback in packetFeedbackVector) {
delayedFeedback = false
incomingPacketFeedback(packetFeedback, msg.feedbackTime)
if (prevDetectorState == BandwidthUsage.kBwUnderusing &&
delayDetector.state() == BandwidthUsage.kBwNormal
) {
recoveredFromOveruse = true
}
prevDetectorState = delayDetector.state()
}
if (delayedFeedback) {
// TODO(bugs.webrtc.org/10125): Design a better mechanism to safe-guard
// against building very large network queues.
return Result()
}
rateControl.inAlr = inAlr
// rateControl.networkStateEstimate = networkStateEstimate
return maybeUpdateEstimate(
ackedBitrate,
probeBitrate,
/* networkEstimate, */
recoveredFromOveruse,
inAlr,
msg.feedbackTime
)
}
private fun incomingPacketFeedback(packetFeedback: PacketResult, atTime: Instant) {
// Reset if the stream has timed out.
if (lastSeenPacket == NEVER || Duration.between(lastSeenPacket, atTime) > kStreamTimeOut) {
interArrivalDelta = InterArrivalDelta(kSendTimeGroupLength)
}
lastSeenPacket = atTime
val packetSize = packetFeedback.sentPacket.size
val calculatedDeltas = interArrivalDelta!!.computeDeltas(
packetFeedback.sentPacket.sendTime,
packetFeedback.receiveTime,
atTime,
packetSize.bytes.toLong()
)
delayDetector.update(
calculatedDeltas.arrivalTimeDelta.toDoubleMillis(),
calculatedDeltas.sendTimeDelta.toDoubleMillis(),
packetFeedback.sentPacket.sendTime.toRoundedEpochMilli(),
packetFeedback.receiveTime.toRoundedEpochMilli(),
packetSize.bytes.toLong(),
calculatedDeltas.computed
)
}
fun triggerOveruse(atTime: Instant, linkCapacity: Bandwidth?): Bandwidth {
val input = RateControlInput(BandwidthUsage.kBwOverusing, linkCapacity)
return rateControl.update(input, atTime)
}
private fun maybeUpdateEstimate(
ackedBitrate: Bandwidth?,
probeBitrate: Bandwidth?,
/* networkEstimate: NetworkStateEstimate?, */
recoveredFromOveruse: Boolean,
inAlr: Boolean,
atTime: Instant
): Result {
val result =
// Currently overusing the bandwidth.
if (delayDetector.state() == BandwidthUsage.kBwOverusing) {
if (ackedBitrate != null && rateControl.timeToReduceFurther(atTime, ackedBitrate)) {
val targetBitrate = updateEstimate(atTime, ackedBitrate)
if (targetBitrate != null) {
Result(updated = true, targetBitrate = targetBitrate)
} else {
Result(updated = false)
}
} else if (ackedBitrate == null && rateControl.validEstimate() &&
rateControl.initialTimeToReduceFurther(atTime)
) {
rateControl.setEstimate(rateControl.latestEstimate() / 2, atTime)
Result(updated = true, probe = false, targetBitrate = rateControl.latestEstimate())
} else {
Result()
}
} else {
if (probeBitrate != null) {
rateControl.setEstimate(probeBitrate, atTime)
Result(probe = true, updated = true, targetBitrate = rateControl.latestEstimate())
} else {
val targetBitrate = updateEstimate(atTime, ackedBitrate)
if (targetBitrate != null) {
Result(
updated = true,
targetBitrate = targetBitrate,
recoveredFromOveruse = recoveredFromOveruse
)
} else {
Result(updated = false, recoveredFromOveruse = recoveredFromOveruse)
}
}
}
val detectorState = delayDetector.state()
if ((result.updated && prevBitrate != result.targetBitrate) ||
detectorState != prevState
) {
val bitrate = if (result.updated) {
result.targetBitrate
} else {
prevBitrate
}
timeSeriesLogger.trace {
diagnosticContext.makeTimeSeriesPoint("bwe_update_delay_based", atTime)
.addField("bitrate_bps", bitrate.bps)
.addField("detector_state", detectorState.name)
}
prevBitrate = bitrate
prevState = detectorState
}
result.delayDetectorState = detectorState
return result
}
private fun updateEstimate(atTime: Instant, ackedBitrate: Bandwidth?): Bandwidth? {
val input = RateControlInput(delayDetector.state(), ackedBitrate)
val targetRate = rateControl.update(input, atTime)
return if (rateControl.validEstimate()) {
targetRate
} else {
null
}
}
fun onRttUpdate(avgRtt: Duration) {
rateControl.rtt = avgRtt
}
fun latestEstimate(): Bandwidth? {
if (!rateControl.validEstimate()) {
return null
}
return rateControl.latestEstimate()
}
fun setStartBitrate(startBitrate: Bandwidth) {
logger.info("BWE setting start bitrate to $startBitrate")
rateControl.setStartBitrate(startBitrate)
}
fun setMinBitrate(minBitrate: Bandwidth) {
rateControl.setMinBitrate(minBitrate)
}
fun getExpectedBwePeriod(): Duration = rateControl.getExpectedBandwidthPeriod()
fun lastEstimate(): Bandwidth = prevBitrate
fun lastState(): BandwidthUsage = prevState
companion object {
val kStreamTimeOut = 2.secs
val kSendTimeGroupLength = 5.ms
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(DelayBasedBwe::class.java)
}
data class Result(
val updated: Boolean = false,
val probe: Boolean = false,
val targetBitrate: Bandwidth = 0.bps,
val recoveredFromOveruse: Boolean = false,
var delayDetectorState: BandwidthUsage = BandwidthUsage.kBwNormal
)
}

36
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayIncreaseDetectorInterface.kt Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
/**
* Interface to detect delay increase.
* *
* Based on WebRTC modules/congestion_controller/goog_cc/delay_increase_detector_interface.h in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
interface DelayIncreaseDetectorInterface {
fun update(
recvDeltaMs: Double,
sendDeltaMs: Double,
sendTimeMs: Long,
arrivalTimeMs: Long,
packetSize: Long,
calculatedDeltas: Boolean
)
fun state(): BandwidthUsage
}

47
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcFactory.kt Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.utils.ms
import java.time.Duration
/** Google CC Network Controller Factory,
* based on WebRTC api/transport/goog_cc_factory.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings, and features not used in
* Chrome have been removed.
*/
class GoogCcFactoryConfig(
val feedbackOnly: Boolean = false,
/* Not in this object in WebRTC: This is a field trial parameter there */
val rateControlSettings: CongestionWindowConfig = CongestionWindowConfig()
)
class GoogCcNetworkControllerFactory(
private val factoryConfig: GoogCcFactoryConfig = GoogCcFactoryConfig()
) : NetworkControllerFactoryInterface {
override fun create(config: NetworkControllerConfig): NetworkControllerInterface {
val googCcConfig = GoogCcConfig(feedbackOnly = factoryConfig.feedbackOnly)
return GoogCcNetworkController(config, googCcConfig)
}
override fun getProcessInterval(): Duration {
val kUpdateIntervalMs = 25L
return kUpdateIntervalMs.ms
}
}

817
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkController.kt Normal file
View File

@@ -0,0 +1,817 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bps
import org.jitsi.nlj.util.bytes
import org.jitsi.nlj.util.max
import org.jitsi.nlj.util.min
import org.jitsi.nlj.util.times
import org.jitsi.utils.MAX_DURATION
import org.jitsi.utils.MIN_DURATION
import org.jitsi.utils.isFinite
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.max
import org.jitsi.utils.min
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import org.jitsi.utils.toDouble
import org.jitsi.utils.toRoundedMillis
import java.time.Duration
import java.time.Instant
import kotlin.math.roundToLong
/** Top-level Google CC Network Controller,
* based on WebRTC modules/congestion_controller/goog_cc/goog_cc_network_control.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings,
* except where needed by unit tests.
*/
class GoogCcConfig(
val feedbackOnly: Boolean = false,
/* Not in this object in WebRTC: This is a field trial parameter there */
val rateControlSettings: CongestionWindowConfig = CongestionWindowConfig()
)
class GoogCcNetworkController(
config: NetworkControllerConfig,
googCcConfig: GoogCcConfig
) : NetworkControllerInterface {
private val logger = config.parentLogger.createChildLogger(javaClass.name)
private val diagnosticContext = config.diagnosticContext
override fun onNetworkAvailability(msg: NetworkAvailability): NetworkControlUpdate = NetworkControlUpdate(
probeClusterConfigs = probeController.onNetworkAvailability(msg)
)
override fun onNetworkRouteChange(msg: NetworkRouteChange): NetworkControlUpdate {
if (safeResetOnRouteChange) {
var estimatedBitrate: Bandwidth?
if (safeResetAcknowledgedRate) {
estimatedBitrate = acknowledgedBitrateEstimator.bitrate()
if (estimatedBitrate == null) {
estimatedBitrate = acknowledgedBitrateEstimator.peekRate()
}
} else {
estimatedBitrate = bandwidthEstimation.targetRate()
}
if (estimatedBitrate != null) {
if (msg.constraints.startingRate != null) {
msg.constraints.startingRate = min(msg.constraints.startingRate!!, estimatedBitrate)
} else {
msg.constraints.startingRate = estimatedBitrate
}
}
}
acknowledgedBitrateEstimator = AcknowledgedBitrateEstimatorInterface.create()
probeBitrateEstimator = ProbeBitrateEstimator(logger, diagnosticContext)
delayBasedBwe = DelayBasedBwe(logger, diagnosticContext)
bandwidthEstimation.onRouteChange()
probeController.reset(msg.atTime)
val update = MutableNetworkControlUpdate(
probeClusterConfigs = resetConstraints(msg.constraints)
)
maybeTriggerOnNetworkChanged(update, msg.atTime)
return update
}
override fun onProcessInterval(msg: ProcessInterval): NetworkControlUpdate {
val update = MutableNetworkControlUpdate()
if (initialConfig != null) {
update.probeClusterConfigs = resetConstraints(initialConfig!!.constraints)
update.pacerConfig = getPacingRates(msg.atTime)
if (initialConfig!!.streamBasedConfig.requestsAlrProbing != null) {
probeController.enablePeriodicAlrProbing(initialConfig!!.streamBasedConfig.requestsAlrProbing!!)
}
if (initialConfig!!.streamBasedConfig.enableRepeatedInitialProbing != null) {
probeController.enableRepeatedInitialProbing(
initialConfig!!.streamBasedConfig.enableRepeatedInitialProbing!!
)
}
val totalBitrate = initialConfig!!.streamBasedConfig.maxTotalAllocatedBitrate
if (totalBitrate != null) {
val probes = probeController.onMaxTotalAllocatedBitrate(totalBitrate, msg.atTime)
update.probeClusterConfigs.addAll(probes)
}
initialConfig = null
}
if (congestionWindowPushbackController != null && msg.pacerQueue != null) {
congestionWindowPushbackController.updatePacingQueue(msg.pacerQueue.bytes.roundToLong())
}
bandwidthEstimation.updateEstimate(msg.atTime)
val startTimeMs = alrDetector.getApplicationLimitedRegionStartTime()
probeController.setAlrStartTimeMs(startTimeMs)
val probes = probeController.process(msg.atTime)
update.probeClusterConfigs.addAll(probes)
if (rateControlSettings.useCongestionWindow() &&
!feedbackMaxRtts.isEmpty()
) {
updateCongestionWindowSize()
}
if (congestionWindowPushbackController != null && currentDataWindow != null) {
congestionWindowPushbackController.setDataWindow(currentDataWindow!!)
} else {
update.congestionWindow = currentDataWindow
}
maybeTriggerOnNetworkChanged(update, msg.atTime)
return update
}
override fun onRemoteBitrateReport(msg: RemoteBitrateReport): NetworkControlUpdate {
if (packetFeedbackOnly) {
logger.error("Received REMB for packet feedback only GoogCC")
return NetworkControlUpdate()
}
bandwidthEstimation.updateReceiverEstimate(msg.receiveTime, msg.bandwidth)
timeSeriesLogger.trace {
diagnosticContext.makeTimeSeriesPoint("REMB_BW", msg.receiveTime)
.addField("REMB_kbps", msg.bandwidth.kbps)
}
return NetworkControlUpdate()
}
override fun onRoundTripTimeUpdate(msg: RoundTripTimeUpdate): NetworkControlUpdate {
if (packetFeedbackOnly || msg.smoothed) {
return NetworkControlUpdate()
}
assert(msg.roundTripTime != Duration.ZERO)
delayBasedBwe.onRttUpdate(msg.roundTripTime)
bandwidthEstimation.updateRtt(msg.roundTripTime, msg.receiveTime)
return NetworkControlUpdate()
}
override fun onSentPacket(sentPacket: SentPacket): NetworkControlUpdate {
alrDetector.onBytesSent(sentPacket.size.bytes.roundToLong(), sentPacket.sendTime.toEpochMilli())
acknowledgedBitrateEstimator.setAlr(alrDetector.getApplicationLimitedRegionStartTime() != null)
if (!firstPacketSent) {
firstPacketSent = true
// Initialize feedback time to send time to allow estimation of RTT until
// first feedback is received.
bandwidthEstimation.updatePropagationRtt(sentPacket.sendTime, Duration.ZERO)
}
bandwidthEstimation.onSentPacket(sentPacket)
if (congestionWindowPushbackController != null) {
congestionWindowPushbackController.updateOutstandingData(sentPacket.dataInFlight.bytes.roundToLong())
val update = MutableNetworkControlUpdate()
maybeTriggerOnNetworkChanged(update, sentPacket.sendTime)
return update
} else {
return NetworkControlUpdate()
}
}
override fun onStreamsConfig(msg: StreamsConfig): NetworkControlUpdate {
val update = MutableNetworkControlUpdate()
if (msg.requestsAlrProbing != null) {
probeController.enablePeriodicAlrProbing(msg.requestsAlrProbing)
}
if (msg.maxTotalAllocatedBitrate != null) {
update.probeClusterConfigs =
probeController.onMaxTotalAllocatedBitrate(msg.maxTotalAllocatedBitrate, msg.atTime)
}
var pacingChanged = false
if (msg.pacingFactor != null && msg.pacingFactor != pacingFactor) {
pacingFactor = msg.pacingFactor
pacingChanged = true
}
if (msg.minTotalAllocatedBitrate != null &&
minTotalAllocatedBitrate != msg.minTotalAllocatedBitrate
) {
minTotalAllocatedBitrate = msg.minTotalAllocatedBitrate
pacingChanged = true
if (useMinAllocatableAsLowerBound) {
clampConstraints()
delayBasedBwe.setMinBitrate(minDataRate)
bandwidthEstimation.setMinMaxBitrate(minDataRate, maxDataRate)
}
}
if (msg.maxPaddingRate != null && msg.maxPaddingRate != maxPaddingRate) {
maxPaddingRate = msg.maxPaddingRate
pacingChanged = true
}
if (pacingChanged) {
update.pacerConfig = getPacingRates(msg.atTime)
}
return update
}
override fun onTargetRateConstraints(constraints: TargetRateConstraints): NetworkControlUpdate {
val update = MutableNetworkControlUpdate()
update.probeClusterConfigs = resetConstraints(constraints)
maybeTriggerOnNetworkChanged(update, constraints.atTime)
return update
}
private fun clampConstraints() {
// TODO(holmer): We should make sure the default bitrates are set to 10 kbps,
// and that we don't try to set the min bitrate to 0 from any applications.
// The congestion controller should allow a min bitrate of 0.
minDataRate = max(minTargetRate, kCongestionControllerMinBitrate)
if (useMinAllocatableAsLowerBound) {
minDataRate = max(minDataRate, minTotalAllocatedBitrate)
}
if (maxDataRate < minDataRate) {
logger.warn("max bitrate $maxDataRate smaller than min bitrate $minDataRate")
maxDataRate = minDataRate
}
if (startingRate != null && startingRate!! < minDataRate) {
logger.warn("start bitrate ${startingRate!!} smaller than min bitrate $minDataRate")
startingRate = minDataRate
}
}
private fun resetConstraints(newConstraints: TargetRateConstraints): MutableList<ProbeClusterConfig> {
minTargetRate = newConstraints.minDataRate ?: Bandwidth.ZERO
maxDataRate = newConstraints.maxDataRate ?: Bandwidth.INFINITY
startingRate = newConstraints.startingRate
clampConstraints()
bandwidthEstimation.setBitrates(startingRate, minDataRate, maxDataRate, newConstraints.atTime)
if (startingRate != null) {
delayBasedBwe.setStartBitrate(startingRate!!)
}
delayBasedBwe.setMinBitrate(minDataRate)
return probeController.setBitrates(
minDataRate,
startingRate ?: Bandwidth.ZERO,
maxDataRate,
newConstraints.atTime
)
}
override fun onTransportLossReport(msg: TransportLossReport): NetworkControlUpdate {
if (packetFeedbackOnly) {
return NetworkControlUpdate()
}
val totalPacketsDelta = msg.packetsReceivedDelta + msg.packetsLostDelta
bandwidthEstimation.updatePacketsLost(msg.packetsLostDelta, totalPacketsDelta, msg.receiveTime)
return NetworkControlUpdate()
}
private fun updateCongestionWindowSize() {
val minFeedbackMaxRtt = feedbackMaxRtts.min().ms
val kMinCwnd = (2 * 1500).bytes
val timeWindow = minFeedbackMaxRtt + rateControlSettings.getCongestionWindowAdditionalTimeMs().ms
var dataWindow = lastLossBasedTargetRate * timeWindow
if (currentDataWindow != null) {
dataWindow = max(kMinCwnd, (dataWindow + currentDataWindow!!) / 2.0)
} else {
dataWindow = max(kMinCwnd, dataWindow)
}
currentDataWindow = dataWindow
}
override fun onTransportPacketsFeedback(report: TransportPacketsFeedback): NetworkControlUpdate {
if (report.packetFeedbacks.isEmpty()) {
// TODO(bugs.webrtc.org/10125): Design a better mechanism to safe-guard
// against building very large network queues.
return NetworkControlUpdate()
}
if (congestionWindowPushbackController != null) {
congestionWindowPushbackController.updateOutstandingData(report.dataInFlight.bytes.roundToLong())
}
var maxFeedbackRtt = MIN_DURATION
var minPropagationRtt = MAX_DURATION
var maxRecvTime = Instant.MIN
val feedbacks = report.receivedWithSendInfo()
for (feedback in feedbacks) {
maxRecvTime = max(maxRecvTime, feedback.receiveTime)
}
for (feedback in feedbacks) {
val feedbackRtt = Duration.between(feedback.sentPacket.sendTime, report.feedbackTime)
val minPendingTime = Duration.between(feedback.receiveTime, maxRecvTime)
val propagationRtt = feedbackRtt - minPendingTime
maxFeedbackRtt = max(maxFeedbackRtt, feedbackRtt)
minPropagationRtt = min(minPropagationRtt, propagationRtt)
}
if (maxFeedbackRtt.isFinite()) {
feedbackMaxRtts.add(maxFeedbackRtt.toRoundedMillis())
val kMaxFeedbackRttWindow = 32
if (feedbackMaxRtts.size > kMaxFeedbackRttWindow) {
feedbackMaxRtts.removeFirst()
}
// TODO(srte): Use time since last unacknowledged packet.
bandwidthEstimation.updatePropagationRtt(report.feedbackTime, minPropagationRtt)
}
if (packetFeedbackOnly) {
if (!feedbackMaxRtts.isEmpty()) {
val sumRttMs = feedbackMaxRtts.sum()
val meanRttMs = sumRttMs / feedbackMaxRtts.size
delayBasedBwe.onRttUpdate(meanRttMs.ms)
}
var feedbackMinRtt = MAX_DURATION
for (packetFeedback in feedbacks) {
val pendingTime = Duration.between(packetFeedback.receiveTime, maxRecvTime)
val rtt = Duration.between(packetFeedback.sentPacket.sendTime, report.feedbackTime) - pendingTime
// Value used for predicting NACK round trip time in FEC controller.
feedbackMinRtt = min(rtt, feedbackMinRtt)
}
if (feedbackMinRtt.isFinite()) {
bandwidthEstimation.updateRtt(feedbackMinRtt, report.feedbackTime)
}
expectedPacketsSinceLastLossUpdate += report.packetsWithFeedback().size
for (packetFeedback in report.packetsWithFeedback()) {
if (!packetFeedback.isReceived()) {
lostPacketsSinceLastLossUpdate += 1
}
}
if (report.feedbackTime > nextLossUpdate) {
nextLossUpdate = report.feedbackTime + kLossUpdateInterval
bandwidthEstimation.updatePacketsLost(
lostPacketsSinceLastLossUpdate.toLong(),
expectedPacketsSinceLastLossUpdate.toLong(),
report.feedbackTime
)
expectedPacketsSinceLastLossUpdate = 0
lostPacketsSinceLastLossUpdate = 0
}
}
val alrStartTime = alrDetector.getApplicationLimitedRegionStartTime()
if (previouslyInAlr && alrStartTime == null) {
val nowMs = report.feedbackTime.toEpochMilli()
acknowledgedBitrateEstimator.setAlrEndedTime(report.feedbackTime)
probeController.setAlrEndedTimeMs(nowMs)
}
previouslyInAlr = alrStartTime != null
acknowledgedBitrateEstimator.incomingPacketFeedbackVector(report.sortedByReceiveTime())
val acknowledgedBitrate = acknowledgedBitrateEstimator.bitrate()
bandwidthEstimation.setAcknowledgedRate(acknowledgedBitrate, report.feedbackTime)
for (feedback in report.sortedByReceiveTime()) {
if (feedback.sentPacket.pacingInfo.probeClusterId != PacedPacketInfo.kNotAProbe) {
probeBitrateEstimator.handleProbeAndEstimateBitrate(feedback)
}
}
/* Skipped network_estimator code */
var probeBitrate = probeBitrateEstimator.fetchAndResetLastEstimatedBitrate()
/* Skipped network_estimator code */
if (limitProbesLowerThanThroughputEstimate && probeBitrate != null && acknowledgedBitrate != null) {
// Limit the backoff to something slightly below the acknowledged
// bitrate. ("Slightly below" because we want to drain the queues
// if we are actually overusing.)
// The acknowledged bitrate shouldn't normally be higher than the delay
// based estimate, but it could happen e.g. due to packet bursts or
// encoder overshoot. We use std::min to ensure that a probe result
// below the current BWE never causes an increase.
val limit = min(delayBasedBwe.lastEstimate(), acknowledgedBitrate * kProbeDropThroughputFraction)
probeBitrate = max(probeBitrate, limit)
}
val update = MutableNetworkControlUpdate()
var recoveredFromOveruse = false
val result = delayBasedBwe.incomingPacketFeedbackVector(
report,
acknowledgedBitrate,
probeBitrate,
alrStartTime != null
)
if (result.updated) {
if (result.probe) {
bandwidthEstimation.setSendBitrate(result.targetBitrate, report.feedbackTime)
}
// Since SetSendBitrate now resets the delay-based estimate, we have to
// call UpdateDelayBasedEstimate after SetSendBitrate.
bandwidthEstimation.updateDelayBasedEstimate(report.feedbackTime, result.targetBitrate)
}
bandwidthEstimation.updateLossBasedEstimator(
report,
result.delayDetectorState,
probeBitrate,
alrStartTime != null
)
if (result.updated) {
// Update the estimate in the ProbeController, in case we want to probe.
maybeTriggerOnNetworkChanged(update, report.feedbackTime)
}
recoveredFromOveruse = result.recoveredFromOveruse
if (recoveredFromOveruse) {
probeController.setAlrStartTimeMs(alrStartTime)
val probes = probeController.requestProbe(report.feedbackTime)
update.probeClusterConfigs.addAll(probes)
}
// No valid RTT could be because send-side BWE isn't used, in which case
// we don't try to limit the outstanding packets.
if (rateControlSettings.useCongestionWindow() && maxFeedbackRtt.isFinite()) {
updateCongestionWindowSize()
}
if (congestionWindowPushbackController != null && currentDataWindow != null) {
congestionWindowPushbackController.setDataWindow(currentDataWindow!!)
} else {
update.congestionWindow = currentDataWindow
}
return update
}
fun getNetworkState(atTime: Instant): NetworkControlUpdate {
val update = MutableNetworkControlUpdate()
update.targetRate = TargetTransferRate().apply {
networkEstimate.atTime = atTime
networkEstimate.lossRateRatio = (lastEstimatedFractionLoss ?: 0u).toFloat() / 255.0f
networkEstimate.roundTripTime = lastEstimatedRoundTripTime
networkEstimate.bwePeriod = delayBasedBwe.getExpectedBwePeriod()
this.atTime = atTime
if (rateControlSettings.useCongestionWindowDropFrameOnly()) {
targetRate = lastLossBasedTargetRate
} else {
targetRate = lastPushbackTargetRate
}
targetRate = lastPushbackTargetRate
stableTargetRate = bandwidthEstimation.getEstimatedLinkCapacity()
}
update.pacerConfig = getPacingRates(atTime)
update.congestionWindow = currentDataWindow
return update
}
private fun maybeTriggerOnNetworkChanged(update: MutableNetworkControlUpdate, atTime: Instant) {
val fractionLoss = bandwidthEstimation.fractionLoss()
val roundTripTime = bandwidthEstimation.roundTripTime()
val lossBasedTargetRate = bandwidthEstimation.targetRate()
val lossBasedState = bandwidthEstimation.lossBasedState()
var pushbackTargetRate = lossBasedTargetRate
/* TODO: plotting */
var cwndReduceRatio = 0.0
if (congestionWindowPushbackController != null) {
var pushbackRate = congestionWindowPushbackController.updateTargetBitrate(lossBasedTargetRate.bps.toInt())
pushbackRate = Math.max(bandwidthEstimation.getMinBitrate(), pushbackRate)
pushbackTargetRate = pushbackRate.bps
if (rateControlSettings.useCongestionWindowDropFrameOnly()) {
cwndReduceRatio = (lossBasedTargetRate.bps - pushbackTargetRate.bps).toDouble() /
lossBasedTargetRate.bps
}
}
var stableTargetRate = bandwidthEstimation.getEstimatedLinkCapacity()
stableTargetRate = min(stableTargetRate, pushbackTargetRate)
if (lossBasedTargetRate != lastLossBasedTargetRate ||
lossBasedState != lastLossBasedState ||
fractionLoss != lastEstimatedFractionLoss ||
roundTripTime != lastEstimatedRoundTripTime ||
pushbackTargetRate != lastPushbackTargetRate ||
stableTargetRate != lastStableTargetRate
) {
lastLossBasedTargetRate = lossBasedTargetRate
lastPushbackTargetRate = pushbackTargetRate
lastEstimatedFractionLoss = fractionLoss
lastEstimatedRoundTripTime = roundTripTime
lastStableTargetRate = stableTargetRate
lastLossBasedState = lossBasedState
alrDetector.setEstimatedBitrate(lossBasedTargetRate.bps.toInt())
val bwePeriod = delayBasedBwe.getExpectedBwePeriod()
val targetRateMsg = TargetTransferRate()
targetRateMsg.atTime = atTime
if (rateControlSettings.useCongestionWindowDropFrameOnly()) {
targetRateMsg.targetRate = lossBasedTargetRate
targetRateMsg.cwndReduceRatio = cwndReduceRatio
} else {
targetRateMsg.targetRate = pushbackTargetRate
}
targetRateMsg.stableTargetRate = stableTargetRate
targetRateMsg.networkEstimate.atTime = atTime
targetRateMsg.networkEstimate.roundTripTime = roundTripTime
targetRateMsg.networkEstimate.lossRateRatio = fractionLoss.toFloat() / 255.0f
targetRateMsg.networkEstimate.bwePeriod = bwePeriod
update.targetRate = targetRateMsg
val probes = probeController.setEstimatedBitrate(
lossBasedTargetRate,
getBandwidthLimitedCause(
bandwidthEstimation.lossBasedState(),
bandwidthEstimation.isRttAboveLimit(),
delayBasedBwe.lastState()
),
atTime
)
update.probeClusterConfigs.addAll(probes)
update.pacerConfig = getPacingRates(atTime)
logger.debug(
"bwe $atTime: pushback_target_bps=${lastPushbackTargetRate.bps} " +
"estimate_bps=${lossBasedTargetRate.bps}"
)
}
}
private fun getPacingRates(atTime: Instant): PacerConfig {
// Pacing rate is based on target rate before congestion window pushback,
// because we don't want to build queues in the pacer when pushback occurs.
val pacingRate = max(minTotalAllocatedBitrate, lastLossBasedTargetRate) * pacingFactor
var paddingRate = if (lastLossBasedState == LossBasedState.kIncreaseUsingPadding) {
max(maxPaddingRate, lastLossBasedTargetRate)
} else {
maxPaddingRate
}
paddingRate = min(paddingRate, lastPushbackTargetRate)
val msg = PacerConfig()
msg.atTime = atTime
msg.timeWindow = 1.secs
msg.dataWindow = pacingRate * msg.timeWindow
msg.padWindow = paddingRate * msg.timeWindow
return msg
}
/** Jitsi local addition.
* Fields based on WebRTC modules/congestion_controller/goog_cc/test/goog_cc_printer.{cc,h}
*/
class StatisticsSnapshot(
val time: Instant,
val rtt: Duration,
val target: Bandwidth,
val stableTarget: Bandwidth,
val pacing: Bandwidth?,
val padding: Bandwidth?,
val window: DataSize,
val rateControlState: AimdRateControl.RateControlState,
val stableEstimate: Bandwidth?,
val trendline: Double,
val trendlineModifiedOffset: Double,
val trendlineOffsetThreshold: Double,
val acknowledgedRate: Bandwidth?,
/* Skipped, based on NetworkStateEstimate
* estCapacity
* estCapacityDev
* estCapacityMin
* estCrossDelay
* estSpikeDelay
* estPreBuffer
* estPostBuffer
* estPropagation
*/
val lossRatio: Float,
/* Fields where data from LossBasedBweV1 are printed, even though LossBasedBweV2 is the default.
TODO: print state out of LossBasedBweV2.
val lossAverage: Double,
val lossAverageMax: Double,
val lossThresInc: Double,
val lossThresDec: Double,
val lossBasedRate: Bandwidth,
val lossAckRate: Bandwidth,
*/
/* SendSideBandwidthEstimator populates itself from LossBasedBwe's estimate. */
val sendSideTarget: Bandwidth,
val lossBasedState: LossBasedState,
val dataWindow: DataSize?,
val pushbackTarget: Bandwidth,
/* Additions to the fields from goog_cc_printer */
val inAlr: Boolean
) {
fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("time", time.toEpochMilli())
put("rtt", rtt.toDouble())
put("target", target.bps)
put("stable_target", stableTarget.bps)
put("pacing", pacing?.bps?.toDouble() ?: Double.NaN)
put("padding", padding?.bps?.toDouble() ?: Double.NaN)
put("window", window.bytes)
put("rate_control_state", rateControlState.name)
put("stable_estimate", stableEstimate?.bps?.toDouble() ?: Double.NaN)
put("trendline", trendline)
put("trendline_modified_offset", trendlineModifiedOffset)
put("trendline_modified_threshold", trendlineOffsetThreshold)
put("acknowledged_rate", acknowledgedRate?.bps?.toDouble() ?: Double.NaN)
put("loss_ratio", lossRatio)
put("send_side_target", sendSideTarget.bps)
put("last_loss_based_state", lossBasedState.name)
put("data_window", dataWindow?.bytes ?: Double.NaN)
put("pushback_target", pushbackTarget.bps)
put("in_alr", inAlr)
}
fun addToTimeSeriesPoint(point: DiagnosticContext.TimeSeriesPoint) {
point.apply {
addField("rtt", rtt.toDouble())
addField("target", target.bps)
addField("stable_target", stableTarget.bps)
addField("pacing", pacing?.bps ?: Double.NaN)
addField("padding", padding?.bps ?: Double.NaN)
addField("window", window.bytes)
addField("rate_control_state", rateControlState.name)
addField("stable_estimate", stableEstimate?.bps ?: Double.NaN)
addField("trendline", trendline)
addField("trendline_modified_offset", trendlineModifiedOffset)
addField("trendline_modified_threshold", trendlineOffsetThreshold)
addField("acknowledged_rate", acknowledgedRate?.bps ?: Double.NaN)
addField("loss_ratio", lossRatio)
addField("send_side_target", sendSideTarget.bps)
addField("last_loss_based_state", lossBasedState.name)
addField("data_window", dataWindow?.bytes ?: Double.NaN)
addField("pushback_target", pushbackTarget.bps)
addField("in_alr", inAlr)
}
}
}
private fun trend() = delayBasedBwe.delayDetector as TrendlineEstimator
fun getStatistics(now: Instant): StatisticsSnapshot {
val stateUpdate = getNetworkState(now)
val target = stateUpdate.targetRate!!
val pacing = stateUpdate.pacerConfig
val congestionWindow = stateUpdate.congestionWindow ?: DataSize.INFINITY
return StatisticsSnapshot(
time = target.atTime,
rtt = target.networkEstimate.roundTripTime,
target = target.targetRate,
stableTarget = target.stableTargetRate,
pacing = pacing?.dataRate(),
padding = pacing?.padRate(),
window = congestionWindow,
rateControlState = delayBasedBwe.rateControl.rateControlState,
stableEstimate = delayBasedBwe.rateControl.linkCapacity.estimate,
trendline = trend().prevTrend,
trendlineModifiedOffset = trend().prevModifiedTrend,
trendlineOffsetThreshold = trend().threshold,
acknowledgedRate = acknowledgedBitrateEstimator.bitrate(),
lossRatio = target.networkEstimate.lossRateRatio,
sendSideTarget = bandwidthEstimation.targetRate(),
lossBasedState = lastLossBasedState,
dataWindow = currentDataWindow,
pushbackTarget = lastPushbackTargetRate,
inAlr = previouslyInAlr
)
}
private val packetFeedbackOnly = googCcConfig.feedbackOnly
private val safeResetOnRouteChange = false
private val safeResetAcknowledgedRate = false
private val useMinAllocatableAsLowerBound = true
private val limitProbesLowerThanThroughputEstimate = true
private val rateControlSettings = googCcConfig.rateControlSettings
private val probeController = ProbeController(logger, diagnosticContext)
private val congestionWindowPushbackController = if (rateControlSettings.useCongestionWindowPushback()) {
CongestionWindowPushbackController()
} else {
null
}
private val bandwidthEstimation = SendSideBandwidthEstimation(logger, diagnosticContext)
private val alrDetector = AlrDetector()
private var probeBitrateEstimator = ProbeBitrateEstimator(logger, diagnosticContext)
private var delayBasedBwe = DelayBasedBwe(logger, diagnosticContext).also {
it.setMinBitrate(kCongestionControllerMinBitrate)
}
private var acknowledgedBitrateEstimator = AcknowledgedBitrateEstimatorInterface.create()
private var initialConfig: NetworkControllerConfig? = config
private var minTargetRate = Bandwidth.ZERO
private var minDataRate = Bandwidth.ZERO
private var maxDataRate = Bandwidth.ZERO
private var startingRate: Bandwidth? = null
private var firstPacketSent = false
/* Skipping NetworkStateEstimate */
private var nextLossUpdate = Instant.MIN
private var lostPacketsSinceLastLossUpdate = 0
private var expectedPacketsSinceLastLossUpdate = 0
private val feedbackMaxRtts = ArrayDeque<Long>()
private var lastLossBasedTargetRate = config.constraints.startingRate!!
private var lastPushbackTargetRate = lastLossBasedTargetRate
private var lastStableTargetRate = lastLossBasedTargetRate
private var lastLossBasedState: LossBasedState = LossBasedState.kDelayBasedEstimate
private var lastEstimatedFractionLoss: UByte? = 0u
private var lastEstimatedRoundTripTime = MAX_DURATION
private var pacingFactor = config.streamBasedConfig.pacingFactor ?: kDefaultPaceMultiplier
private var minTotalAllocatedBitrate = config.streamBasedConfig.minTotalAllocatedBitrate ?: Bandwidth.ZERO
private var maxPaddingRate = config.streamBasedConfig.maxPaddingRate ?: Bandwidth.ZERO
private var previouslyInAlr = false
private var currentDataWindow: DataSize? = null
companion object {
// From RTCPSender video report interval.
private val kLossUpdateInterval = 1000.ms
// Pacing-rate relative to our target send rate.
// Multiplicative factor that is applied to the target bitrate to calculate
// the number of bytes that can be transmitted per interval.
// Increasing this factor will result in lower delays in cases of bitrate
// overshoots from the encoder.
private const val kDefaultPaceMultiplier = 2.5
// If the probe result is far below the current throughput estimate
// it's unlikely that the probe is accurate, so we don't want to drop too far.
// However, if we actually are overusing, we want to drop to something slightly
// below the current throughput estimate to drain the network queues.
private const val kProbeDropThroughputFraction = 0.85
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(GoogCcNetworkController::class.java)
private fun getBandwidthLimitedCause(
lossBasedState: LossBasedState,
isRttAboveLimit: Boolean,
bandwidthUsage: BandwidthUsage
): BandwidthLimitedCause {
if (bandwidthUsage == BandwidthUsage.kBwOverusing ||
bandwidthUsage == BandwidthUsage.kBwUnderusing
) {
return BandwidthLimitedCause.kDelayBasedLimitedDelayIncreased
} else if (isRttAboveLimit) {
return BandwidthLimitedCause.kRttBasedBackOffHighRtt
}
return when (lossBasedState) {
LossBasedState.kDecreasing ->
// Probes may not be sent in this state.
BandwidthLimitedCause.kLossLimitedBwe
LossBasedState.kIncreaseUsingPadding ->
// Probes may not be sent in this state.
BandwidthLimitedCause.kLossLimitedBwe
LossBasedState.kIncreasing ->
// Probes may be sent in this state.
BandwidthLimitedCause.kLossLimitedBweIncreasing
LossBasedState.kDelayBasedEstimate ->
BandwidthLimitedCause.kDelayBasedLimited
}
}
}
}
/* Congestion window config, from WebRTC rtc_base/experiments/rate_control_settings.{h,cc}
* stripped down to used fields only. */
class CongestionWindowConfig(
private val queueSizeMs: Int? = 350,
private val minBitrateBps: Int? = 30000,
private val dropFrameOnly: Boolean = true
) {
fun useCongestionWindow() = queueSizeMs != null
fun getCongestionWindowAdditionalTimeMs() = queueSizeMs ?: kDefaultAcceptedQueueMs
fun useCongestionWindowPushback() = queueSizeMs != null && minBitrateBps != null
fun useCongestionWindowDropFrameOnly() = dropFrameOnly
companion object {
private const val kDefaultAcceptedQueueMs = 350
}
}

371
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcTransportCcEngine.kt Normal file
View File

@@ -0,0 +1,371 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorConfig
import org.jitsi.nlj.rtp.bandwidthestimation2.PacedPacketInfo.Companion.kNotAProbe
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bytes
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.RtcpReportBlock
import org.jitsi.rtp.rtcp.RtcpRrPacket
import org.jitsi.rtp.rtcp.RtcpSrPacket
import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.*
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit
/**
* Transport CC engine invoking GoogCc NetworkController. Contains some code based loosely on
* WebRTC call/rtp_transport_controller_send.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138)
*
*/
class GoogCcTransportCcEngine(
val diagnosticContext: DiagnosticContext,
parentLogger: Logger,
private val scheduledExecutor: ScheduledExecutorService,
private val sendProbing: (DataSize, Any?) -> Int,
val clock: Clock = Clock.systemUTC(),
) : TransportCcEngine() {
private val logger = createChildLogger(parentLogger)
private val feedbackAdapter = TransportFeedbackAdapter(logger)
private val networkController = factory.create(
NetworkControllerConfig(
logger,
diagnosticContext,
constraints = TargetRateConstraints(
atTime = clock.instant(),
startingRate = BandwidthEstimatorConfig.initBw,
maxDataRate = BandwidthEstimatorConfig.maxBw,
minDataRate = BandwidthEstimatorConfig.minBw
)
)
)
private val bitrateProber = BitrateProber(logger)
private var probeTask: ScheduledFuture<*>? = null
private var processTask: ScheduledFuture<*>? = null
private val listeners = LinkedList<BandwidthListener>()
@Synchronized
override fun onRttUpdate(rtt: Duration) {
val update =
networkController.onRoundTripTimeUpdate(
RoundTripTimeUpdate(receiveTime = clock.instant(), roundTripTime = rtt)
)
processUpdate(update)
}
@Synchronized
override fun rtcpPacketReceived(rtcpPacket: RtcpPacket, receivedTime: Instant?) {
when (rtcpPacket) {
is RtcpFbTccPacket -> {
val time = receivedTime ?: clock.instant()
val feedback = feedbackAdapter.processTransportFeedback(rtcpPacket, time)
if (feedback != null) {
val update = networkController.onTransportPacketsFeedback(feedback)
processUpdate(update)
feedback.packetFeedbacks.forEach { fb ->
lossListeners.forEach { l ->
if (fb.isReceived()) {
l.packetReceived(fb.previouslyReportedLost)
} else if (!fb.previouslyReportedLost) {
l.packetLost(1)
}
}
}
}
}
is RtcpFbRembPacket -> {
/* Ignore REMB packets - if we're supposed to be receiving them they'll be handled by [RembHandler],
* and if we're not we're getting mysterious spurious REMB messages which we want to ignore.
*/
/*
val time = receivedTime ?: clock.instant()
val msg = RemoteBitrateReport(receiveTime = time, bandwidth = rtcpPacket.bitrate.bps)
val update = networkController.onRemoteBitrateReport(msg)
processUpdate(update)
*/
}
is RtcpSrPacket -> {
val time = receivedTime ?: clock.instant()
onReport(time, rtcpPacket.reportBlocks)
}
is RtcpRrPacket -> {
val time = receivedTime ?: clock.instant()
onReport(time, rtcpPacket.reportBlocks)
}
}
}
private class LossReport {
var extendedHighestSequenceNumber: Long = 0
var cumulativeLost = 0
}
private val lastReportBlocks = HashMap<Long, LossReport>()
private var lastReportBlockTime = clock.instant()
private fun onReport(receiveTime: Instant, reportBlocks: List<RtcpReportBlock>) {
if (reportBlocks.isEmpty()) {
return
}
var totalPacketsLostDelta = 0L
var totalPacketsDelta = 0L
reportBlocks.forEach { reportBlock ->
val newLossReport = LossReport()
val lastLossReport = lastReportBlocks.putIfAbsent(reportBlock.ssrc, newLossReport)
if (lastLossReport != null) {
totalPacketsDelta += reportBlock.extendedHighestSeqNum - lastLossReport.extendedHighestSequenceNumber
totalPacketsLostDelta += reportBlock.cumulativePacketsLost - lastLossReport.cumulativeLost
}
val lossReport = lastLossReport ?: newLossReport
lossReport.extendedHighestSequenceNumber = reportBlock.extendedHighestSeqNum
lossReport.cumulativeLost = reportBlock.cumulativePacketsLost
}
// Can only compute delta if there has been previous blocks to compare to. If
// not, total_packets_delta will be unchanged and there's nothing more to do.
if (totalPacketsDelta == 0L) {
return
}
val packetsReceivedDelta = totalPacketsDelta - totalPacketsLostDelta
// To detect lost packets, at least one packet has to be received.
if (packetsReceivedDelta < 1) {
return
}
val msg = TransportLossReport(
packetsLostDelta = totalPacketsLostDelta,
packetsReceivedDelta = packetsReceivedDelta,
receiveTime = receiveTime,
startTime = lastReportBlockTime,
endTime = receiveTime
)
val update = networkController.onTransportLossReport(msg)
processUpdate(update)
lastReportBlockTime = receiveTime
}
@Synchronized
override fun mediaPacketTagged(packetInfo: org.jitsi.nlj.PacketInfo, tccSeqNum: Long) {
val now = clock.instant()
val length = packetInfo.packet.length.bytes
val pacedPacketInfo = packetInfo.probingInfo as? PacedPacketInfo
feedbackAdapter.addPacket(
packetInfo,
tccSeqNum,
DataSize.ZERO, // TODO: network overhead
creationTime = now
)
if (pacedPacketInfo == null) {
bitrateProber.onIncomingPacket(length)
}
maybeScheduleProbing(now)
}
@Synchronized
override fun mediaPacketSent(packetInfo: org.jitsi.nlj.PacketInfo, tccSeqNum: Long) {
val now = clock.instant()
val length = packetInfo.packet.length.toLong()
val sentPacketInfo = SentPacketInfo(
packetId = tccSeqNum,
sendTime = now,
info = PacketInfo(
// TODO I think these should always be true when tccSeqNum is defined?
includedInAllocation = true,
includedInFeedback = true,
packetSizeBytes = length
)
)
val sentPacket = feedbackAdapter.processSentPacket(sentPacketInfo)
if (sentPacket != null) {
val update = networkController.onSentPacket(sentPacket)
processUpdate(update)
}
}
@Synchronized
override fun getStatistics(): StatisticsSnapshot {
val now = clock.instant()
return StatisticsSnapshot(
feedbackAdapter.getStatisitics(),
(networkController as GoogCcNetworkController).getStatistics(now)
)
}
@Synchronized
override fun addBandwidthListener(listener: BandwidthListener) {
listeners.add(listener)
}
@Synchronized
override fun removeBandwidthListener(listener: BandwidthListener) {
listeners.remove(listener)
}
@Synchronized
override fun start() {
val startTime = clock.instant()
var update =
networkController.onNetworkAvailability(NetworkAvailability(atTime = startTime, networkAvailable = true))
processUpdate(update) // Does this make sense to do during init?
update = networkController.onProcessInterval(ProcessInterval(atTime = startTime))
processUpdate(update)
val processInterval = factory.getProcessInterval().toMillis()
processTask = scheduledExecutor.scheduleAtFixedRate({
synchronized(this@GoogCcTransportCcEngine) {
val now = clock.instant()
update = networkController.onProcessInterval(ProcessInterval(atTime = now))
processUpdate(update)
}
}, processInterval, processInterval, TimeUnit.MILLISECONDS)
}
@Synchronized
override fun stop() {
// Stop bitrateProber from initiating any new probes
bitrateProber.setEnabled(false)
probeTask?.cancel(false)
processTask?.cancel(false)
}
private fun processUpdate(update: NetworkControlUpdate) {
update.targetRate?.let { targetRate ->
logger.debug { "GoogleCcEstimator setting TargetRate to $targetRate" }
listeners.forEach { it.bandwidthEstimationChanged(targetRate.targetRate) }
}
update.congestionWindow?.let { congestionWindow ->
/* We don't use a congestion window */
/* TODO: does this do anything bad to the estimator? */
logger.trace { "GoogleCcEstimator wants to set CongestionWindow to $congestionWindow" }
}
update.pacerConfig?.let { pacerConfig ->
/* We don't use a pacer */
/* TODO: does this do anything bad to the estimator? */
logger.trace { "GoogleCcEstimator wants to set PacerConfig to $pacerConfig" }
}
update.probeClusterConfigs.let { configs ->
if (configs.isNotEmpty()) {
logger.debug { "GoogleCcEstimator creating ${configs.size} ProbeClusterConfigs: $configs" }
configs.forEach { config ->
bitrateProber.createProbeCluster(config)
}
maybeScheduleProbing(clock.instant())
}
}
if (timeSeriesLogger.isTraceEnabled && update.isNotEmpty()) {
val now = update.atTime ?: clock.instant()
val stats = (networkController as GoogCcNetworkController).getStatistics(now)
val statsPoint = diagnosticContext.makeTimeSeriesPoint("goog_cc_stats", now)
stats.addToTimeSeriesPoint(statsPoint)
timeSeriesLogger.trace(statsPoint)
val updatePoint = diagnosticContext.makeTimeSeriesPoint("goog_cc_update", now)
update.addToTimeSeriesPoint(updatePoint)
timeSeriesLogger.trace(updatePoint)
}
}
/** Schedule bitrate probing if needed and not current scheduled.
* Should be synchronized on this@GoogCcTransportCcEngine. */
private fun maybeScheduleProbing(now: Instant) {
if (bitrateProber.isProbing() && probeTask == null) {
val nextProbeTime = bitrateProber.nextProbeTime(now)
if (nextProbeTime == Instant.MAX) {
return
}
val delay = if (nextProbeTime == Instant.MIN) {
0
} else {
Duration.between(now, nextProbeTime).toMillis().coerceAtLeast(0)
}
probeTask = scheduledExecutor.schedule({
val cluster: PacedPacketInfo
val probeSize: DataSize
synchronized(this@GoogCcTransportCcEngine) {
probeTask = null
val now = clock.instant()
cluster = bitrateProber.currentCluster(now) ?: PacedPacketInfo()
if (cluster.probeClusterId == kNotAProbe) {
return@schedule
}
probeSize = bitrateProber.recommendedMinProbeSize()
val probeSent = sendProbing(probeSize, cluster)
bitrateProber.probeSent(now, probeSent.bytes)
maybeScheduleProbing(now)
}
}, delay, TimeUnit.MILLISECONDS)
}
}
companion object {
private val factory = GoogCcNetworkControllerFactory()
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(GoogCcTransportCcEngine::class.java)
/* Default config settings to use when this version of the Google transport CC estimator engine is used. */
val defaultRateTrackerWindowSize: Duration by config {
"jmt.bwe.estimator.GoogleCc2.default-window-size".from(JitsiConfig.newConfig)
}
val defaultRateTrackerBucketSize: Duration by config {
"jmt.bwe.estimator.GoogleCc2.default-bucket-size".from(JitsiConfig.newConfig)
}
val defaultInitialIgnoreBwePeriod: Duration by config {
"jmt.bwe.estimator.GoogleCc2.default-initial-ignore-bwe-period".from(JitsiConfig.newConfig)
}
}
class StatisticsSnapshot(
val transportAdapterState: TransportFeedbackAdapter.StatisticsSnapshot,
val networkControllerState: GoogCcNetworkController.StatisticsSnapshot
) : TransportCcEngine.StatisticsSnapshot() {
override val unmatchedFeedback: Long get() = transportAdapterState.totalUnmatchedReports
override fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("name", GoogCcTransportCcEngine::class.java.simpleName)
set<ObjectNode>("transport_adapter", transportAdapterState.toJson())
set<ObjectNode>("network_controller", networkControllerState.toJson())
}
}
}

181
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/InterArrivalDelta.kt Normal file
View File

@@ -0,0 +1,181 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.utils.NEVER
import org.jitsi.utils.logging2.createLogger
import org.jitsi.utils.max
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import java.time.Duration
import java.time.Instant
/**
* Helper class to compute the inter-arrival time delta and the size delta
* between two timestamp groups. This code is branched from
* modules/remote_bitrate_estimator/inter_arrival.
* *
* Based on WebRTC modules/congestion_controller/goog_cc/inter_arrival_delta.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
class InterArrivalDelta(
private val sendTimeGroupLength: Duration
) {
// TODO: pass parent logger in so we have log contexts
private val logger = createLogger()
private var currentTimestampGroup: SendTimeGroup = SendTimeGroup()
private var prevTimestampGroup: SendTimeGroup = SendTimeGroup()
private var numConsecutiveReorderedPackets: Int = 0
data class ComputeDeltasResult(
val computed: Boolean,
val sendTimeDelta: Duration = Duration.ZERO,
val arrivalTimeDelta: Duration = Duration.ZERO,
val packetSizeDelta: Int = 0
)
fun computeDeltas(
sendTime: Instant,
arrivalTime: Instant,
systemTime: Instant,
packetSize: Long
): ComputeDeltasResult {
var sendTimeDelta = Duration.ZERO
var arrivalTimeDelta = Duration.ZERO
var packetSizeDelta = 0
var calculatedDeltas = false
if (currentTimestampGroup.isFirstPacket()) {
// We don't have enough data to update the filter, so we store it until we
// have two frames of data to process.
currentTimestampGroup.sendTime = sendTime
currentTimestampGroup.firstSendTime = sendTime
currentTimestampGroup.firstArrival = arrivalTime
} else if (currentTimestampGroup.firstSendTime > sendTime) {
// Reordered packet
return ComputeDeltasResult(false)
} else if (newTimestampGroup(arrivalTime, sendTime)) {
// First packet of a later frame, the previous frame sample is ready.
if (prevTimestampGroup.completeTime != NEVER) {
sendTimeDelta = Duration.between(prevTimestampGroup.sendTime, currentTimestampGroup.sendTime)
arrivalTimeDelta = Duration.between(prevTimestampGroup.completeTime, currentTimestampGroup.completeTime)
val systemTimeDelta =
Duration.between(prevTimestampGroup.lastSystemTime, currentTimestampGroup.lastSystemTime)
if (arrivalTimeDelta - systemTimeDelta >= kArrivalTimeOffsetThreshold) {
logger.warn(
"The arrival clock offset has changed (diff = " +
"${arrivalTimeDelta - systemTimeDelta}), resetting"
)
reset()
return ComputeDeltasResult(false)
}
if (arrivalTimeDelta < Duration.ZERO) {
// The group of packets has been reordered since receiving its local
// arrival timestamp.
++numConsecutiveReorderedPackets
if (numConsecutiveReorderedPackets >= kReorderedResetThreshold) {
logger.warn(
"Packets between send burst arrived out of order, restting: " +
"arrivalTimeDelta=$arrivalTimeDelta, sendTimeDelta=$sendTimeDelta"
)
reset()
}
return ComputeDeltasResult(false)
} else {
numConsecutiveReorderedPackets = 0
}
packetSizeDelta = currentTimestampGroup.size.toInt() - prevTimestampGroup.size.toInt()
calculatedDeltas = true
}
prevTimestampGroup = currentTimestampGroup.copy()
// The new timestamp is now the current frame
currentTimestampGroup.firstSendTime = sendTime
currentTimestampGroup.sendTime = sendTime
currentTimestampGroup.firstArrival = arrivalTime
currentTimestampGroup.size = 0
} else {
currentTimestampGroup.sendTime = max(currentTimestampGroup.sendTime, sendTime)
}
// Accumulate the frame size.
currentTimestampGroup.size += packetSize
currentTimestampGroup.completeTime = arrivalTime
currentTimestampGroup.lastSystemTime = systemTime
return ComputeDeltasResult(calculatedDeltas, sendTimeDelta, arrivalTimeDelta, packetSizeDelta)
}
private fun newTimestampGroup(arrivalTime: Instant, sendTime: Instant): Boolean {
if (currentTimestampGroup.isFirstPacket()) {
return false
} else if (belongsToBurst(arrivalTime, sendTime)) {
return false
} else {
return Duration.between(currentTimestampGroup.firstSendTime, sendTime) > sendTimeGroupLength
}
}
private fun belongsToBurst(arrivalTime: Instant, sendTime: Instant): Boolean {
check(currentTimestampGroup.completeTime != NEVER)
val arrivalTimeDelta = Duration.between(currentTimestampGroup.completeTime, arrivalTime)
val sendTimeDelta = Duration.between(currentTimestampGroup.sendTime, sendTime)
if (sendTimeDelta == Duration.ZERO) {
return true
}
val propagationDelta = arrivalTimeDelta - sendTimeDelta
if (propagationDelta < Duration.ZERO &&
arrivalTimeDelta <= kBurstDeltaThreshold &&
Duration.between(currentTimestampGroup.firstArrival, arrivalTime) < kMaxBurstDuration
) {
return true
}
return false
}
private fun reset() {
numConsecutiveReorderedPackets = 0
currentTimestampGroup = SendTimeGroup()
prevTimestampGroup = SendTimeGroup()
}
data class SendTimeGroup(
var size: Long = 0,
var firstSendTime: Instant = NEVER,
var sendTime: Instant = NEVER,
var firstArrival: Instant = NEVER,
var completeTime: Instant = NEVER,
var lastSystemTime: Instant = NEVER
) {
fun isFirstPacket(): Boolean = completeTime == NEVER
}
companion object {
/**
* After this many packet groups received out of order InterArrival will
* reset, assuming that clocks have made a jump.
*/
const val kReorderedResetThreshold = 3
val kArrivalTimeOffsetThreshold = 3.secs
val kBurstDeltaThreshold = 5.ms
val kMaxBurstDuration = 100.ms
}
}

77
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudget.kt Normal file
View File

@@ -0,0 +1,77 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file uses WebRTC's naming style for constants
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import kotlin.math.max
import kotlin.math.min
/** Interval budget,
* based on WebRTC modules/pacing/interval_budget.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
class IntervalBudget(
initialTargetRateKbps: Int,
private val canBuildUpUnderuse: Boolean = false
) {
private var targetRateKbps: Int = 0
private var maxBytesInBudget: Long = 0
private var bytesRemaining: Long = 0
init {
setTargetRateKbps(initialTargetRateKbps)
}
fun setTargetRateKbps(targetRateKbps: Int) {
this.targetRateKbps = targetRateKbps
maxBytesInBudget = (kWindowMs * targetRateKbps) / 8
bytesRemaining = min(max(-maxBytesInBudget, bytesRemaining), maxBytesInBudget)
}
// TODO(tschumim): Unify IncreaseBudget and UseBudget to one function.
fun increaseBudget(deltaTimeMs: Long) {
val bytes = targetRateKbps * deltaTimeMs / 8
if (bytesRemaining < 0 || canBuildUpUnderuse) {
// We overused last interval, compensate this interval.
bytesRemaining = min(bytesRemaining + bytes, maxBytesInBudget)
} else {
// If we underused last interval we can't use it this interval.
bytesRemaining = min(bytes, maxBytesInBudget)
}
}
fun useBudget(bytes: Long) {
bytesRemaining = max(bytesRemaining - bytes, -maxBytesInBudget)
}
fun bytesRemaining(): Long = max(0, bytesRemaining)
fun budgetRatio(): Double {
if (maxBytesInBudget == 0L) {
return 0.0
}
return bytesRemaining.toDouble() / maxBytesInBudget
}
fun targetRateKbps(): Int = targetRateKbps
companion object {
private const val kWindowMs = 500L
}
}

75
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LinkCapacityEstimator.kt Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.kbps
import kotlin.math.max
import kotlin.math.sqrt
/** Link capacity estimator,
* based on WebRTC modules/congestion_controller/goog_cc/link_capacity_estimator.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
class LinkCapacityEstimator {
private var estimateKbps: Double? = null
private var deviationKbps: Double = 0.4
fun upperBound(): Bandwidth = estimateKbps?.let {
(it + 3 * deviationEstimateKbps()).kbps
} ?: Bandwidth.INFINITY
fun lowerBound(): Bandwidth = estimateKbps?.let {
(max(0.0, it - 3 * deviationEstimateKbps())).kbps
} ?: Bandwidth.ZERO
fun reset() {
estimateKbps = null
}
fun onOveruseDetected(acknowledgedRate: Bandwidth) = update(acknowledgedRate, 0.05)
fun onProbeRate(probeRate: Bandwidth) = update(probeRate, 0.5)
fun hasEstimate(): Boolean = estimateKbps != null
val estimate: Bandwidth?
get() = estimateKbps?.kbps
private fun update(capacitySample: Bandwidth, alpha: Double) {
// This double-cast is probably a bug in the Google CC code (it calls `capacity_sample.kbps()` rather than
// `capacity_sample.kbps<double>()`) but emulate it to be bit-exact.
val sampleKbps = capacitySample.kbps.toLong().toDouble()
estimateKbps = if (estimateKbps == null) {
sampleKbps
} else {
(1 - alpha) * estimateKbps!! + alpha * sampleKbps
}
// Estimate the variance of the link capacity estimate and normalize the
// variance with the link capacity estimate.
val norm = max(estimateKbps!!, 1.0)
val errorKbps = estimateKbps!! - sampleKbps
deviationKbps = (1 - alpha) * deviationKbps + alpha * errorKbps * errorKbps / norm
// 0.4 ~= 14 kbit/s at 500 kbit/s
// 2.5 ~= 35 kbit/s at 500 kbit/s
deviationKbps = deviationKbps.coerceIn(0.4, 2.5)
}
private fun deviationEstimateKbps(): Double = // Calculate the max bit rate std dev given the normalized
// variance and the current throughput bitrate. The standard deviation will
// only be used if estimateKbps has a value.
sqrt(deviationKbps * estimateKbps!!)
}

1169
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2.kt Normal file
View File

File diff suppressed because it is too large Load Diff

102
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkController.kt Normal file
View File

@@ -0,0 +1,102 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import java.time.Duration
/* Base type for network controller,
* based on WebRTC api/transport/network_control.h in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
/** Configuration sent to factory create function. The parameters here are
* optional to use for a network controller implementation. */
class NetworkControllerConfig(
val parentLogger: Logger,
val diagnosticContext: DiagnosticContext,
/** The initial constraints to start with, these can be changed at any later
* time by calls to OnTargetRateConstraints. Note that the starting rate
* has to be set initially to provide a starting state for the network
* controller, even though the field is marked as nullable.
*/
val constraints: TargetRateConstraints = TargetRateConstraints(),
/** Initial stream specific configuration, these are changed at any later time
* by calls to OnStreamsConfig.
*/
val streamBasedConfig: StreamsConfig = StreamsConfig()
)
/** NetworkControllerInterface is implemented by network controllers. A network
* controller is a class that uses information about network state and traffic
* to estimate network parameters such as round trip time and bandwidth. Network
* controllers does not guarantee thread safety, the interface must be used in a
* non-concurrent fashion.
*/
interface NetworkControllerInterface {
/** Called when network availabilty changes */
fun onNetworkAvailability(msg: NetworkAvailability): NetworkControlUpdate
/** Called when the receiving or sending endpoint changes address.*/
fun onNetworkRouteChange(msg: NetworkRouteChange): NetworkControlUpdate
/** Called periodically with a periodicy as specified by
* [NetworkControllerFactoryInterface.getProcessInterval].
*/
fun onProcessInterval(msg: ProcessInterval): NetworkControlUpdate
/** Called when remotely calculated bitrate is received. */
fun onRemoteBitrateReport(msg: RemoteBitrateReport): NetworkControlUpdate
/** Called round trip time has been calculated by protocol specific mechanisms. */
fun onRoundTripTimeUpdate(msg: RoundTripTimeUpdate): NetworkControlUpdate
/** Called when a packet is sent on the network. */
fun onSentPacket(sentPacket: SentPacket): NetworkControlUpdate
// Omitted: onReceivedPacket: Not used
/** Called when the stream specific configuration has been updated. */
fun onStreamsConfig(msg: StreamsConfig): NetworkControlUpdate
/** Called when target transfer rate constraints has been changed. */
fun onTargetRateConstraints(constraints: TargetRateConstraints): NetworkControlUpdate
/** Called when a protocol specific calculation of packet loss has been made. */
fun onTransportLossReport(msg: TransportLossReport): NetworkControlUpdate
/** Called with per packet feedback regarding receive time. */
fun onTransportPacketsFeedback(report: TransportPacketsFeedback): NetworkControlUpdate
// Omitted: onNetworkStateUpdate: Not used
}
/** NetworkControllerFactoryInterface is an interface for creating a network
* controller. */
interface NetworkControllerFactoryInterface {
/** Used to create a new network controller, requires an observer to be
* provided to handle callbacks.
*/
fun create(config: NetworkControllerConfig): NetworkControllerInterface
/** Returns the interval by which the network controller expects
* OnProcessInterval calls.
*/
fun getProcessInterval(): Duration
}

322
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkTypes.kt Normal file
View File

@@ -0,0 +1,322 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bps
import org.jitsi.nlj.util.per
import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.EcnMarking
import org.jitsi.utils.MAX_DURATION
import org.jitsi.utils.NEVER
import org.jitsi.utils.isFinite
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.ms
import org.jitsi.utils.toDoubleMillis
import java.time.Duration
import java.time.Instant
/* Common network types used for bandwidth estimation,
* based on WebRTC api/transport/network_types.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
/* Configuration */
/** Represents constraints and rates related to the currently enabled streams.
* This is used as input to the congestion controller via the StreamsConfig
* struct.
*/
class BitrateAllocationLimits(
/** The total minimum send bitrate required by all sending streams. */
val minAllocatableRate: Bandwidth = Bandwidth.ZERO,
/** The total maximum allocatable bitrate for all currently available streams. */
val maxAllocatableRate: Bandwidth = Bandwidth.ZERO,
/** The max bitrate to use for padding. The sum of the per-stream max padding
* rate. */
val maxPaddingRate: Bandwidth = Bandwidth.ZERO
)
/**
* Use StreamsConfig for information about streams that is required for specific
* adjustments to the algorithms in network controllers. Especially useful
* for experiments.
*/
class StreamsConfig(
val atTime: Instant = Instant.MAX,
val requestsAlrProbing: Boolean? = null,
// If `enable_repeated_initial_probing` is set to true, Probes are sent
// periodically every 1s during the first 5s after the network becomes
// available. The probes ignores max_total_allocated_bitrate.
val enableRepeatedInitialProbing: Boolean? = null,
val pacingFactor: Double? = null,
// TODO(srte): Use BitrateAllocationLimits here.
val minTotalAllocatedBitrate: Bandwidth? = null,
val maxPaddingRate: Bandwidth? = null,
val maxTotalAllocatedBitrate: Bandwidth? = null
)
class TargetRateConstraints(
val atTime: Instant = Instant.MAX,
val minDataRate: Bandwidth? = null,
val maxDataRate: Bandwidth? = null,
// The initial bandwidth estimate to base target rate on. This should be used
// as the basis for initial OnTargetTransferRate and OnPacerConfig callbacks.
var startingRate: Bandwidth? = null
)
/** Send side information */
class NetworkAvailability(
val atTime: Instant = Instant.MAX,
val networkAvailable: Boolean = false
)
class NetworkRouteChange(
val atTime: Instant = Instant.MAX,
// The TargetRateConstraints are set here so they can be changed synchronously
// when network route changes.
val constraints: TargetRateConstraints = TargetRateConstraints()
)
/**
* Information about a paced packet
*/
data class PacedPacketInfo(
val probeClusterId: Int = kNotAProbe,
val probeClusterMinProbes: Int = -1,
val probeClusterMinBytes: Int = -1,
var sendBitrate: Bandwidth = 0.bps
) {
// TODO(srte): Move probing info to a separate, optional struct.
var probeClusterBytesSent = 0
companion object {
const val kNotAProbe = -1
}
}
/**
* A sent packet
*/
data class SentPacket(
var sendTime: Instant = NEVER,
/** Size of packet with overhead up to IP layer. */
var size: DataSize = DataSize.ZERO,
/** Size of preceeding packets that are not part of feedback */
var priorUnackedData: DataSize = DataSize.ZERO,
/** Probe cluster id and parameters including bitrate, number of packets and
number of bytes. */
var pacingInfo: PacedPacketInfo = PacedPacketInfo(),
/** True if the packet is an audio packet, false for video, padding, RTX, etc. */
var audio: Boolean = false,
/** Transport independent sequence number, any tracked packet should have a
sequence number that is unique over the whole call and increasing by 1 for
each packet. */
var sequenceNumber: Long = 0,
/** Tracked data in flight when the packet was sent, excluding unacked data. */
var dataInFlight: DataSize = DataSize.ZERO
)
/** Transport level feedback */
class RemoteBitrateReport(
val receiveTime: Instant = Instant.MAX,
val bandwidth: Bandwidth = Bandwidth.INFINITY
)
class RoundTripTimeUpdate(
val receiveTime: Instant = Instant.MAX,
val roundTripTime: Duration = MAX_DURATION,
val smoothed: Boolean = false
)
class TransportLossReport(
val receiveTime: Instant = Instant.MAX,
val startTime: Instant = Instant.MAX,
val endTime: Instant = Instant.MAX,
val packetsLostDelta: Long = 0,
val packetsReceivedDelta: Long = 0
)
/**
* The result of packet feedback
*/
class PacketResult(
var sentPacket: SentPacket = SentPacket(),
var receiveTime: Instant = NEVER,
var ecn: EcnMarking = EcnMarking.kNotEct,
// `rtp_packet_info` is only set if the feedback is related to a RTP packet.
var rtpPacketInfo: RtpPacketInfo? = null,
// Jitsi extension: Whether a packet was previously reported lost
var previouslyReportedLost: Boolean = false
) {
data class RtpPacketInfo(
val ssrc: Long = 0,
val rtpSequenceNumber: Int = 0,
val isRetransmission: Boolean = false
)
fun isReceived() = receiveTime.isFinite()
fun copy(): PacketResult = PacketResult(sentPacket.copy(), receiveTime, ecn, rtpPacketInfo, previouslyReportedLost)
}
/**
* Summary of transport packets feedback
*/
class TransportPacketsFeedback {
var feedbackTime: Instant = NEVER
var dataInFlight: DataSize = DataSize.ZERO
var packetFeedbacks: MutableList<PacketResult> = ArrayList()
var transportSupportsEcn: Boolean = false
/** Arrival times for messages without send times information */
val sendlessArrivalTimes = ArrayList<Instant>()
fun receivedWithSendInfo(): List<PacketResult> = packetFeedbacks.filter { it.isReceived() }
fun lostWithSendInfo(): List<PacketResult> = packetFeedbacks.filterNot { it.isReceived() }
fun packetsWithFeedback(): List<PacketResult> = packetFeedbacks
fun sortedByReceiveTime(): List<PacketResult> = receivedWithSendInfo().sortedBy { it.receiveTime }
}
// Network estimation
class NetworkEstimate {
var atTime = Instant.MAX
// Deprecated, use TargetTransferRate::target_rate instead.
var bandwidth = Bandwidth.INFINITY
var roundTripTime = MAX_DURATION
var bwePeriod = MAX_DURATION
var lossRateRatio = 0.0f
/* Jitsi local */
override fun toString(): String = "atTime $atTime: " +
"bandwidth $bandwidth, rtt $roundTripTime, bwePeriod $bwePeriod, lossRateRatio $lossRateRatio"
}
class PacerConfig {
var atTime = Instant.MAX
// Pacer should send at most data_window data over time_window duration.
var dataWindow = DataSize.INFINITY
var timeWindow = MAX_DURATION
// Pacer should send at least pad_window data over time_window duration.
var padWindow = DataSize.ZERO
fun dataRate() = dataWindow.per(timeWindow)
fun padRate() = padWindow.per(timeWindow)
/* Jitsi Local */
override fun toString(): String = "Data rate ${dataRate()} ($dataWindow per $timeWindow), " +
"pad rate ${padRate()} ($padWindow per $timeWindow)"
}
data class ProbeClusterConfig(
var atTime: Instant = Instant.MAX,
var targetDataRate: Bandwidth = Bandwidth.ZERO,
// Duration of a probe.
var targetDuration: Duration = Duration.ZERO,
// Delta time between sent bursts of packets during probe.
var minProbeDelta: Duration = 2.ms,
var targetProbeCount: Int = 0,
var id: Int = 0
) {
/* Jitsi local */
override fun toString(): String = "atTime $atTime: ID=$id: DataRate $targetDataRate Duration $targetDuration " +
"ProbeDelta $minProbeDelta ProbeCount $targetProbeCount"
}
class TargetTransferRate {
var atTime = Instant.MAX
// The estimate on which the target rate is based on.
var networkEstimate = NetworkEstimate()
var targetRate = Bandwidth.ZERO
var stableTargetRate = Bandwidth.ZERO
var cwndReduceRatio = 0.0
/* Jitsi local */
override fun toString(): String = "atTime $atTime: networkEstimate {$networkEstimate}, " +
"targetRate $targetRate, stableTargetRate $stableTargetRate, cwndReduceRatio $cwndReduceRatio"
}
// Contains updates of network controller comand state. Using nullables to
// indicate whether a member has been updated. The array of probe clusters
// should be used to send out probes if not empty.
open class NetworkControlUpdate(
open val congestionWindow: DataSize? = null,
open val pacerConfig: PacerConfig? = null,
open val probeClusterConfigs: List<ProbeClusterConfig> = listOf(),
open val targetRate: TargetTransferRate? = null
) {
/* Jitsi local */
fun isEmpty(): Boolean =
congestionWindow == null && pacerConfig == null && probeClusterConfigs.isEmpty() && targetRate == null
fun isNotEmpty() = !isEmpty()
val atTime: Instant?
get() = targetRate?.atTime ?: probeClusterConfigs.firstOrNull()?.atTime ?: pacerConfig?.atTime
fun addToTimeSeriesPoint(point: DiagnosticContext.TimeSeriesPoint) {
targetRate?.let {
point.addField("target_rate_bps", it.targetRate.bps)
point.addField("stable_target_rate_bps", it.stableTargetRate.bps)
point.addField("cwnd_reduce_ratio", it.cwndReduceRatio)
point.addField("rtt_ms", it.networkEstimate.roundTripTime.toDoubleMillis())
point.addField("bwe_period_ms", it.networkEstimate.bwePeriod.toDoubleMillis())
point.addField("loss_rate_ratio", it.networkEstimate.lossRateRatio)
}
congestionWindow?.let {
point.addField("congestion_window_bytes", it.bytes)
}
pacerConfig?.let {
point.addField("pacer_data_rate_bps", it.dataRate().bps)
point.addField("pacer_pad_rate_bps", it.padRate().bps)
}
probeClusterConfigs.forEachIndexed { i, it ->
point.addField("probe_cluster_" + i + "_id", it.id)
point.addField("probe_cluster_" + i + "_data_rate_bps", it.targetDataRate.bps)
point.addField("probe_cluster_" + i + "_duration_ms", it.targetDuration.toDoubleMillis())
point.addField("probe_cluster_" + i + "_delta_ms", it.minProbeDelta.toDoubleMillis())
point.addField("probe_cluster_" + i + "_count", it.targetProbeCount)
}
}
}
class MutableNetworkControlUpdate(
override var congestionWindow: DataSize? = null,
override var pacerConfig: PacerConfig? = null,
override var probeClusterConfigs: MutableList<ProbeClusterConfig> = mutableListOf(),
override var targetRate: TargetTransferRate? = null
) : NetworkControlUpdate()
/** Process control */
class ProcessInterval(
val atTime: Instant = Instant.MAX,
val pacerQueue: DataSize? = null
)

204
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimator.kt Normal file
View File

@@ -0,0 +1,204 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.DataSize
import org.jitsi.nlj.util.bytes
import org.jitsi.nlj.util.min
import org.jitsi.nlj.util.per
import org.jitsi.nlj.util.times
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.secs
import java.time.Duration
import java.time.Instant
import java.util.*
/**
* Basic implementation to estimate bitrate of probes.
*
* Based on WebRTC modules/congestion_controller/goog_cc/probe_bitrate_estimator.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*/
class ProbeBitrateEstimator(
parentLogger: Logger,
private val diagnosticContext: DiagnosticContext
) {
// TODO: pass parent logger in so we have log contexts
private val logger = createChildLogger(parentLogger)
private val clusters = TreeMap<Int, AggregatedCluster>()
private var estimatedDataRate: Bandwidth? = null
fun handleProbeAndEstimateBitrate(packetFeedback: PacketResult): Bandwidth? {
val clusterId = packetFeedback.sentPacket.pacingInfo.probeClusterId
check(clusterId != PacedPacketInfo.kNotAProbe)
eraseOldClusters(packetFeedback.receiveTime)
val cluster = clusters.getOrPut(clusterId) { AggregatedCluster() }
if (packetFeedback.sentPacket.sendTime < cluster.firstSend) {
cluster.firstSend = packetFeedback.sentPacket.sendTime
}
if (packetFeedback.sentPacket.sendTime > cluster.lastSend) {
cluster.lastSend = packetFeedback.sentPacket.sendTime
cluster.sizeLastSend = packetFeedback.sentPacket.size
}
if (packetFeedback.receiveTime < cluster.firstReceive) {
cluster.firstReceive = packetFeedback.receiveTime
cluster.sizeFirstReceive = packetFeedback.sentPacket.size
}
if (packetFeedback.receiveTime > cluster.lastReceive) {
cluster.lastReceive = packetFeedback.receiveTime
}
cluster.sizeTotal += packetFeedback.sentPacket.size
cluster.numProbes += 1
check(packetFeedback.sentPacket.pacingInfo.probeClusterMinProbes > 0)
check(packetFeedback.sentPacket.pacingInfo.probeClusterMinBytes > 0)
val minProbes = packetFeedback.sentPacket.pacingInfo.probeClusterMinProbes *
kMinReceivedProbesRatio
val minSize = packetFeedback.sentPacket.pacingInfo.probeClusterMinBytes.bytes *
kMinReceivedBytesRatio
if (cluster.numProbes < minProbes || cluster.sizeTotal < minSize) {
return null
}
val sendInterval = Duration.between(cluster.firstSend, cluster.lastSend)
val receiveInterval = Duration.between(cluster.firstReceive, cluster.lastReceive)
if (sendInterval <= Duration.ZERO || sendInterval > kMaxProbeInterval ||
receiveInterval <= Duration.ZERO ||
receiveInterval > kMaxProbeInterval
) {
logger.info(
"Probing unsuccessful, invalid send/receive interval " +
"[cluster id: $clusterId] [sendInterval: $sendInterval] " +
"[receive interval: $receiveInterval]"
)
return null
}
// Since the `send_interval` does not include the time it takes to actually
// send the last packet the size of the last sent packet should not be
// included when calculating the send bitrate.
check(cluster.sizeTotal >= cluster.sizeLastSend)
val sendSize = cluster.sizeTotal - cluster.sizeLastSend
val sendRate = sendSize.per(sendInterval)
// Since the `receive_interval` does not include the time it takes to
// actually receive the first packet the size of the first received packet
// should not be included when calculating the receive bitrate.
check(cluster.sizeTotal >= cluster.sizeFirstReceive)
val receiveSize = cluster.sizeTotal - cluster.sizeFirstReceive
val receiveRate = receiveSize.per(receiveInterval)
val ratio = receiveRate / sendRate
if (ratio > kMaxValidRatio) {
logger.info(
"Probing unsuccessful, receive/send ratio too high " +
"[cluster id: $clusterId] [send: $sendSize/$sendInterval = $sendRate] " +
"[receive: $receiveSize/$receiveInterval = $receiveRate] " +
"[ratio: $receiveRate / $sendRate = $ratio > kMaxValidRatio ($kMaxValidRatio)]"
)
return null
}
logger.info(
"Probing successful [cluster id: $clusterId] " +
"[send: $sendSize / $sendInterval = $sendRate]" +
"[receive: $receiveSize / $receiveInterval = $receiveRate]"
)
var res = min(sendRate, receiveRate)
// If we're receiving at significantly lower bitrate than we were sending at,
// it suggests that we've found the true capacity of the link. In this case,
// set the target bitrate slightly lower to not immediately overuse.
if (receiveRate < kMinRatioForUnsaturatedLink * sendRate) {
check(sendRate > receiveRate)
res = kTargetUtilizationFraction * receiveRate
}
timeSeriesLogger.trace {
diagnosticContext.makeTimeSeriesPoint("probe_result_success")
.addField("id", clusterId)
.addField("bitrate_bps", res.bps)
}
estimatedDataRate = res
return estimatedDataRate
}
fun fetchAndResetLastEstimatedBitrate(): Bandwidth? {
val estimatedDataRate = this.estimatedDataRate
this.estimatedDataRate = null
return estimatedDataRate
}
private fun eraseOldClusters(timestamp: Instant) {
clusters.entries.removeIf {
it.value.lastReceive + kMaxClusterHistory < timestamp
}
}
private data class AggregatedCluster(
var numProbes: Int = 0,
var firstSend: Instant = Instant.MAX,
var lastSend: Instant = Instant.MIN,
var firstReceive: Instant = Instant.MAX,
var lastReceive: Instant = Instant.MIN,
var sizeLastSend: DataSize = DataSize.ZERO,
var sizeFirstReceive: DataSize = DataSize.ZERO,
var sizeTotal: DataSize = DataSize.ZERO
)
companion object {
/** The minumum number of probes we need to receive feedback about in percent
* in order to have a valid estimate. */
const val kMinReceivedProbesRatio = 0.80
/** The minumum number of bytes we need to receive feedback about in percent
* in order to have a valid estimate. */
const val kMinReceivedBytesRatio = 0.80
/** The maximum |receive rate| / |send rate| ratio for a valid estimate. */
const val kMaxValidRatio = 2.0f
/** The minimum |receive rate| / |send rate| ratio assuming that the link is
* not saturated, i.e. we assume that we will receive at least
* kMinRatioForUnsaturatedLink * |send rate| if |send rate| is less than the
* link capacity. */
const val kMinRatioForUnsaturatedLink = 0.9
/* The target utilization of the link. If we know true link capacity
* we'd like to send at 95% of that rate. */
const val kTargetUtilizationFraction = 0.95
/* The maximum time period over which the cluster history is retained.
* This is also the maximum time period beyond which a probing burst is not
* expected to last. */
val kMaxClusterHistory = 1.secs
/* The maximum time interval between first and the last probe on a cluster
* on the sender side as well as the receive side. */
val kMaxProbeInterval = 1.secs
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(ProbeBitrateEstimator::class.java)
}
}

607
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeController.kt Normal file
View File

@@ -0,0 +1,607 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:enum-entry-name-case")
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.kbps
import org.jitsi.nlj.util.min
import org.jitsi.nlj.util.times
import org.jitsi.utils.MAX_DURATION
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.utils.max
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import java.time.Duration
import java.time.Instant
/** Probe controller,
* based on WebRTC modules/congestion_controller/goog_cc/probe_controller.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings.
*/
// Maximum waiting time from the time of initiating probing to getting
// the measured results back.
private val kMaxWaitingTimeForProbingResult = 1.secs
// Default probing bitrate limit. Applied only when the application didn't
// specify max bitrate.
private val kDefaultMaxProbingBitrate = 5000.kbps
// If the bitrate drops to a factor `kBitrateDropThreshold` or lower
// and we recover within `kBitrateDropTimeoutMs`, then we'll send
// a probe at a fraction `kProbeFractionAfterDrop` of the original bitrate.
private const val kBitrateDropThreshold = 0.66
private val kBitrateDropTimeout = 5.secs
private const val kProbeFractionAfterDrop = 0.85
// Timeout for probing after leaving ALR. If the bitrate drops significantly,
// (as determined by the delay based estimator) and we leave ALR, then we will
// send a probe if we recover within `kLeftAlrTimeoutMs` ms.
private val kAlrEndedTimeout = 3.secs
// This is a limit on how often probing can be done when there is a BW
// drop detected in ALR.
private val kMinTimeBetweenAlrProbes = 5.secs
// The expected uncertainty of probe result (as a fraction of the target probe
// bitrate). Used to avoid probing if the probe bitrate is close to our current
// estimate.
private const val kProbeUncertainty = 0.05
class ProbeControllerConfig(
// These parameters configure the initial probes. First we send one or two
// probes of sizes p1 * start_bitrate_ and p2 * start_bitrate_.
// Then whenever we get a bitrate estimate of at least further_probe_threshold
// times the size of the last sent probe we'll send another one of size
// step_size times the new estimate.
val firstExponentialProbeScale: Double = 3.0,
val secondExponentialProbeScale: Double? = 6.0,
val furtherExponentialProbeScale: Double = 2.0,
val furtherProbeThreshold: Double = 0.7,
val abortFurtherProbeIfMaxLowerThanCurrent: Boolean = false,
val repeatedInitialProbingTimePeriod: Duration = 5.secs,
// The minimum probing duration of an individual probe during
// the repeated_initial_probing_time_period.
val initialProbeDuration: Duration = 100.ms,
// Delta time between sent bursts of packets in a probe during
// the repeated_initial_probing_time_period.
val initialMinProbeDelta: Duration = 20.ms,
// Configures how often we send ALR probes and how big they are.
val alrProbingInterval: Duration = 5.secs,
val alrProbeScale: Double = 2.0,
// Configures how often we send probes if NetworkStateEstimate is available.
val networkStateEstimateProbingInterval: Duration = MAX_DURATION,
// Periodically probe as long as the ratio between current estimate and
// NetworkStateEstimate is lower then this.
val probeIfEstimateLowerThanNetworkStateEstimateRatio: Double = 0.0,
val estimateLowerThanNetworkStateProbingInterval: Duration = 3.secs,
val networkStateProbeScale: Double = 1.0,
// Overrides min_probe_duration if network_state_estimate_probing_interval
// is set and a network state estimate is known.
val networkStateProbeDuration: Duration = 15.ms,
// Overrides min_probe_delta if network_state_estimate_probing_interval
// is set and a network state estimate is known and equal or higher than the
// probe target.
val networkStateMinProbeDelta: Duration = 20.ms,
// Configures the probes emitted by changed to the allocated bitrate.
val probeOnMaxAllocatedBitrateChange: Boolean = true,
val firstAllocationProbeScale: Double? = 1.0,
val secondAllocationProbeScale: Double? = 2.0,
val allocationProbeLimitByCurrentScale: Double = 2.0,
// The minimum number probing packets used.
val minProbePacketsSent: Int = 5,
// The minimum probing duration.
val minProbeDuration: Duration = 15.ms,
// Delta time between sent bursts of packets in a probe.
val minProbeDelta: Duration = 2.ms,
val lossLimitedProbeScale: Double = 1.5,
// Don't send a probe if min(estimate, network state estimate) is larger than
// this fraction of the set max or max allocated bitrate.
val skipIfEstimateLargerThanFractionOfMax: Double = 0.0,
// Scale factor of the max allocated bitrate. Used when deciding if a probe
// can be skiped due to that the estimate is already high enough.
val skipProbeMaxAllocatedScale: Double = 1.0,
)
/* Reason that bandwidth estimate is limited. Bandwidth estimate can be limited
* by either delay based bwe, or loss based bwe when it increases/decreases the
* estimate.
*/
enum class BandwidthLimitedCause {
kLossLimitedBweIncreasing,
kLossLimitedBwe,
kDelayBasedLimited,
kDelayBasedLimitedDelayIncreased,
kRttBasedBackOffHighRtt
}
/* This class controls initiation of probing to estimate initial channel
* capacity. There is also support for probing during a session when max
* bitrate is adjusted by an application.
*/
class ProbeController(
parentLogger: Logger,
private val diagnosticContext: DiagnosticContext,
private val config: ProbeControllerConfig = ProbeControllerConfig()
) {
private val logger = createChildLogger(parentLogger)
private var networkAvailable = false
private var repeatedInitialProbingEnabled = false
private var lastAllowedRepeatedInitialProbe = Instant.MIN
private var bandwidthLimitedCause = BandwidthLimitedCause.kDelayBasedLimited
private var state = State.kInit
private var minBitrateToProbeFurther = Bandwidth.INFINITY
private var timeLastProbingInitiated = Instant.MIN
private var estimatedBitrate = Bandwidth.ZERO
/* Skipping network_estimate */
private var startBitrate = Bandwidth.ZERO
private var maxBitrate = Bandwidth.INFINITY
private var lastBweDropProbingTime = Instant.MIN
private var alrStartTime: Instant? = null
private var alrEndTime: Instant? = null
private var enablePeriodicAlrProbing = false
private var timeOfLastLargeDrop = Instant.MIN
private var bitrateBeforeLastLargeDrop = Bandwidth.ZERO
private var maxTotalAllocatedBitrate = Bandwidth.ZERO
private val inRapidRecoveryExperiment = false
private var nextProbeClusterId = 1
fun setBitrates(
minBitrate: Bandwidth,
startBitrate: Bandwidth,
maxBitrate: Bandwidth,
atTime: Instant
): MutableList<ProbeClusterConfig> {
if (startBitrate > Bandwidth.ZERO) {
this.startBitrate = startBitrate
estimatedBitrate = startBitrate
} else if (this.startBitrate == Bandwidth.ZERO) {
this.startBitrate = minBitrate
}
// The reason we use the variable `old_max_bitrate_pbs` is because we
// need to set `max_bitrate_` before we call InitiateProbing.
val oldMaxBitrate = this.maxBitrate
this.maxBitrate = if (maxBitrate.isFinite()) {
maxBitrate
} else {
kDefaultMaxProbingBitrate
}
when (state) {
State.kInit ->
if (networkAvailable) {
return initiateExponentialProbing(atTime)
}
State.kWaitingForProbingResult ->
Unit
State.kProbingComplete ->
// If the new max bitrate is higher than both the old max bitrate and the
// estimate then initiate probing.
if (estimatedBitrate != Bandwidth.ZERO && oldMaxBitrate < this.maxBitrate &&
estimatedBitrate < this.maxBitrate
) {
return initiateProbing(atTime, listOf(maxBitrate), false)
}
}
return mutableListOf()
}
// The total bitrate, as opposed to the max bitrate, is the sum of the
// configured bitrates for all active streams.
fun onMaxTotalAllocatedBitrate(
maxTotalAllocatedBitrate: Bandwidth,
atTime: Instant
): MutableList<ProbeClusterConfig> {
val inAlr = alrStartTime != null
val allowAllocationProbe = inAlr
if (config.probeOnMaxAllocatedBitrateChange &&
state == State.kProbingComplete &&
maxTotalAllocatedBitrate != this.maxTotalAllocatedBitrate &&
estimatedBitrate < maxBitrate &&
estimatedBitrate < maxTotalAllocatedBitrate &&
allowAllocationProbe
) {
this.maxTotalAllocatedBitrate = maxTotalAllocatedBitrate
if (config.firstAllocationProbeScale == null) {
return mutableListOf()
}
var firstProbeRate = maxTotalAllocatedBitrate * config.firstAllocationProbeScale
val currentBweLimit =
config.allocationProbeLimitByCurrentScale *
estimatedBitrate
var limitedByCurrentBwe = currentBweLimit < firstProbeRate
if (limitedByCurrentBwe) {
firstProbeRate = currentBweLimit
}
val probes = mutableListOf(firstProbeRate)
if (!limitedByCurrentBwe && config.secondAllocationProbeScale != null) {
var secondProbeRate = maxTotalAllocatedBitrate * config.secondAllocationProbeScale
limitedByCurrentBwe = currentBweLimit < secondProbeRate
if (limitedByCurrentBwe) {
secondProbeRate = currentBweLimit
}
if (secondProbeRate > firstProbeRate) {
probes.add(secondProbeRate)
}
}
val allowFurtherProbing = limitedByCurrentBwe
return initiateProbing(atTime, probes, allowFurtherProbing)
}
if (maxTotalAllocatedBitrate != Bandwidth.ZERO) {
lastAllowedRepeatedInitialProbe = atTime
}
this.maxTotalAllocatedBitrate = maxTotalAllocatedBitrate
return mutableListOf()
}
fun onNetworkAvailability(msg: NetworkAvailability): MutableList<ProbeClusterConfig> {
networkAvailable = msg.networkAvailable
if (!networkAvailable && state == State.kWaitingForProbingResult) {
state = State.kProbingComplete
minBitrateToProbeFurther = Bandwidth.INFINITY
}
if (networkAvailable && state == State.kInit && startBitrate != Bandwidth.ZERO) {
return initiateExponentialProbing(msg.atTime)
}
return mutableListOf()
}
private fun updateState(newState: State) {
when (newState) {
State.kInit ->
state = State.kInit
State.kWaitingForProbingResult ->
state = State.kWaitingForProbingResult
State.kProbingComplete -> {
state = State.kProbingComplete
minBitrateToProbeFurther = Bandwidth.INFINITY
}
}
}
private fun initiateExponentialProbing(atTime: Instant): MutableList<ProbeClusterConfig> {
assert(networkAvailable)
assert(state == State.kInit)
assert(startBitrate > Bandwidth.ZERO)
// When probing at 1.8 Mbps ( 6x 300), this represents a threshold of
// 1.2 Mbps to continue probing.
val probes = mutableListOf(config.firstExponentialProbeScale * startBitrate)
if (config.secondExponentialProbeScale != null && config.secondExponentialProbeScale > 0.0) {
probes.add(config.secondExponentialProbeScale * startBitrate)
}
if (repeatedInitialProbingEnabled && maxTotalAllocatedBitrate == Bandwidth.ZERO) {
lastAllowedRepeatedInitialProbe =
atTime + config.repeatedInitialProbingTimePeriod
logger.info {
"Repeated initial probing enabled, last allowed probe: $lastAllowedRepeatedInitialProbe now: $atTime"
}
}
return initiateProbing(atTime, probes, true)
}
fun setEstimatedBitrate(
bitrate: Bandwidth,
bandwidthLimitedCause: BandwidthLimitedCause,
atTime: Instant
): MutableList<ProbeClusterConfig> {
this.bandwidthLimitedCause = bandwidthLimitedCause
if (bitrate < kBitrateDropThreshold * estimatedBitrate) {
timeOfLastLargeDrop = atTime
bitrateBeforeLastLargeDrop = estimatedBitrate
}
estimatedBitrate = bitrate
if (state == State.kWaitingForProbingResult) {
// Continue probing if probing results indicate channel has greater
// capacity unless we already reached the needed bitrate.
if (config.abortFurtherProbeIfMaxLowerThanCurrent && (
bitrate > maxBitrate || (
maxTotalAllocatedBitrate != Bandwidth.ZERO &&
bitrate > 2 * maxTotalAllocatedBitrate
)
)
) {
// No need to continue probing
minBitrateToProbeFurther = Bandwidth.INFINITY
}
val networkStateEstimateProbeFurtherLimit =
/* Skipping networkEstimate */
Bandwidth.INFINITY
logger.info(
"Measured bitrate: $bitrate Minimum to probe further: $minBitrateToProbeFurther " +
"upper limit: $networkStateEstimateProbeFurtherLimit"
)
if (bitrate > minBitrateToProbeFurther &&
bitrate <= networkStateEstimateProbeFurtherLimit
) {
return initiateProbing(atTime, listOf(config.furtherExponentialProbeScale * bitrate), true)
}
}
return mutableListOf()
}
fun enablePeriodicAlrProbing(enable: Boolean) {
enablePeriodicAlrProbing = enable
}
// Probes are sent periodically every 1s during the first 5s after the network
// becomes available or until OnMaxTotalAllocatedBitrate is invoked with a
// none zero max_total_allocated_bitrate (there are active streams being
// sent.) Probe rate is up to max configured bitrate configured via
// SetBitrates.
fun enableRepeatedInitialProbing(enable: Boolean) {
repeatedInitialProbingEnabled = enable
}
fun setAlrStartTimeMs(alrStartTimeMs: Long?) {
if (alrStartTimeMs != null) {
alrStartTime = Instant.ofEpochMilli(alrStartTimeMs)
} else {
alrStartTime = null
}
}
fun setAlrEndedTimeMs(alrEndTimeMs: Long) {
alrEndTime = Instant.ofEpochMilli(alrEndTimeMs)
}
fun requestProbe(atTime: Instant): MutableList<ProbeClusterConfig> {
// Called once we have returned to normal state after a large drop in
// estimated bandwidth. The current response is to initiate a single probe
// session (if not already probing) at the previous bitrate.
//
// If the probe session fails, the assumption is that this drop was a
// real one from a competing flow or a network change.
val inAlr = alrStartTime != null
val alrEndedRecently = (
alrEndTime != null &&
Duration.between(alrEndTime, atTime) < kAlrEndedTimeout
)
if (inAlr || alrEndedRecently || inRapidRecoveryExperiment) {
if (state == State.kProbingComplete) {
val suggestedProbe = kProbeFractionAfterDrop * bitrateBeforeLastLargeDrop
val minExpectedProbeResult = (1 - kProbeUncertainty) * suggestedProbe
val timeSinceDrop = Duration.between(timeOfLastLargeDrop, atTime)
val timeSinceProbe = Duration.between(lastBweDropProbingTime, atTime)
if (minExpectedProbeResult > estimatedBitrate &&
timeSinceDrop < kBitrateDropTimeout &&
timeSinceProbe > kMinTimeBetweenAlrProbes
) {
logger.info("Detected big bandwidth drop, start probing")
// Track how often we probe in response to bandwidth drop in ALR.
// TODO: histogram
lastBweDropProbingTime = atTime
return initiateProbing(atTime, listOf(suggestedProbe), false)
}
}
}
return mutableListOf()
}
/* Skipping setNetworkStateEstimate */
/**
* Resets the ProbeController to a state equivalent to as if it was just
* created EXCEPT for configuration settings like
* `enable_periodic_alr_probing_` `network_available_` and
* `max_total_allocated_bitrate_`.
*/
fun reset(atTime: Instant) {
bandwidthLimitedCause = BandwidthLimitedCause.kDelayBasedLimited
state = State.kInit
minBitrateToProbeFurther = Bandwidth.INFINITY
timeLastProbingInitiated = Instant.MIN
estimatedBitrate = Bandwidth.ZERO
startBitrate = Bandwidth.ZERO
maxBitrate = kDefaultMaxProbingBitrate
val now = atTime
lastBweDropProbingTime = now
alrEndTime = null
timeOfLastLargeDrop = now
bitrateBeforeLastLargeDrop = Bandwidth.ZERO
}
private fun timeForAlrProbe(atTime: Instant): Boolean {
if (enablePeriodicAlrProbing && alrStartTime != null) {
val nextProbeTime =
max(alrStartTime!!, timeLastProbingInitiated) +
config.alrProbingInterval
return atTime >= nextProbeTime
}
return false
}
private fun timeForNetworkStateProbe(atTime: Instant): Boolean {
/* Not using network_estimate */
return false
}
private fun timeForNextRepeatedInitialProbe(atTime: Instant): Boolean {
if (state != State.kWaitingForProbingResult &&
lastAllowedRepeatedInitialProbe > atTime
) {
val nextProbeTime = timeLastProbingInitiated + kMaxWaitingTimeForProbingResult
if (atTime >= nextProbeTime) {
return true
}
}
return false
}
private fun createProbeClusterConfig(atTime: Instant, bitrate: Bandwidth): ProbeClusterConfig {
val config = ProbeClusterConfig()
config.atTime = atTime
config.targetDataRate = bitrate
if (atTime < lastAllowedRepeatedInitialProbe) {
config.targetDuration = this.config.initialProbeDuration
config.minProbeDelta = this.config.initialMinProbeDelta
} else {
config.targetDuration = this.config.minProbeDuration
config.minProbeDelta = this.config.minProbeDelta
}
config.targetProbeCount = this.config.minProbePacketsSent
config.id = nextProbeClusterId
nextProbeClusterId++
maybeLogProbeClusterCreated(diagnosticContext, config)
return config
}
fun process(atTime: Instant): MutableList<ProbeClusterConfig> {
if (Duration.between(timeLastProbingInitiated, atTime) > kMaxWaitingTimeForProbingResult) {
if (state == State.kWaitingForProbingResult) {
logger.info("kWaitingForProbingResult: timeout")
updateState(State.kProbingComplete)
}
}
if (estimatedBitrate == Bandwidth.ZERO || state != State.kProbingComplete) {
return mutableListOf()
}
if (timeForNextRepeatedInitialProbe(atTime)) {
return initiateProbing(atTime, listOf(estimatedBitrate * config.firstExponentialProbeScale), true)
}
if (timeForAlrProbe(atTime) || timeForNetworkStateProbe(atTime)) {
return initiateProbing(atTime, listOf(estimatedBitrate * config.alrProbeScale), true)
}
return mutableListOf()
}
private enum class State {
/** Initial state where no probing has been triggrered yet */
kInit,
/** Waiting for probing results to continue further probing. */
kWaitingForProbingResult,
/** Probing is complete. */
kProbingComplete
}
private fun initiateProbing(
now: Instant,
bitratesToProbe: List<Bandwidth>,
probeFurtherIn: Boolean
): MutableList<ProbeClusterConfig> {
var probeFurther = probeFurtherIn
if (config.skipIfEstimateLargerThanFractionOfMax > 0.0) {
val networkEstimate = Bandwidth.INFINITY
val maxProbeRate = if (maxTotalAllocatedBitrate == Bandwidth.ZERO) {
maxBitrate
} else {
min(config.skipProbeMaxAllocatedScale * maxTotalAllocatedBitrate, maxBitrate)
}
if (min(networkEstimate, estimatedBitrate) > config.skipIfEstimateLargerThanFractionOfMax * maxProbeRate) {
updateState(State.kProbingComplete)
return mutableListOf()
}
}
var maxProbeBitrate = maxBitrate
if (maxTotalAllocatedBitrate > Bandwidth.ZERO) {
// If a max allocated bitrate has been configured, allow probing up to 2x
// that rate. This allows some overhead to account for bursty streams,
// which otherwise would have to ramp up when the overshoot is already in
// progress.
// It also avoids minor quality reduction caused by probes often being
// received at slightly less than the target probe bitrate.
maxProbeBitrate = min(maxProbeBitrate, maxTotalAllocatedBitrate * 2)
}
when (bandwidthLimitedCause) {
BandwidthLimitedCause.kRttBasedBackOffHighRtt,
BandwidthLimitedCause.kDelayBasedLimitedDelayIncreased,
BandwidthLimitedCause.kLossLimitedBwe -> {
logger.info { "Not sending probe in bandwidth limited state. $bandwidthLimitedCause" }
return mutableListOf()
}
BandwidthLimitedCause.kLossLimitedBweIncreasing ->
maxProbeBitrate =
min(maxProbeBitrate, estimatedBitrate * config.lossLimitedProbeScale)
BandwidthLimitedCause.kDelayBasedLimited ->
Unit
}
/* Skipping use of networkEstimate */
val pendingProbes = mutableListOf<ProbeClusterConfig>()
for (b in bitratesToProbe) {
assert(b != Bandwidth.ZERO)
var bitrate = b
if (bitrate >= maxProbeBitrate) {
bitrate = maxProbeBitrate
probeFurther = false
}
pendingProbes.add(createProbeClusterConfig(now, bitrate))
}
timeLastProbingInitiated = now
if (probeFurther) {
updateState(State.kWaitingForProbingResult)
// Dont expect probe results to be larger than a fraction of the actual
// probe rate.
minBitrateToProbeFurther = pendingProbes.last().targetDataRate * config.furtherProbeThreshold
} else {
updateState(State.kProbingComplete)
}
return pendingProbes
}
companion object {
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(ProbeController::class.java)
private fun maybeLogProbeClusterCreated(diagnosticContext: DiagnosticContext, probe: ProbeClusterConfig) {
val minDataSize = probe.targetDataRate * probe.targetDuration
timeSeriesLogger.trace {
diagnosticContext.makeTimeSeriesPoint("ProbeClusterCreated")
.addField("probe_id", probe.id)
.addField("probe_target_data_rate_bps", probe.targetDataRate.bps)
.addField("probe_target_probe_count", probe.targetProbeCount)
.addField("probe_min_data_size", minDataSize.bytes)
}
}
}
}

233
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimator.kt Normal file
View File

@@ -0,0 +1,233 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bytes
import org.jitsi.nlj.util.min
import org.jitsi.nlj.util.per
import org.jitsi.utils.isFinite
import org.jitsi.utils.isInfinite
import org.jitsi.utils.logging2.createLogger
import org.jitsi.utils.max
import org.jitsi.utils.min
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import java.time.Duration
import java.time.Instant
import java.util.*
import kotlin.Comparator
import kotlin.collections.ArrayDeque
/**
* Robust throughput estimator.
*
* Based on WebRTC modules/congestion_controller/goog_cc/robust_throughput_estimator.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138)
*/
class RobustThroughputEstimator(val settings: RobustThroughputEstimatorSettings) :
AcknowledgedBitrateEstimatorInterface {
// TODO: pass parent logger in so we have log contexts
private val logger = createLogger()
init {
require(settings.enabled)
}
override fun incomingPacketFeedbackVector(packetFeedbackVector: List<PacketResult>) {
require(packetFeedbackVector.isSorted(compareBy { it.receiveTime }))
packetFeedbackVector.forEach { packet ->
// Ignore packets without valid send or receive times.
// (This should not happen in production since lost packets are filtered
// out before passing the feedback vector to the throughput estimator.
// However, explicitly handling this case makes the estimator more robust
// and avoids a hard-to-detect bad state.)
if (packet.receiveTime.isInfinite() ||
packet.sentPacket.sendTime.isInfinite()
) {
return@forEach
}
// Insert the new packet.
window.addLast(packet)
window.last().sentPacket.priorUnackedData =
window.last().sentPacket.priorUnackedData *
settings.unackedWeight
// In most cases, receive timestamps should already be in order, but in the
// rare case where feedback packets have been reordered, we do some swaps to
// ensure that the window is sorted.
var i = window.size - 1
while (i > 0 && window[i].receiveTime < window[i - 1].receiveTime) {
Collections.swap(window, i, i - 1)
i--
}
val kMaxReorderingTime = 1.secs
val receiveDelta = Duration.between(packet.receiveTime, window.last().receiveTime)
if (receiveDelta > kMaxReorderingTime) {
logger.warn("Severe packet re-ordering or timestamps offset changed: $receiveDelta")
window.clear()
latestDiscardedSendTime = Instant.MIN
}
}
// Remove old packets.
while (firstPacketOutsideWindow()) {
latestDiscardedSendTime = max(latestDiscardedSendTime, window.first().sentPacket.sendTime)
window.removeFirst()
}
}
override fun bitrate(): Bandwidth? {
if (window.isEmpty() || window.size < settings.requiredPackets) {
return null
}
var largestRecvGap = Duration.ZERO
var secondLargestRecvGap = Duration.ZERO
for (i in 1 until window.size) {
// Find receive time gaps.
val gap = Duration.between(window[i - 1].receiveTime, window[i].receiveTime)
if (gap > largestRecvGap) {
secondLargestRecvGap = largestRecvGap
largestRecvGap = gap
} else if (gap > secondLargestRecvGap) {
secondLargestRecvGap = gap
}
}
var firstSendTime = Instant.MAX
var lastSendTime = Instant.MIN
var firstRecvTime = Instant.MAX
var lastRecvTime = Instant.MIN
var recvSize = 0.bytes
var sendSize = 0.bytes
var firstRecvSize = 0.bytes
var lastSendSize = 0.bytes
var numSentPacketsInWindow = 0
window.forEach { packet ->
if (packet.receiveTime < firstRecvTime) {
firstRecvTime = packet.receiveTime
firstRecvSize =
packet.sentPacket.size + packet.sentPacket.priorUnackedData
}
lastRecvTime = max(lastRecvTime, packet.receiveTime)
recvSize += packet.sentPacket.size
recvSize += packet.sentPacket.priorUnackedData
if (packet.sentPacket.sendTime < latestDiscardedSendTime) {
// If we have dropped packets from the window that were sent after
// this packet, then this packet was reordered. Ignore it from
// the send rate computation (since the send time may be very far
// in the past, leading to underestimation of the send rate.)
// However, ignoring packets creates a risk that we end up without
// any packets left to compute a send rate.
return@forEach
}
if (packet.sentPacket.sendTime > lastSendTime) {
lastSendTime = packet.sentPacket.sendTime
lastSendSize =
packet.sentPacket.size + packet.sentPacket.priorUnackedData
}
firstSendTime = min(firstSendTime, packet.sentPacket.sendTime)
sendSize += packet.sentPacket.size
sendSize += packet.sentPacket.priorUnackedData
++numSentPacketsInWindow
}
// Suppose a packet of size S is sent every T milliseconds.
// A window of N packets would contain N*S bytes, but the time difference
// between the first and the last packet would only be (N-1)*T. Thus, we
// need to remove the size of one packet to get the correct rate of S/T.
// Which packet to remove (if the packets have varying sizes),
// depends on the network model.
// Suppose that 2 packets with sizes s1 and s2, are received at times t1
// and t2, respectively. If the packets were transmitted back to back over
// a bottleneck with rate capacity r, then we'd expect t2 = t1 + r * s2.
// Thus, r = (t2-t1) / s2, so the size of the first packet doesn't affect
// the difference between t1 and t2.
// Analoguously, if the first packet is sent at time t1 and the sender
// paces the packets at rate r, then the second packet can be sent at time
// t2 = t1 + r * s1. Thus, the send rate estimate r = (t2-t1) / s1 doesn't
// depend on the size of the last packet.
recvSize -= firstRecvSize
sendSize -= lastSendSize
// Remove the largest gap by replacing it by the second largest gap.
// This is to ensure that spurious "delay spikes" (i.e. when the
// network stops transmitting packets for a short period, followed
// by a burst of delayed packets), don't cause the estimate to drop.
// This could cause an overestimation, which we guard against by
// never returning an estimate above the send rate.
check(firstRecvTime.isFinite())
check(lastRecvTime.isFinite())
val recvDuration = (
Duration.between(firstRecvTime, lastRecvTime) -
largestRecvGap + secondLargestRecvGap
).coerceAtLeast(1.ms)
if (numSentPacketsInWindow < settings.requiredPackets) {
// Too few send times to calculate a reliable send rate.
return recvSize.per(recvDuration)
}
check(firstSendTime.isFinite())
check(lastSendTime.isFinite())
val sendDuration = Duration.between(firstSendTime, lastSendTime).coerceAtLeast(1.ms)
return min(sendSize.per(sendDuration), recvSize.per(recvDuration))
}
override fun peekRate(): Bandwidth? = bitrate()
override fun setAlr(inAlr: Boolean) {}
override fun setAlrEndedTime(alrEndedTime: Instant) {}
private fun firstPacketOutsideWindow(): Boolean {
if (window.isEmpty()) {
return false
}
if (window.size > settings.maxWindowPackets) {
return true
}
val currentWindowDuration = Duration.between(window.first().receiveTime, window.last().receiveTime)
if (currentWindowDuration > settings.maxWindowDuration) {
return true
}
if (window.size > settings.windowPackets &&
currentWindowDuration > settings.minWindowDuration
) {
return true
}
return false
}
private val window = ArrayDeque<PacketResult>()
private var latestDiscardedSendTime: Instant = Instant.MIN
}
/* TODO: move this to a util somewhere */
fun <E> Collection<E>.isSorted(comparator: Comparator<E>): Boolean {
this.asSequence().windowed(2).forEach { (a, b) ->
if (comparator.compare(a, b) > 0) {
return@isSorted false
}
}
return true
}

595
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimation.kt Normal file
View File

@@ -0,0 +1,595 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file uses WebRTC's naming style for enums and constants
@file:Suppress("ktlint:standard:enum-entry-name-case", "ktlint:standard:property-naming")
package org.jitsi.nlj.rtp.bandwidthestimation2
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bps
import org.jitsi.nlj.util.kbps
import org.jitsi.nlj.util.max
import org.jitsi.nlj.util.min
import org.jitsi.utils.div
import org.jitsi.utils.isFinite
import org.jitsi.utils.isInfinite
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.max
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import org.jitsi.utils.times
import java.time.Duration
import java.time.Instant
import java.util.*
import kotlin.math.exp
import kotlin.math.min
/** Send-side bandwidth estimation,
* based on WebRTC modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138).
*
* Field trial settings have been generally removed, set to their default settings.
*/
class LinkCapacityTracker {
private var capacityEstimateBps = 0.0
private var lastLinkCapcityUpdate = Instant.MIN
private var lastDelayBasedEstimate = Bandwidth.INFINITY
fun updateDelayBasedEstimate(atTime: Instant, delayBasedEstimate: Bandwidth) {
if (delayBasedEstimate < lastDelayBasedEstimate) {
capacityEstimateBps = min(capacityEstimateBps, delayBasedEstimate.bps.toDouble())
lastLinkCapcityUpdate = atTime
}
lastDelayBasedEstimate = delayBasedEstimate
}
fun onStartingRate(startRate: Bandwidth) {
if (lastLinkCapcityUpdate.isInfinite()) {
capacityEstimateBps = startRate.bps.toDouble()
}
}
fun onRateUpdate(acknowledged: Bandwidth?, target: Bandwidth, atTime: Instant) {
if (acknowledged == null) {
return
}
val acknowledgedTarget = min(acknowledged, target)
if (acknowledgedTarget.bps > capacityEstimateBps) {
val alpha = if (lastLinkCapcityUpdate.isFinite() && atTime.isFinite()) {
val delta = Duration.between(lastLinkCapcityUpdate, atTime)
exp(-(delta / 10.secs))
} else {
0.0
}
capacityEstimateBps = alpha * capacityEstimateBps * alpha + (1 - alpha) * acknowledgedTarget.bps
}
lastLinkCapcityUpdate = atTime
}
fun onRttBackoff(backoffRate: Bandwidth, atTime: Instant) {
capacityEstimateBps = min(capacityEstimateBps, backoffRate.bps.toDouble())
lastLinkCapcityUpdate = atTime
}
fun estimate(): Bandwidth = capacityEstimateBps.bps
}
class RttBasedBackoff {
val disabled = false
val configuredLimit = 3.secs
val dropFraction = 0.8
val dropInterval = 1.secs
val bandwidthFloor = 5.kbps
var rttLimit = configuredLimit
var lastPropagationRttUpdate = Instant.MAX
var lastPropagationRtt = Duration.ZERO
var lastPacketSent = Instant.MIN
fun updatePropagationRtt(atTime: Instant, propagationRtt: Duration) {
lastPropagationRttUpdate = atTime
lastPropagationRtt = propagationRtt
}
fun isRttAboveLimit(): Boolean = correctedRtt() > rttLimit
private fun correctedRtt(): Duration {
// Avoid timeout when no packets are being sent.
val timeoutCorrection = max(Duration.between(lastPropagationRttUpdate, lastPacketSent), Duration.ZERO)
return timeoutCorrection + lastPropagationRtt
}
}
private val kBweIncreaseInterval = 1000.ms
private val kBweDecreaseInterval = 300.ms
private val kStartPhase = 2000.ms
private val kBweConverganceTime = 20000.ms
private const val kLimitNumPackets = 20
private val kDefaultMaxBitrate = 1000000000.bps
private val kLowBitrateLogPeriod = 10000.ms
private val kRtcEventLogPeriod = 5000.ms
// Expecting that RTCP feedback is sent uniformly within [0.5, 1.5]s intervals.
private val kMaxRtcpFeedbackInterval = 5000.ms
private const val kDefaultLowLossThreshold = 0.02f
private const val kDefaultHighLossThreshold = 0.1f
private val kDefaultBitrateThreshold = Bandwidth.ZERO
class SendSideBandwidthEstimation(
parentLogger: Logger,
private val diagnosticContext: DiagnosticContext
) {
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass)
private val logger = parentLogger.createChildLogger(javaClass.name)
private val rttBackoff = RttBasedBackoff()
private val linkCapacity = LinkCapacityTracker()
private val minBitrateHistory = ArrayDeque<Pair<Instant, Bandwidth>>()
// incoming filters
private var lostPacketsSinceLastLossUpdate = 0L
private var expectedPacketsSinceLastLossUpdate = 0L
private var acknowledgedRate: Bandwidth? = null
private var currentTarget: Bandwidth = Bandwidth.ZERO
private var lastLoggedTarget: Bandwidth = Bandwidth.ZERO
private var minBitrateConfigured: Bandwidth = kCongestionControllerMinBitrate
private var maxBitrateConfigured: Bandwidth = kDefaultMaxBitrate
private var lastLowBitrateLog: Instant = Instant.MIN
private var hasDecreasedSinceLastFractionLoss: Boolean = false
@SuppressFBWarnings("URF_UNREAD_FIELD")
private var lastLossFeedback: Instant = Instant.MIN
private var lastLossPacketReport: Instant = Instant.MIN
private var lastFractionLoss: UByte = 0u
private var lastLoggedFractionLoss: UByte = 0u
private var lastRoundTripTime: Duration = Duration.ZERO
// The max bitrate as set by the receiver in the call. This is typically
// signalled using the REMB RTCP message and is used when we don't have any
// send side delay based estimate.
private var receiverLimit: Bandwidth = Bandwidth.INFINITY
private var delayBasedLimit: Bandwidth = Bandwidth.INFINITY
private var timeLastDecrease: Instant = Instant.MIN
private var firstReportTime: Instant = Instant.MIN
private var initiallyLostPackets: Int = 0
private var bitrateAt2Seconds: Bandwidth = Bandwidth.ZERO
private var umaUpdateState: UmaState = UmaState.kNoUpdate
private var umaRttState: UmaState = UmaState.kNoUpdate
private val rampupUmaStatsUpdated = ArrayList<Boolean>(kUmaRampupMetrics.size)
private var lastRtcEventLog: Instant = Instant.MIN
private val lowLossThreshold: Float = kDefaultLowLossThreshold
private val highLossThreshold: Float = kDefaultHighLossThreshold
private val bitrateThreshold: Bandwidth = kDefaultBitrateThreshold
var lossBasedBandwidthEstimatorV2 = LossBasedBweV2().apply {
setMinMaxBitrate(minBitrateConfigured, maxBitrateConfigured)
}
private set
private var lossBasedState: LossBasedState = LossBasedState.kDelayBasedEstimate
fun onRouteChange() {
lostPacketsSinceLastLossUpdate = 0
expectedPacketsSinceLastLossUpdate = 0
currentTarget = Bandwidth.ZERO
minBitrateConfigured = kCongestionControllerMinBitrate
maxBitrateConfigured = kDefaultMaxBitrate
lastLowBitrateLog = Instant.MIN
hasDecreasedSinceLastFractionLoss = false
lastLossFeedback = Instant.MIN
lastLossPacketReport = Instant.MIN
lastFractionLoss = 0u
lastLoggedFractionLoss = 0u
lastRoundTripTime = Duration.ZERO
receiverLimit = Bandwidth.INFINITY
delayBasedLimit = Bandwidth.INFINITY
timeLastDecrease = Instant.MIN
firstReportTime = Instant.MIN
initiallyLostPackets = 0
bitrateAt2Seconds = Bandwidth.ZERO
umaUpdateState = UmaState.kNoUpdate
umaRttState = UmaState.kNoUpdate
lastRtcEventLog = Instant.MIN
if (lossBasedBandwidthEstimatorV2.useInStartPhase()) {
lossBasedBandwidthEstimatorV2 = LossBasedBweV2()
}
}
fun targetRate(): Bandwidth {
var target = currentTarget
if (!disableReceiverLimitCapsOnly) {
target = min(target, receiverLimit)
}
return max(minBitrateConfigured, target)
}
fun lossBasedState(): LossBasedState = lossBasedState
// Return whether the current rtt is higher than the rtt limited configured in
// RttBasedBackoff.
fun isRttAboveLimit(): Boolean = rttBackoff.isRttAboveLimit()
fun fractionLoss() = lastFractionLoss
fun roundTripTime() = lastRoundTripTime
fun getEstimatedLinkCapacity(): Bandwidth = linkCapacity.estimate()
// Call periodically to update estimate
fun updateEstimate(atTime: Instant) {
if (rttBackoff.isRttAboveLimit()) {
if (Duration.between(timeLastDecrease, atTime) >= rttBackoff.dropInterval &&
currentTarget > rttBackoff.bandwidthFloor
) {
timeLastDecrease = atTime
val newBitrate = (currentTarget * rttBackoff.dropFraction).coerceAtLeast(rttBackoff.bandwidthFloor)
linkCapacity.onRttBackoff(newBitrate, atTime)
updateTargetBitrate(newBitrate, atTime)
return
}
// TODO(srte): This is likely redundant in most cases.
applyTargetLimits(atTime)
return
}
// We trust the REMB and/or delay-based estimate during the first 2 seconds if
// we haven't had any packet loss reported, to allow startup bitrate probing.
if (lastFractionLoss.toUInt() == 0u && isInStartPhase(atTime) &&
!lossBasedBandwidthEstimatorV2.readyToUseInStartPhase()
) {
var newBitrate = currentTarget
// TODO(srte): We should not allow the new_bitrate to be larger than the
// receiver limit here.
if (receiverLimit.isFinite()) {
newBitrate = max(receiverLimit, newBitrate)
}
if (delayBasedLimit.isFinite()) {
newBitrate = max(delayBasedLimit, newBitrate)
}
if (newBitrate != currentTarget) {
minBitrateHistory.clear()
minBitrateHistory.add(Pair(atTime, currentTarget))
updateTargetBitrate(newBitrate, atTime)
return
}
}
updateMinHistory(atTime)
if (lastLossPacketReport.isInfinite()) {
// No feedback received.
// TODO(srte): This is likely redundant in most cases.
applyTargetLimits(atTime)
return
}
if (lossBasedBandwidthEstimatorV2.isReady()) {
val result = lossBasedBandwidthEstimatorV2.getLossBasedResult()
lossBasedState = result.state
updateTargetBitrate(result.bandwidthEstimate, atTime)
return
}
val timeSinceLossPacketReport = Duration.between(lastLossPacketReport, atTime)
if (timeSinceLossPacketReport < 1.2 * kMaxRtcpFeedbackInterval) {
// We only care about loss above a given bitrate threshold.
val loss = lastFractionLoss.toFloat() / 256f
// We only make decisions based on loss when the bitrate is above a
// threshold. This is a crude way of handling loss which is uncorrelated
// to congestion.
if (currentTarget < bitrateThreshold || loss <= lowLossThreshold) {
// Loss < 2%: Increase rate by 8% of the min bitrate in the last
// kBweIncreaseInterval.
// Note that by remembering the bitrate over the last second one can
// rampup up one second faster than if only allowed to start ramping
// at 8% per second rate now. E.g.:
// If sending a constant 100kbps it can rampup immediately to 108kbps
// whenever a receiver report is received with lower packet loss.
// If instead one would do: current_bitrate_ *= 1.08^(delta time),
// it would take over one second since the lower packet loss to achieve
// 108kbps.
var newBitrate = (minBitrateHistory.first().second.bps * 1.08 + 0.5).bps
// Add 1 kbps extra, just to make sure that we do not get stuck
// (gives a little extra increase at low rates, negligible at higher
// rates).
newBitrate += 1000.bps
updateTargetBitrate(newBitrate, atTime)
return
} else if (currentTarget > bitrateThreshold) {
if (loss <= highLossThreshold) {
// Loss between 2% - 10%: Do nothing.
} else {
// Loss > 10%: Limit the rate decreases to once a kBweDecreaseInterval
// + rtt.
if (!hasDecreasedSinceLastFractionLoss &&
Duration.between(timeLastDecrease, atTime) >= (kBweDecreaseInterval + lastRoundTripTime)
) {
timeLastDecrease = atTime
// Reduce rate:
// newRate = rate * (1 - 0.5*lossRate);
// where packetLoss = 256*lossRate;
val newBitrate = (currentTarget.bps * (512u - lastFractionLoss.toUInt()).toDouble() / 512.0).bps
hasDecreasedSinceLastFractionLoss = true
updateTargetBitrate(newBitrate, atTime)
return
}
}
}
}
// TODO(srte): This is likely redundant in most cases.
applyTargetLimits(atTime)
}
fun onSentPacket(sentPacket: SentPacket) {
// Only feedback-triggering packets will be reported here.
rttBackoff.lastPacketSent = sentPacket.sendTime
}
fun updatePropagationRtt(atTime: Instant, propagationRtt: Duration) {
rttBackoff.updatePropagationRtt(atTime, propagationRtt)
}
// Call when we receive a RTCP message with TMMBR or REMB.
fun updateReceiverEstimate(atTime: Instant, bandwidth: Bandwidth) {
// TODO(srte): Ensure caller passes PlusInfinity, not zero, to represent no
// limitation.
receiverLimit = if (bandwidth == Bandwidth.ZERO) {
Bandwidth.INFINITY
} else {
bandwidth
}
applyTargetLimits(atTime)
}
// Call when a new delay-based estimate is available.
fun updateDelayBasedEstimate(atTime: Instant, bitrate: Bandwidth) {
linkCapacity.updateDelayBasedEstimate(atTime, bitrate)
// TODO(srte): Ensure caller passes PlusInfinity, not zero, to represent no
// limitation.
delayBasedLimit = if (bitrate == Bandwidth.ZERO) {
Bandwidth.INFINITY
} else {
bitrate
}
applyTargetLimits(atTime)
}
// Call when we receive a RTCP message with a ReceiveBlock.
fun updatePacketsLost(packetsLost: Long, numberOfPackets: Long, atTime: Instant) {
lastLossFeedback = atTime
if (firstReportTime.isInfinite()) {
firstReportTime = atTime
}
// Check sequence number diff and weight loss report
if (numberOfPackets > 0) {
val expected = expectedPacketsSinceLastLossUpdate + numberOfPackets
// Don't generate a loss rate until it can be based on enough packets.
if (expected < kLimitNumPackets) {
// Accumulate reports
expectedPacketsSinceLastLossUpdate = expected
lostPacketsSinceLastLossUpdate = packetsLost
return
}
hasDecreasedSinceLastFractionLoss = false
val lostQ8 = (lostPacketsSinceLastLossUpdate + packetsLost).coerceAtLeast(0L) shl 8
lastFractionLoss = (lostQ8 / expected).coerceAtMost(255).toUByte()
// Reset accumulators
lostPacketsSinceLastLossUpdate = 0
expectedPacketsSinceLastLossUpdate = 0
lastLossPacketReport = atTime
updateEstimate(atTime)
}
}
// Call when we receive a RTCP message with a ReceiveBlock.
fun updateRtt(rtt: Duration, atTime: Instant) {
// Update RTT if we were able to compute an RTT based on this RTCP.
// FlexFEC doesn't send RTCP SR, which means we won't be able to compute RTT.
if (rtt > Duration.ZERO) {
lastRoundTripTime = rtt
}
if (!isInStartPhase(atTime) && umaRttState == UmaState.kNoUpdate) {
umaRttState = UmaState.kDone
/* TODO: histograms */
}
}
fun setBitrates(sendBitrate: Bandwidth?, minBitrate: Bandwidth, maxBitrate: Bandwidth, atTime: Instant) {
setMinMaxBitrate(minBitrate, maxBitrate)
if (sendBitrate != null) {
linkCapacity.onStartingRate(sendBitrate)
setSendBitrate(sendBitrate, atTime)
}
}
fun setSendBitrate(bitrate: Bandwidth, atTime: Instant) {
assert(bitrate >= Bandwidth.ZERO)
// Reset to avoid being capped by the estimate.
delayBasedLimit = Bandwidth.INFINITY
updateTargetBitrate(bitrate, atTime)
minBitrateHistory.clear()
}
fun setMinMaxBitrate(minBitrate: Bandwidth, maxBitrate: Bandwidth) {
minBitrateConfigured = max(minBitrate, kCongestionControllerMinBitrate)
if (maxBitrate > Bandwidth.ZERO && maxBitrate.isFinite()) {
maxBitrateConfigured = max(minBitrateConfigured, maxBitrate)
} else {
maxBitrateConfigured = kDefaultMaxBitrate
}
lossBasedBandwidthEstimatorV2.setMinMaxBitrate(minBitrateConfigured, maxBitrateConfigured)
}
fun getMinBitrate(): Int = minBitrateConfigured.bps.toInt()
fun setAcknowledgedRate(acknowledgedRate: Bandwidth?, atTime: Instant) {
this.acknowledgedRate = acknowledgedRate
if (acknowledgedRate == null) {
return
}
lossBasedBandwidthEstimatorV2.setAcknowledgedBitrate(acknowledgedRate)
}
fun updateLossBasedEstimator(
report: TransportPacketsFeedback,
delayDetectorState: BandwidthUsage,
probeBitrate: Bandwidth?,
inAlr: Boolean
) {
lossBasedBandwidthEstimatorV2.updateBandwidthEstimate(
report.packetFeedbacks,
delayBasedLimit,
inAlr
)
updateEstimate(report.feedbackTime)
}
private fun isInStartPhase(atTime: Instant): Boolean = firstReportTime.isInfinite() ||
Duration.between(firstReportTime, atTime) <= kStartPhase
private fun updateUmaStatsPacketsLost(atTime: Instant, packetsLost: Int) {
val bitrateKbps = ((currentTarget.bps + 500) / 1000).kbps
for (i in kUmaRampupMetrics.indices) {
if (!rampupUmaStatsUpdated[i] &&
bitrateKbps.kbps >= kUmaRampupMetrics[i].bitrateKbps
) {
/* TODO: histograms */
rampupUmaStatsUpdated[i] = true
}
}
if (isInStartPhase(atTime)) {
initiallyLostPackets += packetsLost
} else if (umaUpdateState == UmaState.kNoUpdate) {
umaUpdateState = UmaState.kFirstDone
bitrateAt2Seconds = bitrateKbps
/* TODO: histograms */
} else if (umaUpdateState == UmaState.kFirstDone &&
Duration.between(firstReportTime, atTime) >= kBweConverganceTime
) {
umaUpdateState = UmaState.kDone
val bitrateDiffKbps = bitrateAt2Seconds.kbps.toInt().coerceAtLeast(0)
/* TODO: histograms */
}
}
// Updates history of min bitrates.
// After this method returns min_bitrate_history_.front().second contains the
// min bitrate used during last kBweIncreaseIntervalMs.
private fun updateMinHistory(atTime: Instant) {
// Remove old data points from history.
// Since history precision is in ms, add one so it is able to increase
// bitrate if it is off by as little as 0.5ms.
while (!minBitrateHistory.isEmpty() &&
(Duration.between(minBitrateHistory.first().first, atTime) + 1.ms) > kBweIncreaseInterval
) {
minBitrateHistory.removeFirst()
}
// Typical minimum sliding-window algorithm: Pop values higher than current
// bitrate before pushing it.
while (!minBitrateHistory.isEmpty() &&
currentTarget <= minBitrateHistory.last().second
) {
minBitrateHistory.removeLast()
}
minBitrateHistory.add(Pair(atTime, currentTarget))
}
// Gets the upper limit for the target bitrate. This is the minimum of the
// delay based limit, the receiver limit and the loss based controller limit.
private fun getUpperLimit(): Bandwidth {
var upperLimit = delayBasedLimit
if (disableReceiverLimitCapsOnly) {
upperLimit = min(upperLimit, receiverLimit)
}
return min(upperLimit, maxBitrateConfigured)
}
// Prints a warning if `bitrate` if sufficiently long time has past since last
// warning.
private fun maybeLogLowBitrateWarning(bitrate: Bandwidth, atTime: Instant) {
if (Duration.between(lastLowBitrateLog, atTime) > kLowBitrateLogPeriod) {
logger.warn("Estimated available bandwidth $bitrate is below configured min bitrate $minBitrateConfigured.")
lastLowBitrateLog = atTime
}
}
// Stores an update to the event log if the loss rate has changed, the target
// has changed, or sufficient time has passed since last stored event.
private fun maybeLogLossBasedEvent(atTime: Instant) {
if (currentTarget != lastLoggedTarget ||
lastFractionLoss != lastLoggedFractionLoss ||
Duration.between(lastRtcEventLog, atTime) > kRtcEventLogPeriod
) {
timeSeriesLogger.trace {
diagnosticContext.makeTimeSeriesPoint("RtcEventBweUpdateLossBased", atTime)
.addField("currentTarget", currentTarget.bps)
.addField("lastFractionLoss", lastFractionLoss)
.addField("expectedPacketsSinceLastLossUpdate", expectedPacketsSinceLastLossUpdate)
}
lastLoggedFractionLoss = lastFractionLoss
lastLoggedTarget = currentTarget
lastRtcEventLog = atTime
}
}
// Cap `bitrate` to [min_bitrate_configured_, max_bitrate_configured_] and
// set `current_bitrate_` to the capped value and updates the event log.
private fun updateTargetBitrate(bitrate: Bandwidth, atTime: Instant) {
val newBitrate = min(bitrate, getUpperLimit())
if (newBitrate < minBitrateConfigured) {
maybeLogLowBitrateWarning(newBitrate, atTime)
}
currentTarget = newBitrate
maybeLogLossBasedEvent(atTime)
linkCapacity.onRateUpdate(acknowledgedRate, currentTarget, atTime)
}
// Applies lower and upper bounds to the current target rate.
// TODO(srte): This seems to be called even when limits haven't changed, that
// should be cleaned up.
private fun applyTargetLimits(atTime: Instant) {
updateTargetBitrate(currentTarget, atTime)
}
private enum class UmaState { kNoUpdate, kFirstDone, kDone }
companion object {
const val disableReceiverLimitCapsOnly = false
}
}
private data class UmaRampUpMetric(
val metricName: String,
val bitrateKbps: Int
)
private val kUmaRampupMetrics = arrayOf(
UmaRampUpMetric("WebRTC.BWE.RampUpTimeTo500kbpsInMs", 500),
UmaRampUpMetric("WebRTC.BWE.RampUpTimeTo1000kbpsInMs", 1000),
UmaRampUpMetric("WebRTC.BWE.RampUpTimeTo2000kbpsInMs", 2000)
)

37
jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SentPacketInfo.kt Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright @ 2019 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.nlj.rtp.bandwidthestimation2
import org.jitsi.utils.NEVER
import java.time.Instant
/** Sent packet info,
* based loosely on WebRTC rtc_base/network/sent_packet.{h,cc} in
* WebRTC tag branch-heads/7204 (Chromium 138)
* stripped down to only the fields needed.
*/
class PacketInfo(
var includedInFeedback: Boolean = false,
var includedInAllocation: Boolean = false,
var packetSizeBytes: Long = 0
)
class SentPacketInfo(
var packetId: Long = -1,
var sendTime: Instant = NEVER,
val info: PacketInfo = PacketInfo(),
)

Some files were not shown because too many files have changed in this diff Show More