upstream/upstream-jitsi-videobridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

snapshot of https://github.com/jitsi/jitsi-videobridge.git @ fcbf741c646bc237d2ed85a1e86910532532b831 (2026-09-15)
Some checks failed
Java CI with Maven / Java 17 (push) Failing after 7m2s
Details
Java CI with Maven / Java 21 (push) Failing after 5m0s
Details

Browse Source
This commit is contained in:
upstream-mirror
2026-09-15 16:12:38 +05:00
commit e1257b7a05
647 changed files with 110946 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
jvb/checkstyle.xml Normal file
View File

@@ -0,0 +1,45 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker">
<module name="LineLength">
<property name="fileExtensions" value="java"/>
<property name="max" value="120"/>
<property name="ignorePattern" value="http://|https://"/>
</module>
<module name="TreeWalker">
<module name="LeftCurly">
<property name="option" value="nl"/>
<property name="tokens" value="LITERAL_CATCH"/>
<property name="tokens" value="LITERAL_ELSE"/>
<property name="tokens" value="LITERAL_FINALLY"/>
<property name="tokens" value="LITERAL_IF"/>
<property name="tokens" value="LITERAL_TRY"/>
<property name="tokens" value="LITERAL_SYNCHRONIZED"/>
<property name="tokens" value="ANNOTATION_DEF"/>
<property name="tokens" value="CLASS_DEF"/>
<property name="tokens" value="CTOR_DEF"/>
<property name="tokens" value="ENUM_DEF"/>
<property name="tokens" value="INTERFACE_DEF"/>
<property name="tokens" value="METHOD_DEF"/>
<property name="tokens" value="RECORD_DEF"/>
</module>
<!-- Skips empty blocks {}, to not skip them: value="alone" -->
<module name="RightCurly">
<property name="option" value="alone_or_singleline"/>
</module>
<module name="WhitespaceAfter">
<!-- Drops TYPECAST from default tokens -->
<property name="tokens" value="COMMA , SEMI , LITERAL_IF , LITERAL_ELSE , LITERAL_WHILE , LITERAL_DO , LITERAL_FOR , DO_WHILE"/>
</module>
</module>
<module name="FileTabCharacter">
<property name="eachLine" value="true"/>
</module>
</module>

25
jvb/lib/logging.properties Normal file
View File

@@ -0,0 +1,25 @@
handlers= java.util.logging.ConsoleHandler
#handlers= java.util.logging.ConsoleHandler, io.sentry.jul.SentryHandler
java.util.logging.ConsoleHandler.level = ALL
java.util.logging.ConsoleHandler.formatter = org.jitsi.utils.logging2.JitsiLogFormatter
org.jitsi.utils.logging2.JitsiLogFormatter.programname=JVB
.level=INFO
# Sentry (uncomment handler to use)
io.sentry.jul.SentryHandler.level=WARNING
# time series logging
java.util.logging.SimpleFormatter.format= %5$s%n
java.util.logging.FileHandler.level = ALL
java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
java.util.logging.FileHandler.pattern = /tmp/jvb-series.log
java.util.logging.FileHandler.limit = 200000000
java.util.logging.FileHandler.count = 1
java.util.logging.FileHandler.append = false
timeseries.level=OFF
timeseries.useParentHandlers = false
# time series logging is disabled by default. Uncomment the line below to enable it.
#timeseries.handlers = java.util.logging.FileHandler

8
jvb/lib/videobridge.rc Normal file
View File

@@ -0,0 +1,8 @@
# The amount of memory that the java process can consume.
# Use the format that is in use for java's -Xmx switch
# The default is 3072m
# Only effective on linux, 64 bit
# VIDEOBRIDGE_MAX_MEMORY=3072m
# Uncomment the next line to enable the remote debugging of the video bridge
# VIDEOBRIDGE_DEBUG_OPTIONS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8000"

611
jvb/pom.xml Normal file
View File

@@ -0,0 +1,611 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jitsi</groupId>
<artifactId>jvb-parent</artifactId>
<version>2.3-SNAPSHOT</version>
</parent>
<artifactId>jitsi-videobridge</artifactId>
<packaging>jar</packaging>
<name>jitsi-videobridge</name>
<url>https://github.com/jitsi/jitsi-videobridge</url>
<properties>
<assembly.skipAssembly>false</assembly.skipAssembly>
<exec.mainClass>org.jitsi.videobridge.MainKt</exec.mainClass>
<slf4j.version>2.0.16</slf4j.version>
<exec-maven-plugin.version>3.6.3</exec-maven-plugin.version>
<ktlint.version>1.8.0</ktlint.version>
</properties>
<profiles>
<profile>
<!-- Building a fat jar is disabled by default. To build it
pass '-P buildFatJar'
-->
<id>buildFatJar</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<executions>
<execution>
<id>build-fat-jar</id>
<phase>package</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<dependencies>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>33.4.0-jre</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-rewrite</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.ee10</groupId>
<artifactId>jetty-ee10-servlets</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.ee10.websocket</groupId>
<artifactId>jetty-ee10-websocket-jetty-server</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>jetty-websocket-jetty-client</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>24.1.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.spotbugs/spotbugs-annotations -->
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
<version>${spotbugs.version}</version>
</dependency>
<!-- org.jitsi -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-media-transform</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>rtp</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-dcsctp</artifactId>
<version>1.0-7-gb548df2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-api -->
<!-- we inherit an old version of slf4j-api from tinder, which pcap4j doesn't work with. Adding slf4j-api 1.7.30 (the current stable) as a dep of jvb fixes the problem. -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>ice4j</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-config</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-health-checker</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-jetty</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-mediajson</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-metrics</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-mucclient</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-tracing</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-xmpp-extensions</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-utils</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jitsi-metaconfig</artifactId>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
<version>${kotlin.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-jetty-http</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-kotlin</artifactId>
<version>${jackson.version}</version>
</dependency>
<!-- jackson-module-kotlin uses kotlin-reflect; avoid pulling
in an old version. -->
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-reflect</artifactId>
<version>${kotlin.version}</version>
</dependency>
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>0.9.11</version>
</dependency>
<!-- runtime -->
<dependency>
<groupId>io.sentry</groupId>
<artifactId>sentry</artifactId>
<version>7.20.0</version>
<scope>runtime</scope>
</dependency>
<!-- test -->
<dependency>
<groupId>io.mockk</groupId>
<artifactId>mockk-jvm</artifactId>
<version>${mockk.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.test-framework</groupId>
<artifactId>jersey-test-framework-core</artifactId>
<version>${jersey.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.test-framework.providers</groupId>
<artifactId>jersey-test-framework-provider-jetty</artifactId>
<version>${jersey.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.test-framework.providers</groupId>
<artifactId>jersey-test-framework-provider-grizzly2</artifactId>
<version>${jersey.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- We need this so that maven-surefire-plugin can run junit4 tests -->
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- Specify this dependency here because kotest needs it, and otherwise an older
version will be pulled in -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<version>${junit.platform.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.kotest</groupId>
<artifactId>kotest-runner-junit5-jvm</artifactId>
<version>${kotest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.kotest</groupId>
<artifactId>kotest-assertions-core-jvm</artifactId>
<version>${kotest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jicoco-test-kotlin</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.7.1</version>
<executions>
<execution>
<id>assemble-archive</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<appendAssemblyId>true</appendAssemblyId>
<descriptors>
<descriptor>src/assembly/archive.xml</descriptor>
</descriptors>
<skipAssembly>${assembly.skipAssembly}</skipAssembly>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<executions>
<execution>
<id>build-fat-jar</id>
<goals>
<goal>shade</goal>
</goals>
<!-- Building a fat jar is disabled by default. See the note
on the buildFatJar profile at the top of this file
-->
<phase>none</phase>
<configuration>
<finalName>${project.name}-${project.version}-jar-with-dependencies</finalName>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<!-- Exclude signatures as they don't work correctly with a
fat jar.
-->
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
<!-- ktlint: check runs in the verify phase; auto-format with "mvn exec:exec@ktlint-format" -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>${exec-maven-plugin.version}</version>
<executions>
<execution>
<id>exec-exec</id>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>java</executable>
<arguments>
<argument>-classpath</argument>
<classpath/>
<argument>-Djava.util.logging.config.file=${project.basedir}/lib/logging.properties</argument>
<argument>${exec.mainClass}</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>exec-java</id>
<goals>
<goal>java</goal>
</goals>
<configuration>
<mainClass>${exec.mainClass}</mainClass>
</configuration>
</execution>
<execution>
<id>ktlint-check</id>
<phase>verify</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>java</executable>
<includePluginDependencies>true</includePluginDependencies>
<arguments>
<argument>-classpath</argument>
<classpath>
<dependency>com.pinterest.ktlint:ktlint-cli</dependency>
</classpath>
<argument>com.pinterest.ktlint.Main</argument>
<argument>--relative</argument>
<argument>src/main/kotlin/**/*.kt</argument>
<argument>src/test/kotlin/**/*.kt</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>ktlint-format</id>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>java</executable>
<includePluginDependencies>true</includePluginDependencies>
<arguments>
<argument>-classpath</argument>
<classpath>
<dependency>com.pinterest.ktlint:ktlint-cli</dependency>
</classpath>
<argument>com.pinterest.ktlint.Main</argument>
<argument>--format</argument>
<argument>--relative</argument>
<argument>src/main/kotlin/**/*.kt</argument>
<argument>src/test/kotlin/**/*.kt</argument>
</arguments>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>com.pinterest.ktlint</groupId>
<artifactId>ktlint-cli</artifactId>
<version>${ktlint.version}</version>
<classifier>all</classifier>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.eclipse.jetty.ee10</groupId>
<artifactId>jetty-ee10-maven-plugin</artifactId>
<version>${jetty.version}</version>
</plugin>
<plugin>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-maven-plugin</artifactId>
<version>${kotlin.version}</version>
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/main/kotlin</sourceDir>
<sourceDir>${project.basedir}/src/main/java</sourceDir>
</sourceDirs>
</configuration>
</execution>
<execution>
<id>test-compile</id>
<phase>test-compile</phase>
<goals>
<goal>test-compile</goal>
</goals>
<configuration>
<sourceDirs>
<sourceDir>${project.basedir}/src/test/kotlin</sourceDir>
<sourceDir>${project.basedir}/src/test/java</sourceDir>
</sourceDirs>
</configuration>
</execution>
</executions>
<configuration>
<jvmTarget>17</jvmTarget>
<args>
<arg>-Xallow-result-return-type</arg>
</args>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<executions>
<!-- Replacing default-compile as it is treated specially by maven -->
<execution>
<id>default-compile</id>
<phase>none</phase>
</execution>
<!-- Replacing default-testCompile as it is treated specially by maven -->
<execution>
<id>default-testCompile</id>
<phase>none</phase>
</execution>
<execution>
<id>java-compile</id>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>java-test-compile</id>
<phase>test-compile</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<source>17</source>
<target>17</target>
<compilerArgs>
<arg>-Xlint:all,-serial</arg>
</compilerArgs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire-plugin.version}</version>
<configuration>
<systemPropertyVariables>
<java.util.logging.config.file>${project.basedir}/lib/logging.properties</java.util.logging.config.file>
<kotest.framework.classpath.scanning.autoscan.disable>true</kotest.framework.classpath.scanning.autoscan.disable>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
<version>${spotbugs-maven-plugin.version}</version>
<configuration>
<excludeFilterFile>${project.basedir}/spotbugs-exclude.xml</excludeFilterFile>
<includeTests>true</includeTests>
</configuration>
<dependencies>
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs</artifactId>
<version>${spotbugs.version}</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<!-- Instructs the plugin to add the current package version in the jar manifest
So we can access it runtime with: .class.getPackage().getImplementationVersion()
-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.4.2</version>
<configuration>
<archive>
<manifest>
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<configLocation>checkstyle.xml</configLocation>
</configuration>
<dependencies>
<dependency>
<groupId>com.puppycrawl.tools</groupId>
<artifactId>checkstyle</artifactId>
<version>10.17.0</version>
</dependency>
</dependencies>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>jitsi-maven-repository-releases</id>
<layout>default</layout>
<name>Jitsi Maven Repository (Releases)</name>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<url>https://github.com/jitsi/jitsi-maven-repository/raw/master/releases/</url>
</repository>
<repository>
<id>jitsi-maven-repository-snapshots</id>
<layout>default</layout>
<name>Jitsi Maven Repository (Snapshots)</name>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
<url>https://github.com/jitsi/jitsi-maven-repository/raw/master/snapshots/</url>
</repository>
</repositories>
</project>

14
jvb/resources/jvb.bat Normal file
View File

@@ -0,0 +1,14 @@
@echo off
:begin
:: needed to overcome weird loop behavior in conjunction with variable expansion
SETLOCAL enabledelayedexpansion
set mainClass=org.jitsi.videobridge.MainKt
set cp=jitsi-videobridge.jar
FOR %%F IN (lib/*.jar) DO (
SET cp=!cp!;lib/%%F%
)
java -Djava.util.logging.config.file=lib/logging.properties -Djdk.tls.ephemeralDHKeySize=2048 -cp %cp% %mainClass% %*

22
jvb/resources/jvb.sh Executable file
View File

@@ -0,0 +1,22 @@
#!/bin/bash
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
mainClass="org.jitsi.videobridge.MainKt"
cp=$SCRIPT_DIR/jitsi-videobridge.jar:$SCRIPT_DIR/lib/*
logging_config="$SCRIPT_DIR/lib/logging.properties"
videobridge_rc="$SCRIPT_DIR/lib/videobridge.rc"
# if there is a logging config file in lib folder use it (running from source)
if [ -f $logging_config ]; then
LOGGING_CONFIG_PARAM="-Djava.util.logging.config.file=$logging_config"
fi
if [ -f $videobridge_rc ]; then
source $videobridge_rc
fi
if [ -z "$VIDEOBRIDGE_MAX_MEMORY" ]; then VIDEOBRIDGE_MAX_MEMORY=3072m; fi
if [ -z "$VIDEOBRIDGE_GC_TYPE" ]; then VIDEOBRIDGE_GC_TYPE=G1GC; fi
exec java -Xmx$VIDEOBRIDGE_MAX_MEMORY $VIDEOBRIDGE_DEBUG_OPTIONS -XX:+Use$VIDEOBRIDGE_GC_TYPE -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djdk.tls.ephemeralDHKeySize=2048 $LOGGING_CONFIG_PARAM $JAVA_SYS_PROPS -cp $cp $mainClass $@

39
jvb/spotbugs-exclude.xml Normal file
View File

@@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<FindBugsFilter>
<Match>
<!-- Our Packet hierarchy intentionally uses clone() without calling super.clone(). -->
<Bug pattern="CN_IDIOM_NO_SUPER_CALL"/>
</Match>
<Match>
<!-- Internationalization is not a concern. -->
<Bug pattern="DM_DEFAULT_ENCODING"/>
</Match>
<Match>
<!-- This seems to get triggered by certain kotlin constructs though they look safe -->
<Bug pattern="BC_BAD_CAST_TO_ABSTRACT_COLLECTION"/>
<Source name="~.*\.kt" />
</Match>
<Match>
<!-- Serialization -->
<Bug pattern="SE_BAD_FIELD"/>
</Match>
<Match>
<!-- Serialization -->
<Bug pattern="SE_BAD_FIELD_STORE"/>
</Match>
<Match>
<!-- Tests have a valid reason to ignore return values (expecting exceptions, etc). -->
<Bug pattern="RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT"/>
<Or>
<Source name="~.*\.*Test.kt" />
<Source name="~.*\.*Test.java" />
</Or>
</Match>
</FindBugsFilter>

56
jvb/src/assembly/archive.xml Normal file
View File

@@ -0,0 +1,56 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>archive</id>
<formats>
<format>zip</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>${project.artifactId}-${project.version}</baseDirectory>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<scope>runtime</scope>
<useProjectArtifact>false</useProjectArtifact>
<unpack>false</unpack>
</dependencySet>
</dependencySets>
<files>
<file>
<source>${project.basedir}/target/${project.artifactId}-${project.version}.jar</source>
<outputDirectory>${file.separator}</outputDirectory>
<destName>jitsi-videobridge.jar</destName>
</file>
</files>
<fileSets>
<fileSet>
<directory>${project.build.directory}</directory>
<outputDirectory>${file.separator}</outputDirectory>
<includes>
<include>*.jar</include>
</includes>
<excludes>
<exclude>*-with-dependencies.jar</exclude>
<exclude>${project.artifactId}-${project.version}.jar</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>${project.basedir}/lib</directory>
<outputDirectory>lib</outputDirectory>
<excludes>
<exclude>**/*.jar</exclude>
<exclude>README</exclude>
</excludes>
</fileSet>
<fileSet>
<directory>${project.basedir}/resources/</directory>
<outputDirectory>${file.separator}</outputDirectory>
<fileMode>755</fileMode>
<includes>
<include>jvb.sh</include>
<include>jvb.bat</include>
</includes>
</fileSet>
</fileSets>
</assembly>

1860
jvb/src/main/java/org/jitsi/videobridge/Conference.java Normal file
View File

File diff suppressed because it is too large Load Diff

448
jvb/src/main/java/org/jitsi/videobridge/ConferenceSpeechActivity.java Normal file
View File

@@ -0,0 +1,448 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
import org.jitsi.utils.dsi.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.util.*;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.*;
import java.util.stream.*;
/**
* Represents the speech activity of the <tt>Endpoint</tt>s in a
* <tt>Conference</tt>. Identifies the dominant speaker <tt>Endpoint</tt> in the
* <tt>Conference</tt> and maintains an ordered list of the <tt>Endpoint</tt>s
* in the <tt>Conference</tt> sorted by recentness of speaker domination and/or
* speech activity.
*
* @author Lyubomir Marinov
*/
public class ConferenceSpeechActivity
{
/**
* The <tt>Logger</tt> used by the <tt>ConferenceSpeechActivity</tt> class
* and its instances to print debug information.
*/
private final Logger logger;
/**
* The <tt>ActiveSpeakerChangedListener</tt> which listens to
* {@link #dominantSpeakerIdentification} about changes in the
* active/dominant speaker in this multipoint conference.
*/
private final ActiveSpeakerChangedListener<String> activeSpeakerChangedListener
= ConferenceSpeechActivity.this::activeSpeakerChanged;
/**
* The <tt>DominantSpeakerIdentification</tt> instance which detects/identifies the active/dominant speaker in a
* conference.
*/
private DominantSpeakerIdentification<String> dominantSpeakerIdentification;
/**
* The listener to be notified when the dominant speaker or endpoint order changes.
*/
private Listener listener;
/**
* The list of endpoints ordered by speech activity alone with the dominant speaker at the beginning of the list
* i.e. the dominant speaker history. This contains all endpoints in the conference, including those that have
* never been the dominant speaker. Such endpoints are listed after endpoints which have been the dominant speaker,
* but the order among themselves is not specified.
* This is used internally when the list of all endpoints is required (e.g. for bandwidth allocation decisions).
*/
private final List<AbstractEndpoint> endpointsBySpeechActivity = new ArrayList<>();
/**
* The list of endpoints in "LastN" order. That is, endpoints currently sending video are at the top of the list,
* ordered by speech activity, followed by the rest of the endpoints (again in speech activity order).
*/
private @NotNull List<AbstractEndpoint> endpointsInLastNOrder = new ArrayList<>();
/**
* The list of endpoints which have been "dominant speaker" ordered by speech activity (with the current dominant
* speaker at the beginning). This differs from {@link #endpointsBySpeechActivity} in that it does not necessarily
* contain all endpoints in the conference.
*
* This is used when signaling "recent speakers" to endpoints.
*
* We add 1 to the configured number of recent speakers to account for the dominant speaker.
*/
@NotNull
private final RecentSpeakersList<AbstractEndpoint> recentSpeakers
= new RecentSpeakersList<>(ConferenceSpeechActivityConfig.config.getRecentSpeakersCount() + 1);
/**
* Whether we're currently in a period of silence. With silence detection enabled we initialize to `true` because
* (the {@link #dominantSpeakerIdentification} will fire an initial "silence" event and we don't want to act on it.
*/
private boolean inSilence = ConferenceSpeechActivityConfig.config.getEnableSilenceDetection();
/**
* The <tt>Object</tt> used to synchronize the access to the state of this
* instance.
*/
private final Object syncRoot = new Object();
public ConferenceSpeechActivity(@NotNull Listener listener)
{
this(listener, null);
}
/**
* Initializes a new <tt>ConferenceSpeechActivity</tt> instance.
*
* @param listener the listener to be notified when the dominant speaker or endpoint order change.
*/
public ConferenceSpeechActivity(@NotNull Listener listener, Logger parentLogger)
{
this.listener = Objects.requireNonNull(listener, "listener");
logger =
parentLogger == null ?
new LoggerImpl(ConferenceSpeechActivity.class.getName()) :
parentLogger.createChildLogger(ConferenceSpeechActivity.class.getName());
long silenceTimeoutMs = -1;
if (ConferenceSpeechActivityConfig.config.getEnableSilenceDetection())
{
silenceTimeoutMs = ConferenceSpeechActivityConfig.config.getSilenceDetectionTimeout().toMillis();
}
dominantSpeakerIdentification = new DominantSpeakerIdentification<>(silenceTimeoutMs);
dominantSpeakerIdentification.addActiveSpeakerChangedListener(activeSpeakerChangedListener);
int numLoudestToTrack = LoudestConfig.Companion.getRouteLoudestOnly() ?
LoudestConfig.Companion.getNumLoudest() : 0;
dominantSpeakerIdentification.setLoudestConfig(numLoudestToTrack,
(int)(LoudestConfig.Companion.getEnergyExpireTime().toMillis()),
LoudestConfig.Companion.getEnergyAlphaPct());
}
boolean isInSilence()
{
return inSilence;
}
/**
* Notifies this instance that the underlying {@code dominant speaker identification} has elected a new
* active/dominant speaker.
*
* @param id the ID of the new active/dominant speaker or null if a period of silence began.
*/
protected void activeSpeakerChanged(@Nullable String id)
{
final Listener listener = this.listener;
if (listener == null)
{
return;
}
logger.trace(() -> "The dominant speaker is now " + id + ".");
boolean endpointListChanged;
boolean dominantSpeakerChanged;
synchronized (syncRoot)
{
if (id == null)
{
endpointListChanged = false;
dominantSpeakerChanged = false;
if (!inSilence)
{
inSilence = true;
}
}
else
{
dominantSpeakerChanged = true;
if (inSilence)
{
inSilence = false;
}
AbstractEndpoint endpoint
= endpointsBySpeechActivity.stream()
.filter(e -> id.equals(e.getId()))
.findFirst().orElse(null);
// Move this endpoint to the top of our sorted list
if (!endpointsBySpeechActivity.remove(endpoint))
{
logger.warn("Got active speaker notification for an unknown endpoint: " + id + ", ignoring");
return;
}
endpointsBySpeechActivity.add(0, endpoint);
recentSpeakers.promote(endpoint);
endpointListChanged = updateLastNEndpoints();
}
}
TaskPools.IO_POOL.execute(() -> {
listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence);
if (endpointListChanged)
{
listener.lastNEndpointsChanged();
}
});
}
/**
* Re-calculates the list of endpoints in LastN order ({@link #endpointsInLastNOrder}) based on the speech activity
* and video availability.
*
* @return {@code true} if the call resulted in a change of the ordered list of endpoints, and {@code false}
* otherwise.
*/
boolean updateLastNEndpoints()
{
synchronized (syncRoot)
{
Map<Boolean, List<AbstractEndpoint>> byVideoAvailable
= endpointsBySpeechActivity.stream()
.collect(Collectors.groupingBy(AbstractEndpoint::hasVideoAvailable));
List<AbstractEndpoint> newEndpointsInLastNOrder = new ArrayList<>(endpointsBySpeechActivity.size());
newEndpointsInLastNOrder.addAll(byVideoAvailable.getOrDefault(true, Collections.emptyList()));
newEndpointsInLastNOrder.addAll(byVideoAvailable.getOrDefault(false, Collections.emptyList()));
if (!newEndpointsInLastNOrder.equals(endpointsInLastNOrder))
{
endpointsInLastNOrder = Collections.unmodifiableList(newEndpointsInLastNOrder);
return true;
}
return false;
}
}
void expire()
{
synchronized (syncRoot)
{
if (dominantSpeakerIdentification != null)
{
dominantSpeakerIdentification.removeActiveSpeakerChangedListener(activeSpeakerChangedListener);
}
this.listener = null;
this.dominantSpeakerIdentification = null;
endpointsBySpeechActivity.clear();
endpointsInLastNOrder = Collections.emptyList();
}
}
/**
* Gets the <tt>Endpoint</tt> which is the dominant speaker in the
* multipoint conference represented by this instance.
*
* @return the <tt>Endpoint</tt> which is the dominant speaker in the
* multipoint conference represented by this instance or <tt>null</tt>
*/
public AbstractEndpoint getDominantEndpoint()
{
synchronized (syncRoot)
{
return recentSpeakers.getRecentSpeakers().stream().findFirst().orElse(null);
}
}
/**
* Gets a read-only list of endpoints in "LastN" order.
*
* @return a read-only list of endpoints in "LastN" order.
*/
public List<AbstractEndpoint> getOrderedEndpoints()
{
return endpointsInLastNOrder;
}
/**
* Get a list of recent speakers (including the current dominant one at the top of the list).
*/
@NotNull
public List<String> getRecentSpeakers()
{
return recentSpeakers.getRecentSpeakers().stream()
.map(AbstractEndpoint::getId).collect(Collectors.toList());
}
/**
* Query whether an endpoint is a recent speaker.
*/
public boolean isRecentSpeaker(AbstractEndpoint endpoint)
{
return recentSpeakers.isRecentSpeaker(endpoint);
}
public boolean isAmongLoudest(String endpointId)
{
return dominantSpeakerIdentification.isAmongLoudest(endpointId);
}
/**
* Notifies this instance that a new audio level was received or measured by an <tt>Endpoint</tt>.
*
* @param endpoint the endpoint for which a new audio level was received or measured
* @param level the new audio level which was received or measured
* @return The current energy rank and related data.
*/
public SpeakerRanking levelChanged(@NotNull AbstractEndpoint endpoint, long level)
{
DominantSpeakerIdentification<String> dsi = this.dominantSpeakerIdentification;
return dsi != null ? dsi.levelChanged(endpoint.getId(), (int) level) : null;
}
/**
* Notifies this instance that the list of endpoints changed.
*/
public void endpointsChanged(List<AbstractEndpoint> conferenceEndpoints)
{
boolean endpointsListChanged;
boolean dominantSpeakerChanged;
boolean recentSpeakersChanged;
// The list of endpoints may have changed, sync our list to make sure it matches.
synchronized (syncRoot)
{
// Remove any endpoints we have that are no longer in the conference
AbstractEndpoint previousDominantSpeaker
= endpointsBySpeechActivity.isEmpty() ? null : endpointsBySpeechActivity.get(0);
endpointsListChanged = endpointsBySpeechActivity.removeIf(ep -> !conferenceEndpoints.contains(ep));
recentSpeakersChanged = recentSpeakers.removeAllExcept(conferenceEndpoints);
// Add any endpoints from the conf we don't have to the end of our list
for (AbstractEndpoint conferenceEndpoint : conferenceEndpoints)
{
if (!endpointsBySpeechActivity.contains(conferenceEndpoint))
{
endpointsBySpeechActivity.add(conferenceEndpoint);
endpointsListChanged = true;
}
}
AbstractEndpoint newDominantSpeaker
= endpointsBySpeechActivity.isEmpty() ? null : endpointsBySpeechActivity.get(0);
dominantSpeakerChanged = !Objects.equals(previousDominantSpeaker, newDominantSpeaker);
recentSpeakersChanged |= dominantSpeakerChanged;
if (endpointsListChanged)
{
endpointsListChanged = updateLastNEndpoints();
}
}
if (recentSpeakersChanged || endpointsListChanged)
{
final boolean finalRecentSpeakersChanged = recentSpeakersChanged;
final boolean finalEndpointsChanged = endpointsListChanged;
final Listener listener = this.listener;
if (listener == null)
{
return;
}
TaskPools.IO_POOL.execute(() -> {
if (finalRecentSpeakersChanged)
{
listener.recentSpeakersChanged(
recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence);
}
if (finalEndpointsChanged)
{
listener.lastNEndpointsChanged();
}
});
}
}
public void endpointVideoAvailabilityChanged()
{
boolean endpointsListChanged;
synchronized (syncRoot)
{
endpointsListChanged = updateLastNEndpoints();
}
if (endpointsListChanged)
{
TaskPools.IO_POOL.execute(() -> {
try
{
listener.lastNEndpointsChanged();
}
catch (Throwable e)
{
logger.warn("Failed to fire event", e);
}
});
}
}
/**
* Gets a JSON representation of the parts of this object's state that
* are deemed useful for debugging.
*/
public ObjectNode getDebugState(@NotNull DebugStateMode mode)
{
ObjectMapper mapper = new ObjectMapper();
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
AbstractEndpoint dominantEndpoint = getDominantEndpoint();
debugState.put("dominantEndpoint", dominantEndpoint == null ? "null" : dominantEndpoint.getId());
if (mode != DebugStateMode.FULL)
{
return debugState;
}
DominantSpeakerIdentification<String> dsi = this.dominantSpeakerIdentification;
if (dsi != null)
{
debugState.set("dominantSpeakerIdentification", dsi.doGetJSON());
}
else
{
debugState.putNull("dominantSpeakerIdentification");
}
synchronized (syncRoot)
{
debugState.set("endpointsBySpeechActivity",
mapper.valueToTree(endpointsBySpeechActivity.stream().map(AbstractEndpoint::getId)
.collect(Collectors.toList())));
debugState.set("endpointsInLastNOrder",
mapper.valueToTree(endpointsInLastNOrder.stream().map(AbstractEndpoint::getId)
.collect(Collectors.toList())));
}
return debugState;
}
interface Listener
{
/**
* The list of recent speakers changed (either because a new dominant speaker was promoted, or because an
* endpoint was removed).
* @param recentSpeakers the new list of recent speakers (including the dominant speaker at index 0).
* @param dominantSpeakerChanged whether the dominant speaker changed.
* @param silence whether we're in a period of silence
*/
void recentSpeakersChanged(
List<AbstractEndpoint> recentSpeakers,
boolean dominantSpeakerChanged,
boolean silence);
void lastNEndpointsChanged();
}
}

159
jvb/src/main/java/org/jitsi/videobridge/EncodingsManager.java Normal file
View File

@@ -0,0 +1,159 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge;
import org.jitsi.nlj.rtp.*;
import java.util.*;
import java.util.concurrent.*;
/**
* Process information signaled about encodings (payload types, SSRCs, SSRC
* associations, etc.) and gather it in a single place where it can be published
* out to all interested parties.
*
* The idea is to alleviate the following problem:
* When a new endpoint joins, it can use the notification of new local encoding
* information to trigger updating all other endpoints with that new information,
* but how does this new endpoint learn of this information from all existing
* endpoints? Existing endpoints don't currently have a good trigger to notify
* new endpoints about their encoding information, so this was devised as a
* single location to handle dispersing this information to all interested
* parties.
*
* TODO: Provide a way to remove associations that are no longer used.
*
* @author Brian Baldino
*/
public class EncodingsManager
{
/**
* Maps an endpoint ID to the list of its SSRC associations.
*/
private Map<String, List<SsrcAssociation>> ssrcAssociations
= new ConcurrentHashMap<>();
/**
* Set of listeners to be notified of associations.
*/
private Set<EncodingsUpdateListener> listeners
= ConcurrentHashMap.newKeySet();
/**
* Adds an SSRC association for a specific endpoint.
*
* @param endpointId the ID of the endpoint.
* @param primarySsrc the primary SSRC in the SSRC association.
* @param secondarySsrc the secondary SSRC in the SSRC association.
* @param type the association type.
*/
public void addSsrcAssociation(
String endpointId,
long primarySsrc,
long secondarySsrc,
SsrcAssociationType type)
{
List<SsrcAssociation> epSsrcAssociations
= ssrcAssociations.computeIfAbsent(
endpointId, k -> new ArrayList<>());
epSsrcAssociations.add(
new SsrcAssociation(primarySsrc, secondarySsrc, type));
listeners.forEach(
listener
-> listener.onNewSsrcAssociation(
endpointId, primarySsrc, secondarySsrc, type));
}
/**
* Subscribe to future updates and be notified of any existing SSRC
* associations.
* @param listener
*/
public void subscribe(EncodingsUpdateListener listener)
{
listeners.add(listener);
ssrcAssociations.forEach(
(endpointId, ssrcAssociations)
-> ssrcAssociations.forEach(ssrcAssociation ->
{
listener.onNewSsrcAssociation(
endpointId,
ssrcAssociation.primarySsrc,
ssrcAssociation.secondarySsrc,
ssrcAssociation.type);
}));
}
/**
* Unsubscribe from SSRC updates.
*/
public void unsubscribe(EncodingsUpdateListener listener)
{
listeners.remove(listener);
}
/**
* An interface for listening to new associations.
*/
public interface EncodingsUpdateListener
{
/**
* Notify this listener of a new association.
*/
void onNewSsrcAssociation(
String endpointId,
long primarySsrc,
long secondarySsrc,
SsrcAssociationType type);
}
/**
* Represents an SSRC association.
*/
private static class SsrcAssociation
{
/**
* The primary SSRC.
*/
private long primarySsrc;
/**
* The secondary SSRC.
*/
private long secondarySsrc;
/**
* The association type.
*/
private SsrcAssociationType type;
/**
* Initializes a new {@link SsrcAssociation} instance.
*/
private SsrcAssociation(
long primarySsrc,
long secondarySsrc,
SsrcAssociationType type)
{
this.primarySsrc = primarySsrc;
this.secondarySsrc = secondarySsrc;
this.type = type;
}
}
}

601
jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java Normal file
View File

@@ -0,0 +1,601 @@
/*
* Copyright @ 2017 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge;
import org.eclipse.jetty.websocket.api.Callback;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.core.CloseStatus;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.datachannel.*;
import org.jitsi.videobridge.datachannel.protocol.*;
import org.jitsi.videobridge.message.*;
import org.jitsi.videobridge.metrics.*;
import org.jitsi.videobridge.relay.*;
import org.jitsi.videobridge.websocket.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.lang.ref.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.stream.*;
import static org.jitsi.videobridge.VersionConfig.config;
import static org.jitsi.videobridge.util.MultiStreamCompatibilityKt.endpointIdToSourceName;
/**
* Handles the functionality related to sending and receiving COLIBRI messages
* for an {@link Endpoint}. Supports two underlying transport mechanisms --
* WebRTC data channels and {@code WebSocket}s.
*
* @author Boris Grozev
*/
public class EndpointMessageTransport
extends AbstractEndpointMessageTransport
implements DataChannelStack.DataChannelMessageListener,
ColibriWebSocket.EventHandler
{
/**
* The last accepted web-socket by this instance, if any.
*/
private ColibriWebSocket webSocket;
/**
* User to synchronize access to {@link #webSocket}
*/
private final Object webSocketSyncRoot = new Object();
/**
* Whether the last active transport channel (i.e. the last to receive a
* message from the remote endpoint) was the web socket (if {@code true}),
* or the WebRTC data channel (if {@code false}).
*/
private boolean webSocketLastActive = false;
private WeakReference<DataChannel> dataChannel = new WeakReference<>(null);
private final EndpointMessageTransportEventHandler eventHandler;
private final AtomicInteger numOutgoingMessagesDropped = new AtomicInteger(0);
/**
* The number of sent message by type.
*/
private final Map<String, AtomicLong> sentMessagesCounts = new ConcurrentHashMap<>();
@NotNull
private final Endpoint endpoint;
/**
* Initializes a new {@link EndpointMessageTransport} instance.
* @param endpoint the associated {@link Endpoint}.
*/
EndpointMessageTransport(
@NotNull Endpoint endpoint,
EndpointMessageTransportEventHandler eventHandler,
Logger parentLogger)
{
super(parentLogger);
this.endpoint = endpoint;
this.eventHandler = eventHandler;
}
/**
* {@inheritDoc}
*/
@Override
protected void notifyTransportChannelConnected()
{
endpoint.endpointMessageTransportConnected();
eventHandler.endpointMessageTransportConnected(endpoint);
}
/**
* {@inheritDoc}
*/
@Override
public BridgeChannelMessage clientHello(ClientHelloMessage message)
{
// ClientHello was introduced for functional testing purposes. It
// triggers a ServerHello response from Videobridge. The exchange
// reveals (to the client) that the transport channel between the
// remote endpoint and the Videobridge is operational.
// We take care to send the reply using the same transport channel on
// which we received the request..
return createServerHello();
}
@Override
public BridgeChannelMessage videoType(VideoTypeMessage videoTypeMessage)
{
return sourceVideoType(
new SourceVideoTypeMessage(
videoTypeMessage.getVideoType(),
endpointIdToSourceName(endpoint.getId()),
videoTypeMessage.getEndpointId())
);
}
@Override
public BridgeChannelMessage sourceVideoType(SourceVideoTypeMessage sourceVideoTypeMessage)
{
String sourceName = sourceVideoTypeMessage.getSourceName();
if (getLogger().isDebugEnabled())
{
getLogger().debug("Received video type of " + sourceName +": " + sourceVideoTypeMessage.getVideoType());
}
endpoint.setVideoType(sourceName, sourceVideoTypeMessage.getVideoType());
Conference conference = endpoint.getConference();
if (conference.isExpired())
{
getLogger().warn("Unable to forward SourceVideoTypeMessage, conference is expired");
return null;
}
sourceVideoTypeMessage.setEndpointId(endpoint.getId());
/* Forward videoType messages to Relays. */
conference.sendMessage(sourceVideoTypeMessage, Collections.emptyList(), true);
return null;
}
@Override
public BridgeChannelMessage receiverAudioSubscription(
@NotNull ReceiverAudioSubscriptionMessage receiverAudioSubscriptionMessage
)
{
if (getLogger().isDebugEnabled())
{
getLogger().debug("Received audio subscription: " + receiverAudioSubscriptionMessage);
}
Conference conference = endpoint.getConference();
conference.setEndpointAudioSubscription(endpoint.getId(), receiverAudioSubscriptionMessage);
return null;
}
@Override
public void unhandledMessage(@NotNull BridgeChannelMessage message)
{
getLogger().warn("Received a message with an unexpected type: " + message.getClass().getSimpleName());
}
/**
* Sends a string via a particular transport channel.
* @param dst the transport channel.
* @param message the message to send.
*/
protected void sendMessage(Object dst, BridgeChannelMessage message)
{
super.sendMessage(dst, message); // Log message
if (dst instanceof ColibriWebSocket)
{
sendMessage((ColibriWebSocket) dst, message);
}
else if (dst instanceof DataChannel)
{
sendMessage((DataChannel)dst, message);
}
else
{
throw new IllegalArgumentException("unknown transport:" + dst);
}
}
/**
* Sends a string via a particular {@link DataChannel}.
* @param dst the data channel to send through.
* @param message the message to send.
*/
private void sendMessage(DataChannel dst, BridgeChannelMessage message)
{
dst.sendString(message.toJson());
VideobridgeMetrics.dataChannelMessagesSent.inc();
}
/**
* Sends a string via a particular {@link ColibriWebSocket} instance.
* @param dst the {@link ColibriWebSocket} through which to send the message.
* @param message the message to send.
*/
private void sendMessage(ColibriWebSocket dst, BridgeChannelMessage message)
{
dst.sendString(message.toJson());
VideobridgeMetrics.colibriWebSocketMessagesSent.inc();
}
@Override
public void onDataChannelMessage(DataChannelMessage dataChannelMessage)
{
webSocketLastActive = false;
VideobridgeMetrics.dataChannelMessagesReceived.inc();
if (dataChannelMessage instanceof DataChannelStringMessage)
{
DataChannelStringMessage dataChannelStringMessage = (DataChannelStringMessage)dataChannelMessage;
onMessage(dataChannel.get(), dataChannelStringMessage.data);
}
}
/**
* {@inheritDoc}
*/
@Override
protected void sendMessage(@NotNull BridgeChannelMessage msg)
{
Object dst = getActiveTransportChannel();
if (dst == null)
{
getLogger().debug("No available transport channel, can't send a message");
numOutgoingMessagesDropped.incrementAndGet();
}
else
{
sentMessagesCounts.computeIfAbsent(
msg.getClass().getSimpleName(),
(k) -> new AtomicLong()).incrementAndGet();
sendMessage(dst, msg);
}
}
/**
* @return the active transport channel for this
* {@link EndpointMessageTransport} (either the {@link #webSocket}, or
* the WebRTC data channel represented by a {@link DataChannel}).
* </p>
* The "active" channel is determined based on what channels are available,
* and which one was the last to receive data. That is, if only one channel
* is available, it will be returned. If two channels are available, the
* last one to have received data will be returned. Otherwise, {@code null}
* will be returned.
*/
//TODO(brian): seems like it'd be nice to have the websocket and datachannel
// share a common parent class (or, at least, have a class that is returned
// here and provides a common API but can wrap either a websocket or
// datachannel)
private Object getActiveTransportChannel()
{
DataChannel dataChannel = this.dataChannel.get();
ColibriWebSocket webSocket = this.webSocket;
Object dst = null;
if (webSocketLastActive)
{
dst = webSocket;
}
// Either the socket was not the last active channel,
// or it has been closed.
if (dst == null)
{
if (dataChannel != null && dataChannel.isReady())
{
dst = dataChannel;
}
}
// Maybe the WebRTC data channel is the last active, but it is not
// currently available. If so, and a web-socket is available -- use it.
if (dst == null && webSocket != null)
{
dst = webSocket;
}
return dst;
}
@Override
public boolean isConnected()
{
return getActiveTransportChannel() != null;
}
/**
* {@inheritDoc}
*/
@Override
public void webSocketConnected(ColibriWebSocket ws)
{
synchronized (webSocketSyncRoot)
{
// If we already have a web-socket, discard it and use the new one.
if (webSocket != null)
{
Session session = webSocket.getSession();
if (session != null)
{
session.close(CloseStatus.NORMAL, "replaced", Callback.NOOP);
}
}
webSocket = ws;
webSocketLastActive = true;
sendMessage(ws, createServerHello());
}
try
{
notifyTransportChannelConnected();
}
catch (Exception e)
{
getLogger().warn("Caught an exception in notifyTransportConnected", e);
}
}
private ServerHelloMessage createServerHello()
{
if (config.announceVersion())
{
return new ServerHelloMessage(endpoint.getConference().getVideobridge().getVersion().toString());
}
else
{
return new ServerHelloMessage();
}
}
/**
* {@inheritDoc}
*/
@Override
public void webSocketClosed(ColibriWebSocket ws, int statusCode, String reason)
{
synchronized (webSocketSyncRoot)
{
if (ws != null && ws.equals(webSocket))
{
webSocket = null;
webSocketLastActive = false;
getLogger().info(() -> "Websocket closed, statusCode " + statusCode + " ( " + reason + ").");
// 1000 is normal, 1001 is e.g. a tab closing. 1005 is "No Status Rcvd" and we see the majority of
// sockets close this way.
if (statusCode == 1000 || statusCode == 1001 || statusCode == 1005)
{
VideobridgeMetrics.colibriWebSocketCloseNormal.inc();
}
else
{
VideobridgeMetrics.colibriWebSocketCloseAbnormal.inc();
}
}
}
}
/**
* {@inheritDoc}
*/
@Override
public void webSocketError(ColibriWebSocket ws, Throwable cause)
{
getLogger().error("Colibri websocket error: " + cause.getMessage());
VideobridgeMetrics.colibriWebSocketErrors.inc();
}
/**
* {@inheritDoc}
*/
@Override
public void close()
{
synchronized (webSocketSyncRoot)
{
if (webSocket != null)
{
// 1001 indicates that an endpoint is "going away", such as a server
// going down or a browser having navigated away from a page.
Session session = webSocket.getSession();
if (session != null)
{
session.close(CloseStatus.SHUTDOWN, "endpoint closed", Callback.NOOP);
}
webSocket = null;
getLogger().debug(() -> "Endpoint expired, closed colibri web-socket.");
}
}
}
/**
* {@inheritDoc}
*/
@Override
public void webSocketTextReceived(ColibriWebSocket ws, String message)
{
if (ws == null || !ws.equals(webSocket))
{
getLogger().warn("Received text from an unknown web socket.");
return;
}
VideobridgeMetrics.colibriWebSocketMessagesReceived.inc();
webSocketLastActive = true;
onMessage(ws, message);
}
/**
* Sets the data channel for this endpoint.
* @param dataChannel the {@link DataChannel} to use for this transport
*/
void setDataChannel(DataChannel dataChannel)
{
DataChannel prevDataChannel = this.dataChannel.get();
if (prevDataChannel == null)
{
this.dataChannel = new WeakReference<>(dataChannel);
// We install the handler first, otherwise the 'ready' might fire after we check it but before we
// install the handler
dataChannel.onDataChannelEvents(this::notifyTransportChannelConnected);
if (dataChannel.isReady())
{
notifyTransportChannelConnected();
}
dataChannel.onDataChannelMessage(this);
}
else if (prevDataChannel == dataChannel)
{
//TODO: i think we should be able to ensure this doesn't happen,
// so throwing for now. if there's a good
// reason for this, we can make this a no-op
throw new Error("Re-setting the same data channel");
}
else
{
throw new Error("Overwriting a previous data channel!");
}
}
@Override
public ObjectNode getDebugState()
{
ObjectNode debugState = super.getDebugState();
debugState.put("numOutgoingMessagesDropped", numOutgoingMessagesDropped.get());
ObjectNode sentCounts = JsonNodeFactory.instance.objectNode();
sentMessagesCounts.forEach((k, v) -> sentCounts.put(k, v.get()));
debugState.set("sent_counts", sentCounts);
return debugState;
}
@Nullable
@Override
public BridgeChannelMessage receiverVideoConstraints(@NotNull ReceiverVideoConstraintsMessage message)
{
endpoint.setBandwidthAllocationSettings(message);
return null;
}
/**
* Notifies this {@code Endpoint} that a {@link LastNMessage} has been
* received.
*
* @param message the message that was received.
*/
@Override
public BridgeChannelMessage lastN(LastNMessage message)
{
endpoint.setLastN(message.getLastN());
return null;
}
/**
* Handles an opaque message from this {@code Endpoint} that should be forwarded to either: a) another client in
* this conference (1:1 message) or b) all other clients in this conference (broadcast message).
*
* @param message the message that was received from the endpoint.
*/
@Override
public BridgeChannelMessage endpointMessage(EndpointMessage message)
{
if (endpoint.getVisitor())
{
getLogger().warn("Not forwarding endpoint message from visitor endpoint");
return null;
}
// First insert/overwrite the "from" to prevent spoofing.
String from = endpoint.getId();
message.setFrom(from);
Conference conference = endpoint.getConference();
if (conference == null || conference.isExpired())
{
getLogger().warn("Unable to send EndpointMessage, conference is null or expired");
return null;
}
if (message.isBroadcast())
{
// Broadcast message to all local endpoints and relays.
List<Endpoint> targets = new LinkedList<>(conference.getLocalEndpoints());
targets.remove(endpoint);
conference.sendMessage(message, targets, /* sendToRelays */ true);
}
else
{
// 1:1 message
String to = message.getTo();
AbstractEndpoint targetEndpoint = conference.getEndpoint(to);
if (targetEndpoint instanceof Endpoint)
{
((Endpoint)targetEndpoint).sendMessage(message);
}
else if (targetEndpoint instanceof RelayedEndpoint)
{
((RelayedEndpoint)targetEndpoint).getRelay().sendMessage(message);
}
else if (targetEndpoint != null)
{
conference.sendMessage(message, Collections.emptyList(), /* sendToRelays */ true);
}
else
{
getLogger().warn("Unable to find endpoint to send EndpointMessage to: " + to);
}
}
return null;
}
/**
* Handles an endpoint statistics message from this {@code Endpoint} that should be forwarded to
* other endpoints as appropriate, and also to relays.
*
* @param message the message that was received from the endpoint.
*/
@Override
public BridgeChannelMessage endpointStats(@NotNull EndpointStats message)
{
if (endpoint.getVisitor())
{
getLogger().warn("Not forwarding endpoint stats from visitor endpoint");
return null;
}
// First insert/overwrite the "from" to prevent spoofing.
String from = endpoint.getId();
message.setFrom(from);
Conference conference = endpoint.getConference();
if (conference.isExpired())
{
getLogger().warn("Unable to send EndpointStats, conference is null or expired");
return null;
}
List<Endpoint> targets = conference.getLocalEndpoints().stream()
.filter((ep) -> ep != endpoint && ep.wantsStatsFrom(endpoint))
.collect(Collectors.toList());
conference.sendMessage(message, targets, true);
return null;
}
}

37
jvb/src/main/java/org/jitsi/videobridge/PotentialPacketHandler.java Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
public interface PotentialPacketHandler
{
/**
* Checks whether or not this PotentialPacketHandler is interested
* in the RTP/RTCP packet 'packet' from 'source'
* @param packet the RTP/RTCP packet
* @return true if this handler wants the given packet, false otherwise
*/
boolean wants(@NotNull PacketInfo packet);
/**
* Send the given RTP/RTCP 'packet' (which came from 'source')
* @param packet the RTP/RTCP packet
*/
void send(@NotNull PacketInfo packet);
}

591
jvb/src/main/java/org/jitsi/videobridge/Videobridge.java Normal file
View File

@@ -0,0 +1,591 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.*;
import org.jitsi.health.Result;
import org.jitsi.nlj.*;
import org.jitsi.shutdown.*;
import org.jitsi.utils.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.utils.version.*;
import org.jitsi.videobridge.health.*;
import org.jitsi.videobridge.load_management.*;
import org.jitsi.videobridge.metrics.*;
import org.jitsi.videobridge.shutdown.*;
import org.jitsi.videobridge.stats.*;
import org.jitsi.videobridge.xmpp.*;
import org.jitsi.xmpp.extensions.colibri2.*;
import org.jitsi.xmpp.extensions.health.*;
import org.jivesoftware.smack.packet.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.jxmpp.jid.*;
import org.jxmpp.jid.impl.*;
import org.jxmpp.stringprep.*;
import java.time.*;
import java.util.*;
import static org.jitsi.videobridge.colibri2.Colibri2UtilKt.*;
import static org.jitsi.xmpp.util.ErrorUtilKt.createError;
/**
* Represents the Jitsi Videobridge which creates, lists and destroys
* {@link Conference} instances.
*
* @author Lyubomir Marinov
* @author Hristo Terezov
* @author Boris Grozev
* @author Brian Baldino
*/
@SuppressWarnings("JavadocReference")
public class Videobridge
{
/**
* The <tt>Logger</tt> used by the <tt>Videobridge</tt> class and its
* instances to print debug information.
*/
private static final Logger logger = new LoggerImpl(Videobridge.class.getName());
/**
* The pseudo-random generator which is to be used when generating
* {@link Conference} IDs in order to minimize busy
* waiting for the value of {@link System#currentTimeMillis()} to change.
*/
public static final Random RANDOM = new Random();
/**
* The <tt>Conference</tt>s of this <tt>Videobridge</tt> mapped by their local IDs.
* <p/>
* TODO: The only remaining uses of this ID are for the HTTP debug interface and the colibri WebSocket conference
* identifier. This should be replaced with meetingId (while making sure jvb-rtcstats-push doesn't break).
*/
private final Map<String, Conference> conferencesById = new HashMap<>();
/**
* The <tt>Conference</tt>s of this <tt>Videobridge</tt> mapped by their
* meeting IDs.
*/
private final Map<String, Conference> conferencesByMeetingId = new HashMap<>();
private final JvbHealthChecker jvbHealthChecker = new JvbHealthChecker();
/**
* The clock to use, pluggable for testing purposes.
* <p/>
* Note that currently most code uses the system clock directly.
*/
@NotNull
private final Clock clock;
/**
* The {@link JvbLoadManager} instance used for this bridge.
*/
@NotNull
private final JvbLoadManager<?> jvbLoadManager;
@NotNull private final Version version;
@NotNull private final ShutdownManager shutdownManager;
/**
* Initializes a new <tt>Videobridge</tt> instance.
*/
public Videobridge(
@Nullable XmppConnection xmppConnection,
@NotNull ShutdownServiceImpl shutdownService,
@NotNull Version version,
@NotNull Clock clock)
{
this.clock = clock;
jvbLoadManager = JvbLoadManager.create(this);
if (xmppConnection != null)
{
xmppConnection.setEventHandler(new XmppConnectionEventHandler());
}
this.version = version;
this.shutdownManager = new ShutdownManager(shutdownService, logger);
jvbHealthChecker.start();
}
@NotNull
public JvbHealthChecker getJvbHealthChecker()
{
return jvbHealthChecker;
}
/**
* Generate conference IDs until one is found that isn't in use and create a new {@link Conference}
* object using that ID
*/
private @NotNull Conference doCreateConference(
@Nullable EntityBareJid name,
String meetingId,
boolean isRtcStatsEnabled)
{
Conference conference = null;
do
{
String id = generateConferenceID();
synchronized (conferencesById)
{
if (meetingId != null && conferencesByMeetingId.containsKey(meetingId))
{
throw new IllegalStateException("Already have a meeting with meetingId " + meetingId);
}
if (!conferencesById.containsKey(id))
{
conference = new Conference(this, id, name, meetingId, isRtcStatsEnabled);
conferencesById.put(id, conference);
VideobridgeMetrics.currentConferences.inc();
if (meetingId != null)
{
conferencesByMeetingId.put(meetingId, conference);
}
}
}
}
while (conference == null);
return conference;
}
void localEndpointCreated(boolean visitor)
{
VideobridgeMetrics.currentLocalEndpoints.inc();
if (visitor)
{
VideobridgeMetrics.currentVisitors.inc();
}
}
void localEndpointExpired(boolean visitor)
{
long remainingEndpoints = VideobridgeMetrics.currentLocalEndpoints.decAndGet();
if (visitor)
{
VideobridgeMetrics.currentVisitors.dec();
}
if (remainingEndpoints < 0)
{
logger.warn("Invalid endpoint count " + remainingEndpoints + ". Disabling endpoint-count based shutdown!");
return;
}
shutdownManager.maybeShutdown(remainingEndpoints);
}
/**
* Initializes a new {@link Conference} instance with an ID unique to the
* <tt>Conference</tt> instances listed by this <tt>Videobridge</tt> and
* adds the new instance to the list of existing <tt>Conference</tt>
* instances.
*
* @param gid the "global" id of the conference (or
* {@link Conference#GID_NOT_SET} if it is not specified.
* @param name world readable name of the conference to create.
* @return a new <tt>Conference</tt> instance with an ID unique to the
* <tt>Conference</tt> instances listed by this <tt>Videobridge</tt>
*/
private @NotNull Conference createConference(
@Nullable EntityBareJid name,
String meetingId,
boolean isRtcStatsEnabled)
{
final Conference conference = doCreateConference(name, meetingId, isRtcStatsEnabled);
logger.info(() -> "create_conf, id=" + conference.getID() + " meeting_id=" + meetingId);
return conference;
}
/**
* Expires a specific <tt>Conference</tt> of this <tt>Videobridge</tt> (i.e.
* if the specified <tt>Conference</tt> is not in the list of
* <tt>Conference</tt>s of this <tt>Videobridge</tt>, does nothing).
*
* @param conference the <tt>Conference</tt> to be expired by this
* <tt>Videobridge</tt>
*/
public void expireConference(Conference conference)
{
String id = conference.getID();
String meetingId = conference.getMeetingId();
synchronized (conferencesById)
{
if (conference.equals(conferencesById.get(id)))
{
conferencesById.remove(id);
VideobridgeMetrics.currentConferences.dec();
if (meetingId != null)
{
if (conference.equals(conferencesByMeetingId.get(meetingId)))
{
conferencesByMeetingId.remove(meetingId);
}
}
conference.expire();
}
}
}
/**
* Generates a new <tt>Conference</tt> ID which is not guaranteed to be
* unique.
*
* @return a new <tt>Conference</tt> ID which is not guaranteed to be unique
*/
private String generateConferenceID()
{
return Long.toHexString(System.currentTimeMillis() + RANDOM.nextLong());
}
/**
* Gets an existing {@link Conference} with a specific ID.
*
* @param id the ID of the existing <tt>Conference</tt> to get
* @return an existing <tt>Conference</tt> with the specified ID.
*/
public Conference getConference(String id)
{
synchronized (conferencesById)
{
return conferencesById.get(id);
}
}
/**
* Gets an existing {@link Conference} with a specific meeting ID.
*/
public Conference getConferenceByMeetingId(@NotNull String meetingId)
{
/* Note that conferenceByMeetingId is synchronized on conferencesById. */
synchronized (conferencesById)
{
return conferencesByMeetingId.get(meetingId);
}
}
/**
* Gets the <tt>Conference</tt>s of this <tt>Videobridge</tt>.
*
* @return the <tt>Conference</tt>s of this <tt>Videobridge</tt>
*/
public Collection<Conference> getConferences()
{
synchronized (conferencesById)
{
return new HashSet<>(conferencesById.values());
}
}
/**
* Handles a COLIBRI2 request synchronously.
* @param conferenceModifyIQ The COLIBRI request.
* @return The response in the form of an {@link IQ}. It is either an error or a {@link ConferenceModifiedIQ}.
*/
public IQ handleConferenceModifyIq(ConferenceModifyIQ conferenceModifyIQ)
{
Conference conference;
try
{
conference = getOrCreateConference(conferenceModifyIQ);
}
catch (ConferenceNotFoundException e)
{
return createConferenceNotFoundError(conferenceModifyIQ, conferenceModifyIQ.getMeetingId());
}
catch (ConferenceAlreadyExistsException e)
{
return createConferenceAlreadyExistsError(conferenceModifyIQ, conferenceModifyIQ.getMeetingId());
}
catch (InGracefulShutdownException e)
{
return createGracefulShutdownErrorResponse(conferenceModifyIQ);
}
catch (XmppStringprepException e)
{
return createError(
conferenceModifyIQ,
StanzaError.Condition.bad_request,
"Invalid conference name (not a JID)");
}
return conference.handleConferenceModifyIQ(conferenceModifyIQ);
}
/**
* Handles a COLIBRI request asynchronously.
*/
private void handleColibriRequest(XmppConnection.ColibriRequest request)
{
ConferenceModifyIQ iq = request.getRequest();
String id = request.getRequest().getMeetingId();
Conference conference;
try
{
conference = getOrCreateConference(request.getRequest());
}
catch (ConferenceNotFoundException e)
{
request.getCallback().invoke(createConferenceNotFoundError(iq, id));
return;
}
catch (ConferenceAlreadyExistsException e)
{
request.getCallback().invoke(createConferenceAlreadyExistsError(iq, id));
return;
}
catch (InGracefulShutdownException e)
{
request.getCallback().invoke(createGracefulShutdownErrorResponse(iq));
return;
}
catch (XmppStringprepException e)
{
request.getCallback().invoke(
createError(iq, StanzaError.Condition.bad_request, "Invalid conference name (not a JID)"));
return;
}
// It is now the responsibility of Conference to send a response.
conference.enqueueColibriRequest(request);
}
private @NotNull Conference getOrCreateConference(ConferenceModifyIQ conferenceModifyIQ)
throws InGracefulShutdownException, XmppStringprepException,
ConferenceAlreadyExistsException, ConferenceNotFoundException
{
String meetingId = conferenceModifyIQ.getMeetingId();
synchronized(conferencesById)
{
Conference conference = getConferenceByMeetingId(meetingId);
if (conferenceModifyIQ.getCreate())
{
if (conference != null)
{
logger.warn("Will not create conference, conference already exists for meetingId=" + meetingId);
throw new ConferenceAlreadyExistsException();
}
if (isInGracefulShutdown())
{
logger.warn("Will not create conference in shutdown mode.");
throw new InGracefulShutdownException();
}
String conferenceName = conferenceModifyIQ.getConferenceName();
return createConference(
conferenceName == null ? null : JidCreate.entityBareFrom(conferenceName),
meetingId,
conferenceModifyIQ.isRtcstatsEnabled()
);
}
else
{
if (conference == null)
{
logger.warn("Conference with meeting_id=" + meetingId + " not found.");
throw new ConferenceNotFoundException();
}
return conference;
}
}
}
/**
* Handles a shutdown request.
*/
public void shutdown(boolean graceful)
{
shutdownManager.initiateShutdown(graceful);
shutdownManager.maybeShutdown(VideobridgeMetrics.currentLocalEndpoints.get());
}
/**
* Whether the bridge is in "drain" mode. The bridge will not be assigned to new
* conferences when in this mode.
*/
private boolean drainMode = VideobridgeConfig.Companion.getInitialDrainMode();
/**
* Handles a drain request.
*/
public void setDrainMode(boolean enable)
{
logger.info("Received drain request. enable=" + enable);
VideobridgeMetrics.INSTANCE.getDrainMode().set(enable);
drainMode = enable;
}
/**
* Query the drain state.
*/
public boolean getDrainMode()
{
return drainMode;
}
/**
* Returns {@code true} if this instance has entered graceful shutdown mode.
*
* @return {@code true} if this instance has entered graceful shutdown mode;
* otherwise, {@code false}
*/
public boolean isInGracefulShutdown()
{
return shutdownManager.getState() == ShutdownState.GRACEFUL_SHUTDOWN;
}
public ShutdownState getShutdownState()
{
return shutdownManager.getState();
}
/**
* Stops this {@link Videobridge}.
*/
void stop()
{
jvbLoadManager.stop();
}
/**
* Creates a JSON for debug purposes. If a specific {@code conferenceId}
* is requested, the result will include all of the state of the conference
* considered useful for debugging (note that this is a LOT of data).
* Otherwise (if {@code conferenceId} is {@code null}), the result includes
* a shallow list of the active conferences and their endpoints.
*
* @param conferenceId the ID of a specific conference to include. If not
* specified, a shallow list of all conferences will be returned.
* @param endpointId the ID of a specific endpoint in {@code conferenceId}
* to include. If not specified, all of the conference's endpoints will be
* included.
*/
public ObjectNode getDebugState(String conferenceId, String endpointId, DebugStateMode mode)
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
if (mode == DebugStateMode.FULL || mode == DebugStateMode.SHORT)
{
debugState.put("shutdown_state", shutdownManager.getState().toString());
debugState.put("drain", drainMode);
debugState.put("time", System.currentTimeMillis());
debugState.set("load_management", jvbLoadManager.getStats());
Double jitter = PacketTransitStats.getBridgeJitter();
if (jitter != null)
{
debugState.put("overall_bridge_jitter", jitter);
}
}
ObjectNode conferences = JsonNodeFactory.instance.objectNode();
debugState.set("conferences", conferences);
if (StringUtils.isBlank(conferenceId))
{
getConferences().stream()
.filter(c -> mode != DebugStateMode.STATS || c.isRtcStatsEnabled())
.forEach(conference ->
conferences.set(
conference.getID(),
conference.getDebugState(mode, null)));
}
else
{
// Using getConference will 'touch' it and prevent it from expiring
Conference conference;
synchronized (conferences)
{
conference = this.conferencesById.get(conferenceId);
}
if (conference == null)
{
conferences.put(conferenceId, "null");
}
else
{
conferences.set(conferenceId, conference.getDebugState(mode, endpointId));
}
}
return debugState;
}
@NotNull
public Version getVersion()
{
return version;
}
private class XmppConnectionEventHandler implements XmppConnection.EventHandler
{
@Override
public void colibriRequestReceived(@NotNull XmppConnection.ColibriRequest request)
{
handleColibriRequest(request);
}
@NotNull
@Override
public IQ versionIqReceived(@NotNull org.jivesoftware.smackx.iqversion.packet.Version iq)
{
org.jivesoftware.smackx.iqversion.packet.Version versionResult =
new org.jivesoftware.smackx.iqversion.packet.Version(
version.getApplicationName(),
version.toString(),
System.getProperty("os.name")
);
versionResult.setType(IQ.Type.result);
return versionResult;
}
@NotNull
@Override
public IQ healthCheckIqReceived(@NotNull HealthCheckIQ iq)
{
Result result = jvbHealthChecker.getResult();
if (result.getSuccess())
{
return IQ.createResultIQ(iq);
}
else
{
return IQ.createErrorResponse(
iq,
StanzaError.from(StanzaError.Condition.internal_server_error, result.getMessage()).build());
}
}
}
private static class ConferenceNotFoundException extends Exception {}
private static class ConferenceAlreadyExistsException extends Exception {}
private static class InGracefulShutdownException extends Exception {}
}

164
jvb/src/main/java/org/jitsi/videobridge/VideobridgeExpireThread.java Normal file
View File

@@ -0,0 +1,164 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge;
import java.time.*;
import java.util.*;
import java.util.concurrent.*;
import org.jitsi.utils.concurrent.*;
import org.jitsi.utils.logging2.*;
import static org.jitsi.videobridge.VideobridgeExpireThreadConfig.config;
/**
* Implements a <tt>Thread</tt> which expires the {@link AbstractEndpoint}s and
* {@link Conference}s of a specific <tt>Videobridge</tt>.
*
* @author Lyubomir Marinov
*/
public class VideobridgeExpireThread
{
/**
* The <tt>Logger</tt> used by the <tt>VideobridgeExpireThread</tt> class
* and its instances to print debug information.
*/
private static final Logger logger =
new LoggerImpl(VideobridgeExpireThread.class.getName());
/**
* The executor which periodically calls {@link #expire(Videobridge)} (if
* this {@link VideobridgeExpireThread} has been started).
*/
private static final RecurringRunnableExecutor EXECUTOR
= new RecurringRunnableExecutor(
VideobridgeExpireThread.class.getSimpleName());
/**
* The executor used to expire the individual {@link AbstractEndpoint}s
* or {@link Conference}s.
*/
private static final Executor EXPIRE_EXECUTOR
= ExecutorUtils.newCachedThreadPool(
true, VideobridgeExpireThread.class.getSimpleName() + "-channel");
/**
* The {@link PeriodicRunnable} registered with {@link #EXECUTOR} which is
* to run the expire task for this {@link VideobridgeExpireThread} instance.
*/
private PeriodicRunnable expireRunnable;
/**
* The {@link Videobridge} which has its {@link Conference}s expired by this
* instance.
*/
private Videobridge videobridge;
/**
* Initializes a new {@link VideobridgeExpireThread} instance which is to
* expire the {@link Conference}s of a specific {@link Videobridge}.
*
* @param videobridge the {@link Videobridge} which is to have its
* {@link Conference}s expired by the new instance.
*/
public VideobridgeExpireThread(Videobridge videobridge)
{
this.videobridge = Objects.requireNonNull(videobridge);
}
/**
* Starts this {@link VideobridgeExpireThread}
*/
void start()
{
Duration expireCheckSleepDuration = config.getInterval();
logger.info(
"Starting with " + expireCheckSleepDuration.getSeconds() + " second interval.");
expireRunnable = new PeriodicRunnable(expireCheckSleepDuration.toMillis())
{
@Override
public void run()
{
super.run();
Videobridge videobridge
= VideobridgeExpireThread.this.videobridge;
if (videobridge != null)
{
expire(videobridge);
}
// The current implementation of the executor fails with a
// concurrent modification exception if we de-register from
// the thread running run(). So we can not de-register here
// if videobridge==null, and we will keep running until we get
// explicitly stop()ed, which is fine.
}
};
EXECUTOR.registerRecurringRunnable(expireRunnable);
}
/**
* Stops this {@link VideobridgeExpireThread}.
*/
void stop()
{
logger.info("Stopping.");
if (expireRunnable != null)
{
EXECUTOR.deRegisterRecurringRunnable(expireRunnable);
}
expireRunnable = null;
videobridge = null;
}
/**
* Expires the {@link Conference}s and/or endpoints of a specific <tt>Videobridge</tt> if they
* have been inactive for more than their advertised <tt>expire</tt> number
* of seconds.
*
* @param videobridge the <tt>Videobridge</tt> which is to have its
* <tt>Channel</tt>s expired if they have been inactive for more than their
* advertised <tt>expire</tt> number of seconds
*/
private void expire(Videobridge videobridge)
{
logger.debug("Running expire()");
for (Conference conference : videobridge.getConferences())
{
// The Conferences will live an iteration more than the Contents.
if (conference.shouldExpire())
{
logger.info("Conference "
+ conference.getID() + " should expire, expiring it");
EXPIRE_EXECUTOR.execute(
() -> videobridge.expireConference(conference));
}
else
{
for (AbstractEndpoint endpoint : conference.getEndpoints())
{
if (endpoint.shouldExpire())
{
logger.info("Expiring endpoint " + endpoint.getId());
EXPIRE_EXECUTOR.execute(endpoint::expire);
}
}
}
}
}
}

407
jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java Normal file
View File

@@ -0,0 +1,407 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
import org.jitsi.nlj.rtp.*;
import org.jitsi.nlj.rtp.codec.av1.*;
import org.jitsi.nlj.rtp.codec.vp8.*;
import org.jitsi.nlj.rtp.codec.vp9.*;
import org.jitsi.rtp.rtcp.*;
import org.jitsi.utils.logging.*;
import org.jitsi.utils.logging2.Logger;
import org.jitsi.videobridge.cc.av1.*;
import org.jitsi.videobridge.cc.vp8.*;
import org.jitsi.videobridge.cc.vp9.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.lang.*;
import java.util.*;
/**
* Filters the packets coming from a specific {@link MediaSourceDesc}
* based on the currently forwarded subjective quality index. It's also taking
* care of upscaling and downscaling. It is also responsible for rewriting the
* forwarded packets so that the the quality switches are transparent from the
* receiver. See svc.md in the doc folder for more details.
*
* XXX A projection is a "function" of taking an element of one space and
* somehow putting it on another space. This is what we're doing here, we're
* taking a source of the ingress and we're projecting it to the egress,
* depending on the layers/encodings of the input source and how these encodings are
* implemented (which depends on the codec that is used).
*
* Instances of this class are thread-safe.
*
* @author George Politis
*/
public class AdaptiveSourceProjection
{
/**
* The <tt>Logger</tt> used by the <tt>AdaptiveSourceProjection</tt> class
* and its instances for logging output.
*/
private final Logger logger;
/**
* The main SSRC of the source (if simulcast is used, this is the SSRC
* of the low-quality layer). We use it as the SSRC of the source projection
* and also request keyframes from this SSRC.
*/
private final long targetSsrc;
private final DiagnosticContext diagnosticContext;
/**
* The payload specific source projection context that's responsible for
* rewriting the packets of a projected source.
*
* XXX The more general scheme and more correct approach would be to have
* the adaptive source projection manage a context per payload type. The
* intention was to implement the more general approach but, at the time of
* this righting, we have no use case for that right now nor an easy way to
* test this dynamic payload type changes. So practically what happens is
* that we create the context once, and it remains the same throughout the
* life of the this instance. It's not really possible to simplify and make
* the context final (=> create it during construction) because we need a
* payload type (could be VP9, could be H264, could be VP8) so it has to be
* created on packet arrival.
*/
private AdaptiveSourceProjectionContext context;
/**
* The target quality index for this source projection.
*/
private int targetIndex = RtpLayerDesc.SUSPENDED_INDEX;
/**
* The map for persistent states.
* Currently, we only expect it for AV1.
*/
private final HashMap<Class<? extends AdaptiveSourceProjectionContext>, Object> persistentStates = new HashMap<>();
/**
* Ctor.
*
* @param source the {@link MediaSourceDesc} that owns the packets
* that this instance filters.
*/
public AdaptiveSourceProjection(
@NotNull DiagnosticContext diagnosticContext,
@NotNull MediaSourceDesc source,
Runnable keyframeRequester,
Logger parentLogger
)
{
targetSsrc = source.getPrimarySSRC();
this.diagnosticContext = diagnosticContext;
this.logger = parentLogger.createChildLogger(AdaptiveSourceProjection.class.getName(),
Map.of("targetSsrc", Long.toString(targetSsrc),
"srcEpId", Objects.toString(source.getOwner(), ""),
"srcName", source.getSourceName()
));
this.keyframeRequester = keyframeRequester;
}
/**
* Sets the target index value for this source projection.
*/
public void setTargetIndex(int value)
{
targetIndex = value;
}
/**
* The callback we'll invoke when we want to request a keyframe for a stream.
*/
private final Runnable keyframeRequester;
/**
* Determines whether an RTP packet needs to be accepted or not.
*
* @param packetInfo packet info for the video RTP packet to determine
* whether to accept or not.
* @return true if the packet is accepted, false otherwise.
*/
public boolean accept(@NotNull PacketInfo packetInfo)
{
VideoRtpPacket videoRtpPacket = packetInfo.packetAs();
AdaptiveSourceProjectionContext contextCopy = getContext(videoRtpPacket);
if (contextCopy == null)
{
return false;
}
// XXX We want to let the context know that the stream has been
// suspended so that it can raise the needsKeyframe flag and also allow
// it to compute a sequence number delta when the target becomes > -1.
if (videoRtpPacket.getEncodingId() == RtpLayerDesc.SUSPENDED_ENCODING_ID)
{
logger.warn(
"Dropping an RTP packet, because egress was unable to find " +
"an associated layer/encoding. rtpPacket=" + videoRtpPacket);
return false;
}
int targetIndexCopy = targetIndex;
boolean accept = contextCopy.accept(packetInfo, targetIndexCopy);
// We check if the context needs a keyframe regardless of whether or not
// the packet was accepted.
//
// XXX Upon reading this for the first time, one may think it's
// sufficient to only check for needing a key frame if the packet wasn't
// accepted. But this wouldn't be enough, as we may be accepting packets
// of low-quality, while we wish to switch to high-quality.
if (contextCopy.needsKeyframe()
&& targetIndexCopy > RtpLayerDesc.SUSPENDED_INDEX)
{
keyframeRequester.run();
}
return accept;
}
/**
* Gets or creates the adaptive source projection context that corresponds to
* the parsed class of the RTP packet that is specified as a parameter. If
* the payload type is different from the appropriate one for the context, then a
* new adaptive source projection context is created that is appropriate for
* the new payload type.
*
* We make no attempt for thread safety, assuming no concurrent access.
*
* @param rtpPacket the RTP packet of the adaptive source projection context
* to get or create.
* @return the adaptive source projection context that corresponds to
* the payload type of the RTP packet that is specified as a parameter.
*/
private AdaptiveSourceProjectionContext getContext(@NotNull VideoRtpPacket rtpPacket)
{
int payloadType = rtpPacket.getPayloadType();
if (rtpPacket instanceof Vp8Packet)
{
// Context switch between VP8 simulcast and VP8 non-simulcast (sort
// of pretend that they're different codecs).
//
// HACK: When simulcast is activated, we also get temporal
// scalability, conversely if temporal scalability is disabled
// then simulcast is disabled.
/* Check whether this stream is projectable by the VP8AdaptiveSourceProjectionContext. */
boolean projectable = ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex() &&
((Vp8Packet)rtpPacket).getHasPictureId();
if (projectable
&& !(context instanceof VP8AdaptiveSourceProjectionContext))
{
// context switch
RtpState rtpState = getRtpState();
logger.debug(() -> "adaptive source projection " +
(context == null ? "creating new" : "changing to") +
" VP8 context for source packet ssrc " + rtpPacket.getSsrc());
context = new VP8AdaptiveSourceProjectionContext(
diagnosticContext, rtpState, logger);
}
else if (!projectable
&& (!(context instanceof GenericAdaptiveSourceProjectionContext) ||
((GenericAdaptiveSourceProjectionContext)context).getPayloadType() != payloadType))
{
RtpState rtpState = getRtpState();
// context switch
logger.debug(() -> {
boolean hasTemporalLayer = ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex();
boolean hasPictureId = ((Vp8Packet)rtpPacket).getHasPictureId();
return "adaptive source projection " +
(context == null ? "creating new" : "changing to") +
" generic context for non-scalable VP8 payload " +
" (packet is " + rtpPacket.getClass().getSimpleName() +
", ssrc " + rtpPacket.getSsrc() +
", hasTL=" + hasTemporalLayer + ", hasPID=" + hasPictureId + ")";
});
context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, logger);
}
// no context switch
return context;
}
else if (rtpPacket instanceof Vp9Packet)
{
if (!(context instanceof Vp9AdaptiveSourceProjectionContext))
{
// context switch
RtpState rtpState = getRtpState();
logger.debug(() -> "adaptive source projection " +
(context == null ? "creating new" : "changing to") +
" VP9 context for source packet ssrc " + rtpPacket.getSsrc());
context = new Vp9AdaptiveSourceProjectionContext(
diagnosticContext, rtpState, logger);
}
return context;
}
else if (rtpPacket instanceof Av1DDPacket)
{
if (!(context instanceof Av1DDAdaptiveSourceProjectionContext))
{
// context switch
RtpState rtpState = getRtpState();
logger.debug(() -> "adaptive source projection " +
(context == null ? "creating new" : "changing to") +
" AV1 DD context for source packet ssrc " + rtpPacket.getSsrc());
context = new Av1DDAdaptiveSourceProjectionContext(
diagnosticContext, rtpState,
persistentStates.get(Av1DDAdaptiveSourceProjectionContext.class),
logger);
}
return context;
}
else
{
if (!(context instanceof GenericAdaptiveSourceProjectionContext) ||
((GenericAdaptiveSourceProjectionContext)context).getPayloadType() != payloadType)
{
RtpState rtpState = getRtpState();
logger.debug(() -> "adaptive source projection " +
(context == null ? "creating new" : "changing to") +
" generic context for payload type " + rtpPacket.getPayloadType());
context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, logger);
}
return context;
}
}
/**
* Gets the {@link RtpState}.
*/
private @NotNull RtpState getRtpState()
{
if (context == null)
{
// TODO If '1' are the starting seq number and timestamp, should
// we use random values?
return new RtpState(
targetSsrc,
1 /* maxSequenceNumber */,
1 /* maxTimestamp */) ;
}
else
{
savePersistentState();
return context.getRtpState();
}
}
private void savePersistentState()
{
if (context == null)
{
return;
}
Object state = context.getPersistentState();
if (state != null)
{
if (context.getClass() != Av1DDAdaptiveSourceProjectionContext.class)
{
logger.warn("Got unexpected context persistent state from class " +
context.getClass().getSimpleName());
}
persistentStates.put(context.getClass(), state);
}
else if (persistentStates.get(context.getClass()) != null)
{
persistentStates.remove(context.getClass());
}
}
/**
* Rewrites an RTP packet for projection.
*
* @param packetInfo the RTP packet to rewrite.
*/
public void rewriteRtp(@NotNull PacketInfo packetInfo)
throws RewriteException
{
AdaptiveSourceProjectionContext contextCopy = context;
if (contextCopy != null)
{
contextCopy.rewriteRtp(packetInfo);
}
}
/**
* Rewrites an RTCP packet.
*
* @param rtcpSrPacket the RTCP SR packet to rewrite.
* @return true to let the RTCP packet through, false to drop.
*/
public boolean rewriteRtcp(@NotNull RtcpSrPacket rtcpSrPacket)
{
AdaptiveSourceProjectionContext contextCopy = context;
if (contextCopy == null)
{
return true;
}
return contextCopy.rewriteRtcp(rtcpSrPacket);
}
/**
* @return the SSRC of the source projection.
*/
public long getTargetSsrc()
{
return targetSsrc;
}
/**
* Gets a JSON representation of the parts of this object's state that
* are deemed useful for debugging.
*/
public ObjectNode getDebugState(@NotNull DebugStateMode mode)
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
debugState.put("targetSsrc", targetSsrc);
AdaptiveSourceProjectionContext contextCopy = context;
if (mode == DebugStateMode.FULL)
{
if (contextCopy == null)
{
debugState.putNull("context");
}
else
{
debugState.set("context", contextCopy.getDebugState());
}
}
else if (mode == DebugStateMode.STATS)
{
debugState.put(
"context",
contextCopy == null ? "null" : contextCopy.getClass().getSimpleName());
}
debugState.put("needsKeyframe", contextCopy == null ? "N/A" : Boolean.toString(contextCopy.needsKeyframe()));
debugState.put("targetIndex", targetIndex);
return debugState;
}
}

99
jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java Normal file
View File

@@ -0,0 +1,99 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc;
import edu.umd.cs.findbugs.annotations.Nullable;
import org.jitsi.nlj.*;
import org.jitsi.rtp.rtcp.*;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* Implementations of this interface are responsible for projecting a specific
* video source of a specific payload type.
*
* One can imagine signaling a specific encoding layout (i.e. 3 temporal layers)
* and multiple codec support (e.g. VP8 SVC and VP9 SVC). In the bridge such
* signaling would translate into an AdaptiveSourceProjection instance that would
* remain active throughout the life of the source video source and would receive
* updates from the bitrate controller (i.e. ideal index and target index). The
* specific way of projecting VP9 SVC or VP8 SVC is implemented in "context"
* classes that know how to deal with codec specificities.
*
* @author George Politis
*/
public interface AdaptiveSourceProjectionContext
{
/**
* Determines whether an RTP packet should be accepted or not.
*
* @param packetInfo the RTP packet to determine whether to accept or not.
* @param targetIndex the target quality index
* @return true if the packet should be accepted, false otherwise.
*/
boolean accept(PacketInfo packetInfo, int targetIndex);
/**
* @return true if this stream context needs a keyframe in order to either
* start rendering again or there's a pending simulcast switch (depending
* on the implementation).
*/
boolean needsKeyframe();
/**
* Rewrites the timestamp, sequence number, ssrc and other codec dependent
* fields of the RTP packet that is specified as an argument. Projecting a
* video source needs to be invisible to the receiving endpoint so goal here
* is to make the resulting rtp stream continuous.
*
* @param packetInfo the RTP packet info to rewrite.
* @throws RewriteException the underlying code has failed to rewrite the
* RTP packet that is specified as an argument.
*/
void rewriteRtp(PacketInfo packetInfo)
throws RewriteException;
/**
* Rewrites the RTCP packet that is specified as an argument.
*
* @param rtcpSrPacket the RTCP packet to transform.
* @return true if the RTCP packet is accepted, false otherwise, in which
* case it needs to be dropped.
*/
boolean rewriteRtcp(RtcpSrPacket rtcpSrPacket);
/**
* @return the RTP state that describes the max sequence number, max
* timestamp and other RTP-level details.
*/
RtpState getRtpState();
/**
* @return Persistent state, if any, that should be associated with this
* particular projection context type, so that when it comes back it can
* resume data, if necessary.
*/
@Nullable
default Object getPersistentState()
{
return null;
}
/**
* Gets a JSON representation of the parts of this object's state that
* are deemed useful for debugging.
*/
ObjectNode getDebugState();
}

349
jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java Normal file
View File

@@ -0,0 +1,349 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
import org.jitsi.nlj.rtp.*;
import org.jitsi.rtp.rtcp.*;
import org.jitsi.rtp.util.*;
import org.jitsi.utils.logging2.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
/**
* A generic implementation of an adaptive source projection context that can be
* used with non-SVC codecs or when simulcast is not enabled/used or when
* support for these advanced features is not implemented in the bridge. In this
* restricted case the source projection can have only two states (or qualities),
* either off or on (or -1, 0).
*
* Instances of this class suspend a source when the target quality is set to -1.
* When the target quality is set back to 0, the request key frame flag is
* raised and the source is re-activated after a key frame is received.
* (consequently support for key frame detection for the specific media format
* of the source that is being adapted is necessary).
*
* In order to make the suspend/resume operation transparent to the receiver (at
* least in the RTP level), instances of this class rewrite the RTP sequence
* numbers of the source to hide the gaps caused by the suspend/resume
* operation.
*
* This may not be sufficient for fluid playback at the receiver as the decoder
* may be unable to handle codec specific discontinuities (such as discontinuous
* picture IDs in VP8). In this case a codec specific adaptive source projection
* context implementation will have to be used instead.
*
* Instances of this class are thread-safe.
*
* @author George Politis
*/
class GenericAdaptiveSourceProjectionContext
implements AdaptiveSourceProjectionContext
{
/**
* The <tt>Logger</tt> used by the
* <tt>GenericAdaptiveSourceProjectionContext</tt> class and its instances to
* log debug information.
*/
private final Logger logger;
private final long ssrc;
/**
* Raised when a source has been resumed (after being suspended).
*/
private boolean needsKeyframe = true;
/**
* Useful to determine when we need to change generic projection contexts.
*/
private final int payloadType;
/**
* The maximum sequence number that we have sent.
*/
private int maxDestinationSequenceNumber;
/**
* The delta to apply to the timestamps of the RTP packets of the incoming
* source.
*/
private long timestampDelta;
/**
* A boolean that indicates whether or not the timestamp delta has been
* initialized. This is only necessary upon adaptive source projection
* context switches.
*/
private boolean timestampDeltaInitialized = false;
/**
* The maximum timestamp that we have sent.
*/
private long maxDestinationTimestamp;
/**
* The delta to apply to the sequence numbers of the RTP packets of the
* incoming source.
*/
private int sequenceNumberDelta;
/**
* Ctor.
*
* @param payloadType the media format to expect
* @param rtpState the RTP state (i.e. seqnum, timestamp to start with, etc).
*/
GenericAdaptiveSourceProjectionContext(
int payloadType,
@NotNull RtpState rtpState,
@NotNull Logger parentLogger)
{
this.payloadType = payloadType;
this.ssrc = rtpState.ssrc;
this.maxDestinationSequenceNumber = rtpState.maxSequenceNumber;
this.maxDestinationTimestamp = rtpState.maxTimestamp;
this.logger = parentLogger.createChildLogger(
GenericAdaptiveSourceProjectionContext.class.getName());
}
/**
* Determines whether an RTP packet from the incoming source should be accepted
* or not. If the source is currently suspended, a key frame is necessary to
* start accepting packets again.
* <p>
* Note that, at the time of this writing, there's no practical need for a
* synchronized keyword because there's only one thread (the translator
* thread) accessing this method at a time.
*
* @param packetInfo the RTP packet to determine whether to accept or not.
* @param targetIndex the target quality index
* @return true if the packet should be accepted, false otherwise.
*/
@Override
public synchronized boolean
accept(@NotNull PacketInfo packetInfo, int targetIndex)
{
VideoRtpPacket rtpPacket = packetInfo.packetAs();
if (targetIndex == RtpLayerDesc.SUSPENDED_INDEX)
{
// suspend the stream.
needsKeyframe = true;
return false;
}
int sourceSequenceNumber = rtpPacket.getSequenceNumber();
boolean accept;
if (needsKeyframe)
{
if (!(rtpPacket instanceof ParsedVideoPacket))
{
/* We don't know how to parse this packet's codec type, so
* isKeyframe will never be true. The best we can do is
* to start forwarding it immediately. Hopefully the receiver
* will send a PLI/FIR if it needs a keyframe.
*/
needsKeyframe = false;
accept = true;
}
else if (((ParsedVideoPacket)rtpPacket).isKeyframe())
{
needsKeyframe = false;
// resume after being suspended, we compute the new seqnum delta
// delta = destination - source <=> destination = source + delta
// In other words, we compute the using this formula
// "delta = destination - source" and in order to find the
// destination sequence number we use the equivalent formula
// "destination = source + delta".
int destinationSequenceNumber
= maxDestinationSequenceNumber + 1;
sequenceNumberDelta
= RtpUtils.getSequenceNumberDelta(
destinationSequenceNumber,
sourceSequenceNumber);
logger.trace(() -> "delta ssrc=" + rtpPacket.getSsrc()
+ ",src_sequence=" + sourceSequenceNumber
+ ",dst_sequence=" + destinationSequenceNumber
+ ",max_sequence=" + maxDestinationSequenceNumber
+ ",delta=" + sequenceNumberDelta);
accept = true;
}
else
{
accept = false;
}
}
else
{
accept = true;
}
if (accept)
{
maybeInitializeTimestampDelta(rtpPacket.getTimestamp());
int destinationSequenceNumber
= RtpUtils.applySequenceNumberDelta(
sourceSequenceNumber, sequenceNumberDelta);
long destinationTimestamp
= RtpUtils.applyTimestampDelta(
rtpPacket.getTimestamp(), timestampDelta);
if (RtpUtils.isOlderSequenceNumberThan(
maxDestinationSequenceNumber, destinationSequenceNumber))
{
maxDestinationSequenceNumber = destinationSequenceNumber;
}
if (RtpUtils.isNewerTimestampThan(
destinationTimestamp, maxDestinationTimestamp))
{
maxDestinationTimestamp = destinationTimestamp;
}
logger.trace(() -> "accept ssrc=" + rtpPacket.getSsrc()
+ ",src_sequence=" + sourceSequenceNumber
+ ",dst_sequence=" + destinationSequenceNumber
+ ",max_sequence=" + maxDestinationSequenceNumber);
}
else
{
logger.trace(() -> "reject ssrc=" + rtpPacket.getSsrc()
+ ",src_sequence=" + sourceSequenceNumber);
}
return accept;
}
/**
* Initializes {@link #timestampDelta} if it hasn't been initialized
* already.
* @param sourceTimestamp
*/
private synchronized void maybeInitializeTimestampDelta(long sourceTimestamp)
{
if (timestampDeltaInitialized)
{
return;
}
if (RtpUtils.isNewerTimestampThan(
maxDestinationTimestamp, sourceTimestamp))
{
long destinationTimestamp =
RtpUtils.applyTimestampDelta(maxDestinationTimestamp, 3000);
timestampDelta
= RtpUtils.getTimestampDiff(
destinationTimestamp, sourceTimestamp);
}
timestampDeltaInitialized = true;
}
/**
* @return true when a source has been resumed (after being suspended).
*/
@Override
public boolean needsKeyframe()
{
return needsKeyframe;
}
/**
* Applies a delta to the sequence number of the RTP packet that is
* specified as an argument in order to make suspending/resuming of the
* incoming source transparent at the RTP level.
*
* @param packetInfo the RTP packet info to rewrite.
*/
@Override
public void rewriteRtp(
@NotNull PacketInfo packetInfo)
{
VideoRtpPacket rtpPacket = packetInfo.packetAs();
int sourceSequenceNumber = rtpPacket.getSequenceNumber();
int destinationSequenceNumber
= RtpUtils.applySequenceNumberDelta(
sourceSequenceNumber, sequenceNumberDelta);
if (sourceSequenceNumber != destinationSequenceNumber)
{
rtpPacket.setSequenceNumber(destinationSequenceNumber);
}
long sourceTimestamp = rtpPacket.getTimestamp();
long destinationTimestamp
= RtpUtils.applyTimestampDelta(sourceTimestamp, timestampDelta);
if (sourceTimestamp != destinationTimestamp)
{
rtpPacket.setTimestamp(destinationTimestamp);
}
logger.debug(() -> "rewrite ssrc=" + rtpPacket.getSsrc()
+ ",src_sequence=" + sourceSequenceNumber
+ ",dst_sequence=" + destinationSequenceNumber
+ ",max_sequence=" + maxDestinationSequenceNumber);
}
/**
* If the first RTCP packet of the compound RTCP packet that is specified as
* a parameter is an SR, then this method updates the transmitted bytes and
* transmitted packets of that first SR.
*
* @param rtcpSrPacket the compound RTCP packet to rewrite.
* @return true.
*/
@Override
public boolean rewriteRtcp(@NotNull RtcpSrPacket rtcpSrPacket)
{
return true;
}
@Override
public RtpState getRtpState()
{
return new RtpState(
ssrc, maxDestinationSequenceNumber, maxDestinationTimestamp);
}
public int getPayloadType()
{
return payloadType;
}
/**
* {@inheritDoc}
* @return
*/
@Override
public ObjectNode getDebugState()
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
debugState.put(
"class",
GenericAdaptiveSourceProjectionContext.class.getSimpleName());
debugState.put("TODO", "export more state (or refactor)");
return debugState;
}
}

35
jvb/src/main/java/org/jitsi/videobridge/cc/RewriteException.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc;
/**
* Thrown if the underlying code failed to rewrite an RTP packet.
*
* @author George Politis
*/
public class RewriteException
extends Exception
{
public RewriteException()
{
super();
}
public RewriteException(String message)
{
super(message);
}
}

52
jvb/src/main/java/org/jitsi/videobridge/cc/RtpState.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc;
/**
* This class represents the RTP state and is used in adaptive source context
* switches (see {@link AdaptiveSourceProjection}).
*
* @author George Politis
*/
public class RtpState
{
/**
* The SSRC of the RTP stream that this information pertains to.
*/
public final long ssrc;
/**
* The highest sent RTP timestamp.
*/
public final long maxTimestamp;
/**
* The highest sent sequence number.
*/
public final int maxSequenceNumber;
/**
* @param ssrc the SSRC of the RTP stream that this information pertains to.
* @param maxSequenceNumber the highest sent sequence number.
* @param maxTimestamp the highest sent RTP timestamp.
*/
public RtpState(long ssrc, int maxSequenceNumber, long maxTimestamp)
{
this.ssrc = ssrc;
this.maxSequenceNumber = maxSequenceNumber;
this.maxTimestamp = maxTimestamp;
}
}

731
jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java Normal file
View File

@@ -0,0 +1,731 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.vp8;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
import org.jitsi.nlj.codec.vpx.*;
import org.jitsi.nlj.rtp.codec.vp8.*;
import org.jitsi.rtp.rtcp.*;
import org.jitsi.rtp.util.*;
import org.jitsi.utils.logging.*;
import org.jitsi.utils.logging2.Logger;
import org.jitsi.videobridge.cc.*;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.time.*;
import java.util.*;
/**
* This class represents a projection of a VP8 RTP stream in the RFC 7667 sense
* and it is the main entry point for VP8 simulcast/svc RTP/RTCP rewriting. Read
* svc.md for implementation details. Instances of this class are thread-safe.
*
* @author George Politis
*/
public class VP8AdaptiveSourceProjectionContext
implements AdaptiveSourceProjectionContext
{
private final Logger logger;
/**
* A map that stores the per-encoding VP8 frame maps.
*/
private final Map<Long, VP8FrameMap>
vp8FrameMaps = new HashMap<>();
/**
* The {@link VP8QualityFilter} instance that does quality filtering on the
* incoming frames.
*/
private final VP8QualityFilter vp8QualityFilter;
/**
* The diagnostic context of this instance.
*/
private final DiagnosticContext diagnosticContext;
/**
* The "last" {@link VP8FrameProjection} that this instance has accepted.
* In this context, last here means with the highest sequence number
* and not, for example, the last one received by the bridge.
*/
private VP8FrameProjection lastVP8FrameProjection;
/**
* Ctor.
*
* @param rtpState the RTP state to begin with.
*/
public VP8AdaptiveSourceProjectionContext(
@NotNull DiagnosticContext diagnosticContext,
@NotNull RtpState rtpState,
@NotNull Logger parentLogger)
{
this.diagnosticContext = diagnosticContext;
this.logger = parentLogger.createChildLogger(
VP8AdaptiveSourceProjectionContext.class.getName());
this.vp8QualityFilter = new VP8QualityFilter(parentLogger);
lastVP8FrameProjection = new VP8FrameProjection(diagnosticContext,
rtpState.ssrc, rtpState.maxSequenceNumber, rtpState.maxTimestamp);
}
/** Lookup a Vp8Frame for a packet. */
private VP8Frame lookupVP8Frame(@NotNull Vp8Packet vp8Packet)
{
VP8FrameMap frameMap = vp8FrameMaps.get(vp8Packet.getSsrc());
if (frameMap == null)
return null;
return frameMap.findFrame(vp8Packet);
}
/**
* Insert a packet in the appropriate Vp8FrameMap.
*/
private VP8FrameMap.FrameInsertionResult insertPacketInMap(
@NotNull Vp8Packet vp8Packet)
{
VP8FrameMap frameMap = vp8FrameMaps.computeIfAbsent(vp8Packet.getSsrc(),
ssrc -> new VP8FrameMap(logger));
/* TODO: add more context (ssrc?) to frame map's logger? */
return frameMap.insertPacket(vp8Packet);
}
/**
* Find the previous frame before the given one.
*/
@Nullable
private synchronized VP8Frame prevFrame(@NotNull VP8Frame frame)
{
VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc());
if (frameMap == null)
{
return null;
}
return frameMap.prevFrame(frame);
}
/**
* Find the next frame after the given one.
*/
@Nullable
private synchronized VP8Frame nextFrame(@NotNull VP8Frame frame)
{
VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc());
if (frameMap == null)
{
return null;
}
return frameMap.nextFrame(frame);
}
/**
* Find the previous accepted frame before the given one.
*/
@Nullable
private VP8Frame findPrevAcceptedFrame(@NotNull VP8Frame frame)
{
VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc());
if (frameMap == null)
{
return null;
}
return frameMap.findPrevAcceptedFrame(frame);
}
/**
* Find the next accepted frame after the given one.
*/
@Nullable
private VP8Frame findNextAcceptedFrame(@NotNull VP8Frame frame)
{
VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc());
if (frameMap == null)
{
return null;
}
return frameMap.findNextAcceptedFrame(frame);
}
/**
* Find a subsequent Tid==0 frame after the given frame
* @param frame The frame to query
* @return A subsequent TL0 frame, or null
*/
@Nullable
private VP8Frame findNextTl0(@NotNull VP8Frame frame)
{
VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc());
if (frameMap == null)
{
return null;
}
return frameMap.findNextTl0(frame);
}
/**
* Calculate the projected sequence number gap between two frames (of the same encoding),
* allowing collapsing for unaccepted frames.
*/
private int seqGap(@NotNull VP8Frame frame1, @NotNull VP8Frame frame2)
{
int seqGap = RtpUtils.getSequenceNumberDelta(
frame2.getEarliestKnownSequenceNumber(),
frame1.getLatestKnownSequenceNumber()
);
if (!frame1.isAccepted() && !frame2.isAccepted() &&
frame2.isImmediatelyAfter(frame1))
{
/* If neither frame is being projected, and they have consecutive
picture IDs, we don't need to leave any gap. */
seqGap = 0;
}
else
{
/* If the earlier frame wasn't projected, and we haven't seen its
* final packet, we know it has to consume at least one more sequence number. */
if (!frame1.isAccepted() && !frame1.hasSeenEndOfFrame() && seqGap > 1)
{
seqGap--;
}
/* Similarly, if the later frame wasn't projected and we haven't seen
* its first packet. */
if (!frame2.isAccepted() && !frame2.hasSeenStartOfFrame() && seqGap > 1)
{
seqGap--;
}
if (!frame1.isAccepted() && seqGap > 0)
{
seqGap--;
}
}
return seqGap;
}
/**
* Calculate the projected picture ID gap between two frames (of the same encoding),
* allowing collapsing for unaccepted frames.
*/
private int picGap(@NotNull VP8Frame frame1, @NotNull VP8Frame frame2)
{
int picGap = VpxUtils.getExtendedPictureIdDelta(frame2.getPictureId(), frame1.getPictureId());
if (!frame1.isAccepted() && picGap > 0)
{
picGap--;
}
return picGap;
}
private boolean frameIsNewSsrc(VP8Frame frame)
{
return lastVP8FrameProjection.getVP8Frame() == null ||
!frame.matchesSSRC(lastVP8FrameProjection.getVP8Frame());
}
/**
* Determines whether a packet should be accepted or not.
*
* @param packetInfo the RTP packet to determine whether to project or not.
* @param targetIndex the target quality index we want to achieve
* @return true if the packet should be accepted, false otherwise.
*/
@Override
public synchronized boolean accept(
@NotNull PacketInfo packetInfo, int targetIndex)
{
if (!(packetInfo.getPacket() instanceof Vp8Packet))
{
logger.warn("Packet is not VP8 packet");
return false;
}
Vp8Packet vp8Packet = packetInfo.packetAs();
int incomingEncoding = vp8Packet.getEncodingId();
VP8FrameMap.FrameInsertionResult result = insertPacketInMap(vp8Packet);
if (result == null)
{
/* Very old frame, more than Vp8FrameMap.FRAME_MAP_SIZE old,
or something wrong with the stream. */
return false;
}
VP8Frame frame = result.getFrame();
if (result.isNewFrame())
{
if (vp8Packet.isKeyframe() && frameIsNewSsrc(frame))
{
/* If we're not currently projecting this SSRC, check if we've
already decided to drop a subsequent TL0 frame of this SSRC.
If we have, we can't turn on the encoding starting from this
packet, so treat this frame as though it weren't a keyframe.
*/
VP8Frame f = findNextTl0(frame);
if (f != null && !f.isAccepted())
{
frame.setKeyframe(false);
}
}
Instant receivedTime = packetInfo.getReceivedTime();
boolean accepted = vp8QualityFilter
.acceptFrame(frame, incomingEncoding, targetIndex, receivedTime);
if (accepted)
{
accepted = checkDecodability(frame);
}
frame.setAccepted(accepted);
if (accepted)
{
VP8FrameProjection projection;
try
{
projection = createProjection(frame, vp8Packet, result.isReset(),
receivedTime);
}
catch (Exception e)
{
logger.warn("Failed to create frame projection", e);
/* Make sure we don't have an accepted frame without a projection in the map. */
frame.setAccepted(false);
return false;
}
frame.setProjection(projection);
if (RtpUtils.isNewerSequenceNumberThan(projection.getEarliestProjectedSequence(),
lastVP8FrameProjection.getLatestProjectedSequence()))
{
lastVP8FrameProjection = projection;
}
}
}
return frame.isAccepted() && frame.getProjection().accept(vp8Packet);
}
/**
* For a frame that's been accepted by the quality filter, verify that
* it's decodable given the projection decisions about previous frames
* (in case the targetIndex has changed).
*/
private boolean checkDecodability(@NotNull VP8Frame frame)
{
if (frame.isKeyframe() || frame.getTemporalLayer() <= 0)
{
/* We'll always project all TL0 or unknown-TL frames, and TL0PICIDX lets the
* decoder know if it's missed something, so no need to check.
*/
return true;
}
VP8Frame f = frame, prev;
while ((prev = prevFrame(f)) != null)
{
if (!f.isImmediatelyAfter(prev))
{
/* If we have a gap in the projection history, we don't know
* what will be sent. Default to assuming it'll be decodable.
*/
return true;
}
if (prev.isKeyframe() || prev.getTemporalLayer() <= frame.getTemporalLayer())
{
/* Assume temporal nesting - if the previous frame of a lower
* or equal layer was accepted, this frame is decodable, otherwise
* it probably isn't.
*/
return prev.isAccepted();
}
f = prev;
}
/* If we ran off the beginning of our history, we don't know what was
* sent before; assume it'll be decodable. */
return true;
}
/**
* Create a projection for this frame.
*/
@NotNull
private VP8FrameProjection createProjection(
@NotNull VP8Frame frame,
@NotNull Vp8Packet initialPacket,
boolean isReset,
@Nullable Instant receivedTime)
{
if (frameIsNewSsrc(frame))
{
return createLayerSwitchProjection(frame, initialPacket, receivedTime);
}
else if (isReset)
{
return createResetProjection(frame, initialPacket, receivedTime);
}
return createInLayerProjection(frame, initialPacket, receivedTime);
}
/**
* Create a projection for this frame. It is the first frame sent for a layer.
*/
@NotNull
private VP8FrameProjection createLayerSwitchProjection(
@NotNull VP8Frame frame,
@NotNull Vp8Packet initialPacket,
@Nullable Instant receivedTime)
{
assert(frame.isKeyframe());
assert(initialPacket.isStartOfFrame());
int projectedSeqGap = 1;
if (lastVP8FrameProjection.getVP8Frame() != null &&
!lastVP8FrameProjection.getVP8Frame().hasSeenEndOfFrame())
{
/* Leave a gap to signal to the decoder that the previously routed
frame was incomplete. */
projectedSeqGap++;
/* Make sure subsequent packets of the previous projection won't
overlap the new one. (This means the gap, above, will never be
filled in.)
*/
lastVP8FrameProjection.close();
}
int projectedSeq =
RtpUtils.applySequenceNumberDelta(lastVP8FrameProjection.getLatestProjectedSequence(), projectedSeqGap);
// this is a simulcast switch. The typical incremental value =
// 90kHz / 30 = 90,000Hz / 30 = 3000 per frame or per 33ms
long tsDelta;
if (receivedTime != null && lastVP8FrameProjection.getCreated() != null)
{
tsDelta = 3000 * Math.max(1,
Duration.between(lastVP8FrameProjection.getCreated(), receivedTime).dividedBy(33).toMillis());
}
else
{
tsDelta = 3000;
}
long projectedTs = RtpUtils.applyTimestampDelta(lastVP8FrameProjection.getTimestamp(), tsDelta);
int picId;
int tl0PicIdx;
if (lastVP8FrameProjection.getVP8Frame() != null)
{
picId = VpxUtils.applyExtendedPictureIdDelta(lastVP8FrameProjection.getPictureId(),
1);
tl0PicIdx = VpxUtils.applyTl0PicIdxDelta(lastVP8FrameProjection.getTl0PICIDX(),
1);
}
else
{
picId = frame.getPictureId();
tl0PicIdx = frame.getTl0PICIDX();
}
VP8FrameProjection projection =
new VP8FrameProjection(diagnosticContext,
frame, lastVP8FrameProjection.getSSRC(), projectedTs,
RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.getSequenceNumber()),
picId, tl0PicIdx, receivedTime
);
return projection;
}
/**
* Create a projection for this frame. It follows a large gap in the stream's projected frames.
*/
@NotNull
private VP8FrameProjection createResetProjection(@NotNull VP8Frame frame,
@NotNull Vp8Packet initialPacket, @Nullable Instant receivedTime)
{
VP8Frame lastFrame = lastVP8FrameProjection.getVP8Frame();
/* Apply the latest projected frame's projections out, linearly. */
int seqDelta = RtpUtils.getSequenceNumberDelta(lastVP8FrameProjection.getLatestProjectedSequence(),
lastFrame.getLatestKnownSequenceNumber());
long tsDelta = RtpUtils.getTimestampDiff(lastVP8FrameProjection.getTimestamp(),
lastFrame.getTimestamp());
int picIdDelta = VpxUtils.getExtendedPictureIdDelta(lastVP8FrameProjection.getPictureId(),
lastFrame.getPictureId());
int tl0PicIdxDelta = VpxUtils.getTl0PicIdxDelta(lastVP8FrameProjection.getTl0PICIDX(),
lastFrame.getTl0PICIDX());
long projectedTs = RtpUtils.applyTimestampDelta(frame.getTimestamp(), tsDelta);
int projectedPicId = VpxUtils.applyExtendedPictureIdDelta(frame.getPictureId(), picIdDelta);
int projectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(frame.getTl0PICIDX(), tl0PicIdxDelta);
VP8FrameProjection projection =
new VP8FrameProjection(diagnosticContext,
frame, lastVP8FrameProjection.getSSRC(), projectedTs,
seqDelta,
projectedPicId, projectedTl0PicIdx, receivedTime
);
return projection;
}
/**
* Create a projection for this frame. It is being sent subsequent to other projected frames
* of this layer.
*/
@NotNull
private VP8FrameProjection createInLayerProjection(@NotNull VP8Frame frame,
@NotNull VP8Frame refFrame, @NotNull Vp8Packet initialPacket,
@Nullable Instant receivedTime)
{
long tsGap = RtpUtils.getTimestampDiff(frame.getTimestamp(), refFrame.getTimestamp());
int tl0Gap = VpxUtils.getTl0PicIdxDelta(frame.getTl0PICIDX(), refFrame.getTl0PICIDX());
int seqGap = 0;
int picGap = 0;
VP8Frame f1 = refFrame, f2;
int refSeq;
int picIdDelta = VpxUtils.getExtendedPictureIdDelta(refFrame.getPictureId(), frame.getPictureId());
if (picIdDelta < 0)
{
do
{
f2 = nextFrame(f1);
if (f2 == null)
{
throw new IllegalStateException("No next frame found after frame with picId " + f1.getPictureId() +
", even though refFrame " + refFrame.getPictureId() + " is before frame " +
frame.getPictureId() + "!");
}
seqGap += seqGap(f1, f2);
picGap += picGap(f1, f2);
f1 = f2;
}
while (f2 != frame);
refSeq = refFrame.getProjection().getLatestProjectedSequence();
}
else
{
do
{
f2 = prevFrame(f1);
if (f2 == null)
{
throw new IllegalStateException("No previous frame found before frame with picId " +
f1.getPictureId() + ", even though refFrame " + refFrame.getPictureId() + " is after frame " +
frame.getPictureId() + "!");
}
seqGap += -seqGap(f2, f1);
picGap += -picGap(f2, f1);
f1 = f2;
}
while (f2 != frame);
refSeq = refFrame.getProjection().getEarliestProjectedSequence();
}
int projectedSeq = RtpUtils.applySequenceNumberDelta(refSeq, seqGap);
long projectedTs = RtpUtils.applyTimestampDelta(refFrame.getProjection().getTimestamp(), tsGap);
int projectedPicId = VpxUtils.applyExtendedPictureIdDelta(refFrame.getProjection().getPictureId(), picGap);
int projectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(refFrame.getProjection().getTl0PICIDX(), tl0Gap);
VP8FrameProjection projection =
new VP8FrameProjection(diagnosticContext,
frame, lastVP8FrameProjection.getSSRC(), projectedTs,
RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.getSequenceNumber()),
projectedPicId, projectedTl0PicIdx, receivedTime
);
return projection;
}
/**
* Create a projection for this frame. It is being sent subsequent to other projected frames
* of this layer.
*/
@NotNull
private VP8FrameProjection createInLayerProjection(
@NotNull VP8Frame frame,
@NotNull Vp8Packet initialPacket,
@Nullable Instant receivedTime)
{
VP8Frame prevFrame = findPrevAcceptedFrame(frame);
if (prevFrame != null)
{
return createInLayerProjection(frame, prevFrame, initialPacket, receivedTime);
}
/* prev frame has rolled off beginning of frame map, try next frame */
VP8Frame nextFrame = findNextAcceptedFrame(frame);
if (nextFrame != null)
{
return createInLayerProjection(frame, nextFrame, initialPacket, receivedTime);
}
/* Neither previous or next is found. Very big frame? Use previous projected.
(This must be valid because we don't execute this function unless
frameIsNewSsrc has returned false.)
*/
return createInLayerProjection(frame, lastVP8FrameProjection.getVP8Frame(),
initialPacket, receivedTime);
}
@Override
public boolean needsKeyframe()
{
if (vp8QualityFilter.needsKeyframe())
{
return true;
}
if (lastVP8FrameProjection.getVP8Frame() == null)
{
/* Never sent anything */
return true;
}
return false;
}
/**
* Rewrites the RTCP packet that is specified as an argument.
*
* @param rtcpSrPacket the RTCP packet to transform.
* @return true if the RTCP packet is accepted, false otherwise, in which
* case it needs to be dropped.
*/
@Override
public boolean rewriteRtcp(@NotNull RtcpSrPacket rtcpSrPacket)
{
VP8FrameProjection lastVP8FrameProjectionCopy = lastVP8FrameProjection;
if (lastVP8FrameProjectionCopy.getVP8Frame() == null
|| rtcpSrPacket.getSenderSsrc() != lastVP8FrameProjectionCopy.getVP8Frame().getSsrc())
{
return false;
}
rtcpSrPacket.setSenderSsrc(lastVP8FrameProjectionCopy.getSSRC());
long srcTs = rtcpSrPacket.getSenderInfo().getRtpTimestamp();
long delta = RtpUtils.getTimestampDiff(
lastVP8FrameProjectionCopy.getTimestamp(),
lastVP8FrameProjectionCopy.getVP8Frame().getTimestamp());
long dstTs = RtpUtils.applyTimestampDelta(srcTs, delta);
if (srcTs != dstTs)
{
rtcpSrPacket.getSenderInfo().setRtpTimestamp(dstTs);
}
return true;
}
@Override
public RtpState getRtpState()
{
return new RtpState(
lastVP8FrameProjection.getSSRC(),
lastVP8FrameProjection.getLatestProjectedSequence(),
lastVP8FrameProjection.getTimestamp());
}
/**
* Rewrites the RTP packet that is specified as an argument.
*
* @param packetInfo the packet info for the RTP packet to rewrite.
* @throws RewriteException if a VP8 frame projection is not found
* for the RTP packet that is specified as a parameter.
*/
@Override
public void rewriteRtp(
@NotNull PacketInfo packetInfo)
throws RewriteException
{
if (!(packetInfo.getPacket() instanceof Vp8Packet))
{
logger.info("Got a non-VP8 packet.");
throw new RewriteException("Non-VP8 packet in VP8 source projection");
}
Vp8Packet vp8Packet = packetInfo.packetAs();
if (vp8Packet.getPictureId() == -1)
{
/* Should have been rejected in accept(). */
logger.info("VP8 packet does not have picture ID, cannot track in frame map.");
throw new RewriteException("VP8 packet without picture ID in VP8 source projection");
}
VP8Frame vp8Frame = lookupVP8Frame(vp8Packet);
if (vp8Frame == null)
{
// This packet does not belong to an accepted frame.
// Possibly it aged off the frame map since accept was called?
throw new RewriteException("Frame not in tracker (aged off?)");
}
if (vp8Frame.getProjection() == null)
{
/* Shouldn't happen for an accepted packet whose frame is still known? */
throw new RewriteException("Frame does not have projection?");
}
vp8Frame.getProjection().rewriteRtp(vp8Packet);
}
/**
* {@inheritDoc}
*/
@Override
public synchronized ObjectNode getDebugState()
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
debugState.put(
"class",
VP8AdaptiveSourceProjectionContext.class.getSimpleName());
ArrayNode mapSizes = JsonNodeFactory.instance.arrayNode();
for (Map.Entry<Long, VP8FrameMap> entry: vp8FrameMaps.entrySet())
{
ObjectNode sizeInfo = JsonNodeFactory.instance.objectNode();
sizeInfo.put("ssrc", entry.getKey());
sizeInfo.put("size", entry.getValue().size());
mapSizes.add(sizeInfo);
}
debugState.set("vp8FrameMaps", mapSizes);
debugState.set("vp8QualityFilter", vp8QualityFilter.getDebugState());
return debugState;
}
}

412
jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8Frame.java Normal file
View File

@@ -0,0 +1,412 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.vp8;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.codec.vpx.*;
import org.jitsi.nlj.rtp.*;
import org.jitsi.nlj.rtp.codec.vp8.*;
import org.jitsi.rtp.util.*;
/**
* Groups together some RTP/VP8 fields that refer to a specific incoming VP8
* frame. Most of those fields are final and cannot be changed, with the
* exception of the ending and max sequence number that may be unknown at the
* time of the creation of this instance.
*
* Instances of this class are *NOT* thread safe. While most internal state of
* this class instances is final, the sequence number ranges, haveStart/haveEnd,
* and isKeyframe are not.
*
* @author George Politis
* @author Jonathan Lennox
*/
class VP8Frame
{
/**
* The RTP SSRC of the incoming frame that this instance refers to
* (RFC3550).
*/
private final long ssrc;
/**
* The RTP timestamp of the incoming frame that this instance refers to
* (RFC3550).
*/
private final long timestamp;
/**
* The earliest RTP sequence number seen of the incoming frame that this instance
* refers to (RFC3550).
*/
private int earliestKnownSequenceNumber;
/**
* The latest RTP sequence number seen of the incoming frame that this instance
* refers to (RFC3550).
*/
private int latestKnownSequenceNumber;
/**
* A boolean that indicates whether or not we've seen the first packet of the frame.
* If so, its sequence is earliestKnownSequenceNumber.
*/
private boolean seenStartOfFrame;
/**
* A boolean that indicates whether or not we've seen the last packet of the frame.
* If so, its sequence is latestKnownSequenceNumber.
*/
private boolean seenEndOfFrame;
/**
* The temporal layer of this frame.
*/
private final int temporalLayer;
/**
* The VP8 PictureID of the incoming VP8 frame that this instance refers to
* (RFC7741).
*/
private final int pictureId;
/**
* The VP8 TL0PICIDX of the incoming VP8 frame that this instance refers to
* (RFC7741).
*/
private final int tl0PICIDX;
/**
* A boolean that indicates whether the incoming VP8 frame that this
* instance refers to is a keyframe (RFC7741).
*/
private boolean isKeyframe;
/**
* A record of how this frame was projected, or null if not.
*/
private VP8FrameProjection projection;
/**
* A boolean that records whether this frame was accepted.
*/
private boolean accepted;
/**
* Ctor.
*
* @param packet A packet from the frame to be constructed.
*/
VP8Frame(@NotNull Vp8Packet packet)
{
this.ssrc = packet.getSsrc();
this.timestamp = packet.getTimestamp();
this.earliestKnownSequenceNumber = packet.getSequenceNumber();
this.latestKnownSequenceNumber = packet.getSequenceNumber();
this.seenStartOfFrame = packet.isStartOfFrame();
this.seenEndOfFrame = packet.isEndOfFrame();
this.tl0PICIDX = packet.getTL0PICIDX();
this.isKeyframe = packet.isKeyframe();
this.pictureId = packet.getPictureId();
this.temporalLayer = packet.getTemporalLayerIndex();
}
/**
* Remember another packet of this frame.
* Note: this assumes every packet is received only once, i.e. a filter
* like {@link org.jitsi.nlj.transform.node.incoming.PaddingTermination} is in use.
* @param packet The packet to remember. This should be a packet which
* has tested true with {@link #matchesFrame(Vp8Packet)}.
*/
void addPacket(@NotNull Vp8Packet packet)
{
if (!matchesFrame(packet))
{
throw new IllegalArgumentException("Non-matching packet added to frame");
}
int seq = packet.getSequenceNumber();
if (RtpUtils.isOlderSequenceNumberThan(seq, earliestKnownSequenceNumber))
{
earliestKnownSequenceNumber = seq;
}
if (RtpUtils.isNewerSequenceNumberThan(seq, latestKnownSequenceNumber))
{
latestKnownSequenceNumber = seq;
}
if (packet.isStartOfFrame())
{
seenStartOfFrame = true;
}
if (packet.isEndOfFrame())
{
seenEndOfFrame = true;
}
}
/**
* @return true if the incoming VP8 frame that this instance refers to is a
* keyframe (RFC7741), false otherwise.
*/
boolean isKeyframe()
{
return isKeyframe;
}
/**
* Update a frame's notion of whether it is a keyframe.
*/
void setKeyframe(boolean k)
{
isKeyframe = k;
}
/**
* @return the temporal layer of this frame.
*/
int getTemporalLayer()
{
return temporalLayer;
}
/**
* @return true if this is a base temporal layer frame, false otherwise
* @note We treat unknown temporal layer frames as TL0.
*/
boolean isTL0()
{
return temporalLayer <= 0;
}
/**
* Gets the SSRC.
*/
long getSsrc()
{
return ssrc;
}
/**
* Gets the timestamp.
*/
long getTimestamp()
{
return timestamp;
}
/**
* Gets the earliest sequence number seen for this frame.
*/
public int getEarliestKnownSequenceNumber()
{
return earliestKnownSequenceNumber;
}
/**
* Gets the latest sequence number seen for this frame.
*/
public int getLatestKnownSequenceNumber()
{
return latestKnownSequenceNumber;
}
/**
* Whether the start of this frame has been seen.
*/
public boolean hasSeenStartOfFrame()
{
return seenStartOfFrame;
}
/**
* Whether the end of this frame has been seen.
*/
public boolean hasSeenEndOfFrame()
{
return seenEndOfFrame;
}
/**
* Get the projection of this frame, or null.
*/
public VP8FrameProjection getProjection()
{
return projection;
}
/**
* Get whether this frame has been accepted.
*/
public boolean isAccepted()
{
return accepted;
}
/**
* Set whether this frame has been accepted.
*/
public void setAccepted(boolean a)
{
accepted = a;
}
/**
* Set the projection record of this frame.
*/
public void setProjection(VP8FrameProjection projection)
{
this.projection = projection;
}
/**
* Get the picture ID of this frame.
*/
public int getPictureId()
{
return pictureId;
}
/**
* Get the tl0picidx of this frame.
*/
public int getTl0PICIDX()
{
return tl0PICIDX;
}
/**
* Small utility method that checks whether the {@link VP8Frame} that is
* specified as a parameter belongs to the same RTP stream as the frame that
* this instance refers to.
*
* @param vp8Frame the {@link VP8Frame} to check whether it belongs to the
* same RTP stream as the frame that this instance refers to.
* @return true if the {@link VP8Frame} that is specified as a parameter
* belongs to the same RTP stream as the frame that this instance refers to,
* false otherwise.
*/
boolean matchesSSRC(@NotNull VP8Frame vp8Frame)
{
return ssrc == vp8Frame.ssrc;
}
/**
* Determines whether the {@link VideoRtpPacket} that is specified as an
* argument is part of the VP8 picture that is represented by this
* {@link VP8Frame} instance.
*
* @param pkt the {@link VideoRtpPacket} instance to check whether it's part
* of the VP8 picture that is represented by this {@link VP8Frame}
* instance.
* @return true if the {@link VideoRtpPacket} that is specified as an
* argument is part of the VP8 picture that is represented by this
* {@link VP8Frame} instance, false otherwise.
*/
private boolean matchesSSRC(@NotNull VideoRtpPacket pkt)
{
return ssrc == pkt.getSsrc();
}
/**
* Checks whether the specified RTP packet is part of this frame.
*
* @param pkt the RTP packet to check whether it's part of this frame.
* @return true if the specified RTP packet is part of this frame, false
* otherwise.
*/
boolean matchesFrame(@NotNull Vp8Packet pkt)
{
return matchesSSRC(pkt) && timestamp == pkt.getTimestamp();
}
/**
* Validates that the specified RTP packet consistently matches all the
* parameters of this frame.
*
* This can be useful for diagnosing invalid streams if this fails when
* {@link #matchesFrame(Vp8Packet)} is true.
*
* @param pkt the RTP packet to check whether its parameters match this frame.
* @throws RuntimeException if the specified RTP packet is inconsistent with this frame
*/
void validateConsistency(@NotNull Vp8Packet pkt)
{
if (temporalLayer == pkt.getTemporalLayerIndex() &&
tl0PICIDX == pkt.getTL0PICIDX() &&
pictureId == pkt.getPictureId())
/* TODO: also check start, end, seq nums? */
{
return;
}
StringBuilder s = new StringBuilder().append("Packet")
.append(" ssrc ").append(pkt.getSsrc())
.append(", seq ").append(pkt.getSequenceNumber())
.append(", picture id ").append(pkt.getPictureId())
.append(", timestamp ").append(pkt.getTimestamp())
.append(" is not consistent with frame")
.append(" ssrc ").append(ssrc)
.append(", seq ").append(earliestKnownSequenceNumber).append("-").append(latestKnownSequenceNumber)
.append(", picture id ").append(pictureId)
.append(", timestamp ").append(timestamp)
.append(": ");
boolean complained = false;
if (temporalLayer != pkt.getTemporalLayerIndex())
{
s.append("packet temporal layer ")
.append(pkt.getTemporalLayerIndex())
.append(" != frame temporal layer ")
.append(temporalLayer);
complained = true;
}
if (tl0PICIDX != pkt.getTL0PICIDX())
{
if (complained)
{
s.append("; ");
}
s.append("packet TL0PICIDX ")
.append(pkt.getTL0PICIDX())
.append(" != frame TL0PICIDX ")
.append(tl0PICIDX);
complained = true;
}
if (pictureId != pkt.getPictureId())
{
if (complained)
{
s.append("; ");
}
s.append("packet PictureID ")
.append(pkt.getPictureId())
.append(" != frame PictureID ")
.append(pictureId);
}
throw new RuntimeException(s.toString());
}
/**
* Check whether this frame is immediately after another one, according
* to their extended picture IDs.
*/
boolean isImmediatelyAfter(@NotNull VP8Frame otherFrame)
{
return pictureId ==
VpxUtils.applyExtendedPictureIdDelta(otherFrame.getPictureId(), 1);
}
}

400
jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameMap.java Normal file
View File

@@ -0,0 +1,400 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.vp8;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.codec.vpx.*;
import org.jitsi.nlj.rtp.codec.vp8.*;
import org.jitsi.nlj.util.*;
import org.jitsi.rtp.util.*;
import org.jitsi.utils.logging2.*;
import java.time.*;
import java.util.function.*;
import static java.lang.Long.max;
import static java.lang.Long.min;
/**
* A history of recent frames on a VP8 stream.
*/
public class VP8FrameMap
{
static final int FRAME_MAP_SIZE = 500; /* Matches PacketCache default size. */
/** Cache mapping picture IDs to frames. */
private final FrameHistory frameHistory = new FrameHistory(FRAME_MAP_SIZE);
private final Logger logger;
/**
* Ctor.
*
*/
public VP8FrameMap(
@NotNull Logger parentLogger)
{
this.logger = parentLogger.createChildLogger(VP8FrameMap.class.getName());
}
/** Find a frame in the frame map, based on a packet. */
public synchronized VP8Frame findFrame(@NotNull Vp8Packet packet)
{
return frameHistory.get(packet.getPictureId());
}
/** Get the current size of the map. */
public int size()
{
return frameHistory.numCached;
}
/** Helper function to insert a packet into an existing frame. */
@NotNull
@Contract("_, _ -> new")
private FrameInsertionResult doFrameInsert(@NotNull VP8Frame frame, Vp8Packet packet)
{
try
{
frame.validateConsistency(packet);
}
catch (Exception e)
{
logger.warn(e);
}
frame.addPacket(packet);
return new FrameInsertionResult(frame, false, false);
}
/** Check whether this is a large jump from previous state, so the map should be reset. */
private boolean isLargeJump(@NotNull Vp8Packet packet)
{
VP8Frame latestFrame = frameHistory.getLatestFrame();
if (latestFrame == null)
{
return false;
}
int picDelta = VpxUtils.getExtendedPictureIdDelta(packet.getPictureId(), latestFrame.getPictureId());
if (picDelta > FRAME_MAP_SIZE)
{
return true;
}
long tsDelta = RtpUtils.getTimestampDiff(packet.getTimestamp(), latestFrame.getTimestamp());
if (picDelta < 0)
{
/* if picDelta is negative but timestamp or sequence delta is positive, we've cycled. */
if (tsDelta > 0)
{
return true;
}
if (RtpUtils.getSequenceNumberDelta(
packet.getSequenceNumber(), latestFrame.getLatestKnownSequenceNumber()) > 0)
{
return true;
}
}
/* If tsDelta is more than twice the frame map size at 1 fps, we've cycled. */
if (tsDelta > FRAME_MAP_SIZE * 90000 * 2)
{
return true;
}
return false;
}
/** Insert a packet into the frame map. Return a FrameInsertionResult
* describing what happened.
* @param packet The packet to insert.
* @return What happened. null if insertion failed.
*/
public synchronized FrameInsertionResult insertPacket(@NotNull Vp8Packet packet)
{
int pictureId = packet.getPictureId();
if (pictureId == -1)
{
/* Frame map indexes by picture ID. All supported browsers should currently be setting it. */
/* Log message will have been logged by Vp8Parser in jmt. */
return null;
}
if (isLargeJump(packet))
{
frameHistory.indexTracker.resetAt(pictureId);
VP8Frame frame = new VP8Frame(packet);
if (!frameHistory.insert(pictureId, frame))
{
return null;
}
return new FrameInsertionResult(frame, true, true);
}
VP8Frame frame = frameHistory.get(pictureId);
if (frame != null)
{
if (!frame.matchesFrame(packet))
{
if (frame.getPictureId() != pictureId)
{
throw new IllegalStateException("Frame map returned frame with picture ID " +
frame.getPictureId() +
" when asked for frame with picture ID " + pictureId);
}
logger.warn("Cannot insert packet in frame map: " +
"frame with ssrc " + frame.getSsrc() +
", timestamp " + frame.getTimestamp() +
", and sequence number range " + frame.getEarliestKnownSequenceNumber() +
"-" + frame.getLatestKnownSequenceNumber() +
", and packet " + packet.getSequenceNumber() +
" with ssrc " + packet.getSsrc() +
", timestamp " + packet.getTimestamp() +
", and sequence number " + packet.getSequenceNumber() +
" both have picture ID " + pictureId);
return null;
}
return doFrameInsert(frame, packet);
}
frame = new VP8Frame(packet);
if (!frameHistory.insert(pictureId, frame))
{
return null;
}
return new FrameInsertionResult(frame, true, false);
}
@Nullable
public synchronized VP8Frame nextFrame(@NotNull VP8Frame frame)
{
return frameHistory.findAfter(frame, (VP8Frame f) -> true );
}
@Nullable
public synchronized VP8Frame nextFrameWith(@NotNull VP8Frame frame, Predicate<VP8Frame> pred)
{
return frameHistory.findAfter(frame, pred);
}
@Nullable
public synchronized VP8Frame findNextTl0(@NotNull VP8Frame frame)
{
return nextFrameWith(frame, VP8Frame::isTL0);
}
@Nullable
public synchronized VP8Frame findNextAcceptedFrame(@NotNull VP8Frame frame)
{
return nextFrameWith(frame, VP8Frame::isAccepted);
}
@Nullable
public synchronized VP8Frame prevFrame(@NotNull VP8Frame frame)
{
return frameHistory.findBefore(frame, (VP8Frame f) -> true );
}
@Nullable
public synchronized VP8Frame prevFrameWith(@NotNull VP8Frame frame, Predicate<VP8Frame> pred)
{
return frameHistory.findBefore(frame, pred);
}
@Nullable
public synchronized VP8Frame findPrevAcceptedFrame(@NotNull VP8Frame frame)
{
return prevFrameWith(frame, VP8Frame::isAccepted);
}
/**
* The result of calling {@link #insertPacket(Vp8Packet).}
*/
public static class FrameInsertionResult
{
/** The frame corresponding to the packet that was inserted. */
private VP8Frame frame;
/** Whether inserting the frame created a new frame. */
private boolean newFrame;
/** Whether inserting the frame caused a reset */
private boolean reset;
/** Construct a FrameInsertionResult. */
private FrameInsertionResult(VP8Frame frame, boolean newFrame, boolean reset)
{
this.frame = frame;
this.newFrame = newFrame;
this.reset = reset;
}
/** Get the frame corresponding to the packet that was inserted. */
public VP8Frame getFrame()
{
return frame;
}
/** Get whether inserting the frame created a new frame. */
public boolean isNewFrame()
{
return newFrame;
}
/** Get whether inserting the frame caused a reset. */
public boolean isReset()
{
return reset;
}
}
private static class FrameHistory extends ArrayCache<VP8Frame>
{
FrameHistory(int size)
{
super(size, (k) -> k, false, Clock.systemUTC());
}
int numCached = 0;
long firstIndex = -1;
PictureIdIndexTracker indexTracker = new PictureIdIndexTracker();
/**
* Gets a frame with a given VP8 picture ID from the cache.
*/
public VP8Frame get(int pictureId)
{
long index = indexTracker.interpret(pictureId);
return getIndex(index);
}
/**
* Gets a frame with a given VP8 picture ID index from the cache.
*/
private VP8Frame getIndex(long index)
{
if (index <= getLastIndex() - getSize())
{
/* We don't want to remember old frames even if they're still
tracked; their neighboring frames may have been evicted,
so findBefore / findAfter will return bogus data. */
return null;
}
ArrayCache<VP8Frame>.Container c = getContainer(index);
if (c == null)
{
return null;
}
return c.getItem();
}
/** Get the latest frame in the tracker. */
private VP8Frame getLatestFrame()
{
return getIndex(getLastIndex());
}
public boolean insert(int pictureId, VP8Frame frame)
{
long index = indexTracker.update(pictureId);
boolean ret = super.insertItem(frame, index);
if (ret)
{
numCached++;
if (firstIndex == -1 || index < firstIndex)
{
firstIndex = index;
}
}
return ret;
}
/**
* Called when an item in the cache is replaced/discarded.
*/
@Override
protected void discardItem(VP8Frame frame)
{
numCached--;
}
@Nullable
public VP8Frame findBefore(VP8Frame frame, Predicate<VP8Frame> pred)
{
long lastIndex = getLastIndex();
if (lastIndex == -1)
{
return null;
}
long index = indexTracker.interpret(frame.getPictureId());
long searchStartIndex = min(index - 1, lastIndex);
long searchEndIndex = max(lastIndex - getSize(), firstIndex - 1);
return doFind(pred, searchStartIndex, searchEndIndex, -1);
}
@Nullable
public VP8Frame findAfter(VP8Frame frame, Predicate<VP8Frame> pred)
{
long lastIndex = getLastIndex();
if (lastIndex == -1)
{
return null;
}
long index = indexTracker.interpret(frame.getPictureId());
if (index >= lastIndex)
{
return null;
}
long searchStartIndex = max(index + 1, max(lastIndex - getSize() + 1, firstIndex));
return doFind(pred, searchStartIndex, lastIndex + 1, 1);
}
@Nullable
private VP8Frame doFind(Predicate<VP8Frame> pred, long startIndex, long endIndex, int increment)
{
if (!((increment > 0 && startIndex <= endIndex) || (increment < 0 && startIndex >= endIndex)))
{
throw new IllegalArgumentException("Values of startIndex=" + startIndex + ", endIndex=" + endIndex +
" , and increment=" + increment + " could lead to infinite loop");
}
for (long index = startIndex; index != endIndex; index += increment)
{
VP8Frame frame = getIndex(index);
if (frame != null && pred.test(frame))
{
return frame;
}
}
return null;
}
}
}

297
jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameProjection.java Normal file
View File

@@ -0,0 +1,297 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.vp8;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.rtp.codec.vp8.*;
import org.jitsi.rtp.util.*;
import org.jitsi.utils.logging.*;
import java.time.*;
/**
* Represents a VP8 frame projection. It puts together all the necessary bits
* and pieces that are useful when projecting an accepted VP8 frame. A
* projection is responsible for rewriting a VP8 packet. Instances of this class
* are thread-safe.
*
* @author George Politis
*/
public class VP8FrameProjection
{
/**
* The time series logger for this instance.
*/
private static final TimeSeriesLogger timeSeriesLogger
= TimeSeriesLogger.getTimeSeriesLogger(VP8FrameProjection.class);
/**
* The diagnostic context for this instance.
*/
private final DiagnosticContext diagnosticContext;
/**
* A timestamp of when this instance was created. It's used to calculate
* RTP timestamps when we switch encodings.
*/
private final @Nullable Instant created;
/**
* The projected {@link VP8Frame}.
*/
private final VP8Frame vp8Frame;
/**
* The RTP SSRC of the projection (RFC7667, RFC3550).
*/
private final long ssrc;
/**
* The RTP timestamp of the projection (RFC7667, RFC3550).
*/
private final long timestamp;
/**
* The sequence number delta for packets of this frame.
*/
private final int sequenceNumberDelta;
/**
* The VP8 picture ID of the projection (RFC7667, RFC7741).
*/
private final int extendedPictureId;
/**
* The VP8 TL0PICIDX of the projection (RFC7741).
*/
private final int tl0PICIDX;
/**
* -1 if this projection is still "open" for new, later packets.
* Projections can be closed when we switch away from their encodings.
*/
private int closedSeq = -1;
/**
* Ctor.
*
* @param ssrc the SSRC of the destination VP8 picture.
* @param timestamp The RTP timestamp of the projected frame that this
* instance refers to (RFC3550).
* @param sequenceNumberDelta The starting RTP sequence number of the
* projected frame that this instance refers to (RFC3550).
*/
VP8FrameProjection(
@NotNull DiagnosticContext diagnosticContext,
long ssrc, int sequenceNumberDelta, long timestamp)
{
this(diagnosticContext, null /* vp8Frame */, ssrc, timestamp,
sequenceNumberDelta, 0 /* extendedPictureId */,
0 /* tl0PICIDX */, null /* created */);
}
/**
* Ctor.
*
* @param vp8Frame The {@link VP8Frame} that's projected.
* @param ssrc The RTP SSRC of the projected frame that this instance refers
* to (RFC3550).
* @param timestamp The RTP timestamp of the projected frame that this
* instance refers to (RFC3550).
* @param sequenceNumberDelta The starting RTP sequence number of the
* projected frame that this instance refers to (RFC3550).
* @param extendedPictureId The VP8 extended picture ID of the projected VP8
* frame that this instance refers to (RFC7741).
* @param tl0PICIDX The VP8 TL0PICIDX of the projected VP8 frame that this
* instance refers to (RFC7741).
*/
VP8FrameProjection(
@NotNull DiagnosticContext diagnosticContext,
VP8Frame vp8Frame,
long ssrc, long timestamp, int sequenceNumberDelta,
int extendedPictureId, int tl0PICIDX, @Nullable Instant created)
{
this.diagnosticContext = diagnosticContext;
this.ssrc = ssrc;
this.timestamp = timestamp;
this.sequenceNumberDelta = sequenceNumberDelta;
this.extendedPictureId = extendedPictureId;
this.tl0PICIDX = tl0PICIDX;
this.vp8Frame = vp8Frame;
this.created = created;
}
public int rewriteSeqNo(int seq)
{
return RtpUtils.applySequenceNumberDelta(seq, sequenceNumberDelta);
}
/**
* Rewrites an RTP packet.
*
* @param pkt the RTP packet to rewrite.
*/
void rewriteRtp(@NotNull Vp8Packet pkt)
{
int sequenceNumber = rewriteSeqNo(pkt.getSequenceNumber());
if (timeSeriesLogger.isTraceEnabled())
{
timeSeriesLogger.trace(diagnosticContext
.makeTimeSeriesPoint("rtp_vp8_rewrite")
.addField("orig.rtp.ssrc", pkt.getSsrc())
.addField("orig.rtp.timestamp", pkt.getTimestamp())
.addField("orig.rtp.seq", pkt.getSequenceNumber())
.addField("orig.vp8.pictureid", pkt.getPictureId())
.addField("orig.vp8.tl0picidx", pkt.getTL0PICIDX())
.addField("proj.rtp.ssrc", ssrc)
.addField("proj.rtp.timestamp", timestamp)
.addField("proj.rtp.seq", sequenceNumber)
.addField("proj.vp8.pictureid", extendedPictureId)
.addField("proj.vp8.tl0picidx", tl0PICIDX));
}
// update ssrc, sequence number, timestamp, pictureId and tl0picidx
pkt.setSsrc(ssrc);
pkt.setTimestamp(timestamp);
pkt.setSequenceNumber(sequenceNumber);
if (pkt.getTL0PICIDX() != -1)
{
pkt.setTL0PICIDX(tl0PICIDX);
}
pkt.setPictureId(extendedPictureId);
}
/**
* Determines whether a packet can be forwarded as part of this
* {@link VP8FrameProjection} instance. The check is based on the sequence
* of the incoming packet and whether or not the {@link VP8FrameProjection}
* has been "closed" or not.
*
* @param rtpPacket the {@link Vp8Packet} that will be examined.
* @return true if the packet can be forwarded as part of this
* {@link VP8FrameProjection}, false otherwise.
*/
public boolean accept(@NotNull Vp8Packet rtpPacket)
{
if (vp8Frame == null || !vp8Frame.matchesFrame(rtpPacket))
{
// The packet does not belong to this VP8 picture.
return false;
}
synchronized (vp8Frame)
{
if (closedSeq < 0)
{
return true;
}
return RtpUtils
.isOlderSequenceNumberThan(rtpPacket.getSequenceNumber(),
closedSeq);
}
}
/**
* @return The projected {@link VP8Frame}.
*/
VP8Frame getVP8Frame()
{
return vp8Frame;
}
/**
* @return The RTP SSRC of this projection.
*/
public long getSSRC()
{
return ssrc;
}
/**
* @return The RTP timestamp of this projection.
*/
public long getTimestamp()
{
return timestamp;
}
/**
* @return The picture ID of this projection.
*/
public int getPictureId()
{
return extendedPictureId;
}
/**
* @return The TL0PICIDX of this projection.
*/
public int getTl0PICIDX()
{
return tl0PICIDX;
}
/**
* @return The system time (in ms) this projection was created.
*/
public @Nullable Instant getCreated()
{
return created;
}
public int getEarliestProjectedSequence()
{
if (vp8Frame == null)
{
return sequenceNumberDelta;
}
synchronized (vp8Frame)
{
return rewriteSeqNo(vp8Frame.getEarliestKnownSequenceNumber());
}
}
public int getLatestProjectedSequence()
{
if (vp8Frame == null)
{
return sequenceNumberDelta;
}
synchronized (vp8Frame)
{
return rewriteSeqNo(vp8Frame.getLatestKnownSequenceNumber());
}
}
/**
* Prevents the max sequence number of this frame to grow any further.
*/
public void close()
{
if (vp8Frame != null)
{
synchronized (vp8Frame)
{
closedSeq = vp8Frame.getLatestKnownSequenceNumber();
}
}
}
}

387
jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java Normal file
View File

@@ -0,0 +1,387 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.vp8;
import edu.umd.cs.findbugs.annotations.*;
import org.jetbrains.annotations.*;
import org.jetbrains.annotations.Nullable;
import org.jitsi.nlj.*;
import org.jitsi.utils.logging2.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.lang.SuppressWarnings;
import java.time.*;
/**
* This class is responsible for dropping VP8 simulcast/svc packets based on
* their quality, i.e. packets that correspond to qualities that are above a
* given quality target. Instances of this class are thread-safe.
*
* @author George Politis
*/
class VP8QualityFilter
{
/**
* The {@link Logger} to be used by this instance to print debug
* information.
*/
private final Logger logger;
/**
* The default maximum frequency at which the media engine
* generates key frame.
*/
private static final Duration MIN_KEY_FRAME_WAIT = Duration.ofMillis(300);
/**
* The HD, SD, LD and suspended spatial/quality layer IDs.
*/
private static final int SUSPENDED_ENCODING_ID = -1;
/**
* Holds the arrival time of the most recent keyframe group.
* Reading/writing of this field is synchronized on this instance.
*/
private @Nullable Instant mostRecentKeyframeGroupArrivalTime = null;
/**
* A boolean flag that indicates whether a simulcast switch is pending. This
* condition is equivalent to:
*
* internalSpatialLayerIdTarget != externalSpatialLayerIdTarget.
*
* Reading/writing of this field is synchronized on this instance.
*/
private boolean needsKeyframe = false;
/**
* The encoding id that this instance tries to achieve. Upon
* receipt of a packet, we check whether externalSpatialLayerIdTarget
* (that's specified as an argument to the
* {@link #acceptFrame(VP8Frame, int, int, Instant)} method) is set to something
* different, in which case we set {@link #needsKeyframe} equal to true and
* update.
*/
private int internalEncodingIdTarget = SUSPENDED_ENCODING_ID;
/**
* The encoding layer ID that we're currently forwarding. -1
* indicates that we're not forwarding anything. Reading/writing of this
* field is synchronized on this instance.
*/
private int currentEncodingId = SUSPENDED_ENCODING_ID;
public VP8QualityFilter(Logger parentLogger)
{
this.logger = parentLogger.createChildLogger(VP8QualityFilter.class.getName());
}
/**
* @return true if a the target encoding id has changed and a
* keyframe hasn't been received yet, false otherwise.
*/
boolean needsKeyframe()
{
return needsKeyframe;
}
/**
* Determines whether to accept or drop a VP8 frame.
*
* Note that, at the time of this writing, there's no practical need for a
* synchronized keyword because there's only one thread accessing this
* method at a time.
*
* @param frame the VP8 frame.
* @param incomingEncoding the encoding index of the incoming RTP packet
* @param externalTargetIndex the target quality index that the user of this
* instance wants to achieve.
* @param receivedTime the current time
* @return true to accept the VP8 frame, otherwise false.
*/
synchronized boolean acceptFrame(
@NotNull VP8Frame frame,
int incomingEncoding,
int externalTargetIndex, Instant receivedTime)
{
// We make local copies of the externalTemporalLayerIdTarget and the
// externalEncodingTarget (as they may be updated by some other
// thread).
int externalTemporalLayerIdTarget
= RtpLayerDesc.getTidFromIndex(externalTargetIndex);
int externalEncodingIdTarget
= RtpLayerDesc.getEidFromIndex(externalTargetIndex);
if (externalEncodingIdTarget != internalEncodingIdTarget)
{
// The externalEncodingIdTarget has changed since accept last
// run; perhaps we should request a keyframe.
internalEncodingIdTarget = externalEncodingIdTarget;
if (externalEncodingIdTarget > SUSPENDED_ENCODING_ID)
{
needsKeyframe = true;
}
}
if (externalEncodingIdTarget < 0
|| externalTemporalLayerIdTarget < 0)
{
// We stop forwarding immediately. We will need a keyframe in order
// to resume.
currentEncodingId = SUSPENDED_ENCODING_ID;
return false;
}
int temporalLayerIdOfFrame = frame.getTemporalLayer();
if (temporalLayerIdOfFrame < 0)
{
// temporal scalability is not enabled. Pretend that
// this is the base temporal layer.
temporalLayerIdOfFrame = 0;
}
if (frame.isKeyframe())
{
logger.debug(() -> "Quality filter got keyframe for stream "
+ frame.getSsrc());
return acceptKeyframe(incomingEncoding, receivedTime);
}
else if (currentEncodingId > SUSPENDED_ENCODING_ID)
{
if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding))
{
// XXX(george) i've noticed some "rogue" base layer keyframes
// that trigger this. what happens is the client sends a base
// layer key frame, the bridge switches to that layer because
// for all it knows it may be the only keyframe sent by the
// client engine. then the bridge notices that packets from the
// higher quality streams are flowing and execution ends-up
// here. it is a mystery why the engine is "leaking" base layer
// key frames
needsKeyframe = true;
}
if (incomingEncoding != currentEncodingId)
{
// for non-keyframes, we can't route anything but the current encoding
return false;
}
// This branch reads the {@link #currentEncodingId} and it
// filters packets based on their temporal layer.
if (currentEncodingId > externalEncodingIdTarget)
{
// pending downscale, decrease the frame rate until we
// downscale.
return temporalLayerIdOfFrame < 1;
}
else if (currentEncodingId < externalEncodingIdTarget)
{
// pending upscale, increase the frame rate until we upscale.
return true;
}
else
{
// The currentSpatialLayerId matches exactly the target
// currentSpatialLayerId.
return temporalLayerIdOfFrame <= externalTemporalLayerIdTarget;
}
}
else
{
// In this branch we're not processing a keyframe and the
// currentEncodingId is in suspended state, which means we need
// a keyframe to start streaming again.
// We should have already requested a keyframe, either above or when the
// internal target encoding was first moved off SUSPENDED_ENCODING.
return false;
}
}
/**
* Returns a boolean that indicates whether we are in layer switching phase
* or not.
*
* @param receivedTime the time the latest frame was received
* @return false if we're in layer switching phase, true otherwise.
*/
private synchronized boolean isOutOfSwitchingPhase(@Nullable Instant receivedTime)
{
if (receivedTime == null)
{
return false;
}
if (mostRecentKeyframeGroupArrivalTime == null)
{
return true;
}
Duration delta = Duration.between(mostRecentKeyframeGroupArrivalTime, receivedTime);
return delta.compareTo(MIN_KEY_FRAME_WAIT) > 0;
}
/**
* @return true if it looks like we can re-scale (see implementation of
* method for specific details).
*/
private synchronized boolean isPossibleToSwitch(int encodingId)
{
if (encodingId == -1)
{
// We failed to resolve the spatial/quality layer of the packet.
return false;
}
if (encodingId > currentEncodingId
&& currentEncodingId < internalEncodingIdTarget)
{
// It looks like upscaling is possible.
return true;
}
else if (encodingId < currentEncodingId
&& currentEncodingId > internalEncodingIdTarget)
{
// It looks like downscaling is possible.
return true;
}
else
{
return false;
}
}
/**
* Determines whether to accept or drop a VP8 keyframe. This method updates
* the spatial layer id.
*
* Note that, at the time of this writing, there's no practical need for a
* synchronized keyword because there's only one thread accessing this
* method at a time.
*
* @param receivedTime the time the frame was received
* @return true to accept the VP8 keyframe, otherwise false.
*/
private synchronized boolean acceptKeyframe(
int encodingIdOfKeyframe, @Nullable Instant receivedTime)
{
// This branch writes the {@link #currentSpatialLayerId} and it
// determines whether or not we should switch to another simulcast
// stream.
if (encodingIdOfKeyframe < 0)
{
// something went terribly wrong, normally we should be able to
// extract the layer id from a keyframe.
logger.error("unable to get layer id from keyframe");
return false;
}
logger.debug(() -> "Received a keyframe of encoding: "
+ encodingIdOfKeyframe);
// The keyframe request has been fulfilled at this point, regardless of
// whether we'll be able to achieve the internalEncodingIdTarget.
needsKeyframe = false;
if (isOutOfSwitchingPhase(receivedTime))
{
// During the switching phase we always project the first
// keyframe because it may very well be the only one that we
// receive (i.e. the endpoint is sending low quality only). Then
// we try to approach the target.
mostRecentKeyframeGroupArrivalTime = receivedTime;
logger.debug(() -> "First keyframe in this kf group " +
"currentEncodingId: " + encodingIdOfKeyframe +
". Target is " + internalEncodingIdTarget);
if (encodingIdOfKeyframe <= internalEncodingIdTarget)
{
// If the target is 180p and the first keyframe of a group of
// keyframes is a 720p keyframe we don't project it. If we
// receive a 720p keyframe, we know that there MUST be a 180p
// keyframe shortly after.
currentEncodingId = encodingIdOfKeyframe;
return true;
}
else
{
return false;
}
}
else
{
// We're within the 300ms window since the reception of the
// first key frame of a key frame group, let's check whether an
// upscale/downscale is possible.
if (currentEncodingId <= encodingIdOfKeyframe
&& encodingIdOfKeyframe <= internalEncodingIdTarget)
{
// upscale or current quality case
currentEncodingId = encodingIdOfKeyframe;
logger.debug(() -> "Upscaling to encoding "
+ encodingIdOfKeyframe
+ ". The target is " + internalEncodingIdTarget);
return true;
}
else if (encodingIdOfKeyframe <= internalEncodingIdTarget
&& internalEncodingIdTarget < currentEncodingId)
{
// downscale case
currentEncodingId = encodingIdOfKeyframe;
logger.debug(() -> " Downscaling to encoding "
+ encodingIdOfKeyframe + ". The target is + "
+ internalEncodingIdTarget);
return true;
}
else
{
return false;
}
}
}
/**
* Gets a JSON representation of the parts of this object's state that
* are deemed useful for debugging.
*/
@SuppressWarnings("unchecked")
@SuppressFBWarnings(
value = "IS2_INCONSISTENT_SYNC",
justification = "We intentionally avoid synchronizing while reading" +
" fields only used in debug output.")
public ObjectNode getDebugState()
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
debugState.put(
"mostRecentKeyframeGroupArrivalTimeMs",
mostRecentKeyframeGroupArrivalTime != null ? mostRecentKeyframeGroupArrivalTime.toEpochMilli() : -1L);
debugState.put("needsKeyframe", needsKeyframe);
debugState.put(
"internalEncodingIdTarget",
internalEncodingIdTarget);
debugState.put("currentEncodingId", currentEncodingId);
return debugState;
}
}

155
jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannel.java Normal file
View File

@@ -0,0 +1,155 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel;
import org.jitsi.rtp.extensions.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.datachannel.protocol.*;
import java.nio.*;
/**
* Models a WebRTC Data Channel
*
* @author Brian Baldino
*/
public class DataChannel
{
private final DataChannelStack.DataChannelDataSender dataChannelDataSender;
protected final int channelType;
protected final int priority;
protected final long reliability;
protected final int sid;
protected final String label;
protected boolean ready = false;
protected final Logger logger;
private DataChannelStack.DataChannelEventListener eventListener;
private DataChannelStack.DataChannelMessageListener messageListener;
/**
* Initializes a new {@link DataChannel} instance.
*/
public DataChannel(
DataChannelStack.DataChannelDataSender dataChannelDataSender,
Logger parentLogger,
int channelType, int priority, long reliability, int sid, String label
)
{
this.dataChannelDataSender = dataChannelDataSender;
this.logger = parentLogger.createChildLogger(this.getClass().getName());
this.channelType = channelType;
this.priority = priority;
this.reliability = reliability;
this.sid = sid;
this.label = label;
}
public void open()
{
OpenChannelMessage openMessage = new OpenChannelMessage(
channelType,
priority,
reliability,
label,
DataChannelProtocolConstants.PROTOCOL_STRING);
ByteBuffer msg = openMessage.getBuffer();
if (dataChannelDataSender.send(msg, sid, DataChannelProtocolConstants.WEBRTC_DCEP_PPID) < 0)
{
logger.error("Error sending data channel open message");
}
}
/**
* Checks if this data channel is ready.
*/
public boolean isReady()
{
return ready;
}
/**
* Sets the listener for data channel events.
* @param listener
*/
public void onDataChannelEvents(
DataChannelStack.DataChannelEventListener listener)
{
this.eventListener = listener;
}
/**
* Sets the message listener.
*/
public void onDataChannelMessage(
DataChannelStack.DataChannelMessageListener
dataChannelMessageListener)
{
this.messageListener = dataChannelMessageListener;
}
/**
* Handles an incoming message.
* @param message
*/
public void onIncomingMsg(DataChannelMessage message)
{
if (message instanceof OpenChannelAckMessage)
{
ready = true;
eventListener.onDataChannelOpened();
}
else if (message instanceof DataChannelStringMessage)
{
DataChannelStringMessage dataChannelStringMessage = (DataChannelStringMessage)message;
logger.debug(() -> "Received data channel string message: " + dataChannelStringMessage.data);
}
else if (message instanceof DataChannelBinaryMessage)
{
DataChannelBinaryMessage dataChannelBinaryMessage = (DataChannelBinaryMessage)message;
logger.debug(() -> "Received data channel binary message: " +
ByteBufferKt.toHex(ByteBuffer.wrap(dataChannelBinaryMessage.data)));
}
messageListener.onDataChannelMessage(message);
}
/**
* Sends data through the sender.
*/
protected int sendData(ByteBuffer data, int sid, int ppid)
{
return dataChannelDataSender.send(data, sid, ppid);
}
/**
* Sends a string through this data channel.
* @param message the string to send.
*/
public void sendString(String message)
{
if (logger.isDebugEnabled())
{
logger.debug("Sending string data channel message: '" + message + "'");
}
DataChannelStringMessage stringMessage = new DataChannelStringMessage(message);
dataChannelDataSender.send(stringMessage.getBuffer(), sid, DataChannelProtocolConstants.WEBRTC_PPID_STRING);
}
}

181
jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannelStack.java Normal file
View File

@@ -0,0 +1,181 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.datachannel.protocol.*;
import org.jitsi.videobridge.metrics.*;
import org.jitsi.videobridge.sctp.*;
import java.nio.*;
import java.util.*;
/**
* We need the stack to look at all incoming messages so that it can listen for an 'open channel'
* message from the remote side
*
* Handles DataChannel negotiation and the routing of all Data Channel messages
* to specific {@link DataChannel} instances.
*
* @author Brian Baldino
*/
//TODO: revisit thread safety in here
//TODO: add an arbitrary ID
public class DataChannelStack
{
private final Map<Integer, DataChannel> dataChannels = new HashMap<>();
private final DataChannelDataSender dataChannelDataSender;
private final Logger logger;
private DataChannelStackEventListener listener;
/**
* Initializes a new {@link DataChannelStack} with a specific sender.
* @param dataChannelDataSender the sender.
*/
public DataChannelStack(DataChannelDataSender dataChannelDataSender, Logger parentLogger)
{
this.dataChannelDataSender = dataChannelDataSender;
logger = parentLogger.createChildLogger(DataChannelStack.class.getName());
}
/**
* Handles a received packet.
*/
public void onIncomingDataChannelPacket(ByteBuffer data, int sid, int ppid)
{
logger.debug(() -> "Data channel stack received SCTP message");
DataChannelMessage message = DataChannelProtocolMessageParser.parse(data.array(), ppid);
if (message instanceof OpenChannelMessage)
{
logger.debug("Received data channel open message");
if (dataChannels.size() >= SctpConfig.config.getMaxChannels())
{
logger.warn("Data channel limit exceeded.");
VideobridgeMetrics.rejectedDataChannels.inc();
return;
}
OpenChannelMessage openChannelMessage = (OpenChannelMessage)message;
// Remote side wants to open a channel
DataChannel dataChannel = new RemotelyOpenedDataChannel(
dataChannelDataSender,
logger,
openChannelMessage.channelType,
openChannelMessage.priority,
openChannelMessage.reliability,
sid,
openChannelMessage.label);
dataChannels.put(sid, dataChannel);
listener.onDataChannelOpenedRemotely(dataChannel);
}
else
{
DataChannel dataChannel= dataChannels.get(sid);
if (dataChannel == null)
{
logger.error("Could not find data channel for sid " + sid);
return;
}
dataChannel.onIncomingMsg(message);
}
}
public void onDataChannelStackEvents(DataChannelStackEventListener listener)
{
this.listener = listener;
}
/**
* Opens new WebRTC data channel using specified parameters.
* @param channelType channel type as defined in control protocol description.
* Use 0 for "reliable".
* @param priority channel priority. The higher the number, the lower
* the priority.
* @param reliability Reliability Parameter<br/>
*
* This field is ignored if a reliable channel is used.
* If a partial reliable channel with limited number of
* retransmissions is used, this field specifies the number of
* retransmissions. If a partial reliable channel with limited
* lifetime is used, this field specifies the maximum lifetime in
* milliseconds. The following table summarizes this:<br/></br>
+------------------------------------------------+------------------+
| Channel Type | Reliability |
| | Parameter |
+------------------------------------------------+------------------+
| DATA_CHANNEL_RELIABLE | Ignored |
| DATA_CHANNEL_RELIABLE_UNORDERED | Ignored |
| DATA_CHANNEL_PARTIAL_RELIABLE_REXMIT | Number of RTX |
| DATA_CHANNEL_PARTIAL_RELIABLE_REXMIT_UNORDERED | Number of RTX |
| DATA_CHANNEL_PARTIAL_RELIABLE_TIMED | Lifetime in ms |
| DATA_CHANNEL_PARTIAL_RELIABLE_TIMED_UNORDERED | Lifetime in ms |
+------------------------------------------------+------------------+
* @param sid SCTP stream id that will be used by new channel
* (it must not be already used).
* @param label text label for the channel.
* @return new instance of <tt>WebRtcDataStream</tt> that represents opened
* WebRTC data channel.
*/
public DataChannel createDataChannel(int channelType, int priority, long reliability, int sid, String label)
{
synchronized (dataChannels)
{
DataChannel dataChannel = new DataChannel(
dataChannelDataSender, logger, channelType, priority, reliability, sid, label);
dataChannels.put(sid, dataChannel);
return dataChannel;
}
}
/**
* TODO: these 2 feel a bit awkward since they are so similar, but we use
* a different one for a remote channel (fired by the stack) and a
* locally-created channel (fired by the data channel itself).
*/
public interface DataChannelStackEventListener
{
/**
* The data channel was opened by the remote side.
*/
void onDataChannelOpenedRemotely(DataChannel dataChannel);
}
public interface DataChannelEventListener
{
/**
* The data channel was opened.
*/
void onDataChannelOpened();
}
public interface DataChannelMessageListener
{
/**
* A message received.
*/
void onDataChannelMessage(DataChannelMessage dataChannelMessage);
}
public interface DataChannelDataSender
{
/**
* Sends a message.
*/
int send(ByteBuffer data, int sid, int ppid);
}
}

56
jvb/src/main/java/org/jitsi/videobridge/datachannel/RemotelyOpenedDataChannel.java Normal file
View File

@@ -0,0 +1,56 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.datachannel.protocol.*;
import java.nio.*;
/**
* Same as {@link DataChannel} but automatically sends the open channel ack message upon creation
*
* @author Brian Baldino
*/
public class RemotelyOpenedDataChannel extends DataChannel
{
/**
* Initializes a new {@link RemotelyOpenedDataChannel} instance.
*/
public RemotelyOpenedDataChannel(
DataChannelStack.DataChannelDataSender dataChannelDataSender,
Logger parentLogger,
int channelType, int priority, long reliability, int sid,
String label)
{
super(dataChannelDataSender, parentLogger, channelType, priority,
reliability, sid, label);
ready = true;
sendOpenChannelAck();
}
/**
* Sends an {@link OpenChannelAckMessage} message.
*/
protected void sendOpenChannelAck()
{
OpenChannelAckMessage openChannelAckMessage = new OpenChannelAckMessage();
ByteBuffer msg = openChannelAckMessage.getBuffer();
sendData(msg, sid, DataChannelProtocolConstants.WEBRTC_DCEP_PPID);
}
}

65
jvb/src/main/java/org/jitsi/videobridge/datachannel/Transport.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel;
import java.nio.*;
/**
* TODO: This needs documentation
*
* @author Brian Baldino
*/
public interface Transport
{
interface DataChannelDataCallback
{
/**
* A packet was received.
*/
void dataChannelPacketReceived(ByteBuffer data, int sid, int ppid);
}
interface DataChannelTransportEventHandler
{
/**
* The transport connected.
*/
void transportConnected();
/**
* The transport disconnected.
*/
void transportDisconnected();
}
/**
* Sets the handler for data.
* @param dataChannelDataCallback
*/
void onData(DataChannelDataCallback dataChannelDataCallback);
/**
* Sends a {@link ByteBuffer}.
*/
int send(ByteBuffer data, boolean ordered, int sid, int ppid);
/**
* Sets the handler for events.
*/
void onEvent(
DataChannelTransportEventHandler dataChannelTransportEventHandler);
}

45
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelBinaryMessage.java Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import java.nio.*;
/**
* @author Brian Baldino
*/
public class DataChannelBinaryMessage extends DataChannelMessage
{
public final byte[] data;
/**
* Initializes a new {@link DataChannelBinaryMessage} from a byte array.
* @param data the byte array.
*/
public DataChannelBinaryMessage(byte[] data)
{
this.data = data;
}
/**
* {@inheritDoc}
*/
@Override
public ByteBuffer getBuffer()
{
return ByteBuffer.wrap(data);
}
}

30
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelMessage.java Normal file
View File

@@ -0,0 +1,30 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import java.nio.*;
/**
* @author Brian Baldino
*/
public abstract class DataChannelMessage
{
/**
* Gets the buffer which holds the message.
*/
public abstract ByteBuffer getBuffer();
}

59
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import org.jetbrains.annotations.*;
import org.jitsi.dcsctp4j.DcSctpMessage;
import org.jitsi.rtp.*;
/**
* @author Brian Baldino
*/
public class DataChannelPacket extends Packet
{
public final int sid;
public final int ppid;
/**
* Initializes a new {@link DataChannelPacket}.
*/
public DataChannelPacket(
byte[] data, int offset, int length, int sid, int ppid)
{
super(data, offset, length);
this.sid = sid;
this.ppid = ppid;
}
public DataChannelPacket(DcSctpMessage message)
{
super(message.getPayload(), 0, message.getPayload().length);
this.sid = message.getStreamID();
this.ppid = message.getPpid();
}
/**
* {@inheritDoc}
*/
@NotNull
@Override
public Packet clone()
{
return new DataChannelPacket(
getBuffer().clone(), getOffset(), getLength(), sid, ppid);
}
}

61
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolConstants.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
public class DataChannelProtocolConstants
{
/**
* The <tt>String</tt> value of the <tt>Protocol</tt> field of the
* <tt>DATA_CHANNEL_OPEN</tt> message.
*/
public static final String PROTOCOL_STRING = "http://jitsi.org/protocols/colibri";
/**
* DataChannel message types
//TODO: enum?
*/
public static final int MSG_TYPE_CHANNEL_OPEN = 0x3;
public static final int MSG_TYPE_CHANNEL_ACK = 0x2;
/**
* Channel types
* TODO: enum?
*/
public static final int RELIABLE = 0x00;
public static final int RELIABLE_UNORDERED = 0x80;
public static final int PARTIAL_RELIABLE_REXMIT = 0x01;
public static final int PARTIAL_RELIABLE_REXMIT_UNORDERED = 0x81;
public static final int PARTIAL_RELIABLE_TIMED = 0x02;
public static final int PARTIAL_RELIABLE_TIMED_UNORDERED = 0x82;
/**
* This is the SCTP PPID used for datachannel establishment protocol
* https://tools.ietf.org/html/draft-ietf-rtcweb-data-protocol-08#section-8.1
*/
public static final int WEBRTC_DCEP_PPID = 50;
/**
* Payload protocol id that identifies text data UTF8 encoded in WebRTC data
* channels.
*/
public static final int WEBRTC_PPID_STRING = 51;
/**
* Payload protocol id that identifies binary data in WebRTC data channel.
*/
public static final int WEBRTC_PPID_BIN = 53;
}

68
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessage.java Normal file
View File

@@ -0,0 +1,68 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import java.nio.*;
/**
* https://tools.ietf.org/html/draft-ietf-rtcweb-data-protocol-08#section-5
* Base class for Data Channel 'protocol' messages; that is, messages
* used in Data Channel setup.
*
* @author Brian Baldino
*/
public class DataChannelProtocolMessage extends DataChannelMessage
{
/**
* The only common field is the message type, so we'll call that the 'header'
*/
private static final int HEADER_SIZE_BYTES = 1;
static final int MAX_LABEL_LENGTH = 0xFFFF;
static final int MAX_PROTOCOL_LENGTH = 0xFFFF;
private final int messageType;
/**
* Initializes a new {@link DataChannelProtocolMessage} instance.
* @param messageType the message type.
*/
public DataChannelProtocolMessage(int messageType)
{
this.messageType = messageType;
}
protected int getSizeBytes()
{
return HEADER_SIZE_BYTES;
}
/**
* {@inheritDoc}
*/
@Override
public ByteBuffer getBuffer()
{
ByteBuffer buf = ByteBuffer.allocate(getSizeBytes());
serialize(buf);
return buf;
}
public void serialize(ByteBuffer destination)
{
destination.put((byte)messageType);
}
}

78
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessageParser.java Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import org.jitsi.utils.logging2.*;
import java.nio.*;
/**
* Parses Data Channel protocol messages
*
* @author Brian Baldino
*/
public class DataChannelProtocolMessageParser
{
private static Logger logger = new LoggerImpl(DataChannelProtocolMessageParser.class.getName());
/**
* Reads the message type from a byte array.
* @param data
* @return
*/
private static int getMessageType(byte[] data)
{
return ByteBuffer.wrap(data).get(0) & 0xFF;
}
/**
* TODO(brian): change data to be a ByteBuffer
*/
public static DataChannelMessage parse(byte[] data, long ppid)
{
if (ppid == DataChannelProtocolConstants.WEBRTC_DCEP_PPID)
{
int messageType = getMessageType(data);
switch (messageType)
{
case DataChannelProtocolConstants.MSG_TYPE_CHANNEL_ACK: {
return new OpenChannelAckMessage();
}
case DataChannelProtocolConstants.MSG_TYPE_CHANNEL_OPEN: {
return OpenChannelMessage.parse(data);
}
default:
{
logger.warn("Unrecognized datachannel control message type: " + messageType);
return null;
}
}
}
else if (ppid == DataChannelProtocolConstants.WEBRTC_PPID_STRING)
{
return DataChannelStringMessage.parse(data);
}
else if (ppid == DataChannelProtocolConstants.WEBRTC_PPID_BIN)
{
return new DataChannelBinaryMessage(data);
}
else
{
logger.warn("Unrecognized data channel ppid: " + ppid);
}
return null;
}
}

57
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelStringMessage.java Normal file
View File

@@ -0,0 +1,57 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import java.nio.*;
import java.nio.charset.*;
/**
* @author Brian Baldino
*/
public class DataChannelStringMessage extends DataChannelMessage
{
public final String data;
/**
* Initializes a new {@link DataChannelStringMessage} instance.
* @param data
*/
public DataChannelStringMessage(String data)
{
this.data = data;
}
/**
* Parses a {@link DataChannelStringMessage} from a byte array.
* @param data
* @return
*/
public static DataChannelStringMessage parse(byte[] data)
{
String stringData = new String(data, StandardCharsets.UTF_8);
return new DataChannelStringMessage(stringData);
}
/**
* {@inheritDoc}
*/
@Override
public ByteBuffer getBuffer()
{
return ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8));
}
}

31
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelAckMessage.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
/**
* @author Brian Baldino
*/
public class OpenChannelAckMessage extends DataChannelProtocolMessage
{
/**
* Initializes a new {@link OpenChannelAckMessage} instance.
*/
public OpenChannelAckMessage()
{
super(DataChannelProtocolConstants.MSG_TYPE_CHANNEL_ACK);
}
}

115
jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelMessage.java Normal file
View File

@@ -0,0 +1,115 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.datachannel.protocol;
import java.nio.*;
import java.nio.charset.*;
/**
* TODO: this needs documentation
* @author Brian Baldino
*/
public class OpenChannelMessage extends DataChannelProtocolMessage
{
public final int channelType;
public final int priority;
public final long reliability;
public final String label;
public final String protocol;
/**
* Initializes a new {@link OpenChannelMessage} instance.
*/
public OpenChannelMessage(
int channelType, int priority,
long reliability, String label, String protocol)
{
super(DataChannelProtocolConstants.MSG_TYPE_CHANNEL_OPEN);
this.channelType = channelType;
this.priority = priority;
this.reliability = reliability;
this.label = label;
this.protocol = protocol;
}
public static OpenChannelMessage parse(byte[] data)
{
ByteBuffer buf = ByteBuffer.wrap(data);
int channelType = buf.get(1);
int priority = buf.getShort(2);
long reliability = buf.getInt(4);
int labelLength = buf.getShort(8);
int protocolLength = buf.getShort(10);
String label = null;
if (labelLength > 0)
{
// The cast is needed when the code is compiled with JDK9+ and run on JRE8.
// https://stackoverflow.com/questions/48693695/java-nio-buffer-not-loading-clear-method-on-runtime
((Buffer) buf).position(12);
byte[] labelBytes = new byte[labelLength];
buf.get(labelBytes, 0, labelLength);
label = new String(labelBytes);
}
String protocol = null;
if (protocolLength > 0)
{
// The cast is needed when the code is compiled with JDK9+ and run on JRE8.
// https://stackoverflow.com/questions/48693695/java-nio-buffer-not-loading-clear-method-on-runtime
((Buffer) buf).position(12 + labelLength);
byte[] protocolBytes = new byte[protocolLength];
buf.get(protocolBytes, 0, protocolLength);
protocol = new String(protocolBytes);
}
return new OpenChannelMessage(
channelType, priority, reliability, label, protocol);
}
/**
* {@inheritDoc}
*/
@Override
protected int getSizeBytes()
{
byte[] labelBytes = label.getBytes(StandardCharsets.UTF_8);
int labelLength = Math.min(labelBytes.length, MAX_LABEL_LENGTH);
byte[] protocolBytes = protocol.getBytes(StandardCharsets.UTF_8);
int protocolLength = Math.min(protocolBytes.length, MAX_PROTOCOL_LENGTH);
return super.getSizeBytes() + 11 + labelLength + protocolLength;
}
/**
* {@inheritDoc}
*/
@Override
public void serialize(ByteBuffer destination)
{
super.serialize(destination);
destination.put((byte)channelType);
destination.putShort((short)priority);
destination.putInt((int)reliability);
byte[] labelBytes = label.getBytes(StandardCharsets.UTF_8);
int labelLength = Math.min(labelBytes.length, MAX_LABEL_LENGTH);
byte[] protocolBytes = protocol.getBytes(StandardCharsets.UTF_8);
int protocolLength = Math.min(protocolBytes.length, MAX_PROTOCOL_LENGTH);
destination.putShort((short)labelLength);
destination.putShort((short)protocolLength);
destination.put(labelBytes, 0, labelLength);
destination.put(protocolBytes, 0, protocolLength);
}
}

46
jvb/src/main/java/org/jitsi/videobridge/ice/TransportUtils.java Normal file
View File

@@ -0,0 +1,46 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.ice;
import org.ice4j.ice.*;
public class TransportUtils
{
/**
* Determines whether at least one <tt>LocalCandidate</tt> of a specific ICE
* <tt>Component</tt> can reach (in the terms of the ice4j library) a
* specific <tt>RemoteCandidate</tt>
*
* @param component the ICE <tt>Component</tt> which contains the
* <tt>LocalCandidate</tt>s to check whether at least one of them can reach
* the specified <tt>remoteCandidate</tt>
* @param remoteCandidate the <tt>RemoteCandidate</tt> to check whether at
* least one of the <tt>LocalCandidate</tt>s of the specified
* <tt>component</tt> can reach it
* @return <tt>true</tt> if at least one <tt>LocalCandidate</tt> of the
* specified <tt>component</tt> can reach the specified
* <tt>remoteCandidate</tt>
*/
public static boolean canReach(
Component component,
RemoteCandidate remoteCandidate)
{
return component.getLocalCandidates().stream().
anyMatch(
localCandidate -> localCandidate.canReach(remoteCandidate));
}
}

39
jvb/src/main/java/org/jitsi/videobridge/relay/RelayedPacketInfo.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright @ 2019-Present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.relay;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
import org.jitsi.rtp.*;
/**
* A packet received from a {@link Relay}.
*/
public class RelayedPacketInfo extends PacketInfo
{
@Nullable private final String meshId;
public RelayedPacketInfo(@NotNull Packet packet, @Nullable String meshId)
{
super(packet);
this.meshId = meshId;
}
public String getMeshId()
{
return meshId;
}
}

34
jvb/src/main/java/org/jitsi/videobridge/rest/annotations/EnabledByConfig.java Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.annotations;
import org.jitsi.videobridge.rest.*;
import java.lang.annotation.*;
/**
* An annotation which allows a REST resource to be enabled/disabled by a property value in {@link RestConfig}.
*
* See {@link org.jitsi.videobridge.rest.filters.ConfigFilter}
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
@Inherited
public @interface EnabledByConfig
{
RestApis value();
}

35
jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/BadRequestExceptionWithMessage.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.exceptions;
import jakarta.servlet.http.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
/**
* The ctor for {@link BadRequestException} which takes in a String doesn't
* actually include that String in the response. A much longer syntax (seen
* in the constructor below) is necessary. This class exists in order to expose
* that behavior in a more concise way
*/
public class BadRequestExceptionWithMessage extends BadRequestException
{
public BadRequestExceptionWithMessage(String message)
{
super(Response.status(HttpServletResponse.SC_BAD_REQUEST, message).build());
}
}

35
jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/InternalServerErrorExceptionWithMessage.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.exceptions;
import jakarta.servlet.http.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
/**
* The ctor for {@link InternalServerErrorException} which takes in a String doesn't
* actually include that String in the response. A much longer syntax (seen
* in the constructor below) is necessary. This class exists in order to expose
* that behavior in a more concise way
*/
public class InternalServerErrorExceptionWithMessage extends InternalServerErrorException
{
public InternalServerErrorExceptionWithMessage(String message)
{
super(Response.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, message).build());
}
}

49
jvb/src/main/java/org/jitsi/videobridge/rest/filters/ConfigFilter.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.filters;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.container.*;
import jakarta.ws.rs.core.*;
/**
* A filter which returns 404 not found for any path which:
* 1) Is marked with the {@link EnabledByConfig} annotation and
* 2) The resulting config value in {@link RestConfig} is `false`.
*/
public class ConfigFilter implements ContainerRequestFilter
{
@Context
protected ResourceInfo resourceInfo;
@Override
public void filter(ContainerRequestContext containerRequestContext)
{
Class<?> clazz = resourceInfo.getResourceClass();
if (clazz.isAnnotationPresent(EnabledByConfig.class))
{
EnabledByConfig anno = clazz.getAnnotation(EnabledByConfig.class);
if (!RestConfig.config.isEnabled(anno.value()))
{
throw new NotFoundException();
}
}
}
}

69
jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java Normal file
View File

@@ -0,0 +1,69 @@
/*
* Copyright @ 2022 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.prometheus;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
import kotlin.*;
import org.jetbrains.annotations.NotNull;
import org.jitsi.metrics.*;
import java.util.*;
import java.util.stream.*;
/**
* A REST endpoint exposing JVB stats for Prometheus.
* Any scraper supporting Prometheus' text-based formats ({@code text/plain; version=0.0.4} or OpenMetrics)
* is compatible with this {@code /metrics} endpoint.<br>
* JSON is provided when the client performs a request with the 'Accept' header set to {@code application/json}.<br>
* The response defaults to {@code text/plain; version=0.0.4} formatted output.
*
* @see <a href="https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md">
* Prometheus' exposition formats</a>
*/
@Path("/metrics")
public class Prometheus
{
static private final Comparator<MediaType> comparator = Comparator.comparing(Prometheus::getQValue).reversed();
private static double getQValue(MediaType m)
{
return m.getParameters().get("q") == null ? 1.0 : Double.parseDouble(m.getParameters().get("q"));
}
@NotNull
private final MetricsContainer metricsContainer;
public Prometheus(@NotNull MetricsContainer metricsContainer)
{
this.metricsContainer = metricsContainer;
}
@GET
public Response x(@HeaderParam("Accept") String accept)
{
List<String> acceptMediaTypes
= Arrays.stream(accept.split(","))
.map(MediaType::valueOf)
.sorted(comparator)
.map(m -> m.getType() + "/" + m.getSubtype())
.collect(Collectors.toList());
Pair<String, String> m = metricsContainer.getMetrics(acceptMediaTypes);
return Response.ok(m.getFirst(), m.getSecond()).build();
}
}

67
jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java Normal file
View File

@@ -0,0 +1,67 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root;
import org.glassfish.jersey.server.*;
import org.jetbrains.annotations.*;
import org.jitsi.utils.version.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.health.*;
import org.jitsi.videobridge.metrics.*;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.binders.*;
import org.jitsi.videobridge.rest.filters.*;
import org.jitsi.videobridge.rest.prometheus.*;
import org.jitsi.videobridge.xmpp.*;
import static org.jitsi.videobridge.rest.RestConfig.config;
public class Application extends ResourceConfig
{
public Application(
Videobridge videobridge,
XmppConnection xmppConnection,
@NotNull Version version,
@NotNull JvbHealthChecker healthChecker)
{
register(
new ServiceBinder(
videobridge,
xmppConnection,
healthChecker
)
);
// Filters
register(ConfigFilter.class);
// Register all resources in the package
packages("org.jitsi.videobridge.rest.root");
if (config.isEnabled(RestApis.HEALTH))
{
register(new org.jitsi.rest.Health(healthChecker));
}
if (config.isEnabled(RestApis.VERSION))
{
register(new org.jitsi.rest.Version(version));
}
if (config.isEnabled(RestApis.PROMETHEUS))
{
register(new Prometheus(VideobridgeMetricsContainer.getInstance()));
}
}
}

57
jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/debug/Debug.java Normal file
View File

@@ -0,0 +1,57 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.colibri.debug;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
import jakarta.ws.rs.core.Response;
import java.net.*;
/**
* We moved /colibri/debug to /debug.
*/
@Path("/colibri/debug")
@EnabledByConfig(RestApis.COLIBRI)
public class Debug
{
@POST
@Path("{path:.+}")
public Response post(@PathParam("path") String path, @Context UriInfo uriInfo)
{
return create301(path, uriInfo);
}
@GET
@Path("{path:.+}")
public Response get(@PathParam("path") String path, @Context UriInfo uriInfo)
{
return create301(path, uriInfo);
}
private Response create301(String path, UriInfo uriInfo)
{
String target = uriInfo.getBaseUri() + "debug/" + path;
if (uriInfo.getQueryParameters().get("full") != null)
{
target += "?full=" + uriInfo.getQueryParameters().get("full");
}
return Response.status(301).location(URI.create(target)).build();
}
}

75
jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/drain/Drain.java Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.colibri.drain;
import org.eclipse.jetty.http.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.inject.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
/**
* A resource for putting the videobridge in drain via REST.
*/
@Path("/colibri/drain")
@EnabledByConfig(RestApis.DRAIN)
public class Drain
{
@Inject
protected Videobridge videobridge;
private Response setDrainMode(boolean enable)
{
try
{
videobridge.setDrainMode(enable);
return Response.ok().build();
}
catch (Throwable t)
{
return Response.status(HttpStatus.INTERNAL_SERVER_ERROR_500).build();
}
}
@POST
@Path("/enable")
public Response drainEnable()
{
return setDrainMode(true);
}
@POST
@Path("/disable")
public Response drainDisable()
{
return setDrainMode(false);
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public String getDrainState()
{
ObjectNode obj = JsonNodeFactory.instance.objectNode();
obj.put("drain", videobridge.getDrainMode());
return obj.toString();
}
}

100
jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClient.java Normal file
View File

@@ -0,0 +1,100 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.colibri.mucclient;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import org.jitsi.videobridge.xmpp.*;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import jakarta.inject.*;
import jakarta.servlet.http.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
/**
* Add or remove XMPP environments to which the bridge will connect
*/
@Path("/colibri/muc-client")
@EnabledByConfig(RestApis.COLIBRI)
public class MucClient
{
@Inject
protected XmppConnection xmppConnection;
@Path("/add")
@POST
@Consumes(MediaType.APPLICATION_JSON)
public Response addMucClient(String requestBody)
{
//NOTE: unfortunately MucClientConfiguration is not a compliant bean (it doesn't have
// a no-arg ctor) so we can't parse the json directly into a MucClientConfiguration
// instance and just take that as an argument here, we have to read the json
// ourselves.
try
{
JsonNode node = new ObjectMapper().readTree(requestBody);
if (!(node instanceof ObjectNode))
{
return Response.status(HttpServletResponse.SC_BAD_REQUEST).build();
}
if (xmppConnection.addMucClient((ObjectNode)node))
{
return Response.ok().build();
}
return Response.status(HttpServletResponse.SC_BAD_REQUEST).build();
}
catch (Exception e)
{
return Response.status(HttpServletResponse.SC_BAD_REQUEST).build();
}
}
@Path("/remove")
@POST
@Consumes(MediaType.APPLICATION_JSON)
public Response removeMucClient(String requestBody)
{
try
{
JsonNode node = new ObjectMapper().readTree(requestBody);
if (!(node instanceof ObjectNode))
{
return Response.status(HttpServletResponse.SC_BAD_REQUEST).build();
}
if (xmppConnection.removeMucClient((ObjectNode)node))
{
return Response.ok().build();
}
return Response.status(HttpServletResponse.SC_BAD_REQUEST).build();
}
catch (Exception e)
{
return Response.status(HttpServletResponse.SC_BAD_REQUEST).build();
}
}
@Path("/list")
@GET
@Produces(MediaType.APPLICATION_JSON)
public String listMucClientIDs()
{
return xmppConnection.getMucClientIds();
}
}

79
jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/shutdown/Shutdown.java Normal file
View File

@@ -0,0 +1,79 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.colibri.shutdown;
import com.fasterxml.jackson.annotation.*;
import org.eclipse.jetty.http.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import jakarta.inject.*;
import jakarta.servlet.http.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
/**
* A resource for shutting down the videobridge via REST.
*/
@Path("/colibri/shutdown")
@EnabledByConfig(RestApis.SHUTDOWN)
public class Shutdown
{
@Inject
protected Videobridge videobridge;
@POST
@Consumes(MediaType.APPLICATION_JSON)
public Response shutdown(ShutdownJson shutdown, @Context HttpServletRequest request)
{
try
{
videobridge.shutdown(shutdown.isGraceful);
return Response.ok().build();
}
catch (Throwable t)
{
return Response.status(HttpStatus.INTERNAL_SERVER_ERROR_500).build();
}
}
/**
* A class binding for the shutdown JSON passed to the shutdown request
* Currently, it only has a single field:
* {
* graceful-shutdown: Boolean [required]
* }
*/
public static class ShutdownJson
{
// Unfortunately, using @JsonProperty here alone is not enough to throw an error
// if the property is missing from the JSON, but we can't leave it out entirely
// as it's needed for serialization (since the constructor param JsonProperty
// is only for the parameter, not the member, so it isn't used when
// serializing).
@JsonProperty(value = "graceful-shutdown", required = true)
private Boolean isGraceful;
@JsonCreator
public ShutdownJson(@JsonProperty(value = "graceful-shutdown", required = true) Boolean isGraceful)
{
this.isGraceful = isGraceful;
}
}
}

36
jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java Normal file
View File

@@ -0,0 +1,36 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.colibri.stats;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import org.jitsi.videobridge.stats.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
@Path("/colibri/stats")
@EnabledByConfig(RestApis.COLIBRI)
public class Stats
{
@GET
@Produces(MediaType.APPLICATION_JSON)
public String getStats()
{
return VideobridgeStatisticsShim.INSTANCE.getStatsJson().toString();
}
}

184
jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/v2/conferences/Conferences.java Normal file
View File

@@ -0,0 +1,184 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.colibri.v2.conferences;
import org.jitsi.nlj.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import org.jitsi.videobridge.rest.exceptions.*;
import org.jitsi.xmpp.extensions.colibri2.*;
import org.jitsi.xmpp.extensions.colibri2.json.*;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.jivesoftware.smack.packet.*;
import jakarta.inject.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
import java.util.*;
@Path("/colibri/v2/conferences")
@EnabledByConfig(RestApis.COLIBRI)
public class Conferences
{
private static final ObjectMapper jsonMapper = new ObjectMapper();
@Inject
private Videobridge videobridge;
@GET
@Produces(MediaType.APPLICATION_JSON)
public String getConferences()
{
/* Not clear what this should do. If you have a use case please contact us with requirements. */
throw new ServerErrorException(Response.Status.NOT_IMPLEMENTED);
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{meetingId}")
public String getConference(@PathParam("meetingId") String meetingId)
{
Conference conference = videobridge.getConferenceByMeetingId(meetingId);
if (conference == null)
{
throw new NotFoundException();
}
/* Not clear what this should do. If you have a use case please contact us with requirements. */
throw new ServerErrorException(Response.Status.NOT_IMPLEMENTED);
}
@GET
@Produces(MediaType.APPLICATION_JSON)
@Path("/{meetingId}/dominant-speaker-identification")
public String getDominantSpeakerIdentification(@PathParam("meetingId") String meetingId)
{
Conference conference = videobridge.getConferenceByMeetingId(meetingId);
if (conference == null)
{
throw new NotFoundException();
}
ConferenceSpeechActivity conferenceSpeechActivity = conference.getSpeechActivity();
return conferenceSpeechActivity.getDebugState(DebugStateMode.FULL).toString();
}
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public String createConference(String requestBody)
{
JsonNode requestJson;
try
{
requestJson = jsonMapper.readTree(requestBody);
if (!(requestJson instanceof ObjectNode))
{
throw new BadRequestException();
}
}
catch (JsonProcessingException pe)
{
throw new BadRequestExceptionWithMessage(
"Failed to create conference, could not parse JSON: " + pe.getMessage());
}
ConferenceModifyIQ conferenceModifyIQ;
try
{
conferenceModifyIQ
= Colibri2JSONDeserializer.deserializeConferenceModify((ObjectNode) requestJson).build();
}
catch (Exception e)
{
throw new BadRequestExceptionWithMessage("Failed to parse ConferenceModify JSON:" + e.getMessage());
}
return getVideobridgeIqResponseAsJson(conferenceModifyIQ);
}
@PATCH
@Path("/{meetingId}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public String patchConference(@PathParam("meetingId") String meetingId, String requestBody)
{
Conference conference = videobridge.getConferenceByMeetingId(meetingId);
if (conference == null)
{
throw new NotFoundException();
}
JsonNode requestJson;
try
{
requestJson = jsonMapper.readTree(requestBody);
if (!(requestJson instanceof ObjectNode))
{
throw new BadRequestException();
}
}
catch (JsonProcessingException e)
{
throw new BadRequestException();
}
ConferenceModifyIQ.Builder conferenceModifyIQBuilder;
try
{
conferenceModifyIQBuilder
= Colibri2JSONDeserializer.deserializeConferenceModify((ObjectNode) requestJson);
}
catch (Exception e)
{
throw new BadRequestExceptionWithMessage("Failed to parse ConferenceModify JSON:" + e.getMessage());
}
conferenceModifyIQBuilder.setMeetingId(meetingId);
ConferenceModifyIQ conferenceModifyIQ = conferenceModifyIQBuilder.build();
return getVideobridgeIqResponseAsJson(conferenceModifyIQ);
}
private String getVideobridgeIqResponseAsJson(ConferenceModifyIQ request)
{
IQ responseIq = videobridge.handleConferenceModifyIq(request);
if (responseIq.getError() != null)
{
throw new BadRequestExceptionWithMessage(
"Failed to create conference: " + responseIq.getError().getDescriptiveText());
}
if (!(responseIq instanceof ConferenceModifiedIQ))
{
throw new InternalServerErrorExceptionWithMessage("Non-error, non-colibri IQ result");
}
ObjectNode responseJson = Colibri2JSONSerializer.serializeConferenceModified((ConferenceModifiedIQ)responseIq);
return responseJson.toString();
}
}

431
jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java Normal file
View File

@@ -0,0 +1,431 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.debug;
import org.jitsi.health.*;
import org.jitsi.nlj.*;
import org.jitsi.nlj.transform.node.*;
import org.jitsi.nlj.transform.node.debug.*;
import org.jitsi.utils.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.utils.logging2.Logger;
import org.jitsi.utils.queue.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.metrics.*;
import org.jitsi.videobridge.relay.*;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
import org.jitsi.videobridge.stats.*;
import org.jitsi.videobridge.transport.ice.*;
import org.jitsi.videobridge.util.*;
import org.jitsi.videobridge.xmpp.*;
import jakarta.inject.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.*;
import jakarta.ws.rs.core.MediaType;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.net.*;
/**
* A REST interface for retrieving debug information about the bridge.
*
* Note that using this interface MAY disrupt running conferences or even
* cause a deadlock. It is really meant only for debugging, which is why it is
* disabled by default. Use at your own risk.
*
* @author bbaldino
*/
@Path("/debug")
@EnabledByConfig(RestApis.DEBUG)
public class Debug
{
@Inject
@SuppressWarnings("unused")
private Videobridge videobridge;
@Inject
private HealthCheckService healthCheckService;
private final Logger logger = new LoggerImpl(Debug.class.getName());
// Functions to enable or disable features
/**
* Set the state of a given JVB feature
* @param feature the feature to enable or disable
* @param enabled whether the feature should be enabled
* @return HTTP response
*/
@POST
@Path("/features/jvb/{feature}/{enabled}")
public Response setJvbFeatureState(
@PathParam("feature") DebugFeatures feature,
@PathParam("enabled") Boolean enabled)
{
logger.info((enabled ? "Enabling" : "Disabling") + " feature " + feature.getValue());
setFeature(feature, enabled);
return Response.ok().build();
}
@GET
@Path("/features/jvb/{feature}/{enabled}")
public Response setJvbFeatureState2(
@PathParam("feature") DebugFeatures feature,
@PathParam("enabled") Boolean enabled)
{
logger.info((enabled ? "Enabling" : "Disabling") + " feature " + feature.getValue());
setFeature(feature, enabled);
return Response.ok().build();
}
/**
* Find out whether the given JVB feature is currently enabled or disabled
* @param feature the feature to check
* @return true if the feature is enabled, false otherwise
*/
@GET
@Path("/features/jvb/{feature}")
public Boolean getJvbFeatureState(@PathParam("feature") DebugFeatures feature)
{
switch (feature)
{
case PAYLOAD_VERIFICATION: {
return Node.Companion.isPayloadVerificationEnabled();
}
case NODE_STATS: {
return StatsKeepingNode.Companion.getEnableStatistics();
}
case POOL_STATS: {
return ByteBufferPool.statisticsEnabled();
}
case POOL_BOOKKEEPING: {
return ByteBufferPool.bookkeepingEnabled();
}
case QUEUE_STATS: {
return QueueStatistics.DEBUG;
}
case QUEUE_TIMING_STATS: {
return QueueStatistics.TRACK_TIMES;
}
case NODE_TRACING: {
return Node.Companion.isNodeTracingEnabled();
}
case TRANSIT_STATS: {
// Always enabled (worth modeling as a 'feature' then?)
return true;
}
case TASK_POOL_STATS: {
// Always enabled (worth modeling as a 'feature' then?)
return true;
}
case TOSSED_PACKET_STATS: {
return true;
}
default: {
throw new NotFoundException();
}
}
}
@POST
@Path("/features/endpoint/{confId}/{epId}/{feature}/{enabled}")
public Response setEndpointFeatureState(
@PathParam("confId") String confId,
@PathParam("epId") String epId,
@PathParam("feature") EndpointDebugFeatures feature,
@PathParam("enabled") Boolean enabled)
{
Conference conference = videobridge.getConference(confId);
if (conference == null)
{
throw new NotFoundException("No conference was found with the specified id.");
}
Endpoint endpoint = conference.getLocalEndpoint(epId);
if (endpoint == null)
{
throw new NotFoundException("No endpoint was found with the specified id.");
}
logger.info("Setting feature state: feature=" + feature.getValue() + ", enabled? " + enabled);
try
{
endpoint.setFeature(feature, enabled);
}
catch (IllegalStateException e)
{
return Response.status(403, e.getMessage()).build();
}
return Response.ok().build();
}
@GET
@Path("/features/endpoint/{confId}/{epId}/{feature}/")
public Boolean getEndpointFeatureState(
@PathParam("confId") String confId,
@PathParam("epId") String epId,
@PathParam("feature") EndpointDebugFeatures feature)
{
Conference conference = videobridge.getConference(confId);
if (conference == null)
{
throw new NotFoundException("No conference was found with the specified id.");
}
Endpoint endpoint = conference.getLocalEndpoint(epId);
if (endpoint == null)
{
throw new NotFoundException("No endpoint was found with the specified id.");
}
try
{
return endpoint.isFeatureEnabled(feature);
}
catch (Exception e)
{
throw new ServerErrorException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR);
}
}
@POST
@Path("/features/relay/{confId}/{rId}/{feature}/{enabled}")
public Response setRelayFeatureState(
@PathParam("confId") String confId,
@PathParam("rId") String rId,
@PathParam("feature") EndpointDebugFeatures feature,
@PathParam("enabled") Boolean enabled)
{
Conference conference = videobridge.getConference(confId);
if (conference == null)
{
throw new NotFoundException("No conference was found with the specified id.");
}
Relay relay = conference.getRelay(rId);
if (relay == null)
{
throw new NotFoundException("No relay was found with the specified id.");
}
logger.info("Setting feature state: feature=" + feature.getValue() + ", enabled? " + enabled);
try
{
relay.setFeature(feature, enabled);
}
catch (IllegalStateException e)
{
return Response.status(403, e.getMessage()).build();
}
return Response.ok().build();
}
@GET
@Path("/features/relay/{confId}/{rId}/{feature}/")
public Boolean getRelayFeatureState(
@PathParam("confId") String confId,
@PathParam("rId") String rId,
@PathParam("feature") EndpointDebugFeatures feature)
{
Conference conference = videobridge.getConference(confId);
if (conference == null)
{
throw new NotFoundException("No conference was found with the specified id.");
}
Relay relay = conference.getRelay(rId);
if (relay == null)
{
throw new NotFoundException("No relay was found with the specified id.");
}
try
{
return relay.isFeatureEnabled(feature);
}
catch (Exception e)
{
throw new ServerErrorException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR);
}
}
private void setFeature(DebugFeatures feature, boolean enabled)
{
switch (feature)
{
case PAYLOAD_VERIFICATION: {
Node.Companion.enablePayloadVerification(enabled);
break;
}
case NODE_STATS: {
StatsKeepingNode.Companion.setEnableStatistics(enabled);
break;
}
case POOL_STATS: {
ByteBufferPool.enableStatistics(enabled);
break;
}
case POOL_BOOKKEEPING: {
ByteBufferPool.enableBookkeeping(enabled);
break;
}
case QUEUE_STATS: {
QueueStatistics.DEBUG = enabled;
break;
}
case QUEUE_TIMING_STATS: {
QueueStatistics.TRACK_TIMES = enabled;
break;
}
case NODE_TRACING: {
Node.Companion.enableNodeTracing(enabled);
break;
}
case TRANSIT_STATS: {
//TODO
break;
}
case TASK_POOL_STATS: {
//TODO
break;
}
default: {
throw new NotFoundException();
}
}
}
// Functions to actually get statistics
@GET
@Produces(MediaType.APPLICATION_JSON)
public String bridgeDebug(@DefaultValue("false") @QueryParam("full") boolean full)
{
ObjectNode debugState = videobridge.getDebugState(
null,
null,
full ? DebugStateMode.FULL : DebugStateMode.SHORT);
// Append the health status.
Result result = healthCheckService.getResult();
ObjectNode health = JsonNodeFactory.instance.objectNode();
health.put("success", result.getSuccess());
health.put("hardFailure", result.getHardFailure());
health.put("responseCode", result.getResponseCode());
health.put("message", result.getMessage());
debugState.set("health", health);
return debugState.toString();
}
@GET
@Path("/{confId}")
@Produces(MediaType.APPLICATION_JSON)
public String confDebug(
@PathParam("confId") String confId,
@DefaultValue("true") @QueryParam("full") boolean full)
{
ObjectNode confJson = videobridge.getDebugState(
confId,
null,
full ? DebugStateMode.FULL : DebugStateMode.SHORT);
return confJson.toString();
}
@GET
@Path("/{confId}/{epId}")
@Produces(MediaType.APPLICATION_JSON)
public String epDebug(
@PathParam("confId") String confId,
@PathParam("epId") String epId,
@DefaultValue("true") @QueryParam("full") boolean full)
{
ObjectNode confJson = videobridge.getDebugState(
confId,
epId,
full ? DebugStateMode.FULL : DebugStateMode.SHORT);
return confJson.toString();
}
@GET
@Path("/stats/jvb/{feature}")
@Produces(MediaType.APPLICATION_JSON)
public String getJvbFeatureStats(@PathParam("feature") DebugFeatures feature)
{
switch (feature)
{
case NODE_STATS: {
return StatsKeepingNode.Companion.getStatsJson().toString();
}
case POOL_STATS: {
return ByteBufferPool.getStatsJson().toString();
}
case QUEUE_STATS: {
return QueueStats.getQueueStats().toString();
}
case TRANSIT_STATS: {
return PacketTransitStats.getStatsJson().toString();
}
case TASK_POOL_STATS: {
return TaskPools.getStatsJson().toString();
}
case XMPP_DELAY_STATS: {
return XmppConnection.getStatsJson().toString();
}
case PAYLOAD_VERIFICATION: {
return PayloadVerificationPlugin.getStatsJson().toString();
}
case ICE_STATS: {
return IceStatistics.Companion.getStats().toJson().toString();
}
case CONFERENCE_PACKET_STATS: {
return ConferencePacketStats.stats.toJson().toString();
}
case TOSSED_PACKET_STATS: {
return VideobridgeMetrics.tossedPacketsEnergy.get().toString();
}
default: {
throw new NotFoundException();
}
}
}
// Old deprecated paths
/**
* Depreacted, use {@link Debug#getJvbFeatureStats(DebugFeatures)}
* @param featureName the feature name
* @param uriInfo the URI info of the request
* @return HTTP response
*/
@Deprecated
@GET
@Path("/stats/{feature_name:.+}")
@Produces(MediaType.APPLICATION_JSON)
public Response getStats(@PathParam("feature_name") String featureName, @Context UriInfo uriInfo)
{
// Redirect to the new location
String newTarget = uriInfo.getBaseUri() + "debug/stats/jvb/" + featureName;
return Response.status(302).location(URI.create(newTarget)).build();
}
}

67
jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java Normal file
View File

@@ -0,0 +1,67 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.debug;
/**
* Enumerates different debug features which can be enabled and disabled via
* the debug REST API.
*/
public enum DebugFeatures
{
PAYLOAD_VERIFICATION("payload-verification"),
NODE_STATS("node-stats"),
POOL_STATS("pool-stats"),
POOL_BOOKKEEPING("pool-bookkeeping"),
QUEUE_STATS("queue-stats"),
QUEUE_TIMING_STATS("queue-timing-stats"),
TRANSIT_STATS("transit-stats"),
TASK_POOL_STATS("task-pool-stats"),
NODE_TRACING("node-tracing"),
ICE_STATS("ice-stats"),
XMPP_DELAY_STATS("xmpp-delay-stats"),
TOSSED_PACKET_STATS("tossed-packet-stats"),
CONFERENCE_PACKET_STATS("conference-packet-stats");
private final String value;
DebugFeatures(String value)
{
this.value = value;
}
public String getValue()
{
return this.value;
}
/**
* A custom 'fromString' implementation which allows creating an instance of
* this enum from its value. This assumes that the String values are derived using
* the following transformation of the 'keys':
* 1) lower case
* 2) underscores are replaced with hyphens
*
* @param value the String value of the enum
* @return an instance of the enum, if one can be derived by reversing the transformation
* detailed above
*/
public static DebugFeatures fromString(String value)
{
String normalized = value.toUpperCase().replace("-", "_");
return DebugFeatures.valueOf(normalized);
}
}

52
jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.debug;
public enum EndpointDebugFeatures
{
PCAP_DUMP("pcap-dump"),
SCTP_PCAP_DUMP("sctp-pcap-dump");
private final String value;
EndpointDebugFeatures(String value)
{
this.value = value;
}
public String getValue()
{
return this.value;
}
/**
* A custom 'fromString' implementation which allows creating an instance of
* this enum from its value. This assumes that the String values are derived using
* the following transformation of the 'keys':
* 1) lower case
* 2) underscores are replaced with hyphens
*
* @param value the String value of the enum
* @return an instance of the enum, if one can be derived by reversing the transformation
* detailed above
*/
public static EndpointDebugFeatures fromString(String value)
{
String normalized = value.toUpperCase().replace("-", "_");
return EndpointDebugFeatures.valueOf(normalized);
}
}

45
jvb/src/main/java/org/jitsi/videobridge/rest/root/stats/Stats.java Normal file
View File

@@ -0,0 +1,45 @@
/*
* Copyright @ 2025 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.rest.root.stats;
import jakarta.inject.*;
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import org.jitsi.nlj.*;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.rest.*;
import org.jitsi.videobridge.rest.annotations.*;
@Path("/stats")
@EnabledByConfig(RestApis.RTCSTATS)
public class Stats
{
@Inject
private Videobridge videobridge;
@GET
@Produces(MediaType.APPLICATION_JSON)
public String rtcstats()
{
ObjectNode debugState = videobridge.getDebugState(
null,
null,
DebugStateMode.STATS);
return debugState.toString();
}
}

327
jvb/src/main/java/org/jitsi/videobridge/util/ByteBufferPool.java Normal file
View File

@@ -0,0 +1,327 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.util;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.util.*;
import org.jitsi.utils.logging2.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.stream.*;
/**
* A pool of reusable byte[]
*
* @author Brian Baldino
* @author Boris Grozev
*/
public class ByteBufferPool
{
/**
* The pool of buffers is segmented by size to make the overall memory
* footprint smaller.
* These thresholds were chosen to minimize the used memory for a trace of
* requests from a test conference. Using finer segmentation (4 thresholds)
* results in only a marginal improvement.
*/
private static int T1 = 220;
private static int T2 = 775;
private static int T3 = 1500;
/**
* The pool of buffers with size <= T1
*/
private static final PartitionedByteBufferPool pool1
= new PartitionedByteBufferPool(T1);
/**
* The pool of buffers with size in (T1, T2]
*/
private static final PartitionedByteBufferPool pool2
= new PartitionedByteBufferPool(T2);
/**
* The pool of buffers with size in (T2, T3]
*/
private static final PartitionedByteBufferPool pool3
= new PartitionedByteBufferPool(T3);
/**
* The {@link Logger}
*/
private static final Logger logger = new LoggerImpl(ByteBufferPool.class.getName());
private static final Set<byte[]> outstandingBuffers =
Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>()));
private static final Set<byte[]> returnedBuffers =
Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>()));
/**
* A debug data structure which tracks all events (requests and returns) for a given
* buffer
*/
private static final Map<byte[], Queue<BufferEvent>> bufferEvents =
Collections.synchronizedMap(new IdentityHashMap<>());
/**
* Whether to enable keeping track of statistics.
*/
private static boolean enableStatistics = false;
/**
* Whether to enable or disable book keeping.
*/
private static Boolean bookkeepingEnabled = false;
/**
* Total number of buffers requested.
*/
private static final LongAdder numRequests = new LongAdder();
/**
* The number of requests which were larger than our threshold and were
* allocated from java instead.
*/
private static final LongAdder numLargeRequests = new LongAdder();
/**
* Total number of buffers returned.
*/
private static final LongAdder numReturns = new LongAdder();
/**
* Gets the current thread ID.
*/
private static long threadId()
{
return Thread.currentThread().getId();
}
/**
* Gets a stack trace as a multi-line string.
*/
private static String stackTraceAsString(StackTraceElement[] stack)
{
StringBuilder sb = new StringBuilder();
for (StackTraceElement ste : stack)
{
sb.append(ste.toString()).append("\n");
}
return sb.toString();
}
/**
* Returns a buffer from the pool.
*
* @param size the minimum size.
*/
public static byte[] getBuffer(int size)
{
if (enableStatistics)
{
numRequests.increment();
}
byte[] buf;
if (size <= T1)
{
buf = pool1.getBuffer(size);
}
else if (size <= T2)
{
buf = pool2.getBuffer(size);
}
else if (size <= T3)
{
buf = pool3.getBuffer(size);
}
else
{
buf = new byte[size];
numLargeRequests.increment();
}
if (bookkeepingEnabled)
{
Exception stackTrace = new Exception();
outstandingBuffers.add(buf);
returnedBuffers.remove(buf);
bufferEvents.computeIfAbsent(buf, k -> new LinkedBlockingQueue<>())
.add(new BufferEvent(BufferEvent.ALLOCATION, System.currentTimeMillis(), stackTrace));
}
return buf;
}
/**
* Returns a buffer to the pool.
* @param buf
*/
public static void returnBuffer(@NotNull byte[] buf)
{
if (enableStatistics)
{
numReturns.increment();
}
int len = buf.length;
if (bookkeepingEnabled)
{
int arrayId = System.identityHashCode(buf);
Exception s;
Exception stackTrace = new Exception();
bufferEvents.computeIfAbsent(buf, k -> new LinkedBlockingQueue<>())
.add(new BufferEvent(BufferEvent.RETURN, System.currentTimeMillis(), stackTrace));
if (outstandingBuffers.remove(buf))
{
returnedBuffers.add(buf);
}
else if (returnedBuffers.contains(buf))
{
String bufferTimeline = bufferEvents.get(buf).stream()
.map(BufferEvent::toString)
.collect(Collectors.joining());
logger.error("Thread " + threadId()
+ " returned a previously-returned " + len + "-byte buffer " + arrayId
+ " its timeline:\n" + bufferTimeline);
}
else
{
logger.error("Thread " + threadId()
+ " returned a " + len + "-byte buffer we didn't give out from\n"
+ stackTraceAsString(stackTrace.getStackTrace()));
}
}
if (len <= T1)
{
pool1.returnBuffer(buf);
}
else if (len <= T2)
{
pool2.returnBuffer(buf);
}
else if (len <= T3)
{
pool3.returnBuffer(buf);
}
else
{
logger.warn(
"Received a suspiciously large buffer (size = " + len + ")\n +" +
UtilKt.getStackTrace());
}
}
/**
* Gets a JSON representation of the statistics about the pool.
*/
public static ObjectNode getStatsJson()
{
ObjectNode stats = JsonNodeFactory.instance.objectNode();
long numLargeRequestsSum = numLargeRequests.sum();
stats.put("num_large_requests", numLargeRequestsSum);
if (enableStatistics)
{
long numRequestsSum = numRequests.sum();
long allAllocations = numLargeRequestsSum + pool1.getNumAllocations()
+ pool2.getNumAllocations() + pool3.getNumAllocations();
long storedBytes = pool1.getStoredBytes() + pool2.getStoredBytes()
+ pool3.getStoredBytes();
stats.put("num_requests", numRequestsSum);
stats.put("num_returns", numReturns.sum());
stats.put("num_allocations", allAllocations);
stats.put(
"allocation_percent",
(100.0 * allAllocations) / numRequestsSum);
stats.put("stored_bytes", storedBytes);
stats.set("pool1", pool1.getStats());
stats.set("pool2", pool2.getStats());
stats.set("pool3", pool3.getStats());
}
if (bookkeepingEnabled)
{
stats.put("outstanding_buffers", outstandingBuffers.size());
}
return stats;
}
/**
* Enables of disables tracking of statistics for the pool.
* @param enable whether to enable it or disable it.
*/
public static void enableStatistics(boolean enable)
{
enableStatistics = enable;
pool1.enableStatistics(enable);
pool2.enableStatistics(enable);
pool3.enableStatistics(enable);
}
public static boolean statisticsEnabled()
{
return enableStatistics;
}
public static void enableBookkeeping(boolean enable)
{
bookkeepingEnabled = enable;
if (!enable)
{
outstandingBuffers.clear();
returnedBuffers.clear();
bufferEvents.clear();
}
}
public static boolean bookkeepingEnabled()
{
return bookkeepingEnabled;
}
private static class BufferEvent
{
final String context;
final Long timestamp;
final Exception trace;
public BufferEvent(String context, Long timestamp, Exception trace)
{
this.context = context;
this.timestamp = timestamp;
this.trace = trace;
}
@Override
public String toString()
{
return context + " BufferEvent: timestamp=" + timestamp +
"\n" + stackTraceAsString(trace.getStackTrace());
}
public static final String ALLOCATION = "allocation";
public static final String RETURN = "return";
}
}

420
jvb/src/main/java/org/jitsi/videobridge/util/PartitionedByteBufferPool.java Normal file
View File

@@ -0,0 +1,420 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.util;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.utils.stats.*;
import java.time.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
/**
* Implements a byte array pool based on a number of independent partitions.
* Buffers are requested and returned to a random partition, which helps with
* contention.
*
* @author Brian Baldino
* @author Boris Grozev
*/
class PartitionedByteBufferPool
{
/**
* The number of partitions.
*/
private static final int NUM_PARTITIONS = 8;
/**
* How many buffers to pre-allocate in each partition.
*/
private static final int INITIAL_SIZE = 10;
/**
* Whether to accept small buffers (<1500) that are returned.
*/
private static final boolean ACCEPT_SMALL_BUFFERS = false;
/**
* The {@link Logger}
*/
private static final Logger logger = new LoggerImpl(PartitionedByteBufferPool.class.getName());
/**
* The partitions.
*/
private final Partition[] partitions = new Partition[NUM_PARTITIONS];
/**
* Whether to keep track of request/return rates and other basic statistics.
* As opposed to {@link ByteBufferPool#bookkeepingEnabled()} this has a
* relatively low overhead and can be kept on in production if necessary.
*/
private boolean enableStatistics = false;
private final int defaultBufferSize;
/**
* Initializes a new {@link PartitionedByteBufferPool} instance with
* a given initial size for each partition.
*/
PartitionedByteBufferPool(int defaultBufferSize)
{
this.defaultBufferSize = defaultBufferSize;
for (int i = 0; i < NUM_PARTITIONS; ++i)
{
partitions[i] = new Partition(i, INITIAL_SIZE);
}
logger.info("Initialized a new " + getClass().getSimpleName()
+ " with " + NUM_PARTITIONS + " partitions.");
}
/**
* Enables or disables tracking of statistics.
* @param enable whether to enable or disable.
*/
void enableStatistics(boolean enable)
{
enableStatistics = enable;
}
/**
* Returns a random partition.
*/
private Partition getPartition()
{
return partitions[ThreadLocalRandom.current().nextInt(NUM_PARTITIONS)];
}
/**
* {@inheritDoc}
*/
byte[] getBuffer(int size)
{
return getPartition().getBuffer(size);
}
/**
* {@inheritDoc}
*/
void returnBuffer(byte[] buf)
{
getPartition().returnBuffer(buf);
}
/**
* Adds statistics for this pool to the given JSON object.
*/
ObjectNode getStats()
{
ObjectNode stats = JsonNodeFactory.instance.objectNode();
stats.put("default_size", defaultBufferSize);
long requests = 0;
long returns = 0;
long allocations = 0;
long storedBytes = 0;
ArrayNode partitionStats = JsonNodeFactory.instance.arrayNode();
for (Partition p : partitions)
{
requests += p.numRequests.sum();
returns += p.numReturns.sum();
allocations += p.numAllocations.sum();
storedBytes += p.storedBytes.get();
partitionStats.add(p.getStatsJson());
}
stats.put("num_requests", requests);
stats.put("num_returns", returns);
stats.put("num_allocations", allocations);
stats.put("stored_bytes", storedBytes);
stats.put(
"allocation_percent",
100D * allocations / Math.max(1, requests));
stats.set("partitions", partitionStats);
return stats;
}
/**
* Gets the total number of times a new byte[] was allocated.
*/
long getNumAllocations()
{
long allocations = 0;
for (int i = 0; i < NUM_PARTITIONS; i++)
{
allocations += partitions[i].numAllocations.sum();
}
return allocations;
}
/**
* Gets the total number of bytes currently stored in the pool
*/
long getStoredBytes()
{
long storedBytes = 0;
for (int i = 0; i < NUM_PARTITIONS; i++)
{
storedBytes += partitions[i].storedBytes.get();
}
return storedBytes;
}
/**
* A byte array pool with a single {@link LinkedBlockingQueue}.
*/
private class Partition
{
/**
* The queue in which we store available packets.
*/
private final LinkedBlockingQueue<byte[]> pool
= new LinkedBlockingQueue<>();
/**
* The ID of the partition (used for debugging).
*/
private final int id;
/**
* The number of times a request was satisfied with a buffer from the
* pool.
*/
private final LongAdder numNoAllocationNeeded = new LongAdder();
/**
* The number of times a new {@code byte[]} had to be allocated.
*/
private final LongAdder numAllocations = new LongAdder();
/**
* The number of times a new {@code byte[]} had to be allocated because
* the pool was empty.
*/
private final LongAdder numEmptyPoolAllocations = new LongAdder();
/**
* The number of times a new {@code byte[]} had to be allocated because
* the pool did not have a buffer with the required size (but it was
* not empty).
*/
private final LongAdder numWrongSizeAllocations = new LongAdder();
/**
* Total number of requests.
*/
private final LongAdder numRequests = new LongAdder();
/**
* Total number of returned buffers.
*/
private final LongAdder numReturns = new LongAdder();
/**
* The number of times a small buffer (<1500 bytes) was returned.
*/
private final LongAdder numSmallReturns = new LongAdder();
/**
* The number of times a buffer from the pool was discarded because it
* was too small to satisfy a request.
*/
private final LongAdder numSmallBuffersDiscarded = new LongAdder();
/**
* The number of times a large buffer (>1500 bytes) was requested.
*/
private final LongAdder numLargeRequests = new LongAdder();
/**
* The total size of buffers stored in the pool.
*/
private final AtomicLong storedBytes = new AtomicLong();
/**
* Request rate in requests per second over the last 10 seconds.
*/
private final RateTracker requestRate = new RateTracker(Duration.ofSeconds(10), Duration.ofMillis(100));
/**
* Return rate in requests per second over the last 10 seconds.
*/
private final RateTracker returnRate = new RateTracker(Duration.ofSeconds(10), Duration.ofMillis(100));
/**
* Initializes a new partition.
* @param id
* @param initialSize
*/
Partition(int id, int initialSize)
{
this.id = id;
for (int i = 0; i < initialSize; ++i)
{
pool.add(new byte[defaultBufferSize]);
}
}
/**
* Returns a buffer from this pool (allocates a new one if necessary).
* @param requiredSize the minimum size.
*
* @return a {@code byte[]} with size at least {@code requiredSize}.
*/
private byte[] getBuffer(int requiredSize)
{
if (enableStatistics)
{
numRequests.increment();
requestRate.update(1);
if (requiredSize > defaultBufferSize)
{
numLargeRequests.increment();
}
}
byte[] buf = pool.poll();
if (buf == null)
{
buf = new byte[Math.max(defaultBufferSize, requiredSize)];
if (enableStatistics)
{
numAllocations.increment();
numEmptyPoolAllocations.increment();
}
}
else if (buf.length < requiredSize)
{
if (ByteBufferPool.bookkeepingEnabled())
{
logger.info("Needed buffer of size " + requiredSize
+ ", got size " + buf.length + " retrying");
}
// This is unusual and shouldn't happen often, so we will just
// allocate a new buffer.
// If the size is smaller than the default AND we just witnessed
// the buffer to be too small in practice, we'll throw it away.
// This makes sure that if someone returns a small buffer (in
// the practical sense) it will not get stuck in the pool.
if (buf.length >= defaultBufferSize)
{
pool.offer(buf);
}
else if (enableStatistics)
{
storedBytes.getAndAdd(-buf.length);
numSmallBuffersDiscarded.increment();
}
buf = new byte[Math.max(defaultBufferSize, requiredSize)];
if (enableStatistics)
{
numAllocations.increment();
numWrongSizeAllocations.increment();
}
}
else if (enableStatistics)
{
storedBytes.getAndAdd(-buf.length);
numNoAllocationNeeded.increment();
}
return buf;
}
/**
* Returns a buffer to this pool.
* @param buf
*/
private void returnBuffer(@NotNull byte[] buf)
{
if (enableStatistics)
{
numReturns.increment();
returnRate.update(1);
}
if (buf.length < defaultBufferSize)
{
if (enableStatistics)
{
numSmallReturns.increment();
}
if (ACCEPT_SMALL_BUFFERS)
{
pool.offer(buf);
if (enableStatistics)
{
storedBytes.getAndAdd(buf.length);
}
}
}
else
{
pool.offer(buf);
if (enableStatistics)
{
storedBytes.getAndAdd(buf.length);
}
}
}
/**
* Gets a snapshot of the statistics of this partition in JSON format.
*/
private ObjectNode getStatsJson()
{
long now = System.currentTimeMillis();
ObjectNode stats = JsonNodeFactory.instance.objectNode();
stats.put("id", id);
long numRequestsSum = numRequests.sum();
long numAllocationsSum = numAllocations.sum();
stats.put("num_requests", numRequestsSum);
stats.put("num_returns", numReturns.sum());
stats.put("requests_rate_rps", requestRate.getRate(now));
stats.put("returns_rate_rps", returnRate.getRate(now));
stats.put("current_size", pool.size());
stats.put("stored_bytes", storedBytes.get());
stats.put("num_allocations", numAllocationsSum);
stats.put("num_allocations_empty_pool", numEmptyPoolAllocations.sum());
stats.put("num_allocations_wrong_size", numWrongSizeAllocations.sum());
stats.put("num_no_allocation_needed", numNoAllocationNeeded.sum());
stats.put(
"allocation_percent",
100D * numAllocationsSum / Math.max(1, numRequestsSum));
stats.put("num_small_returns", numSmallReturns.sum());
stats.put("num_large_requests", numLargeRequests.sum());
stats.put("num_small_discarded", numSmallBuffersDiscarded.sum());
return stats;
}
}
}

90
jvb/src/main/java/org/jitsi/videobridge/util/TaskPools.java Normal file
View File

@@ -0,0 +1,90 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.util;
import org.jitsi.utils.concurrent.*;
import org.jitsi.utils.logging2.*;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.util.concurrent.*;
public class TaskPools
{
private static final Logger classLogger = new LoggerImpl(TaskPools.class.getName());
/**
* A global executor service which can be used for non-CPU-intensive tasks.
*/
public static final ExecutorService IO_POOL =
Executors.newCachedThreadPool(new CustomizableThreadFactory("Global IO pool", false));
/**
* An executor to be used for CPU-intensive tasks. NOTE that tasks which block should
* NOT use this pool!
*/
public static final ExecutorService CPU_POOL =
Executors.newFixedThreadPool(
Runtime.getRuntime().availableProcessors(),
new CustomizableThreadFactory("Global CPU pool", false)
);
private final static ScheduledExecutorService DEFAULT_SCHEDULED_POOL =
Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("Global scheduled pool", false));
public static ScheduledExecutorService SCHEDULED_POOL = DEFAULT_SCHEDULED_POOL;
public static void resetScheduledPool()
{
SCHEDULED_POOL = DEFAULT_SCHEDULED_POOL;
}
public static ObjectNode getStatsJson(ExecutorService es)
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
debugState.put("executor_class", es.getClass().getSimpleName());
if (es instanceof ThreadPoolExecutor)
{
ThreadPoolExecutor ex = (ThreadPoolExecutor)es;
debugState.put("pool_size", ex.getPoolSize());
debugState.put("active_task_count", ex.getActiveCount());
debugState.put("completed_task_count", ex.getCompletedTaskCount());
debugState.put("core_pool_size", ex.getCorePoolSize());
debugState.put("maximum_pool_size", ex.getMaximumPoolSize());
debugState.put("largest_pool_size", ex.getLargestPoolSize());
debugState.put("queue_class", ex.getQueue().getClass().getSimpleName());
debugState.put("pending_task_count", ex.getQueue().size());
}
return debugState;
}
public static ObjectNode getStatsJson()
{
ObjectNode debugState = JsonNodeFactory.instance.objectNode();
debugState.set("IO_POOL", getStatsJson(IO_POOL));
debugState.set("CPU_POOL", getStatsJson(CPU_POOL));
return debugState;
}
static {
classLogger.info("TaskPools detected " + Runtime.getRuntime().availableProcessors() +
" processors, creating the CPU pool with that many threads");
}
}

113
jvb/src/main/java/org/jitsi/videobridge/util/UlimitCheck.java Normal file
View File

@@ -0,0 +1,113 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.util;
import org.jitsi.utils.logging2.*;
import java.io.*;
/**
* Checks the `ulimit` values for the process and prints a warning message if
* they are too low.
*
* @author Boris Grozev
*/
public class UlimitCheck
{
/**
* The {@link Logger} to be used by the {@link UlimitCheck} class
* and its instances to print debug information.
*/
private static final Logger logger = new LoggerImpl(UlimitCheck.class.getName());
/**
* Executes a command in {@code bash} and returns the output ({@code stdin}
* and {@code stderr} combined), or {@code null} on failure.
* @param command the command to execute.
* @return the output of the command or {@code null}.
*/
private static String getOutputFromCommand(String command)
{
ProcessBuilder pb = new ProcessBuilder("bash", "-c", command);
pb.redirectErrorStream(true);
try
{
Process p = pb.start();
BufferedReader br
= new BufferedReader(
new InputStreamReader(p.getInputStream()));
String output = br.lines().reduce(String::concat).orElse("null?");
br.close();
return output;
}
catch (IOException e)
{
return null;
}
}
private static int parseInt(String s)
{
try
{
return Integer.parseInt(s);
}
catch (NumberFormatException n)
{
return Integer.MAX_VALUE;
}
}
/**
* Extracts the current limits for number of open files and user processes
* (threads) by running {@code bash}'s {@code ulimit} builtin, and logs
* the values.
*/
public static void printUlimits()
{
String fileLimit = getOutputFromCommand("ulimit -n");
String fileLimitHard = getOutputFromCommand("ulimit -Hn");
String threadLimit = getOutputFromCommand("ulimit -u");
String threadLimitHard = getOutputFromCommand("ulimit -Hu");
StringBuilder sb
= new StringBuilder("Running with open files limit ")
.append(fileLimit)
.append(" (hard ").append(fileLimitHard).append(')')
.append(", thread limit ").append(threadLimit)
.append(" (hard ").append(threadLimitHard).append(").");
// These are not precise limits, they are based on numbers observed on a moderately loaded 8-core machine in
// production (Nov 2022).
boolean warn = parseInt(fileLimit) <= 4096 || parseInt(threadLimit) <= 2048;
if (warn)
{
sb.append(" These values are too low and they will limit the ")
.append("number of participants that the bridge can serve ")
.append("simultaneously.");
logger.warn(sb);
}
else
{
logger.info(sb);
}
}
}

204
jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java Normal file
View File

@@ -0,0 +1,204 @@
/*
* Copyright @ 2017 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.websocket;
import org.eclipse.jetty.websocket.api.Callback;
import org.eclipse.jetty.websocket.api.Session;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.util.*;
import org.jitsi.videobridge.websocket.config.*;
import java.nio.*;
import java.time.*;
import java.util.*;
import java.util.concurrent.*;
/**
* @author Boris Grozev
*/
public class ColibriWebSocket implements Session.Listener.AutoDemanding
{
/**
* The logger instance used by this {@link ColibriWebSocket}.
*/
private final Logger logger;
/**
* The {@link EventHandler}, if any, associated with this web socket.
*/
private final EventHandler eventHandler;
/**
* The clock used to compute lastSendTime.
*/
private final Clock clock = Clock.systemUTC();
/** The last time something was sent on this web socket */
private Instant lastSendTime = Instant.MIN;
/** The recurring task to send pings on the connection, if needed. */
private ScheduledFuture<?> pinger = null;
/** The {@link Session} associated with this socket, set once the upgrade completes. */
private volatile Session session;
/**
* Initializes a new {@link ColibriWebSocket} instance.
*/
public ColibriWebSocket(
String id,
EventHandler eventHandler
)
{
this.logger = new LoggerImpl(getClass().getName(), new LogContext(Map.of("id", id)));
this.eventHandler = Objects.requireNonNull(eventHandler, "eventHandler");
}
public Session getSession()
{
return session;
}
/**
* Handles an a text message received on this web socket.
* @param message the message.
*/
@Override
public void onWebSocketText(String message)
{
logger.debug(() -> "Received text: " + message);
eventHandler.webSocketTextReceived(this, message);
}
/**
* Handles the event of this web socket being connected. Finds the
* destination COLIBRI {@link Endpoint} and authenticates the request
* based on the password.
*/
@Override
public void onWebSocketOpen(Session sess)
{
this.session = sess;
if (WebsocketServiceConfig.config.getSendKeepalivePings())
{
pinger = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(this::maybeSendPing,
WebsocketServiceConfig.config.getKeepalivePingInterval().toMillis(),
WebsocketServiceConfig.config.getKeepalivePingInterval().toMillis(),
TimeUnit.MILLISECONDS);
}
eventHandler.webSocketConnected(this);
}
/**
* {@inheritDoc}
*/
@Override
public void onWebSocketClose(int statusCode, String reason)
{
session = null;
eventHandler.webSocketClosed(this, statusCode, reason);
if (pinger != null)
{
pinger.cancel(true);
}
}
public void sendString(String message)
{
Session s = session;
if (s != null)
{
// We'll use the async version of sendText since this may be called
// from multiple threads. It's just fire-and-forget though, so we
// don't wait on the result.
s.sendText(message, Callback.NOOP);
synchronized (this)
{
lastSendTime = clock.instant();
}
}
}
private void maybeSendPing()
{
try
{
Instant now = clock.instant();
synchronized (this)
{
if (Duration.between(lastSendTime, now).
compareTo(WebsocketServiceConfig.config.getKeepalivePingInterval()) < 0)
{
Session s = session;
if (s != null)
{
s.sendPing(ByteBuffer.allocate(0), Callback.NOOP);
lastSendTime = clock.instant();
}
}
}
}
catch (Exception e)
{
logger.error("Error sending websocket ping", e);
}
}
/**
* {@inheritDoc}
*/
@Override
public void onWebSocketError(Throwable cause)
{
eventHandler.webSocketError(this, cause);
if (pinger != null)
{
pinger.cancel(true);
}
}
public interface EventHandler
{
/**
* Notifies that a specific {@link ColibriWebSocket}
* instance associated with it has been closed.
* @param ws the {@link ColibriWebSocket} which has been closed.
*/
void webSocketClosed(ColibriWebSocket ws, int statusCode, String reason);
/**
* Notifies that a specific {@link ColibriWebSocket}
* instance associated with it has connected.
* @param ws the {@link ColibriWebSocket} which has connected.
*/
void webSocketConnected(ColibriWebSocket ws);
/**
* Notifies that a message has been received from a
* specific {@link ColibriWebSocket} instance associated with it.
* @param ws the {@link ColibriWebSocket} from which a message was received.
*/
void webSocketTextReceived(ColibriWebSocket ws, String message);
/**
* An error occurred for a specific websocket.
* @param ws the websocket.
* @param cause the cause of the error.
*/
void webSocketError(ColibriWebSocket ws, Throwable cause);
}
}

234
jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java Normal file
View File

@@ -0,0 +1,234 @@
/*
* Copyright @ 2017 Atlassian Pty Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.websocket;
import org.eclipse.jetty.ee10.websocket.server.*;
import org.eclipse.jetty.http.HttpStatus;
import org.eclipse.jetty.websocket.api.*;
import org.jetbrains.annotations.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.videobridge.*;
import org.jitsi.videobridge.relay.*;
import org.jitsi.videobridge.websocket.config.*;
import static org.jitsi.videobridge.websocket.config.WebsocketServiceConfig.config;
import java.io.*;
import java.time.*;
import java.util.*;
import java.util.stream.*;
/**
* @author Boris Grozev
*/
class ColibriWebSocketServlet
extends JettyWebSocketServlet
{
/**
* The logger instance used by this {@link ColibriWebSocketServlet}.
*/
private static final Logger logger = new LoggerImpl(ColibriWebSocketServlet.class.getName());
/**
* The server ID of this websocket servlet. Incoming requests must have a matching
* server ID to be handled.
*/
private final String serverId;
@NotNull
private final Videobridge videobridge;
/**
* Initializes a new {@link ColibriWebSocketServlet} instance.
*/
ColibriWebSocketServlet(
@NotNull String serverId,
@NotNull Videobridge videobridge)
{
this.serverId = serverId;
this.videobridge = videobridge;
}
/**
* {@inheritDoc}
*/
@Override
public void configure(JettyWebSocketServletFactory webSocketServletFactory)
{
// set a timeout of 1min
webSocketServletFactory.setIdleTimeout(WebsocketServiceConfig.config.getIdleTimeout());
webSocketServletFactory.setCreator((request, response) ->
{
try
{
return
ColibriWebSocketServlet
.this.createWebSocket(request, response);
}
catch (IOException ioe)
{
response.setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500);
return null;
}
});
}
/**
* Handles a request for a web-socket. Validates the request URI and either
* accepts the request and creates a {@link ColibriWebSocket} instance, or
* rejects the request and sends an error.
*/
private ColibriWebSocket createWebSocket(
JettyServerUpgradeRequest request,
JettyServerUpgradeResponse response)
throws IOException
{
// A valid request URI looks like this:
// /colibri-ws/server-id/conf-id/endpoint-id?pwd=password
// or /colibri-ws/relay/server-id/conf-id/relay-id?pwd=password
// The "path" does not include "?pwd=password", which is in the "query"
String path = request.getRequestURI().getPath();
logger.debug(() -> "Got a create websocket request at path " + path);
boolean isRelay;
if (path == null)
{
logger.debug(() -> "Received request for an invalid path: " + path);
response.sendError(404, "invalid path");
return null;
}
else if (path.startsWith(ColibriWebSocketService.COLIBRI_WS_PATH))
{
isRelay = false;
}
else if (path.startsWith(ColibriWebSocketService.COLIBRI_RELAY_WS_PATH))
{
isRelay = true;
}
else
{
logger.debug(() -> "Received request for an invalid path: " + path);
response.sendError(404, "invalid path");
return null;
}
String wsPath
= path.substring((isRelay ?
ColibriWebSocketService.COLIBRI_RELAY_WS_PATH :
ColibriWebSocketService.COLIBRI_WS_PATH).length());
String[] ids = wsPath.split("/");
if (ids.length < 3)
{
logger.debug(() -> "Received request for an invalid path: " + wsPath);
response.sendError(404, "invalid path");
return null;
}
if (!serverId.equals(ids[0]))
{
logger.warn("Received request with a mismatching server ID "
+ "(expected '" + serverId
+ "', received '" + ids[0] + "').");
response.sendError(404, "server ID mismatch");
return null;
}
// Note: intentionally fail with the exact same message (as to not leak
// info)
String authFailed = "authentication failed";
Conference conference = videobridge.getConference(ids[1]);
if (conference == null)
{
logger.warn("Received request for an nonexistent conference: "
+ ids[1]);
response.sendError(403, authFailed);
return null;
}
ColibriWebSocket.EventHandler eventHandler;
if (isRelay)
{
Relay relay = conference.getRelay(ids[2]);
if (relay == null)
{
logger.warn("Received request for a nonexistent relay: "
+ ids[2] + " (conference " + conference.getID() + ")");
response.sendError(403, authFailed);
return null;
}
String pwd = getPwd(request.getRequestURI().getQuery());
if (!relay.acceptWebSocket(pwd))
{
response.sendError(403, authFailed);
return null;
}
eventHandler = relay.getMessageTransport();
}
else
{
Endpoint endpoint = conference.getLocalEndpoint(ids[2]);
if (endpoint == null)
{
logger.warn("Received request for a nonexistent endpoint: "
+ ids[2] + " (conference " + conference.getID() + ")");
response.sendError(403, authFailed);
return null;
}
String pwd = getPwd(request.getRequestURI().getQuery());
if (!endpoint.acceptWebSocket(pwd))
{
response.sendError(403, authFailed);
return null;
}
eventHandler = endpoint.getMessageTransport();
}
if (!config.shouldEnableCompression())
{
List<ExtensionConfig> extensions = response.getExtensions().stream().
filter((ext) -> !ext.getName().equals("permessage-deflate")).
collect(Collectors.toList());
response.setExtensions(extensions);
}
return new ColibriWebSocket(ids[2], eventHandler);
}
/**
* Extracts the password from the "query" part of a URI (e.g. "pwd=12345").
* @param query the "query" part of a URI (e.g. "pwd=12345").
* @return the password (e.g. "12345"), or null if parsing failed.
*/
private String getPwd(String query)
{
// TODO: this only deals with the simplest case.
if (query == null)
{
return null;
}
if (!query.startsWith("pwd="))
{
return null;
}
return query.substring("pwd=".length());
}
}

840
jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java Normal file
View File

@@ -0,0 +1,840 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.xmpp;
import org.jetbrains.annotations.*;
import org.jitsi.nlj.*;
import org.jitsi.nlj.rtp.*;
import org.jitsi.nlj.rtp.codec.vpx.*;
import org.jitsi.utils.logging2.*;
import org.jitsi.xmpp.extensions.colibri.*;
import org.jitsi.xmpp.extensions.jingle.*;
import org.jitsi.xmpp.extensions.jitsimeet.*;
import org.jivesoftware.smack.packet.*;
import org.jxmpp.jid.parts.*;
import org.jxmpp.util.*;
import java.util.*;
import java.util.Objects;
import java.util.stream.*;
/**
* A factory of {@link MediaSourceDesc}s from jingle signaling.
*
* @author George Politis
*/
public class MediaSourceFactory
{
/**
* The {@link Logger} used by the {@link MediaSourceDesc} class and its
* instances for logging output.
*/
private static final Logger logger = new LoggerImpl(MediaSourceFactory.class.getName());
/**
* The resolution of the base stream when activating simulcast for VP8.
*
* FIXME: hardcoded ugh.. this should be either signaled or somehow included
* in the RTP stream.
*/
private static final int VP8_SIMULCAST_BASE_LAYER_HEIGHT = 180;
/**
* libjitsi isn't aware of the group semantics names defined in
* {@link SourceGroupPacketExtension}, which is how we distinguish secondary
* ssrcs, so we'll translate them into constants defined in libjitsi
*/
private static Map<String, SsrcAssociationType> secondarySsrcTypeMap = null;
private static synchronized
Map<String, SsrcAssociationType> getSecondarySsrcTypeMap()
{
if (secondarySsrcTypeMap == null)
{
secondarySsrcTypeMap = new HashMap<>();
secondarySsrcTypeMap.put(
SourceGroupPacketExtension.SEMANTICS_FID, SsrcAssociationType.RTX);
}
return secondarySsrcTypeMap;
}
private static final VpxRtpLayerDesc[] noDependencies = new VpxRtpLayerDesc[0];
/*
* Creates layers for an encoding.
*
* @param spatialLen the number of spatial encodings per simulcast stream.
* @param temporalLen the number of temporal encodings per simulcast stream.
* @param height the maximum height of the top spatial layer
* @return an array that holds the layer descriptions.
*/
private static RtpLayerDesc[] createRTPLayerDescs(
int spatialLen, int temporalLen, int encodingIdx, int height)
{
VpxRtpLayerDesc[] rtpLayers
= new VpxRtpLayerDesc[spatialLen * temporalLen];
for (int spatialIdx = 0; spatialIdx < spatialLen; spatialIdx++)
{
double frameRate = (double) 30 / (1 << (temporalLen - 1));
for (int temporalIdx = 0;
temporalIdx < temporalLen; temporalIdx++)
{
int idx = idx(spatialIdx, temporalIdx,
temporalLen);
VpxRtpLayerDesc[] dependencies;
if (spatialIdx > 0 && temporalIdx > 0)
{
// this layer depends on spatialIdx-1 and temporalIdx-1.
dependencies = new VpxRtpLayerDesc[]{
rtpLayers[
idx(spatialIdx, temporalIdx - 1,
temporalLen)],
rtpLayers[
idx(spatialIdx - 1, temporalIdx,
temporalLen)]
};
}
else if (spatialIdx > 0)
{
// this layer depends on spatialIdx-1.
dependencies = new VpxRtpLayerDesc[]
{rtpLayers[
idx(spatialIdx - 1, temporalIdx,
temporalLen)]};
}
else if (temporalIdx > 0)
{
// this layer depends on temporalIdx-1.
dependencies = new VpxRtpLayerDesc[]
{rtpLayers[
idx(spatialIdx, temporalIdx - 1,
temporalLen)]};
}
else
{
// this is a base layer without any dependencies.
dependencies = noDependencies;
}
int temporalId = temporalLen > 1 ? temporalIdx : -1;
int spatialId = spatialLen > 1 ? spatialIdx : -1;
rtpLayers[idx]
= new VpxRtpLayerDesc(encodingIdx,
temporalId, spatialId, height, frameRate, dependencies);
frameRate *= 2;
}
}
return rtpLayers;
}
/**
* Creates an RTP encoding.
* @param primarySsrc the primary SSRC for the encoding.
* @param spatialLen the number of spatial layers of the encoding.
* @param temporalLen the number of temporal layers of the encodings.
* @param secondarySsrcs a list of pairs, where each
* pair has the secondary ssrc as its key, and the type (rtx, etc.) as its
* value
* @param encodingIdx the index of the encoding
* @return a description of the encoding.
*/
private static RtpEncodingDesc createRtpEncodingDesc(Long primarySsrc,
int spatialLen, int temporalLen, SecondarySsrcs secondarySsrcs,
int encodingIdx, int height)
{
RtpLayerDesc[] layers = createRTPLayerDescs(spatialLen, temporalLen,
encodingIdx, height);
RtpEncodingDesc enc = new RtpEncodingDesc(primarySsrc, layers, encodingIdx);
if (secondarySsrcs != null)
{
secondarySsrcs.forEach(ssrcSecondarySsrc -> {
SsrcAssociationType type
= getSecondarySsrcTypeMap()
.get(ssrcSecondarySsrc.type);
if (type == null)
{
logger.error("Unable to find a mapping for" +
" secondary ssrc type " +
ssrcSecondarySsrc.type +
" will NOT included this secondary ssrc as" +
" an encoding");
}
else
{
enc.addSecondarySsrc(
ssrcSecondarySsrc.ssrc, type);
}
});
}
return enc;
}
/**
* Get the 'secondary' ssrcs for the given primary ssrc. 'Secondary' here
* is defined as things like rtx or fec ssrcs.
* @param ssrc the primary ssrc for which to get the secondary ssrcs
* @param sourceGroups the source groups
* @return a map of secondary ssrc -> type (rtx, fec, etc.)
*/
private static List<SecondarySsrc> getSecondarySsrcs(
long ssrc, Collection<SourceGroupPacketExtension> sourceGroups)
{
List<SecondarySsrc> secondarySsrcs = new ArrayList<>();
for (SourceGroupPacketExtension sourceGroup : sourceGroups)
{
if (sourceGroup.getSemantics().equalsIgnoreCase(
SourceGroupPacketExtension.SEMANTICS_SIMULCAST))
{
// Simulcast does not fall under the definition of 'secondary'
// we want here.
continue;
}
long groupPrimarySsrc = sourceGroup.getSources().get(0).getSSRC();
long groupSecondarySsrc = sourceGroup.getSources().get(1).getSSRC();
if (groupPrimarySsrc == ssrc)
{
secondarySsrcs.add(
new SecondarySsrc(
groupSecondarySsrc, sourceGroup.getSemantics()));
}
}
return secondarySsrcs;
}
/**
* Build a map of source ssrc -> a list of secondary ssrc, secondary ssrc
* type
* @param ssrcs the ssrcs to get secondary ssrcs for
* @param sourceGroups the signaled source groups
* @return map of source ssrc -> a list of secondary ssrc, secondary ssrc
* type
*/
private static Map<Long, SecondarySsrcs> getAllSecondarySsrcs(
SourceSsrcs ssrcs, Collection<SourceGroupPacketExtension> sourceGroups)
{
Map<Long, SecondarySsrcs> allSecondarySsrcs = new HashMap<>();
for (long ssrc : ssrcs)
{
List<SecondarySsrc> secondarySsrcs
= getSecondarySsrcs(ssrc, sourceGroups);
allSecondarySsrcs.put(ssrc, new SecondarySsrcs(secondarySsrcs));
}
return allSecondarySsrcs;
}
/**
* Get all groups which have the given semantics
* @param semantics
* @param groups
* @return
*/
private static List<SourceGroupPacketExtension> getGroups(
final String semantics, final List<SourceGroupPacketExtension> groups)
{
return groups.stream()
.filter(sg -> sg.getSemantics().equalsIgnoreCase(semantics))
.collect(Collectors.toList());
}
/**
* Removes all sources which correspond to an ssrc in sourceSsrcs, or any
* groups with a primary ssrc in sourceSsrcs. NOTE: modifies
* sources and sourceGroups in place
* @param sourceSsrcs the primary ssrcs for which all corresponding ssrcs
* will be removed from the given sources and groups
* @param sources the set of sources
* @param sourceGroups the set of groups
*/
private static void removeReferences(
SourceSsrcs sourceSsrcs,
List<SourcePacketExtension> sources,
List<SourceGroupPacketExtension> sourceGroups)
{
// Remove any groups to which any of the ssrcs of this source belong
List<SourceGroupPacketExtension> groupsToRemove
= sourceGroups.stream()
.filter(
group -> group.getSources().stream().anyMatch(
source -> sourceSsrcs.contains(source.getSSRC())))
.collect(Collectors.toList());
sourceGroups.removeAll(groupsToRemove);
/*
* Remove not only the ssrcs in the source itself, but any ssrcs that
* were in groups along with ssrcs from the source. E.g. if we have:`
* SIM 1 2 3
* RTX 1 10
* RTX 2 20
* RTX 3 30
* then we need to make sure ssrcs 10, 20 and 30 don't create sources of
* their own and are removed along with the processing of the source
* with ssrcs 1, 2 and 3.
*/
Set<Long> ssrcsToRemove = extractSsrcs(groupsToRemove);
sources.removeIf(
source ->
sourceSsrcs.contains(source.getSSRC())
|| ssrcsToRemove.contains(source.getSSRC()));
}
/**
* Extracts all SSRCs from all sources of a list of source groups.
* @param groups the list of groups.
* @return the set of SSRCs contained in one of the groups.
*/
private static Set<Long> extractSsrcs(
List<SourceGroupPacketExtension> groups)
{
Set<Long> ssrcs = new HashSet<>();
groups.forEach(
group -> group.getSources().forEach(
source -> ssrcs.add(source.getSSRC())));
return ssrcs;
};
/**
* Given the sources and groups, return a list of the ssrcs for each
* unique source
* @param sources the set of sources
* @param sourceGroups the set of source groups
* @return a list of {@link SourceSsrcs}. Each SourceSsrc object represents
* a set of primary video ssrcs belonging to a single source (video source)
*/
private static List<SourceSsrcs> getSourceSsrcs(
Collection<SourcePacketExtension> sources,
Collection<SourceGroupPacketExtension> sourceGroups
)
{
//FIXME: determining the individual sources should be done via msid,
// but somewhere along the line we seem to lose the msid information
// in the packet extensions
List<SourceSsrcs> sourceSsrcsList = new ArrayList<>();
// We'll need to keep track of which sources and groups have been
// processed, so make copies of the lists we've been given so we can
// modify them.
List<SourcePacketExtension> sourcesCopy = new ArrayList<>(sources);
List<SourceGroupPacketExtension> sourceGroupsCopy
= new ArrayList<>(sourceGroups);
// Note that the order we process these groups is important
Arrays.asList(
SourceGroupPacketExtension.SEMANTICS_SIMULCAST,
SourceGroupPacketExtension.SEMANTICS_FID,
SourceGroupPacketExtension.SEMANTICS_FEC
).forEach(groupSem -> {
List<SourceGroupPacketExtension> groups
= getGroups(groupSem, sourceGroupsCopy);
groups.forEach(group -> {
// An empty group is the signal that we want to clear all
// the groups.
// https://github.com/jitsi/jitsi/blob/7eabaab0fca37711813965d66a0720d1545f6c48/src/net/java/sip/communicator/impl/protocol/jabber/extensions/colibri/ColibriBuilder.java#L188
if (group.getSources() == null || group.getSources().isEmpty())
{
if (groups.size() > 1)
{
logger.warn("Received empty group, which is " +
"a signal to clear all groups, but there were " +
"other groups present, which shouldn't happen");
}
return;
}
List<Long> ssrcs;
// For a simulcast group, all the ssrcs are considered primary
// ssrcs, but for others, only the main ssrc of the group is
if (groupSem.equalsIgnoreCase(
SourceGroupPacketExtension.SEMANTICS_SIMULCAST))
{
ssrcs
= group.getSources().stream()
.map(SourcePacketExtension::getSSRC)
.collect(Collectors.toList());
}
else
{
ssrcs = Arrays.asList(group.getSources().get(0).getSSRC());
}
SourceSsrcs sourceSsrcs = new SourceSsrcs(ssrcs);
// Now we need to remove any groups with these ssrcs as their
// primary, or sources that correspond to one of these ssrcs
removeReferences(sourceSsrcs, sourcesCopy, sourceGroupsCopy);
sourceSsrcsList.add(sourceSsrcs);
});
});
if (!sourceGroupsCopy.isEmpty())
{
logger.warn(
"Unprocessed source groups: " +
sourceGroupsCopy.stream()
.map(Element::toXML)
.map(Object::toString)
.reduce(String::concat));
}
// The remaining sources are not part of any group. Add them as sources
// with their own primary SSRC.
// NOTE: we need to ignore sources with an ssrc of -1, because the
// ColibriBuilder will use that as a signal to remove sources
// https://github.com/jitsi/jitsi/blob/7eabaab0fca37711813965d66a0720d1545f6c48/src/net/java/sip/communicator/impl/protocol/jabber/extensions/colibri/ColibriBuilder.java#L162
sourcesCopy.forEach(source -> {
if (source.getSSRC() != -1)
{
sourceSsrcsList.add(new SourceSsrcs(source.getSSRC()));
}
else
{
if (sourcesCopy.size() > 1)
{
logger.warn("Received an empty source, which is " +
"a signal to clear all sources, but there were " +
"other sources present, which shouldn't happen");
}
}
});
setOwnersAndNames(sources, sourceSsrcsList);
return sourceSsrcsList;
}
/**
* Updates the given list of {@link SourceSsrcs}, setting the {@code owner}
* field according to the {@code owner} attribute in {@code ssrc-info}
* extensions contained in {@code sources}, if not specified externally
* @param sources the list of {@link SourcePacketExtension} which may contain
* the {@code owner} as an attribute of a {@code ssrc-info} child. The
* list or the objects in the list will not be modified.
* @param sourceSsrcsList the list of {@link SourceSsrcs} to update.
*/
private static void setOwnersAndNames(
Collection<SourcePacketExtension> sources,
Collection<SourceSsrcs> sourceSsrcsList
)
{
for (SourceSsrcs sourceSsrcs : sourceSsrcsList)
{
// Look for the "owner" tag in the sources. We assume that all
// sources contain the "owner" tag so we just check for the
// first SSRC of the source's SSRCs.
long primarySsrc = sourceSsrcs.get(0);
SourcePacketExtension sourceSource
= sources.stream()
.filter(source -> source.getSSRC() == primarySsrc)
.findAny().orElse(null);
sourceSsrcs.owner = getOwner(sourceSource);
sourceSsrcs.name = sourceSource != null ? sourceSource.getName() : null;
}
}
/**
* Extracts the owner/endpoint ID as a {@link String} from a
* {@link SourcePacketExtension}.
* Jicofo includes the full occupant JID of the endpoint as the owner of
* a {@link SSRCInfoPacketExtension}, but in jitsi-videobridge we only
* care about the resource part, which coincides with the ID of the Colibri
* endpoint associated with the owner.
*
* @param source the {@link SourcePacketExtension} from which to extract
* the owner.
* @return the owner/endpoint ID as a {@link String}.
*/
public static String getOwner(SourcePacketExtension source)
{
SSRCInfoPacketExtension ssrcInfoPacketExtension
= source == null
? null : source.getFirstChildOfType(
SSRCInfoPacketExtension.class);
if (ssrcInfoPacketExtension != null)
{
/*
* N.B.: this is semantically the same as
* return
* ssrcInfoPacketExtension.getOwner()
* .getResourceOrEmpty().toString();
*
* but avoids expensive construction of a lot of JID parts we just throw away.
*/
String ownerAttr = ssrcInfoPacketExtension.getAttributeAsString(SSRCInfoPacketExtension.OWNER_ATTR_NAME);
if (ownerAttr != null)
{
return getResourceFromJid(ownerAttr);
}
}
return null;
}
/**
* Given a string containing a JID, return its StringPrep'd Resource part,
* or {@link Resourcepart#EMPTY}, as a string.
*
* Helper for {@link #getOwner}
*/
private static String getResourceFromJid(String jid)
{
String unpreppedResource = XmppStringUtils.parseResource(jid);
if (unpreppedResource.isEmpty())
{
return Resourcepart.EMPTY.toString();
}
return Resourcepart.fromOrThrowUnchecked(unpreppedResource).toString();
}
/**
* Creates {@link MediaSourceDesc}s from signaling params
*
* will receive the created {@link MediaSourceDesc}s.
* @param sources The {@link List} of {@link SourcePacketExtension} that
* describes the list of jingle sources.
* @param sourceGroups The {@link List} of
* {@link SourceGroupPacketExtension} that describes the list of jingle
* source groups.
* @return an array of {@link MediaSourceDesc} that are described in the
* jingle sources and source groups.
* @deprecated Use createMediaSource.
*/
@Deprecated
public static MediaSourceDesc[] createMediaSources(
Collection<SourcePacketExtension> sources,
Collection<SourceGroupPacketExtension> sourceGroups)
{
final Collection<SourceGroupPacketExtension> finalSourceGroups
= sourceGroups == null ? new ArrayList<>() : sourceGroups;
final Collection<SourcePacketExtension> finalSources
= sources == null ? new ArrayList<>() : sources;
List<SourceSsrcs> sourceSsrcsList = getSourceSsrcs(finalSources, finalSourceGroups);
List<MediaSourceDesc> mediaSources = new ArrayList<>();
sourceSsrcsList.forEach(sourceSsrcs -> {
// As of now, we only ever have 1 spatial layer per stream
int numSpatialLayersPerStream = 1;
// Previously, we assumed that when simulcast was enabled, 3 temporal layers would
// be present in each stream and if simulcast wasn't enabled, there would be only
// 1 temporal layer per stream, but we found that screenshare started sending 2
// temporal layers, and we'd therefore start dropping layer 1 packets since we didn't
// recognize them. The truth is we have no way of _knowing_ how many temporal layers
// there are going to be (as it's not signaled) unless we dynamically read the layers
// as the packets come in (which we may want to do in the future)--but for now we'll
// over-estimate how many layers there are per stream to avoid dropping packets for a
// layer we don't recognize. Since there are already situations where chrome sends fewer
// temporal layers than we assume, this should be safe.
final int numTemporalLayersPerStream = 3;
Map<Long, SecondarySsrcs> secondarySsrcs
= getAllSecondarySsrcs(sourceSsrcs, finalSourceGroups);
MediaSourceDesc mediaSource
= createSource(
sourceSsrcs,
numSpatialLayersPerStream,
numTemporalLayersPerStream,
secondarySsrcs,
sourceSsrcs.owner,
sourceSsrcs.name,
getVideoType(finalSources),
false
);
mediaSources.add(mediaSource);
});
return mediaSources.toArray(new MediaSourceDesc[0]);
}
// This method is to replace createMediaSources when Colibri V1 is removed
public static MediaSourceDesc createMediaSource(
Collection<SourcePacketExtension> sources,
Collection<SourceGroupPacketExtension> sourceGroups,
String owner,
String name)
{
return createMediaSource(sources, sourceGroups, owner, name, false);
}
public static MediaSourceDesc createMediaSource(
Collection<SourcePacketExtension> sources,
Collection<SourceGroupPacketExtension> sourceGroups,
String owner,
String name,
boolean synthetic)
{
Objects.requireNonNull(owner, "owner is required");
Objects.requireNonNull(name, "name is required");
final Collection<SourceGroupPacketExtension> finalSourceGroups
= sourceGroups == null ? new ArrayList<>() : sourceGroups;
if (sources == null)
{
return null;
}
List<SourceSsrcs> sourceSsrcsList = getSourceSsrcs(sources, finalSourceGroups);
if (sourceSsrcsList.size() != 1)
{
logger.warn("sourceSsrcsList.size() != 1 for: " + sources + " groups: " + finalSourceGroups);
}
if (sourceSsrcsList.size() > 0)
{
SourceSsrcs sourceSsrcs = sourceSsrcsList.get(0);
// As of now, we only ever have 1 spatial layer per stream
int numSpatialLayersPerStream = 1;
// Previously, we assumed that when simulcast was enabled, 3 temporal layers would
// be present in each stream and if simulcast wasn't enabled, there would be only
// 1 temporal layer per stream, but we found that screenshare started sending 2
// temporal layers, and we'd therefore start dropping layer 1 packets since we didn't
// recognize them. The truth is we have no way of _knowing_ how many temporal layers
// there are going to be (as it's not signaled) unless we dynamically read the layers
// as the packets come in (which we may want to do in the future)--but for now we'll
// over-estimate how many layers there are per stream to avoid dropping packets for a
// layer we don't recognize. Since there are already situations where chrome sends fewer
// temporal layers than we assume, this should be safe.
final int numTemporalLayersPerStream = 3;
Map<Long, SecondarySsrcs> secondarySsrcs = getAllSecondarySsrcs(sourceSsrcs, finalSourceGroups);
return createSource(
sourceSsrcs,
numSpatialLayersPerStream,
numTemporalLayersPerStream,
secondarySsrcs,
owner,
name,
getVideoType(sources),
synthetic
);
}
else
{
return null;
}
}
/**
* Calculates the array position of an RTP layer description specified by its
* spatial index (SVC) and temporal index (SVC).
*
* @param spatialIdx the spatial layer index.
* @param temporalIdx the temporal layer index.
*
* @return the subjective quality index of the flow specified in the
* arguments.
*/
private static int idx(int spatialIdx, int temporalIdx,
int temporalLen)
{
return spatialIdx * temporalLen + temporalIdx;
}
/**
* Describes a single secondary ssrc and its group semantics to
* its primary ssrc (e.g. FID or FEC-FR)
*/
private static class SecondarySsrc
{
public long ssrc;
public String type;
/**
* Initializes a new {@link SecondarySsrc} instance.
*/
public SecondarySsrc(long ssrc, String type)
{
this.ssrc = ssrc;
this.type = type;
}
}
/**
* Groups a set of secondary ssrcs for a single primary ssrc (e.g. its
* rtx and fec ssrcs)
*/
private static class SecondarySsrcs
implements Iterable<SecondarySsrc>
{
public List<SecondarySsrc> secondarySsrcs;
/**
* Initializes a new {@link SecondarySsrcs} instance.
*/
public SecondarySsrcs(List<SecondarySsrc> secondarySsrcs)
{
this.secondarySsrcs = secondarySsrcs;
}
/**
* {@inheritDoc}
*/
@Override
public Iterator<SecondarySsrc> iterator()
{
return secondarySsrcs.iterator();
}
}
/**
* Describes one or more primary video ssrcs for a single
* {@link MediaSourceDesc}
*/
private static class SourceSsrcs
implements Iterable<Long>
{
private String name;
private List<Long> sourceSsrcs;
private String owner;
/**
* Initializes a new {@link SourceSsrcs} instance for a single SSRC.
*/
private SourceSsrcs(Long ssrc)
{
this(Collections.singletonList(ssrc));
}
/**
* Initializes a new {@link SourceSsrcs} instance for a list of SSRCs.
*/
public SourceSsrcs(List<Long> sourceSsrcs)
{
this.sourceSsrcs = sourceSsrcs;
}
/**
* Checks whether this instance contains a specific SSRC.
*/
public boolean contains(Long ssrc)
{
return sourceSsrcs.contains(ssrc);
}
/**
* @return the number of SSRCs in this instance.
*/
public int size()
{
return sourceSsrcs.size();
}
/**
* Gets the SSRC at a specific index.
*/
public Long get(int index)
{
return sourceSsrcs.get(index);
}
/**
* {@inheritDoc}
*/
@Override
public Iterator<Long> iterator()
{
return sourceSsrcs.iterator();
}
}
/**
* Creates a single MediaSourceDesc with the given information
* @param primarySsrcs the set of primary video ssrcs belonging to this source
* @param numSpatialLayersPerStream the number of spatial layers per stream
* for this source
* @param numTemporalLayersPerStream the number of temporal layers per stream
* for this source
* @param allSecondarySsrcs a map of primary ssrc -> SecondarySsrcs, which lists
* the ssrc and type of all the secondary ssrcs for a given primary (e.g.
* its corresponding rtx and fec ssrcs)
* @param owner An externally specified owner, if available.
* @param name An externally specified name, if available.
* @return the created MediaSourceDesc
*/
private static MediaSourceDesc createSource(
SourceSsrcs primarySsrcs,
int numSpatialLayersPerStream,
int numTemporalLayersPerStream,
Map<Long, SecondarySsrcs> allSecondarySsrcs,
String owner,
String name,
VideoType videoType,
boolean synthetic
)
{
RtpEncodingDesc[] encodings =
new RtpEncodingDesc[primarySsrcs.size()];
// this loop builds a subjective quality index array that looks like
// this:
//
// [s0t0, s0t1, s0t2, s1t0, s1t1, s1t2, s2t0, s2t1, s2t2]
//
// The spatial layer is offered either by simulcast (VP8) or spatial
// scalability (VP9). Exotic cases might do simulcast + spatial
// scalability.
//TODO(brian): this is only correct if the highest res is 720p
int height = VP8_SIMULCAST_BASE_LAYER_HEIGHT;
for (int encodingIdx = 0; encodingIdx < primarySsrcs.size(); encodingIdx++) {
Long primarySsrc = primarySsrcs.get(encodingIdx);
SecondarySsrcs ssrcSecondarySsrcs = allSecondarySsrcs.get(primarySsrc);
encodings[encodingIdx] = createRtpEncodingDesc(primarySsrc,
numSpatialLayersPerStream, numTemporalLayersPerStream,
ssrcSecondarySsrcs, encodingIdx, height);
height *= 2;
}
if (name == null)
{
throw new IllegalArgumentException("The 'name' is missing in the source description");
}
if (owner == null)
{
throw new IllegalArgumentException("The 'owner' is missing in the source description");
}
return new MediaSourceDesc(encodings, owner, name, videoType, synthetic);
}
private static VideoType getVideoType(@NotNull Collection<SourcePacketExtension> sources)
{
for (SourcePacketExtension source : sources)
{
if (source.getVideoType() != null)
{
return VideoType.valueOf(source.getVideoType().toUpperCase());
}
}
return VideoType.CAMERA;
}
}

407
jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt Normal file
View File

@@ -0,0 +1,407 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.DebugStateMode
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.VideoType
import org.jitsi.nlj.format.PayloadType
import org.jitsi.nlj.rtp.RtpExtension
import org.jitsi.utils.NEVER
import org.jitsi.utils.event.EventEmitter
import org.jitsi.utils.event.SyncEventEmitter
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.queue.PacketQueue
import org.jitsi.videobridge.cc.allocation.MediaSourceContainer
import org.jitsi.videobridge.cc.allocation.ReceiverConstraintsMap
import org.jitsi.videobridge.cc.allocation.VideoConstraints
import org.jitsi.videobridge.relay.AudioSourceDesc
import org.jitsi.videobridge.util.TaskPools
import java.time.Instant
import java.util.concurrent.ConcurrentHashMap
/**
* Represents an endpoint in a conference (i.e. the entity associated with
* a participant in the conference, which connects the participant's audio
* and video channel). This might be an endpoint connected to this instance of
* jitsi-videobridge, or a relayed endpoint connected to another bridge in the
* same conference.
*
* @author Boris Grozev
* @author Brian Baldino
*/
abstract class AbstractEndpoint protected constructor(
/**
* The [Conference] which this endpoint is to be part of.
*/
val conference: Conference,
/**
* The ID of the endpoint.
*/
final override val id: String,
parentLogger: Logger
) : MediaSourceContainer {
/**
* The [Logger] used by the [AbstractEndpoint] class to print debug
* information.
*/
protected val logger: Logger = parentLogger.createChildLogger(this.javaClass.name, mapOf("epId" to id))
/**
* The map of source name -> ReceiverConstraintsMap.
*/
private val receiverVideoConstraints = ConcurrentHashMap<String, ReceiverConstraintsMap>()
private data class SourceAndHeight(
val sourceName: String,
val maxHeight: Int
)
private val receiverVideoConstraintsQueue = PacketQueue<SourceAndHeight>(
128,
null,
"receiver-constraints-$id",
{ (sourceName, newMaxHeight) ->
receiverVideoConstraintsChanged(sourceName, newMaxHeight)
true
},
TaskPools.IO_POOL
)
/**
* The statistics id of this <tt>Endpoint</tt>.
*/
var statsId: String? = null
set(value) {
field = value
if (value != null) {
logger.addContext("stats_id", value)
}
}
/**
* The indicator which determines whether [.expire] has been called
* on this <tt>Endpoint</tt>.
*/
var isExpired = false
private set
/**
* A map of source name -> VideoConstraints.
*
* Stores the maximum set of constraints applied by all receivers for each media source sent by this endpoint.
* The client needs to send _at least_ this for a media source to satisfy all its receivers.
*/
protected var maxReceiverVideoConstraints = mutableMapOf<String, VideoConstraints>()
protected val eventEmitter: EventEmitter<EventHandler> = SyncEventEmitter()
private val videoTypeCache = mutableMapOf<String, VideoType>()
fun hasVideoAvailable(): Boolean {
for (source in mediaSources) {
if (source.videoType !== VideoType.NONE) {
return true
}
}
return videoTypeCache.values.any { it !== VideoType.NONE }
}
fun setVideoType(sourceName: String, videoType: VideoType) {
val mediaSourceDesc = findMediaSourceDesc(sourceName)
if (mediaSourceDesc != null) {
if (mediaSourceDesc.videoType !== videoType) {
if (mediaSourceDesc.videoType.isEnabled() && videoType.isEnabled()) {
logger.warn(
"Changing video type from ${mediaSourceDesc.videoType} to $videoType for $sourceName. " +
"This will not trigger re-signaling the mapping."
)
}
mediaSourceDesc.videoType = videoType
conference.speechActivity.endpointVideoAvailabilityChanged()
}
}
videoTypeCache[sourceName] = videoType
}
protected fun applyVideoTypeCache(mediaSourceDescs: Array<MediaSourceDesc>) {
// Video types are signaled over JVB data channel while MediaStreamDesc over Colibri. The two channels need
// to be synchronized.
for (mediaSourceDesc in mediaSourceDescs) {
val videoType = videoTypeCache[mediaSourceDesc.sourceName]
if (videoType != null) {
mediaSourceDesc.videoType = videoType
videoTypeCache.remove(mediaSourceDesc.sourceName)
}
}
}
/**
* Checks whether a specific SSRC belongs to this endpoint.
* @param ssrc
* @return
*/
abstract fun receivesSsrc(ssrc: Long): Boolean
/**
* The set of SSRCs received from this endpoint.
* @return
*/
abstract val ssrcs: Set<Long>
/**
* The [AbstractEndpointMessageTransport] associated with
* this endpoint.
*/
open val messageTransport: AbstractEndpointMessageTransport?
get() = null
/** Whether this endpoint represents a visitor. */
abstract val visitor: Boolean
/**
* Whether diarization is requested for this endpoint's audio, as signaled via the colibri2 `diarize` attribute.
* Propagated to the transcriber via the mediajson start event. Defaults to false for endpoint types that don't
* signal it (e.g. relayed endpoints).
*/
open val diarize: Boolean
get() = false
/**
* Gets the description of the video [MediaSourceDesc] that this endpoint has advertised, or `null` if
* it hasn't advertised any video sources.
*/
protected val mediaSource: MediaSourceDesc?
get() = mediaSources.firstOrNull()
fun findMediaSourceDesc(sourceName: String): MediaSourceDesc? = mediaSources.firstOrNull {
sourceName == it.sourceName
}
fun addEventHandler(eventHandler: EventHandler) {
eventEmitter.addHandler(eventHandler)
}
fun removeEventHandler(eventHandler: EventHandler) {
eventEmitter.removeHandler(eventHandler)
}
override fun toString() = "${javaClass.name} $id"
/**
* Expires this [AbstractEndpoint].
*/
open fun expire() {
logger.debug("Expiring.")
isExpired = true
conference.endpointExpired(this)
}
/**
* Return true if this endpoint should expire (based on whatever logic is
* appropriate for that endpoint implementation.
*
* @return true if this endpoint should expire, false otherwise
*/
abstract fun shouldExpire(): Boolean
/**
* Get the last 'incoming activity' (packets received) this endpoint has seen
* @return the timestamp, in milliseconds, of the last activity of this endpoint
*/
open val lastIncomingActivity: Instant?
get() = NEVER
/**
* Requests a keyframe from this endpoint for the specified media SSRC.
*
* @param mediaSsrc the media SSRC to request a keyframe from.
*/
abstract fun requestKeyframe(requesterID: String, mediaSsrc: Long)
/**
* Requests a keyframe from this endpoint on the first video SSRC
* it finds. Being able to request a keyframe without passing a specific
* SSRC is useful for things like requesting a pre-emptive keyframes when a new
* active speaker is detected (where it isn't convenient to try and look up
* a particular SSRC).
*/
abstract fun requestKeyframe()
/** A JSON representation of the parts of this object's state that are deemed useful for debugging. */
open fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
val receiverVideoConstraints = JsonNodeFactory.instance.objectNode()
this@AbstractEndpoint.receiverVideoConstraints.forEach { (sourceName, receiverConstraints) ->
receiverVideoConstraints.set<ObjectNode>(sourceName, receiverConstraints.getDebugState())
}
set<ObjectNode>("receiver_video_constraints", receiverVideoConstraints)
put("max_receiver_video_constraints", HashMap(maxReceiverVideoConstraints).toString())
put("expired", isExpired)
put("stats_id", statsId)
}
/**
* Enqueues a call to send a receiver video constrains message.
* We use a queue to prevent races in sending these constraint messages.
*/
private fun sendReceiverVideoConstraintsChanged(sourceName: String, newMaxHeight: Int) {
receiverVideoConstraintsQueue.add(SourceAndHeight(sourceName, newMaxHeight))
}
/**
* Computes and sets the [.maxReceiverVideoConstraints] from the specified video constraints of the media
* source identified by the given source name.
*
* @param sourceName the name of the media source for which the constraints have changed.
* @param newMaxHeight the maximum height resulting from the current set of constraints.
* (Currently we only support constraining the height, and not frame rate.)
*/
private fun receiverVideoConstraintsChanged(sourceName: String, newMaxHeight: Int) {
val oldReceiverMaxVideoConstraints = maxReceiverVideoConstraints[sourceName]
val newReceiverMaxVideoConstraints = VideoConstraints(newMaxHeight, -1.0)
if (newReceiverMaxVideoConstraints != oldReceiverMaxVideoConstraints) {
maxReceiverVideoConstraints[sourceName] = newReceiverMaxVideoConstraints
sendVideoConstraints(sourceName, newReceiverMaxVideoConstraints)
}
}
/**
* Whether the remote endpoint is currently sending (non-silence) audio.
*/
abstract val isSendingAudio: Boolean
/**
* Whether the remote endpoint is currently sending video.
*/
abstract val isSendingVideo: Boolean
/**
* The set of [AudioSourceDesc]s that this endpoint has advertised.
*/
abstract var audioSources: List<AudioSourceDesc>
/**
* Index of this endpoint's audio sources by SSRC, for O(1) lookup on the per-packet audio path (e.g.
* [Conference.levelChanged]). Rebuilt only when [audioSources] changes; subclasses must call
* [audioSourcesChanged] from their [audioSources] setter to keep it current.
*/
@Volatile
private var audioSourcesBySsrc: Map<Long, AudioSourceDesc> = emptyMap()
/** Returns the advertised audio source with the given [ssrc], or null if there is none. */
fun getAudioSource(ssrc: Long): AudioSourceDesc? = audioSourcesBySsrc[ssrc]
/** Keeps the [getAudioSource] index in sync; call from the [audioSources] setter whenever it changes. */
protected fun audioSourcesChanged(sources: List<AudioSourceDesc>) {
audioSourcesBySsrc = sources.associateBy { it.ssrc }
}
/**
* Adds a payload type to this endpoint.
*/
abstract fun addPayloadType(payloadType: PayloadType)
/**
* Adds an RTP extension to this endpoint
*/
abstract fun addRtpExtension(rtpExtension: RtpExtension)
/**
* Sets extmap-allow-mixed for this endpoint
*/
abstract fun setExtmapAllowMixed(allow: Boolean)
/**
* Notifies this instance that the max video constraints that the bridge needs to receive from a source of this
* endpoint has changed. Each implementation handles this notification differently.
*
* @param sourceName the name of the media source
* @param maxVideoConstraints the max video constraints that the bridge needs to receive from the source
*/
protected abstract fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints)
/**
* Notifies this instance that a specified received wants to receive the specified video constraints from the media
* source with the given source name.
*
* The receiver can be either another endpoint, or a remote bridge.
*
* @param receiverId the id that specifies the receiver endpoint.
* @param sourceName the name of the media source for which the constraints are to be applied.
* @param newVideoConstraints the video constraints that the receiver wishes to receive.
*/
fun addReceiver(receiverId: String, sourceName: String, newVideoConstraints: VideoConstraints) {
val sourceConstraints = receiverVideoConstraints.computeIfAbsent(sourceName) { ReceiverConstraintsMap() }
synchronized(sourceConstraints) {
val oldVideoConstraints = sourceConstraints.put(receiverId, newVideoConstraints)
if (oldVideoConstraints == null || oldVideoConstraints != newVideoConstraints) {
logger.debug {
"Changed receiver constraints: $receiverId->$sourceName: ${newVideoConstraints.maxHeight}"
}
sendReceiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight)
}
}
}
/** Notify this endpoint that another endpoint expired */
open fun otherEndpointExpired(expired: AbstractEndpoint) {
removeReceiver(expired.id)
}
/**
* Notifies this instance that the specified receiver no longer wants or
* needs to receive anything from the endpoint attached to this
* instance (the sender).
*
* @param receiverId the id that specifies the receiver endpoint
*/
fun removeReceiver(receiverId: String) {
for ((sourceName, sourceConstraints) in receiverVideoConstraints) {
synchronized(sourceConstraints) {
if (sourceConstraints.remove(receiverId) != null) {
logger.debug { "Removed receiver $receiverId for $sourceName" }
sendReceiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight)
}
}
}
}
/**
* Notifies this instance that the specified receiver no longer wants or needs to receive anything from the media
* source attached to this instance (the sender).
*
* @param receiverId the id that specifies the receiver endpoint
* @param sourceName the media source name
*/
fun removeSourceReceiver(receiverId: String, sourceName: String) {
val sourceConstraints = receiverVideoConstraints[sourceName]
if (sourceConstraints != null) {
synchronized(sourceConstraints) {
if (sourceConstraints.remove(receiverId) != null) {
logger.debug { "Removed receiver $receiverId for $sourceName" }
sendReceiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight)
}
}
}
}
interface EventHandler {
fun sourcesChanged()
}
}

125
jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt Normal file
View File

@@ -0,0 +1,125 @@
/*
* Copyright @ 2017 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.queue.CountingErrorHandler
import org.jitsi.utils.queue.PacketQueue
import org.jitsi.videobridge.message.BridgeChannelMessage
import org.jitsi.videobridge.message.BridgeChannelMessage.Companion.parse
import org.jitsi.videobridge.message.MessageHandler
import org.jitsi.videobridge.metrics.QueueMetrics
import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer
import org.jitsi.videobridge.util.TaskPools
import java.io.IOException
import java.time.Clock
abstract class AbstractEndpointMessageTransport(parentLogger: Logger) : MessageHandler() {
protected val logger: Logger = parentLogger.createChildLogger(javaClass.name)
abstract val isConnected: Boolean
private val incomingMessageQueue: PacketQueue<MessageAndSource> = PacketQueue<MessageAndSource>(
50,
true,
INCOMING_MESSAGE_QUEUE_ID,
{ messageAndSource ->
try {
handleMessage(messageAndSource.message)?.let { response ->
sendMessage(messageAndSource.source, response)
}
} catch (e: Exception) {
logger.warn("Failed to handle message: ", e)
}
true
},
TaskPools.IO_POOL,
Clock.systemUTC()
).apply { setErrorHandler(queueErrorCounter) }
/**
* Fires the message transport ready event for the associated endpoint.
*/
protected open fun notifyTransportChannelConnected() {}
/**
* Notifies this [EndpointMessageTransport] that a specific message
* has been received on a specific transport channel.
* @param src the transport channel on which the message has been received.
* @param msg the message that was received.
*/
fun onMessage(src: Any?, msg: String) {
val message = try {
parse(msg)
} catch (ioe: IOException) {
logger.warn("Invalid message received (${ioe.message}: $msg")
return
}
logger.debug { "RECV: $msg" }
incomingMessageQueue.add(MessageAndSource(message, src))
}
/**
* Sends [msg] over the active transport channel of this [EndpointMessageTransport].
*/
protected open fun sendMessage(msg: BridgeChannelMessage) {}
protected open fun sendMessage(dst: Any?, message: BridgeChannelMessage) =
logger.debug { "SENT: " + message.toJson() }
open fun close() {}
open val debugState: ObjectNode
get() = JsonNodeFactory.instance.objectNode().apply {
val counts = JsonNodeFactory.instance.objectNode()
getReceivedCounts().forEach { (k, v) -> counts.put(k, v) }
set<ObjectNode>("received_counts", counts)
}
/**
* Events generated by [AbstractEndpointMessageTransport] types which are of interest to other entities.
*/
interface EndpointMessageTransportEventHandler {
fun endpointMessageTransportConnected(endpoint: AbstractEndpoint)
}
private data class MessageAndSource(val message: BridgeChannelMessage, val source: Any?)
companion object {
const val INCOMING_MESSAGE_QUEUE_ID = "bridge-channel-message-incoming-queue"
private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter(
"endpoint_receive_message_queue_dropped_packets",
"Number of packets dropped out of the Endpoint receive message queue."
)
private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter(
"endpoint_receive_message_queue_exceptions",
"Number of exceptions from the Endpoint receive message queue."
)
val queueErrorCounter = object : CountingErrorHandler() {
override fun packetDropped() = super.packetDropped().also {
droppedPacketsMetric.inc()
QueueMetrics.droppedPackets.inc()
}
override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also {
exceptionsMetric.inc()
QueueMetrics.exceptions.inc()
}
}
}
}

81
jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscription.kt Normal file
View File

@@ -0,0 +1,81 @@
/*
* Copyright @ 2025 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.videobridge.message.ReceiverAudioSubscriptionMessage
import org.jitsi.videobridge.relay.AudioSourceDesc
class AudioSubscription {
/** Whether to receive sources not named in [include]/[exclude] (synthetic sources excepted -- see below). */
private var all: Boolean = false
/** The source names being explicitly included / excluded, and the SSRCs they currently resolve to. */
private var include: Set<String> = emptySet()
private var exclude: Set<String> = emptySet()
private var includedSsrcs: Set<Long> = emptySet()
private var excludedSsrcs: Set<Long> = emptySet()
private val lock: Any = Any()
fun updateSubscription(subscription: ReceiverAudioSubscriptionMessage, sources: List<AudioSourceDesc>) =
synchronized(lock) {
all = subscription.all
include = subscription.include.toSet()
exclude = subscription.exclude.toSet()
includedSsrcs = ssrcsForNames(include, sources)
excludedSsrcs = ssrcsForNames(exclude, sources)
}
private fun ssrcsForNames(names: Set<String>, sources: Collection<AudioSourceDesc>): Set<Long> =
sources.filter { it.sourceName != null && it.sourceName in names }.map(AudioSourceDesc::ssrc).toSet()
fun isSsrcWanted(ssrc: Long): Boolean = when {
excludedSsrcs.contains(ssrc) -> false
includedSsrcs.contains(ssrc) -> true
else -> all
}
/**
* Whether the given SSRC is explicitly wanted, i.e. named in the [include] list (and not excluded). Unlike
* [isSsrcWanted] this is not satisfied by [all], so it gates synthetic sources, which are forwarded only on an
* explicit subscription.
*/
fun isExplicitlyWanted(ssrc: Long): Boolean = includedSsrcs.contains(ssrc) && !excludedSsrcs.contains(ssrc)
fun onConferenceSourceAdded(descs: Set<AudioSourceDesc>) = synchronized(lock) {
includedSsrcs = includedSsrcs.union(ssrcsForNames(include, descs))
excludedSsrcs = excludedSsrcs.union(ssrcsForNames(exclude, descs))
}
fun onConferenceSourceRemoved(descs: Set<AudioSourceDesc>) = synchronized(lock) {
val removed = descs.map(AudioSourceDesc::ssrc).toSet()
includedSsrcs = includedSsrcs.subtract(removed)
excludedSsrcs = excludedSsrcs.subtract(removed)
}
fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
synchronized(lock) {
put("all", all)
putArray("include").apply { include.forEach { add(it) } }
putArray("exclude").apply { exclude.forEach { add(it) } }
putArray("included_ssrcs").apply { includedSsrcs.forEach { add(it) } }
putArray("excluded_ssrcs").apply { excludedSsrcs.forEach { add(it) } }
}
}
}

208
jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscriptionManager.kt Normal file
View File

@@ -0,0 +1,208 @@
/*
* Copyright @ 2025 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.videobridge.message.ReceiverAudioSubscriptionMessage
import org.jitsi.videobridge.relay.AudioSourceDesc
import java.util.concurrent.ConcurrentHashMap
class AudioSubscriptionManager() {
/**
* A map of endpoint IDs to their audio subscriptions.
*/
private val audioSubscriptions = ConcurrentHashMap<String, AudioSubscription>()
/**
* A map of local audio source names to a set of endpoint IDs that subscribe to the source with an "Include" type.
*/
private val subscribedLocalAudioSources = ConcurrentHashMap<String, MutableSet<String>>()
/**
* The audio sources currently known to the conference. This is the authoritative set against which subscriptions
* resolve their source names to SSRCs. It is maintained solely via [onSourcesAdded]/[removeSources] under [lock],
* which is the same lock that guards subscription updates. Resolving against this (rather than a source list
* captured by the caller outside [lock]) closes the race where a subscription arriving concurrently with the
* colibri signaling of a source would resolve against a stale snapshot and permanently drop the source.
*/
private val knownSources = mutableSetOf<AudioSourceDesc>()
/**
* An immutable snapshot classifying each currently-known audio SSRC as synthetic (true) or regular (false). An
* SSRC absent from the map is not a currently-known source and is never routed (see [isEndpointAudioWanted]).
*
* This is the single value read on the per-packet hot path: reading one volatile reference to an immutable map
* yields a consistent "is it known? is it synthetic?" answer with no tearing between two separate sets. It is
* rebuilt from [knownSources] under [lock] whenever sources change.
*
* Gating routing on membership here is what closes the synthetic-source leak in both directions: a synthetic
* SSRC that is not yet known (media arriving before colibri signals the source) or no longer known (media still
* arriving just after the source was removed) is absent from this map and so is dropped, rather than falling
* through to the permissive default and being forwarded as ordinary audio to non-subscribers.
*/
@Volatile
private var ssrcSynthetic: Map<Long, Boolean> = emptyMap()
private val lock: Any = Any()
/**
* Sets the audio subscription for a given endpoint.
* @param endpointId the ID of the endpoint
* @param subscription the audio subscription message
*/
fun setEndpointAudioSubscription(endpointId: String, subscription: ReceiverAudioSubscriptionMessage) =
synchronized(lock) {
val audioSubscription = audioSubscriptions.getOrPut(endpointId) {
AudioSubscription()
}
// Update subscribed local sources before updating subscription
updateSubscribedLocalAudioSourcesForEndpoint(endpointId, subscription)
// Resolve against the authoritative set of known sources, not a snapshot supplied by the caller: the caller
// gathers sources outside [lock], so a source signaled concurrently could otherwise be missed permanently.
audioSubscription.updateSubscription(subscription, knownSources.toList())
}
/*
* Checks if audio from a given SSRC is wanted by a specific endpoint.
* @param endpointId the ID of the endpoint
* @param ssrc the SSRC to check
* @return true if the audio is wanted, false otherwise
*/
/** Whether the given SSRC belongs to a (currently-known) synthetic source. */
fun isSynthetic(ssrc: Long): Boolean = ssrcSynthetic[ssrc] == true
/**
* Returns the synthetic source with the given name, or null if there is none. Resolved against [knownSources]
* under [lock] so it is consistent with [isSynthetic]/[ssrcSynthetic]: callers (e.g. injecting translated media)
* see a synthetic source as resolvable exactly while it is treated as synthetic at routing time, rather than
* relying on a separately-maintained, lock-skewed view of the conference's sources.
*/
fun findSyntheticSource(sourceName: String): AudioSourceDesc? = synchronized(lock) {
knownSources.firstOrNull { it.synthetic && it.sourceName == sourceName }
}
fun isEndpointAudioWanted(endpointId: String, ssrc: Long): Boolean {
// Single consistent read: null => not a currently-known source.
val synthetic = ssrcSynthetic[ssrc]
// Don't route audio that doesn't correspond to a currently-known source. Real audio always has a source
// signaled via colibri; an unrecognized SSRC is either a synthetic source whose desc hasn't arrived yet
// or was just removed (which must not leak as ordinary audio), or a stray SSRC -- none should be routed.
?: return false
if (synthetic) {
// Synthetic sources are never routed automatically; only to an endpoint that explicitly (Include)
// subscribed to them by name.
return audioSubscriptions[endpointId]?.isExplicitlyWanted(ssrc) ?: false
}
val subscription = audioSubscriptions[endpointId]
return subscription?.isSsrcWanted(ssrc) ?: true
}
/**
* Checks if a specific local audio source is explicitly subscribed to by any endpoint.
* @param sourceName the name of the audio source
* @return true if the source is explicitly subscribed, false otherwise
*/
fun isExplicitlySubscribed(sourceName: String?): Boolean = subscribedLocalAudioSources.containsKey(sourceName)
/**
* Updates the subscribed local audio sources for a specific endpoint based on their subscription.
* @param endpointId the ID of the endpoint
* @param subscription the audio subscription message
*/
private fun updateSubscribedLocalAudioSourcesForEndpoint(
endpointId: String,
subscription: ReceiverAudioSubscriptionMessage
) {
subscribedLocalAudioSources.values.forEach { endpointSet ->
endpointSet.remove(endpointId)
}
subscribedLocalAudioSources.entries.removeIf { it.value.isEmpty() }
// Only the explicit `include` list counts as an explicit subscription.
subscription.include.forEach { sourceName ->
subscribedLocalAudioSources.getOrPut(sourceName) { mutableSetOf() }.add(endpointId)
}
}
/**
* Called when new audio sources are added to the conference.
* @param sources the new audio source descriptions
*/
fun onSourcesAdded(sources: Set<AudioSourceDesc>) = synchronized(lock) {
knownSources.addAll(sources)
rebuildSsrcClassification()
audioSubscriptions.values.forEach { subscription ->
subscription.onConferenceSourceAdded(sources)
}
}
/** Rebuilds the [ssrcSynthetic] snapshot from [knownSources]. Must be called under [lock]. */
private fun rebuildSsrcClassification() {
ssrcSynthetic = knownSources.associate { it.ssrc to it.synthetic }
}
/**
* Called when an endpoint is removed from the conference.
* @param id the endpoint ID that was removed
*/
fun removeEndpoint(id: String) = synchronized(lock) {
// Remove the endpoint from all sets
// This is necessary to precisely maintain the number of subscriptions to a source
subscribedLocalAudioSources.values.forEach { endpointSet ->
endpointSet.remove(id)
}
subscribedLocalAudioSources.entries.removeIf { it.value.isEmpty() }
audioSubscriptions.remove(id)
}
fun removeSources(sources: Set<AudioSourceDesc>) = synchronized(lock) {
knownSources.removeAll(sources)
rebuildSsrcClassification()
subscribedLocalAudioSources.keys.removeAll(sources.mapNotNull { it.sourceName })
audioSubscriptions.values.forEach { subscription ->
subscription.onConferenceSourceRemoved(sources)
}
}
fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
synchronized(lock) {
// The conference's known audio sources, including which are synthetic. This is the authoritative set that
// routing (isEndpointAudioWanted) and subscription name resolution are gated on.
val sources = putArray("known_sources")
knownSources.forEach { src ->
sources.addObject().apply {
put("ssrc", src.ssrc)
put("name", src.sourceName)
put("owner", src.owner)
put("synthetic", src.synthetic)
}
}
// Local source names explicitly subscribed (Include) to by at least one endpoint -> the subscribers.
val subscribed = putObject("subscribed_local_sources")
subscribedLocalAudioSources.forEach { (name, endpointIds) ->
subscribed.putArray(name).apply { endpointIds.forEach { add(it) } }
}
// Per-endpoint subscription state (mode + resolved SSRCs).
val subscriptions = putObject("subscriptions")
audioSubscriptions.forEach { (endpointId, subscription) ->
subscriptions.set<ObjectNode>(endpointId, subscription.debugState())
}
}
}
}

68
jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt Normal file
View File

@@ -0,0 +1,68 @@
/*
* Copyright @ 2024 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.utils.queue.CountingErrorHandler
import org.jitsi.utils.queue.PacketQueue
import org.jitsi.videobridge.metrics.QueueMetrics
import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer
import org.jitsi.videobridge.util.TaskPools
import org.jitsi.videobridge.xmpp.XmppConnection
import java.time.Clock
import kotlin.Int.Companion.MAX_VALUE
abstract class ColibriQueue(packetHandler: PacketHandler<XmppConnection.ColibriRequest>) :
PacketQueue<XmppConnection.ColibriRequest>(
MAX_VALUE,
true,
QUEUE_NAME,
packetHandler,
TaskPools.IO_POOL,
// TODO: using the Videobridge clock breaks tests somehow
Clock.systemUTC(),
// Allow running tasks to complete (so we can close the queue from within the task).
false,
) {
init {
setErrorHandler(queueErrorCounter)
}
companion object {
val QUEUE_NAME = "colibri-queue"
val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter(
"colibri_queue_dropped_packets",
"Number of packets dropped out of the Colibri queue."
)
val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter(
"colibri_queue_exceptions",
"Number of exceptions from the Colibri queue."
)
/** Count the number of dropped packets and exceptions. */
val queueErrorCounter = object : CountingErrorHandler() {
override fun packetDropped() = super.packetDropped().also {
droppedPacketsMetric.inc()
QueueMetrics.droppedPackets.inc()
}
override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also {
exceptionsMetric.inc()
QueueMetrics.exceptions.inc()
}
}
}
}

39
jvb/src/main/kotlin/org/jitsi/videobridge/ConferenceSpeechActivityConfig.kt Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright @ 2021 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import java.time.Duration
class ConferenceSpeechActivityConfig {
val recentSpeakersCount: Int by config {
"videobridge.speech-activity.recent-speakers-count".from(JitsiConfig.newConfig)
}
val enableSilenceDetection: Boolean by config {
"videobridge.speech-activity.enable-silence-detection".from(JitsiConfig.newConfig)
}
val silenceDetectionTimeout: Duration by config {
"videobridge.speech-activity.silence-detection-timeout".from(JitsiConfig.newConfig)
}
companion object {
@JvmField
val config = ConferenceSpeechActivityConfig()
}
}

32
jvb/src/main/kotlin/org/jitsi/videobridge/ConnectionStatsConfig.kt Normal file
View File

@@ -0,0 +1,32 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import java.time.Duration
class ConnectionStatsConfig private constructor() {
companion object {
val enabled: Boolean by config(
"videobridge.connection-stats.enabled".from(JitsiConfig.newConfig)
)
val interval: Duration by config(
"videobridge.connection-stats.interval".from(JitsiConfig.newConfig)
)
}
}

31
jvb/src/main/kotlin/org/jitsi/videobridge/CryptexConfig.kt Normal file
View File

@@ -0,0 +1,31 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
class CryptexConfig private constructor() {
companion object {
val endpoint: Boolean by config(
"videobridge.cryptex.endpoint".from(JitsiConfig.newConfig)
)
val relay: Boolean by config(
"videobridge.cryptex.relay".from(JitsiConfig.newConfig)
)
}
}

1296
jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt Normal file
View File

File diff suppressed because it is too large Load Diff

47
jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusConfig.kt Normal file
View File

@@ -0,0 +1,47 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import java.time.Duration
class EndpointConnectionStatusConfig private constructor() {
val firstTransferTimeout: Duration by config {
"org.jitsi.videobridge.EndpointConnectionStatus.FIRST_TRANSFER_TIMEOUT"
.from(JitsiConfig.legacyConfig)
.convertFrom<Long>(Duration::ofMillis)
"videobridge.ep-connection-status.first-transfer-timeout".from(JitsiConfig.newConfig)
}
val maxInactivityLimit: Duration by config {
"org.jitsi.videobridge.EndpointConnectionStatus.MAX_INACTIVITY_LIMIT"
.from(JitsiConfig.legacyConfig)
.convertFrom<Long>(Duration::ofMillis)
"videobridge.ep-connection-status.max-inactivity-limit".from(JitsiConfig.newConfig)
}
val intervalMs: Long by config {
"videobridge.ep-connection-status.check-interval".from(JitsiConfig.newConfig)
.convertFrom<Duration>(Duration::toMillis)
}
companion object {
@JvmField
val config = EndpointConnectionStatusConfig()
}
}

169
jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt Normal file
View File

@@ -0,0 +1,169 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.utils.NEVER
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.cdebug
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.EndpointConnectionStatusConfig.Companion.config
import org.jitsi.videobridge.message.EndpointConnectionStatusMessage
import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer
import java.time.Clock
import java.time.Duration
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicReference
class EndpointConnectionStatusMonitor @JvmOverloads constructor(
private val conference: Conference,
private val executor: ScheduledExecutorService,
parentLogger: Logger,
private val clock: Clock = Clock.systemUTC()
) {
private val logger = createChildLogger(parentLogger)
/**
* Note that we intentionally do not prune this set when an endpoint expires, because if an endpoint expires and
* is recreated, we need to send an "active" message (the other endpoints in the conference are not aware that the
* object on the bridge was expired and recreated).
* Also note that when an endpoint is moved to another bridge, it will be expired and a RelayedEndpoint with the
* same ID will be created.
*/
private val inactiveEndpointIds = mutableSetOf<String>()
private val taskHandle = AtomicReference<ScheduledFuture<*>>(null)
fun start() {
if (taskHandle.compareAndSet(
null,
executor.scheduleWithFixedDelay(::run, config.intervalMs, config.intervalMs, TimeUnit.MILLISECONDS)
)
) {
logger.info("Starting connection status monitor")
} else {
logger.warn("Task already started, not starting again")
}
}
fun stop() {
taskHandle.getAndSet(null)?.cancel(false)
logger.info("Stopped")
}
private fun run() {
conference.localEndpoints.filter { !it.visitor }.forEach(::monitorEndpointActivity)
}
private fun monitorEndpointActivity(endpoint: Endpoint) {
val now = clock.instant()
val mostRecentChannelCreatedTime = endpoint.creationTime
val lastActivity = endpoint.lastIncomingActivity
val active: Boolean
var changed = false
if (lastActivity == NEVER) {
// Here we check if it's taking too long for the endpoint to connect
// We're doing that by checking how much time has elapsed since
// the first endpoint's channel has been created.
val timeSinceCreation = Duration.between(mostRecentChannelCreatedTime, now)
if (timeSinceCreation > config.firstTransferTimeout) {
active = false
synchronized(inactiveEndpointIds) {
val alreadyInactive = inactiveEndpointIds.contains(endpoint.id)
if (!alreadyInactive) {
logger.cdebug {
"${endpoint.id} is having trouble establishing the connection " +
"and will be marked as inactive"
}
inactiveEndpointIds += endpoint.id
changed = true
}
}
} else {
logger.cdebug { "${endpoint.id} not ready for activity checks yet" }
return
}
} else {
val noActivityTime = Duration.between(lastActivity, now)
active = noActivityTime <= config.maxInactivityLimit
synchronized(inactiveEndpointIds) {
val wasActive = !inactiveEndpointIds.contains(endpoint.id)
if (wasActive && !active) {
logger.info { "${endpoint.id} is considered disconnected. No activity for $noActivityTime" }
inactiveEndpointIds += endpoint.id
endpointsDisconnected.inc()
changed = true
} else if (!wasActive && active) {
logger.info { "${endpoint.id} has reconnected" }
inactiveEndpointIds -= endpoint.id
endpointsReconnected.inc()
changed = true
}
}
}
if (changed) {
notifyStatusChange(endpoint.id, active, null)
}
}
private fun notifyStatusChange(subjectEpId: String, isConnected: Boolean, receiverEpId: String?) {
val msg = EndpointConnectionStatusMessage(subjectEpId, isConnected)
if (receiverEpId == null) {
// We broadcast the message also to the endpoint itself for
// debugging purposes, and we also send it to Relays.
conference.broadcastMessage(msg, true)
} else {
conference.getLocalEndpoint(receiverEpId)?.let {
conference.sendMessage(msg, listOf(it), false)
}
}
}
/**
* Notify this [EndpointConnectionStatusMonitor] that an endpoint in the conference has
* connected
*/
fun endpointConnected(endpointId: String) {
synchronized(inactiveEndpointIds) {
val localEndpointIds = conference.localEndpoints.filter { !it.visitor }.map { it.id }
inactiveEndpointIds.forEach { inactiveEpId ->
// inactiveEndpointIds may contain endpoints that have already expired and/or moved to another bridge.
if (localEndpointIds.contains(inactiveEpId)) {
notifyStatusChange(inactiveEpId, isConnected = false, receiverEpId = endpointId)
}
}
}
}
companion object {
@JvmField
val endpointsDisconnected = VideobridgeMetricsContainer.instance.registerCounter(
"endpoints_disconnected",
"Endpoints detected as temporarily inactive/disconnected due to inactivity."
)
@JvmField
val endpointsReconnected = VideobridgeMetricsContainer.instance.registerCounter(
"endpoints_reconnected",
"Endpoints reconnected after being detected as temporarily inactive/disconnected due to inactivity."
)
}
}

42
jvb/src/main/kotlin/org/jitsi/videobridge/JvbLastN.kt Normal file
View File

@@ -0,0 +1,42 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import java.util.function.Supplier
/**
* A JVB-wide last-n value which will be observed by all endpoints on this bridge.
*
* A value of -1 means no limit is enforced.
*/
class JvbLastN : Supplier<Int> {
private val defaultJvbLastN: Int by config("videobridge.cc.jvb-last-n".from(JitsiConfig.newConfig))
var jvbLastN: Int = defaultJvbLastN
override fun get(): Int = jvbLastN
}
@JvmField
val jvbLastNSingleton: JvbLastN = JvbLastN()
fun calculateLastN(vararg lastN: Int): Int {
val min = lastN.map { if (it == -1) Int.MAX_VALUE else it }.minOrNull() ?: Int.MAX_VALUE
return if (min == Int.MAX_VALUE) -1 else min
}

50
jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt Normal file
View File

@@ -0,0 +1,50 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import java.time.Duration
class LoudestConfig private constructor() {
companion object {
@JvmStatic
val routeLoudestOnly: Boolean by config(
"videobridge.loudest.route-loudest-only".from(JitsiConfig.newConfig)
)
@JvmStatic
val numLoudest: Int by config(
"videobridge.loudest.num-loudest".from(JitsiConfig.newConfig)
)
@JvmStatic
val alwaysRouteDominant: Boolean by config(
"videobridge.loudest.always-route-dominant".from(JitsiConfig.newConfig)
)
@JvmStatic
val energyExpireTime: Duration by config(
"videobridge.loudest.energy-expire-time".from(JitsiConfig.newConfig)
)
@JvmStatic
val energyAlphaPct: Int by config(
"videobridge.loudest.energy-alpha-pct".from(JitsiConfig.newConfig)
)
}
}

274
jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt Normal file
View File

@@ -0,0 +1,274 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.eclipse.jetty.ee10.servlet.ServletHolder
import org.glassfish.jersey.servlet.ServletContainer
import org.ice4j.ice.harvest.AbstractUdpListener
import org.ice4j.ice.harvest.MappingCandidateHarvesters
import org.ice4j.util.Buffer
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.ConfigException
import org.jitsi.metaconfig.MetaconfigLogger
import org.jitsi.metaconfig.MetaconfigSettings
import org.jitsi.nlj.dtls.DtlsConfig
import org.jitsi.rest.JettyBundleActivatorConfig
import org.jitsi.rest.createServer
import org.jitsi.rest.enableCors
import org.jitsi.rest.isEnabled
import org.jitsi.rest.servletContextHandler
import org.jitsi.rtp.Packet
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.shutdown.ShutdownServiceImpl
import org.jitsi.utils.logging2.LoggerImpl
import org.jitsi.utils.queue.PacketQueue
import org.jitsi.videobridge.ice.Harvesters
import org.jitsi.videobridge.metrics.Metrics
import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics
import org.jitsi.videobridge.rest.root.Application
import org.jitsi.videobridge.stats.MucPublisher
import org.jitsi.videobridge.util.ByteBufferPool
import org.jitsi.videobridge.util.TaskPools
import org.jitsi.videobridge.util.UlimitCheck
import org.jitsi.videobridge.version.JvbVersionService
import org.jitsi.videobridge.websocket.ColibriWebSocketService
import org.jitsi.videobridge.xmpp.XmppConnection
import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig
import sun.misc.Signal
import java.time.Clock
import kotlin.concurrent.thread
import kotlin.system.exitProcess
import org.jitsi.videobridge.websocket.singleton as webSocketServiceSingleton
fun main() {
val logger = LoggerImpl("org.jitsi.videobridge.Main")
Thread.setDefaultUncaughtExceptionHandler { thread, exception ->
logger.error("An uncaught exception occurred in thread=$thread", exception)
}
setupMetaconfigLogger()
setSystemPropertyDefaults()
// Some of our dependencies bring in slf4j, which means Jetty will default to using
// slf4j as its logging backend. The version of slf4j brought in, however, is too old
// for Jetty so it throws errors. We use java.util.logging so tell Jetty to use that
// as its logging backend.
// TODO: Instead of setting this here, we should integrate it with the infra/debian scripts
// to be passed.
System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.JavaUtilLog")
Metrics.start()
// Reload the Typesafe config used by ice4j, because the original was initialized before the new system
// properties were set.
JitsiConfig.reloadNewConfig()
logger.info("Starting jitsi-videobridge version ${JvbVersionService.instance.currentVersion}")
UlimitCheck.printUlimits()
startIce4j()
setupBufferPools()
// Initialize, binding on the main ICE port.
Harvesters.init()
org.jitsi.videobridge.xmpp.Smack.initialize()
PacketQueue.setEnableStatisticsDefault(true)
// Trigger an exception early in case the DTLS cipher suites are misconfigured
try {
DtlsConfig.config.cipherSuites
} catch (ce: ConfigException) {
logger.error("Dtls configuration error: $ce")
// According to https://freedesktop.org/software/systemd/man/systemd.exec…html#Process%20Exit%20Code
// 78 means "configuration error"
exitProcess(78)
}
val xmppConnection = XmppConnection().apply { start() }
val shutdownService = ShutdownServiceImpl()
val videobridge = Videobridge(
xmppConnection,
shutdownService,
JvbVersionService.instance.currentVersion,
Clock.systemUTC()
)
val videobridgeExpireThread = VideobridgeExpireThread(videobridge).apply { start() }
Metrics.metricsUpdater.addUpdateTask {
VideobridgePeriodicMetrics.update(videobridge)
}
val healthChecker = videobridge.jvbHealthChecker
val presencePublisher = MucPublisher(
TaskPools.SCHEDULED_POOL,
XmppClientConnectionConfig.config.presenceInterval,
xmppConnection
).apply { start() }
val publicServerConfig = JettyBundleActivatorConfig(
"org.jitsi.videobridge.rest",
"videobridge.http-servers.public"
)
val publicHttpServer = if (publicServerConfig.isEnabled()) {
logger.info("Starting public http server")
val websocketService = ColibriWebSocketService(publicServerConfig.isTls)
webSocketServiceSingleton().setColibriWebSocketService(websocketService)
createServer(publicServerConfig).also {
websocketService.registerServlet(it.servletContextHandler, videobridge)
it.start()
}
} else {
logger.info("Not starting public http server")
null
}
val privateServerConfig = JettyBundleActivatorConfig(
"org.jitsi.videobridge.rest.private",
"videobridge.http-servers.private"
)
val privateHttpServer = if (privateServerConfig.isEnabled()) {
logger.info("Starting private http server")
val restApp = Application(
videobridge,
xmppConnection,
JvbVersionService.instance.currentVersion,
healthChecker
)
createServer(privateServerConfig).also {
it.servletContextHandler.addServlet(
ServletHolder(ServletContainer(restApp)),
"/*"
)
it.servletContextHandler.enableCors()
it.start()
}
} else {
logger.info("Not starting private http server")
null
}
var exitStatus = 0
/* Catch signals and cause them to trigger a clean shutdown. */
listOf("TERM", "HUP", "INT").forEach { signalName ->
try {
Signal.handle(Signal(signalName)) { signal ->
exitStatus = signal.number + 128 // Matches java.lang.Terminator
logger.info("Caught signal $signal, shutting down.")
shutdownService.beginShutdown()
}
} catch (e: IllegalArgumentException) {
/* Unknown signal on this platform, or not allowed to register this signal; that's fine. */
logger.warn("Unable to register signal '$signalName'", e)
}
}
// Block here until the bridge shuts down
shutdownService.waitForShutdown()
logger.info("Bridge shutting down")
healthChecker.stop()
presencePublisher.stop()
xmppConnection.stop()
try {
publicHttpServer?.stop()
privateHttpServer?.stop()
} catch (t: Throwable) {
logger.error("Error shutting down http servers", t)
}
videobridgeExpireThread.stop()
videobridge.stop()
stopIce4j()
Metrics.stop()
TaskPools.SCHEDULED_POOL.shutdownNow()
TaskPools.CPU_POOL.shutdownNow()
TaskPools.IO_POOL.shutdownNow()
exitProcess(exitStatus)
}
private fun setupMetaconfigLogger() {
val configLogger = LoggerImpl("org.jitsi.config")
MetaconfigSettings.logger = object : MetaconfigLogger {
override fun warn(block: () -> String) = configLogger.warn(block)
override fun error(block: () -> String) = configLogger.error(block)
override fun debug(block: () -> String) = configLogger.debug(block)
}
}
private fun setSystemPropertyDefaults() {
val defaults = getSystemPropertyDefaults()
defaults.forEach { (key, value) ->
if (System.getProperty(key) == null) {
System.setProperty(key, value)
}
}
}
private fun getSystemPropertyDefaults(): Map<String, String> {
val defaults = mutableMapOf<String, String>()
// Make legacy ice4j properties system properties.
val cfg = JitsiConfig.SipCommunicatorProps
val ice4jPropNames = cfg.getPropertyNamesByPrefix("org.ice4j", false)
ice4jPropNames?.forEach { key ->
cfg.getString(key)?.let { value ->
defaults.put(key, value)
}
}
return defaults
}
private fun startIce4j() {
AbstractUdpListener.USE_PUSH_API = true
// Start the initialization of the mapping candidate harvesters.
// Asynchronous, because the AWS and STUN harvester may take a long
// time to initialize.
thread(start = true) {
MappingCandidateHarvesters.initialize()
}
}
private fun stopIce4j() {
// Shut down harvesters.
Harvesters.close()
}
/** Configure our libraries to use JVB's global [ByteBufferPool] */
private fun setupBufferPools() {
org.jitsi.rtp.util.BufferPool.getArray = { ByteBufferPool.getBuffer(it) }
org.jitsi.rtp.util.BufferPool.returnArray = { ByteBufferPool.returnBuffer(it) }
org.jitsi.nlj.util.BufferPool.getBuffer = { ByteBufferPool.getBuffer(it) }
org.jitsi.nlj.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it) }
org.ice4j.util.BufferPool.getBuffer = { len ->
val b = ByteBufferPool.getBuffer(len)
Buffer(b, 0, b.size)
}
org.ice4j.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it.buffer) }
org.ice4j.ice.harvest.AbstractUdpListener.BYTES_TO_LEAVE_AT_START_OF_PACKET =
RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET
org.ice4j.ice.harvest.AbstractUdpListener.BYTES_TO_LEAVE_AT_END_OF_PACKET =
Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET
}

71
jvb/src/main/kotlin/org/jitsi/videobridge/RecentSpeakersList.kt Normal file
View File

@@ -0,0 +1,71 @@
/*
* Copyright @ 2021 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import java.util.LinkedList
/**
* Maintain a list of recent speakers. A speaker can be [promote]d, bumping it to the top of the list. The list of the
* top [numRecentSpeakers], ordered by the time they were promoted, is exposed as [recentSpeakers].
*
* Removal is possible via [removeAllExcept].
*/
class RecentSpeakersList<T>(
/**
* The number of speakers to expose via [recentSpeakers]. Internally we store all speakers to allow them to be
* removed without losing information about the order.
*/
private val numRecentSpeakers: Int
) {
/**
* Store the history of all promoted speakers.
*/
val list: MutableList<T> = LinkedList()
/**
* Promote [endpoint] to the top of the list. If it is already in the list it is moved to the top, otherwise it's
* added to the top.
*
* @return true iff [recentSpeakers] changed as a result of this call.
*/
fun promote(endpoint: T): Boolean {
val index = list.indexOf(endpoint)
if (index >= 0) {
list.removeAt(index)
}
list.add(0, endpoint)
return index != 0
}
/**
* Remove all entries from the list except the ones contained in [retain].
*
* @return true iff [recentSpeakers] changed as a result of this call.
*/
fun removeAllExcept(retain: List<T>): Boolean {
val prevRecentSpeakers = recentSpeakers
list.removeIf { !retain.contains(it) }
return recentSpeakers != prevRecentSpeakers
}
/**
* The first [numRecentSpeakers] in the list, ordered by the time they were promoted.
*/
val recentSpeakers: List<T>
get() = list.take(numRecentSpeakers)
fun isRecentSpeaker(endpoint: T) = recentSpeakers.contains(endpoint)
}

786
jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt Normal file
View File

@@ -0,0 +1,786 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.VideoType
import org.jitsi.nlj.codec.vpx.VpxUtils
import org.jitsi.nlj.rtp.SsrcAssociationType
import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket
import org.jitsi.nlj.rtp.codec.av1.applyTemplateIdDelta
import org.jitsi.nlj.rtp.codec.av1.getTemplateIdDelta
import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet
import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet
import org.jitsi.rtp.rtcp.RtcpPacket
import org.jitsi.rtp.rtcp.RtcpSrPacket
import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.rtp.util.RtpUtils
import org.jitsi.utils.LRUCache
import org.jitsi.utils.MediaType
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.message.AudioSourceMapping
import org.jitsi.videobridge.message.AudioSourcesMap
import org.jitsi.videobridge.message.BridgeChannelMessage
import org.jitsi.videobridge.message.VideoSourceMapping
import org.jitsi.videobridge.message.VideoSourcesMap
import org.jitsi.videobridge.relay.AudioSourceDesc
import java.util.concurrent.ConcurrentHashMap
/**
* Align common fields from different source types.
* Perhaps this could become a base class of those types.
*/
class SourceDesc private constructor(
val name: String,
val owner: String,
val videoType: VideoType,
val ssrc1: Long,
val ssrc2: Long
) {
constructor(s: AudioSourceDesc) : this(
s.sourceName ?: "anon",
s.owner ?: "unknown",
VideoType.DISABLED,
s.ssrc,
-1
)
constructor(s: MediaSourceDesc) : this(s.sourceName, s.owner, s.videoType, s.primarySSRC, getRtx(s))
companion object {
fun getRtx(s: MediaSourceDesc): Long {
/* Ignoring any additional entries for now. */
return if (s.rtpEncodings.isEmpty()) -1 else s.rtpEncodings[0].getSecondarySsrc(SsrcAssociationType.RTX)
}
}
/**
* {@inheritDoc}
*/
override fun toString(): String = "$owner:$ssrc1/$ssrc2"
}
/**
* Some RTP header state to track.
*/
class RtpState {
var lastSequenceNumber = 0
var lastTimestamp = 0L
var codecState: CodecState? = null
var av1PersistentState: CodecState? = null // TODO? Generalize if needed in the future
var valid = false
fun update(packet: RtpPacket) {
lastSequenceNumber = packet.sequenceNumber
lastTimestamp = packet.timestamp
codecState = packet.getCodecState()
if (packet is Av1DDPacket) {
av1PersistentState = codecState
}
valid = true
}
/**
* {@inheritDoc}
*/
override fun toString(): String = if (valid) "$lastSequenceNumber/$lastTimestamp/$codecState" else "-"
}
/**
* RTP state for received SSRCs.
*/
class ReceiveSsrc(val props: SourceDesc) {
val state = RtpState()
/**
* True when sequence number and timestamp deltas have been calculated.
* If false, calculate them on the next relayed packet.
*/
var hasDeltas = false
/**
* {@inheritDoc}
*/
override fun toString(): String = "$state" + if (hasDeltas) "" else " (no \u2206)"
}
/**
* RTP state for sent SSRCs.
*/
class SendSsrc(val ssrc: Long) {
private val state = RtpState()
private var sequenceNumberDelta = 0
private var timestampDelta = 0L
private var codecDeltas: CodecDeltas? = null
/**
* Update RTP state and apply deltas.
*/
fun rewriteRtp(packet: RtpPacket, sending: Boolean, recv: ReceiveSsrc) {
if (sending) {
if (!recv.hasDeltas) {
/* Calculate new deltas the first time a receive ssrc is mapped to a send ssrc. */
if (state.valid) {
if (recv.state.valid) {
sequenceNumberDelta =
RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, recv.state.lastSequenceNumber)
timestampDelta =
RtpUtils.getTimestampDiff(state.lastTimestamp, recv.state.lastTimestamp)
codecDeltas = state.codecState?.getDeltas(
recv.state.codecState
)
} else {
val prevSequenceNumber =
RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, -1)
val prevTimestamp =
RtpUtils.applyTimestampDelta(packet.timestamp, -960) // guessing
sequenceNumberDelta =
RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, prevSequenceNumber)
timestampDelta =
RtpUtils.getTimestampDiff(state.lastTimestamp, prevTimestamp)
codecDeltas = state.codecState?.getDeltas(packet)
}
}
recv.hasDeltas = true
}
if (packet is Av1DDPacket && state.codecState !is Av1DDCodecState && state.av1PersistentState != null) {
codecDeltas = state.av1PersistentState?.getDeltas(packet)
}
recv.state.update(packet)
packet.ssrc = ssrc
packet.sequenceNumber = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, sequenceNumberDelta)
packet.timestamp = RtpUtils.applyTimestampDelta(packet.timestamp, timestampDelta)
codecDeltas?.rewritePacket(packet)
state.update(packet)
} else {
/* Don't touch send state if we're dropping the packet. */
recv.state.update(packet)
}
}
/**
* Fix SSRC and timestamps in an RTCP packet.
* For packets in the same direction as media flow; feedback messages handled separately.
*/
fun rewriteRtcp(packet: RtcpPacket) {
packet.senderSsrc = ssrc
if (packet is RtcpSrPacket) {
packet.senderInfo.rtpTimestamp = RtpUtils.applyTimestampDelta(
packet.senderInfo.rtpTimestamp,
timestampDelta
)
}
}
/**
* {@inheritDoc}
*/
override fun toString(): String = "$ssrc{$state,\u2206=$sequenceNumberDelta/$timestampDelta/$codecDeltas}"
}
/**
* Associates primary and secondary send SSRCs.
* Primary constructor preserves state of the existing send SSRCs.
*/
class SendSource(val props: SourceDesc, val send1: SendSsrc, val send2: SendSsrc, val mid: String? = null) {
/**
* If false, do not send on this SSRC until a packet with start=true arrives.
*/
private var started = false
/**
* Create object with new send SSRCs.
*/
constructor(props: SourceDesc, ssrc1: Long, ssrc2: Long) : this(props, SendSsrc(ssrc1), SendSsrc(ssrc2))
/**
* Demux to proper SSRC.
*/
private fun getSender(ssrc: Long) = if (ssrc == props.ssrc2) send2 else send1
/**
* Update RTP state and apply deltas.
* Returns true if packet should be sent.
*/
fun rewriteRtp(packet: RtpPacket, start: Boolean, recv: ReceiveSsrc): Boolean {
if (start) {
started = true
}
getSender(packet.ssrc).rewriteRtp(packet, started, recv)
return started
}
/**
* Fix SSRC and timestamps in an RTCP packet.
* For packets in the same direction as media flow; feedback messages handled separately.
*/
fun rewriteRtcp(packet: RtcpPacket) {
getSender(packet.senderSsrc).rewriteRtcp(packet)
}
/**
* {@inheritDoc}
*/
override fun toString(): String = "$send1, $send2" + if (started) "" else " (not started)"
}
/**
* Interface between an SsrcCache and its owner.
*/
interface SsrcRewriter {
/**
* Find the properties of the video source indicated by the given SSRC. Returns null if not found.
*/
fun findVideoSourceProps(ssrc: Long): MediaSourceDesc?
/**
* Find the properties of the audio source indicated by the given SSRC. Returns null if not found.
*/
fun findAudioSourceProps(ssrc: Long): AudioSourceDesc?
/**
* Get a unique send SSRC.
*/
fun getNextSendSsrc(): Long
/**
* Sends a specific message to this endpoint over its bridge channel.
*/
fun sendMessage(msg: BridgeChannelMessage)
}
/**
* Limit the number of local SSRCs used for the given media type to the number specified.
* If there are more sources in the conference than the limit, then the least recently used
* SSRCs are remapped. RTP packets have their header fields rewritten so the stream appears
* to be a continuation of an already advertised SSRC.
*/
abstract class SsrcCache(
val size: Int,
val ep: SsrcRewriter,
/** Whether to assign and stamp a mid per slot for mid-based demuxing on the client. */
val midDemux: Boolean,
val parentLogger: Logger,
label: String
) {
private val logger = createChildLogger(parentLogger).apply { addContext("type", label) }
/** The prefix used for the mids assigned to slots of this cache (e.g. "a" for audio, "v" for video). */
protected abstract val midPrefix: String
/**
* Maps a (send-side) SSRC to the mid of the slot it belongs to. Populated when [midDemux] is enabled. Both the
* primary and RTX send SSRCs of a slot map to the same mid. Read from the sender pipeline thread (lock-free) while
* written under the [sendSources] lock, hence the concurrent map.
*/
private val sendSsrcToMid = ConcurrentHashMap<Long, String>()
/** The next index to use when assigning a mid to a newly-created slot. */
private var nextMidIndex = 0
/**
* All remote SSRCs that have been seen.
*/
private val receivedSsrcs = HashMap<Long, ReceiveSsrc>()
/**
* The most recently forwarded remote SSRC groups. If this list is full and a new remote SSRC needs to be
* forwarded to the endpoint, the element at the front of this list will be removed and that element's
* local SSRC will be used. Note: indexed by primary SSRC.
*/
private val sendSources = LRUCache<Long, SendSource>(
size,
// accessOrder
true
)
/**
* Whether an incoming RTP packet can automatically activate its source (i.e. acquire a send SSRC).
* If false, sources must be activated using the activate() method.
*/
protected abstract val allowCreateOnPacket: Boolean
/**
* Number of times an SSRC has changed its mapping. (Does not count initial mappings.)
*/
private var remapCount = 0
/**
* Find the properties of the source indicated by the given SSRC. Returns null if not found.
*/
protected abstract fun findSourceProps(ssrc: Long): SourceDesc?
/**
* Notify new SSRC mappings to the client. The list will contain at least one element.
*/
protected abstract fun notifyMappings(sources: List<SendSource>)
/**
* Assign a group of send SSRCs to use for the specified source.
* If remapping the send SSRCs from another source, transfer RTP state from the old source.
* Returns null if no current mapping exists and allowCreate is false.
* Otherwise, returns the send source information to use.
*/
private fun getSendSource(
ssrc: Long,
props: SourceDesc,
allowCreate: Boolean,
/* Collect remapped sources into this list, if provided. */
remappings: MutableList<SendSource>? = null
): SendSource? {
/* Moves to end of LRU when found. */
var sendSource = sendSources[ssrc]
if (sendSource == null) {
if (!allowCreate) {
return null
}
if (sendSources.size == size) {
val eldest = sendSources.eldest()
/* Reuse the eldest slot's send SSRCs *and* its mid: the mid is bound to the slot (i.e. to the send
SSRCs), not to the source, so it stays stable across remaps. */
sendSource = SendSource(props, eldest.value.send1, eldest.value.send2, eldest.value.mid)
logger.debug { "Remapping SSRC: ${props.ssrc1}->$sendSource. ${eldest.key}->inactive" }
/* Request new deltas on next sent packet */
receivedSsrcs[props.ssrc1]?.hasDeltas = false
if (props.ssrc2 != -1L) {
receivedSsrcs[props.ssrc2]?.hasDeltas = false
}
++remapCount
} else {
val ssrc1 = ep.getNextSendSsrc()
val ssrc2 = ep.getNextSendSsrc()
val mid = if (midDemux) "$midPrefix${nextMidIndex++}" else null
sendSource = SendSource(props, SendSsrc(ssrc1), SendSsrc(ssrc2), mid)
logger.debug { "Added send SSRC: ${props.ssrc1}->$sendSource" }
}
sendSources[ssrc] = sendSource
/* Bind both the primary and RTX send SSRCs to the slot's mid so the stamper can resolve either. */
sendSource.mid?.let { mid ->
sendSsrcToMid[sendSource.send1.ssrc] = mid
sendSsrcToMid[sendSource.send2.ssrc] = mid
}
remappings?.add(sendSource)
}
return sendSource
}
/**
* Returns the mid assigned to the slot whose (send-side) SSRC is [ssrc], or null if [ssrc] is not a slot SSRC of
* this cache or mid demux is disabled.
*/
fun getMidBySsrc(ssrc: Long): String? = sendSsrcToMid[ssrc]
/**
* Remove all send sources owned by [owner]. Does not signal anything to the client, as only additions need to be
* signaled.
*/
fun removeByOwner(owner: String) {
synchronized(sendSources) {
sendSources.values.removeIf { sendSource ->
sendSource.props.owner == owner
}
}
}
/**
* Assign send SSRCs to the given sources. Any remapped SSRCs will be notified to the client.
*/
fun activate(sources: List<MediaSourceDesc>) {
val remappings = mutableListOf<SendSource>()
synchronized(sendSources) {
/* Before creating any new send sources, do a first pass that
touches the already active sources, to prevent them from being
bumped out of the LRU by earlier elements of the list. */
sources.filter { source ->
sendSources[source.primarySSRC] == null
}.forEach { source ->
getSendSource(source.primarySSRC, SourceDesc(source), allowCreate = true, remappings)
}
logger.debug { this.toString() }
}
if (remappings.isNotEmpty()) {
notifyMappings(remappings)
}
}
/**
* Send all current mappings to the endpoint.
* Can be used to resynchronize after message transport reconnects.
*/
fun sendAllMappings() {
val remappings: List<SendSource>
synchronized(sendSources) {
remappings = sendSources.values.toList()
}
if (remappings.isNotEmpty()) {
notifyMappings(remappings)
}
}
/**
* Rewrite RTP fields for a relayed packet.
* Activates a send SSRC if necessary.
* @param packet the packet about to be sent.
* @param start whether this packet can be the first packet sent on a new SSRC mapping.
* @return whether to send this packet.
*/
fun rewriteRtp(packet: RtpPacket, start: Boolean = true): Boolean {
val remappings = mutableListOf<SendSource>()
var send = false
try {
synchronized(sendSources) {
var rs = receivedSsrcs[packet.ssrc]
if (rs == null) {
val props = findSourceProps(packet.ssrc) ?: return false
rs = ReceiveSsrc(props)
receivedSsrcs[packet.ssrc] = rs
logger.debug { "Added receive SSRC: ${packet.ssrc}" }
}
val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreateOnPacket, remappings)
if (ss != null) {
send = ss.rewriteRtp(packet, start, rs)
}
}
if (remappings.isNotEmpty()) {
notifyMappings(remappings)
}
} catch (e: Exception) {
logger.error("Error rewriting SSRC, packet $packet", e)
send = false
}
return send
}
/**
* Fix SSRC and timestamps in an RTCP packet.
* For packets in the same direction as media flow; feedback messages handled separately.
*/
fun rewriteRtcp(packet: RtcpPacket): Boolean {
synchronized(sendSources) {
/* Don't activate a source on RTCP. */
val rs = receivedSsrcs[packet.senderSsrc] ?: return false
val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreate = false) ?: return false
ss.rewriteRtcp(packet)
return true
}
}
/**
* Change the SSRC in an RTCP Feedback packet to the receive-side SSRC.
* Return the endpoint identifier of the source owner.
* If the SSRC in the packet does not refer to an active source,
* then do not modify the packet and return null.
*/
fun unmapRtcpFbSsrc(packet: RtcpFbPacket): String? {
val mediaSsrc = packet.mediaSourceSsrc
synchronized(sendSources) {
val ss = sendSources.values.find { sendSource ->
if (mediaSsrc == sendSource.send1.ssrc) {
packet.mediaSourceSsrc = sendSource.props.ssrc1
return@find true
}
if (mediaSsrc == sendSource.send2.ssrc) {
packet.mediaSourceSsrc = sendSource.props.ssrc2
return@find true
}
return@find false
}
if (ss != null) {
logger.debug {
"Received RTCP FB packet. " +
"Translated send SSRC $mediaSsrc to receive SSRC ${packet.mediaSourceSsrc}. " +
"Owner = ${ss.props.owner}."
}
return ss.props.owner
} else {
logger.debug { "Received RTCP FB packet for SSRC $mediaSsrc. Not active." }
return null
}
}
}
/**
* Returns JSON statistics useful for debugging.
*/
fun getDebugState(): ObjectNode {
synchronized(sendSources) {
return JsonNodeFactory.instance.objectNode().apply {
put("max", this@SsrcCache.size)
put("received", receivedSsrcs.size)
put("sent", sendSources.size)
put("remappings", remapCount)
}
}
}
/**
* {@inheritDoc}
*/
override fun toString(): String = "SSRCs: received=" +
receivedSsrcs.entries.joinToString(", ", "[", "]") {
"(${it.key}->${it.value})"
} +
" mappings=" +
sendSources.entries.joinToString(", ", "[", "]") {
"(${it.key}->${it.value})"
}
}
/**
* SSRC Cache for audio.
*/
class AudioSsrcCache(size: Int, ep: SsrcRewriter, midDemux: Boolean, parentLogger: Logger) :
SsrcCache(size, ep, midDemux, parentLogger, MediaType.AUDIO.toString()) {
/* Switching occurs on received packets */
override val allowCreateOnPacket = true
override val midPrefix = "a"
/**
* {@inheritDoc}
*/
override fun findSourceProps(ssrc: Long): SourceDesc? {
val p = ep.findAudioSourceProps(ssrc)
return if (p?.sourceName == null || p.owner == null) {
null
} else {
SourceDesc(p)
}
}
/**
* {@inheritDoc}
*/
override fun notifyMappings(sources: List<SendSource>) {
sources.map {
val props = it.props
AudioSourceMapping(props.name, props.owner, it.send1.ssrc, it.mid)
}.also {
ep.sendMessage(AudioSourcesMap(it))
}
}
}
/**
* SSRC Cache for video.
*/
class VideoSsrcCache(size: Int, ep: SsrcRewriter, midDemux: Boolean, parentLogger: Logger) :
SsrcCache(size, ep, midDemux, parentLogger, MediaType.VIDEO.toString()) {
/* Switching triggered by activate() method only */
override val allowCreateOnPacket = false
override val midPrefix = "v"
/**
* {@inheritDoc}
*/
override fun findSourceProps(ssrc: Long): SourceDesc? = ep.findVideoSourceProps(ssrc)?.let { SourceDesc(it) }
/**
* {@inheritDoc}
*/
override fun notifyMappings(sources: List<SendSource>) {
sources.map {
val props = it.props
VideoSourceMapping(props.name, props.owner, it.send1.ssrc, it.send2.ssrc, props.videoType, it.mid)
}.also {
ep.sendMessage(VideoSourcesMap(it))
}
}
}
/** Codec-specific packet state. */
interface CodecState {
fun getDeltas(otherState: CodecState?): CodecDeltas?
fun getDeltas(packet: RtpPacket): CodecDeltas?
}
/** Codec-specific packet deltas. */
interface CodecDeltas {
fun rewritePacket(packet: RtpPacket)
}
private class Vp8CodecState(val lastTl0Index: Int) : CodecState {
constructor(packet: Vp8Packet) : this(packet.TL0PICIDX)
override fun getDeltas(otherState: CodecState?): CodecDeltas? {
if (otherState !is Vp8CodecState) {
return null
}
val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, otherState.lastTl0Index)
return Vp8CodecDeltas(tl0IndexDelta)
}
override fun getDeltas(packet: RtpPacket): CodecDeltas? {
if (packet !is Vp8Packet) {
return null
}
val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(
lastTl0Index,
VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, -1)
)
return Vp8CodecDeltas(tl0IndexDelta)
}
override fun toString() = "[VP8 TL0Idx]$lastTl0Index"
}
private class Vp8CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas {
override fun rewritePacket(packet: RtpPacket) {
if (packet !is Vp8Packet) {
return
}
packet.TL0PICIDX = VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, tl0IndexDelta)
}
override fun toString() = "[VP8 TL0Idx]$tl0IndexDelta"
}
private class Vp9CodecState(val lastTl0Index: Int) : CodecState {
constructor(packet: Vp9Packet) : this(packet.TL0PICIDX)
override fun getDeltas(otherState: CodecState?): CodecDeltas? {
if (otherState !is Vp9CodecState) {
return null
}
val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, otherState.lastTl0Index)
return Vp9CodecDeltas(tl0IndexDelta)
}
override fun getDeltas(packet: RtpPacket): CodecDeltas? {
if (packet !is Vp9Packet) {
return null
}
val tl0IndexDelta = if (packet.hasTL0PICIDX) {
VpxUtils.getTl0PicIdxDelta(
lastTl0Index,
VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, -1)
)
} else {
0
}
return Vp9CodecDeltas(tl0IndexDelta)
}
override fun toString() = "[VP9 TL0Idx]$lastTl0Index"
}
private class Vp9CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas {
override fun rewritePacket(packet: RtpPacket) {
if (packet !is Vp9Packet) {
return
}
if (packet.hasTL0PICIDX) {
packet.TL0PICIDX = VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, tl0IndexDelta)
}
}
override fun toString() = "[VP9 TL0Idx]$tl0IndexDelta"
}
private class Av1DDCodecState : CodecState {
val lastFrameNum: Int
val nextTemplateIdx: Int
constructor(lastFrameNum: Int, lastTemplateIdx: Int) {
this.lastFrameNum = lastFrameNum
this.nextTemplateIdx = lastTemplateIdx
}
constructor(packet: Av1DDPacket) {
val descriptor = packet.descriptor
requireNotNull(descriptor) { "AV1 Packet being routed must have non-null descriptor" }
this.lastFrameNum = packet.frameNumber
this.nextTemplateIdx = descriptor.structure.templateIdOffset + descriptor.structure.templateCount
}
override fun getDeltas(otherState: CodecState?): CodecDeltas? {
if (otherState !is Av1DDCodecState) {
return null
}
val frameNumDelta = RtpUtils.getSequenceNumberDelta(lastFrameNum, otherState.lastFrameNum)
val templateIdDelta = getTemplateIdDelta(nextTemplateIdx, otherState.nextTemplateIdx)
return Av1DDCodecDeltas(frameNumDelta, templateIdDelta)
}
override fun getDeltas(packet: RtpPacket): CodecDeltas? {
if (packet !is Av1DDPacket) {
return null
}
val descriptor = packet.descriptor ?: return null
val frameNumDelta = RtpUtils.getSequenceNumberDelta(
lastFrameNum,
RtpUtils.applySequenceNumberDelta(packet.frameNumber, -1)
)
val templateIdDelta = getTemplateIdDelta(nextTemplateIdx, descriptor.structure.templateIdOffset)
return Av1DDCodecDeltas(frameNumDelta, templateIdDelta)
}
override fun toString() = "[Av1DD FrameNum]$lastFrameNum [Av1DD TemplateIdx]$nextTemplateIdx"
}
private class Av1DDCodecDeltas(val frameNumDelta: Int, val templateIdDelta: Int) : CodecDeltas {
override fun rewritePacket(packet: RtpPacket) {
if (packet !is Av1DDPacket) {
return
}
val descriptor = packet.descriptor
requireNotNull(descriptor)
descriptor.frameNumber = RtpUtils.applySequenceNumberDelta(descriptor.frameNumber, frameNumDelta)
descriptor.frameDependencyTemplateId =
applyTemplateIdDelta(descriptor.frameDependencyTemplateId, templateIdDelta)
descriptor.structure.templateIdOffset =
applyTemplateIdDelta(descriptor.structure.templateIdOffset, templateIdDelta)
packet.reencodeDdExt()
}
override fun toString() = "[AV1DD FrameNum]$frameNumDelta [Av1DD templateId]$templateIdDelta"
}
private fun RtpPacket.getCodecState(): CodecState? = when {
this is Vp8Packet && isRewritable() -> Vp8CodecState(this)
this is Vp9Packet -> Vp9CodecState(this)
this is Av1DDPacket -> Av1DDCodecState(this)
else -> null
}
private fun Vp8Packet.isRewritable(): Boolean = hasTL0PICIDX

34
jvb/src/main/kotlin/org/jitsi/videobridge/SsrcLimitConfig.kt Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright @ 2022 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
class SsrcLimitConfig private constructor() {
val maxVideoSsrcs: Int by config {
"videobridge.ssrc-limit.video".from(JitsiConfig.newConfig)
}
val maxAudioSsrcs: Int by config {
"videobridge.ssrc-limit.audio".from(JitsiConfig.newConfig)
}
companion object {
@JvmField
val config = SsrcLimitConfig()
}
}

27
jvb/src/main/kotlin/org/jitsi/videobridge/TransportConfig.kt Normal file
View File

@@ -0,0 +1,27 @@
/*
* Copyright @ 2018 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
class TransportConfig private constructor() {
companion object {
@JvmStatic
val queueSize: Int by config("videobridge.transport.send.queue-size".from(JitsiConfig.newConfig))
}
}

34
jvb/src/main/kotlin/org/jitsi/videobridge/VersionConfig.kt Normal file
View File

@@ -0,0 +1,34 @@
/*
* Copyright @ 2020 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig.Companion.newConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import org.jitsi.metaconfig.optionalconfig
class VersionConfig private constructor() {
val announceVersion: Boolean by config("videobridge.version.announce".from(newConfig))
fun announceVersion() = announceVersion
val release: String? by optionalconfig("videobridge.version.release".from(newConfig))
companion object {
@JvmField
val config = VersionConfig()
}
}

38
jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
class VideobridgeConfig private constructor() {
companion object {
val initialDrainMode: Boolean by config {
"videobridge.initial-drain-mode".from(JitsiConfig.newConfig)
}
@JvmStatic
val redactRemoteAddresses: Boolean by config {
"videobridge.redact-remote-addresses".from(JitsiConfig.newConfig)
}
@JvmStatic
val redactColibriHttpHeaders: Boolean by config {
"videobridge.redact-colibri-http-headers".from(JitsiConfig.newConfig)
}
}
}

37
jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeExpireThreadConfig.kt Normal file
View File

@@ -0,0 +1,37 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.metaconfig.from
import java.time.Duration
class VideobridgeExpireThreadConfig private constructor() {
val inactivityTimeout: Duration by config("videobridge.entity-expiration.timeout".from(JitsiConfig.newConfig))
val interval: Duration by config {
"org.jitsi.videobridge.EXPIRE_CHECK_SLEEP_SEC".from(JitsiConfig.legacyConfig)
.convertFrom<Long>(Duration::ofSeconds)
"videobridge.entity-expiration.check-interval".from(JitsiConfig.newConfig)
}
companion object {
@JvmField
val config = VideobridgeExpireThreadConfig()
}
}

158
jvb/src/main/kotlin/org/jitsi/videobridge/cc/BandwidthProbing.kt Normal file
View File

@@ -0,0 +1,158 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.rtp.TransportCcEngine
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.BitrateTracker
import org.jitsi.nlj.util.bytes
import org.jitsi.utils.concurrent.PeriodicRunnable
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.ms
import org.jitsi.utils.secs
import org.jitsi.videobridge.cc.allocation.BitrateControllerStatusSnapshot
import org.jitsi.videobridge.cc.config.BandwidthProbingConfig.Companion.config
import java.util.function.Supplier
class BandwidthProbing(
private val probingDataSender: ProbingDataSender,
private val statusSnapshotSupplier: Supplier<BitrateControllerStatusSnapshot>
) : PeriodicRunnable(config.paddingPeriodMs), TransportCcEngine.BandwidthListener {
/** Whether or not probing is currently enabled */
var enabled = false
private var lastTotalNeededBps = 0L
private var lastMaxPaddingBps = 0L
private var lastPaddingBps = 0L
private fun zeroStats() {
lastTotalNeededBps = 0
lastMaxPaddingBps = 0
lastPaddingBps = 0
}
private val probingBitrate = BitrateTracker(5.secs, 100.ms)
/**
* The number of bytes left over from one run of probing to the next. This
* avoids accumulated rounding errors causing us to under-shoot the probing
* bandwidth, and also handles the use when the number of bytes we want to
* send is less than the size of an RTP header.
*/
private var bytesLeftOver: Double = 0.0
private var latestBwe: Long = -1
var diagnosticsContext: DiagnosticContext? = null
override fun bandwidthEstimationChanged(newValue: Bandwidth) {
latestBwe = newValue.bps
}
override fun run() {
super.run()
if (!enabled) {
zeroStats()
return
}
// We calculate how much to probe for based on the total target bps
// (what we're able to reach), the total ideal bps (what we want to
// be able to reach) and the total current bps (what we currently send).
val bitrateControllerStatus = statusSnapshotSupplier.get()
// How much padding do we need?
val totalNeededBps = bitrateControllerStatus.currentIdealBps - bitrateControllerStatus.currentTargetBps
if (totalNeededBps < 1 || !bitrateControllerStatus.hasNonIdealLayer) {
// Don't need to send any probing.
bytesLeftOver = 0.0
zeroStats()
return
}
val latestBweCopy = latestBwe
if (bitrateControllerStatus.currentIdealBps <= latestBweCopy) {
zeroStats()
return
}
// How much padding can we afford?
val maxPaddingBps = latestBweCopy - bitrateControllerStatus.currentTargetBps
val paddingBps = totalNeededBps.coerceAtMost(maxPaddingBps)
lastTotalNeededBps = totalNeededBps
lastMaxPaddingBps = maxPaddingBps
lastPaddingBps = paddingBps
var timeSeriesPoint: DiagnosticContext.TimeSeriesPoint? = null
val newBytesNeeded = (config.paddingPeriodMs * paddingBps / 1000.0 / 8.0)
val bytesNeeded = newBytesNeeded + bytesLeftOver
if (timeSeriesLogger.isTraceEnabled) {
diagnosticsContext?.let { diagnosticsContext ->
timeSeriesPoint = diagnosticsContext
.makeTimeSeriesPoint("sent_padding")
.addField("padding_bps", paddingBps)
.addField("total_ideal_bps", bitrateControllerStatus.currentIdealBps)
.addField("total_target_bps", bitrateControllerStatus.currentTargetBps)
.addField("needed_bps", totalNeededBps)
.addField("max_padding_bps", maxPaddingBps)
.addField("bwe_bps", latestBweCopy)
.addField("bytes_needed", bytesNeeded)
.addField("prev_bytes_left_over", bytesLeftOver)
}
}
if (bytesNeeded >= 1) {
val bytesSent = probingDataSender.sendProbing(bitrateControllerStatus.activeSsrcs, bytesNeeded.toInt())
probingBitrate.update(bytesSent.bytes)
bytesLeftOver = (bytesNeeded - bytesSent).coerceAtLeast(0.0)
timeSeriesPoint?.addField("bytes_sent", bytesSent)?.addField("new_bytes_left_over", bytesLeftOver)
} else {
bytesLeftOver = bytesNeeded.coerceAtLeast(0.0)
}
if (timeSeriesLogger.isTraceEnabled) {
timeSeriesLogger?.trace(timeSeriesPoint)
}
}
fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("enabled", enabled)
put("latest_bwe", latestBwe)
put("last_total_needed_bps", lastTotalNeededBps)
put("last_max_padding_bps", lastMaxPaddingBps)
put("last_padding_bps", lastPaddingBps)
put("probing_bps", probingBitrate.getRateBps())
}
companion object {
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BandwidthProbing::class.java)
}
interface ProbingDataSender {
/**
* Sends a specific number of bytes with a specific set of SSRCs.
* @param mediaSsrcs the SSRCs
* @param numBytes the number of probing bytes we want to send
* @return the number of bytes of probing data actually sent
*/
fun sendProbing(mediaSsrcs: Collection<Long>, numBytes: Int): Int
}
}

184
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt Normal file
View File

@@ -0,0 +1,184 @@
/*
* Copyright @ 2020 - present 8x8, Inc.
* Copyright @ 2021 - Vowel, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.config.JitsiConfig
import org.jitsi.metaconfig.config
import org.jitsi.nlj.util.bps
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.LoggerImpl
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config
import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage
/**
* This class encapsulates all of the client-controlled settings for bandwidth allocation.
*/
data class AllocationSettings @JvmOverloads constructor(
@Deprecated("", ReplaceWith("onStageSources"), DeprecationLevel.WARNING)
val onStageEndpoints: List<String> = emptyList(),
@Deprecated("", ReplaceWith("selectedSources"), DeprecationLevel.WARNING)
val selectedEndpoints: List<String> = emptyList(),
val onStageSources: List<String> = emptyList(),
val selectedSources: List<String> = emptyList(),
val videoConstraints: Map<String, VideoConstraints> = emptyMap(),
val lastN: Int = defaultInitialLastN,
val defaultConstraints: VideoConstraints,
/** A non-negative value is assumed as the available bandwidth in bps. A negative value is ignored. */
val assumedBandwidthBps: Long = -1
) {
fun toJson(): ObjectNode {
val mapper = ObjectMapper()
return JsonNodeFactory.instance.objectNode().apply {
set<ObjectNode>("on_stage_sources", mapper.valueToTree(onStageSources))
set<ObjectNode>("selected_sources", mapper.valueToTree(selectedSources))
set<ObjectNode>("video_constraints", mapper.valueToTree(videoConstraints.mapValues { it.value.toString() }))
put("last_n", lastN)
put("default_constraints", defaultConstraints.toString())
put("assumed_bandwidth_bps", assumedBandwidthBps)
}
}
override fun toString(): String = toJson().toString()
fun getConstraints(endpointId: String) = videoConstraints.getOrDefault(endpointId, defaultConstraints)
companion object {
val defaultInitialLastN: Int by config {
"videobridge.cc.initial-last-n".from(JitsiConfig.newConfig)
}
}
}
/**
* Maintains an [AllocationSettings] instance and allows fields to be set individually, with an indication of whether
* the overall state changed.
*/
internal class AllocationSettingsWrapper(
parentLogger: Logger = LoggerImpl(AllocationSettingsWrapper::class.java.name)
) {
private val logger = createChildLogger(parentLogger)
/**
* The last selected sources set signaled by the receiving endpoint.
*/
private var selectedSources = emptyList<String>()
internal var lastN: Int = AllocationSettings.defaultInitialLastN
private var videoConstraints: Map<String, VideoConstraints> = emptyMap()
private var defaultConstraints: VideoConstraints = VideoConstraints(config.initialMaxHeightPx)
private var assumedBandwidthBps: Long = -1
private var onStageSources: List<String> = emptyList()
private var allocationSettings = create()
private var everSet = false
private fun create(): AllocationSettings = AllocationSettings(
onStageSources = onStageSources,
selectedSources = selectedSources,
videoConstraints = videoConstraints,
defaultConstraints = defaultConstraints,
lastN = lastN,
assumedBandwidthBps = assumedBandwidthBps
)
fun get() = allocationSettings
fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage): Boolean {
var changed = false
message.lastN?.let {
if (lastN != it) {
lastN = it
changed = true
}
}
message.selectedSources?.let {
if (selectedSources != it) {
selectedSources = it
changed = true
}
}
message.onStageSources?.let {
if (onStageSources != it) {
onStageSources = it
changed = true
}
}
val newDefaultConstraints = message.defaultConstraints ?: if (!everSet) {
defaultDefaultConstraints
} else {
null
}
newDefaultConstraints?.let {
if (defaultConstraints != it) {
defaultConstraints = it
changed = true
}
}
message.constraints?.let {
if (this.videoConstraints != it) {
this.videoConstraints = it
changed = true
}
}
message.assumedBandwidthBps?.let {
config.assumedBandwidthLimit?.let { limit ->
val limited = it.coerceAtMost(limit.bps)
if (assumedBandwidthBps != limited) {
logger.warn("Setting assumed bandwidth ${limited.bps} (receiver asked for $it).")
this.assumedBandwidthBps = limited
changed = true
}
} ?: run {
if (it.bps >= 0.bps) {
logger.info("Ignoring assumed-bandwidth-bps, not allowed in config.")
}
}
}
if (changed) {
allocationSettings = create()
}
everSet = true
return changed
}
/**
* Return `true` iff the [AllocationSettings] state changed.
*/
fun setLastN(lastN: Int): Boolean {
if (this.lastN != lastN) {
this.lastN = lastN
allocationSettings = create()
return true
}
return false
}
companion object {
val defaultDefaultConstraints = VideoConstraints(maxHeight = config.defaultMaxHeightPx)
}
}

110
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt Normal file
View File

@@ -0,0 +1,110 @@
/*
* Copyright @ 2020 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.RtpLayerDesc
/**
* The result of bandwidth allocation.
*/
class BandwidthAllocation @JvmOverloads constructor(
val allocations: Set<SingleAllocation>,
val oversending: Boolean = false,
val idealBps: Long = -1,
val targetBps: Long = -1,
/** Whether any of the requested sources were suspended (no layer at all was selected) due to BWE. */
private val suspendedSources: List<String> = emptyList()
) {
val hasSuspendedSources: Boolean = suspendedSources.isNotEmpty()
val forwardedSources: Set<String> =
allocations.filter { it.isForwarded() }.mapNotNull { it.mediaSource?.sourceName }.toSet()
/**
* Whether the two allocations have the same endpoints and same layers.
*/
fun isTheSameAs(other: BandwidthAllocation) = allocations.size == other.allocations.size &&
oversending == other.oversending &&
allocations.all { allocation ->
other.allocations.any { otherAllocation ->
allocation.endpointId == otherAllocation.endpointId &&
allocation.mediaSource?.primarySSRC == otherAllocation.mediaSource?.primarySSRC &&
allocation.mediaSource?.videoType == otherAllocation.mediaSource?.videoType &&
allocation.targetLayer?.index == otherAllocation.targetLayer?.index
}
}
override fun toString(): String = "oversending=$oversending " + allocations.joinToString()
val debugState: ObjectNode
get() = JsonNodeFactory.instance.objectNode().apply {
put("idealBps", idealBps)
put("targetBps", targetBps)
put("oversending", oversending)
put("has_suspended_sources", hasSuspendedSources)
put("suspended_sources", suspendedSources.toString())
val allocationsNode = JsonNodeFactory.instance.objectNode().apply {
allocations.forEach {
val name = it.mediaSource?.sourceName ?: it.endpointId
set<ObjectNode>(name, it.debugState)
}
}
set<ObjectNode>("allocations", allocationsNode)
}
}
/**
* The result of bandwidth allocation for a specific endpoint and a [MediaSourceDesc].
*/
data class SingleAllocation(
/** The ID of the endpoint which owns the media source */
val endpointId: String,
/** The media source */
val mediaSource: MediaSourceDesc?,
/**
* The layer which has been selected to be forwarded.
*/
val targetLayer: RtpLayerDesc? = null,
/**
* The layer which would have been selected without bandwidth constraints.
*/
val idealLayer: RtpLayerDesc? = null
) {
constructor(endpoint: MediaSourceContainer, targetLayer: RtpLayerDesc? = null, idealLayer: RtpLayerDesc? = null) :
this(
endpoint.id,
endpoint.mediaSources.firstOrNull(),
targetLayer,
idealLayer
)
private val targetIndex: Int
get() = targetLayer?.index ?: -1
fun isForwarded(): Boolean = targetIndex > -1
override fun toString(): String = "[epId=$endpointId sourceName=${mediaSource?.sourceName} " +
"target=${targetLayer?.height}/${targetLayer?.frameRate} " +
"(${targetLayer?.indexString()}) " +
"ideal=${idealLayer?.height}/${idealLayer?.frameRate} (${idealLayer?.indexString()})]"
val debugState: ObjectNode
get() = JsonNodeFactory.instance.objectNode().apply {
targetLayer?.debugState()?.let { set<ObjectNode>("target", it) } ?: put("target", null as String?)
idealLayer?.debugState()?.let { set<ObjectNode>("ideal", it) } ?: put("ideal", null as String?)
}
}

427
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt Normal file
View File

@@ -0,0 +1,427 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.util.bps
import org.jitsi.utils.event.EventEmitter
import org.jitsi.utils.event.SyncEventEmitter
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.cc.config.BitrateControllerConfig
import org.jitsi.videobridge.util.TaskPools
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.concurrent.ScheduledFuture
import java.util.concurrent.TimeUnit
import java.util.function.Supplier
import kotlin.math.abs
internal class BandwidthAllocator<T : MediaSourceContainer>(
eventHandler: EventHandler,
/**
* Provide the current list of endpoints (in no particular order).
* TODO: Simplify to avoid the weird (and slow) flow involving `endpointsSupplier` and `sortedEndpointIds`.
*/
private val endpointsSupplier: Supplier<List<T>>,
/**
* Whether bandwidth allocation should be constrained to the available bandwidth (when `true`), or assume
* infinite bandwidth (when `false`).
*/
private val trustBwe: Supplier<Boolean>,
parentLogger: Logger,
private val diagnosticContext: DiagnosticContext,
private val clock: Clock
) {
private val logger = createChildLogger(parentLogger)
/** The estimated available bandwidth in bits per second. */
private var bweBps: Long = -1
/** Whether this bandwidth estimator has been expired. Once expired we stop periodic re-allocation. */
private var expired = false
/**
* The "effective" constraints for an endpoint indicate the maximum resolution/fps that this
* [BandwidthAllocator] would allocate for this endpoint given enough bandwidth.
*
* They are the constraints signaled by the receiver, further reduced to 0 when the endpoint is "outside lastN".
*
* Effective constraints are used to signal to video senders to reduce their resolution to the minimum that
* satisfies all receivers.
*
* With the multi-stream support added, the mapping is stored on a per source name basis instead of an endpoint id.
*
* When an endpoint falls out of the last N, the constraints of all the sources of this endpoint are reduced to 0.
*
* TODO Update this description when the endpoint ID signaling is removed from the JVB.
*/
private var effectiveConstraints: EffectiveConstraintsMap = emptyMap()
private val eventEmitter: EventEmitter<EventHandler> = SyncEventEmitter<EventHandler>().apply {
addHandler(eventHandler)
}
/** The allocations settings signalled by the receiver. */
private var allocationSettings =
AllocationSettings(
defaultConstraints = VideoConstraints(BitrateControllerConfig.config.initialMaxHeightPx)
)
/**
* The last time [BandwidthAllocator.update] was called.
* Initialized as initialization time to prevent triggering an update immediately, because the settings might not
* have been configured yet.
*/
private var lastUpdateTime: Instant = clock.instant()
/** The result of the bitrate control algorithm, the last time it ran. */
var allocation = BandwidthAllocation(emptySet())
private set
/** The task scheduled to call [.update]. */
private var updateTask: ScheduledFuture<*>? = null
init {
rescheduleUpdate()
}
/** Gets a JSON representation of the parts of this object's state that are deemed useful for debugging. */
@get:SuppressFBWarnings(
value = ["IS2_INCONSISTENT_SYNC"],
justification = "We intentionally avoid synchronizing while reading fields only used in debug output."
)
val debugState: ObjectNode
get() {
val debugState = JsonNodeFactory.instance.objectNode()
debugState.put("trustBwe", trustBwe.get())
debugState.put("bweBps", bweBps)
debugState.set<ObjectNode>("allocation", allocation.debugState)
debugState.set<ObjectNode>("allocationSettings", allocationSettings.toJson())
debugState.put("effectiveConstraints", effectiveConstraints.mapKeys { it.key.sourceName }.toString())
return debugState
}
/** Get the available bandwidth, taking into account the `trustBwe` option. */
private val availableBandwidth: Long
get() = if (trustBwe.get()) bweBps else Long.MAX_VALUE
/**
* Notify the [BandwidthAllocator] that the estimated available bandwidth has changed.
* @param newBandwidthBps the newly estimated bandwidth in bps
*/
fun bandwidthChanged(newBandwidthBps: Long) {
if (!bweChangeIsLargerThanThreshold(bweBps, newBandwidthBps)) {
logger.debug {
"New bwe ($newBandwidthBps) is not significantly changed from previous bwe ($bweBps), ignoring."
}
// If this is a "negligible" change in the bandwidth estimation
// wrt the last bandwidth estimation that we reacted to, then
// do not update the bandwidth allocation. The goal is to limit
// the resolution changes due to bandwidth estimation changes,
// as often resolution changes can negatively impact user
// experience, at the risk of clogging the receiver pipe.
} else {
logger.debug { "new bandwidth is $newBandwidthBps, updating" }
bweBps = newBandwidthBps
update()
}
}
/**
* Updates the allocation settings and calculates a new bitrate [BandwidthAllocation].
* @param allocationSettings the new allocation settings.
*/
fun update(allocationSettings: AllocationSettings) {
this.allocationSettings = allocationSettings
update()
}
/**
* Runs the bandwidth allocation algorithm, and fires events if the result is different from the previous result.
*/
@Synchronized
fun update() {
if (expired) {
return
}
lastUpdateTime = clock.instant()
// Order the sources by selection, followed by Endpoint's speech activity.
val sources = endpointsSupplier.get().flatMap { it.mediaSources.toList() }.toMutableList()
val sortedSources = prioritize(sources, selectedSources)
// Extract and update the effective constraints.
val oldEffectiveConstraints = effectiveConstraints
val newEffectiveConstraints = getEffectiveConstraints(sortedSources, allocationSettings)
effectiveConstraints = newEffectiveConstraints
logger.trace {
"Allocating: sortedSources=${sortedSources.map { it.sourceName }}, " +
"effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}"
}
// Compute the bandwidth allocation.
val newAllocation = allocate(sortedSources)
val allocationChanged = !allocation.isTheSameAs(newAllocation)
val effectiveConstraintsChanged = effectiveConstraints != oldEffectiveConstraints
if (timeSeriesLogger.isTraceEnabled) {
timeSeriesLogger.trace(
diagnosticContext.makeTimeSeriesPoint("allocator_update", lastUpdateTime)
.addField("target_bps", newAllocation.targetBps)
.addField("ideal_bps", newAllocation.idealBps)
.addField("bwe_bps", bweBps)
.addField("oversending", newAllocation.oversending)
.addField("allocation_changed", allocationChanged)
.addField("effective_constraints_changed", effectiveConstraintsChanged)
)
}
if (allocationChanged) {
eventEmitter.fireEvent { allocationChanged(newAllocation) }
}
allocation = newAllocation
logger.trace {
"Finished allocation: allocationChanged=$allocationChanged, " +
"effectiveConstraintsChanged=$effectiveConstraintsChanged, " +
"allocation=[$allocation]"
}
if (effectiveConstraintsChanged) {
eventEmitter.fireEvent {
effectiveVideoConstraintsChanged(oldEffectiveConstraints, effectiveConstraints)
}
}
}
// On-stage sources are considered selected (with higher priority).
private val selectedSources: List<String>
get() {
// On-stage sources are considered selected (with higher priority).
val selectedSources = allocationSettings.onStageSources.toMutableList()
allocationSettings.selectedSources.forEach {
if (!selectedSources.contains(it)) {
selectedSources.add(it)
}
}
return selectedSources
}
/**
* Implements the bandwidth allocation algorithm for the given ordered list of media sources.
*
* The new version which works with multiple streams per endpoint.
*
* @param conferenceMediaSources the list of endpoint media sources in order of priority to allocate for.
* @return the new [BandwidthAllocation].
*/
@Synchronized
private fun allocate(conferenceMediaSources: List<MediaSourceDesc>): BandwidthAllocation {
val sourceBitrateAllocations = createAllocations(conferenceMediaSources)
if (sourceBitrateAllocations.isEmpty()) {
return BandwidthAllocation(emptySet())
}
val initialBandwidth = if (allocationSettings.assumedBandwidthBps >= 0) {
logger.warn("Allocating with assumed bandwidth ${allocationSettings.assumedBandwidthBps.bps}.")
allocationSettings.assumedBandwidthBps
} else {
availableBandwidth
}
var remainingBandwidth = initialBandwidth
// Only one source is allowed to oversend: the first on-stage source which actually has an oversend layer.
// We can not use the index in the loop below to identify it, because sources with disabled constraints are
// skipped; and we require an oversend layer so that when there is more than one on-stage source (e.g. a
// camera and a screenshare) the slot is not wasted on one which can not use it.
val oversendingIndex = sourceBitrateAllocations.indexOfFirst {
!it.constraints.isDisabled() && it.isOnStage() && it.layers.oversendIndex >= 0
}
var oldRemainingBandwidth: Long = -1
var oversending = false
while (oldRemainingBandwidth != remainingBandwidth) {
oldRemainingBandwidth = remainingBandwidth
for (i in sourceBitrateAllocations.indices) {
val sourceBitrateAllocation = sourceBitrateAllocations[i]
if (sourceBitrateAllocation.constraints.isDisabled()) {
continue
}
// In stage view improve greedily until preferred, in tile view go step-by-step.
remainingBandwidth -= sourceBitrateAllocation.improve(remainingBandwidth, i == oversendingIndex)
if (remainingBandwidth < 0) {
oversending = true
}
// In stage view, do not allocate bandwidth for thumbnails until the on-stage reaches "preferred".
// This prevents enabling thumbnail only to disable them when bwe slightly increases allowing on-stage
// to take more.
if (sourceBitrateAllocation.isOnStage() && !sourceBitrateAllocation.hasReachedPreferred()) {
break
}
}
}
// The sources which are in lastN, and are sending video, but were suspended due to bwe.
val suspendedSources = sourceBitrateAllocations.filter { it.isSuspended }
val suspendedIds = suspendedSources.map { it.mediaSource.sourceName }.toList()
if (suspendedIds.isNotEmpty()) {
logger.info {
// The bandwidth we started with is not the BWE if the receiver signaled an assumed bandwidth, so log
// it separately when the two differ.
val budget = if (initialBandwidth != availableBandwidth) "budget=$initialBandwidth bps, " else ""
val maxOversend = BitrateControllerConfig.config.maxOversendBitrate
val sources = suspendedSources.mapIndexed { i, source ->
// Only summarize the first few sources in full. They are in order of priority, so these are the
// ones whose suspension is most likely to be of interest. Name the rest without any detail, to
// bound the size of the message when many sources are suspended at once.
if (i < MAX_SOURCES_SUMMARIZED) source.suspendedSummary() else source.mediaSource.sourceName
}
"Sources suspended due to insufficient bandwidth (bwe=$availableBandwidth bps, $budget" +
"remaining=$remainingBandwidth bps, maxOversend=$maxOversend): " + sources.joinToString()
}
}
val allocations = mutableSetOf<SingleAllocation>()
var targetBps: Long = 0
var idealBps: Long = 0
for (sourceBitrateAllocation: SingleSourceAllocation in sourceBitrateAllocations) {
allocations.add(sourceBitrateAllocation.result)
targetBps += sourceBitrateAllocation.targetBitrate
idealBps += sourceBitrateAllocation.idealBitrate
}
return BandwidthAllocation(allocations, oversending, idealBps, targetBps, suspendedIds)
}
/**
* Query whether the allocator has non-zero effective constraints for the given endpoint or source.
*/
internal fun hasNonZeroEffectiveConstraints(source: MediaSourceDesc): Boolean {
val constraints = effectiveConstraints[source] ?: return false
return !constraints.isDisabled()
}
@Synchronized
private fun createAllocations(conferenceMediaSources: List<MediaSourceDesc>): List<SingleSourceAllocation> =
conferenceMediaSources.map { source ->
SingleSourceAllocation(
source.owner,
source,
// Note that we use the effective constraints and not the receiver's constraints
// directly. This means we never even try to allocate bitrate to sources "outside
// lastN". For example, if LastN=1 and the first endpoint sends a non-scalable
// stream with bitrate higher that the available bandwidth, we will forward no
// video at all instead of going to the second endpoint in the list.
// I think this is not desired behavior. However, it is required for the "effective
// constraints" to work as designed.
effectiveConstraints[source]!!,
allocationSettings.onStageSources.contains(source.sourceName),
diagnosticContext,
clock,
logger
)
}.toList()
/**
* Expire this bandwidth allocator.
*/
fun expire() {
expired = true
updateTask?.cancel(false)
}
/**
* Submits a call to `update` in a CPU thread if bandwidth allocation has not been performed recently.
*
* Also, re-schedule the next update in at most `maxTimeBetweenCalculations`. This should only be run
* in the constructor or in the scheduler thread, otherwise it will schedule multiple tasks.
*/
private fun rescheduleUpdate() {
if (expired) {
return
}
val timeSinceLastUpdate = Duration.between(lastUpdateTime, clock.instant())
val period = BitrateControllerConfig.config.maxTimeBetweenCalculations
val delayMs = if (timeSinceLastUpdate > period) {
logger.debug("Running periodic re-allocation.")
TaskPools.CPU_POOL.execute { this.update() }
period.toMillis()
} else {
period.minus(timeSinceLastUpdate).toMillis()
}
// Add 5ms to avoid having to re-schedule right away. This increases the average period at which we
// re-allocate by an insignificant amount.
updateTask = TaskPools.SCHEDULED_POOL.schedule(
{ rescheduleUpdate() },
delayMs + 5,
TimeUnit.MILLISECONDS
)
}
companion object {
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BandwidthAllocator::class.java)
/** The number of suspended sources for which we log the full details of the failed allocation. */
private const val MAX_SOURCES_SUMMARIZED = 3
}
interface EventHandler {
fun allocationChanged(allocation: BandwidthAllocation)
fun effectiveVideoConstraintsChanged(
oldEffectiveConstraints: EffectiveConstraintsMap,
newEffectiveConstraints: EffectiveConstraintsMap
)
}
}
/**
* Returns a boolean that indicates whether the current bandwidth estimation (in bps) has changed above the
* configured threshold with respect to the previous bandwidth estimation.
*
* @param previousBwe the previous bandwidth estimation (in bps).
* @param currentBwe the current bandwidth estimation (in bps).
* @return true if the bandwidth has changed above the configured threshold, * false otherwise.
*/
private fun bweChangeIsLargerThanThreshold(previousBwe: Long, currentBwe: Long): Boolean {
if (previousBwe == currentBwe) { // Even if we're "changing" -1 to -1
return false
}
if (previousBwe == -1L || currentBwe == -1L) {
return true
}
// We supress re-allocation when BWE has changed less than 15% (by default) of its previous value in order to
// prevent excessive changes during ramp-up.
// When BWE increases it should eventually increase past the threshold because of probing.
// When BWE decreases it is probably above the threshold because of AIMD. It's not clear to me whether we need
// the threshold in this case.
// In any case, there are other triggers for re-allocation, so any suppression we do here will only last up to
// a few seconds.
val deltaBwe = abs(currentBwe - previousBwe)
return deltaBwe > previousBwe * BitrateControllerConfig.config.bweChangeThreshold
// If, on the other hand, the bwe has decreased, we require at least a 15% drop in order to update the bitrate
// allocation. This is an ugly hack to prevent too many resolution/UI changes in case the bridge produces too
// low bandwidth estimate, at the risk of clogging the receiver's pipe.
// TODO: do we still need this? Do we ever ever see BWE drop by <%15?
}
typealias EffectiveConstraintsMap = Map<MediaSourceDesc, VideoConstraints>

344
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt Normal file
View File

@@ -0,0 +1,344 @@
/*
* Copyright @ 2020 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.DebugStateMode
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.format.PayloadType
import org.jitsi.nlj.format.PayloadTypeEncoding
import org.jitsi.nlj.util.Bandwidth
import org.jitsi.nlj.util.bps
import org.jitsi.rtp.rtcp.RtcpSrPacket
import org.jitsi.utils.event.SyncEventEmitter
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config
import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage
import org.jitsi.videobridge.util.BooleanStateTimeTracker
import java.time.Clock
import java.time.Duration
import java.util.function.Supplier
/**
* [BitrateController] is responsible for controlling the send bitrate to an `Endpoint`. This includes two tasks:
* 1. Decide how to allocate the available bandwidth between the available streams.
* 2. Implement the allocation via a packet handling interface.
*
* Historically both were implemented in a single class, but they are now split between [BandwidthAllocator] and
* [PacketHandler]. This class was introduced as a lightweight shim in order to preserve the previous API.
*
*/
class BitrateController<T : MediaSourceContainer> @JvmOverloads constructor(
eventHandler: EventHandler,
endpointsSupplier: Supplier<List<T>>,
private val diagnosticContext: DiagnosticContext,
parentLogger: Logger,
private val clock: Clock = Clock.systemUTC()
) {
val eventEmitter = SyncEventEmitter<EventHandler>()
private val logger = createChildLogger(parentLogger)
private val bitrateAllocatorEventHandler = BitrateAllocatorEventHandler()
/**
* Keep track of the "forwarded" sources, i.e. the media sources for which we are forwarding *some* layer.
*/
var forwardedSources: Set<String> = emptySet()
private set
/**
* Keep track of how much time we spend knowingly oversending (due to enableOnstageVideoSuspend being false)
*/
val oversendingTimeTracker = BooleanStateTimeTracker()
val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BitrateController::class.java).let {
if (it.isTraceEnabled) it else null
}
/**
* NOTE(george): this flag acts as an approximation for determining whether or not adaptivity/probing is
* supported. Eventually we need to scrap this and implement something cleaner, i.e. disable adaptivity if the
* endpoint hasn't signaled `goog-remb` nor `transport-cc`.
*
* Unfortunately the channel iq from jicofo lists `goog-remb` and `transport-cc` support, even tho the jingle from
* firefox doesn't (which is the main use case for wanting to disable adaptivity).
*/
private var supportsRtx = false
private val packetHandler: PacketHandler = PacketHandler(clock, parentLogger, diagnosticContext, eventEmitter)
private val bandwidthAllocator: BandwidthAllocator<T> =
BandwidthAllocator(
bitrateAllocatorEventHandler,
endpointsSupplier,
Supplier { trustBwe },
parentLogger,
diagnosticContext,
clock
)
fun hasSuspendedSources() = bandwidthAllocator.allocation.hasSuspendedSources
private val allocationSettingsWrapper = AllocationSettingsWrapper(parentLogger)
val allocationSettings
get() = allocationSettingsWrapper.get()
init {
eventEmitter.addHandler(eventHandler)
}
private var bweSet = false
/**
* Ignore the bandwidth estimations for some initial time because the REMBs don't ramp up fast enough.
* This shouldn't be needed for other bandwidth estimation algorithms.
*/
private val trustBwe: Boolean
get() = config.trustBwe && supportsRtx && bweSet &&
packetHandler.timeSinceFirstMedia() >= config.initialIgnoreBwePeriod
// Proxy to the allocator
fun endpointOrderingChanged() = bandwidthAllocator.update()
var lastN: Int
get() = allocationSettingsWrapper.lastN
set(value) {
if (allocationSettingsWrapper.setLastN(value)) {
bandwidthAllocator.update(allocationSettingsWrapper.get())
}
}
fun expire() = bandwidthAllocator.expire()
/** Return the number of sources currently being forwarded. */
fun numForwardedSources(): Int = forwardedSources.size
fun getTotalOversendingTime(): Duration = oversendingTimeTracker.totalTimeOn()
fun isOversending() = oversendingTimeTracker.state
fun bandwidthChanged(newBandwidthBps: Long) {
bweSet = true
timeSeriesLogger?.logBweChange(newBandwidthBps)
bandwidthAllocator.bandwidthChanged(newBandwidthBps)
}
// Proxy to the packet handler
fun accept(packetInfo: PacketInfo): Boolean {
if (packetInfo.layeringChanged) {
// This needs to be done synchronously, so it's complete before the accept, below.
logger.debug {
"Layering information changed for packet from ${packetInfo.endpointId}, updating bandwidth allocation"
}
bandwidthAllocator.update()
}
return packetHandler.accept(packetInfo)
}
fun accept(rtcpSrPacket: RtcpSrPacket): Boolean = packetHandler.accept(rtcpSrPacket)
fun transformRtcp(rtcpSrPacket: RtcpSrPacket): Boolean = packetHandler.transformRtcp(rtcpSrPacket)
fun transformRtp(packetInfo: PacketInfo): Boolean = packetHandler.transformRtp(packetInfo)
fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
set<ObjectNode>("bitrate_allocator", bandwidthAllocator.debugState)
set<ObjectNode>("packet_handler", packetHandler.debugState(mode))
put("forwarded_sources", forwardedSources.toString())
put("oversending", oversendingTimeTracker.state)
put("total_oversending_time_secs", oversendingTimeTracker.totalTimeOn().seconds)
put("supports_rtx", supportsRtx)
put("trust_bwe", trustBwe)
}
fun addPayloadType(payloadType: PayloadType) {
if (payloadType.encoding == PayloadTypeEncoding.RTX) {
supportsRtx = true
}
}
fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage) {
if (allocationSettingsWrapper.setBandwidthAllocationSettings(message)) {
bandwidthAllocator.update(allocationSettingsWrapper.get())
}
}
/**
* Query whether this source is on stage or selected, as of the most recent
* video constraints
*/
fun isOnStageOrSelected(source: MediaSourceDesc) = allocationSettings.onStageSources.contains(source.sourceName) ||
allocationSettings.selectedSources.contains(source.sourceName)
/**
* Query whether this allocator has non-zero effective constraints for a given source
*/
fun hasNonZeroEffectiveConstraints(source: MediaSourceDesc) =
bandwidthAllocator.hasNonZeroEffectiveConstraints(source)
/**
* Get the target and ideal bitrate of the current [BandwidthAllocation], as well as the list of SSRCs being
* forwarded, for use in probing.
*
* Note that the ideal layers are calculated with the allocation, and inactive layers are not considered. So when a
* higher layer becomes active, it will not be accounted for until until the allocation updates. Conversely, if the
* ideal layer becomes inactive, it will contribute 0 bps to the total ideal bitrate until the allocation updates
* and a lower layer is selected as idea.
*/
fun getStatusSnapshot(): BitrateControllerStatusSnapshot {
var totalTargetBitrate = 0.bps
var totalIdealBitrate = 0.bps
val activeSsrcs = mutableSetOf<Long>()
var hasNonIdealLayer = false
val nowMs = clock.instant().toEpochMilli()
val allocation = bandwidthAllocator.allocation
allocation.allocations.forEach { singleAllocation ->
val allocationTargetBitrate: Bandwidth? = singleAllocation.targetLayer?.getBitrate(nowMs)
allocationTargetBitrate?.let {
totalTargetBitrate += it
singleAllocation.mediaSource?.primarySSRC?.let { primarySsrc -> activeSsrcs.add(primarySsrc) }
}
val allocationIdealBitrate: Bandwidth? = if (config.useVlaTargetBitrate) {
singleAllocation.idealLayer?.targetBitrate ?: singleAllocation.idealLayer?.getBitrate(nowMs)
} else {
singleAllocation.idealLayer?.getBitrate(nowMs)
}
allocationIdealBitrate?.let {
totalIdealBitrate += it
}
if (singleAllocation.idealLayer != null && singleAllocation.idealLayer != singleAllocation.targetLayer) {
hasNonIdealLayer = true
}
}
activeSsrcs.removeIf { it < 0 }
return BitrateControllerStatusSnapshot(
currentTargetBps = totalTargetBitrate.bps,
currentIdealBps = totalIdealBitrate.bps,
activeSsrcs = activeSsrcs,
hasNonIdealLayer = hasNonIdealLayer
)
}
private fun TimeSeriesLogger.logBweChange(newBweBps: Long) {
trace(diagnosticContext.makeTimeSeriesPoint("new_bwe").addField("bwe_bps", newBweBps))
}
private fun TimeSeriesLogger.logAllocationChange(allocation: BandwidthAllocation) {
val nowMs = clock.millis()
var totalTargetBps = 0.0
var totalIdealBps = 0.0
var totalTargetSmoothedBps = 0.0
var totalIdealSmoothedBps = 0.0
var totalTargetVlaBps = 0.0
var totalIdealVlaBps = 0.0
allocation.allocations.forEach {
it.targetLayer?.getBitrate(nowMs)?.let { bitrate -> totalTargetBps += bitrate.bps }
it.idealLayer?.getBitrate(nowMs)?.let { bitrate -> totalIdealBps += bitrate.bps }
it.targetLayer?.getSmoothedBitrate(nowMs)?.let { bitrate -> totalTargetSmoothedBps += bitrate.bps }
it.idealLayer?.getSmoothedBitrate(nowMs)?.let { bitrate -> totalIdealSmoothedBps += bitrate.bps }
it.targetLayer?.targetBitrate?.let { bitrate -> totalTargetVlaBps += bitrate.bps }
it.idealLayer?.targetBitrate?.let { bitrate -> totalIdealVlaBps += bitrate.bps }
trace(
diagnosticContext
.makeTimeSeriesPoint("allocation_for_source", nowMs)
.addField("remote_endpoint_id", it.endpointId)
.addField("target_idx", it.targetLayer?.index ?: -1)
.addField("ideal_idx", it.idealLayer?.index ?: -1)
.addField("target_bps_measured", it.targetLayer?.getBitrate(nowMs)?.bps ?: -1)
.addField("target_bps_smoothed", it.targetLayer?.getSmoothedBitrate(nowMs)?.bps ?: -1)
.addField("target_bps_vla", it.targetLayer?.targetBitrate?.bps ?: -1)
.addField("ideal_bps_measured", it.idealLayer?.getBitrate(nowMs)?.bps ?: -1)
.addField("ideal_bps_smoothed", it.idealLayer?.getSmoothedBitrate(nowMs)?.bps ?: -1)
.addField("ideal_bps_vla", it.idealLayer?.targetBitrate?.bps ?: -1)
)
}
trace(
diagnosticContext
.makeTimeSeriesPoint("allocation", nowMs)
.addField("total_target_measured_bps", totalTargetBps)
.addField("total_ideal_measured_bps", totalIdealBps)
.addField("total_target_smoothed_bps", totalTargetSmoothedBps)
.addField("total_ideal_smoothed_bps", totalIdealSmoothedBps)
.addField("total_target_vla_bps", totalTargetVlaBps)
.addField("total_ideal_vla_bps", totalIdealVlaBps)
)
}
interface EventHandler {
fun forwardedSourcesChanged(forwardedSources: Set<String>)
fun effectiveVideoConstraintsChanged(
oldEffectiveConstraints: EffectiveConstraintsMap,
newEffectiveConstraints: EffectiveConstraintsMap,
)
fun keyframeNeeded(endpointId: String?, ssrc: Long)
/**
* This is meant to be internal to BitrateAllocator, but is exposed here temporarily for the purposes of testing.
*/
fun allocationChanged(allocation: BandwidthAllocation) { }
}
private inner class BitrateAllocatorEventHandler : BandwidthAllocator.EventHandler {
override fun allocationChanged(allocation: BandwidthAllocation) {
timeSeriesLogger?.logAllocationChange(allocation)
// Actually implement the allocation (configure the packet filter to forward the chosen target layers).
packetHandler.allocationChanged(allocation)
val newForwardedSources = allocation.forwardedSources
if (forwardedSources != newForwardedSources) {
forwardedSources = newForwardedSources
eventEmitter.fireEvent { forwardedSourcesChanged(newForwardedSources) }
}
oversendingTimeTracker.setState(allocation.oversending)
// TODO: this is for testing only. Should we change the tests to work with [BitrateAllocator] directly?
eventEmitter.fireEvent { allocationChanged(allocation) }
}
override fun effectiveVideoConstraintsChanged(
oldEffectiveConstraints: EffectiveConstraintsMap,
newEffectiveConstraints: EffectiveConstraintsMap
) {
// Forward to the outer EventHandler.
eventEmitter.fireEvent {
effectiveVideoConstraintsChanged(oldEffectiveConstraints, newEffectiveConstraints)
}
}
}
}
/**
* Abstracts a media source for the purposes of [BandwidthAllocator].
*/
interface MediaSourceContainer {
val id: String
val mediaSources: Array<MediaSourceDesc>
}
data class BitrateControllerStatusSnapshot(
val currentTargetBps: Long = -1L,
val currentIdealBps: Long = -1L,
val activeSsrcs: Collection<Long> = emptyList(),
val hasNonIdealLayer: Boolean
)

77
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Layers.kt Normal file
View File

@@ -0,0 +1,77 @@
/*
* Copyright @ 2021 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import org.jitsi.nlj.RtpLayerDesc
/**
* Saves the bitrate of a specific [RtpLayerDesc] at a specific point in time.
*/
data class LayerSnapshot(
val layer: RtpLayerDesc,
/**
* The bitrate (in bps) to use for allocation: [vlaTargetBitrate] when `use-vla-target-bitrate` is enabled and the
* sender signals one; otherwise the smoothed measured bitrate for screen sharing (bursty) sources, and
* [measuredBitrate] for everything else.
*/
val bitrate: Long,
/**
* The bitrate (in bps) measured for the layer (and its dependencies) over the short window, i.e. the one which
* matches the bandwidth estimator's. Note that this is not necessarily the bitrate used for allocation.
*/
val measuredBitrate: Long = bitrate,
/**
* The target bitrate (in bps) signaled by the sender in the VLA RTP header extension, or `null` if the sender
* doesn't signal VLA.
*/
val vlaTargetBitrate: Long? = null
) {
/**
* A short description of the layer and its bitrate, for logging. The bitrate which was not used for allocation is
* only mentioned when it differs from the one which was, so in the default configuration (measured bitrates, no
* VLA signaled by the sender) this is just the measured bitrate.
*/
fun summary(): String = buildString {
append("${layer.indexString()}/${layer.height}p/${layer.frameRate}fps=$bitrate")
val other = listOfNotNull(
if (measuredBitrate != bitrate) "measured=$measuredBitrate" else null,
vlaTargetBitrate?.takeIf { it != bitrate }?.let { "vla=$it" }
)
if (other.isNotEmpty()) append(other.joinToString(prefix = "(", postfix = ")"))
}
}
/**
* An immutable representation of the layers to be considered when allocating bandwidth for an endpoint. The order is
* ascending by preference (and not necessarily bitrate).
*/
data class Layers(
val layers: List<LayerSnapshot>,
/** The index of the "preferred" layer, i.e. the layer up to which we allocate eagerly. */
val preferredIndex: Int,
/**
* The index of the layer which will be selected if oversending is enabled. If set to -1, oversending is disabled.
*/
val oversendIndex: Int
) : List<LayerSnapshot> by layers {
val preferredLayer = layers.getOrNull(preferredIndex)
val oversendLayer = layers.getOrNull(oversendIndex)
val idealLayer = layers.lastOrNull()
companion object {
val noLayers = Layers(emptyList(), -1, -1)
}
}

203
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt Normal file
View File

@@ -0,0 +1,203 @@
/*
* Copyright @ 2015 - Present, 8x8 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.DebugStateMode
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.PacketInfo.Companion.enablePayloadVerification
import org.jitsi.nlj.RtpLayerDesc
import org.jitsi.nlj.rtp.VideoRtpPacket
import org.jitsi.rtp.rtcp.RtcpSrPacket
import org.jitsi.utils.event.EventEmitter
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.cc.AdaptiveSourceProjection
import org.jitsi.videobridge.cc.RewriteException
import java.time.Clock
import java.time.Duration
import java.time.Instant
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger
/**
* Handles packets for a [BitrateController], implementing a specific [BandwidthAllocation] provided via
* [allocationChanged]. Serves as a bridge between [BitrateController] (which decides which layers are to be forwarded)
* and [AdaptiveSourceProjection] which processes packets and maintains the state of a forwarded stream.
*
* Defines "accept" and "transform" functions for video RTP packets and RTCP Sender Reports.
*/
internal class PacketHandler(
private val clock: Clock,
parentLogger: Logger,
private val diagnosticContext: DiagnosticContext,
private val eventEmitter: EventEmitter<BitrateController.EventHandler>
) {
private val logger = createChildLogger(parentLogger)
/**
* The time (in ms) when this instance first transformed any media. This allows to ignore the BWE during the early
* stages of the call.
*
* NOTE This is only meant to be as a temporary hack and ideally should be fixed.
*/
private var firstMedia: Instant? = null
private val numDroppedPacketsUnknownSsrc = AtomicInteger(0)
/**
* The [AdaptiveSourceProjection]s that this instance is managing, keyed
* by the SSRCs of the associated [MediaSourceDesc].
*/
private val adaptiveSourceProjectionMap: MutableMap<Long, AdaptiveSourceProjection> = ConcurrentHashMap()
/**
* @return true if the packet was transformed successfully, false otherwise.
*/
fun transformRtp(
/** Contains the video RTP packet to be transformed */
packetInfo: PacketInfo
): Boolean {
val videoPacket = packetInfo.packetAs<VideoRtpPacket>()
if (firstMedia == null) {
firstMedia = clock.instant()
}
val adaptiveSourceProjection = adaptiveSourceProjectionMap[videoPacket.ssrc] ?: return false
return try {
adaptiveSourceProjection.rewriteRtp(packetInfo)
// The rewriteRtp operation must not modify the VP8 payload.
if (enablePayloadVerification) {
val expected = packetInfo.payloadVerification
val actual = videoPacket.payloadVerification
if ("" != expected && expected != actual) {
logger.warn("Payload unexpectedly modified! Expected: $expected, actual: $actual")
}
}
true
} catch (e: RewriteException) {
logger.warn("Failed to rewrite a packet.", e)
false
}
}
/**
* @return true if a packet should be accepted according to the current configuration.
*/
fun accept(packetInfo: PacketInfo): Boolean {
val videoPacket = packetInfo.packetAs<VideoRtpPacket>()
val adaptiveSourceProjection = adaptiveSourceProjectionMap[videoPacket.ssrc]
if (adaptiveSourceProjection == null) {
logger.debug { "Dropping an RTP packet for an unknown SSRC: ${videoPacket.ssrc}" }
numDroppedPacketsUnknownSsrc.incrementAndGet()
return false
}
return adaptiveSourceProjection.accept(packetInfo)
}
/**
* @return true if [rtcpSrPacket] should be accepted, false otherwise.
*
* Filters out packets that match one of the streams that this instance manages, but don't match the target SSRC.
* Allows packets for streams not managed by this instance.
*/
fun accept(rtcpSrPacket: RtcpSrPacket): Boolean {
val adaptiveSourceProjection = adaptiveSourceProjectionMap[rtcpSrPacket.senderSsrc]
// This is probably for an audio stream. In any case, if it's for a stream which we are not forwarding it
// will be stripped off at a later stage (in RtcpSrUpdater).
return adaptiveSourceProjection == null ||
// We only accept SRs for the SSRC that we're forwarding with.
adaptiveSourceProjection.targetSsrc == rtcpSrPacket.senderSsrc
}
/**
* @return true if the packet was transformed successfully, false otherwise.
*/
fun transformRtcp(rtcpSrPacket: RtcpSrPacket): Boolean =
adaptiveSourceProjectionMap[rtcpSrPacket.senderSsrc]?.rewriteRtcp(rtcpSrPacket) ?: false
/**
* Utility method that looks-up or creates the adaptive source projection of
* a source.
*/
private fun lookupOrCreateAdaptiveSourceProjection(singleAllocation: SingleAllocation): AdaptiveSourceProjection? {
val source = singleAllocation.mediaSource
val endpointID = singleAllocation.endpointId
if (source == null) {
return null
}
synchronized(adaptiveSourceProjectionMap) {
adaptiveSourceProjectionMap[source.primarySSRC]?.let { return it }
if (source.rtpEncodings.isEmpty()) {
return null
}
val adaptiveSourceProjection = AdaptiveSourceProjection(
diagnosticContext,
source,
{
eventEmitter.fireEvent { keyframeNeeded(endpointID, source.primarySSRC) }
},
logger
)
logger.debug { "new source projection for $source" }
// Route all encodings to the specified bitrate controller.
source.rtpEncodings.forEach {
adaptiveSourceProjectionMap[it.primarySSRC] = adaptiveSourceProjection
}
return adaptiveSourceProjection
}
}
fun timeSinceFirstMedia(): Duration = firstMedia?.let { Duration.between(it, clock.instant()) } ?: Duration.ZERO
fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("num_dropped_packets_unknown_ssrc", numDroppedPacketsUnknownSsrc.toInt())
set<ObjectNode>("adaptive_source_projection_map", adaptiveSourceProjectionMap.debugState(mode))
}
private fun Map<Long, AdaptiveSourceProjection>.debugState(mode: DebugStateMode) =
JsonNodeFactory.instance.objectNode().also {
forEach { (ssrc, adaptiveSourceProjection) ->
it.set<ObjectNode>(ssrc.toString(), adaptiveSourceProjection.getDebugState(mode))
}
}
/**
* Signals to this instance that the allocation chosen by the `BitrateAllocator` has changed.
*/
fun allocationChanged(allocation: BandwidthAllocation) {
if (allocation.allocations.isEmpty()) {
adaptiveSourceProjectionMap.values.forEach { it.setTargetIndex(RtpLayerDesc.SUSPENDED_INDEX) }
} else {
allocation.allocations.forEach {
val sourceTargetIdx = it.targetLayer?.index ?: -1
// Review this.
val adaptiveSourceProjection = lookupOrCreateAdaptiveSourceProjection(it)
adaptiveSourceProjection?.setTargetIndex(sourceTargetIdx)
}
}
}
}

99
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt Normal file
View File

@@ -0,0 +1,99 @@
/*
* Copyright @ 2020 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.videobridge.SsrcLimitConfig
import org.jitsi.videobridge.calculateLastN
import org.jitsi.videobridge.jvbLastNSingleton
import org.jitsi.videobridge.load_management.ConferenceSizeLastNLimits.Companion.singleton as conferenceSizeLimits
fun prioritize(
/**
* All available sources ordered by their endpoint's speech activity and video availability. Note that when an
* endpoint has multiple sources and one of them is disabled it will still sort high because the endpoint has
* video availability.
*/
conferenceSources: MutableList<MediaSourceDesc>,
selectedSourceNames: List<String> = emptyList()
): List<MediaSourceDesc> {
val enabledSelectedSources = mutableListOf<MediaSourceDesc>()
val enabledNonSelectedSources = mutableListOf<MediaSourceDesc>()
val disabledSources = mutableListOf<MediaSourceDesc>()
// conferenceSources can be large, while selectedSourceNames is usually small, so do a single pass over
// conferenceSources.
conferenceSources.forEach { source ->
if (source.videoType.isEnabled()) {
if (selectedSourceNames.contains(source.sourceName)) {
enabledSelectedSources.add(source)
} else {
enabledNonSelectedSources.add(source)
}
} else {
disabledSources.add(source)
}
}
// The enabled selected sources are sorted according to the order in which they are selected and prioritized
// over non-selected.
enabledSelectedSources.sortBy { selectedSourceNames.indexOf(it.sourceName) }
enabledSelectedSources.addAll(enabledNonSelectedSources)
// All disabled sources are sorted last, regardless of whether they are selected.
enabledSelectedSources.addAll(disabledSources)
return enabledSelectedSources
}
/**
* Return the "effective" constraints for the given media sources, i.e. the constraints adjusted for LastN.
*/
fun getEffectiveConstraints(
sources: List<MediaSourceDesc>,
allocationSettings: AllocationSettings
): EffectiveConstraintsMap {
// FIXME figure out before merge - is using source count instead of endpoints
// Add 1 for the receiver endpoint, which is not in the list.
val effectiveLastN = effectiveLastN(allocationSettings.lastN, sources.size + 1)
// Keep track of the number of sources with non-zero constraints. Once [effectiveLastN] of them have been
// added, all other sources have effectiveConstraints 0, because they would never be forwarded by the
// algorithm.
var sourcesWithNonZeroConstraints = 0
return sources.associateWith { source ->
if (!source.videoType.isEnabled() || sourcesWithNonZeroConstraints >= effectiveLastN) {
VideoConstraints.NOTHING
} else {
allocationSettings.getConstraints(source.sourceName).also {
if (!it.isDisabled()) sourcesWithNonZeroConstraints++
}
}
}
}
/**
* The LastN value adjusted according to the limits configured on the bridge, or [Int.MAX_VALUE] if LastN is disabled.
*/
private fun effectiveLastN(lastN: Int, conferenceSize: Int): Int {
val adjustedLastN =
calculateLastN(
lastN,
jvbLastNSingleton.jvbLastN,
conferenceSizeLimits.getLastNLimit(conferenceSize),
SsrcLimitConfig.config.maxVideoSsrcs
)
return if (adjustedLastN < 0) Int.MAX_VALUE else adjustedLastN
}

84
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMap.kt Normal file
View File

@@ -0,0 +1,84 @@
/*
* Copyright @ 2018 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
/**
* A Map of Endpoint IDs to their receiver video constraints. Tracks the max height of all
* of those constraints as new constraints are added and removed.
*/
class ReceiverConstraintsMap {
private val map = mutableMapOf<String, VideoConstraints>()
private val lock = Any()
var maxHeight: Int = 0
private set
fun put(key: String, value: VideoConstraints): VideoConstraints? {
synchronized(lock) {
return map.put(key, value).also { removed ->
maxHeight = when {
value.maxHeight == -1 -> value.maxHeight
maxHeight != -1 && value.maxHeight >= maxHeight -> value.maxHeight
(maxHeight == -1 || value.maxHeight < maxHeight) && removed?.maxHeight == maxHeight ->
findNextMax(maxHeight)
else -> maxHeight
}
}
}
}
fun remove(key: String): VideoConstraints? {
synchronized(lock) {
return map.remove(key)?.also { removed ->
if (removed.maxHeight == maxHeight) {
maxHeight = findNextMax(maxHeight)
}
}
}
}
/**
* Given the previous max, go through the constraints until we either:
* 1) Find a maxHeight equal to the old one or
* 2) Go through the whole list, and track the highest value we've seen.
* Note: this method must be called with the lock held
*/
private fun findNextMax(oldMaxHeight: Int): Int {
var maxSeen = 0
map.values.forEach { constraints ->
if (constraints.maxHeight == oldMaxHeight) {
return oldMaxHeight
} else if (maxSeen >= 0 && (constraints.maxHeight > maxSeen || constraints.maxHeight == -1)) {
maxSeen = constraints.maxHeight
}
}
return maxSeen
}
fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply {
put("maxHeight", maxHeight)
set<ObjectNode>("constraints", ObjectMapper().valueToTree(map.mapValues { it.value.toString() }.toMap()))
}
override fun toString() = getDebugState().toString()
}

471
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt Normal file
View File

@@ -0,0 +1,471 @@
/*
* Copyright @ 2021 - present 8x8, Inc.
* Copyright @ 2021 - Vowel, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import org.jitsi.nlj.MediaSourceDesc
import org.jitsi.nlj.RtpLayerDesc
import org.jitsi.nlj.VideoType
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.LoggerImpl
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config
import java.lang.Integer.max
import java.time.Clock
import kotlin.math.min
/**
* A bitrate allocation that pertains to a specific source. This is the internal representation used in the allocation
* algorithm, as opposed to [SingleAllocation] which is the end result.
*
* @author George Politis
* @author Pawel Domas
*/
internal class SingleSourceAllocation(
val endpointId: String,
val mediaSource: MediaSourceDesc,
/** The constraints to use while allocating bandwidth to this media source. */
val constraints: VideoConstraints,
/** Whether the source is on stage. */
private val onStage: Boolean,
diagnosticContext: DiagnosticContext,
clock: Clock,
parentLogger: Logger = LoggerImpl(SingleSourceAllocation::class.qualifiedName)
) {
val logger = createChildLogger(parentLogger).apply {
addContext(mapOf("remote_endpoint_id" to endpointId))
}
/**
* The immutable list of layers to be considered when allocating bandwidth.
*/
val layers: Layers = selectLayers(mediaSource, onStage, constraints, clock.instant().toEpochMilli())
/**
* The index (into [layers] of the current target layer). It can be improved in the `improve()` step, if there is
* enough bandwidth.
*/
var targetIdx = -1
/**
* The bandwidth (in bps) which was available to this source the last time [improve] was called, or `null` if
* [improve] was never called. The latter means the allocation loop never got as far as this source, because a
* higher priority source had not reached its preferred layer. Note that this is negative when the allocator has
* oversent, so it can not use a negative sentinel for "never called". Only used for logging.
*/
private var lastAvailableBps: Long? = null
/** Whether oversending was allowed the last time [improve] was called. Only used for logging. */
private var lastAllowedOversending = false
init {
if (timeSeriesLogger.isTraceEnabled) {
val ratesTimeSeriesPoint = diagnosticContext.makeTimeSeriesPoint("layers_considered")
.addField("remote_endpoint_id", endpointId)
for (layerSnapshot in layers.layers) {
val l = layerSnapshot.layer
val prefix = "${l.indexString()}_${l.height}p_${l.frameRate}fps"
// Which fields are present must not vary with the data, so that the series can be consumed
// mechanically. Note that the bitrate used for allocation is one of the other two, depending on the
// configuration and on whether the source is bursty.
ratesTimeSeriesPoint.addField("${prefix}_bps", layerSnapshot.bitrate)
ratesTimeSeriesPoint.addField("${prefix}_measured_bps", layerSnapshot.measuredBitrate)
// -1 means the sender does not signal a target bitrate for the layer in the VLA extension.
ratesTimeSeriesPoint.addField("${prefix}_vla_bps", layerSnapshot.vlaTargetBitrate ?: -1)
}
timeSeriesLogger.trace(ratesTimeSeriesPoint)
}
}
fun isOnStage() = onStage
fun hasReachedPreferred(): Boolean = targetIdx >= layers.preferredIndex
/**
* Implements an "improve" step, incrementing [.targetIdx] to the next layer if there is sufficient
* bandwidth. Note that this works eagerly up until the "preferred" layer (if any), and as a single step from
* then on.
*
* @param remainingBps the additional bandwidth which is available on top of the bitrate of the current target
* layer.
* @return the bandwidth "consumed" by the method, i.e. the difference between the resulting and initial target
* bitrate. E.g. if the target bitrate goes from 100 to 300 as a result if the method call, it will return 200.
*/
fun improve(remainingBps: Long, allowOversending: Boolean): Long {
val initialTargetBitrate = targetBitrate
val maxBps = remainingBps + initialTargetBitrate
lastAvailableBps = maxBps
lastAllowedOversending = allowOversending
if (layers.isEmpty()) {
return 0
}
if (targetIdx == -1 && layers.preferredIndex > -1 && onStage) {
// Boost on stage participant to preferred, if there's enough bw.
for (i in layers.indices) {
if (i > layers.preferredIndex || maxBps < layers[i].bitrate) {
break
}
targetIdx = i
}
} else {
// Try the next element in the ratedIndices array.
if (targetIdx + 1 < layers.size && layers[targetIdx + 1].bitrate < maxBps) {
targetIdx++
}
}
if (targetIdx > -1) {
// If there's a higher layer available with a lower bitrate, skip to it.
//
// For example, if 1080p@15fps is configured as a better subjective quality than 720p@30fps (i.e. it sits
// on a higher index in the ratedIndices array) and the bitrate that we measure for the 1080p stream is less
// than the bitrate that we measure for the 720p stream, then we "jump over" the 720p stream and immediately
// select the 1080p stream.
//
// TODO further: Should we just prune the list of layers we consider to not include such layers?
for (i in layers.size - 1 downTo targetIdx + 1) {
if (layers[i].bitrate <= layers[targetIdx].bitrate) {
targetIdx = i
}
}
}
// If oversending is allowed, look for a better layer which doesn't exceed maxBps by more than
// `maxOversendBitrate`.
if (allowOversending && layers.oversendIndex >= 0 && targetIdx < layers.oversendIndex) {
for (i in layers.oversendIndex downTo targetIdx + 1) {
if (layers[i].bitrate <= maxBps + config.maxOversendBitrate.bps) {
targetIdx = i
}
}
}
val resultingTargetBitrate = targetBitrate
return resultingTargetBitrate - initialTargetBitrate
}
/**
* The source is suspended if we've not selected a layer AND the source has active layers.
*
* TODO: this is not exactly correct because it only looks at the layers we consider. E.g. if the receiver set
* a maxHeight=0 constraint for an endpoint, it will appear suspended. This is not critical, because this val is
* only used for logging.
*/
val isSuspended: Boolean
get() = targetIdx == -1 && layers.isNotEmpty() && layers[0].bitrate > 0
/**
* The layer which had to fit in the available bandwidth in order for this source not to be suspended: the first
* layer, which is the one [improve] tries while the source has no target layer.
*/
private val requiredLayer: LayerSnapshot?
get() = layers.firstOrNull()
/**
* The cheapest of the layers which could have kept this source from being suspended, i.e. of [rescuingLayers].
* This is normally [requiredLayer], but the layers are not guaranteed to be ordered by bitrate (notably with VP9),
* and when oversending it is the cheapest layer which decides whether the source can be rescued.
*/
private val cheapestLayer: LayerSnapshot?
get() = rescuingLayers.minByOrNull { it.bitrate }
/**
* The layers which the last [improve] call could have selected while the source had no target layer. That is just
* the first one, unless oversending was allowed, in which case the oversend loop scans everything up to and
* including [Layers.oversendIndex] (but no further).
*/
private val rescuingLayers: List<LayerSnapshot>
get() = layers.take(
if (lastAllowedOversending && layers.oversendIndex >= 0) layers.oversendIndex + 1 else 1
)
/**
* A summary of why this source ended up suspended, for logging. Only meaningful once the allocation algorithm has
* completed, and only for a source which is actually suspended (i.e. [isSuspended] is true) -- for any other source
* the fields describe an allocation which did succeed. All bitrates are in bps:
*
* "required" is the layer which had to fit in the available bandwidth, i.e. what the allocator thought the source
* needed; "cheapest" is only present when a different layer could have rescued the source more cheaply; "ideal" is
* the bitrate of the layer we'd have forwarded with unlimited bandwidth, and is omitted when that is the required
* layer; "oversend" describes the layer we'd have used when oversending (capped by
* videobridge.cc.max-oversend-bitrate) -- its bitrate, or "required" when it is the required layer, or "notAllowed"
* when this source was not the one source which is allowed to oversend -- and is absent when the source has no
* oversend layer at all; "available" is the bandwidth which was left for the source when it was last considered,
* and is negative if the allocator had already oversent. The bitrates of the layers which are not called out here
* are only available in the "layers_considered" time series.
*/
fun suspendedSummary(): String = buildString {
append(mediaSource.sourceName)
append("[type=${mediaSource.videoType}")
append(" onStage=$onStage")
append(" required=${requiredLayer?.summary()}")
cheapestLayer?.let { if (it !== requiredLayer) append(" cheapest=${it.summary()}") }
if (idealBitrate != requiredLayer?.bitrate) append(" ideal=$idealBitrate")
layers.oversendLayer?.let {
val oversend = when {
!lastAllowedOversending -> "notAllowed"
it === requiredLayer -> "required"
else -> it.bitrate.toString()
}
append(" oversend=$oversend")
}
append(" available=${lastAvailableBps ?: "none(never considered)"}")
append(" constraints=$constraints]")
}
/**
* Gets the target bitrate (in bps) for this endpoint allocation, i.e. the bitrate of the currently chosen layer.
*/
val targetBitrate: Long
get() = targetLayer?.bitrate ?: 0
private val targetLayer: LayerSnapshot?
get() = layers.getOrNull(targetIdx)
/**
* Gets the ideal bitrate (in bps) for this endpoint allocation, i.e. the bitrate of the layer the bridge would
* forward if there were no (bandwidth) constraints.
*/
val idealBitrate: Long
get() = layers.idealLayer?.bitrate ?: 0
/**
* Exposed for testing only.
*/
val preferredLayer: RtpLayerDesc?
get() = layers.preferredLayer?.layer
/**
* Exposed for testing only.
*/
val oversendLayer: RtpLayerDesc?
get() = layers.oversendLayer?.layer
/**
* Creates the final immutable result of this allocation. Should be called once the allocation algorithm has
* completed.
*/
val result: SingleAllocation
get() = SingleAllocation(
endpointId,
mediaSource,
targetLayer?.layer,
layers.idealLayer?.layer
)
override fun toString(): String = (
"[id=" + endpointId +
" constraints=" + constraints +
" ratedPreferredIdx=" + layers.preferredIndex +
" ratedTargetIdx=" + targetIdx
)
/**
* Selects from a list of layers the ones which should be considered when allocating bandwidth, as well as the
* "preferred" and "oversend" layers. Logic specific to screensharing: we prioritize resolution over framerate,
* prioritize the full resolution (at the frame rate which is cheapest to send it at) over other endpoints by
* setting it as "preferred", and allow oversending up to the full resolution (with low frame rate). Above the
* preferred layer the screenshare's frame rate competes with the other sources, which may reduce (though not
* disable) their quality as it improves.
*/
private fun selectLayersForScreensharing(
layers: List<LayerSnapshot>,
constraints: VideoConstraints,
onStage: Boolean,
/** Whether the sender has asked for the share's frame rate to be prioritized. */
highFps: Boolean
): Layers {
var activeLayers = layers.filter { it.bitrate > 0 }
// No active layers usually happens when the source has just been signaled and we haven't received
// any packets yet. Add the layers here, so one gets selected and we can start forwarding sooner.
if (activeLayers.isEmpty()) activeLayers = layers
// We select all layers that satisfy the constraints.
var selectedLayers =
if (!constraints.heightIsLimited()) {
activeLayers
} else {
activeLayers.filter { it.layer.height <= constraints.maxHeight }
}
// If no layers satisfy the constraints, we use the layers with the lowest resolution.
if (selectedLayers.isEmpty()) {
val minHeight = activeLayers.minOfOrNull { it.layer.height } ?: return Layers.noLayers
selectedLayers = activeLayers.filter { it.layer.height == minHeight }
// This recognizes the structure used with VP9 (multiple encodings with the same resolution and unknown frame
// rate). In this case, we only want the low quality layer. Unless we're on stage, in which case we should
// consider all layers.
if (!onStage && selectedLayers.isNotEmpty() && selectedLayers[0].layer.frameRate < 0) {
selectedLayers = listOf(selectedLayers[0])
}
}
// A source always has at least one layer here (selectLayers returns early otherwise), and the branches above
// never leave selectedLayers empty, so the maxima below exist.
val maxHeight = selectedLayers.maxOf { it.layer.height }
val maxHeightIndices = selectedLayers.indices.filter { selectedLayers[it].layer.height == maxHeight }
// Of all layers with the highest resolution, the one with the lowest bitrate. In case of VP9 the layers are
// not necessarily ordered by bitrate.
val cheapestMaxHeightIdx = maxHeightIndices.minBy { selectedLayers[it].bitrate }
// Since we prioritize resolution over frame rate, "preferred" is the full resolution at the frame rate which
// is cheapest to send it at. Anything above that improves frame rate only, and competes with the other
// sources on equal terms rather than blocking them entirely. There are two exceptions, for which we keep the
// highest layer as "preferred" (as we previously did for all screen sharing):
// - High frame rate screen sharing, where the sender has explicitly asked for frame rate to be prioritized.
// - The structure used with VP9 (multiple encodings with the same resolution and unknown frame rate), where
// the layers above the cheapest one improve quality rather than frame rate, and quality is exactly what
// this policy exists to protect.
val preferredIdx = if (highFps || maxHeightIndices.any { selectedLayers[it].layer.frameRate < 0 }) {
selectedLayers.size - 1
} else {
cheapestMaxHeightIdx
}
val oversendIdx = if (onStage && config.allowOversendOnStage) cheapestMaxHeightIdx else -1
return Layers(selectedLayers, preferredIdx, oversendIdx)
}
/**
* Selects from the layers of a [MediaSourceContainer] the ones which should be considered when allocating bandwidth for
* an endpoint. Also selects the indices of the "preferred" and "oversend" layers.
*
* @param endpoint the [MediaSourceContainer] that describes the available layers.
* @param constraints the constraints signaled for the endpoint.
* @return the ordered list of [endpoint]'s layers which should be considered when allocating bandwidth, as well as the
* indices of the "preferred" and "oversend" layers.
*/
private fun selectLayers(
/** The endpoint which is the source of the stream(s). */
source: MediaSourceDesc,
onStage: Boolean,
/** The constraints that the receiver specified for [source]. */
constraints: VideoConstraints,
nowMs: Long
): Layers {
if (constraints.maxHeight == 0 || !source.hasRtpLayers()) {
return Layers.noLayers
}
// Screen sharing is bursty: while the screen is static the encoder accumulates rate control credit (about a
// second's worth) and then spends it on a single frame, so the bitrate measured over the short window is not
// representative of the bandwidth the source needs. Allocate for it using the longer window instead.
val isBursty = source.videoType.isScreenshare()
val layers = source.rtpLayers.map {
if (isBursty) {
// Only bursty sources need the extra tracker, so it is not created until we ask for it.
it.enableSmoothedBitrate(nowMs)
} else {
// And when a source stops being bursty, drop the tracker so its layers stop paying for it.
it.disableSmoothedBitrate()
}
val measuredBitrate = it.getBitrate(nowMs).bps
val vlaTargetBitrate = it.targetBitrate?.bps
val allocationBitrate = when {
config.useVlaTargetBitrate && vlaTargetBitrate != null -> vlaTargetBitrate
isBursty -> it.getSmoothedBitrate(nowMs).bps
else -> measuredBitrate
}
LayerSnapshot(it, allocationBitrate, measuredBitrate, vlaTargetBitrate)
}
return when {
source.videoType == VideoType.CAMERA -> selectLayersForCamera(layers, constraints)
source.videoType.isScreenshare() -> selectLayersForScreensharing(
layers,
constraints,
onStage,
highFps = source.videoType == VideoType.DESKTOP_HIGH_FPS
)
else -> Layers.noLayers
}
}
/**
* Selects from a list of layers the ones which should be considered when allocating bandwidth, as well as the
* "preferred" and "oversend" layers. Logic specific to a camera stream: once the "preferred" height is reached we
* require a high frame rate, with preconfigured values for the "preferred" height and frame rate, and we do not allow
* oversending.
*/
private fun selectLayersForCamera(layers: List<LayerSnapshot>, constraints: VideoConstraints): Layers {
val minHeight = layers.minOfOrNull { it.layer.height } ?: return Layers.noLayers
val maxFps = layers.maxOfOrNull { it.layer.frameRate } ?: return Layers.noLayers
val noActiveLayers = layers.none { (_, bitrate) -> bitrate > 0 }
val (preferredHeight, preferredFps) = getPreferred(constraints)
val effectivePreferredFps = if (maxFps > 0) {
min(maxFps, preferredFps)
} else {
preferredFps
}
val ratesList: MutableList<LayerSnapshot> = ArrayList()
// Initialize the list of layers to be considered. These are the layers that satisfy the constraints, with
// a couple of exceptions (see comments below).
for (layerSnapshot in layers) {
val layer = layerSnapshot.layer
val lessThanPreferredHeight = layer.height < preferredHeight
val lessThanOrEqualMaxHeight = layer.height <= constraints.maxHeight || !constraints.heightIsLimited()
// If frame rate is unknown, consider it to be sufficient.
val atLeastPreferredFps = layer.frameRate < 0 || layer.frameRate >= effectivePreferredFps
if (lessThanPreferredHeight ||
(lessThanOrEqualMaxHeight && atLeastPreferredFps) ||
layer.height == minHeight
) {
// No active layers usually happens when the source has just been signaled and we haven't received
// any packets yet. Add the layers here, so one gets selected and we can start forwarding sooner.
if (noActiveLayers || layerSnapshot.bitrate > 0) {
ratesList.add(layerSnapshot)
}
}
}
val effectivePreferredHeight = max(preferredHeight, minHeight)
val preferredIndex = ratesList.lastIndexWhich { it.layer.height <= effectivePreferredHeight }
logger.trace {
"Selected rates list $ratesList, preferred index $preferredIndex " +
"from layers $layers with constraints $constraints"
}
return Layers(ratesList, preferredIndex, -1)
}
companion object {
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BandwidthAllocator::class.java)
}
}
/**
* Returns the index of the last element of this list which satisfies the given predicate, or -1 if no elements do.
*/
private fun <T> List<T>.lastIndexWhich(predicate: (T) -> Boolean): Int {
var lastIndex = -1
forEachIndexed { i, e -> if (predicate(e)) lastIndex = i }
return lastIndex
}
/**
* Gets the "preferred" height and frame rate based on the constraints signaled from the receiver.
*
* For participants with sufficient maxHeight we favor frame rate over resolution. We consider all
* temporal layers for resolutions lower than the preferred, but for resolutions >= preferred, we only
* consider frame rates at least as high as the preferred. In practice this means we consider
* 180p/7.5fps, 180p/15fps, 180p/30fps, 360p/30fps and 720p/30fps.
*/
private fun getPreferred(constraints: VideoConstraints): VideoConstraints =
if (constraints.maxHeight > 180 || !constraints.heightIsLimited()) {
VideoConstraints(config.onstagePreferredHeightPx, config.onstagePreferredFramerate)
} else {
VideoConstraints.UNLIMITED
}

49
jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt Normal file
View File

@@ -0,0 +1,49 @@
/*
* Copyright @ 2020 - present 8x8, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.allocation
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.databind.node.JsonNodeFactory
@JsonIgnoreProperties(ignoreUnknown = true)
data class VideoConstraints @JvmOverloads constructor(
val maxHeight: Int,
val maxFrameRate: Double = UNLIMITED_FRAME_RATE
) {
init {
require(maxHeight == UNLIMITED_HEIGHT || maxHeight >= 0) { "maxHeight must be either -1, 0, or positive." }
require(maxFrameRate == UNLIMITED_FRAME_RATE || maxFrameRate >= 0.0) {
"maxFrameRate must be either -1, or >= 0"
}
}
override fun toString(): String = JsonNodeFactory.instance.objectNode().apply {
put("maxHeight", maxHeight)
put("maxFrameRate", maxFrameRate)
}.toString()
fun heightIsLimited() = maxHeight != UNLIMITED_HEIGHT
fun frameRateIsLimited() = maxFrameRate != UNLIMITED_FRAME_RATE
fun isDisabled() = maxHeight == 0 || maxFrameRate == 0.0
companion object {
const val UNLIMITED_HEIGHT = -1
const val UNLIMITED_FRAME_RATE = -1.0
val NOTHING = VideoConstraints(0)
val UNLIMITED = VideoConstraints(UNLIMITED_HEIGHT, UNLIMITED_FRAME_RATE)
}
}
fun Map<String, VideoConstraints>.prettyPrint(): String = entries.joinToString { "${it.key}->${it.value.maxHeight}" }

694
jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt Normal file
View File

@@ -0,0 +1,694 @@
/*
* Copyright @ 2019-present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.av1
import com.fasterxml.jackson.databind.node.ArrayNode
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import org.jitsi.nlj.PacketInfo
import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex
import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket
import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc
import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex
import org.jitsi.nlj.rtp.codec.av1.getTemplateIdDelta
import org.jitsi.rtp.rtcp.RtcpSrPacket
import org.jitsi.rtp.rtp.header_extensions.DTI
import org.jitsi.rtp.rtp.header_extensions.toShortString
import org.jitsi.rtp.util.RtpUtils
import org.jitsi.rtp.util.isNewerThan
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import org.jitsi.videobridge.cc.AdaptiveSourceProjectionContext
import org.jitsi.videobridge.cc.RewriteException
import org.jitsi.videobridge.cc.RtpState
import java.time.Duration
import java.time.Instant
class Av1DDAdaptiveSourceProjectionContext(
private val diagnosticContext: DiagnosticContext,
rtpState: RtpState,
persistentState: Any?,
parentLogger: Logger
) : AdaptiveSourceProjectionContext {
private val logger: Logger = createChildLogger(parentLogger)
/**
* A map that stores the per-encoding AV1 frame maps.
*/
private val av1FrameMaps = HashMap<Long, Av1DDFrameMap>()
/**
* The [Av1DDQualityFilter] instance that does quality filtering on the
* incoming pictures, to choose encodings and layers to forward.
*/
private val av1QualityFilter = Av1DDQualityFilter(av1FrameMaps, logger)
init {
require(persistentState is Av1PersistentState?)
}
private var lastAv1FrameProjection = Av1DDFrameProjection(
diagnosticContext,
rtpState.ssrc,
rtpState.maxSequenceNumber,
rtpState.maxTimestamp,
(persistentState as? Av1PersistentState)?.frameNumber,
(persistentState as? Av1PersistentState)?.templateId
)
/**
* The frame number index that started the latest stream resumption.
* We can't send frames with frame number less than this, because we don't have
* space in the projected sequence number/frame number counts.
*/
private var lastFrameNumberIndexResumption = -1L
override fun accept(packetInfo: PacketInfo, targetIndex: Int): Boolean {
val packet = packetInfo.packet
if (packet !is Av1DDPacket) {
logger.warn("Packet is not AV1 DD Packet")
return false
}
val incomingEncoding = packet.encodingId
/* If insertPacketInMap returns null, this is a very old picture, more than Av1FrameMap.PICTURE_MAP_SIZE old,
or something is wrong with the stream. */
val result = insertPacketInMap(packet) ?: return false
val frame = result.frame
if (result.isNewFrame) {
if (packet.isKeyframe && frameIsNewSsrc(frame)) {
/* If we're not currently projecting this SSRC, check if we've
* already decided to drop a subsequent required frame of this SSRC for the DT.
* If we have, we can't turn on the encoding starting from this
* packet, so treat this frame as though it weren't a keyframe.
* Note that this may mean re-analyzing the packets with a now-available template dependency structure.
*/
if (haveSubsequentNonAcceptedChain(frame, incomingEncoding, targetIndex)) {
frame.isKeyframe = false
}
}
val receivedTime = packetInfo.receivedTime
val acceptResult = av1QualityFilter
.acceptFrame(frame, incomingEncoding, targetIndex, receivedTime)
frame.isAccepted = acceptResult.accept && frameIsProjectable(frame)
if (frame.isAccepted) {
val projection: Av1DDFrameProjection
try {
projection = createProjection(
frame = frame,
initialPacket = packet,
isResumption = acceptResult.isResumption,
isReset = result.isReset,
mark = acceptResult.mark,
newDt = acceptResult.newDt,
receivedTime = receivedTime
)
} catch (e: Exception) {
logger.warn("Failed to create frame projection", e)
/* Make sure we don't have an accepted frame without a projection in the map. */
frame.isAccepted = false
return false
}
frame.projection = projection
if (projection.earliestProjectedSeqNum isNewerThan lastAv1FrameProjection.latestProjectedSeqNum) {
lastAv1FrameProjection = projection
}
}
}
val accept = frame.isAccepted &&
if (frame.projection?.accept(packet) == true) {
true
} else {
if (frame.projection != null && frame.projection?.closedSeq != -1) {
logger.debug(
"Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}"
)
} else if (frame.projection == null) {
logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it")
} else {
logger.warn("Not accepting $packet, even though frame projection is not closed")
}
false
}
if (timeSeriesLogger.isTraceEnabled) {
val pt = diagnosticContext.makeTimeSeriesPoint("rtp_av1")
.addField("ssrc", packet.ssrc)
.addField("timestamp", packet.timestamp)
.addField("seq", packet.sequenceNumber)
.addField("frameNumber", packet.frameNumber)
.addField("templateId", packet.statelessDescriptor.frameDependencyTemplateId)
.addField("hasStructure", packet.descriptor?.newTemplateDependencyStructure != null)
.addField("spatialLayer", packet.frameInfo?.spatialId)
.addField("temporalLayer", packet.frameInfo?.temporalId)
.addField("dti", packet.frameInfo?.dti?.toShortString())
.addField("hasInterPictureDependency", packet.frameInfo?.hasInterPictureDependency())
// TODO add more relevant fields from AV1 DD for debugging
.addField("targetIndex", Av1DDRtpLayerDesc.indexString(targetIndex))
.addField("new_frame", result.isNewFrame)
.addField("accept", accept)
.addField("payload_length", packet.payloadLength)
.addField("packet_length", packet.length)
av1QualityFilter.addDiagnosticContext(pt)
timeSeriesLogger.trace(pt)
}
return accept
}
/** Look up an Av1DDFrame for a packet. */
private fun lookupAv1Frame(av1Packet: Av1DDPacket): Av1DDFrame? = av1FrameMaps[av1Packet.ssrc]?.findFrame(av1Packet)
/**
* Insert a packet in the appropriate [Av1DDFrameMap].
*/
private fun insertPacketInMap(av1Packet: Av1DDPacket) =
av1FrameMaps.getOrPut(av1Packet.ssrc) { Av1DDFrameMap(logger) }.insertPacket(av1Packet)
private fun haveSubsequentNonAcceptedChain(frame: Av1DDFrame, incomingEncoding: Int, targetIndex: Int): Boolean {
val map = av1FrameMaps[frame.ssrc] ?: return false
val structure = frame.structure ?: return false
val dtsToCheck = if (incomingEncoding == getEidFromIndex(targetIndex)) {
setOf(getDtFromIndex(targetIndex))
} else {
frame.frameInfo?.dtisPresent ?: emptySet()
}
val chainsToCheck = dtsToCheck.mapNotNull { structure.decodeTargetProtectedBy.getOrNull(it) }.toSet()
return map.nextFrameWith(frame) {
if (it.isAccepted) return@nextFrameWith false
if (it.frameInfo == null) {
it.updateParse(structure, logger)
}
return@nextFrameWith chainsToCheck.any { chainIdx ->
it.partOfActiveChain(chainIdx)
}
} != null
}
private fun Av1DDFrame.partOfActiveChain(chainIdx: Int): Boolean {
val structure = structure ?: return false
val frameInfo = frameInfo ?: return false
for (i in structure.decodeTargetProtectedBy.indices) {
if (structure.decodeTargetProtectedBy[i] != chainIdx) {
continue
}
if (frameInfo.dti[i] == DTI.NOT_PRESENT || frameInfo.dti[i] == DTI.DISCARDABLE) {
return false
}
}
return true
}
/**
* Calculate the projected sequence number gap between two frames (of the same encoding),
* allowing collapsing for unaccepted frames.
*/
private fun seqGap(frame1: Av1DDFrame, frame2: Av1DDFrame): Int {
var seqGap = RtpUtils.getSequenceNumberDelta(
frame2.earliestKnownSequenceNumber,
frame1.latestKnownSequenceNumber
)
if (!frame1.isAccepted && !frame2.isAccepted && frame2.isImmediatelyAfter(frame1)) {
/* If neither frame is being projected, and they have consecutive
frame numbers, we don't need to leave any gap. */
seqGap = 0
} else {
/* If the earlier frame wasn't projected, and we haven't seen its
* final packet, we know it has to consume at least one more sequence number. */
if (!frame1.isAccepted && !frame1.seenEndOfFrame && seqGap > 1) {
seqGap--
}
/* Similarly, if the later frame wasn't projected and we haven't seen
* its first packet. */
if (!frame2.isAccepted && !frame2.seenStartOfFrame && seqGap > 1) {
seqGap--
}
/* If the frame wasn't accepted, it has to have consumed at least one sequence number,
* which we can collapse out. */
if (!frame1.isAccepted && seqGap > 0) {
seqGap--
}
}
return seqGap
}
private fun frameIsNewSsrc(frame: Av1DDFrame): Boolean = lastAv1FrameProjection.av1Frame?.matchesSSRC(frame) != true
private fun frameIsProjectable(frame: Av1DDFrame): Boolean =
frameIsNewSsrc(frame) || frame.index >= lastFrameNumberIndexResumption
/**
* Find the previous frame before the given one.
*/
@Synchronized
private fun prevFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.prevFrame(frame)
/**
* Find the next frame after the given one.
*/
@Synchronized
private fun nextFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.nextFrame(frame)
/**
* Find the previous accepted frame before the given one.
*/
private fun findPrevAcceptedFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.findPrevAcceptedFrame(frame)
/**
* Find the next accepted frame after the given one.
*/
private fun findNextAcceptedFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.findNextAcceptedFrame(frame)
/**
* Create a projection for this frame.
*/
private fun createProjection(
frame: Av1DDFrame,
initialPacket: Av1DDPacket,
mark: Boolean,
isResumption: Boolean,
isReset: Boolean,
newDt: Int?,
receivedTime: Instant?
): Av1DDFrameProjection {
if (frameIsNewSsrc(frame)) {
return createEncodingSwitchProjection(frame, initialPacket, mark, newDt, receivedTime)
} else if (isReset) {
return createResetProjection(frame, initialPacket, mark, newDt, receivedTime)
} else if (isResumption) {
return createResumptionProjection(frame, initialPacket, mark, newDt, receivedTime)
}
return createInEncodingProjection(frame, initialPacket, mark, newDt, receivedTime)
}
/**
* Create a projection for the first frame after an encoding switch.
*/
private fun createEncodingSwitchProjection(
frame: Av1DDFrame,
initialPacket: Av1DDPacket,
mark: Boolean,
newDt: Int?,
receivedTime: Instant?
): Av1DDFrameProjection {
// We can only switch on packets that carry a scalability structure, which is the first packet of a keyframe
assert(frame.isKeyframe)
assert(initialPacket.isStartOfFrame)
lastFrameNumberIndexResumption = frame.index
var projectedSeqGap = 1
if (lastAv1FrameProjection.av1Frame?.seenEndOfFrame == false) {
/* Leave a gap to signal to the decoder that the previously routed
frame was incomplete. */
projectedSeqGap++
/* Make sure subsequent packets of the previous projection won't
overlap the new one. (This means the gap, above, will never be
filled in.)
*/
lastAv1FrameProjection.close()
}
val projectedSeq =
RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.latestProjectedSeqNum, projectedSeqGap)
// this is a simulcast switch. The typical incremental value =
// 90kHz / 30 = 90,000Hz / 30 = 3000 per frame or per 33ms
val tsDelta = lastAv1FrameProjection.created?.let { created ->
receivedTime?.let {
3000 * Duration.between(created, receivedTime).dividedBy(33).seconds.coerceAtLeast(1L)
}
} ?: 3000
val projectedTs = RtpUtils.applyTimestampDelta(lastAv1FrameProjection.timestamp, tsDelta)
val frameNumber: Int
val templateIdDelta: Int
val nextTemplateId = lastAv1FrameProjection.getNextTemplateId()
if (nextTemplateId != null) {
frameNumber = RtpUtils.applySequenceNumberDelta(
lastAv1FrameProjection.frameNumber,
1
)
val structure = frame.structure
check(structure != null)
templateIdDelta = getTemplateIdDelta(nextTemplateId, structure.templateIdOffset)
} else {
frameNumber = frame.frameNumber
templateIdDelta = 0
}
return Av1DDFrameProjection(
diagnosticContext = diagnosticContext,
av1Frame = frame,
ssrc = lastAv1FrameProjection.ssrc,
timestamp = projectedTs,
sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber),
frameNumber = frameNumber,
templateIdDelta = templateIdDelta,
dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) },
mark = mark,
created = receivedTime
)
}
/**
* Create a projection for the first frame after a resumption, i.e. when a source is turned back on.
*/
private fun createResumptionProjection(
frame: Av1DDFrame,
initialPacket: Av1DDPacket,
mark: Boolean,
newDt: Int?,
receivedTime: Instant?
): Av1DDFrameProjection {
lastFrameNumberIndexResumption = frame.index
/* These must be non-null because we don't execute this function unless
frameIsNewSsrc and isReset were both false.
*/
val lastFrame = prevFrame(frame)!!
val lastProjectedFrame = lastAv1FrameProjection.av1Frame!!
/* Project timestamps linearly. */
val tsDelta = RtpUtils.getTimestampDiff(
lastAv1FrameProjection.timestamp,
lastProjectedFrame.timestamp
)
val projectedTs = RtpUtils.applyTimestampDelta(frame.timestamp, tsDelta)
/* Increment frameNumber by 1 from the last projected frame. */
val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.frameNumber, 1)
/** If this packet has a template structure, rewrite it to follow after the pre-resumption structure.
* (We could check to see if the structure is unchanged, but that's an unnecessary optimization.)
*/
val templateIdDelta = if (frame.structure != null) {
val nextTemplateId = lastAv1FrameProjection.getNextTemplateId()
if (nextTemplateId != null) {
val structure = frame.structure
check(structure != null)
getTemplateIdDelta(nextTemplateId, structure.templateIdOffset)
} else {
0
}
} else {
0
}
/* Increment sequence numbers based on the last projected frame, but leave a gap
* for packet reordering in case this isn't the first packet of the keyframe.
*/
val seqGap = RtpUtils.getSequenceNumberDelta(initialPacket.sequenceNumber, lastFrame.latestKnownSequenceNumber)
val newSeq = RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.latestProjectedSeqNum, seqGap)
val seqDelta = RtpUtils.getSequenceNumberDelta(newSeq, initialPacket.sequenceNumber)
return Av1DDFrameProjection(
diagnosticContext = diagnosticContext,
av1Frame = frame,
ssrc = lastAv1FrameProjection.ssrc,
timestamp = projectedTs,
sequenceNumberDelta = seqDelta,
frameNumber = projectedFrameNumber,
templateIdDelta = templateIdDelta,
dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) },
mark = mark,
created = receivedTime
)
}
/**
* Create a projection for the first frame after a frame reset, i.e. after a large gap in sequence numbers.
*/
private fun createResetProjection(
frame: Av1DDFrame,
initialPacket: Av1DDPacket,
mark: Boolean,
newDt: Int?,
receivedTime: Instant?
): Av1DDFrameProjection {
/* This must be non-null because we don't execute this function unless
frameIsNewSsrc has returned false.
*/
val lastFrame = lastAv1FrameProjection.av1Frame!!
/* Apply the latest projected frame's projections out, linearly. */
val seqDelta = RtpUtils.getSequenceNumberDelta(
lastAv1FrameProjection.latestProjectedSeqNum,
lastFrame.latestKnownSequenceNumber
)
val tsDelta = RtpUtils.getTimestampDiff(
lastAv1FrameProjection.timestamp,
lastFrame.timestamp
)
val frameNumberDelta = RtpUtils.applySequenceNumberDelta(
lastAv1FrameProjection.frameNumber,
lastFrame.frameNumber
)
val projectedTs = RtpUtils.applyTimestampDelta(frame.timestamp, tsDelta)
val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(frame.frameNumber, frameNumberDelta)
/** If this packet has a template structure, rewrite it to follow after the pre-reset structure.
* (We could check to see if the structure is unchanged, but that's an unnecessary optimization.)
*/
val templateIdDelta = if (frame.structure != null) {
val nextTemplateId = lastAv1FrameProjection.getNextTemplateId()
if (nextTemplateId != null) {
val structure = frame.structure
check(structure != null)
getTemplateIdDelta(nextTemplateId, structure.templateIdOffset)
} else {
0
}
} else {
0
}
return Av1DDFrameProjection(
diagnosticContext = diagnosticContext,
av1Frame = frame,
ssrc = lastAv1FrameProjection.ssrc,
timestamp = projectedTs,
sequenceNumberDelta = seqDelta,
frameNumber = projectedFrameNumber,
templateIdDelta = templateIdDelta,
dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) },
mark = mark,
created = receivedTime
)
}
/**
* Create a frame projection for the normal case, i.e. as part of the same encoding as the
* previously-projected frame.
*/
private fun createInEncodingProjection(
frame: Av1DDFrame,
initialPacket: Av1DDPacket,
mark: Boolean,
newDt: Int?,
receivedTime: Instant?
): Av1DDFrameProjection {
val prevFrame = findPrevAcceptedFrame(frame)
if (prevFrame != null) {
return createInEncodingProjection(frame, prevFrame, initialPacket, mark, newDt, receivedTime)
}
/* prev frame has rolled off beginning of frame map, try next frame */
val nextFrame = findNextAcceptedFrame(frame)
if (nextFrame != null) {
return createInEncodingProjection(frame, nextFrame, initialPacket, mark, newDt, receivedTime)
}
/* Neither previous or next is found. Very big frame? Use previous projected.
(This must be valid because we don't execute this function unless
frameIsNewSsrc has returned false.)
*/
return createInEncodingProjection(
frame,
lastAv1FrameProjection.av1Frame!!,
initialPacket,
mark,
newDt,
receivedTime
)
}
/**
* Create a frame projection for the normal case, i.e. as part of the same encoding as the
* previously-projected frame, based on a specific chosen previously-projected frame.
*/
private fun createInEncodingProjection(
frame: Av1DDFrame,
refFrame: Av1DDFrame,
initialPacket: Av1DDPacket,
mark: Boolean,
newDt: Int?,
receivedTime: Instant?
): Av1DDFrameProjection {
val tsGap = RtpUtils.getTimestampDiff(frame.timestamp, refFrame.timestamp)
val frameNumGap = RtpUtils.getSequenceNumberDelta(frame.frameNumber, refFrame.frameNumber)
var seqGap = 0
var f1 = refFrame
var f2: Av1DDFrame?
val refSeq: Int
if (frameNumGap > 0) {
/* refFrame is earlier than frame in decode order. */
do {
f2 = nextFrame(f1)
checkNotNull(f2) {
"No next frame found after frame with frame number ${f1.frameNumber}, " +
"even though refFrame ${refFrame.frameNumber} is before " +
"frame ${frame.frameNumber}!"
}
seqGap += seqGap(f1, f2)
f1 = f2
} while (f2 !== frame)
/* refFrame is a projected frame, so it has a projection. */
refSeq = refFrame.projection!!.latestProjectedSeqNum
} else {
/* refFrame is later than frame in decode order. */
do {
f2 = prevFrame(f1)
checkNotNull(f2) {
"No previous frame found before frame with frame number ${f1.frameNumber}, " +
"even though refFrame ${refFrame.frameNumber} is after " +
"frame ${frame.frameNumber}!"
}
seqGap += -seqGap(f2, f1)
f1 = f2
} while (f2 !== frame)
refSeq = refFrame.projection!!.earliestProjectedSeqNum
}
val projectedSeq = RtpUtils.applySequenceNumberDelta(refSeq, seqGap)
val projectedTs = RtpUtils.applyTimestampDelta(refFrame.projection!!.timestamp, tsGap)
val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(refFrame.projection!!.frameNumber, frameNumGap)
return Av1DDFrameProjection(
diagnosticContext = diagnosticContext,
av1Frame = frame,
ssrc = lastAv1FrameProjection.ssrc,
timestamp = projectedTs,
sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber),
frameNumber = projectedFrameNumber,
templateIdDelta = lastAv1FrameProjection.templateIdDelta,
dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) },
mark = mark,
created = receivedTime
)
}
override fun needsKeyframe(): Boolean {
if (av1QualityFilter.needsKeyframe) {
return true
}
return lastAv1FrameProjection.av1Frame == null
}
override fun rewriteRtp(packetInfo: PacketInfo) {
if (packetInfo.packet !is Av1DDPacket) {
logger.info("Got a non-AV1 DD packet.")
throw RewriteException("Non-AV1 DD packet in AV1 DD source projection")
}
val av1Packet = packetInfo.packetAs<Av1DDPacket>()
val av1Frame = lookupAv1Frame(av1Packet)
?: throw RewriteException("Frame not in tracker (aged off?)")
val av1Projection = av1Frame.projection
?: throw RewriteException("Frame does not have projection?")
/* Shouldn't happen for an accepted packet whose frame is still known? */
logger.trace { "Rewriting packet with structure ${System.identityHashCode(av1Packet.descriptor?.structure)}" }
av1Projection.rewriteRtp(av1Packet)
}
override fun rewriteRtcp(rtcpSrPacket: RtcpSrPacket): Boolean {
val lastAv1FrameProjectionCopy: Av1DDFrameProjection = lastAv1FrameProjection
if (rtcpSrPacket.senderSsrc != lastAv1FrameProjectionCopy.av1Frame?.ssrc) {
return false
}
rtcpSrPacket.senderSsrc = lastAv1FrameProjectionCopy.ssrc
val srcTs = rtcpSrPacket.senderInfo.rtpTimestamp
val delta = RtpUtils.getTimestampDiff(
lastAv1FrameProjectionCopy.timestamp,
lastAv1FrameProjectionCopy.av1Frame.timestamp
)
val dstTs = RtpUtils.applyTimestampDelta(srcTs, delta)
if (srcTs != dstTs) {
rtcpSrPacket.senderInfo.rtpTimestamp = dstTs
}
return true
}
override fun getRtpState() = RtpState(
lastAv1FrameProjection.ssrc,
lastAv1FrameProjection.latestProjectedSeqNum,
lastAv1FrameProjection.timestamp
)
override fun getPersistentState(): Any = Av1PersistentState(
lastAv1FrameProjection.frameNumber,
lastAv1FrameProjection.getNextTemplateId() ?: 0
)
override fun getDebugState(): ObjectNode {
val debugState = JsonNodeFactory.instance.objectNode()
debugState.put("class", Av1DDAdaptiveSourceProjectionContext::class.java.simpleName)
val mapSizes: ArrayNode = JsonNodeFactory.instance.arrayNode()
for ((key, value) in av1FrameMaps.entries) {
val sizeInfo = JsonNodeFactory.instance.objectNode()
sizeInfo.put("ssrc", key)
sizeInfo.put("size", value.size())
mapSizes.add(sizeInfo)
}
debugState.set<ObjectNode>("av1FrameMaps", mapSizes)
debugState.set<ObjectNode>("av1QualityFilter", av1QualityFilter.debugState)
return debugState
}
companion object {
/**
* The time series logger for this class.
*/
private val timeSeriesLogger =
TimeSeriesLogger.getTimeSeriesLogger(Av1DDAdaptiveSourceProjectionContext::class.java)
}
}
data class Av1PersistentState(
val frameNumber: Int,
val templateId: Int
)

303
jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt Normal file
View File

@@ -0,0 +1,303 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.av1
import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket
import org.jitsi.rtp.rtp.RtpPacket
import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader
import org.jitsi.rtp.rtp.header_extensions.Av1DependencyException
import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure
import org.jitsi.rtp.rtp.header_extensions.FrameInfo
import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta
import org.jitsi.rtp.util.isNewerThan
import org.jitsi.rtp.util.isOlderThan
import org.jitsi.utils.logging2.Logger
class Av1DDFrame internal constructor(
/**
* The RTP SSRC of the incoming frame that this instance refers to
* (RFC3550).
*/
val ssrc: Long,
/**
* The RTP timestamp of the incoming frame that this instance refers to
* (RFC3550).
*/
val timestamp: Long,
/**
* The earliest RTP sequence number seen of the incoming frame that this instance
* refers to (RFC3550).
*/
earliestKnownSequenceNumber: Int,
/**
* The latest RTP sequence number seen of the incoming frame that this instance
* refers to (RFC3550).
*/
latestKnownSequenceNumber: Int,
/**
* A boolean that indicates whether we've seen the first packet of the frame.
* If so, its sequence is earliestKnownSequenceNumber.
*/
seenStartOfFrame: Boolean,
/**
* A boolean that indicates whether we've seen the last packet of the frame.
* If so, its sequence is latestKnownSequenceNumber.
*/
seenEndOfFrame: Boolean,
/**
* A boolean that indicates whether we've seen a packet with the marker bit set.
*/
seenMarker: Boolean,
/**
* AV1 FrameInfo for the frame
*/
frameInfo: FrameInfo?,
/**
* The AV1 DD Frame Number of this frame.
*/
val frameNumber: Int,
/**
* The FrameID index (FrameID plus cycles) of this frame.
*/
val index: Long,
/**
* The template ID of this frame
*/
val templateId: Int,
/**
* The AV1 Template Dependency Structure in effect for this frame, if known
*/
structure: Av1TemplateDependencyStructure?,
/**
* A new activeDecodeTargets specified for this frame, if any.
* TODO: is this always specified in all packets of the frame?
*/
val activeDecodeTargets: Int?,
/**
* A boolean that indicates whether the incoming AV1 frame that this
* instance refers to is a keyframe.
*/
var isKeyframe: Boolean,
/**
* The raw dependency descriptor included in the packet. Stored if it could not be parsed initially.
*/
val rawDependencyDescriptor: RtpPacket.HeaderExtension?
) {
/**
* AV1 FrameInfo for the frame
*/
var frameInfo = frameInfo
private set
/**
* The AV1 Template Dependency Structure in effect for this frame, if known
*/
var structure = structure
private set
/**
* The earliest RTP sequence number seen of the incoming frame that this instance
* refers to (RFC3550).
*/
var earliestKnownSequenceNumber = earliestKnownSequenceNumber
private set
/**
* The latest RTP sequence number seen of the incoming frame that this instance
* refers to (RFC3550).
*/
var latestKnownSequenceNumber: Int = latestKnownSequenceNumber
private set
/**
* A boolean that indicates whether we've seen the first packet of the frame.
* If so, its sequence is earliestKnownSequenceNumber.
*/
var seenStartOfFrame: Boolean = seenStartOfFrame
private set
/**
* A boolean that indicates whether we've seen the last packet of the frame.
* If so, its sequence is latestKnownSequenceNumber.
*/
var seenEndOfFrame: Boolean = seenEndOfFrame
private set
/**
* A boolean that indicates whether we've seen a packet with the marker bit set.
*/
var seenMarker: Boolean = seenMarker
private set
/**
* A record of how this frame was projected, or null if not.
*/
var projection: Av1DDFrameProjection? = null
/**
* A boolean that records whether this frame was accepted, i.e. should be forwarded to the receiver
* given the decoding target currently being forwarded.
*/
var isAccepted = false
// Validate that the index matches the pictureId
init {
assert((index and 0xffff).toInt() == frameNumber)
}
constructor(packet: Av1DDPacket, index: Long) : this(
ssrc = packet.ssrc,
timestamp = packet.timestamp,
earliestKnownSequenceNumber = packet.sequenceNumber,
latestKnownSequenceNumber = packet.sequenceNumber,
seenStartOfFrame = packet.isStartOfFrame,
seenEndOfFrame = packet.isEndOfFrame,
seenMarker = packet.isMarked,
frameInfo = packet.frameInfo,
frameNumber = packet.statelessDescriptor.frameNumber,
index = index,
templateId = packet.statelessDescriptor.frameDependencyTemplateId,
structure = packet.descriptor?.structure,
activeDecodeTargets = packet.activeDecodeTargets,
isKeyframe = packet.isKeyframe,
rawDependencyDescriptor = if (packet.frameInfo == null) {
packet.getHeaderExtension(packet.av1DDHeaderExtensionId)?.clone()
} else {
null
}
)
/**
* Remember another packet of this frame.
* Note: this assumes every packet is received only once, i.e. a filter
* like [org.jitsi.nlj.transform.node.incoming.PaddingTermination] is in use.
* @param packet The packet to remember. This should be a packet which
* has tested true with [matchesFrame].
*/
fun addPacket(packet: Av1DDPacket) {
require(matchesFrame(packet)) { "Non-matching packet added to frame" }
val seq = packet.sequenceNumber
if (seq isOlderThan earliestKnownSequenceNumber) {
earliestKnownSequenceNumber = seq
}
if (seq isNewerThan latestKnownSequenceNumber) {
latestKnownSequenceNumber = seq
}
if (packet.isStartOfFrame) {
seenStartOfFrame = true
}
if (packet.isEndOfFrame) {
seenEndOfFrame = true
}
if (packet.isMarked) {
seenMarker = true
}
if (structure == null && packet.descriptor?.structure != null) {
structure = packet.descriptor?.structure
}
if (frameInfo == null && packet.frameInfo != null) {
frameInfo = packet.frameInfo
}
}
fun updateParse(templateDependencyStructure: Av1TemplateDependencyStructure, logger: Logger) {
if (rawDependencyDescriptor == null) {
return
}
val parser = Av1DependencyDescriptorReader(rawDependencyDescriptor)
val descriptor = try {
parser.parse(templateDependencyStructure)
} catch (e: Av1DependencyException) {
logger.warn("Could not parse updated AV1 Dependency Descriptor: ${e.message}")
return
}
structure = descriptor.structure
frameInfo = try {
descriptor.frameInfo
} catch (e: Av1DependencyException) {
logger.warn("Could not extract frame info from updated AV1 Dependency Descriptor: ${e.message}")
null
}
}
/**
* Small utility method that checks whether the [Av1DDFrame] that is
* specified as a parameter belongs to the same RTP stream as the frame that
* this instance refers to.
*
* @param av1Frame the [Av1DDFrame] to check whether it belongs to the
* same RTP stream as the frame that this instance refers to.
* @return true if the [Av1DDFrame] that is specified as a parameter
* belongs to the same RTP stream as the frame that this instance refers to,
* false otherwise.
*/
fun matchesSSRC(av1Frame: Av1DDFrame): Boolean = ssrc == av1Frame.ssrc
/**
* Checks whether the specified RTP packet is part of this frame.
*
* @param pkt the RTP packet to check whether it's part of this frame.
* @return true if the specified RTP packet is part of this frame, false
* otherwise.
*/
fun matchesFrame(pkt: Av1DDPacket): Boolean = ssrc == pkt.ssrc && timestamp == pkt.timestamp &&
frameNumber == pkt.frameNumber
fun validateConsistency(pkt: Av1DDPacket) {
if (frameInfo == null) {
return
}
if (frameInfo == pkt.frameInfo) {
return
}
throw RuntimeException(
buildString {
with(pkt) {
append("Packet ssrc $ssrc, seq $sequenceNumber, frame number $frameNumber, timestamp $timestamp ")
append("packet template ${statelessDescriptor.frameDependencyTemplateId} ")
append("frame info $frameInfo ")
}
append("is not consistent with frame ${this@Av1DDFrame}")
}
)
}
fun isImmediatelyAfter(otherFrame: Av1DDFrame): Boolean = frameNumber ==
applySequenceNumberDelta(otherFrame.frameNumber, 1)
override fun toString() = buildString {
append("$ssrc, ")
append("seq $earliestKnownSequenceNumber-$latestKnownSequenceNumber ")
append("frame number $frameNumber, timestamp $timestamp: ")
append("frame template $templateId info $frameInfo")
}
}

246
jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt Normal file
View File

@@ -0,0 +1,246 @@
package org.jitsi.videobridge.cc.av1
import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket
import org.jitsi.nlj.util.ArrayCache
import org.jitsi.nlj.util.RtpSequenceIndexTracker
import org.jitsi.rtp.util.RtpUtils
import org.jitsi.rtp.util.isNewerThan
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import kotlin.math.max
import kotlin.math.min
/**
* A history of recent frames on a Av1 stream.
*/
class Av1DDFrameMap(
parentLogger: Logger
) {
/** Cache mapping frame IDs to frames. */
private val frameHistory = FrameHistory(FRAME_MAP_SIZE)
private val logger: Logger = createChildLogger(parentLogger)
/** Find a frame in the frame map, based on a packet. */
@Synchronized
fun findFrame(packet: Av1DDPacket): Av1DDFrame? = frameHistory[packet.frameNumber]
/** Get the current size of the map. */
fun size(): Int = frameHistory.numCached
/** Check whether this is a large jump from previous state, so the map should be reset. */
private fun isLargeJump(packet: Av1DDPacket): Boolean {
val latestFrame: Av1DDFrame = frameHistory.latestFrame ?: return false
val picDelta = RtpUtils.getSequenceNumberDelta(packet.frameNumber, latestFrame.frameNumber)
if (picDelta > FRAME_MAP_SIZE) {
return true
}
val tsDelta: Long = RtpUtils.getTimestampDiff(packet.timestamp, latestFrame.timestamp)
if (picDelta < 0) {
/* if picDelta is negative but timestamp or sequence delta is positive, we've cycled. */
if (tsDelta > 0) {
return true
}
if (packet.sequenceNumber isNewerThan latestFrame.latestKnownSequenceNumber) {
return true
}
}
/* If tsDelta is more than twice the frame map size at 1 fps, we've cycled. */
return tsDelta > FRAME_MAP_SIZE * 90000 * 2
}
/** Insert a packet into the frame map. Return a frameInsertionResult
* describing what happened.
* @param packet The packet to insert.
* @return What happened. null if insertion failed.
*/
@Synchronized
fun insertPacket(packet: Av1DDPacket): PacketInsertionResult? {
val frameNumber = packet.frameNumber
if (isLargeJump(packet)) {
frameHistory.indexTracker.resetAt(frameNumber)
val frame = frameHistory.insert(packet) ?: return null
return PacketInsertionResult(frame, true, isReset = true)
}
val frame = frameHistory[frameNumber]
if (frame != null) {
if (!frame.matchesFrame(packet)) {
check(frame.frameNumber == frameNumber) {
"frame map returned frame with frame number ${frame.frameNumber} "
"when asked for frame with frame ID $frameNumber"
}
logger.warn(
"Cannot insert packet in frame map: " +
with(frame) {
"frame with ssrc $ssrc, timestamp $timestamp, " +
"and sequence number range $earliestKnownSequenceNumber-$latestKnownSequenceNumber, "
} +
with(packet) {
"and packet $sequenceNumber with ssrc $ssrc, timestamp $timestamp, " +
"and sequence number $sequenceNumber"
} +
" both have frame ID $frameNumber"
)
return null
}
try {
frame.validateConsistency(packet)
} catch (e: Exception) {
logger.warn(e)
}
frame.addPacket(packet)
return PacketInsertionResult(frame, isNewFrame = false)
}
val newframe = frameHistory.insert(packet) ?: return null
return PacketInsertionResult(newframe, true)
}
/** Insert a frame. Only used for unit testing. */
@Synchronized
internal fun insertFrame(frame: Av1DDFrame) {
frameHistory.insert(frame)
}
@Synchronized
fun getIndex(frameIndex: Long) = frameHistory.getIndex(frameIndex)
@Synchronized
fun nextFrame(frame: Av1DDFrame): Av1DDFrame? = frameHistory.findAfter(frame) { true }
@Synchronized
fun nextFrameWith(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? =
frameHistory.findAfter(frame, pred)
@Synchronized
fun prevFrame(frame: Av1DDFrame): Av1DDFrame? = frameHistory.findBefore(frame) { true }
@Synchronized
fun prevFrameWith(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? =
frameHistory.findBefore(frame, pred)
fun findPrevAcceptedFrame(frame: Av1DDFrame): Av1DDFrame? = prevFrameWith(frame) { it.isAccepted }
fun findNextAcceptedFrame(frame: Av1DDFrame): Av1DDFrame? = nextFrameWith(frame) { it.isAccepted }
companion object {
const val FRAME_MAP_SIZE = 500 // Matches PacketCache default size.
}
}
internal class FrameHistory(size: Int) :
ArrayCache<Av1DDFrame>(
size,
cloneItem = { k -> k },
synchronize = false
) {
var numCached = 0
var firstIndex = -1L
var indexTracker = RtpSequenceIndexTracker()
/**
* Gets a frame with a given frame number from the cache.
*/
operator fun get(frameNumber: Int): Av1DDFrame? {
val index = indexTracker.interpret(frameNumber)
return getIndex(index)
}
/**
* Gets a frame with a given frame number index from the cache.
*/
fun getIndex(index: Long): Av1DDFrame? {
if (index <= lastIndex - size) {
/* We don't want to remember old frames even if they're still
tracked; their neighboring frames may have been evicted,
so findBefore / findAfter will return bogus data. */
return null
}
val c = getContainer(index) ?: return null
return c.item
}
/** Get the latest frame in the tracker. */
val latestFrame: Av1DDFrame?
get() = getIndex(lastIndex)
fun insert(frame: Av1DDFrame): Boolean {
val ret = super.insertItem(frame, frame.index)
if (ret) {
numCached++
if (firstIndex == -1L || frame.index < firstIndex) {
firstIndex = frame.index
}
}
return ret
}
fun insert(packet: Av1DDPacket): Av1DDFrame? {
val index = indexTracker.update(packet.frameNumber)
val frame = Av1DDFrame(packet, index)
return if (insert(frame)) frame else null
}
/**
* Called when an item in the cache is replaced/discarded.
*/
override fun discardItem(item: Av1DDFrame) {
numCached--
}
fun findBefore(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? {
val lastIndex = lastIndex
if (lastIndex == -1L) {
return null
}
val index = frame.index
val searchStartIndex = min(index - 1, lastIndex)
val searchEndIndex = max(lastIndex - size, firstIndex - 1)
return doFind(pred, searchStartIndex, searchEndIndex, -1)
}
fun findAfter(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? {
val lastIndex = lastIndex
if (lastIndex == -1L) {
return null
}
val index = frame.index
if (index >= lastIndex) {
return null
}
val searchStartIndex = max(index + 1, max(lastIndex - size + 1, firstIndex))
return doFind(pred, searchStartIndex, lastIndex + 1, 1)
}
private fun doFind(pred: (Av1DDFrame) -> Boolean, startIndex: Long, endIndex: Long, increment: Int): Av1DDFrame? {
require((increment > 0 && startIndex <= endIndex) || (increment < 0 && startIndex >= endIndex)) {
"Values of startIndex=$startIndex, endIndex=$endIndex, and increment=$increment " +
"could lead to infinite loop"
}
var index = startIndex
while (index != endIndex) {
val frame = getIndex(index)
if (frame != null && pred(frame)) {
return frame
}
index += increment
}
return null
}
}
/**
* The result of calling [insertPacket]
*/
class PacketInsertionResult(
/** The frame corresponding to the packet that was inserted. */
val frame: Av1DDFrame,
/** Whether inserting the packet created a new frame. */
val isNewFrame: Boolean,
/** Whether inserting the packet caused a reset */
val isReset: Boolean = false
)

246
jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt Normal file
View File

@@ -0,0 +1,246 @@
/*
* Copyright @ 2019 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.av1
import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket
import org.jitsi.nlj.rtp.codec.av1.applyTemplateIdDelta
import org.jitsi.rtp.rtp.header_extensions.toShortString
import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta
import org.jitsi.rtp.util.isOlderThan
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging.TimeSeriesLogger
import java.time.Instant
/**
* Represents an AV1 DD frame projection. It puts together all the necessary bits
* and pieces that are useful when projecting an accepted AV1 DD frame. A
* projection is responsible for rewriting a AV1 DD packet. Instances of this class
* are thread-safe.
*/
class Av1DDFrameProjection internal constructor(
/**
* The diagnostic context for this instance.
*/
private val diagnosticContext: DiagnosticContext,
/**
* The projected [Av1DDFrame].
*/
val av1Frame: Av1DDFrame?,
/**
* The RTP SSRC of the projection (RFC7667, RFC3550).
*/
val ssrc: Long,
/**
* The RTP timestamp of the projection (RFC7667, RFC3550).
*/
val timestamp: Long,
/**
* The sequence number delta for packets of this frame.
*/
private val sequenceNumberDelta: Int,
/**
* The AV1 frame number of the projection.
*/
val frameNumber: Int,
/**
* The template ID delta for this frame. This applies both to the frame's own template ID, and to
* the template ID offset in the dependency structure, if present.
*/
val templateIdDelta: Int,
/**
* The decode target indication to set on this frame, if any.
*/
val dti: Int?,
/**
* Whether to add a marker bit to the last packet of this frame.
* (Note this will not clear already-existing marker bits.)
*/
val mark: Boolean,
/**
* A timestamp of when this instance was created. It's used to calculate
* RTP timestamps when we switch encodings.
*/
val created: Instant?
) {
/**
* -1 if this projection is still "open" for new, later packets.
* Projections can be closed when we switch away from their encodings.
*/
var closedSeq = -1
private set
/**
* Ctor.
*
* @param ssrc the SSRC of the destination AV1 frame.
* @param timestamp The RTP timestamp of the projected frame that this
* instance refers to (RFC3550).
* @param sequenceNumberDelta The starting RTP sequence number of the
* projected frame that this instance refers to (RFC3550).
*/
internal constructor(
diagnosticContext: DiagnosticContext,
ssrc: Long,
sequenceNumberDelta: Int,
timestamp: Long,
frameNumber: Int?,
templateId: Int?
) : this(
diagnosticContext = diagnosticContext,
av1Frame = null,
ssrc = ssrc,
timestamp = timestamp,
sequenceNumberDelta = sequenceNumberDelta,
frameNumber = frameNumber ?: 0,
templateIdDelta = templateId ?: -1,
dti = null,
mark = false,
created = null
)
fun rewriteSeqNo(seq: Int): Int = applySequenceNumberDelta(seq, sequenceNumberDelta)
fun rewriteTemplateId(id: Int): Int = applyTemplateIdDelta(id, templateIdDelta)
/**
* Rewrites an RTP packet.
*
* @param pkt the RTP packet to rewrite.
*/
fun rewriteRtp(pkt: Av1DDPacket) {
val sequenceNumber = rewriteSeqNo(pkt.sequenceNumber)
val templateId = rewriteTemplateId(pkt.statelessDescriptor.frameDependencyTemplateId)
if (timeSeriesLogger.isTraceEnabled) {
timeSeriesLogger.trace(
diagnosticContext
.makeTimeSeriesPoint("rtp_av1_rewrite")
.addField("orig.rtp.ssrc", pkt.ssrc)
.addField("orig.rtp.timestamp", pkt.timestamp)
.addField("orig.rtp.seq", pkt.sequenceNumber)
.addField("orig.av1.framenum", pkt.frameNumber)
.addField("orig.av1.dti", pkt.frameInfo?.dti?.toShortString() ?: "-")
.addField("orig.av1.templateid", pkt.statelessDescriptor.frameDependencyTemplateId)
.addField("proj.rtp.ssrc", ssrc)
.addField("proj.rtp.timestamp", timestamp)
.addField("proj.rtp.seq", sequenceNumber)
.addField("proj.av1.framenum", frameNumber)
.addField("proj.av1.dti", dti ?: -1)
.addField("proj.av1.templateid", templateId)
.addField("proj.rtp.mark", mark)
)
}
// update ssrc, sequence number, timestamp, frameNumber, and templateID
pkt.ssrc = ssrc
pkt.timestamp = timestamp
pkt.sequenceNumber = sequenceNumber
if (mark && pkt.isEndOfFrame) pkt.isMarked = true
val descriptor = pkt.descriptor
if (descriptor != null && (
frameNumber != pkt.frameNumber || templateId != descriptor.frameDependencyTemplateId ||
dti != null
)
) {
descriptor.frameNumber = frameNumber
descriptor.frameDependencyTemplateId = templateId
val structure = descriptor.structure
check(
descriptor.newTemplateDependencyStructure == null ||
descriptor.newTemplateDependencyStructure === descriptor.structure
)
structure.templateIdOffset = rewriteTemplateId(structure.templateIdOffset)
if (dti != null && (
descriptor.newTemplateDependencyStructure == null ||
dti != (1 shl structure.decodeTargetCount) - 1
)
) {
descriptor.activeDecodeTargetsBitmask = dti
}
pkt.descriptor = descriptor
pkt.reencodeDdExt()
}
}
/**
* Determines whether a packet can be forwarded as part of this
* [Av1DDFrameProjection] instance. The check is based on the sequence
* of the incoming packet and whether or not the [Av1DDFrameProjection]
* has been "closed" or not.
*
* @param rtpPacket the [Av1DDPacket] that will be examined.
* @return true if the packet can be forwarded as part of this
* [Av1DDFrameProjection], false otherwise.
*/
fun accept(rtpPacket: Av1DDPacket): Boolean {
if (av1Frame?.matchesFrame(rtpPacket) != true) {
// The packet does not belong to this AV1 picture.
return false
}
synchronized(av1Frame) {
return if (closedSeq < 0) {
true
} else {
rtpPacket.sequenceNumber isOlderThan closedSeq
}
}
}
val earliestProjectedSeqNum: Int
get() {
if (av1Frame == null) {
return sequenceNumberDelta
}
synchronized(av1Frame) { return rewriteSeqNo(av1Frame.earliestKnownSequenceNumber) }
}
val latestProjectedSeqNum: Int
get() {
if (av1Frame == null) {
return sequenceNumberDelta
}
synchronized(av1Frame) { return rewriteSeqNo(av1Frame.latestKnownSequenceNumber) }
}
/**
* Prevents the max sequence number of this frame to grow any further.
*/
fun close() {
if (av1Frame != null) {
synchronized(av1Frame) { closedSeq = av1Frame.latestKnownSequenceNumber }
}
}
/**
* Get the next template ID that would come after the template IDs in this projection's structure
*/
fun getNextTemplateId(): Int? {
if (av1Frame == null && templateIdDelta != -1) {
return templateIdDelta
}
return av1Frame?.structure?.let { rewriteTemplateId(it.templateIdOffset + it.templateCount) }
}
companion object {
/**
* The time series logger for this class.
*/
private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(Av1DDFrameProjection::class.java)
}
}

470
jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt Normal file
View File

@@ -0,0 +1,470 @@
/*
* Copyright @ 2019 - present 8x8, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jitsi.videobridge.cc.av1
import com.fasterxml.jackson.databind.node.JsonNodeFactory
import com.fasterxml.jackson.databind.node.ObjectNode
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings
import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_ENCODING_ID
import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_INDEX
import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex
import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc
import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.SUSPENDED_DT
import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex
import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getIndex
import org.jitsi.nlj.rtp.codec.av1.containsDecodeTarget
import org.jitsi.rtp.rtp.header_extensions.DTI
import org.jitsi.utils.logging.DiagnosticContext
import org.jitsi.utils.logging2.Logger
import org.jitsi.utils.logging2.createChildLogger
import java.time.Duration
import java.time.Instant
/**
* This class is responsible for dropping AV1 simulcast/svc packets based on
* their quality, i.e. packets that correspond to qualities that are above a
* given quality target. Instances of this class are thread-safe.
*/
internal class Av1DDQualityFilter(
val av1FrameMap: Map<Long, Av1DDFrameMap>,
parentLogger: Logger
) {
/**
* The [Logger] to be used by this instance to print debug
* information.
*/
private val logger: Logger = createChildLogger(parentLogger)
/**
* Holds the arrival time of the most recent keyframe group.
* Reading/writing of this field is synchronized on this instance.
*/
private var mostRecentKeyframeGroupArrivalTime: Instant? = null
/**
* A boolean flag that indicates whether a keyframe is needed, due to an
* encoding or (in some cases) a decode target switch.
*/
var needsKeyframe = false
private set
/**
* The encoding ID that this instance tries to achieve. Upon
* receipt of a packet, we check whether encoding in the externalTargetIndex
* (that's specified as an argument to the
* [acceptFrame] method) is set to something different,
* in which case we set [needsKeyframe] equal to true and
* update.
*/
private var internalTargetEncoding = SUSPENDED_ENCODING_ID
/**
* The decode target that this instance tries to achieve.
*/
private var internalTargetDt = SUSPENDED_DT
/**
* The layer index that we're currently forwarding. [SUSPENDED_INDEX]
* indicates that we're not forwarding anything. Reading/writing of this
* field is synchronized on this instance.
*/
private var currentIndex = SUSPENDED_INDEX
/**
* Determines whether to accept or drop an AV1 frame.
*
* Note that, at the time of this writing, there's no practical need for a
* synchronized keyword because there's only one thread accessing this
* method at a time.
*
* @param frame the AV1 frame.
* @param incomingEncoding The encoding ID of the incoming packet
* @param externalTargetIndex the target quality index that the user of this
* instance wants to achieve.
* @param receivedTime the current time (as an Instant)
* @return true to accept the AV1 frame, otherwise false.
*/
@Synchronized
fun acceptFrame(
frame: Av1DDFrame,
incomingEncoding: Int,
externalTargetIndex: Int,
receivedTime: Instant?
): AcceptResult {
val prevIndex = currentIndex
val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime)
val currentDt = getDtFromIndex(currentIndex)
val mark = currentDt != SUSPENDED_DT &&
(frame.frameInfo?.spatialId == frame.structure?.decodeTargetLayers?.getOrNull(currentDt)?.spatialId)
val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX)
if (isResumption) {
check(accept) {
// Every code path that can turn off SUSPENDED_INDEX also accepts
"isResumption=$isResumption but accept=$accept for frame ${frame.frameNumber}, " +
"frameInfo=${frame.frameInfo}"
}
}
val dtChanged = (prevIndex != currentIndex)
if (dtChanged && currentDt != SUSPENDED_DT) {
check(accept) {
// Every code path that changes DT also accepts
"dtChanged=$dtChanged but accept=$accept for frame ${frame.frameNumber}, frameInfo=${frame.frameInfo}"
}
}
val newDt = if (dtChanged || frame.activeDecodeTargets != null) currentDt else null
return AcceptResult(accept = accept, isResumption = isResumption, mark = mark, newDt = newDt)
}
private fun doAcceptFrame(
frame: Av1DDFrame,
incomingEncoding: Int,
externalTargetIndex: Int,
receivedTime: Instant?
): Boolean {
val externalTargetEncoding = getEidFromIndex(externalTargetIndex)
val currentEncoding = getEidFromIndex(currentIndex)
if (externalTargetEncoding != internalTargetEncoding) {
// The externalEncodingIdTarget has changed since accept last
// ran; perhaps we should request a keyframe.
internalTargetEncoding = externalTargetEncoding
if (externalTargetEncoding != SUSPENDED_ENCODING_ID &&
externalTargetEncoding != currentEncoding
) {
needsKeyframe = true
}
}
if (externalTargetEncoding == SUSPENDED_ENCODING_ID) {
// We stop forwarding immediately. We will need a keyframe in order
// to resume.
currentIndex = SUSPENDED_INDEX
return false
}
return if (frame.isKeyframe) {
logger.debug {
"Quality filter got keyframe for stream ${frame.ssrc}"
}
acceptKeyframe(frame, incomingEncoding, externalTargetIndex, receivedTime)
} else if (currentEncoding != SUSPENDED_ENCODING_ID) {
if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding)) {
// XXX(george) i've noticed some "rogue" base layer keyframes
// that trigger this. what happens is the client sends a base
// layer key frame, the bridge switches to that layer because
// for all it knows it may be the only keyframe sent by the
// client engine. then the bridge notices that packets from the
// higher quality streams are flowing and execution ends-up
// here. it is a mystery why the engine is "leaking" base layer
// key frames
needsKeyframe = true
}
if (incomingEncoding != currentEncoding) {
// for non-keyframes, we can't route anything but the current encoding
return false
}
/** Logic to forward a non-keyframe:
* If the frame does not have FrameInfo, reject and set needsKeyframe (we couldn't decode its templates).
* If we're trying to switch DTs in the current encoding, check the template structure to ensure that
* there is at least one template which is SWITCH for the target DT and not NOT_PRESENT for the current DT.
* If there is not, request a keyframe.
* If the current frame is SWITCH for the target DT, and we've forwarded all the frames on which (by
* its fdiffs) it depends, forward it, and change the current DT to the target DT.
* In normal circumstances (when the target index == the current index), or when we're trying to switch
* *up* encodings, forward all frames whose DT for the current DT is not NOT_PRESENT.
* If we're trying to switch *down* encodings, only forward frames which are REQUIRED or SWITCH for the
* current DT.
*/
val frameInfo = frame.frameInfo ?: run {
needsKeyframe = true
return@doAcceptFrame false
}
var currentDt = getDtFromIndex(currentIndex)
val externalTargetDt = if (currentEncoding == externalTargetEncoding) {
getDtFromIndex(externalTargetIndex)
} else {
currentDt
}
if (
frame.activeDecodeTargets != null &&
!frame.activeDecodeTargets.containsDecodeTarget(externalTargetDt)
) {
/* This shouldn't happen, because we should have set layeringChanged for this packet. */
logger.warn {
"External target DT $externalTargetDt not present in current decode targets 0x" +
Integer.toHexString(frame.activeDecodeTargets) + " for frame $frame."
}
return false
}
if (currentDt != externalTargetDt) {
val frameMap = av1FrameMap[frame.ssrc]
if (frameInfo.dti.getOrNull(externalTargetDt) == null) {
logger.warn { "Target DT $externalTargetDt not present for frame $frame [frameInfo $frameInfo]" }
}
if (frameInfo.dti[externalTargetDt] == DTI.SWITCH &&
frameMap != null &&
frameInfo.fdiff.all {
frameMap.getIndex(frame.index - it)?.isAccepted == true
}
) {
logger.debug { "Switching to DT $externalTargetDt from $currentDt" }
currentDt = externalTargetDt
currentIndex = externalTargetIndex
} else {
if (frame.structure?.canSwitchWithoutKeyframe(currentDt, externalTargetDt) != true) {
logger.debug {
"Want to switch to DT $externalTargetDt from $currentDt, requesting keyframe"
}
needsKeyframe = true
}
}
}
val currentFrameDti = frameInfo.dti[currentDt]
if (currentEncoding > externalTargetEncoding) {
(currentFrameDti == DTI.SWITCH || currentFrameDti == DTI.REQUIRED)
} else {
(currentFrameDti != DTI.NOT_PRESENT)
}
} else {
// In this branch we're not processing a keyframe and the
// currentEncoding is in suspended state, which means we need
// a keyframe to start streaming again.
// We should have already requested a keyframe, either above or when the
// internal target encoding was first moved off SUSPENDED_ENCODING.
false
}
}
/**
* Returns a boolean that indicates whether we are in layer switching phase
* or not.
*
* @param receivedTime the time the latest frame was received
* @return false if we're in layer switching phase, true otherwise.
*/
@Synchronized
private fun isOutOfSwitchingPhase(receivedTime: Instant?): Boolean {
if (receivedTime == null) {
return false
}
if (mostRecentKeyframeGroupArrivalTime == null) {
return true
}
val delta = Duration.between(mostRecentKeyframeGroupArrivalTime, receivedTime)
return delta > MIN_KEY_FRAME_WAIT
}
/**
* @return true if it looks like we can re-scale (see implementation of
* method for specific details).
*/
@Synchronized
private fun isPossibleToSwitch(incomingEncoding: Int): Boolean {
val currentEncoding = getEidFromIndex(currentIndex)
if (incomingEncoding == SUSPENDED_ENCODING_ID) {
// We failed to resolve the spatial/quality layer of the packet.
return false
}
return when {
incomingEncoding > currentEncoding && currentEncoding < internalTargetEncoding ->
// It looks like upscaling is possible
true
incomingEncoding < currentEncoding && currentEncoding > internalTargetEncoding ->
// It looks like downscaling is possible.
true
else ->
false
}
}
/**
* Determines whether to accept or drop an AV1 keyframe. This method updates
* the encoding id.
*
* Note that, at the time of this writing, there's no practical need for a
* synchronized keyword because there's only one thread accessing this
* method at a time.
*
* @param receivedTime the time the frame was received
* @return true to accept the AV1 keyframe, otherwise false.
*/
@Synchronized
private fun acceptKeyframe(
frame: Av1DDFrame,
incomingEncoding: Int,
externalTargetIndex: Int,
receivedTime: Instant?
): Boolean {
// This branch writes the {@link #currentSpatialLayerId} and it
// determines whether or not we should switch to another simulcast
// stream.
if (incomingEncoding < 0) {
// something went terribly wrong, normally we should be able to
// extract the layer id from a keyframe.
logger.error("unable to get layer id from keyframe")
return false
}
val frameInfo = frame.frameInfo ?: run {
// something went terribly wrong, normally we should be able to
// extract the frame info from a keyframe.
logger.error("unable to get frame info from keyframe")
return@acceptKeyframe false
}
logger.debug {
"Received a keyframe of encoding: $incomingEncoding"
}
val currentEncoding = getEidFromIndex(currentIndex)
val externalTargetEncoding = getEidFromIndex(externalTargetIndex)
val indexIfSwitched = when {
incomingEncoding == externalTargetEncoding -> externalTargetIndex
incomingEncoding == internalTargetEncoding && internalTargetDt != -1 ->
getIndex(currentEncoding, internalTargetDt)
else -> frameInfo.dtisPresent.maxOrNull()!!
}
val dtIfSwitched = getDtFromIndex(indexIfSwitched)
val dtiIfSwitched = frameInfo.dti[dtIfSwitched]
val acceptIfSwitched = dtiIfSwitched != DTI.NOT_PRESENT
// The keyframe request has been fulfilled at this point, regardless of
// whether we'll be able to achieve the internalEncodingIdTarget.
needsKeyframe = false
return if (isOutOfSwitchingPhase(receivedTime)) {
// During the switching phase we always project the first
// keyframe because it may very well be the only one that we
// receive (i.e. the endpoint is sending low quality only). Then
// we try to approach the target.
mostRecentKeyframeGroupArrivalTime = receivedTime
logger.debug {
"First keyframe in this kf group " +
"currentEncodingId: $incomingEncoding. " +
"Target is $internalTargetEncoding"
}
if (incomingEncoding <= internalTargetEncoding) {
// If the target is 180p and the first keyframe of a group of
// keyframes is a 720p keyframe we don't project it. If we
// receive a 720p keyframe, we know that there MUST be a 180p
// keyframe shortly after.
if (acceptIfSwitched) {
currentIndex = indexIfSwitched
}
acceptIfSwitched
} else {
false
}
} else {
// We're within the 300ms window since the reception of the
// first key frame of a key frame group, let's check whether an
// upscale/downscale is possible.
when {
currentEncoding <= incomingEncoding &&
incomingEncoding <= internalTargetEncoding -> {
// upscale or current quality case
if (acceptIfSwitched) {
currentIndex = indexIfSwitched
logger.debug {
"Upscaling to encoding $incomingEncoding. " +
"The target is $internalTargetEncoding"
}
}
acceptIfSwitched
}
incomingEncoding <= internalTargetEncoding &&
internalTargetEncoding < currentEncoding -> {
// downscale case
if (acceptIfSwitched) {
currentIndex = indexIfSwitched
logger.debug {
"Downscaling to encoding $incomingEncoding. " +
"The target is $internalTargetEncoding"
}
}
acceptIfSwitched
}
else -> {
false
}
}
}
}
/**
* Adds internal state to a diagnostic context time series point.
*/
@SuppressFBWarnings(
value = ["IS2_INCONSISTENT_SYNC"],
justification = "We intentionally avoid synchronizing while reading fields only used in debug output."
)
internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) {
pt.addField("qf.currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex))
.addField("qf.internalTargetEncoding", internalTargetEncoding)
.addField("qf.internalTargetDt", internalTargetDt)
.addField("qf.needsKeyframe", needsKeyframe)
.addField(
"qf.mostRecentKeyframeGroupArrivalTimeMs",
mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1
)
/* TODO any other fields necessary */
}
/**
* Gets a JSON representation of the parts of this object's state that
* are deemed useful for debugging.
*/
@get:SuppressFBWarnings(
value = ["IS2_INCONSISTENT_SYNC"],
justification = "We intentionally avoid synchronizing while reading fields only used in debug output."
)
val debugState: ObjectNode
get() {
val debugState = JsonNodeFactory.instance.objectNode()
debugState.put(
"mostRecentKeyframeGroupArrivalTimeMs",
mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1
)
debugState.put("needsKeyframe", needsKeyframe)
debugState.put("internalTargetEncoding", internalTargetEncoding)
debugState.put("internalTargetDt", internalTargetDt)
debugState.put("currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex))
return debugState
}
data class AcceptResult(
val accept: Boolean,
val isResumption: Boolean,
val mark: Boolean,
val newDt: Int?
)
companion object {
/**
* The default maximum frequency at which the media engine
* generates key frame.
*/
private val MIN_KEY_FRAME_WAIT = Duration.ofMillis(300)
}
}

Some files were not shown because too many files have changed in this diff Show More