commit e1257b7a059d11b860f2eefb766671ac2b9ddbb7 Author: upstream-mirror Date: Tue Sep 15 16:12:38 2026 +0500 snapshot of https://github.com/jitsi/jitsi-videobridge.git @ fcbf741c646bc237d2ed85a1e86910532532b831 (2026-09-15) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c9d9750 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +[*.{kt,kts}] +# Keep constructor parameters wrapped one per line, and kotest specs unindented. +ktlint_standard_class-signature = disabled +max_line_length=120 +ktlint_code_style = intellij_idea + +# I find trailing commas annoying +ktlint_standard_trailing-comma-on-call-site = disabled +ktlint_standard_trailing-comma-on-declaration-site = disabled + +# This forbids underscores in package names, which we use +ktlint_standard_package-name = disabled diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..488f808 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.java text eol=lf diff=java diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..80cfdc6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,27 @@ +--- +name: Bug Report +about: Before posting, please make sure you check https://community.jitsi.org + +--- + +*This Issue tracker is only for reporting bugs and tracking code related issues.* + +Before posting, please make sure you check community.jitsi.org to see if the same or similar bugs have already been discussed. General questions, installation help, and feature requests can also be posted to community.jitsi.org. + +## Description +--- + +## Current behavior +--- + +## Expected Behavior +--- + +## Possible Solution +--- + +## Steps to reproduce +--- + +# Environment details +--- diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f3fcd4b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "maven" + directories: + - "/" + - "/rtp" + - "/jitsi-media-transform" + - "/jvb" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..d89fd31 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,44 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +env: + # Java version to use for the release + RELEASE_JAVA_VERSION: 17 + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + java: [ 17, 21 ] + + name: Java ${{ matrix.java }} + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up Java ${{ matrix.java }} + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + + - name: Build with Maven + run: ./resources/build.sh -Pcoverage -B + + - name: Upload coverage report + if: matrix.java == env.RELEASE_JAVA_VERSION + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0251908 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +/classes/ +/dist/ +/junit-reports/ +/.settings/ +target/ +/openfire/target/ +*.class +/.classpath +*.jar +/.project +*.swp +**/.idea/ +*.iml +.mvn/ +.kotlintest +**/.DS_Store diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..b66cfb6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,154 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Jitsi Videobridge (JVB) is a WebRTC-compatible Selective Forwarding Unit (SFU) that routes media streams between participants in video conferences. It is a backend component of the Jitsi Meet stack. + +## Build System & Commands + +This project uses Maven with Java 11+ and Kotlin. Requires Maven 3.6.3+. + +### Build Commands + +```bash +# Build the entire project +mvn install + +# Build without running tests +mvn install -DskipTests + +# Run all tests +mvn test + +# Run tests in a specific module +cd jvb && mvn test + +# Run a specific test class +mvn test -Dtest=ClassName + +# Run a specific test method +mvn test -Dtest=ClassName#methodName + +# Build package (creates archive in jvb/target/) +mvn install +# Output: jvb/target/jitsi-videobridge-2.3-SNAPSHOT-archive.zip +``` + +### Code Quality Commands + +```bash +# Run checkstyle (Java formatting) +mvn checkstyle:check + +# Run ktlint (Kotlin formatting, must be run inside a submodule directory, e.g. ./jvb/; sibling modules must be +# resolvable, so run "mvn install -DskipTests" from the root first if they are not) +mvn exec:exec@ktlint-check + +# Auto-format Kotlin code (must be run inside a submodule directory, e.g. ./jvb/) +mvn exec:exec@ktlint-format +``` + +## Project Structure + +This is a multi-module Maven project with three main modules: + +### 1. `rtp/` - RTP Protocol Layer +Low-level RTP/RTCP packet handling and parsing. + +### 2. `jitsi-media-transform/` - Media Processing +Core media transformation logic including: +- Codec-specific processing (VP8, VP9, AV1, H.264) +- Simulcast and SVC (Scalable Video Coding) handling +- Bandwidth allocation and quality adaptation +- SRTP encryption/decryption + +### 3. `jvb/` - Main Videobridge Application +The main application that ties everything together: + +**Key Java Classes:** +- `Videobridge.java` - Main service managing conferences +- `Conference.java` - Represents a single video conference, manages endpoints and media routing +- `Endpoint` (Kotlin) - Represents a participant in a conference +- `EncodingsManager.java` - Manages simulcast encodings for endpoints + +**Key Kotlin Packages:** +- `cc/allocation/` - Bandwidth allocation and bitrate control +- `cc/vp8/`, `cc/vp9/`, `cc/av1/` - Codec-specific frame processing +- `load_management/` - CPU and packet rate based load management +- `relay/` - Multi-region support (Octo relay) +- `websocket/` - WebSocket transport for signaling +- `xmpp/` - XMPP/Colibri signaling + +**Main Entry Point:** `org.jitsi.videobridge.Main.kt` + +## Architecture Principles + +### Media Forwarding Pipeline +1. **Ingress**: Media arrives at endpoints via ICE/DTLS/SRTP +2. **Processing**: Packets flow through codec-specific handlers that understand simulcast/SVC structure +3. **Bandwidth Allocation**: `BitrateController` determines which quality layers to forward to each receiver +4. **Rewriting**: RTP headers (sequence numbers, timestamps, picture IDs) are rewritten to make stream switches transparent +5. **Egress**: Forwarded packets are sent to receiving endpoints + +### Simulcast & SVC Handling +The bridge supports temporal and spatial scalability in VP8, VP9, and AV1. It performs transparent stream switching by rewriting RTP sequence numbers, timestamps, and codec-specific picture IDs. See `doc/svc.md` for detailed rewriting principles. + +**Key Constraint**: Switches must be completely transparent to receivers - no gaps in sequence numbers, continuous timestamps, and continuous picture IDs. + +### Configuration System +Uses Typesafe Config (HOCON format): +- `reference.conf` contains all default values with documentation +- User config in `/etc/jitsi/videobridge/jvb.conf` or `~/.jvb/jvb.conf` overrides defaults +- Config is accessed via `jitsi-metaconfig` library with type-safe property delegates + +## Code Style & Conventions + +### Java Code +- **Brace Style**: Opening braces on new line for methods, classes, control structures +- **Line Length**: Maximum 120 characters +- **Indentation**: 4 spaces, NO tabs +- Validated by Checkstyle (see `jvb/checkstyle.xml`) + +### Kotlin Code +- Follows ktlint conventions (standard Kotlin style) +- **Line Length**: Maximum 120 characters +- ktlint runs from Maven via exec-maven-plugin (version pinned by `ktlint.version` in the root pom); `mvn verify` runs the check + +### Testing +- Uses JUnit 5 for new tests (JUnit 4 vintage engine for legacy tests) +- Kotlin tests use Kotest framework +- Mocking with MockK (Kotlin) or Mockito (Java) + +## Important Implementation Notes + +### Thread Safety +- `Conference.endpointsById` is a `ConcurrentHashMap` but writes must be synchronized on the map itself to keep it in sync with `endpointsCache` +- Most core components use concurrent data structures but may require explicit synchronization for compound operations + +### Performance Considerations +- `endpointsCache` is a read-only cache updated on writes - used in hot path (per-packet processing) +- Buffer pools (`ByteBufferPool`) are used throughout to reduce GC pressure +- Packet queues are bounded to prevent memory exhaustion under load + +### Logging +- Uses `org.jitsi.utils.logging2.Logger` (custom logging framework) +- Can optionally redact remote IP addresses (controlled by `videobridge.redact-remote-addresses`) + +### Dependencies +- **ice4j**: ICE/STUN/TURN implementation +- **jicoco**: Common Jitsi infrastructure (config, metrics, REST) +- **Jetty**: HTTP server for REST APIs and WebSockets +- **Smack**: XMPP client library + +## Testing & CI + +Tests are organized by module. When implementing features, consider whether tests are needed and discuss with the user if implementation is non-trivial. + +Run module-specific tests: +```bash +cd jvb && mvn test +cd jitsi-media-transform && mvn test +cd rtp && mvn test +``` diff --git a/CONFIG.md b/CONFIG.md new file mode 100644 index 0000000..1a204ed --- /dev/null +++ b/CONFIG.md @@ -0,0 +1,63 @@ +# Configuration + +##### 10/14/19 +We've begun the migration to a new configuration framework. The 'old' method (the `sip-communicator.properties` file and program arguments) will be supported throughout the transition and the bridge will be fully backwards-compatible with the old config. No changes are necessary until we pull the plug on the old config completely, and we'll give ample warning before that time. It's also worth noting that any new properties will only be supported in the new config files. + +Moving forward, we'll be using lightbend's [config library](https://github.com/lightbend/config). Their GitHub page gives lots of details, but below are the changes most relevant to configuring the JVB. + +### Configuration sources +During the transition, we'll continue to read from the original locations (`sip-communicator.properties` and program arguments) for all configuration properties, falling back to the new `reference.conf` file for defaults. The `reference.conf` file holds defaults for all configuration properties, and shouldn't be modified in a deployment--though it may be interesting for developers who want to see the defaults. + +To override values, users can either put a file named `application.conf` in the classpath or pass a path to a file as a system property via `-Dconfig.file=/path/to/config.conf`. Values in the file given will override ones in `reference.conf`. The lightbend docs do a good job of describing [how configuration is loaded](https://github.com/lightbend/config#standard-behavior), but here's an example of overriding a configuration value for JVB: + +### Overriding default values +Say we want to override some of the default values in `reference.conf`, namely: the XMPP API server values, the health check interval, and whether or not onstage video suspension is enabled. We'll create an `application.conf` files with the values we want to change: + +``` +videobridge { + health { + // Override the health check interval + interval=60 seconds + } + cc { + // Override the onstage video suspension setting + onstage-video-suspension-enabled=true + } +} +``` +You can override just the values you want to change, all others will fallback to the defaults set in `reference.conf` + +### Migrating from old config +Below are the mappings from old property values to new ones. Note: for new property names, the 'flat paths' are given, i.e. a value of `videobridge.health.interval` corresponds to the setting: +``` +videobridge { + health { + interval + } +} +``` +This list will be updated as properties are migrated: + +| Old property name | New property name | Notes | +| -------- | ------- | ------- | +| org.jitsi.videobridge.health.INTERVAL | videobridge.health.interval | The new config models this as a duration, rather than an amount of milliseconds | +| org.jitsi.videobridge.health.TIMEOUT | videobridge.health.timeout | The new config models this as a duration, rather than an amount of milliseconds | +| org.jitsi.videobridge.health.STICKY_FAILURES | videobridge.health.sticky-failures | | +| org.jitsi.videobridge.EndpointConnectionStatus.FIRST_TRANSFER_TIMEOUT | videobridge.ep-connection-status.first-transfer-timeout | The new config models this as a duration, rather than an amount of milliseconds | +| org.jitsi.videobridge.EndpointConnectionStatus.MAX_INACTIVITY_LIMIT | videobridge.ep-connection-status.max-inactivity-limit | The new config models this as a duration, rather than an amount of milliseconds | +| org.jitsi.videobridge.BWE_CHANGE_THRESHOLD_PCT | videobridge.cc.bwe-change-threshold-pct | | +| org.jitsi.videobridge.THUMBNAIL_MAX_HEIGHT | videobridge.cc.thumbnail-max-height-px | | +| org.jitsi.videobridge.ONSTAGE_PREFERRED_HEIGHT | videobridge.cc.onstage-preferred-height-px | | +| org.jitsi.videobridge.ONSTAGE_PREFERRED_FRAME_RATE | videobridge.cc.onstage-preferred-framerate | | +| org.jitsi.videobridge.ENABLE_ONSTAGE_VIDEO_SUSPEND | videobridge.cc.enable-onstage-video-suspend | | +| org.jitsi.videobridge.TRUST_BWE | videobridge.cc.trust-bwe | | +| org.jitsi.videobridge.PADDING_PERIOD_MS | videobridge.cc.padding-period | The new config models this as a duration, rather than an amount of milliseconds | +| org.jitsi.videobridge.DISABLE_RTX_PROBING | n/a | This property has been deprecated | +| org.jitsi.videobridge.rest.COLIBRI_WS_DISABLE | videobridge.websockets.enabled | The semantics of this property have been inverted (disable -> enable) | +| org.jitsi.videobridge.rest.COLIBRI_WS_DOMAIN | videobridge.websockets.domain | | +| org.jitsi.videobridge.rest.COLIBRI_WS_TLS | videobridge.websockets.tls | | +| org.jitsi.videobridge.rest.COLIBRI_WS_SERVER_ID | videobridge.websockets.server-id | | +| org.jitsi.videobridge.SINGLE_PORT_HARVESTER_PORT | videobridge.ice.udp.port | | +| org.jitsi.videobridge.ICE_UFRAG_PREFIX | videobridge.ice.ufrag-prefix | | +| org.jitsi.videobridge.KEEP_ALIVE_STRATEGY | videobridge.ice.keep-alive-strategy | | + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..abf11eb --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# Intro + +Jitsi Videobridge is a WebRTC-compatible Selective Forwarding Unit (SFU), i.e. a +multimedia router. It is one of the backend components in the [Jitsi Meet](https://github.com/jitsi/jitsi-meet) stack. + +You can find more documentation in the +[doc/ directory in the source tree](https://github.com/jitsi/jitsi-videobridge/tree/master/doc) and in the +[Jitsi Meet Handbook](https://jitsi.github.io/handbook/). + +If you have questions about the project, please post on the [Jitsi Community Forum](https://community.jitsi.org/). +GitHub issues are only used to track actionable items. + +# Packages + +## Debian/Ubuntu +You can download binary packages for Debian/Ubuntu: +* [stable](https://download.jitsi.org/stable/) ([instructions](https://jitsi.org/downloads/ubuntu-debian-installations-instructions/)) +* [testing](https://download.jitsi.org/testing/) ([instructions](https://jitsi.org/downloads/ubuntu-debian-installations-instructions-for-testing/)) +* [nightly](https://download.jitsi.org/unstable/) ([instructions](https://jitsi.org/downloads/ubuntu-debian-installations-instructions-nightly/)) + +## Building your own package +You can build a custom package with just `mvn install` in the root directory of the repo. Look for the package in +`jvb/target/jitsi-videobridge-2.1-SNAPSHOT-archive.zip`. + +# Running locally +You can run jitsi-videobridge locally with maven (or in your IDE). First create a `~/.jvb/jvb.conf` to configure the +environment to connect to and other options (see +[reference.conf](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf) for the +available options). + +```sh +JVB_HOME="/path/to/the/cloned/repo" +JVB_CONFIG_DIR_LOCATION="~/" +JVB_CONFIG_DIR_NAME=".jvb" +JVB_CONFIG_FILE="$JVB_CONFIG_DIR_LOCATION/$JVB_JVB_CONFIG_DIR_NAME/jvb.conf" + +mvn compile exec:exec -Dexec.executable=java -Dexec.args="-cp %classpath org.jitsi.videobridge.MainKt -Djava.library.path=$JVB_HOME/lib/native/linux-64 -Djava.util.logging.config.file=$JVB_HOME/lib/logging.properties -Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION=$JVB_CONFIG_DIR_LOCATION -Dnet.java.sip.communicator.SC_HOME_DIR_NAME=$JVB_CONFIG_DIR_NAME -Dconfig.file=$JVB_CONFIG_FILE" +``` + +# Configuration +Application level configuration comes from a config file, usually installed in `/etc/jitsi/videobridge/jvb.conf`. The +values in that file override the defaults defined in +[reference.conf](https://github.com/jitsi/ice4j/blob/master/src/main/resources/reference.conf). + +## Debian +On debian systems the `/etc/jitsi/videobridge/config` file can be used to set configuration for the Java virtual machine. +Notable examples: +```commandline +# Increase the java heap to 8GB +VIDEOBRIDGE_MAX_MEMORY=8192m +# Change the garbage collector (defaults to G1GC) +VIDEOBRIDGE_GC_TYPE=G1GC +``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..cf2773d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security + +## Reporting security issues + +We take security very seriously and develop all Jitsi projects to be secure and safe. + +If you find (or simply suspect) a security issue in any of the Jitsi projects, please report it to us via [HackerOne](https://hackerone.com/8x8-bounty) or send us an email to security@jitsi.org. + +**We encourage responsible disclosure for the sake of our users, so please reach out before posting in a public space.** diff --git a/config/20-jvb-udp-buffers.conf b/config/20-jvb-udp-buffers.conf new file mode 100644 index 0000000..4d8b81f --- /dev/null +++ b/config/20-jvb-udp-buffers.conf @@ -0,0 +1,3 @@ +# this sets the max, so that we can bump the JVB UDP single port buffer size. +net.core.rmem_max=10485760 +net.core.netdev_max_backlog=100000 diff --git a/config/logrotate b/config/logrotate new file mode 100644 index 0000000..190e54f --- /dev/null +++ b/config/logrotate @@ -0,0 +1,12 @@ +/var/log/jitsi/jvb.log { + hourly + size 100M + missingok + rotate 24 + compress + delaycompress + notifempty + copytruncate + su jvb jitsi +} + diff --git a/debian/README b/debian/README new file mode 100644 index 0000000..3c99f9b --- /dev/null +++ b/debian/README @@ -0,0 +1,2 @@ +Jitsi Videobridge - WebRTC compatible Selective Forwarding Unit + (SFU) for multiuser video communication diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 0000000..d14b99d --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,8 @@ +jitsi-videobridge for Debian +--------------------- + +If you want a WebRTC videoconferencing solution you have to install +jitsi-meet package which uses Jitsi Videobridge as a SFU (Selective +Forwarding Unit) + + -- Yasen Pramatarov Tue, 22 Apr 2014 21:40:37 +0300 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..e4dba95 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,6 @@ +jitsi-videobridge2 (2.1-0-g676fb3d-1) unstable; urgency=low + + * Initial release. + + -- Damian Minkov Fri, 17 Jan 2020 15:51:09 +0000 + diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +12 diff --git a/debian/config b/debian/config new file mode 100644 index 0000000..4a2bc0c --- /dev/null +++ b/debian/config @@ -0,0 +1,8 @@ +#!/bin/sh -e + +# Source debconf library. +. /usr/share/debconf/confmodule + +# server hostname +db_input critical jitsi-videobridge/jvb-hostname || true +db_go diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..b1110ee --- /dev/null +++ b/debian/control @@ -0,0 +1,20 @@ +Source: jitsi-videobridge2 +Section: net +Priority: extra +Maintainer: Jitsi Team +Uploaders: Emil Ivov , Damian Minkov +Build-Depends: debhelper (>= 9), dh-systemd, openjdk-17-jdk | openjdk-21-jdk, maven +Standards-Version: 3.9.3 +Homepage: https://jitsi.org/videobridge + +Package: jitsi-videobridge2 +Replaces: jitsi-videobridge +Conflicts: jitsi-videobridge (<= 1400-1) +Architecture: all +Pre-Depends: openjdk-17-jre-headless | openjdk-17-jre | openjdk-21-jre-headless | openjdk-21-jre, libssl3 | libssl1.1 +Depends: ${misc:Depends}, procps, uuid-runtime, ruby-hocon +Recommends: libpcap0.8 +Description: WebRTC compatible Selective Forwarding Unit (SFU) + Jitsi Videobridge is a WebRTC compatible Selective Forwarding Unit + (SFU) for multiuser video communication. It is an essential part + of Jitsi Meet diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..06c2299 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,13 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Jitsi Videobridge +Upstream-Contact: Emil Ivov +Source: https://github.com/jitsi/jitsi-videobridge + +Files: * +Copyright: 2015-present Atlassian Pty Ltd +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the full text of the Apache + License version 2 can be found in the file + '/usr/share/common-licenses/Apache-2.0'. diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..e69de29 diff --git a/debian/init.d b/debian/init.d new file mode 100644 index 0000000..536bd6d --- /dev/null +++ b/debian/init.d @@ -0,0 +1,118 @@ +#! /bin/sh +# +# INIT script for Jitsi Videobridge +# Version: 1.0 01-May-2014 yasen@bluejimp.com +# +### BEGIN INIT INFO +# Provides: jitsi-videobridge +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Jitsi Videobridge +# Description: WebRTC compatible Selective Forwarding Unit (SFU) +### END INIT INFO + +. /lib/lsb/init-functions + +# Include videobridge defaults if available +if [ -f /etc/jitsi/videobridge/config ]; then + . /etc/jitsi/videobridge/config +fi + +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +DAEMON=/usr/share/jitsi-videobridge/jvb.sh +NAME=jvb +USER=jvb +# A tmpfs backed directory just for the JVB process. This is introduced +# to hold packet arrival times, but it may be otherwise useful in the future. +TMPPATH=/var/run/jitsi-videobridge +PIDFILE=/var/run/jitsi-videobridge.pid +LOGFILE=/var/log/jitsi/jvb.log +DESC=jitsi-videobridge + +if [ ! -d "$TMPPATH" ]; then + mkdir "$TMPPATH" + chown $USER:adm "$TMPPATH" +fi + +if [ ! $JVB_HOST ]; then + JVB_HOST=localhost +fi +# TODO: remove this when support for --apis is removed +DAEMON_OPTS="$JVB_OPTS" + +if [ ! -x $DAEMON ] ;then + echo "Daemon not executable: $DAEMON" + exit 1 +fi + +set -e + +stop() { + if [ -f $PIDFILE ]; then + PID=$(cat $PIDFILE) + fi + echo -n "Stopping $DESC: " + if [ $PID ]; then + kill $PID || true + rm $PIDFILE || true + echo "$NAME stopped." + else + echo "$NAME doesn't seem to be running." + fi +} + +start() { + if [ -f $PIDFILE ]; then + echo "$DESC seems to be already running, we found pidfile $PIDFILE." + exit 1 + fi + echo -n "Starting $DESC: " + DAEMON_START_CMD="JAVA_SYS_PROPS=\"$JAVA_SYS_PROPS\" exec $DAEMON $DAEMON_OPTS < /dev/null >> $LOGFILE 2>&1" + AUTHBIND_CMD="" + if [ "$AUTHBIND" = "yes" ]; then + AUTHBIND_CMD="/usr/bin/authbind --deep /bin/bash -c " + DAEMON_START_CMD="'$DAEMON_START_CMD'" + fi + start-stop-daemon --start --quiet --background --chuid $USER --make-pidfile --pidfile $PIDFILE \ + --exec /bin/bash -- -c "$AUTHBIND_CMD $DAEMON_START_CMD" + echo "$NAME started." +} + +reload() { + echo 'Not yet implemented.' +} + +status() { + status_of_proc -p $PIDFILE java "$NAME" && exit 0 || exit $? +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + reload) + reload + ;; + force-reload) + reload + ;; + status) + status + ;; + *) + N=/etc/init.d/$NAME + echo "Usage: $N {start|stop|restart|reload|status}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/debian/install b/debian/install new file mode 100644 index 0000000..d523ecf --- /dev/null +++ b/debian/install @@ -0,0 +1,10 @@ +jvb/target/dependency/* usr/share/jitsi-videobridge/lib +jvb/lib/videobridge.rc usr/share/jitsi-videobridge/lib +jvb/lib/logging.properties etc/jitsi/videobridge +jitsi-videobridge.jar usr/share/jitsi-videobridge +jvb/resources/jvb.sh usr/share/jitsi-videobridge +resources/graceful_shutdown.sh usr/share/jitsi-videobridge +resources/collect-dump-logs.sh usr/share/jitsi-videobridge + +jitsi-videobridge etc/logrotate.d +config/20-jvb-udp-buffers.conf etc/sysctl.d diff --git a/debian/jitsi-videobridge2.service b/debian/jitsi-videobridge2.service new file mode 100644 index 0000000..073b4fb --- /dev/null +++ b/debian/jitsi-videobridge2.service @@ -0,0 +1,29 @@ +[Unit] +Description=Jitsi Videobridge +After=network-online.target +Wants=network-online.target + +[Service] +SuccessExitStatus=143 +# configuration error prevents restart loops +RestartPreventExitStatus=78 +# allow bind to 80 and 443 +AmbientCapabilities=CAP_NET_BIND_SERVICE +EnvironmentFile=/etc/jitsi/videobridge/config +Environment=LOGFILE=/var/log/jitsi/jvb.log +User=jvb +RuntimeDirectory=jitsi-videobridge +RuntimeDirectoryMode=0750 +PIDFile=/var/run/jitsi-videobridge/jitsi-videobridge.pid +# more threads for this process +TasksMax=65000 +# allow more open files for this process +LimitNPROC=65000 +LimitNOFILE=65000 +ExecStart=/bin/bash -c "exec /usr/share/jitsi-videobridge/jvb.sh ${JVB_OPTS} < /dev/null >> ${LOGFILE} 2>&1" +ExecStartPost=/bin/bash -c "echo $MAINPID > /var/run/jitsi-videobridge/jitsi-videobridge.pid" +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/debian/manpage.1 b/debian/manpage.1 new file mode 100644 index 0000000..f94ef05 --- /dev/null +++ b/debian/manpage.1 @@ -0,0 +1,40 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH JITSI-VIDEOBRIDGE SECTION "April 09, 2014" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) +.SH NAME +jitsi-videobridge \- WebRTC compatible Selective Forwarding Unit +(SFU) for multiuser video communication +.SH SYNOPSIS +.B /usr/share/jitsi-videobridge/jvb.sh +.RI [ options ] ... +.br +.B /etc/init.d/jitsi-videobridge (start|stop|restart) +.SH DESCRIPTION +This manual page documents briefly the +.B jitsi-videobridge +service. +.PP +.\" TeX users may be more comfortable with the \fB\fP and +.\" \fI\fP escape sequences to invoke bold face and italics, +.\" respectively. +\fBjitsi-videobridge\fP is a Selective Forwarding Unit (SFU) for +building applications for WebRTC compatible video conferences. +.SH AUTHOR +jitsi-videobridge was written by Jitsi.org and Bluejimp.com. +.PP +This manual page was written by Yasen Pramatarov , +for the Debian project (and may be used by others). diff --git a/debian/po/POTFILES.in b/debian/po/POTFILES.in new file mode 100644 index 0000000..cef83a3 --- /dev/null +++ b/debian/po/POTFILES.in @@ -0,0 +1 @@ +[type: gettext/rfc822deb] templates diff --git a/debian/po/templates.pot b/debian/po/templates.pot new file mode 100644 index 0000000..6c4e575 --- /dev/null +++ b/debian/po/templates.pot @@ -0,0 +1,30 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: jitsi-videobridge\n" +"Report-Msgid-Bugs-To: jitsi-videobridge@packages.debian.org\n" +"POT-Creation-Date: 2014-08-11 10:41+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: 8bit\n" + +#. Type: string +#. Description +#: ../templates:1001 +msgid "Hostname:" +msgstr "" + +#. Type: string +#. Description +#: ../templates:1001 +msgid "The jitsi-videobridge package needs the DNS hostname of your instance." +msgstr "" diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 0000000..726241f --- /dev/null +++ b/debian/postinst @@ -0,0 +1,184 @@ +#!/bin/bash +# postinst script for jitsi-videobridge + +set -e + +case "$1" in + configure) + + CONFIG="/etc/jitsi/videobridge/config" + + # we don't want to regenerate config on upgrade + OLDCONFIG="false" + # migrate any old config found + if [ -f "/etc/default/jitsi-videobridge" ]; then + mv /etc/default/jitsi-videobridge $CONFIG + fi + + if [ -f $CONFIG ]; then + . $CONFIG + if [ -n "$JVB_HOSTNAME" ] && [ -n "$JVB_SECRET" ]; then + OLDCONFIG="true" + fi + fi + + # debconf hostname question + . /usr/share/debconf/confmodule + + # the default vars + db_get jitsi-videobridge/jvb-hostname + JVB_HOSTNAME_IN=$(echo "$RET" | xargs echo -n) + + # generate config on new install or when we are reconfiguring + # and all install is different than current one + if [ "$OLDCONFIG" = "false" ] || [ "$JVB_HOSTNAME" != "$JVB_HOSTNAME_IN" ]; then + + JVB_HOSTNAME="$JVB_HOSTNAME_IN" + if [ "$OLDCONFIG" = "false" ]; then + # 8-chars random secret, alternative to pwgen 8 + db_get jitsi-videobridge/jvbsecret + if [ -n "$RET" ]; then + JVB_SECRET="$RET" + else + JVB_SECRET=`head -c 8 /dev/urandom | tr '\0-\377' 'a-zA-Z0-9a-zA-Z0-9a-zA-Z0-9a-zA-Z0-9@@@@####'` + fi + fi + fi + + # Store it, so we can use it if xmpp server is on same machine + # we are currently doing this on every upgrade in order to be able to successfully upgrade + # from old packages, otherwise we can do it only when secret is first time generated + db_set jitsi-videobridge/jvbsecret $JVB_SECRET + + # and we're done with debconf + db_stop + + HOCON_CONFIG="/etc/jitsi/videobridge/jvb.conf" + + # let's check whether there is a setting in the $CONFIG + # for home folder and logging props file, if missing add it + if ! grep -q "JAVA_SYS_PROPS" "$CONFIG"; then + echo >> $CONFIG + echo '# adds java system props that are passed to jvb (default are for home and logging config file)' >> $CONFIG + echo "JAVA_SYS_PROPS=\"-Dnet.java.sip.communicator.SC_HOME_DIR_LOCATION=/etc/jitsi\ + -Dnet.java.sip.communicator.SC_HOME_DIR_NAME=videobridge\ + -Dnet.java.sip.communicator.SC_LOG_DIR_LOCATION=/var/log/jitsi\ + -Djava.util.logging.config.file=/etc/jitsi/videobridge/logging.properties\"" >> $CONFIG + fi + + # Updates config so new and old installs will start using the new config file + if ! grep -q "\-Dconfig.file" "$CONFIG"; then + sed -i 's|JAVA_SYS_PROPS="|JAVA_SYS_PROPS="-Dconfig.file='"$HOCON_CONFIG"' |g' $CONFIG + fi + + # unused old parameter, systemd unit files does not resolve bash variables, + # and this breaks startup script + sed -i 's/\$JVB_EXTRA_JVM_PARAMS//g' $CONFIG + + if [ ! -f $HOCON_CONFIG ]; then + echo "Generating a default hocon config" + echo "videobridge { + apis.xmpp-client.configs { + shard { + HOSTNAME=localhost + DOMAIN=\"auth.$JVB_HOSTNAME\" + USERNAME=jvb + PASSWORD=\"$JVB_SECRET\" + MUC_JIDS=\"jvbbrewery@internal.auth.$JVB_HOSTNAME\" + MUC_NICKNAME=$(uuidgen) + } + } +} +ice4j { + harvest { + mapping { + aws { + enabled = false + } + stun { + addresses = [\"meet-jit-si-turnrelay.jitsi.net:443\"] + } + } + } +}" >> $HOCON_CONFIG + fi + + # The REST API is disabled by default. It it was previously enabled via --apis, enable it in jvb.conf + if grep -v '^\s*#' /etc/jitsi/videobridge/config | grep -- '--apis=' | grep -i rest >/dev/null 2>&1 ;then + if ! hocon -f $HOCON_CONFIG get "videobridge.apis.rest.enabled" > /dev/null 2>&1 ;then + hocon -f $HOCON_CONFIG set "videobridge.apis.rest.enabled" "true" + fi + # Remove it from /etc/jitsi/videobridge/config to prevent it perpetually updating jvb.conf + sed -i 's/.*--apis.*//' $CONFIG + fi + + # we don't want to start the daemon as root + if ! getent group jitsi > /dev/null ; then + groupadd jitsi + fi + if ! getent passwd jvb > /dev/null ; then + useradd -r -g jitsi --shell /bin/bash --create-home -d /usr/share/jitsi-videobridge jvb + fi + if ! groups jvb | grep '\bjitsi\b' > /dev/null ; then + usermod -g jitsi jvb + fi + + mkdir -p /usr/share/jitsi-videobridge + + # we claim the home folder of jvb in case it is owned by someone else + OWNER=$(stat -c '%U' /usr/share/jitsi-videobridge) + GROUP=$(stat -c '%G' /usr/share/jitsi-videobridge) + if ! dpkg-statoverride --list /usr/share/jitsi-videobridge/* >/dev/null && [ "$OWNER:$GROUP" != "jvb:jitsi" ]; then + chown -R jvb:jitsi /usr/share/jitsi-videobridge + OWNER=jvb + GROUP=jitsi + fi + + CONFIG_DIR=$(dirname $CONFIG) + if ! dpkg-statoverride --list "$CONFIG_DIR" >/dev/null; then + chown -R jvb:jitsi "$CONFIG_DIR" + chmod 750 "$CONFIG_DIR" + fi + + # die logz + if [ ! -d /var/log/jitsi ]; then + mkdir -p /var/log/jitsi + fi + chown $OWNER:$GROUP /var/log/jitsi + chmod 770 /var/log/jitsi + ls /var/log/jitsi/jvb* 1>/dev/null 2>&1 && chown -f -R $OWNER:$GROUP /var/log/jitsi/jvb* + ls /var/log/jitsi/jvb* 1>/dev/null 2>&1 && chmod -f -R 640 /var/log/jitsi/jvb* + + # ensure videobridge is not running - it will be started at the end + if [ -d /run/systemd/system ]; then + systemctl stop jitsi-videobridge2.service >/dev/null || true + fi + + # clean up old jvb group + if getent group jvb > /dev/null; then + groupdel jvb + fi + + # load the UDP buffer sizes required by the JVB. As reported in + # https://github.com/jitsi/jitsi-videobridge/issues/461 + # OpenVZ containers do not typically allow permission to modify the + # kernel with sysctl config values. + sysctl --system || true + + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/debian/postrm b/debian/postrm new file mode 100644 index 0000000..ab7a2f4 --- /dev/null +++ b/debian/postrm @@ -0,0 +1,53 @@ +#!/bin/bash +# postrm script for jitsi-videobridge + +set -e + +# Load debconf +. /usr/share/debconf/confmodule + + +case "$1" in + purge) + + # Clear the debconf variable + db_purge + + # and then remove the user + if getent passwd jvb > /dev/null ; then + deluser jvb || true + fi + + # clear logs + if [ -d "/var/log/jitsi" ]; then + rm -rf "/var/log/jitsi" + fi + + rm -f /etc/jitsi/videobridge/config + rm -f /etc/jitsi/videobridge/jvb.conf + rm -f /etc/jitsi/videobridge/sip-communicator.properties + rm -f /etc/jitsi/videobridge/logging.properties + rm -rf /usr/share/jitsi-videobridge/.java + rm -rf /usr/share/jitsi-videobridge/.sip-communicator + + # make sure we remove keys we used ('jitsi-videobridge/jvb-hostname') + db_unregister jitsi-videobridge/jvb-hostname || true + ;; + + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +db_stop + +exit 0 diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..3e9dd6e --- /dev/null +++ b/debian/rules @@ -0,0 +1,34 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 +#export DEB_VERBOSE_ALL=true + +%: + dh $@ + +override_dh_auto_clean: + mvn -Dmaven.repo.local=${HOME}/.m2/repository clean + +override_dh_auto_configure: + # do nothing + +override_dh_auto_test: + # do nothing + +override_dh_auto_build: + mvn -Dmaven.repo.local=${HOME}/.m2/repository -DskipTests -Dassembly.skipAssembly=true install + mvn -Dmaven.repo.local=${HOME}/.m2/repository dependency:copy-dependencies -DincludeScope=runtime + +override_dh_install: + mkdir debian/tmp + cp config/logrotate debian/tmp/jitsi-videobridge + cp jvb/target/jitsi-videobridge*.jar debian/tmp/jitsi-videobridge.jar + dh_install + +override_dh_installinit: + dh_installinit --noscripts + +override_dh_auto_install: + dh_auto_install + dh_installsystemd --restart-after-upgrade diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/templates b/debian/templates new file mode 100644 index 0000000..d378b1f --- /dev/null +++ b/debian/templates @@ -0,0 +1,10 @@ +Template: jitsi-videobridge/jvb-hostname +Type: string +Default: localhost +_Description: Hostname: + The jitsi-videobridge package needs the DNS hostname of your instance. + +Template: jitsi-videobridge/jvbsecret +Type: password +_Description: The secret used to connect to xmpp server as component + Component secret: diff --git a/doc/allocation.md b/doc/allocation.md new file mode 100644 index 0000000..7b84c9f --- /dev/null +++ b/doc/allocation.md @@ -0,0 +1,305 @@ +# Bandwidth Allocation Algorithm + +Bandwidth allocation is the process of selecting the set of layers to forward to a specific endpoint (the "receiver"), +or "allocating" the available bandwidth among the available layers. When conditions change, the algorithm is re-run, +and the set of forwarded layers is updated according to the new result. + +The overall goal is to provide the most relevant and suitable set of streams to the receiver, given a limited bandwidth. + +## Input +### Available bandwidth +The available bandwidth (or the Bandwidth Estimation, BWE) is the estimated bandwidth between the bridge and the +receiver. It is calculated elsewhere, and is only used as input for bandwidth allocation. + +### Available sources +This is the list of streams being sent from the other endpoints in the conference. Streams have multiple layers, and +the algorithm selects one layer (or no layers) for each source. + +For example, a simulcast sender can encode its video in 3 different encodings, with 3 different frame rates each. This +gives the allocator 9 layers to choose from. + +The list of available sources changes when endpoints join or leave the conference, or when they signal a change in their +streams (such as when an endpoint switches from a video camera to screensharing or vice-versa). + +### Receiver-specified settings +The following settings are controlled by the receiver, with messages over the "bridge channel". + +#### LastN +LastN is the maximum number of video streams that the receiver wants to receive. To effectively stop receiving video +(for example to conserve bandwidth), the receiver can set LastN=0. + +#### Selected sources +This is a list of sources to be prioritized first, overriding the natural speech activity order of the endpoints which +own the sources. + +For example, if the receiver wants to always receive a screensharing source, regardless of who is speaking +in the conference, it can "select" this source. + +#### On-stage sources +This is a list of sources for which allocation should be prioritized up to a higher resolution (since they are +displayed "on stage"). On-stage sources are prioritized higher than selected sources, and in addition: +1. Allocation for them is greedy up to the preferred resolution +([360p by default](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L40)) +2. Above the preferred resolution, only frame rates of +[at least 30 fps](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L41) +are considered. + +#### Video Constraints +Video constraints are resolution (`maxHeight`) and frame rate (`maxFrameRate`) constraints for each source. These are +"soft" constraints in the sense that the bridge may exceed them in some circumstances (see below). + +When set to a negative number, they indicate no constraints. + +When set to 0, they signal that no video should be forwarded for the associated source, and this is never exceeded. + +When set to a positive number, the algorithm will attempt to select a layer which satisfies the constraints. If no +layers satisfy the constraints, and there is sufficient bandwidth, the algorithm will exceed the constraints and +select the lowest layer. In practice this is relevant only in the case where a sender does not use simulcast (or SVC), +and encodes a single high-resolution stream. Given enough bandwidth, the stream will be forwarded even when the receiver +signaled low constraints. + +## Implementation +The bandwidth allocation algorithm is implemented in [BandwidthAllocator](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt) + +It consists of 3 phases: +### 1. Prioritize +This phase orders the available sources in the desired way. It starts with sources coming from the endpoints ordered by +speech activity (dominant speaker, followed by the previous dominant speaker, etc). Then, it moves the sources from +the endpoints which are NOT sending video to the bottom of the list (this is actually implemented in +[ConferenceSpeechActivity](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/java/org/jitsi/videobridge/ConferenceSpeechActivity.java). +Finally, the selected sources are moved to the TOP of the list. + +TODO: Update the algorithm, to only move selected endpoint when they are sending video. + +### 2. Apply LastN +This phase disables the video sources in the list that are not among the first `LastN`. Note that the effective +`LastN` value comes from the number signaled by the client, potentially also limited by [static](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/kotlin/org/jitsi/videobridge/JvbLastN.kt) +and [dynamic](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt) +configuration of the bridge. This is implemented by setting the `maxHeight` constraint to 0. + +The resulting constraints are the "effective" constraints used by the rest of the algorithm. Once calculated, they are +announced via an event, so that the sender-side constraints can be applied. Doing this step here, early in the process, +allows us to do "aggressive layer suspension" (i.e. set sender-side constraints based on LastN). + +### 3. Allocation +The final phase is the actual allocation. + +#### 3.1 Initialize potential layers +The first step is to initialize a list of layers to consider for each source. It starts with the list of all layers +for the source, and prunes ones which should not be considered: + +A) The ones with resolution and frame rate higher than the constraints + +B) The ones which are inactive (the sending endpoint is currently not transmitting them) + +C) Layers with high resolution but insufficient frame rate, that is at least the [preferred resolution](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L41), +and frame rate less than the [preferred frame rate](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L42). +For example, with the defaults of preferred resolution 360p and preferred frame rate 30 fps, the following layers will +not be considered: 360p/7.5fps, 360p/15fps, 720p/7.5fps, 720p/15fps. + +#### 3.2 Allocation loop +It starts with no layers selected for any source, and remaining bandwidth equal to the total available bandwidth. +Until there is remaining bandwidth, it loops over the sources in the order obtained in [phase 1](#1.-Prioritize), +and tries to `improve()` the layer of each. + +The normal `improve()` step selects the next higher layer if there is sufficient bandwidth. For on-stage sources +the `improve()` step works eagerly up to the "preferred" resolution. +The preferred resolution [can be configured](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L40). + +# Signaling +This section describes the signaling between the client and the bridge that affects bandwidth allocation. + +## Message format + +The receiver's video constraint message is used to signal the preference of the client in regard to which media streams +it wants to receive. Usually only a portion of all available videos is displayed on the client. Each video is identified +by a source name and each endpoint can send multiple videos. The default format used in jitsi-meet follows a pattern +where the first part is the endpoint ID followed by '-v' and the zero-based index of the video source (see the example +below). + +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "lastN": 2, + "selectedSources": ["A-v0", "B-v0"], + "onStageSources": ["A-v1","C-v0", "D-v0"], + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "A-v1": { "maxHeight": 720 }, + "B-v0": { "maxHeight": 360 } + } +} +``` + +All fields are optional. The ones which are included will be updated, and the ones which are not included are not +changed. + +The `defaultConstraints` are used for sources not explicitly included in `constraints` (including new sources). + +The initial values are `lastN: -1` (unlimited), `defaultConstraints: {maxHeight: 180}` +([configurable](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L38)), +and the rest empty. + +### Examples + +#### Stage view (1) +Stage view with source `A-v0` in high definition and all other sources in 180p: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "onStageSources": ["A-v0"], + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "A-v0": { "maxHeight": 720 } + } +} +``` + +#### Stage view (2) +Stage view with source `A-v0` in high definition, `B-v0`, `C-v0`, `D-v0` in 180p and all others disabled: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "onStageSources": ["A-v0"], + "defaultConstraints": { "maxHeight": 0 }, + "constraints": { + "A-v0": { "maxHeight": 720 }, + "B-v0": { "maxHeight": 180 }, + "C-v0": { "maxHeight": 180 }, + "D-v0": { "maxHeight": 180 } + } +} +``` + +#### Stage view (3) +Stage view with source `A-v0` in high definition, `B-v0`, `C-v0`, `D-v0` disabled and all others in 180p: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "onStageSources": ["A-v0"], + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "A-v0": { "maxHeight": 720 }, + "B-v0": { "maxHeight": 0 }, + "C-v0": { "maxHeight": 0 }, + "D-v0": { "maxHeight": 0 } + } +} +``` + +#### Stage view (4) +Stage view with source `A-v0` in high definition and all other sources in 180p, with "D-v0" prioritized higher than +the dominant speaker's video source: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "onStageEndpoints": ["A-v0"], + "selectedEndpoints": ["D-v0"], + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "A-v0": { "maxHeight": 720 } + } +} +``` + +#### Tile view (1) +Tile view with all sources in 180p/15fps: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "defaultConstraints": { "maxHeight": 180, "maxFrameRate": 15 } +} +``` + +#### Tile view (2) +Tile view with all sources in 360p: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "defaultConstraints": { "maxHeight": 360 } +} +``` + +#### Tile view (3) +Tile view with 180p, sources `A-v0` and `B-v0` prioritized, and sources `C-v0` and `D-v0` disabled: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "selectedSources": ["A-v0", "B-v0"], + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "C-v0": { "maxHeight": 0 }, + "D-v0": { "maxHeight": 0 } + } +} +``` + +#### Tile view (4) +Tile view with all sources disabled except `A-v0`, `B-v0`, `C-v0`: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "defaultConstraints": { "maxHeight": 0 }, + "constraints": { + "A-v0": { "maxHeight": 180 }, + "B-v0": { "maxHeight": 180 }, + "C-v0": { "maxHeight": 180 } + } +} +``` +#### Multi-stage view (1) +With two on-stage sources, and up-to 4 other sources at 180p: +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "onStageSources": ["A-v0", "B-v0"], + "lastN": 6, + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "A-v0": { "maxHeight": 720 }, + "B-v0": { "maxHeight": 720 } + } +} +``` + + +## Old message format +The old format works with endpoint IDs rather than source names. It also uses `selectedEndpoints` and `onStageEndpoints` +instead of `selectedSources` and `onStageSources`. All use cases described in the examples above are valid here as +well, but the assumption is that every endpoint has only one video source. + +```json +{ + "colibriClass": "ReceiverVideoConstraints", + "lastN": 2, + "selectedEndpoints": ["A", "B"], + "onStageEndpoints": ["C", "D"], + "defaultConstraints": { "maxHeight": 180 }, + "constraints": { + "A": { "maxHeight": 720 }, + "B": { "maxHeight": 360 } + } +} +``` + +The support for source names vs legacy endpoint ID based format is determined on the ColibriV2 level. The endpoint +create request must indicate "source-names" capability on the endpoint that will be using the new format. This is done +automatically by Jicofo when multi stream support is enabled in jitsi-meet, but if you're using JVB without Jicofo then +remember to do that yourself. + +```xml + + + + + + ... + + + ... + + + + + +``` diff --git a/doc/constraints.md b/doc/constraints.md new file mode 100644 index 0000000..9bb0c0c --- /dev/null +++ b/doc/constraints.md @@ -0,0 +1,18 @@ +## Video Constraints + +###### Sender video constraints + +The bridge sends the following message to a sender to notify it that resolutions +higher than the specified need not be transmitted for a specific video source: + +``` +{ + "colibriClass": "SenderSourceConstraints", + "sourceName": "endpoint1-v0", + "maxHeight": 180 +} +``` + +Note that in order to handle portrait-mode video correctly, +despite the name `maxHeight`, this parameter is actually used to specify the smaller +of the dimensions of the video (which for portrait mode will be its width). diff --git a/doc/datachannel-communication.md b/doc/datachannel-communication.md new file mode 100644 index 0000000..f4c6bb0 --- /dev/null +++ b/doc/datachannel-communication.md @@ -0,0 +1,36 @@ +# Messages by COLIBRI class: + +## EndpointMessage: + + +This type of messages could be used by the clients for sending JSON objects to the other clients in the conference. A client can send a message to specific endpoint or a broadcast message. + + + +* The messages processed by Jitsi Videobridge has the following format: + +``` +{ + colibriClass: "EndpointMessage", + to: ``, + msgPayload: `` +} +``` + +`` is valid JSON string. +`` is endpoint id of existing participant or `""` for broadcast message. + +* The messages sent by Jitsi Videobridge has the following format: + + +``` +{ + colibriClass: "EndpointMessage", + to: ``, + from: ``, + msgPayload: +} +``` + + +The only modification made by Jitsi Videobridge before the message is forwarded is adding `from` property to it. The value of this property is the endpoint id of the sender of the message. diff --git a/doc/debugging.md b/doc/debugging.md new file mode 100644 index 0000000..4db4cf4 --- /dev/null +++ b/doc/debugging.md @@ -0,0 +1,201 @@ +# Introduction + +This document describes some HTTP endpoints that can be useful when debugging jitsi-videobridge. With a few exceptions, +these are NOT intended to be used against production systems. Most endpoints return valid JSON which can be fed to e.g. `jq`: +``` +curl http://localhost:8080/debug | jq . +``` + +# Basic queries + +### Health +Returns `200 OK` if the instance is currently healthy, and `4XX/5XX` otherwise. +``` +GET /about/health +``` + +### Version +Returns the currently running version. +``` +GET /about/version +``` + +### Get Dominant speaker +Returns a dominant speaker identification +``` +GET /colibri/v2/conferences/{meetingId}/dominant-speaker-identification +``` + + +### Overall statistics +Returns overall bridge statistics. These include number of current and total conferences and endpoints, number of packets/bytes received/sent, current packet rate and bit rate. They are generally suitable for storing as timeseries in order to monitor a service. +``` +GET /colibri/stats +``` + + +# Detailed debug state +The following can be used to get very detailed state of running conferences and endpoints. + +### List conferences +This includes a list of all conferences and their endpoints, but not the full state. +``` +GET /debug +``` + +### List all conferences with full state +This includes the full state of all conferences and their endpoints. The output can be substantial (~25KB per endpoint). +``` +GET /debug?full=true +``` + +## List a specific conference with full state +This includes the full state of a specific conference and its endpoints. The output can be substantial (~25KB per endpoint). +The full output can be suspended (leaving just the list of endpoints) by adding `?full=false`. +``` +GET /debug/CONFERENCE_ID +``` + +## Get full state for an endpoint +This includes the full state of a specific endpoint in a specific conference. +``` +GET /debug/CONFERENCE_ID/ENDPOINT_ID +``` + + +# Global debug options and statistic features +These are features that affect the entire bridge (as opposed to a single endpoint or conference). + +### Payload verification +This mode enables verification of the RTP payload between each step of the packet pipeline. +This is very CPU intensive and disabled by default (it is only useful for testing new code). + +Enable: +``` +POST /debug/stats/jvb/payload-verification/true +``` + +Disable: +``` +POST /debug/stats/jvb/payload-verification/false +``` + +Query: +``` +GET /debug/stats/jvb/payload-verification +``` + +### Pipeline statistics +This keeps track of processing delay, number of discarded packets, +number of packets by various types, etc. from each Node in the packet +processing pipeline. The data is organized by Node type. This has low overhead +and is enabled by default. Note that the data does not update in real-time, but +only when endpoints expire. + +Enable: +``` +POST /debug/stats/jvb/node-stats/true +``` + +Disable: +``` +POST /debug/stats/jvb/node-stats/false +``` +Query: +``` +GET /debug/stats/jvb/node-stats +``` + +### Memory pool statistics +These include current size, number of requests, allocation rates, etc. for the application memory pools. This is disabled by default. + +Enable: +``` +POST /debug/stats/jvb/pool-stats/true +``` + +Disable: +``` +POST /debug/stats/jvb/pool-stats/false +``` +Query: +``` +GET /debug/stats/jvb/pool-stats +``` + +### Packet queue statistics +This keeps track of the number of dropped packets and exception caught in the +various packet queues. It is enabled by default. + +Enable: +``` +POST /debug/stats/jvb/queue-stats/true +``` + +Disable: +``` +POST /debug/stats/jvb/queue-stats/false +``` + +Query: +``` +GET /debug/stats/jvb/queue-stats +``` + +### Packet transit time statistics +This keeps track of the overall transit time for RTP/RTCP packets (average, +max, and a distribution), as well as the jitter introduced in processing. It +is always enabled. + +Query: +``` +GET /debug/stats/jvb/transit-stats +``` + +### Task pool stats +This keeps track of statistics for the various task pools (CPU, IO, scheduled). +It is always enabled. + +Query: +``` +GET /debug/stats/jvb/task-pool-stats +``` + +### XMPP Delay stats +This keeps track of the response time for requests received over XMPP. + +Query: +``` +GET /debug/stats/jvb/xmpp-delay-stats +``` + +### Node tracing +This adds an entry to the stack trace from each Node in the packet processing pipeline. + +Enable: +``` +POST /debug/stats/jvb/node-tracing/true +``` + +Disable: +``` +POST /debug/stats/jvb/node-tracing/false +``` + +# Endpoint-specific debug options and features +These are features that can be enabled for a specific endpoint in a specific conference. + +### Saving RTP/RTCP in PCAP +This enables saving the RTP and RTCP traffic sent/received from a specific endpoint to a PCAP file in `/tmp/`. +The use of this feature is disabled by default, and the bridge needs to be explicitly configured to +allow it by setting `jmt.debug.pcap.enabled=true` in `/etc/jitsi/videobridge/jvb.conf`. + +Enable: +``` +POST /debug/features/endpoint/CONFERENCE_ID/ENDPOINT_ID/pcap-dump/true +``` + +Disable: +``` +POST /debug/features/endpoint/CONFERENCE_ID/ENDPOINT_ID/pcap-dump/false +``` diff --git a/doc/health-checks.md b/doc/health-checks.md new file mode 100644 index 0000000..c3e9c4d --- /dev/null +++ b/doc/health-checks.md @@ -0,0 +1,19 @@ +The REST API allows querying Videobridge whether it deems itself in a healthy state (i.e. the application is operational and the functionality it provides should perform as expected) at the time of the query or not. Videobridge performs periodic internal tests, and returns the latest result in response to requests to the `/about/health` endpoint. + + + + + + + + + + + + +
HTTP MethodResourceResponse
GET/about/health +
    +
  • 200 OK The service is healthy. The response is with Content-Type: application/json but no JSON value is provided i.e. Content-Length: 0.
  • +
  • 500 Internal Server Error or any other 5xx status code if Videobridge determined that it is in an unhealthy state.
  • +
+
diff --git a/doc/muc.md b/doc/muc.md new file mode 100644 index 0000000..dc0ce29 --- /dev/null +++ b/doc/muc.md @@ -0,0 +1,66 @@ +# Intro +When the XMPP API is enabled jitsi-videobridge uses an XMPP client connection and advertises presence in a Multi-User +Chat (MUC) room. + +With this mode a jitsi-videobridge instance can connect to a set of XMPP servers, and new servers +can be added [at runtime](https://github.com/jitsi/jitsi-videobridge/blob/master/doc/rest-muc-client.md). + +# XMPP server configuration +The XMPP server can be provisioned with a single user account to be shared between all jitsi-videobridge instances that +connect to it. For prosody this can be done using: + +``` +prosodyctl register jvb $DOMAIN $PASSWORD +``` + +We recommend using a separate XMPP domain, not accessible by anonymous users. +See the [jitsi-meet-prosody postinst +script](https://github.com/jitsi/jitsi-meet/blob/master/debian/jitsi-meet-prosody.postinst) +for a full example. It is also possible to use a completely separate XMPP server accessible only to jitsi-videobridge +and jicofo (not publicly accessible at all). + +# Jicofo configuration +To make use of the MUC mode in the bridge, jicofo needs to be configured to +join the same MUC. To do this set `jicofo.bridge.brewery-jid` to the MUC address in `jicofo.conf`. + +# Videobridge configuration +To enable the MUC mode add the following to the `videobridge` section of the config file: +``` +stats { + # Enable broadcasting stats/presence in a MUC + enabled = true + transports = [ + { type = "muc" } + ] +} + +apis { + xmpp-client { + configs { + # Connect to the first XMPP server + xmpp-server-1 { + hostname="example.net" + domain = "auth.example.net" + username = "jvb" + password = "$PASSWORD" + muc_jids = "JvbBrewery@internal.auth.example.net" + # The muc_nickname must be unique across all jitsi-videobridge instances + muc_nickname = "unique-instance-id" + # disable_certificate_verification = true + } + # Connect to a second XMPP server + xmpp-server-2 { + hostname="another.example.net" + domain = "auth.example.net" + username = "jvb" + password = "$PASSWORD" + muc_jids = "JvbBrewery@internal.auth.example.net" + # The muc_nickname must be unique across all jitsi-videobridge instances + muc_nickname = "unique-instance-id2" + # disable_certificate_verification = true + } + } + } +} + +``` \ No newline at end of file diff --git a/doc/network-configuration.md b/doc/network-configuration.md new file mode 100644 index 0000000..c95e337 --- /dev/null +++ b/doc/network-configuration.md @@ -0,0 +1,41 @@ +# Network Configuration + +## IPv6 Configuration + +JVB supports IPv6 in dual-stack (IPv4 + IPv6) environments. +To enable it, add the following to your `jvb.conf`: +```hocon +ice4j { + harvest { + use-ipv6 = true + use-link-local-addresses = false + mapping { + stun { + addresses = [ "your-stun-server:3478" ] + } + static-mappings = [ + { + local-address = "YOUR_PRIVATE_IPV4" + public-address = "YOUR_PUBLIC_IPV4" + } + ] + } + } +} +``` + +### Notes +- Jitsi Web and the Videobridge can have different IPv6 addresses +- When IPv6 is enabled, a UDP6 listener will open on port 10000 + +### Verifying ICE Candidates +To confirm correct candidates are advertised, enable verbose logging in `/etc/jitsi/videobridge/logging.properties`: +``` +org.jitsi.videobridge.xmpp.XmppConnection.level=ALL +``` + +Then grep the logs for `SENT` to inspect the ICE candidates being sent. + +### Related Issues +- [#1722](https://github.com/jitsi/jitsi-videobridge/issues/1722) + diff --git a/doc/relay.md b/doc/relay.md new file mode 100644 index 0000000..7d75542 --- /dev/null +++ b/doc/relay.md @@ -0,0 +1,86 @@ +# Setting up relays (cascaded bridges) + +### Relays +Relays (aka secure octo) use ICE and DTLS/SRTP between each pair of bridges, so a secure +network is not required. It uses and requires either SCTP or colibri websockets for the +bridge-bridge connections. + +## Jitsi Videobridge configuration + +### Relay configuration +Relays can be configured with the following properties in `/etc/jitsi/videobridge/jvb.conf` (also see +[reference.conf](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L132)). +``` + videobridge { + relay { + enabled=true + region="region1" + relay-id="unique-id-of-the-jitsi-videobridge-instance" + } + } +``` + +Legacy configuration is detected and automatically adapted, so no configuration changes are necessary when upgrading. + +Relays require colibri websockets for the bridge-to-bridge connections, which can be enabled as described in +[this document](https://github.com/jitsi/jitsi-videobridge/blob/master/doc/web-sockets.md). Note that the `colibri-relay-ws` +endpoints also need to be proxied. + +## Jicofo configuration +Octo needs to be enabled, and a suitable bridge selection strategy needs to be configured in `/etc/jitsi/jicofo/jicofo.conf`: + +``` +jicofo { + bridge { + selection-strategy = RegionBasedBridgeSelectionStrategy + } + octo { + enabled = true + } +} +``` + +The `RegionBasedBridgeSelectionStrategy` matches the region of the clients to +the region of the Jitsi Videobridge instances. That is, it always attempts to select a bridge +in the region of the client. + +The `SplitBridgeSelectionStrategy` can be used for testing. It tries to select a new bridge +for each client, regardless of the regions. This is useful while testing, because you can +verify that Octo works before setting up the region configuration for the clients. + +*Important note*: Jicofo will not mix bridges with different versions in the same conference. Even with +`SplitBridgeSelectionStrategy` if the bridges have different versions only bridges with the same version will be used. +To prevent this behavior (for testing purposes only) set `jicofo.ocfo.allow-mixed-versions=true` in `jicofo.conf`. + +## Debugging +Jicofo's debug interface can be used for troubleshooting. + +To list the bridges available to jicofo (with their versions, regions, etc): +```commandline +curl "http://localhost:8888/debug" | jq .bridge_selector +``` + +To list currently running conferences: +```commandline +curl "http://localhost:8888/debug" | jq .focus_manager +``` + +To list the full state of a conference (get actual conference ID with the command above, escaping the '@' sign): +```commandline +curl "http://localhost:8888/debug/conference/test\@conference.example.com" | jq . +``` + +Look under `colibri_session_manager` for the different bridge sessions. + + +## Configuring client regions +Clients learn about the region that they are in via the value of +`config.deploymentInfo.userRegion`. Thus, in order for relays to actually work +for geolocation, `config.js` must be served with the correct values. The best +way to implement this is outside the scope of this document, and it depends on +the environment in which Jitsi Meet is installed. + +One easy solution can be to set the variable based on the domain name, and then +access the conference through the desired domain name, e.g. use +`us-east.jitsi.example.com`, `us-west.jitsi.example.com`, +`eu.jitsi.example.com` and have them serve custom versions of `config.js`. diff --git a/doc/rest-colibri2.md b/doc/rest-colibri2.md new file mode 100644 index 0000000..4e9aa64 --- /dev/null +++ b/doc/rest-colibri2.md @@ -0,0 +1,315 @@ +# Introduction + +This document describes the Colibri2 REST APIs + +See [this document](rest.md) for how to configure the HTTP(S) interfaces of jitsi-videobridge. + +We envision using the HTTP/JSON version of the colibri2 protocol you can see some examples [here](https://github.com/jitsi/jitsi-xmpp-extensions/blob/master/src/test/kotlin/org/jitsi/xmpp/extensions/colibri2/json/Colibri2JSONSerializerTest.kt). + +# Implementation + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HTTP MethodResourceResponse
GET/colibri/v2/conferences/{meetingId}/dominant-speaker-identification + 200 OK with JSON objects which represents endpointsBySpeechActivity,dominantEndpoint, endpointsInLastNOrder, dominantSpeakerIdentification
+ For example: +
+{
+    "endpointsBySpeechActivity": [
+        "79f0273d"
+    ],
+    "dominantSpeakerIdentification": null,
+    "endpointsInLastNOrder": [
+        "79f0273d"
+    ],
+    "dominantEndpoint": "null"
+}
+
POST/colibri/v2/conferences/ + 200 OK with a JSON object which represents the created conference if the request was with Content-Type: application/json and was a JSON object which + consist field like meeting-id, endpoints, media types, media payloads
+ See the test example + + Colibri2JSONSerializerTest +
+ For example, a request could look like: +
+{
+  "meeting-id":"beccf2ed-5441-4bfe-96d6-f0f3a6796378",
+  "name":"hardencounterssinkright@muc.meet.jitsi",
+  "create":true,
+  "endpoints":[
+    {
+      "create":true,
+      "id":"79f0273d",
+      "stats-id":"Garett-w1o",
+      "muc-role":"moderator",
+      "medias":[
+        {
+          "type":"audio",
+          "payload-types":[
+            {
+              "name": "red", "id": "112", "channels": "2", "clockrate": "48000",
+              "parameters": { "null": "111/111" }
+            },
+            {
+              "name": "opus", "id": "111", "channels": "2", "clockrate": "48000",
+              "parameters": {"useinbandfec": "1", "minptime": "10" },
+              "rtcp-fbs": [{"type": "transport-cc"}]
+            }
+          ],
+          "rtp-hdrexts":[
+            { "uri":"urn:ietf:params:rtp-hdrext:ssrc-audio-level", "id":1 },
+            { "uri":"http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", "id":5 }
+          ],
+          "extmap-allow-mixed":true
+        },
+        {
+          "type": "video",
+          "payload-types":[
+            {
+              "name": "VP8", "id": "100", "clockrate": "90000",
+              "parameters": {"x-google-start-bitrate": "800"},
+              "rtcp-fbs":[
+                { "type": "ccm", "subtype": "fir" },
+                { "type": "nack" },
+                { "type": "nack", "subtype": "pli" },
+                { "type": "transport-cc" }
+              ]
+            },
+            {
+              "name": "VP9", "id": "101", "clockrate": "90000",
+              "parameters": {"x-google-start-bitrate": "800"},
+              "rtcp-fbs":[
+                { "type": "ccm", "subtype": "fir" },
+                { "type": "nack" },
+                { "type": "nack", "subtype": "pli" },
+                { "type": "transport-cc" }
+              ]
+            }
+          ],
+          "rtp-hdrexts":[
+            { "uri":"http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time", "id":3 },
+            { "uri":"http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", "id":5 }
+          ],
+          "extmap-allow-mixed":true
+        }
+      ],
+      "transport": { "ice-controlling": true },
+      "capabilities": [ "source-names" ]
+    }
+  ],
+  "connects": [
+    { "url": "wss://example.com/audio", "protocol": "mediajson", "type": "transcriber", "audio": true },
+    { "url": "wss://example.com/video", "protocol": "mediajson", "type": "recorder", "video": true }
+  ]
+}
+ +The respective response could look like: + +
+{
+    "endpoints": [
+        {
+            "id": "79f0273d",
+            "transport": {
+                "transport": {
+                    "candidates": [
+                        {
+                            "generation": "0",
+                            "component": "1",
+                            "protocol": "udp",
+                            "port": "10000",
+                            "ip": "172.18.0.4",
+                            "foundation": "1",
+                            "id": "5bbb5d5e101f870f0ffffffffdf286239",
+                            "priority": "2130706431",
+                            "type": "host",
+                            "network": "0"
+                        },
+                        {
+                            "generation": "0",
+                            "rel-port": "10000",
+                            "component": "1",
+                            "protocol": "udp",
+                            "port": "10000",
+                            "ip": "192.168.1.1",
+                            "foundation": "2",
+                            "id": "56f8f898101f870f0fffffffff3be6336",
+                            "rel-addr": "172.18.0.4",
+                            "priority": "1694498815",
+                            "type": "srflx",
+                            "network": "0"
+                        },
+                        {
+                            "generation": "0",
+                            "rel-port": "10000",
+                            "component": "1",
+                            "protocol": "udp",
+                            "port": "10000",
+                            "ip": "152.58.154.133",
+                            "foundation": "2",
+                            "id": "c6ac5101f870f0ffffffffcb50fcba",
+                            "rel-addr": "172.18.0.4",
+                            "priority": "1694498815",
+                            "type": "srflx",
+                            "network": "0"
+                        }
+                    ],
+                    "xmlns": "urn:xmpp:jingle:transports:ice-udp:1",
+                    "ufrag": "26ant1ioqu7qnr",
+                    "rtcp-mux": true,
+                    "pwd": "9q8uj7tg6j4rc4i6ulc1pmsq9",
+                    "web-sockets": [
+                        "wss://localhost:8443/colibri-ws/172.18.0.4/7e441eda3701676e/79f0273d?pwd=9q8uj7tg6j4rc4i6ulc1pmsq9"
+                    ],
+                    "fingerprints": [
+                        {
+                            "fingerprint": "F1:05:72:F8:F3:F4:B4:65:5B:9A:FC:81:24:11:23:78:D5:42:54:B1:F2:66:2F:82:CC:2B:01:1C:DB:98:BE:C0",
+                            "setup": "actpass",
+                            "hash": "sha-256"
+                        }
+                    ]
+                }
+            }
+        }
+    ],
+    "sources": [
+        {
+            "sources": [
+                {
+                    "ssrc": 3191927963,
+                    "name": "jvb-a0"
+                }
+            ],
+            "id": "jvb-a0",
+            "type": "audio"
+        },
+        {
+            "sources": [
+                {
+                    "ssrc": 1863439907,
+                    "name": "jvb-v0"
+                }
+            ],
+            "id": "jvb-v0",
+            "type": "video"
+        }
+    ]
+}
+
PATCH/colibri/v2/conferences/{mettingId} + Expire conference
+ 200 OK which represents success.
+ For example: +
+The Request body to expire conference: 
+{
+  "expire":true
+}
+
+
PATCH/colibri/v2/conferences/{mettingId} + 200 OK which represents success.
+ Here, I am doing it for + + Conference-modified with transport for an endpoint and feedback sources +
+In the same way you can do it for + + Update endpoint with transport and sources +
+For example, a request could look like: +
+{
+  "endpoints": [
+    {
+      "id":"79f0273e",
+      "transport": {
+        "transport": {
+          "candidates": [
+            {
+              "generation": "0",
+              "rel-port": "9",
+              "component": "1",
+              "protocol": "udp",
+              "port": "10000",
+              "ip": "129.80.210.199",
+              "foundation": "2",
+              "id": "653aa1ba295b62480ffffffffdc52c0d9",
+              "rel-addr": "0.0.0.0",
+              "priority": "1694498815",
+              "type": "srflx",
+              "network": "0"
+            }
+          ],
+          "xmlns": "urn:xmpp:jingle:transports:ice-udp:1",
+          "ufrag": "2ivqh1fvtf0l3h",
+          "rtcp-mux": true,
+          "pwd": "1a5ejbent91k6io6a3fauikg22",
+          "web-sockets": [
+            "wss://beta-us-ashburn-1-global-2808-jvb-83-102-26.jitsi.net:443/colibri-ws/default-id/3d937bbdf97a23e0/79f0273e?pwd=1a5ejbent91k6io6a3fauikg22"
+          ],
+          "fingerprints": [
+            {
+              "fingerprint": "2E:CC:85:71:32:5B:B5:60:64:C8:F6:7B:6D:45:D4:34:2B:51:A0:06:B5:EA:2F:84:BC:7B:64:1F:A3:0A:69:23",
+              "setup": "actpass",
+              "hash": "sha-256",
+              "cryptex": true
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "sources": [
+    {
+      "type": "audio",
+      "id": "jvb-a0",
+      "sources": [
+        { "ssrc":411312308, "name": "jvb-a0", "parameters": { "msid": "mixedmslabel mixedlabelaudio0" } }
+      ]
+    },
+    {
+      "type": "video",
+      "id": "jvb-v0",
+      "sources": [
+        { "ssrc":3929652146, "name": "jvb-v0", "parameters": { "msid": "mixedmslabel mixedlabelvideo0" } }
+      ]
+    }
+  ]
+}
+ +The respective response could look like: + +
+{
+    "endpoints": [
+        {
+            "id": "79f0273e"
+        }
+    ]
+}
+
diff --git a/doc/rest-muc-client.md b/doc/rest-muc-client.md new file mode 100644 index 0000000..6a8b4f6 --- /dev/null +++ b/doc/rest-muc-client.md @@ -0,0 +1,34 @@ +The configuration for the XMPP client connections that jitsi-videobridge uses can be modified at run time using REST calls to `/colibri/muc-client/`. + +# Adding an XMPP client connection +A new XMPP client connection (i.e. a MucClient) can be added by posting a JSON which contains its configuration to `/colibri/muc-client/add`: +``` +{ + "id": "new-client-connection", + "domain":"xmpp.example.com", + "hostname":"10.0.0.1", + "username":"xmpp-username", + "password":"xmpp-password", + "muc_jids":"JvbBrewery@conference.xmpp.example.com", + "muc_nickname":"unique-resource", + "disable_certificate_verification":"false" +} +``` + +If a configuration with the specified ID already exists the behavior depends on whether the `hostname`, `port`, +`domain` and `username` fields match the existing client. If they match no changes are made, otherwise the existing +`MucClient` is removed and a new one is added. + +# Removing an XMPP client connection +An XMPP client connection (i.e. a MucClient) can be removed by posting a JSON which contains its ID to `/colibri/muc-client/remove`: +``` +{ + "id": "new-client-connection" +} +``` + +The request will be successful (return 200) if an XMPP client connection was removed. + + +# Listing XMPP client connections +IDs of previously added XMPP client connections can be listed using a GET call to `/colibri/muc-client/list`. diff --git a/doc/rest.md b/doc/rest.md new file mode 100644 index 0000000..0703bba --- /dev/null +++ b/doc/rest.md @@ -0,0 +1,61 @@ +Introduction +============== +Jitsi-videobridge supports two HTTP(S) interfaces, a _public_ and a _private_ one. +The two interfaces use different ports. The _private_ interface exposes +HTTP endpoints which are not meant to be publicly accessible (but might be used +by other components of the infrastructure, e.g. a signaling server), such as: + +* The [COLIBRI control interface](rest-colibri2.md) (```/colibri/```) +* The health-check interface (```/about/health```) +* The version interface (```/about/version```) + +The _public_ interface is used for Colibri Web Sockets that clients connect to. + +Configuration +============== + +The following configuration properties can be added to the jitsi-videobridge configuration file (whichever file is +passed to `-Dconfig.file` when running the bridge, usually `/etc/jitsi/videobridge/jvb.conf`) to control the behavior +of the HTTP interfaces. + +For the _private_ interface, use the following block and values: + +```hocon +videobridge { + apis { + rest { + enabled = true + } + } + http-servers { + private { + // The port for the private HTTP interface (or -1 to disable it). Defaults to 8080. + port = + // The port for the private HTTP interface if TLS is to be used (or -1 to disable). Defaults to 8443. + tls-port = + // The address on which the server will listen + host = + // The file path to the keystore to be used with HTTPS for the private interface. If this is not specified, + // HTTPS is disabled for the private interface + key-store-path = + // The password to be used by the SslContextFactory when HTTPS is used + key-store-password = + // Whether or not client certificate authentication is to be required when + // HTTPS is used + need-client-auth: + } + } +} +``` + +The _public_ interface uses the same values, but under a different scope: + +```hocon +videobridge { + http-servers { + public { + // Same values as above + } + } +} +``` diff --git a/doc/statistics.md b/doc/statistics.md new file mode 100644 index 0000000..2eb58f3 --- /dev/null +++ b/doc/statistics.md @@ -0,0 +1,129 @@ +Introduction +============ +Jitsi Videobridge exports statistics/metrics as key-value pairs in three ways: in a custom JSON format, as Prometheus +metrics and as periodic reports published in an XMPP MUC. + +# REST +The statistics are available through the `/colibri/stats` endpoint on the [private REST interface](rest.md) +(if it has been enabled) in JSON format: +```json +{ + "key": "value" +} +``` + +Note that the report itself is generated periodically and a cached version is returned. The period defaults to 5 seconds +and can be configured with the `videobridge.stats.interval` property in `jvb.conf`. + +# Prometheus +Metrics are exposed in a Prometheus/openmetrics format at `/metrics` on the private HTTP interface. Not all metrics are +currently available in this format, and improving this is a work in progress. +```commandline +# Query prometheus metrics in openmetrics format +curl 0:8080/metrics -H 'Accept: application/openmetrics-text' + +# Query prometheus metrics in 004 format +curl 0:8080/metrics -H 'Accept: text/plain; version=0.0.4;' + +# Query prometheus metrics in a JSON key-value format +curl 0:8080/metrics -H 'Accept: application/json' +``` + +Note: Conference size distribution metrics (by total endpoints, by audio senders, and by video senders) are available as +Prometheus histograms (`conferences_by_size`, `conferences_by_audio_sender`, `conferences_by_video_sender`) but are not +exposed via the JSON `/colibri/stats` endpoint. + +# XMPP MUC +The statistics can also be published periodically via XMPP (which allows jicofo to monitor a set of bridges and perform +load balancing, or allows an application to monitor the MUC and collect metrics from multiple bridges). In this case the +key-value pairs are represented in XML format with a `stats` element like this: +```xml + + + +``` + +By default, statistics are pushed every 5 seconds and this can be configured in `jvb.conf` with the +[`videobridge.apis.xmpp-client.presence-interval`](https://github.com/jitsi/jitsi-videobridge/blob/master/jvb/src/main/resources/reference.conf#L65) property. + + +# Supported metrics +Below is a non-exhaustive list of currently supported metrics. In the descriptions "current" means an instantaneous +value (when the report was generated), and "total" means a cumulative value since the application was started. + +* `bit_rate_download` - the current incoming bitrate (RTP) in kilobits per second. +* `bit_rate_upload` - the current outgoing bitrate (RTP) in kilobits per second. +* `conferences` - The current number of conferences. +* `current_timestamp` - the UTC time at which the report was generated. +* `dtls_failed_endpoints` - the total number of endpoints which failed to establish a DTLS connection. +* `endpoints` - the current number of endpoints, including `octo` endpoints. +* `endpoints_sending_audio` - current number of endpoints sending (non-silence) audio. +* `endpoints_sending_video` - current number of endpoints sending video. +* `endpoints_with_spurious_remb` - total number of endpoints which have sent an RTCP REMB packet when REMB was not +signaled. +* `graceful_shutdown` - whether jitsi-videobridge is currently in graceful shutdown mode (hosting existing conferences, +but not accepting new ones). +* `inactive_conferences` - current number of conferences in which no endpoints are sending audio nor video. Note that +this includes conferences which are currently using a peer-to-peer transport. +* `inactive_endpoints` - current number of endpoints in inactive conferences (see `inactive_conferences`). +* `largest_conference` - the size of the current largest conference (counting all endpoints, including `octo` +endpoints which are connected to a different jitsi-videobridge instance) +* `local_active_endpoints` - the current number of local endpoints (not `octo`) which are in an active conference. This +includes endpoints which are not sending audio or video, but are in an active conference (i.e. they are receive-only). +* `local_endpoints` - the current number of local (non-`octo`) endpoints. +* `num_eps_oversending` - current number of endpoints to which we are oversending. +* `octo_conferences` - current number of conferences in which `octo` is enabled. +* `octo_endpoints` - current number of `octo` endpoints (connected to remove jitsi-videobridge instances). +* `octo_receive_bitrate` - current incoming bitrate on the `octo` channel (combined for all conferences) in bits per +second. +* `octo_receive_packet_rate` - current incoming packet rate on the `octo` channel (combined for all conferences) in +packets per second. +* `octo_send_bitrate` - current outgoing bitrate on the `octo` channel (combined for all conferences) in bits per +second. +* `octo_send_packet_rate` - current outgoing packet rate on the `octo` channel (combined for all conferences) in +packets per second. +* `p2p_conferences` - current number of peer-to-peer conferences. These are conferences of size 2 in which no endpoint +is sending audio not video. Presumably the endpoints are using a peer-to-peer transport at this time. +* `packet_rate_download` - current RTP incoming packet rate in packets per second. +* `packet_rate_upload` - current RTP outgoing packet rate in packets per second. +* `participants` - current number of endpoints, including `octo` endpoints. Deprecated. +* `preemptive_kfr_sent` - total number of preemptive keyframe requests sent. +* `receive_only_endpoints` - current number of endpoints which are not sending audio nor video. +* `region` - preconfigured region used for bridge selection in jicofo. +* `relay_id` - encodes the `octo` address of this bridge. +* `rtt_aggregate` - round-trip-time measured via RTCP averaged over all local endpoints with a valid RTT measurement in +milliseconds. +* `stress_level` - current stress level on the bridge, with 0 indicating no load and 1 indicating the load is at full +capacity (though values >1 are permitted). +* `threads` - current number of JVM threads. +* `total_bytes_received` - total number of bytes received in RTP. +* `total_bytes_received_octo` - total number of bytes received on the `octo` channel. +* `total_bytes_sent` - total number of bytes sent in RTP. +* `total_bytes_sent_octo` - total number of bytes sent on the `octo` channel. +* `total_colibri_web_socket_messages_received` - total number of messages received on a Colibri "bridge channel" +messages received on a WebSocket. +* `total_colibri_web_socket_messages_sent` - total number of messages sent over a Colibri "bridge channel" messages +sent over a WebSocket. +* `total_conference_seconds` - total number of conference-seconds served (only updates once a conference expires). +* `total_conferences_completed` - total number of conferences completed. +* `total_conferences_created` - total number of conferences created. +* `total_data_channel_messages_received` - total number of Colibri "bridge channel" messages received on SCTP data +channels. +* `total_data_channel_messages_sent` - total number of Colibri "bridge channel" messages sent over SCTP data +channels. +* `total_dominant_speaker_changes` - total number of times the dominant speaker in a conference changed. +* `total_failed_conferences` - total number of conferences in which no endpoints succeeded to establish an ICE +connection. +* `total_ice_failed` - total number of endpoints which failed to establish an ICE connection. +* `total_ice_succeeded` - total number of endpoints which successfully established an ICE connection. +* `total_ice_succeeded_relayed` - total number of endpoints which connected through a TURN relay (currently broken). +* `total_packets_dropped_octo` - total number of packets dropped on the `octo` channel. +* `total_packets_received` - total number of RTP packets received. +* `total_packets_received_octo` - total number packets received on the `octo` channel. +* `total_packets_sent` - total number of RTP packets sent. +* `total_packets_sent_octo` - total number packets sent over the `octo` channel. +* `total_partially_failed_conferences` - total number of conferences in which at least one endpoint failed to establish +an ICE connection. +* `total_participants` - total number of endpoints created. +* `version` - the version of jitsi-videobridge. +* `videochannels` - current number of endpoints with a video channel (i.e. which support receiving video). Deprecated. diff --git a/doc/svc.md b/doc/svc.md new file mode 100644 index 0000000..46b26a2 --- /dev/null +++ b/doc/svc.md @@ -0,0 +1,124 @@ +# SSRC rewriting principles for VP8 simulcast/SVC + +WebRTC has no notion of [simulcast][simulcast] when receiving a video stream, +and it is therefore critical that any switch made by the SFU must be completely +transparent to the receiver [1]. + +1) In the case of sending an RTP stream without using picture ids the SFU has +to rewrite the RTP sequence numbers so that the stream looks continuous, and +rewrite the RTP timestamps so that there is no significant jump compared to the +last frame. The timestamp may not jump backward. + +2) If picture ids are used, besides following the steps outlined in 1), the SFU +also has to rewrite the picture ids to make them continuous. Note that it is +not possible to suddenly drop picture ids from the stream, as the receiver only +keeps state relevant to picture ids, if they are used and therefore won't fall +back to RTP sequence numbers. + +3) If picture ids, tl0 picture indexes and temporal indexes are used, besides +following the steps outlined in 2), the SFU also has to rewrite the tl0 picture +indexes. Note that it is possible to drop the tl0 picture indexes and the +temporal indexes from the stream, but if they are used again the tl0 picture +index has to continue from where it left off. Also note that the state of the +tl0 picture indexes is only updated by the receiver when both the tl0 picture +index and temporal index is received. + +# Analysis + +With simple simulcast, the SFU simply needs to apply a fixed delta (i.e. linear +translation) to the RTP sequence numbers/timestamps [4] and VP8 +picture IDs/tl0picidx [3] that it forwards. The specific deltas are determined +when the stream switching occurs. This straightforward approach doesn't work +well if the SFU wants to leverage VP8 temporal scalability. We start our +analysis by exploring the implications of the first guideline and, as we will +see, 2 and 3 follow naturally. + +With temporal scalability in VP8 (and other codecs)[2], where the SFU can drop frames to achieve +lower frame rates, applying a fixed delta would leave gaps in the sequence +numbers (because of the packets that the SFU drops). So the SFU needs to keep +track of what it's sent and manage the sequence number space of the +SFU-receiver leg (i.e. generate sequence numbers for every egress packet of a +frame). This mode of operation has several implications. + +First, it means that if the SFU decides to skip a frame for whatever reason, +it cannot go back and change that decision because there would be no space left +in the sequence numbers. Furthermore, if packet re-ordering or loss has occurred, +a decision must be made immediately as to how much of a gap to leave in the +sequence numbers so delayed packets have a place to be transmitted. + +When a frame arrives and it is to be forwarded, the SFU needs to determine how to +assign its projected sequence numbers. Decisions on how to assign sequence numbers +are done by comparing the _new_ frame with an _existing_ (previously-projected) frame. +(In most cases this existing frame will be the previous projected frame, in sequence number +ordering to the new frame, though in some corner cases involving very old frames +a different existing frame is chosen.) + +If there is a gap in receive sequence numbers between the existing and the new frame, +the SFU can determine the worst-case gap size of projected sequence numbers to +leave such that any not-yet-received packets can be forwarded correctly. +However, the SFU can't (in general) know the temporal layer of frames it has +not yet received. If it turns out that the not-yet-received frames were in fact +not ones that the SFU wanted to forward, this will result in sequence number +gaps in the projected packet sequence, which the receiver may interpret as packet +loss. (Care must be taken in bandwidth estimation algorithms not to incorrectly +reduce bandwidth based on this "false" packet loss; in particular, this means +that senders operating on receiver-side REMB-based bandwidth estimation mustn't +adjust the estimates based on received loss.) + +# Examples of projection + +## Scenario A: Happy path (target TL0) + +Sender sends: TL0 (1, 2, 3), TL2 (4, 5), TL1 (6, 7), TL2 (8), TL0 (9, 10, 11) + +Bridge receives: what the sender sends + +Bridge sends: 101, 102, 103, (9 -> 104), (10 -> 105), (11 -> 106) + +When packet 9 arrives, the bridge can determine that sequence numbers 4 - 8 +belong to non-routed packets and can be suppressed + +## Scenario B: packet loss (target TL2) + +Sender sends: TL0 (1, 2, 3), TL2 (4, 5), TL1 (6, 7), TL2 (8), TL0 (9, 10, 11) + +Bridge receives: 1, 2, 3, 4, _(X)_, 6, 7, 8, 9, 10, 11, 5 + +Bridge sends: 101, 102, 103, 104, 106, 107, 108, 109, 110, 111, 105 + +When packet 6 arrives, the bridge computes the gap from the previously projected +frame. Because it wants to route both the TL2 and TL1 frames, it leaves +a gap in the outgoing sequence numbers, into which packet 5 can later be slotted. + +## Scenario C: packet loss 1 (target TL0) + +Sender sends: TL0 (1, 2, 3), TL2 (4, 5), TL1 (6, 7), TL2 (8), TL0 (9, 10, 11) + +Bridge receives: 1, 2, 3, 4, _(X)_, 6, 7, 8, 9, 10, 11, 5 + +Bridge sends: 101, 102, 103, (9 -> 104), (10 -> 105), (11 -> 106) + +Similarly to Scenario A, when packet 9 arrives, the bridge can determine that +sequence numbers 4-8 belong to non-routed packets. The status of the +unreceived packet 5 can be determined because packet 4 will not have its +"end of frame" flag set. + +## Scenario D: packet loss 2 (target TL1) + +Sender sends: TL0 (1, 2, 3), TL2 (4, 5), TL1 (6, 7), TL2 (8), TL0 (9, 10, 11) + +Bridge receives: 1, 2, 3, 4, 5, 6, 7, _(X)_, 9, 10, 11, 8 + +Bridge sends: 101, 102, 103, (6 -> 104), (7 -> 105), (9 -> 107), (10 -> 108), (11 -> 109) + +In this case, when packet 9 arrives, the bridge cannot determine what type of +frame packet 8 was -- it could have been a TL1. Therefore it must leave a gap in +the outgoing sequence numbers. + + +[simulcast]: https://ieeexplore.ieee.org/abstract/document/7992929 +[1]: https://groups.google.com/d/topic/discuss-webrtc/gik2VH4hUjk/discussion +[2]: https://webrtchacks.com/sfu-simulcast/ +[3]: https://tools.ietf.org/html/rfc7741 +[4]: https://tools.ietf.org/html/rfc3550 + diff --git a/doc/web-sockets.md b/doc/web-sockets.md new file mode 100644 index 0000000..52eefdc --- /dev/null +++ b/doc/web-sockets.md @@ -0,0 +1,130 @@ +# Introduction +WebSockets can be used instead of WebRTC Data Channels for transport of Colibri +client-to-bridge messages. This needs support from the bridge as well as the +client. + +When this is enabled, a bridge will advertise a Colibri WebSocket URL together with +its ICE candidates. The URL will be specific to an endpoint (in fact the ICE username +fragment is reused, encoded as a URL parameter, for authentication), and connections +to it will be routed to the Endpoint representation in the bridge. + +The URL has the following format: +``` +wss://example.com/colibri-ws/server-id/conf-id/endpoint-id?pwd=123 +``` + +For secure-octo relays the URL has the following format: +``` +wss://example.com/colibri-relay-ws/server-id/conf-id/relay-id?pwd=123 +``` + +# Bridge configuration +To enable WebSockets on the bridge, both the publicly-accessible HTTP server and WebSockets +themselves must be enabled. + +The following block in jvb.conf configures the publicly-accessible HTTP server: + +To enable the publicly-accessible HTTP server without TLS: +``` +videobridge { + http-servers { + public { + port = 9090 + } + } +} +``` + +To enable the publicly-accessible HTTP server with TLS: +``` +videobridge { + http-servers { + public { + tls-port = 443 + key-store-path=/etc/jitsi/videobridge/ssl.store + key-store-password=KEY_STORE_PASSWORD + } + } +} +``` + +To enable WebSockets: +``` +videobridge { + websockets { + enabled = true + tls = true # or false, depending on your HTTP server config + # The port here is the 'advertise port' for websockets, which means the publicly-accessible + # port clients will use. This may match the public http server port, but could also be different + # if a proxy is being used. + domain = "example.com:443" + # A server ID can optionally be provided. This is useful when a set of jitsi-videobridge instances + # are fronted by an HTTP proxy and they advertise the same domain. + server-id = jvb2 + } +} +``` + +# Proxy configuration +If you are using an HTTP proxy, it needs to support WebSocket. The following is +an example `nginx` configuration which fronts two bridges. The two bridges use +ports `9090` and `9091` without TLS, and are configured with +`COLIBRI_WS_SERVER_ID=jvb1` and `jvb2` (see above) respectively. + +``` + # colibri (JVB) websockets for jvb1 + location ~ ^/colibri-ws/jvb1/(.*) { + proxy_pass http://127.0.0.1:9090/colibri-ws/jvb1/$1$is_args$args; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + tcp_nodelay on; + } + location ~ ^/colibri-ws/jvb2/(.*) { + proxy_pass http://127.0.0.1:9091/colibri-ws/jvb2/$1$is_args$args; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host alpha.jitsi.net; + tcp_nodelay on; + } +``` + +This configuration allows two jitsi-videobridge instances to run on the same +machine, which is useful while testing Octo. + +For secure-octo the `/colibri-relay-ws` endpoints also need to be proxied: +``` + # colibri secure-octo relay websockets for jvb1 + location ~ ^/colibri-relay-ws/jvb1/(.*) { + proxy_pass http://127.0.0.1:9090/colibri-relay-ws/jvb1/$1$is_args$args; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + tcp_nodelay on; + } + location ~ ^/colibri-relay-ws/jvb2/(.*) { + proxy_pass http://127.0.0.1:9091/colibri-relay-ws/jvb2/$1$is_args$args; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host alpha.jitsi.net; + tcp_nodelay on; + } +``` + +# Troubleshooting +To verify that WebSockets are configured and used, first check that the Colibri +WebSocket URL is advertised to the clients. Open a conference and look for +"session-initiate" in the javascript console logs. Extend the XML and look for +`description -> content -> transport`. You should see a `web-socket` element +like this (you can verify this on meet.jit.si): + +```xml + +``` + +Make sure that the URL is correct, and that your infrastructure routes it to +the correct jitsi-videobridge instance. Finally, check the `Network` tab in the +Chrome dev console and look for requests to this URL. You should see a websocket +opened to this URL, and messages exchanged every few seconds. diff --git a/jitsi-media-transform/README.md b/jitsi-media-transform/README.md new file mode 100644 index 0000000..84364f8 --- /dev/null +++ b/jitsi-media-transform/README.md @@ -0,0 +1,21 @@ +# Jitsi Media Transform +Jitsi Media Transform contains classes for processing and transforming RTP and RTCP packets + +# Code style +We use [ktlint](https://pinterest.github.io/ktlint/) for linting and autoformatting. It runs as part of +`mvn verify` (the build fails on style violations), and the version used is pinned by `ktlint.version` in the +root `pom.xml`, so no separate installation is needed. + +To run only the check, or to autoformat, run these in this module's directory: +``` +mvn exec:exec@ktlint-check +mvn exec:exec@ktlint-format +``` + +If you also want the `ktlint` command-line tool (for editor integration or a git hook), install a version matching +`ktlint.version`, e.g. on macOS with Homebrew: +``` +brew install ktlint +``` +It can then be run directly with `ktlint` (or `ktlint -F` to autoformat), install a pre-commit hook with +`ktlint installGitPreCommitHook`, and configure IntelliJ IDEA with `ktlint applyToIDEAProject`. diff --git a/jitsi-media-transform/checkstyle.xml b/jitsi-media-transform/checkstyle.xml new file mode 100644 index 0000000..8b1229c --- /dev/null +++ b/jitsi-media-transform/checkstyle.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/jitsi-media-transform/pom.xml b/jitsi-media-transform/pom.xml new file mode 100644 index 0000000..6585917 --- /dev/null +++ b/jitsi-media-transform/pom.xml @@ -0,0 +1,408 @@ + + + 4.0.0 + + + org.jitsi + jvb-parent + 2.3-SNAPSHOT + + + org.jitsi + jitsi-media-transform + + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + + + ${project.groupId} + jitsi-srtp + 1.1-23-gaf3cd06 + + + ${project.groupId} + rtp + ${project.version} + + + ${project.groupId} + jitsi-utils + ${jitsi.utils.version} + + + ${project.groupId} + jicoco-config + ${jicoco.version} + + + ${project.groupId} + jitsi-metaconfig + + + org.jetbrains.kotlin + kotlin-reflect + ${kotlin.version} + + + org.bouncycastle + bctls-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.pcap4j + pcap4j-core + 1.8.2 + + + org.pcap4j + pcap4j-packetfactory-static + 1.8.2 + + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs.version} + + + + + ${project.groupId} + jicoco-test-kotlin + ${jicoco.version} + + + io.pkts + pkts-core + 3.0.10 + test + + + org.junit.platform + junit-platform-launcher + ${junit.platform.version} + test + + + io.kotest + kotest-runner-junit5-jvm + ${kotest.version} + test + + + io.kotest + kotest-assertions-core-jvm + ${kotest.version} + test + + + io.kotest + kotest-property-jvm + ${kotest.version} + test + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + io.mockk + mockk-jvm + ${mockk.version} + test + + + javax.xml.bind + jaxb-api + 2.3.1 + test + + + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + compile + compile + + + ${project.basedir}/src/main/kotlin + ${project.basedir}/src/main/java + + + + + test-compile + test-compile + + + ${project.basedir}/src/test/kotlin + ${project.basedir}/src/test/java + + + + + + 17 + + + + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper-maven-plugin.version} + + + generate-sources + + add-source + + + + ${project.basedir}/src/main/kotlin + ${project.basedir}/src/main/java + + + + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + verify + + jar + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + + default-compile + none + + + + default-testCompile + none + + + java-compile + compile + compile + + + java-test-compile + test-compile + testCompile + + + + 17 + 17 + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + true + + + + DtlsStackTest.* + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.6.3 + + + com.pinterest.ktlint + ktlint-cli + ${ktlint.version} + all + + + + + ktlint-check + verify + + exec + + + java + true + + -classpath + + com.pinterest.ktlint:ktlint-cli + + com.pinterest.ktlint.Main + --relative + src/main/kotlin/**/*.kt + src/test/kotlin/**/*.kt + + + + + ktlint-format + + exec + + + java + true + + -classpath + + com.pinterest.ktlint:ktlint-cli + + com.pinterest.ktlint.Main + --format + --relative + src/main/kotlin/**/*.kt + src/test/kotlin/**/*.kt + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.6.0 + + checkstyle.xml + + + + + check + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${project.basedir}/spotbugs-exclude.xml + true + + + + verify + check + + + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + + + + + + jitsi-maven-repository-releases + default + Jitsi Maven Repository (Releases) + + true + + + false + + https://github.com/jitsi/jitsi-maven-repository/raw/master/releases/ + + + jitsi-maven-repository-snapshots + default + Jitsi Maven Repository (Snapshots) + + false + + + true + + https://github.com/jitsi/jitsi-maven-repository/raw/master/snapshots/ + + + + + + coverage + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + prepare-agent + + + + + report + test + + report + + + + + + + + + diff --git a/jitsi-media-transform/spotbugs-exclude.xml b/jitsi-media-transform/spotbugs-exclude.xml new file mode 100644 index 0000000..a56e93d --- /dev/null +++ b/jitsi-media-transform/spotbugs-exclude.xml @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/DePacketizer.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/DePacketizer.java new file mode 100644 index 0000000..e3db5a7 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/DePacketizer.java @@ -0,0 +1,114 @@ +/* + * Copyright @ 2015 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.codec.video.h264; + +import org.jitsi.utils.logging2.*; + +import static org.jitsi_modified.impl.neomedia.codec.video.h264.H264.*; + +/** + * Implements Codec to represent a depacketizer of H.264 RTP packets + * into NAL units. + * + * @author Lyubomir Marinov + * @author Damian Minkov + */ +public class DePacketizer +{ + /** + * The Logger used by the DePacketizer class and its + * instances for logging output. + */ + private static final Logger logger = new LoggerImpl(DePacketizer.class.toString()); + + /** + * Returns true if the buffer contains a H264 key frame at offset + * offset. + * + * @param buf the byte buffer to check + * @param off the offset in the byte buffer where the actuall data starts + * @param len the length of the data in the byte buffer + * @return true if the buffer contains a H264 key frame at offset + * offset. + */ + public static boolean isKeyFrame(byte[] buf, int off, int len) + { + if (buf == null || buf.length < off + Math.max(len, 1)) + { + return false; + } + + int nalType = buf[off] & kTypeMask; + // Single NAL Unit Packet + if (nalType == kFuA) + { + // Fragmented NAL units (FU-A). + if (parseFuaNaluForKeyFrame(buf, off, len)) + { + return true; + } + } + else + { + if (parseSingleNaluForKeyFrame(buf, off, len)) + { + return true; + } + } + + return false; + } + + /** + * Checks if a fragment of a NAL unit from a specific FU-A RTP packet + * payload is keyframe or not. + */ + private static boolean parseFuaNaluForKeyFrame(byte[] buf, int off, int len) { + if (len < kFuAHeaderSize) + { + return false; + } + return ((buf[off + 1] & kTypeMask) == kIdr); + } + + /** + * Checks if a fragment of a NAL unit from a specific FU-A RTP packet + * payload is keyframe or not. + */ + private static boolean parseSingleNaluForKeyFrame(byte[] buf, int off, int len) + { + int naluStart = off + kNalHeaderSize; + int naluLength = len - kNalHeaderSize; + int nalType = buf[off] & kTypeMask; + if (nalType == kStapA) + { + // Skip the StapA header (StapA nal type + length). + if (len <= kStapAHeaderSize) + { + logger.error("StapA header truncated."); + return false; + } + if (!verifyStapANaluLengths(buf, naluStart, naluLength)) + { + logger.error("StapA packet with incorrect NALU packet lengths."); + return false; + } + nalType = buf[off + kStapAHeaderSize] & kTypeMask; + } + return (nalType == kIdr || nalType == kSps || + nalType == kPps || nalType == kSei); + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264.java new file mode 100644 index 0000000..51d1084 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264.java @@ -0,0 +1,83 @@ +/* + * Copyright @ 2015 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.codec.video.h264; + +/** + * Common utilities and constants for H264. + */ +public class H264 +{ + /** + * The bytes to prefix any NAL unit to be output by this + * DePacketizer and given to a H.264 decoder. Includes + * start_code_prefix_one_3bytes. According to "B.1 Byte stream NAL unit + * syntax and semantics" of "ITU-T Rec. H.264 Advanced video coding for + * generic audiovisual services", zero_byte "shall be present" when "the + * nal_unit_type within the nal_unit() is equal to 7 (sequence parameter + * set) or 8 (picture parameter set)" or "the byte stream NAL unit syntax + * structure contains the first NAL unit of an access unit in decoding + * order". + */ + static final byte[] NAL_PREFIX = {0, 0, 0, 1}; + + /** + * Constants used to detect H264 keyframes in rtp packet + */ + static final byte kTypeMask = 0x1F; // Nalu + static final byte kIdr = 5; + static final byte kSei = 6; + static final byte kSps = 7; + static final byte kPps = 8; + static final byte kStapA = 24; + static final byte kFuA = 28; // Header sizes + static final int kNalHeaderSize = 1; + static final int kFuAHeaderSize = 2; + static final int kLengthFieldSize = 2; + static final int kStapAHeaderSize = kNalHeaderSize + kLengthFieldSize; + static final int kNalUSize = 2; + + /** + * Check if Single-Time Aggregation Packet (STAP-A) NAL unit is correctly formed. + * + * @param data STAP-A payload + * @param offset Starting position of NAL unit + * @param lengthRemaining Bytes left in STAP-A + * @return True if STAP-A NAL Unit is correct + */ + static boolean verifyStapANaluLengths(byte[] data, int offset, int lengthRemaining) + { + if (data.length < offset + lengthRemaining) + { + return false; + } + while (lengthRemaining != 0) + { + if (lengthRemaining < kNalUSize) + { + return false; + } + int naluSize = kNalUSize + getUint16(data, offset); + offset += naluSize; + lengthRemaining -= naluSize; + } + return true; + } + + static int getUint16(byte[] data, int offset) + { + return ((data[offset] & 0xff) << 8) | (data[offset + 1] & 0xff); + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp8/DePacketizer.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp8/DePacketizer.java new file mode 100644 index 0000000..47ab085 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp8/DePacketizer.java @@ -0,0 +1,595 @@ +/* + * Copyright @ 2015 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.codec.video.vp8; + +import org.jitsi.utils.*; +import org.jitsi.rtp.extensions.ByteKt; + +/** + * A depacketizer from VP8. + * See {@link "http://tools.ietf.org/html/rfc7741"} + * + * @author Boris Grozev + * @author George Politis + * @author Jonathan Lennox + */ +public class DePacketizer +{ + /** + * Returns true if the buffer contains a VP8 key frame at offset + * offset. + * + * @param buf the byte buffer to check + * @param off the offset in the byte buffer where the actual data starts + * @param len the length of the data in the byte buffer + * @return true if the buffer contains a VP8 key frame at offset + * offset. + */ + public static boolean isKeyFrame(byte[] buf, int off, int len) + { + // Check if this is the start of a VP8 partition in the payload + // descriptor. + if (!DePacketizer.VP8PayloadDescriptor.isValid(buf, off, len)) + { + return false; + } + + if (!DePacketizer.VP8PayloadDescriptor.isStartOfFrame(buf, off)) + { + return false; + } + + int szVP8PayloadDescriptor = DePacketizer + .VP8PayloadDescriptor.getSize(buf, off, len); + + return DePacketizer.VP8PayloadHeader.isKeyFrame( + buf, off + szVP8PayloadDescriptor); + } + + /** + * A class that represents the VP8 Payload Descriptor structure defined + * in {@link "https://tools.ietf.org/html/rfc7741"} + */ + public static class VP8PayloadDescriptor + { + /** + * I bit from the X byte of the Payload Descriptor. + */ + private static final byte I_BIT = (byte) 0x80; + + /** + * K bit from the X byte of the Payload Descriptor. + */ + private static final byte K_BIT = (byte) 0x10; + + /** + * L bit from the X byte of the Payload Descriptor. + */ + private static final byte L_BIT = (byte) 0x40; + + /** + * M bit from the I byte of the Payload Descriptor. + */ + private static final byte M_BIT = (byte) 0x80; + + /** + * Maximum length of a VP8 Payload Descriptor. + */ + public static final int MAX_LENGTH = 6; + + /** + * S bit from the first byte of the Payload Descriptor. + */ + private static final byte S_BIT = (byte) 0x10; + + /** + * T bit from the X byte of the Payload Descriptor. + */ + private static final byte T_BIT = (byte) 0x20; + + /** + * X bit from the first byte of the Payload Descriptor. + */ + private static final byte X_BIT = (byte) 0x80; + + /** + * N bit from the first byte of the Payload Descriptor. + */ + private static final byte N_BIT = (byte) 0x20; + + /** + * Gets whether a VP8 payload has a temporal layer index (TID). + * + * @param buf the byte buffer that holds the VP8 packet. + * @param off the offset in the byte buffer where the VP8 payload starts. + * @param len the length of the VP8 payload. + * @return true if the VP8 payload has a temporal layer index, false if not. + */ + public static boolean hasTemporalLayerIndex(byte[] buf, int off, int len) + { + return getTemporalLayerIndex(buf, off, len) != -1; + } + + /** + * Gets the temporal layer index (TID), if that's set. + * + * @param buf the byte buffer that holds the VP8 packet. + * @param off the offset in the byte buffer where the VP8 packet starts. + * @param len the length of the VP8 packet. + * @return the temporal layer index (TID), if that's set, -1 otherwise. + */ + public static int getTemporalLayerIndex(byte[] buf, int off, int len) + { + if (buf == null || buf.length < off + len || len < 2) + { + return -1; + } + + if ((buf[off] & X_BIT) == 0 || (buf[off + 1] & T_BIT) == 0) + { + return -1; + } + + int sz = getSize(buf, off, len); + if (buf.length < off + sz || sz < 1) + { + return -1; + } + + return (buf[off + sz - 1] & 0xc0) >> 6; + } + + /** + * Sets the temporal layer index (TID), if there's room for it. + * + * @param buf the byte buffer that holds the VP8 packet. + * @param off the offset in the byte buffer where the VP8 packet starts. + * @param len the length of the VP8 packet. + * @param tid the temporal layer value to + * @return true if the layer was set successfully, false otherwise. + */ + public static boolean setTemporalLayerIndex(byte[] buf, int off, int len, int tid) + { + if (tid < 0 || tid > 3) + { + throw new IllegalArgumentException("Bad tid value " + tid); + } + + if (buf == null || buf.length < off + len || len < 2) + { + return false; + } + + if ((buf[off] & X_BIT) == 0 || (buf[off + 1] & T_BIT) == 0) + { + return false; + } + + int sz = getSize(buf, off, len); + if (buf.length < off + sz || sz < 1) + { + return false; + } + + buf[off + sz - 1] = (byte)((tid << 6) | buf[off + sz - 1] & 0x3f); + + return true; + } + + + + /** + * Returns a simple Payload Descriptor, with PartID = 0, the 'start + * of partition' bit set according to startOfPartition, and + * all other bits set to 0. + * + * @param startOfPartition whether to 'start of partition' bit should be + * set + * @return a simple Payload Descriptor, with PartID = 0, the 'start + * of partition' bit set according to startOfPartition, and + * all other bits set to 0. + */ + public static byte[] create(boolean startOfPartition) + { + byte[] pd = new byte[1]; + pd[0] = startOfPartition ? (byte) 0x10 : 0; + return pd; + } + + /** + * The size in bytes of the Payload Descriptor at offset + * offset in input. The size is between 1 and 6. + * + * @param baf the ByteArrayBuffer that holds the VP8 payload + * descriptor. + * @return The size in bytes of the Payload Descriptor at offset + * offset in input, or -1 if the input is not a valid + * VP8 Payload Descriptor. The size is between 1 and 6. + */ + public static int getSize(ByteArrayBuffer baf) + { + if (baf == null) + { + return -1; + } + + return getSize(baf.getBuffer(), baf.getOffset(), baf.getLength()); + } + + /** + * The size in bytes of the Payload Descriptor at offset + * offset in input. The size is between 1 and 6. + * + * @param input input + * @param offset offset + * @param length length + * @return The size in bytes of the Payload Descriptor at offset + * offset in input, or -1 if the input is not a valid + * VP8 Payload Descriptor. The size is between 1 and 6. + */ + public static int getSize(byte[] input, int offset, int length) + { + if (!isValid(input, offset, length)) + return -1; + + if ((input[offset] & X_BIT) == 0) + return 1; + + int size = 2; + if ((input[offset + 1] & I_BIT) != 0) + { + size++; + if ((input[offset + 2] & M_BIT) != 0) + size++; + } + if ((input[offset + 1] & L_BIT) != 0) + size++; + if ((input[offset + 1] & (T_BIT | K_BIT)) != 0) + size++; + + return size; + } + + /** + * Determines whether the VP8 payload specified in the buffer that is + * passed as an argument has a picture ID or not. + * + * @param buf the byte buffer that contains the VP8 payload. + * @param off the offset in the byte buffer where the VP8 payload + * starts. + * @param len the length of the VP8 payload in the byte buffer. + * @return true if the VP8 payload contains a picture ID, false + * otherwise. + */ + public static boolean hasPictureId(byte[] buf, int off, int len) + { + return isValid(buf, off, len) + && (buf[off] & X_BIT) != 0 && (buf[off + 1] & I_BIT) != 0; + } + + /** + * Determines whether the VP8 payload specified in the buffer that is + * passed as an argument has an extended picture ID or not. + * + * @param buf the byte buffer that contains the VP8 payload. + * @param off the offset in the byte buffer where the VP8 payload + * starts. + * @param len the length of the VP8 payload in the byte buffer. + * @return true if the VP8 payload contains an extended picture ID, + * false otherwise. + */ + public static boolean hasExtendedPictureId(byte[] buf, int off, int len) + { + return hasPictureId(buf, off, len) && (buf[off + 2] & M_BIT) != 0; + } + + /** + * Determines whether the VP8 payload specified in the buffer that is + * passed as an argument has a TL0PICIDX or not. + * + * @param buf the byte buffer that contains the VP8 payload. + * @param off the offset in the byte buffer where the VP8 payload + * starts. + * @param len the length of the VP8 payload in the byte buffer. + * @return true if the VP8 payload contains a TL0PICIDX, + * false otherwise. + */ + public static boolean hasTL0PICIDX(byte[] buf, int off, int len) + { + return getLByteOffset(buf, off, len) >= 0; + } + + /** + * Gets the value of the PictureID field of a VP8 Payload Descriptor. + * + * @param input + * @param offset + * @return the value of the PictureID field of a VP8 Payload Descriptor, + * or -1 if the fields is not present. + */ + public static int getPictureId(byte[] input, int offset) + { + if (input == null + || !hasPictureId(input, offset, input.length - offset)) + { + return -1; + } + + boolean isLong = (input[offset + 2] & M_BIT) != 0; + if (isLong) + return (input[offset + 2] & 0x7f) << 8 + | (input[offset + 3] & 0xff); + else + return input[offset + 2] & 0x7f; + + } + + /** + * Sets the extended picture ID for the VP8 payload specified in the + * buffer that is passed as an argument. + * + * @param buf the byte buffer that contains the VP8 payload. + * @param off the offset in the byte buffer where the VP8 payload + * starts. + * @param len the length of the VP8 payload in the byte buffer. + * @return true if the operation succeeded, false otherwise. + */ + public static boolean setExtendedPictureId( + byte[] buf, int off, int len, int val) + { + if (!hasExtendedPictureId(buf, off, len)) + { + return false; + } + + buf[off + 2] = (byte) (0x80 | (val >> 8) & 0x7F); + buf[off + 3] = (byte) (val & 0xFF); + + return true; + } + + /** + * Sets the TL0PICIDX field for the VP8 payload specified in the + * buffer that is passed as an argument. + * + * @param buf the byte buffer that contains the VP8 payload. + * @param off the offset in the byte buffer where the VP8 payload + * starts. + * @param len the length of the VP8 payload in the byte buffer. + * @return true if the operation succeeded, false otherwise. + */ + public static boolean setTL0PICIDX(byte[] buf, int off, int len, + int val) + { + int offL = getLByteOffset(buf, off, len); + if (offL < 0) + { + return false; + } + + buf[off + offL] = (byte) val; + return true; + } + + /** + * Checks whether the arguments specify a valid buffer. + * + * @param buf + * @param off + * @param len + * @return true if the arguments specify a valid buffer, false + * otherwise. + */ + public static boolean isValid(byte[] buf, int off, int len) + { + return buf != null && buf.length >= off + len && off > -1 + && len > 0; + } + + /** + * Checks whether the 'start of partition' bit is set in the + * VP8 Payload Descriptor at offset offset in input. + * + * @param input input + * @param offset offset + * @return true if the 'start of partition' bit is set, + * false otherwise. + */ + public static boolean isStartOfPartition(byte[] input, int offset) + { + return (input[offset] & S_BIT) != 0; + } + + /** + * Sets the 'start of partition' bit in the + * VP8 Payload Descriptor at offset offset in input. + * + * @param input input + * @param offset offset + * @param start whether it is start or not. + */ + public static void setStartOfPartition(byte[] input, int offset, boolean start) + { + input[offset] = ByteKt.putBitWithMask(input[offset], S_BIT, start); + } + + + /** + * Returns true if both the 'start of partition' bit + * is set and the PID fields has value 0 in the VP8 Payload + * Descriptor at offset offset in input. + * + * @param input + * @param offset + * @return true if both the 'start of partition' bit + * is set and the PID fields has value 0 in the VP8 Payload + * Descriptor at offset offset in input. + */ + public static boolean isStartOfFrame(byte[] input, int offset) + { + return isStartOfPartition(input, offset) + && getPartitionId(input, offset) == 0; + } + + /** + * Returns the value of the PID (partition ID) field of the + * VP8 Payload Descriptor at offset offset in input. + * + * @param input + * @param offset + * @return the value of the PID (partition ID) field of the + * VP8 Payload Descriptor at offset offset in input. + */ + public static int getPartitionId(byte[] input, int offset) + { + return input[offset] & 0x07; + } + + /** + * Gets a boolean that indicates whether or not the non-reference bit is + * set. + * + * @param buf the byte buffer that holds the VP8 payload descriptor. + * @param off the offset in the byte buffer where the payload descriptor + * starts. + * @param len the length of the payload descriptor in the byte buffer. + * @return true if the non-reference bit is NOT set, false otherwise. + */ + public static boolean isReference(byte[] buf, int off, int len) + { + return (buf[off] & N_BIT) == 0; + } + + /** + * Gets the TL0PICIDX from the payload descriptor. + * + * @param buf the byte buffer that holds the VP8 payload descriptor. + * @param off the offset in the byte buffer where the payload descriptor + * starts. + * @param len the length of the payload descriptor in the byte buffer. + * @return the TL0PICIDX from the payload descriptor, or -1 if the packet + * does not have one. + */ + public static int getTL0PICIDX(byte[] buf, int off, int len) + { + int offL = getLByteOffset(buf, off, len); + if (offL < 0) + { + return -1; + } + + return buf[off + offL] & 0xff; + } + + /** + * Return the offset of the {@code L} byte relative to the start of + * the payload descriptor, or -1 if the PR has no {@code L} byte. + * + * @param buf the byte buffer that holds the VP8 payload descriptor. + * @param off the offset in the byte buffer where the payload descriptor + * starts. + * @param len the length of the payload descriptor in the byte buffer. + * @return + */ + private static int getLByteOffset(byte[] buf, int off, int len) + { + if (!isValid(buf, off, len) + || (buf[off] & X_BIT) == 0 || (buf[off + 1] & L_BIT) == 0) + { + return -1; + } + + // We could use hasPictureId(), but it would unnecessarily repeat + // the isValid() check. + int offL = 2; + if ((buf[off + 1] & I_BIT) != 0) + { + offL++; + if ((buf[off + 2] & M_BIT) != 0) + { + offL++; + } + } + + return offL; + } + } + + + /** + * A class that represents the VP8 Payload Header structure described + * in {@link "http://tools.ietf.org/html/rfc7741"} + */ + public static class VP8PayloadHeader + { + /** + * S bit of the Payload Header. + */ + private static final byte S_BIT = (byte) 0x01; + + /** + * Returns true if the P (inverse key frame flag) field of the + * VP8 Payload Header at offset offset in input is 0. + * + * @return true if the P (inverse key frame flag) field of the + * VP8 Payload Header at offset offset in input is 0, + * false otherwise. + */ + public static boolean isKeyFrame(byte[] input, int offset) + { + // When set to 0 the current frame is a key frame. When set to 1 + // the current frame is an interframe. Defined in [RFC6386] + + return (input[offset] & S_BIT) == 0; + } + + /* Sets the P (inverse key frame flag) field of the + * VP8 Payload header at offset offset in input. + * Sets P to 0 if keyFrame is false, or to 1 if keyFrame is true. + */ + public static void setKeyFrame(byte[] input, int offset, boolean keyFrame) + { + input[offset] = ByteKt.putBitWithMask(input[offset], S_BIT, !keyFrame); + } + } + + + /** + * A class that represents a keyframe header structure (see RFC 6386, + * paragraph 9.1). + * + * @author George Politis + */ + public static class VP8KeyframeHeader + { + // From RFC 6386, the keyframe header has this format. + // + // Start code byte 0 0x9d + // Start code byte 1 0x01 + // Start code byte 2 0x2a + // + // 16 bits : (2 bits Horizontal Scale << 14) | Width (14 bits) + // 16 bits : (2 bits Vertical Scale << 14) | Height (14 bits) + + /** + * @return the height of this instance. + */ + public static int getHeight(byte[] buf, int off) + { + return (((buf[off + 6] & 0xff) << 8) | buf[off + 5] & 0xff) & 0x3fff; + } + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp9/DePacketizer.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp9/DePacketizer.java new file mode 100644 index 0000000..d1e1816 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/codec/video/vp9/DePacketizer.java @@ -0,0 +1,810 @@ +/* + * Copyright @ 2015 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.codec.video.vp9; + +import org.jitsi.rtp.extensions.*; + +/** + * A depacketizer from VP9. + * See {@link "https://tools.ietf.org/html/draft-ietf-payload-vp9-10"} + * + * @author George Politis + */ +public class DePacketizer +{ + /* + * VP9 Payload header, flexible mode (F=1): + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |I|P|L|F|B|E|V|Z| (REQUIRED) + * +-+-+-+-+-+-+-+-+ + * I: |M| PICTURE ID | (REQUIRED) + * +-+-+-+-+-+-+-+-+ + * M: | EXTENDED PID | (RECOMMENDED) + * +-+-+-+-+-+-+-+-+ + * L: | TID |U| SID |D| (CONDITIONALLY RECOMMENDED) + * +-+-+-+-+-+-+-+-+ -\ + * P,F: | P_DIFF |N| (CONDITIONALLY REQUIRED) - up to 3 times + * +-+-+-+-+-+-+-+-+ -/ + * V: | SS | + * | .. | + * +-+-+-+-+-+-+-+-+ + * + * Non-flexible mode (F=0): + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |I|P|L|F|B|E|V|Z| (REQUIRED) + * +-+-+-+-+-+-+-+-+ + * I: |M| PICTURE ID | (RECOMMENDED) + * +-+-+-+-+-+-+-+-+ + * M: | EXTENDED PID | (RECOMMENDED) + * +-+-+-+-+-+-+-+-+ + * L: | TID |U| SID |D| (CONDITIONALLY RECOMMENDED) + * +-+-+-+-+-+-+-+-+ + * | TL0PICIDX | (CONDITIONALLY REQUIRED) + * +-+-+-+-+-+-+-+-+ + * V: | SS | + * | .. | + * +-+-+-+-+-+-+-+-+ + */ + + /** + * I bit from the first byte of the Payload Descriptor: + * Picture ID present. + */ + private static final byte I_BIT = (byte) (1 << 7); + + /** + * P bit from the first byte of the Payload Descriptor: + * Inter-picture predicted frame. + */ + private static final byte P_BIT = (byte) (1 << 6); + + /** + * L bit from the first byte of the Payload Descriptor: + * Layer indices present. + */ + private static final byte L_BIT = (byte) (1 << 5); + + /** + * F bit from the first byte of the Payload Descriptor: + * flexible mode. + */ + private static final byte F_BIT = (byte) (1 << 4); + + /** + * B bit from the first byte of the Payload Descriptor: + * Start of frame. + */ + private static final byte B_BIT = (byte) (1 << 3); + + /** + * E bit from the first byte of the Payload Descriptor: + * End of frame. + */ + private static final byte E_BIT = (byte) (1 << 2); + + /** + * V bit from the first byte of the Payload Descriptor: + * Scalability structure present. + */ + private static final byte V_BIT = (byte) (1 << 1); + + /** + * Z bit from first byte of the Payload Descriptor: + * Not an upper-level reference frame + */ + private static final byte Z_BIT = (byte)1; + + /** + * M bit from the first picture ID byte of the Payload Descriptor: + * extended picture ID. Only present if I is true in the first byte. + */ + private static final byte M_BIT = (byte) (1 << 7); + + /** + * U bit from the layer indices byte of the Payload Descriptor: + * switching up point. Only present if L is true in the first byte. + */ + private static final byte U_BIT = (byte) (1 << 4); + + /** + * D bit from the layer indices byte of the Payload Descriptor: + * inter-layer dependency. Only present if L is true in the first byte. + */ + private static final byte D_BIT = (byte) (1); + + /** + * N bit from the picture difference byte of the Payload Descriptor: + * more picture diffs present. Only present if P and F are true in the first byte. + */ + private static final byte N_BIT = (byte) (1); + + /** + * A class that represents the VP9 Payload Descriptor structure defined + * in {@link "https://tools.ietf.org/html/draft-ietf-payload-vp9-10"} + */ + public static class VP9PayloadDescriptor + { + /** + * Returns true if the arguments specify a valid non-empty + * VP9 packet. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the arguments specify a valid non-empty + * buffer. + */ + private static boolean isValid(byte[] buf, int off, int len) + { + return + (buf != null && buf.length >= off + len && off > -1 && len > 0); + } + + /** + * Returns true if the B bit from the first byte of the payload + * descriptor has value 0. (Note this is start of frame, not start of picture!) + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the B bit from the first byte of the + * payload descriptor has value 0, false otherwise. + * + */ + public static boolean isStartOfFrame(byte[] buf, int off, int len) + { + // Check if this is the start of a VP9 layer frame in the payload + // descriptor. + + return isValid(buf, off, len) && (buf[off] & B_BIT) != 0; + } + + /** + * Sets the B bit (start of frame) in the first byte of the payload + * descriptor. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * @param start the new value of the start of frame bit. + */ + public static void setStartOfFrame(byte[] buf, int off, int len, boolean start) + { + // Check if this is the start of a VP9 layer frame in the payload + // descriptor. + + if (!isValid(buf, off, len)) + { + throw new IllegalStateException("Can't set startOfFrame for invalid VP9 packet"); + } + + buf[off] = ByteKt.putBitWithMask(buf[off], B_BIT, start); + } + + /** + * Returns true if the E bit from the first byte of the payload + * descriptor has value 0. (Note this is end of frame, not end of picture!) + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the E bit from the first byte of the + * payload descriptor has value 0, false otherwise. + */ + public static boolean isEndOfFrame(byte[] buf, int off, int len) + { + // Check if this is the end of a VP9 layer frame in the payload + // descriptor. + + return isValid(buf, off, len) && (buf[off] & E_BIT) != 0; + } + + /** + * Sets the E bit (end of frame) in the first byte of the payload + * descriptor. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * @param end the new value of the end of frame bit. + */ + public static void setEndOfFrame(byte[] buf, int off, int len, boolean end) + { + if (!isValid(buf, off, len)) + { + throw new IllegalStateException("Can't set endOfFrame for invalid VP9 packet"); + } + + buf[off] = ByteKt.putBitWithMask(buf[off], E_BIT, end); + } + + /** + * Returns true if the packet is encoded in flexible mode + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet is encoded in flexible mode, + * false otherwise + */ + public static boolean isFlexibleMode(byte[] buf, int off, int len) + { + return isValid(buf, off, len) && (buf[off] & F_BIT) != 0; + } + + /** + * Returns true if the packet contains a scalability structure + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet contains a scalability structure, + * false otherwise + */ + public static boolean hasScalabilityStructure(byte[] buf, int off, int len) + { + return isValid(buf, off, len) && (buf[off] & V_BIT) != 0; + } + + /** + * Returns true if the packet might be an upper-level reference. + * + * (Note this is inverted from the sense of the Z bit in the payload header.) + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet contains a scalability structure, + * false otherwise + */ + public static boolean isUpperLevelReference(byte[] buf, int off, int len) + { + return isValid(buf, off, len) && (buf[off] & Z_BIT) == 0; + } + + /** + * Sets the Z bit (upper layer reference) in the first byte of the payload + * descriptor. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * @param ref whether the frame might be an upper-layer reference. + * (Note this is inverted from the sense of the Z bit in the payload header.) + */ + public static void setUpperLevelReference(byte[] buf, int off, int len, boolean ref) + { + if (!isValid(buf, off, len)) + { + throw new IllegalStateException("Can't set upperLevelReference for invalid VP9 packet"); + } + + buf[off] = ByteKt.putBitWithMask(buf[off], Z_BIT, !ref); + } + + /** + * Returns true if the packet has a picture ID. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet has a picture ID, false + * if not. + */ + public static boolean hasPictureId(byte[] buf, int off, int len) + { + return isValid(buf, off, len) && (buf[off] & I_BIT) != 0 && len > 1; + } + + /** + * Returns true if the packet has an extended (15-bit) picture ID. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet has a picture ID, false + * if not. + */ + public static boolean hasExtendedPictureId(byte[] buf, int off, int len) + { + return hasPictureId(buf, off, len) && + (buf[off+1] & M_BIT) != 0 && len > 2; + } + + /** + * Gets the value of the PictureID field of a VP9 Payload Descriptor. + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet has a picture ID, false + * if not. + */ + public static int getPictureId(byte[] buf, int off, int len) + { + if (!hasPictureId(buf, off, len)) + { + return -1; + } + + if (hasExtendedPictureId(buf, off, len)) + { + return (buf[off + 1] & 0x7f) << 8 + | (buf[off + 2] & 0xff); + } + else + { + return buf[off + 1] & 0x7f; + } + } + + /** + * Sets the extended picture ID for the VP9 payload specified in the + * buffer that is passed as an argument. + * + * @param buf the byte buffer that contains the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload + * starts. + * @param len the length of the VP9 payload in the byte buffer. + * @return true if the operation succeeded, false otherwise. + */ + public static boolean setExtendedPictureId( + byte[] buf, int off, int len, int val) + { + if (!hasExtendedPictureId(buf, off, len)) + { + return false; + } + + buf[off + 1] = (byte) (0x80 | (val >> 8) & 0x7F); + buf[off + 2] = (byte) (val & 0xFF); + + return true; + } + + /** + * Returns true if the packet contains an inter-picture + * predicted frame. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet contains an inter-picture + * predicted frame, false if not. + */ + public static boolean isInterPicturePredicted(byte[] buf, int off, int len) + { + return isValid(buf, off, len) && (buf[off] & P_BIT) != 0; + } + + /** + * Sets the P bit (inter-picture predicted) in the first byte of the payload + * descriptor. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * @param pred the new value of the inter-picture predicted bit. + */ + public static void setInterPicturePredicted(byte[] buf, int off, int len, boolean pred) + { + if (!isValid(buf, off, len)) + { + throw new IllegalStateException("Can't set interPicturePredicted for invalid VP9 packet"); + } + + buf[off] = ByteKt.putBitWithMask(buf[off], P_BIT, pred); + } + + + /** + * Returns true if the packet is part of a keyframe. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet is part of a keyframe, false if not. + */ + public static boolean isKeyFrame(byte[] buf, int off, int len) + { + if (!isValid(buf, off, len)) + { + return false; + } + + if ((buf[off] & P_BIT) != 0) + { + return false; + } + + if (!hasLayerIndices(buf, off, len)) + { + /* P bit without layer indices is a key frame. */ + return true; + } + + return (getSpatialLayerIndex(buf, off, len) == 0 || + !usesInterLayerDependency(buf, off, len)); + } + + /** + * Query whether the packet has layer indices values. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return the temporal layer index (TID), if that's set, -1 otherwise. + */ + public static boolean hasLayerIndices(byte[] buf, int off, int len) + { + return (isValid(buf, off, len) && (buf[off] & L_BIT) != 0); + } + + private static int getLayerIndexOffset(byte[] buf, int off, int len) + { + if (!hasLayerIndices(buf, off, len)) + { + return -1; + } + + int loff = off + 1; + if ((buf[off] & I_BIT) != 0) + { + loff += 1; + if ((buf[off + 1] & M_BIT) != 0) + { + // extended pid. + loff += 1; + } + } + + if (loff >= off+len) + { + return -1; + } + + return loff; + } + + /** + * Determines whether the VP9 payload specified in the buffer that is + * passed as an argument has a TL0PICIDX or not. + * + * @param buf the byte buffer that contains the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload + * starts. + * @param len the length of the VP9 payload in the byte buffer. + * @return true if the VP9 payload contains a TL0PICIDX, + * false otherwise. + */ + public static boolean hasTL0PICIDX(byte[] buf, int off, int len) + { + int loff = getLayerIndexOffset(buf, off, len); + if (loff < 0 || loff + 1 >= off+len) + { + return false; + } + return !isFlexibleMode(buf, off, len); + } + + /** + * Gets the TL0PICIDX from the payload descriptor. + * + * @param buf the byte buffer that holds the VP9 payload descriptor. + * @param off the offset in the byte buffer where the payload descriptor + * starts. + * @param len the length of the payload descriptor in the byte buffer. + * @return the TL0PICIDX from the payload descriptor, or -1 if the packet + * does not have one. + */ + public static int getTL0PICIDX(byte[] buf, int off, int len) + { + int loff = getLayerIndexOffset(buf, off, len); + if (!hasTL0PICIDX(buf, off, len)) + return -1; + return buf[loff+1] & 0xff; + } + + /** + * Sets the TL0PICIDX field for the VP9 payload specified in the + * buffer that is passed as an argument. + * + * @param buf the byte buffer that contains the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload + * starts. + * @param len the length of the VP9 payload in the byte buffer. + * @return true if the operation succeeded, false otherwise. + */ + public static boolean setTL0PICIDX(byte[] buf, int off, int len, + int val) + { + if (!hasTL0PICIDX(buf, off, len)) + { + return false; + } + + int loff = getLayerIndexOffset(buf, off, len); + buf[loff+1] = (byte) val; + return true; + } + + /** + * Gets the temporal layer index (TID), if that's set. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return the temporal layer index (TID), if that's set, -1 otherwise. + */ + public static int getTemporalLayerIndex(byte[] buf, int off, int len) + { + int loff = getLayerIndexOffset(buf, off, len); + if (loff < 0) + { + return -1; + } + + return (buf[loff] & 0xE0) >> 5; + } + + /** + * Gets the spatial layer index (SID), if that's set. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return the spatial layer index (SID), if that's set, -1 otherwise. + */ + public static int getSpatialLayerIndex(byte[] buf, int off, int len) + { + int loff = getLayerIndexOffset(buf, off, len); + if (loff < 0) + { + return -1; + } + + return (buf[loff] & 0xE) >> 1; + } + + /** + * Gets whether this frame is a temporal switching-up point, where it is + * valid to switch the stream to a higher temporal layer. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet is a switching-up point, false + * if not. + */ + public static boolean isSwitchingUpPoint(byte[] buf, int off, int len) + { + int loff = getLayerIndexOffset(buf, off, len); + if (loff < 0) + { + return false; + } + + return (buf[loff] & U_BIT) != 0; + } + + /** + * Gets whether this frame uses inter-layer dependency, where the frame + * depends on lower spatial-layer frames of the same picture. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * + * @return true if the packet uses inter-layer dependency, false + * if not. + */ + public static boolean usesInterLayerDependency(byte[] buf, int off, int len) + { + int loff = getLayerIndexOffset(buf, off, len); + if (loff < 0) + { + return false; + } + + return (buf[loff] & D_BIT) != 0; + } + + /** + * Sets the packet's layer indices, if there's room for it in the packet. + * + * @param buf the byte buffer that holds the VP9 payload. + * @param off the offset in the byte buffer where the VP9 payload starts. + * @param len the length of the VP9 payload. + * @param sid the spatial layer of the packet + * @param tid the temporal layer of the packet + * @param isSwitchingUpPoint whether the packet is a switching-up point + * @param usesInterLayerDependency whether the packet uses inter-layer dependency + * + * @return Whether the layer indices were successfully set + */ + public static boolean setLayerIndices(byte[] buf, int off, int len, + int sid, int tid, boolean isSwitchingUpPoint, boolean usesInterLayerDependency) + { + int loff = getLayerIndexOffset(buf, off, len); + if (loff < 0) + { + return false; + } + + if (sid < 0 || sid > 8) + { + throw new IllegalArgumentException("Invalid spatial ID " + sid); + } + + if (tid < 0 || tid > 8) + { + throw new IllegalArgumentException("Invalid spatial ID " + sid); + } + + buf[loff] = (byte)((tid << 5) | + (isSwitchingUpPoint ? U_BIT : 0) | + (sid << 1) | + (usesInterLayerDependency ? D_BIT : 0)); + + return true; + } + + /** + * The size in bytes of the Payload Descriptor at off + * off in buf. The size is at least 1. + * + * @param buf buf + * @param off off + * @param len len + * @return The size in bytes of the Payload Descriptor at off + * off in buf, or -1 if the buffer is not a valid + * VP9 Payload Descriptor. The size is at least 1. + */ + public static int getSize(byte[] buf, int off, int len) + { + if (!isValid(buf, off, len)) + return -1; + + int pos = off + 1; + + if ((buf[off] & I_BIT) != 0) + { + boolean extended = (buf[pos] & M_BIT) != 0; + pos++; + if (extended) + { + pos++; + } + } + + if ((buf[off] & L_BIT) != 0) + { + pos++; + if ((buf[off] & F_BIT) == 0) + { + pos++; + } + } + + if ((buf[off] & F_BIT) != 0 && (buf[off] & P_BIT) != 0) + { + for (int i = 0; i < 3; i++) { + int hasNext = buf[pos] & N_BIT; + pos++; + + if (hasNext == 0) + { + break; + } + } + } + + if ((buf[off] & V_BIT) != 0) + { + /* SS present */ + int n_s = (buf[pos] & 0xE0) >> 5; + boolean resPresent = ((buf[pos] & (1 << 4)) != 0); + boolean pgPresent = ((buf[pos] & (1 << 3)) != 0); + pos++; + + if (resPresent) + { + pos += 4 * (n_s + 1); + } + + if (pgPresent) + { + int n_g = (buf[pos] & 0xff); + int i; + pos++; + + for (i = 0; i < n_g; i++) + { + int r = (buf[pos] & 0x0C) >> 2; + pos += r + 1; + } + } + } + + return pos - off; + } + + /** + * The offset of the scalability structure within the buffer, if + * the buffer contains a VP9 packet with a scalability structure; otherwise -1. + * + * @param buf buf + * @param off off + * @param len len + * @return The size in bytes of the Payload Descriptor at off + * off in buf, or -1 if the buffer is not a valid + * VP9 Payload Descriptor. The size is at least 1. + */ + public static int getScalabilityStructureOffset(byte[] buf, int off, int len) + { + if (!hasScalabilityStructure(buf, off, len)) + { + return -1; + } + + int pos = off + 1; + + if ((buf[off] & I_BIT) != 0) + { + boolean extended = (buf[pos] & M_BIT) != 0; + pos++; + if (extended) + { + pos++; + } + } + + if ((buf[off] & L_BIT) != 0) + { + pos++; + if ((buf[off] & F_BIT) == 0) + { + pos++; + } + } + + if ((buf[off] & F_BIT) != 0 && (buf[off] & P_BIT) != 0) + { + do { + pos++; + } while ((buf[pos] & N_BIT) != 0); + } + + return pos; + } + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java new file mode 100644 index 0000000..3de8db7 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/AimdRateControl.java @@ -0,0 +1,532 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging.DiagnosticContext; +import org.jitsi.utils.logging.TimeSeriesLogger; + +/** + * A rate control implementation based on additive increases of bitrate when no + * over-use is detected and multiplicative decreases when over-uses are + * detected. When we think the available bandwidth has changes or is unknown, we + * will switch to a "slow-start mode" where we increase multiplicatively. + * + * webrtc/modules/remote_bitrate_estimator/aimd_rate_control.cc + * webrtc/modules/remote_bitrate_estimator/aimd_rate_control.h + * + * @author Lyubomir Marinov + */ +class AimdRateControl +{ + /** + * The TimeSeriesLogger used by the + * RemoteBitrateEstimatorAbsSendTime class and its instances for + * logging output. + */ + private static final TimeSeriesLogger logger + = TimeSeriesLogger.getTimeSeriesLogger(AimdRateControl.class); + + private static final int kDefaultRttMs = 200; + + private static final long kInitializationTimeMs = 5000; + + private static final long kFirstIncomingEstimateExpirationMs = 2 * kInitializationTimeMs; + + private static final long kLogIntervalMs = 1000; + + private static final long kMaxFeedbackIntervalMs = 1000; + + private static final long kMinFeedbackIntervalMs = 200; + + private static final int kRtcpSize = 80; + + private static final double kWithinIncomingBitrateHysteresis = 1.05; + + private final DiagnosticContext diagnosticContext; + + private float avgMaxBitrateKbps; + + private float beta; + + private boolean bitrateIsInitialized; + + private long currentBitrateBps; + + private final RateControlInput currentInput + = new RateControlInput(BandwidthUsage.kBwNormal, 0L, 0D); + + private boolean inExperiment; + + private long minConfiguredBitrateBps; + + private RateControlRegion rateControlRegion; + + private RateControlState rateControlState; + + private long rtt; + + private long timeFirstIncomingEstimate; + + private long timeLastBitrateChange; + + private long timeOfLastLog; + + private boolean updated; + + private float varMaxBitrateKbps; + + public AimdRateControl(@NotNull DiagnosticContext diagnosticContext) + { + reset(); + this.diagnosticContext = diagnosticContext; + } + + private long additiveRateIncrease( + long nowMs, + long lastMs, + long responseTimeMs) + { + if (responseTimeMs <= 0) + throw new IllegalArgumentException("responseTimeMs"); + + double beta = 0.0; + + if (lastMs > 0) + { + beta + = Math.min((nowMs - lastMs) / (double) responseTimeMs, 1.0); + if (inExperiment) + beta /= 2.0; + } + + double bitsPerFrame = (double) currentBitrateBps / 30.0; + double packetsPerFrame = Math.ceil(bitsPerFrame / (8.0 * 1200.0)); + double avgPacketSizeBits = bitsPerFrame / packetsPerFrame; + + return (long) Math.max(1000.0, beta * avgPacketSizeBits); + } + + private long changeBitrate( + long currentBitrateBps, + long incomingBitrateBps, + long nowMs) + { + if (!updated) + return this.currentBitrateBps; + // An over-use should always trigger us to reduce the bitrate, even though + // we have not yet established our first estimate. By acting on the over-use, + // we will end up with a valid estimate. + if (!bitrateIsInitialized + && currentInput.bwState != BandwidthUsage.kBwOverusing) + { + return this.currentBitrateBps; + } + + updated = false; + changeState(currentInput, nowMs); + + // Calculated here because it's used in multiple places. + float incomingBitrateKbps = incomingBitrateBps / 1000.0F; + // Calculate the max bit rate std dev given the normalized variance and + // the current incoming bit rate. + float stdMaxBitRate + = (float) Math.sqrt(varMaxBitrateKbps * avgMaxBitrateKbps); + + switch (rateControlState) + { + case kRcHold: + break; + + case kRcIncrease: + { + if (avgMaxBitrateKbps >= 0F + && incomingBitrateKbps + > avgMaxBitrateKbps + 3F * stdMaxBitRate) + { + changeRegion(RateControlRegion.kRcMaxUnknown, nowMs); + avgMaxBitrateKbps = -1F; + } + if (rateControlRegion == RateControlRegion.kRcNearMax) + { + // Approximate the over-use estimator delay to 100 ms. + long responseTime = rtt + 100; + long additiveIncreaseBps + = additiveRateIncrease( + nowMs, + timeLastBitrateChange, + responseTime); + + currentBitrateBps += additiveIncreaseBps; + } + else // kRcMaxUnknown || kRcAboveMax + { + long multiplicativeIncreaseBps + = multiplicativeRateIncrease( + nowMs, + timeLastBitrateChange, + currentBitrateBps); + + currentBitrateBps += multiplicativeIncreaseBps; + } + + timeLastBitrateChange = nowMs; + break; + } + case kRcDecrease: + { + bitrateIsInitialized = true; + if (incomingBitrateBps < minConfiguredBitrateBps) + { + currentBitrateBps = minConfiguredBitrateBps; + } + else + { + // Set bit rate to something slightly lower than max to get rid + // of any self-induced delay. + currentBitrateBps = (long) (beta * incomingBitrateBps + 0.5); + if (currentBitrateBps > this.currentBitrateBps) + { + // Avoid increasing the rate when over-using. + if (rateControlRegion != RateControlRegion.kRcMaxUnknown) + { + currentBitrateBps + = (long) (beta * avgMaxBitrateKbps * 1000F + 0.5F); + } + currentBitrateBps + = Math.min(currentBitrateBps, this.currentBitrateBps); + } + changeRegion(RateControlRegion.kRcNearMax, nowMs); + + if (incomingBitrateKbps + < avgMaxBitrateKbps - 3F * stdMaxBitRate) + { + avgMaxBitrateKbps = -1F; + } + + updateMaxBitRateEstimate(incomingBitrateKbps); + } + // Stay on hold until the pipes are cleared. + changeState(RateControlState.kRcHold, nowMs); + timeLastBitrateChange = nowMs; + break; + } + + default: + throw new IllegalStateException("rateControlState"); + } + if ((incomingBitrateBps > 100000L || currentBitrateBps > 150000L) + && currentBitrateBps > 1.5 * incomingBitrateBps) + { + // Allow changing the bit rate if we are operating at very low rates + // Don't change the bit rate if the send side is too far off + currentBitrateBps = this.currentBitrateBps; + timeLastBitrateChange = nowMs; + } + return currentBitrateBps; + } + + private void changeRegion(RateControlRegion region, long nowMs) + { + if (rateControlRegion == region) + { + return; + } + + rateControlRegion = region; + + if (logger.isTraceEnabled()) + { + logger.trace(diagnosticContext + .makeTimeSeriesPoint("aimd_region", nowMs) + .addField("aimd_id", hashCode()) + .addField("region", region)); + } + } + + private void changeState(RateControlInput input, long nowMs) + { + switch (currentInput.bwState) + { + case kBwNormal: + if (rateControlState == RateControlState.kRcHold) + { + timeLastBitrateChange = nowMs; + changeState(RateControlState.kRcIncrease, nowMs); + } + break; + case kBwOverusing: + if (rateControlState != RateControlState.kRcDecrease) + { + changeState(RateControlState.kRcDecrease, nowMs); + } + break; + case kBwUnderusing: + changeState(RateControlState.kRcHold, nowMs); + break; + default: + throw new IllegalStateException("currentInput.bwState"); + } + } + + private void changeState(RateControlState newState, long nowMs) + { + if (rateControlState == newState) + { + return; + } + + rateControlState = newState; + + if (logger.isTraceEnabled()) + { + logger.trace(diagnosticContext + .makeTimeSeriesPoint("aimd_state", nowMs) + .addField("aimd_id", hashCode()) + .addField("state", rateControlState)); + } + } + + public long getFeedBackInterval() + { + // Estimate how often we can send RTCP if we allocate up to 5% of + // bandwidth to feedback. + long interval + = (long) + (kRtcpSize * 8.0 * 1000.0 / (0.05 * currentBitrateBps) + 0.5); + + return + Math.min( + Math.max(interval, kMinFeedbackIntervalMs), + kMaxFeedbackIntervalMs); + } + + public long getLatestEstimate() + { + return currentBitrateBps; + } + + /** + * Returns true if the bitrate estimate hasn't been changed for + * more than an RTT, or if the incomingBitrate is more than 5% + * above the current estimate. Should be used to decide if we should reduce + * the rate further when over-using. + * + * @param timeNow + * @param incomingBitrateBps + * @return + */ + public boolean isTimeToReduceFurther(long timeNow, long incomingBitrateBps) + { + long bitrateReductionInterval = Math.max(Math.min(rtt, 200L), 10L); + + if (timeNow - timeLastBitrateChange >= bitrateReductionInterval) + return true; + if (isValidEstimate()) + { + long threshold + = (long) (kWithinIncomingBitrateHysteresis * incomingBitrateBps); + long bitrateDifference = getLatestEstimate() - incomingBitrateBps; + + return bitrateDifference > threshold; + } + return false; + } + + /** + * Returns true if there is a valid estimate of the incoming + * bitrate, false otherwise. + * + * @return + */ + public boolean isValidEstimate() + { + return bitrateIsInitialized; + } + + private long multiplicativeRateIncrease( + long nowMs, + long lastMs, + long currentBitrateBps) + { + double alpha = 1.08; + + if (lastMs > -1) + { + long timeSinceLastUpdateMs = Math.min(nowMs - lastMs, 1000); + + alpha = Math.pow(alpha, timeSinceLastUpdateMs / 1000.0); + } + return (long) Math.max(currentBitrateBps * (alpha - 1.0), 1000.0); + } + + public void reset() + { + reset(RemoteBitrateEstimatorAbsSendTime.kDefaultMinBitrateBps); + } + + private void reset(long minBitrateBps) + { + minConfiguredBitrateBps = minBitrateBps; + currentBitrateBps = /* maxConfiguredBitrateBps */ 30000000L; + avgMaxBitrateKbps = -1F; + varMaxBitrateKbps = 0.4F; + rateControlState = RateControlState.kRcHold; + rateControlRegion = RateControlRegion.kRcMaxUnknown; + timeLastBitrateChange = -1L; + currentInput.bwState = BandwidthUsage.kBwNormal; + currentInput.incomingBitRate = 0L; + currentInput.noiseVar = 1D; + updated = false; + timeFirstIncomingEstimate = -1L; + bitrateIsInitialized = false; + beta = 0.85F; + rtt = kDefaultRttMs; + timeOfLastLog = -1L; + inExperiment = false; + } + + public void setEstimate(long bitrateBps, long nowMs) + { + updated = true; + bitrateIsInitialized = true; + currentBitrateBps = changeBitrate(bitrateBps, bitrateBps, nowMs); + } + + public void setMinBitrate(long minBitrateBps) + { + minConfiguredBitrateBps = minBitrateBps; + currentBitrateBps = Math.max(minBitrateBps, currentBitrateBps); + } + + public void setRtt(long nowMs, long rtt) + { + if (logger.isTraceEnabled()) + { + logger.trace(diagnosticContext + .makeTimeSeriesPoint("aimd_rtt", nowMs) + .addField("aimd_id", hashCode()) + .addField("rtt", rtt)); + } + + this.rtt = rtt; + } + + public void update(RateControlInput input, long nowMs) + { + if (input == null) + throw new NullPointerException("input"); + + // Set the initial bit rate value to what we're receiving the first half + // second. + if (!bitrateIsInitialized) + { + if (timeFirstIncomingEstimate < 0L) + { + if (input.incomingBitRate > 0L) + timeFirstIncomingEstimate = nowMs; + } + else + { + long timeSinceFirstIncomingEstimate = nowMs - timeFirstIncomingEstimate; + if (timeSinceFirstIncomingEstimate > kFirstIncomingEstimateExpirationMs) + { + if (input.incomingBitRate > 0L) + { + timeFirstIncomingEstimate = nowMs; + } + else + { + timeFirstIncomingEstimate = -1L; + } + } + else if (timeSinceFirstIncomingEstimate > kInitializationTimeMs + && input.incomingBitRate > 0L) + { + currentBitrateBps = input.incomingBitRate; + bitrateIsInitialized = true; + } + } + } + + if (updated && currentInput.bwState == BandwidthUsage.kBwOverusing) + { + // Only update delay factor and incoming bit rate. We always want to + // react on an over-use. + currentInput.noiseVar = input.noiseVar; + currentInput.incomingBitRate = input.incomingBitRate; + } + else + { + updated = true; + currentInput.copy(input); + } + } + + public long updateBandwidthEstimate(long nowMs) + { + currentBitrateBps + = changeBitrate( + currentBitrateBps, + currentInput.incomingBitRate, + nowMs); + + if (logger.isTraceEnabled() && isValidEstimate()) + { + logger.trace(diagnosticContext + .makeTimeSeriesPoint("aimd_estimate", nowMs) + .addField("aimd_id", hashCode()) + .addField("estimate_bps", currentBitrateBps) + .addField("incoming_bps", currentInput.incomingBitRate)); + } + + if (nowMs - timeOfLastLog > kLogIntervalMs) + timeOfLastLog = nowMs; + return currentBitrateBps; + } + + private void updateMaxBitRateEstimate(float incomingBitrateKbps) + { + float alpha = 0.05F; + + if (avgMaxBitrateKbps == -1F) + { + avgMaxBitrateKbps = incomingBitrateKbps; + } + else + { + avgMaxBitrateKbps + = (1 - alpha) * avgMaxBitrateKbps + alpha * incomingBitrateKbps; + } + + // Estimate the max bit rate variance and normalize the variance with + // the average max bit rate. + float norm = Math.max(avgMaxBitrateKbps, 1F); + + varMaxBitrateKbps + = (1 - alpha) * varMaxBitrateKbps + + alpha + * (avgMaxBitrateKbps - incomingBitrateKbps) + * (avgMaxBitrateKbps - incomingBitrateKbps) + / norm; + // 0.4 ~= 14 kbit/s at 500 kbit/s + if (varMaxBitrateKbps < 0.4F) + varMaxBitrateKbps = 0.4F; + // 2.5f ~= 35 kbit/s at 500 kbit/s + if (varMaxBitrateKbps > 2.5f) + varMaxBitrateKbps = 2.5f; + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java new file mode 100644 index 0000000..16a5cf1 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/BandwidthUsage.java @@ -0,0 +1,35 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +/** + * webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h + * + * @author Lyubomir Marinov + */ +public enum BandwidthUsage +{ + kBwNormal(0), kBwUnderusing(-1), kBwOverusing(1); + + private int value; + BandwidthUsage(int value){ + this.value = value; + } + + public int getValue() { + return value; + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/InterArrival.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/InterArrival.java new file mode 100644 index 0000000..727a68d --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/InterArrival.java @@ -0,0 +1,374 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import org.jitsi.utils.TimestampUtils; +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.utils.logging2.Logger; + +/** + * Helper class to compute the inter-arrival time delta and the size delta + * between two timestamp groups. A timestamp is a 32 bit unsigned number with a + * client defined rate. + * + * webrtc/modules/remote_bitrate_estimator/inter_arrival.cc + * webrtc/modules/remote_bitrate_estimator/inter_arrival.h + * + * @author Lyubomir Marinov + * @author Julian Chukwu + * @author George Politis + */ +class InterArrival +{ + private static final int kBurstDeltaThresholdMs = 5; + + private final Logger logger; + private final TimeSeriesLogger timeSeriesLogger; + + /** + * webrtc/modules/include/module_common_types.h + * + * @param timestamp1 + * @param timestamp2 + * @return + */ + private static long latestTimestamp(long timestamp1, long timestamp2) + { + return TimestampUtils.latestTimestamp(timestamp1, timestamp2); + } + + private boolean burstGrouping; + + private TimestampGroup currentTimestampGroup = new TimestampGroup(); + + private final long kTimestampGroupLengthTicks; + + private final DiagnosticContext diagnosticContext; + + private TimestampGroup prevTimestampGroup = new TimestampGroup(); + + private double timestampToMsCoeff; + + private int numConsecutiveReorderedPackets; + + // After this many packet groups received out of order InterArrival will + // reset, assuming that clocks have made a jump. + private static final int kReorderedResetThreshold = 3; + private static final int kArrivalTimeOffsetThresholdMs = 3000; + + /** + * A timestamp group is defined as all packets with a timestamp which are at + * most {@code timestampGroupLengthTicks} older than the first timestamp in + * that group. + * + * @param timestampGroupLengthTicks + * @param timestampToMsCoeff + * @param enableBurstGrouping + */ + public InterArrival( + long timestampGroupLengthTicks, + double timestampToMsCoeff, + boolean enableBurstGrouping, + @NotNull DiagnosticContext diagnosticContext, + @NotNull Logger parentLogger) + { + kTimestampGroupLengthTicks = timestampGroupLengthTicks; + this.timestampToMsCoeff = timestampToMsCoeff; + burstGrouping = enableBurstGrouping; + numConsecutiveReorderedPackets = 0; + this.diagnosticContext = diagnosticContext; + this.logger = parentLogger.createChildLogger(getClass().getName()); + this.timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(getClass()); + } + + private boolean belongsToBurst(long arrivalTimeMs, long timestamp) + { + if (!burstGrouping) + return false; + + if (currentTimestampGroup.completeTimeMs < 0) + { + throw new IllegalStateException( + "currentTimestampGroup.completeTimeMs"); + } + + long arrivalTimeDeltaMs + = arrivalTimeMs - currentTimestampGroup.completeTimeMs; + + long timestampDiff = TimestampUtils + .subtractAsUnsignedInt32(timestamp, currentTimestampGroup.timestamp); + + long tsDeltaMs = (long) (timestampToMsCoeff * timestampDiff + 0.5); + + if (tsDeltaMs == 0) + return true; + + long propagationDeltaMs = arrivalTimeDeltaMs - tsDeltaMs; + + return + propagationDeltaMs < 0 + && arrivalTimeDeltaMs <= kBurstDeltaThresholdMs; + } + + /** + * Returns {@code true} if a delta was computed, or {@code false} if the + * current group is still incomplete or if only one group has been + * completed. + * + * @param timestamp is the timestamp. + * @param arrivalTimeMs is the local time at which the packet arrived. + * @param packetSize is the size of the packet. + * @param deltas {@code timestampDelta} is the computed timestamp delta, + * {@code arrivalTimeDeltaMs} is the computed arrival-time delta, + * {@code packetSizeDelta} is the computed size delta. + * @return + * + * @Note: We have two {@code computeDeltas}. + * One with a valid {@code systemTimeMs} according to webrtc + * implementation as of June 12,2017 and a previous one + * with a default systemTimeMs (-1L). the later may be removed or + * deprecated. + */ + public boolean computeDeltas( + long timestamp, + long arrivalTimeMs, + int packetSize, + long[] deltas){ + return computeDeltas(timestamp, arrivalTimeMs, packetSize, deltas, -1L); + } + + public boolean computeDeltas( + long timestamp, + long arrivalTimeMs, + int packetSize, + long[] deltas, + long systemTimeMs) + { + if (deltas == null) + throw new NullPointerException("deltas"); + if (deltas.length != 3) + throw new IllegalArgumentException("deltas.length"); + + boolean calculatedDeltas = false; + + if (currentTimestampGroup.isFirstPacket()) + { + // We don't have enough data to update the filter, so we store it + // until we have two frames of data to process. + currentTimestampGroup.timestamp = timestamp; + currentTimestampGroup.firstTimestamp = timestamp; + } + else if (!isPacketInOrder(timestamp)) + { + return false; + } + else if (isNewTimestampGroup(arrivalTimeMs, timestamp)) + { + // First packet of a later frame, the previous frame sample is + // ready. + if (prevTimestampGroup.completeTimeMs >= 0) + { + /* long timestampDelta */ deltas[0] + = TimestampUtils.subtractAsUnsignedInt32( + currentTimestampGroup.timestamp, + prevTimestampGroup.timestamp); + + long arrivalTimeDeltaMs + = deltas[1] + = currentTimestampGroup.completeTimeMs + - prevTimestampGroup.completeTimeMs; + + // Check system time differences to see if we have an unproportional jump + // in arrival time. In that case reset the inter-arrival computations. + long systemTimeDeltaMs = + currentTimestampGroup.lastSystemTimeMs - + prevTimestampGroup.lastSystemTimeMs; + if (prevTimestampGroup.lastSystemTimeMs != -1L && + currentTimestampGroup.lastSystemTimeMs != -1L && + arrivalTimeDeltaMs - systemTimeDeltaMs >= + kArrivalTimeOffsetThresholdMs) + { + logger.warn( "The arrival time clock offset has changed (diff = " + + (arrivalTimeDeltaMs - systemTimeDeltaMs) + + " ms), resetting."); + Reset(); + return false; + } + + if (arrivalTimeDeltaMs < 0) + { + ++numConsecutiveReorderedPackets; + if (numConsecutiveReorderedPackets >= kReorderedResetThreshold) + { + // The group of packets has been reordered since receiving + // its local arrival timestamp. + logger.warn( + "Packets are being reordered on the path from the " + + "socket to the bandwidth estimator. Ignoring " + + "this packet for bandwidth estimation."); + Reset(); + } + return false; + } + else + { + numConsecutiveReorderedPackets = 0; + } + /* int packetSizeDelta */ deltas[2] + = (int) + (currentTimestampGroup.size - prevTimestampGroup.size); + + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("computed_deltas") + .addField("inter_arrival", hashCode()) + .addField("arrival_time_ms", arrivalTimeMs) + .addField("timestamp_delta", deltas[0]) + .addField("arrival_time_ms_delta", deltas[1]) + .addField("payload_size_delta", deltas[2])); + } + + calculatedDeltas = true; + } + prevTimestampGroup.copy(currentTimestampGroup); + // The new timestamp is now the current frame. + currentTimestampGroup.firstTimestamp = timestamp; + currentTimestampGroup.timestamp = timestamp; + currentTimestampGroup.size = 0; + + } + else + { + currentTimestampGroup.timestamp = latestTimestamp( + currentTimestampGroup.timestamp, timestamp); + } + // Accumulate the frame size. + currentTimestampGroup.size += packetSize; + currentTimestampGroup.completeTimeMs = arrivalTimeMs; + currentTimestampGroup.lastSystemTimeMs = systemTimeMs; + + return calculatedDeltas; + } + + /** + * Returns {@code true} if the last packet was the end of the current batch + * and the packet with {@code timestamp} is the first of a new batch. + * + * @param arrivalTimeMs + * @param timestamp + * @return + */ + private boolean isNewTimestampGroup(long arrivalTimeMs, long timestamp) + { + if (currentTimestampGroup.isFirstPacket()) + { + return false; + } + else if (belongsToBurst(arrivalTimeMs, timestamp)) + { + return false; + } + else + { + long timestampDiff + = TimestampUtils.subtractAsUnsignedInt32( + timestamp, currentTimestampGroup.firstTimestamp); + + return timestampDiff > kTimestampGroupLengthTicks; + } + } + + /** + * Returns {@code true} if the packet with timestamp {@code timestamp} + * arrived in order. + * + * @param timestamp + * @return + */ + private boolean isPacketInOrder(long timestamp) + { + if (currentTimestampGroup.isFirstPacket()) + { + return true; + } + else + { + // Assume that a diff which is bigger than half the timestamp + // interval (32 bits) must be due to reordering. This code is almost + // identical to that in isNewerTimestamp() in module_common_types.h. + long timestampDiff + = TimestampUtils.subtractAsUnsignedInt32( + timestamp, currentTimestampGroup.firstTimestamp); + + long tsDeltaMs = (long) (timestampToMsCoeff * timestampDiff + 0.5); + boolean inOrder = timestampDiff < 0x80000000L; + + if (!inOrder && timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("reordered_packet") + .addField("inter_arrival", hashCode()) + .addField("ts_delta_ms", tsDeltaMs)); + } + + return inOrder; + } + } + + private static class TimestampGroup + { + public long completeTimeMs = -1L; + + public long size = 0L; + + public long firstTimestamp = 0L; + + public long timestamp = 0L; + + public long lastSystemTimeMs = -1L; + + /** + * Assigns the values of the fields of source to the respective + * fields of this {@code TimestampGroup}. + * + * @param source the {@code TimestampGroup} the values of the fields of + * which are to be assigned to the respective fields of this + * {@code TimestampGroup} + */ + public void copy(TimestampGroup source) + { + completeTimeMs = source.completeTimeMs; + firstTimestamp = source.firstTimestamp; + size = source.size; + timestamp = source.timestamp; + } + + public boolean isFirstPacket() + { + return completeTimeMs == -1L; + } + } + + public void Reset() { + numConsecutiveReorderedPackets = 0; + currentTimestampGroup = new TimestampGroup(); + prevTimestampGroup = new TimestampGroup(); + + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OverUseDetectorOptions.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OverUseDetectorOptions.java new file mode 100644 index 0000000..5f2f46f --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OverUseDetectorOptions.java @@ -0,0 +1,44 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.config.OveruseDetectorConfig; + +/** + * Bandwidth over-use detector options. These are used to drive experimentation + * with bandwidth estimation parameters. + * + * webrtc/common_types.h + * + * @author Lyubomir Marinov + */ +class OverUseDetectorOptions +{ + public double initialAvgNoise = 0.0D; + + public final double[][] initialE + = new double[][] { { 100, 0 }, { 0, 1e-1 } }; + + public double initialOffset = 0.0D; + + public final double[] initialProcessNoise = new double[] { 1e-10, 1e-2 }; + + public double initialSlope = 8.0D / 512.0D; + + public double initialThreshold = OveruseDetectorConfig.Companion.getInitialThreshold(); + + public double initialVarNoise = 50.0D; +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java new file mode 100644 index 0000000..150f7ed --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseDetector.java @@ -0,0 +1,203 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import edu.umd.cs.findbugs.annotations.*; +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging.*; +import org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.config.OveruseDetectorConfig; + +import java.util.*; + +/** + * webrtc/modules/remote_bitrate_estimator/overuse_detector.cc + * webrtc/modules/remote_bitrate_estimator/overuse_detector.h + * + * @author Lyubomir Marinov + */ +@SuppressFBWarnings( + value = {"FE_FLOATING_POINT_EQUALITY", "DB_DUPLICATE_BRANCHES"}, + justification = "We only compare timeOverUsing with the special value -1") +class OveruseDetector +{ + private final TimeSeriesLogger timeSeriesLogger; + + private static final double kMaxAdaptOffsetMs = 15.0; + + private BandwidthUsage hypothesis = BandwidthUsage.kBwNormal; + + private static final double kDown = 0D; + + private static final double kUp = 0D; + + private long lastUpdateMs = -1L; + + private int overuseCounter; + + private static final double overusingTimeThreshold = OveruseDetectorConfig.Companion.getOverusingTimeThreshold(); + + private double prevOffset; + + private double threshold; + + private double timeOverUsing = -1D; + + private final DiagnosticContext diagnosticContext; + + public OveruseDetector( + OverUseDetectorOptions options, + @NotNull DiagnosticContext diagnosticContext) + { + Objects.requireNonNull(options, "options"); + + threshold = options.initialThreshold; + timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(getClass()); + + this.diagnosticContext = diagnosticContext; + } + + /** + * Update the detection state based on the estimated inter-arrival time + * delta offset. {@code timestampDelta} is the delta between the last + * timestamp which the estimated offset is based on and the last timestamp + * on which the last offset was based on, representing the time between + * detector updates. {@code numOfDeltas} is the number of deltas the offset + * estimate is based on. Returns the state after the detection update. + * + * @param offset + * @param tsDelta + * @param numOfDeltas + * @param nowMs + * @return + */ + public BandwidthUsage detect( + double offset, + double tsDelta, + int numOfDeltas, + long nowMs) + { + if (numOfDeltas < 2) + return BandwidthUsage.kBwNormal; + + double prev_offset = this.prevOffset; + + this.prevOffset = offset; + + double T = Math.min(numOfDeltas, 60) * offset; + + boolean newHypothesis = false; + + if (T > threshold) + { + if (timeOverUsing == -1) + { + // Initialize the timer. Assume that we've been + // over-using half of the time since the previous + // sample. + timeOverUsing = tsDelta / 2; + } + else + { + // Increment timer + timeOverUsing += tsDelta; + } + overuseCounter++; + if (timeOverUsing > overusingTimeThreshold && overuseCounter > 1) + { + if (offset >= prev_offset) + { + timeOverUsing = 0; + overuseCounter = 0; + hypothesis = BandwidthUsage.kBwOverusing; + newHypothesis = true; + } + } + } + else if (T < -threshold) + { + timeOverUsing = -1; + overuseCounter = 0; + hypothesis = BandwidthUsage.kBwUnderusing; + newHypothesis = true; + } + else + { + timeOverUsing = -1; + overuseCounter = 0; + hypothesis = BandwidthUsage.kBwNormal; + newHypothesis = true; + } + + if (newHypothesis && timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("utilization_hypothesis", nowMs) + .addField("detector", hashCode()) + .addField("offset", offset) + .addField("prev_offset", prev_offset) + .addField("T", T) + .addField("threshold", threshold) + .addField("hypothesis", hypothesis.getValue())); + } + + updateThreshold(T, nowMs); + + return hypothesis; + } + + /** + * Returns the current detector state. + * + * @return + */ + public BandwidthUsage getState() + { + return hypothesis; + } + + private void updateThreshold(double modifiedOffset, long nowMs) + { + if (lastUpdateMs == -1) + lastUpdateMs = nowMs; + + if (Math.abs(modifiedOffset) > threshold + kMaxAdaptOffsetMs) + { + // Avoid adapting the threshold to big latency spikes, caused e.g., + // by a sudden capacity drop. + lastUpdateMs = nowMs; + return; + } + + double k = Math.abs(modifiedOffset) < threshold ? kDown : kUp; + + threshold + += k + * (Math.abs(modifiedOffset) - threshold) + * (nowMs - lastUpdateMs); + + final double kMinThreshold = 6; + final double kMaxThreshold = 600; + + threshold = Math.min(Math.max(threshold, kMinThreshold), kMaxThreshold); + + lastUpdateMs = nowMs; + } + + public double getThreshold() + { + return threshold; + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseEstimator.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseEstimator.java new file mode 100644 index 0000000..571f488 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/OveruseEstimator.java @@ -0,0 +1,330 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.utils.logging2.Logger; + +import java.util.*; + +/** + * webrtc/modules/remote_bitrate_estimator/overuse_estimator.cc + * webrtc/modules/remote_bitrate_estimator/overuse_estimator.h + * + * @author Lyubomir Marinov + */ +class OveruseEstimator +{ + /** + * The Logger used by + * OveruseEstimator instances for + * logging output. + */ + private final Logger logger; + + /** + * The TimeSeriesLogger used for time series. + */ + private final TimeSeriesLogger timeSeriesLogger; + + private static final int kDeltaCounterMax = 1000; + + private static final int kMinFramePeriodHistoryLength = 60; + + /** + * Creates and returns a deep copy of a double two-dimensional + * matrix. + * + * @param matrix the double two-dimensional matrix to create and + * return a deep copy of + * @return a deep copy of matrix + */ + private static double[][] clone(double[][] matrix) + { + int length = matrix.length; + double[][] clone; + + clone = new double[length][]; + for (int i = 0; i < length; i++) + clone[i] = matrix[i].clone(); + return clone; + } + + private double avgNoise; + + private final double[][] E; + + private DiagnosticContext diagnosticContext; + + /** + * Reduces the effects of allocations and garbage collection of the method + * {@code update}. + */ + private final double[] Eh = new double[2]; + + /** + * Reduces the effects of allocations and garbage collection of the method + * {@code update}. + */ + private final double[] h = new double[2]; + + /** + * Reduces the effects of allocations and garbage collection of the method + * {@code update}. + */ + private final double[][] IKh = new double[][] { { 0, 0 }, { 0, 0 } }; + + /** + * Reduces the effects of allocations and garbage collection of the method + * {@code update}. + */ + private final double[] K = new double[2]; + + private int numOfDeltas; + + private double offset; + + private double prevOffset; + + private final double[] processNoise; + + private double slope; + + /** + * Store the tsDelta history into a {@code double[]} used as a circular buffer + * Original c++ code uses std::list but in Java this translate + * to {@code List} which causes a lot of autoboxing + */ + private final double[] tsDeltaHist = new double[kMinFramePeriodHistoryLength]; + + /** + * Index to insert next value into {@link #tsDeltaHist} + */ + private int tsDeltaHistInsIdx; + + private double varNoise; + + public OveruseEstimator( + OverUseDetectorOptions options, + @NotNull DiagnosticContext diagnosticContext, + @NotNull Logger parentLogger) + { + logger = parentLogger.createChildLogger(getClass().getName()); + timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(getClass()); + this.diagnosticContext = diagnosticContext; + slope = options.initialSlope; + offset = options.initialOffset; + prevOffset = offset; + avgNoise = options.initialAvgNoise; + varNoise = options.initialVarNoise; + E = clone(options.initialE); + processNoise = options.initialProcessNoise.clone(); + /** + * Initialize {@link tsDeltaHist} with {@code Double.MAX_VALUE} + * to simplify {@link updateMinFramePeriod} + */ + Arrays.fill(tsDeltaHist, Double.MAX_VALUE); + } + + /** + * Returns the number of deltas which the current over-use estimator state + * is based on. + * + * @return + */ + public int getNumOfDeltas() + { + return numOfDeltas; + } + + /** + * Returns the estimated inter-arrival time delta offset in ms. + * + * @return + */ + public double getOffset() + { + return offset; + } + + /** + * Returns the estimated noise/jitter variance in ms^2. + * + * @return + */ + public double getVarNoise() + { + return varNoise; + } + + /** + * Update the estimator with a new sample. The deltas should represent + * deltas between timestamp groups as defined by the InterArrival class. + * {@code currentHypothesis} should be the hypothesis of the over-use + * detector at this time. + * + * @param tDelta + * @param tsDelta + * @param sizeDelta + * @param currentHypothesis + */ + public void update( + long tDelta, + double tsDelta, + int sizeDelta, + BandwidthUsage currentHypothesis, long systemTimeMs) + { + double minFramePeriod = updateMinFramePeriod(tsDelta); + double tTsDelta = tDelta - tsDelta; + + ++numOfDeltas; + if (numOfDeltas > kDeltaCounterMax) + numOfDeltas = kDeltaCounterMax; + + // Update the Kalman filter + E[0][0] += processNoise[0]; + E[1][1] += processNoise[1]; + + if ((currentHypothesis == BandwidthUsage.kBwOverusing + && offset < prevOffset) + || (currentHypothesis == BandwidthUsage.kBwUnderusing + && offset > prevOffset)) + { + E[1][1] += 10D * processNoise[1]; + } + + double[] h = this.h; + double[] Eh = this.Eh; + + h[0] = sizeDelta; + h[1] = 1D; + Eh[0] = E[0][0] * h[0] + E[0][1] * h[1]; + Eh[1] = E[1][0] * h[0] + E[1][1] * h[1]; + + double residual = tTsDelta - slope * h[0] - offset; + boolean inStableState = (currentHypothesis == BandwidthUsage.kBwNormal); + + // We try to filter out very late frames. For instance periodic key + // frames doesn't fit the Gaussian model well. + double maxResidual = 3D * Math.sqrt(varNoise); + double residualForUpdateNoiseEstimate + = (Math.abs(residual) < maxResidual) + ? residual + : (residual < 0 ? -maxResidual : maxResidual); + + updateNoiseEstimate( + residualForUpdateNoiseEstimate, + minFramePeriod, + inStableState); + + double denom = varNoise + h[0]*Eh[0] + h[1]*Eh[1]; + double[] K = this.K; + double[][] IKh = this.IKh; + + K[0] = Eh[0] / denom; + K[1] = Eh[1] / denom; + IKh[0][0] = 1D - K[0]*h[0]; + IKh[0][1] = -K[0]*h[1]; + IKh[1][0] = -K[1]*h[0]; + IKh[1][1] = 1D - K[1]*h[1]; + + double e00 = E[0][0]; + double e01 = E[0][1]; + + // Update state. + E[0][0] = e00 * IKh[0][0] + E[1][0] * IKh[0][1]; + E[0][1] = e01 * IKh[0][0] + E[1][1] * IKh[0][1]; + E[1][0] = e00 * IKh[1][0] + E[1][0] * IKh[1][1]; + E[1][1] = e01 * IKh[1][0] + E[1][1] * IKh[1][1]; + + // Covariance matrix, must be positive semi-definite. + boolean positiveSemiDefinite + = E[0][0] + E[1][1] >= 0 + && E[0][0] * E[1][1] - E[0][1] * E[1][0] >= 0 + && E[0][0] >= 0; + + if (!positiveSemiDefinite) + throw new IllegalStateException("positiveSemiDefinite"); + + slope = slope + K[0] * residual; + prevOffset = offset; + offset = offset + K[1] * residual; + + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("delay_variation_estimation", systemTimeMs) + .addField("estimator", hashCode()) + .addField("time_delta", tDelta) + .addField("ts_delta", tsDelta) + .addField("tts_delta", tTsDelta) + .addField("offset", offset) + .addField("hypothesis", currentHypothesis.getValue())); + } + } + + private double updateMinFramePeriod(double tsDelta) + { + /** + * Change from C++ version: + * We use {@link tsDeltaHist} as a circular buffer initialized + * with {@code Double.MAX_VALUE}, so we insert new {@link tsDelta} + * value at {@link tsDeltaHistInsIdx} and we search for the + * minimum value in {@link tsDeltaHist} + */ + tsDeltaHist[tsDeltaHistInsIdx] = tsDelta; + tsDeltaHistInsIdx = (tsDeltaHistInsIdx + 1) % tsDeltaHist.length; + + double min = tsDelta; + for (double d : tsDeltaHist) + { + if (d < min) + min = d; + } + + return min; + } + + private void updateNoiseEstimate( + double residual, + double tsDelta, + boolean stableState) + { + if (!stableState) + return; + + // Faster filter during startup to faster adapt to the jitter level of + // the network alpha is tuned for 30 frames per second, but is scaled + // according to tsDelta. + double alpha = 0.01D; + + if (numOfDeltas > 10 * 30) + alpha = 0.002D; + + // Only update the noise estimate if we're not over-using beta is a + // function of alpha and the time delta since the previous update. + double beta = Math.pow(1 - alpha, tsDelta * 30D / 1000D); + + avgNoise = beta * avgNoise + (1 - beta) * residual; + varNoise + = beta * varNoise + + (1 - beta) * (avgNoise - residual) * (avgNoise - residual); + if (varNoise < 1) + varNoise = 1; + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlInput.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlInput.java new file mode 100644 index 0000000..a4dda77 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlInput.java @@ -0,0 +1,58 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h + * + * @author Lyubomir Marinov + */ +@SuppressFBWarnings("PA_PUBLIC_PRIMITIVE_ATTRIBUTE") +class RateControlInput +{ + public BandwidthUsage bwState; + + public long incomingBitRate; + + public double noiseVar; + + public RateControlInput( + BandwidthUsage bwState, + long incomingBitRate, + double noiseVar) + { + this.bwState = bwState; + this.incomingBitRate = incomingBitRate; + this.noiseVar = noiseVar; + } + + /** + * Assigns the values of the fields of source to the respective + * fields of this RateControlInput. + * + * @param source the RateControlInput the values of the fields of + * which are to be assigned to the respective fields of this + * RateControlInput + */ + public void copy(RateControlInput source) + { + bwState = source.bwState; + incomingBitRate = source.incomingBitRate; + noiseVar = source.noiseVar; + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlRegion.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlRegion.java new file mode 100644 index 0000000..0187b15 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlRegion.java @@ -0,0 +1,28 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +/** + * webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h + * + * @author Lyubomir Marinov + */ +enum RateControlRegion +{ + kRcNearMax, + kRcAboveMax, + kRcMaxUnknown +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlState.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlState.java new file mode 100644 index 0000000..1bdf871 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RateControlState.java @@ -0,0 +1,28 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +/** + * webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h + * + * @author Lyubomir Marinov + */ +enum RateControlState +{ + kRcHold, + kRcIncrease, + kRcDecrease +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java new file mode 100644 index 0000000..6a8f6da --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateEstimatorAbsSendTime.java @@ -0,0 +1,463 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.util.*; +import org.jitsi.utils.logging.DiagnosticContext; +import org.jitsi.utils.logging.TimeSeriesLogger; +import org.jitsi.utils.logging2.*; + +import java.time.*; + +/** + * webrtc.org abs_send_time implementation as of June 26, 2017. + * commit ID: 23fbd2aa2c81d065b84d17b09b747e75672e1159 + * + * @author Julian Chukwu + * @author George Politis + */ +public class RemoteBitrateEstimatorAbsSendTime +{ + /** + * webrtc/modules/remote_bitrate_estimator/include/bwe_defines.h + */ + static final private int kBitrateWindowMs = 1000; + + static final int kDefaultMinBitrateBps = 30000; + + static final private int kStreamTimeOutMs = 2000; + + /** + * The {@link TimeSeriesLogger} to be used by this instance to print time + * series. + */ + private static final TimeSeriesLogger timeSeriesLogger + = TimeSeriesLogger.getTimeSeriesLogger( + RemoteBitrateEstimatorAbsSendTime.class); + + /** + * Defines the number of digits in the AST representation (24 bits, 6.18 + * fixed point) after the radix. + */ + private final static int kAbsSendTimeFraction = 18; + + /** + * Defines the upshift (left bit-shift) to apply to AST (24 bits, 6.18 fixed + * point) to make it inter-arrival compatible (expanded AST, 32 bits, 6.26 + * fixed point). + */ + private final static int kAbsSendTimeInterArrivalUpshift = 8; + + /** + * This is used in the {@link InterArrival} computations. In this estimator + * a timestamp group is defined as all packets with a timestamp which are at + * most 5ms older than the first timestamp in that group. + */ + private final static int kTimestampGroupLengthMs = 5; + + /** + * Defines the number of digits in the expanded AST representation (32 bits, + * 6.26 fixed point) after the radix. + */ + private final static int kInterArrivalShift + = kAbsSendTimeFraction + kAbsSendTimeInterArrivalUpshift; + + /** + * Converts the {@link #kTimestampGroupLengthMs} into "ticks" for use with + * the {@link InterArrival}. + */ + private static final long kTimestampGroupLengthTicks + = (kTimestampGroupLengthMs << kInterArrivalShift) / 1000; + + /** + * Defines the expanded AST (32 bits) to millis conversion rate. Units are + * ms per timestamp + */ + private static final double + kTimestampToMs = (double) 1000 / (1 << kInterArrivalShift); + + /** + * Reduces the effects of allocations and garbage collection of the method + * {@code incomingPacket}. + */ + private final long[] deltas = new long[3]; + + /** + * Reduces the effects of allocations and garbage collection of the method + * {@link #incomingPacketInfo(long, long, long, int)}} by promoting the + * {@code RateControlInput} instance from a local variable to a field and + * reusing the same instance across method invocations. (Consequently, the + * default values used to initialize the field are of no importance because + * they will be overwritten before they are actually used.) + */ + private final RateControlInput input + = new RateControlInput(BandwidthUsage.kBwNormal, 0L, 0D); + + private long lastPacketTimeMs; + + /** + * The time (in millis) when we saw the first packet. Useful to determine + * the probing period. + */ + private long firstPacketTimeMs; + + /** + * Keeps track of the last time (in millis) that we updated the bitrate + * estimate. + */ + private long lastUpdateMs; + + /** + * The rate control implementation based on additive increases of bitrate + * when no over-use is detected and multiplicative decreases when over-uses + * are detected. + */ + private final AimdRateControl remoteRate; + + /** + * Holds the {@link InterArrival}, {@link OveruseEstimator} and + * {@link OveruseDetector} instances of this RBE. + */ + private Detector detector; + + /** + * Keeps track of how much data we're receiving. + */ + private BitrateTracker incomingBitrate; + + /** + * Determines whether or not the incoming bitrate is initialized or not. + */ + private boolean incomingBitrateInitialized; + + /** + * The {@link DiagnosticContext} of this instance. + */ + private final DiagnosticContext diagnosticContext; + + /** + * The instance logger. + */ + private final Logger logger; + + /** + * Ctor. + * + * @param diagnosticContext the {@link DiagnosticContext} of this instance. + */ + public RemoteBitrateEstimatorAbsSendTime( + @NotNull DiagnosticContext diagnosticContext, + @NotNull Logger parentLogger) + { + this.logger = parentLogger.createChildLogger(getClass().getName()); + this.diagnosticContext = diagnosticContext; + this.remoteRate = new AimdRateControl(diagnosticContext); + this.incomingBitrate = new BitrateTracker(Duration.ofMillis(kBitrateWindowMs)); + this.incomingBitrateInitialized = false; + this.firstPacketTimeMs = -1; + this.lastPacketTimeMs = -1; + this.lastUpdateMs = -1; + } + + /** + * Reset back to the state immediately after construction. + */ + public synchronized void reset() + { + remoteRate.reset(); + this.incomingBitrate = new BitrateTracker(Duration.ofMillis(kBitrateWindowMs)); + this.incomingBitrateInitialized = false; + this.firstPacketTimeMs = -1; + this.lastPacketTimeMs = -1; + this.lastUpdateMs = -1; + + this.detector = null; + } + + /** + * Notifies this instance of an incoming packet. + * + * @param nowMs the current time when this method is called + * @param arrivalTimeMs the arrival time of the packet in millis. + * @param sendTimeMs the send time of the packet in millis + * @param payloadSize the payload size of the packet. + */ + public synchronized void incomingPacketInfo( + long nowMs, + long sendTimeMs, + long arrivalTimeMs, + int payloadSize) + { + + long sendTime24bits = convertMsTo24Bits(sendTimeMs); + // Shift up send time to use the full 32 bits that inter_arrival + // works with, so wrapping works properly. + long timestamp = sendTime24bits << kAbsSendTimeInterArrivalUpshift; + + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("in_pkt", nowMs) + .addField("rbe_id", hashCode()) + .addField("recv_ts_ms", arrivalTimeMs) + .addField("timestamp", timestamp) + .addField("pkt_sz_bytes", payloadSize)); + } + + // should be broken out from here. + // Check if incoming bitrate estimate is valid, and if it + // needs to be reset. + long incomingBitrate_ = incomingBitrate.getRateBps(arrivalTimeMs); + if (incomingBitrate_ != 0) + { + incomingBitrateInitialized = true; + } + else if (incomingBitrateInitialized) + { + // Incoming bitrate had a previous valid value, but now not + // enough data point are left within the current window. + // Reset incoming bitrate estimator so that the window + // size will only contain new data points. + incomingBitrate = new BitrateTracker(Duration.ofMillis(kBitrateWindowMs)); + incomingBitrateInitialized = false; + } + + incomingBitrate.update(DataSizeKt.getBytes(payloadSize), arrivalTimeMs); + + if (firstPacketTimeMs == -1) + { + firstPacketTimeMs = nowMs; + } + + boolean updateEstimate = false; + + checkTimeouts(nowMs); + lastPacketTimeMs = nowMs; + + long[] deltas = this.deltas; + + /* long timestampDelta */ deltas[0] = 0; + /* long timeDelta */ deltas[1] = 0; + /* int sizeDelta */ deltas[2] = 0; + + if (detector == null) + { + detector = new Detector(new OverUseDetectorOptions(), true, logger); + } + + if (detector.interArrival.computeDeltas( + timestamp, arrivalTimeMs, payloadSize, deltas, nowMs)) + { + double tsDeltaMs = deltas[0] * kTimestampToMs; + + detector.estimator.update( + /* timeDelta */ deltas[1], + /* timestampDelta */ tsDeltaMs, + /* sizeDelta */ (int) deltas[2], + detector.detector.getState(), nowMs); + + detector.detector.detect( + detector.estimator.getOffset(), tsDeltaMs, + detector.estimator.getNumOfDeltas(), arrivalTimeMs); + } + + // Check if it's time for a periodic update or if we + // should update because of an over-use. + if (lastUpdateMs == -1 + || nowMs - lastUpdateMs > remoteRate.getFeedBackInterval()) + { + updateEstimate = true; + } + else if ( + detector.detector.getState() == BandwidthUsage.kBwOverusing) + { + long incomingRate_ = incomingBitrate.getRateBps(arrivalTimeMs); + + if (incomingRate_ > 0 && remoteRate + .isTimeToReduceFurther(nowMs, incomingBitrate_)) + { + updateEstimate = true; + } + } + + if (updateEstimate) + { + // The first overuse should immediately trigger a new estimate. + // We also have to update the estimate immediately if we are + // overusing and the target bitrate is too high compared to + // what we are receiving. + input.bwState = detector.detector.getState(); + input.incomingBitRate = incomingBitrate.getRateBps(arrivalTimeMs); + input.noiseVar = detector.estimator.getVarNoise(); + + remoteRate.update(input, nowMs); + remoteRate.updateBandwidthEstimate(nowMs); + updateEstimate = remoteRate.isValidEstimate(); + } + + if (updateEstimate) + { + lastUpdateMs = nowMs; + } + } + + /** + * Timeouts SSRCs that have not received any data for + * kTimestampGroupLengthMs millis. + * + * @param nowMs the current time in millis. + */ + private synchronized void checkTimeouts(long nowMs) + { + if (nowMs - lastPacketTimeMs > kStreamTimeOutMs) + { + detector = null; + // We deliberately don't reset the first_packet_time_ms_ + // here for now since we only probe for bandwidth in the + // beginning of a call right now. + } + } + + /** + * Called when an RTP sender has a new round-trip time estimate. + */ + public synchronized void onRttUpdate(long nowMs, long avgRttMs) + { + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("new_rtt", nowMs) + .addField("avg_ms", avgRttMs)); + } + + remoteRate.setRtt(nowMs, avgRttMs); + } + + public synchronized long getLatestEstimate() + { + long bitrateBps; + if (!remoteRate.isValidEstimate()) + { + return -1; + } + if (detector == null) + { + bitrateBps = 0; + } + else + { + bitrateBps = remoteRate.getLatestEstimate(); + } + return bitrateBps; + } + + /** + * Sets the minimum bitrate for this instance. + * + * @param minBitrateBps the minimum bitrate in bps. + */ + public synchronized void setMinBitrate(int minBitrateBps) + { + // Called from both the configuration thread and the network thread. + // Shouldn't be called from the network thread in the future. + remoteRate.setMinBitrate(minBitrateBps); + } + + /** + * Get various statistics about the estimation process. [Local addition, not in original C++.] + */ + public synchronized @Nullable Statistics getStatistics() + { + if (detector == null) + { + return null; + } + return new Statistics( + detector.estimator.getOffset(), + detector.detector.getThreshold(), + detector.detector.getState() + ); + } + + /** + * Holds the {@link InterArrival}, {@link OveruseEstimator} and + * {@link OveruseDetector} instances that estimate the remote bitrate of a + * stream. + */ + private class Detector + { + /** + * Computes the send-time and recv-time deltas to feed to the estimator. + */ + private final InterArrival interArrival; + + /** + * The Kalman filter implementation that estimates the jitter. + */ + private final OveruseEstimator estimator; + + /** + * The overuse detector that compares the jitter to an adaptive threshold. + */ + private final OveruseDetector detector; + + /** + * Ctor. + * + * @param options the over-use detector options. + * @param enableBurstGrouping true to activate burst detection, false + * otherwise + */ + Detector(OverUseDetectorOptions options, + boolean enableBurstGrouping, + Logger parentLogger) + { + this.interArrival = new InterArrival( + kTimestampGroupLengthTicks, kTimestampToMs, + enableBurstGrouping, diagnosticContext, parentLogger); + this.estimator = new OveruseEstimator(options, diagnosticContext, logger); + this.detector = new OveruseDetector(options, diagnosticContext); + } + } + + /** + * Converts rtp timestamps to 24bit timestamp equivalence + * @param timeMs is the RTP timestamp e.g System.currentTimeMillis(). + * @return time stamp representation in 24 bit representation. + */ + public static long convertMsTo24Bits(long timeMs) + { + return (((timeMs << kAbsSendTimeFraction) + 500) / 1000) & 0x00FFFFFF; + } + + /** + * Various statistics about the estimation process. [Local addition, not in original C++.] + */ + public static class Statistics { + public double offset; + public double threshold; + public BandwidthUsage hypothesis; + + public Statistics(double o, double t, BandwidthUsage h) + { + offset = o; + threshold = t; + hypothesis = h; + } + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateObserver.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateObserver.java new file mode 100644 index 0000000..c833916 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/RemoteBitrateObserver.java @@ -0,0 +1,38 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator; + +import java.util.*; + +/** + * RemoteBitrateObserver is used to signal changes in bitrate estimates + * for the incoming streams. + * + * webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h + * + * @author Lyubomir Marinov + */ +public interface RemoteBitrateObserver +{ + /** + * Called when a receive channel group has a new bitrate estimate for the + * incoming streams. + * + * @param ssrcs + * @param bitrate + */ + void onReceiveBitrateChanged(Collection ssrcs, long bitrate); +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/SendSideBandwidthEstimation.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/SendSideBandwidthEstimation.java new file mode 100644 index 0000000..25e6ca6 --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/SendSideBandwidthEstimation.java @@ -0,0 +1,882 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation; + +import org.jetbrains.annotations.NotNull; +import org.jitsi.utils.logging.DiagnosticContext; +import org.jitsi.utils.logging.TimeSeriesLogger; +import org.jitsi.utils.logging2.Logger; +import org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation.config.SendSideBandwidthEstimationConfig; + +import java.time.Duration; +import java.util.*; + +/** + * Implements the send-side bandwidth estimation described in + * https://tools.ietf.org/html/draft-ietf-rmcat-gcc-01 + * Heavily based on code from webrtc.org (send_side_bandwidth_estimation.cc, + * commit ID 7ad9e661f8a035d49d049ccdb87c77ae8ecdfa35). + * + * @author Boris Grozev + */ +public class SendSideBandwidthEstimation +{ + /** + * send_side_bandwidth_estimation.cc + */ + private static final int kBweIncreaseIntervalMs = 1000; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final long kBweDecreaseIntervalMs = 300; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final int kDefaultMinBitrateBps = 10000; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final int kDefaultMaxBitrateBps = 1000000000; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final int kStartPhaseMs = 2000; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final int kLimitNumPackets = 20; + + /** + * send_side_bandwidth_estimation.cc + * + * Expecting that RTCP feedback is sent uniformly within [0.5, 1.5]s + * intervals. + */ + private static final long kFeedbackIntervalMs = 1500; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final double kPacketReportTimeoutIntervals = 1.2; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final long kFeedbackTimeoutIntervals = 3; + + /** + * send_side_bandwidth_estimation.cc + */ + private static final long kTimeoutIntervalMs = 1000; + + /** + * The random number generator for all instances of this class. + */ + private static final Random kRandom = new Random(); + + /** + * The Logger used by this instance for logging output. + */ + private final Logger logger; + + /** + * The {@link TimeSeriesLogger} to be used by this instance to print time + * series. + */ + private static final TimeSeriesLogger timeSeriesLogger + = TimeSeriesLogger.getTimeSeriesLogger( + SendSideBandwidthEstimation.class); + + /** + * send_side_bandwidth_estimation.h + */ + private final double low_loss_threshold_; + + /** + * send_side_bandwidth_estimation.h + */ + private final double high_loss_threshold_; + + /** + * send_side_bandwidth_estimation.h + */ + private final int bitrate_threshold_bps_; + + /** + * send_side_bandwidth_estimation.h + */ + private long first_report_time_ms_ = -1; + + /** + * send_side_bandwidth_estimation.h + */ + private int lost_packets_since_last_loss_update_Q8_ = 0; + + /** + * send_side_bandwidth_estimation.h + */ + private int expected_packets_since_last_loss_update_ = 0; + + /** + * send_side_bandwidth_estimation.h + */ + private boolean has_decreased_since_last_fraction_loss_ = false; + + /** + * send_side_bandwidth_estimation.h + * + * uint8_t last_fraction_loss_; + */ + private int last_fraction_loss_ = 0; + + /** + * send_side_bandwidth_estimation.h + */ + private long last_feedback_ms_ = -1; + + /** + * send_side_bandwidth_estimation.h + */ + private long last_packet_report_ms_ = -1; + + /** + * send_side_bandwidth_estimation.h + */ + private long last_timeout_ms_ = -1; + + /** + * send_side_bandwidth_estimation.h + */ + private final boolean in_timeout_experiment_; + + /** + * send_side_bandwidth_estimation.h + */ + private int min_bitrate_configured_ = kDefaultMinBitrateBps; + + /** + * send_side_bandwidth_estimation.h + */ + private int max_bitrate_configured_ = kDefaultMaxBitrateBps; + + /** + * send_side_bandwidth_estimation.h + */ + private long time_last_decrease_ms_= 0; + + /** + * send_side_bandwidth_estimation.h + */ + private long bwe_incoming_ = 0; + + /** + * send_side_bandwidth_estimation.h + */ + private long bitrate_; + + /** + * send_side_bandwidth_estimation.h + */ + private Deque min_bitrate_history_ = new LinkedList<>(); + + /** + * The {@link DiagnosticContext} of this instance. + */ + private final DiagnosticContext diagnosticContext; + + /** + * The most recent RTT calculation we've received for our connection with + * the remote endpoint in milliseconds. + */ + private long rttMs; + + /** + * The instance that holds stats for this instance. + */ + private final Statistics statistics = new Statistics(); + + public SendSideBandwidthEstimation(DiagnosticContext diagnosticContext, long startBitrate, + @NotNull Logger parentLogger) + { + logger = parentLogger.createChildLogger(getClass().getName()); + this.diagnosticContext = diagnosticContext; + + double lossExperimentProbability = SendSideBandwidthEstimationConfig.lossExperimentProbability(); + + if (kRandom.nextFloat() < lossExperimentProbability) + { + low_loss_threshold_ = SendSideBandwidthEstimationConfig.experimentalLowLossThreshold(); + high_loss_threshold_ = SendSideBandwidthEstimationConfig.experimentalHighLossThreshold(); + bitrate_threshold_bps_ = (int) SendSideBandwidthEstimationConfig.experimentalBitrateThresholdBps(); + } + else + { + low_loss_threshold_ = SendSideBandwidthEstimationConfig.defaultLowLossThreshold(); + high_loss_threshold_ = SendSideBandwidthEstimationConfig.defaultHighLossThreshold(); + bitrate_threshold_bps_ = (int) SendSideBandwidthEstimationConfig.defaultBitrateThresholdBps(); + } + + + double timeoutExperimentProbability = SendSideBandwidthEstimationConfig.timeoutExperimentProbability(); + + in_timeout_experiment_ + = kRandom.nextFloat() < timeoutExperimentProbability; + + setBitrate(startBitrate); + } + + public synchronized void reset(long startBitrate) + { + first_report_time_ms_ = -1; + lost_packets_since_last_loss_update_Q8_ = 0; + expected_packets_since_last_loss_update_ = 0; + has_decreased_since_last_fraction_loss_ = false; + last_fraction_loss_ = 0; + last_feedback_ms_ = -1; + last_packet_report_ms_ = -1; + last_timeout_ms_ = -1; + + time_last_decrease_ms_= 0; + bwe_incoming_ = 0; + min_bitrate_history_.clear(); + + setBitrate(startBitrate); + + statistics.reset(); + } + + /** + * bool SendSideBandwidthEstimation::IsInStartPhase(int64_t now_ms) + */ + private synchronized boolean isInStartPhase(long now) + { + return first_report_time_ms_ == -1 || + now - first_report_time_ms_ < kStartPhaseMs; + } + + /** + * int SendSideBandwidthEstimation::CapBitrateToThresholds + */ + private synchronized long capBitrateToThresholds(long bitrate) + { + if (bwe_incoming_ > 0 && bitrate > bwe_incoming_) + { + bitrate = bwe_incoming_; + } + if (bitrate > max_bitrate_configured_) + { + bitrate = max_bitrate_configured_; + } + if (bitrate < min_bitrate_configured_) + { + bitrate = min_bitrate_configured_; + } + return bitrate; + } + + /** + * void SendSideBandwidthEstimation::UpdateEstimate(int64_t now_ms) + */ + synchronized public void updateEstimate(long now) + { + long bitrate = bitrate_; + + // We trust the REMB during the first 2 seconds if we haven't had any + // packet loss reported, to allow startup bitrate probing. + if (last_fraction_loss_ == 0 && isInStartPhase(now) && + bwe_incoming_ > bitrate) + { + setBitrate(capBitrateToThresholds(bwe_incoming_)); + min_bitrate_history_.clear(); + min_bitrate_history_.addLast(new BitrateHistoryRecord(now, bitrate_)); + return; + } + updateMinHistory(now); + + if (last_packet_report_ms_ == -1) + { + // No feedback received. + bitrate_ = capBitrateToThresholds(bitrate_); + return; + } + + long time_since_packet_report_ms = now - last_packet_report_ms_; + long time_since_feedback_ms = now - last_feedback_ms_; + + if (time_since_packet_report_ms + < kPacketReportTimeoutIntervals * kFeedbackIntervalMs) + { + // We only care about loss above a given bitrate threshold. + float loss = last_fraction_loss_ / 256.0f; + // We only make decisions based on loss when the bitrate is above a + // threshold. This is a crude way of handling loss which is + // uncorrelated to congestion. + if (bitrate_ < bitrate_threshold_bps_ || loss <= low_loss_threshold_) + { + // Loss < 2%: Increase rate by 8% of the min bitrate in the last + // kBweIncreaseIntervalMs. + // Note that by remembering the bitrate over the last second one can + // rampup up one second faster than if only allowed to start ramping + // at 8% per second rate now. E.g.: + // If sending a constant 100kbps it can rampup immediatly to 108kbps + // whenever a receiver report is received with lower packet loss. + // If instead one would do: bitrate_ *= 1.08^(delta time), it would + // take over one second since the lower packet loss to achieve 108kbps. + bitrate = (long) (min_bitrate_history_.getFirst().bitrate * 1.08 + 0.5); + + // Add 1 kbps extra, just to make sure that we do not get stuck + // (gives a little extra increase at low rates, negligible at higher + // rates). + bitrate += 1000; + + statistics.update(now, false, LossRegion.LossFree); + + } + else if (bitrate_ > bitrate_threshold_bps_) + { + if (loss <= high_loss_threshold_) + { + // Loss between 2% - 10%: Do nothing. + + statistics.update(now, false, LossRegion.LossLimited); + } + else + { + // Loss > 10%: Limit the rate decreases to once a kBweDecreaseIntervalMs + + // rtt. + if (!has_decreased_since_last_fraction_loss_ && + (now - time_last_decrease_ms_) >= + (kBweDecreaseIntervalMs + getRttMs())) + { + time_last_decrease_ms_ = now; + + // Reduce rate: + // newRate = rate * (1 - 0.5*lossRate); + // where packetLoss = 256*lossRate; + bitrate = (long) ( + (bitrate * (512 - last_fraction_loss_)) / 512.0); + has_decreased_since_last_fraction_loss_ = true; + + statistics.update(now, false, LossRegion.LossDegraded); + } + } + } + } + else + { + statistics.update(now, true, null); + + if (time_since_feedback_ms > + kFeedbackTimeoutIntervals * kFeedbackIntervalMs + && (last_timeout_ms_ == -1 + || now - last_timeout_ms_ > kTimeoutIntervalMs)) + { + if (in_timeout_experiment_) + { + bitrate_ *= 0.8; + // Reset accumulators since we've already acted on missing + // feedback and shouldn't to act again on these old lost + // packets. + lost_packets_since_last_loss_update_Q8_ = 0; + expected_packets_since_last_loss_update_ = 0; + last_timeout_ms_ = now; + } + } + } + + setBitrate(capBitrateToThresholds(bitrate)); + } + + /** + * Report that one packet has arrived. + */ + public void reportPacketArrived(long now, boolean previouslyReportedLost) + { + if (previouslyReportedLost) + { + lostPacketArrived(); + } + else + { + updateReceiverBlock(0, 1, now); + } + } + + /** + * Report that one packet was lost. + */ + public void reportPacketLost(long now) + { + updateReceiverBlock(256, 1, now); + } + + /** + * void SendSideBandwidthEstimation::UpdateReceiverBlock + */ + synchronized private void updateReceiverBlock(long fraction_lost, long number_of_packets, long now) + { + last_feedback_ms_ = now; + if (first_report_time_ms_ == -1) + { + first_report_time_ms_ = now; + } + + // Check sequence number diff and weight loss report + if (number_of_packets > 0) + { + // Calculate number of lost packets. + long num_lost_packets_Q8 = fraction_lost * number_of_packets; + // Accumulate reports. + lost_packets_since_last_loss_update_Q8_ += num_lost_packets_Q8; + expected_packets_since_last_loss_update_ += number_of_packets; + + // Don't generate a loss rate until it can be based on enough packets. + if (expected_packets_since_last_loss_update_ < kLimitNumPackets) + return; + + has_decreased_since_last_fraction_loss_ = false; + last_fraction_loss_ = + lost_packets_since_last_loss_update_Q8_ / + expected_packets_since_last_loss_update_; + + // Reset accumulators. + lost_packets_since_last_loss_update_Q8_ = 0; + expected_packets_since_last_loss_update_ = 0; + + last_packet_report_ms_ = now; + } + } + + synchronized private void lostPacketArrived() + { + /* This algorithm can only cancel out losses that have occurred + since the last loss update, unfortunately. */ + if (lost_packets_since_last_loss_update_Q8_ >= 256) + { + lost_packets_since_last_loss_update_Q8_ -= 256; + } + } + + private synchronized void updateMinHistory(long now_ms) + { + // Remove old data points from history. + // Since history precision is in ms, add one so it is able to increase + // bitrate if it is off by as little as 0.5ms. + while (!min_bitrate_history_.isEmpty() && + now_ms - min_bitrate_history_.getFirst().timestampMs + 1 > + kBweIncreaseIntervalMs) + { + min_bitrate_history_.removeFirst(); + } + + // Typical minimum sliding-window algorithm: Pop values higher than current + // bitrate before pushing it. + while (!min_bitrate_history_.isEmpty() && + bitrate_ <= min_bitrate_history_.getLast().bitrate) + { + min_bitrate_history_.removeLast(); + } + + min_bitrate_history_.addLast(new BitrateHistoryRecord(now_ms, bitrate_)); + } + + /** + * {@link SendSideBandwidthEstimation#updateReceiverEstimate} + * This is the entry/update point for the estimated bitrate in the + * REMBPacket or a Delay Based Controller estimated bitrate when the + * Delay based controller and the loss based controller lives on the + * send side. see internet draft on "Congestion Control for RTCWEB" + */ + public synchronized void updateReceiverEstimate(long bandwidth) + { + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("bwe_incoming") + .addField("bitrate_bps", bandwidth)); + } + bwe_incoming_ = bandwidth; + setBitrate(capBitrateToThresholds(bitrate_)); + } + + /** + * void SendSideBandwidthEstimation::SetMinMaxBitrate + */ + synchronized public void setMinMaxBitrate(int min_bitrate, int max_bitrate) + { + min_bitrate_configured_ = Math.max(min_bitrate, kDefaultMinBitrateBps); + if (max_bitrate > 0) + { + max_bitrate_configured_ = + Math.max(min_bitrate_configured_, max_bitrate); + } + else + { + max_bitrate_configured_ = kDefaultMaxBitrateBps; + } + } + + /** + * Sets the value of {@link #bitrate_}. + * @param newValue the value to set + */ + private synchronized void setBitrate(long newValue) + { + bitrate_ = newValue; + } + + /** + * @return the latest estimate. + */ + public long getLatestEstimate() + { + return bitrate_; + } + + /** + * @return the latest values of the Receiver Estimated Maximum Bandwidth. + */ + public long getLatestREMB() + { + return bwe_incoming_; + } + + /** + * @return the latest effective fraction loss calculated. + * The value is between 0 and 256 (corresponding + * to 0% and 100% respectively). + */ + public int getLatestFractionLoss() + { + return last_fraction_loss_; + } + + /** + * @return the statistics specific to this bandwidth estimator. + */ + public Statistics getStatistics() + { + return statistics; + } + + public synchronized void onRttUpdate(Duration newRtt) + { + this.rttMs = newRtt.toMillis(); + } + + /** + * Returns the last calculated RTT to the endpoint in milliseconds. + * @return the last calculated RTT to the endpoint in milliseconds. + */ + private synchronized long getRttMs() + { + if (rttMs < 0) + { + logger.warn("RTT not calculated, using the default of 100ms."); + rttMs = 100; + } + else if (rttMs > 1000) + { + logger.warn( + "RTT suspiciously high (" + rttMs + "ms), capping to 1000ms."); + rttMs = 1000; + } + + return rttMs; + } + + private static final class BitrateHistoryRecord + { + final long timestampMs; + final long bitrate; + + BitrateHistoryRecord(long now, long br) + { + timestampMs = now; + bitrate = br; + } + } + + /** + * This class records statistics information about how much time we spend + * in different loss-states (loss-free, loss-limited and loss-degraded). + */ + public class Statistics + { + /** + * The current state {@link LossRegion}. + */ + private LossRegion currentState = null; + + /** + * Keeps the time (in millis) of the last transition (including a loop). + */ + private long lastTransitionTimestampMs = -1; + + /** + * The cumulative duration (in millis) of the current state + * {@link #currentState} after having looped + * {@link #currentStateConsecutiveVisits} times. + */ + private long currentStateCumulativeDurationMs; + + /** + * The number of loops over the current state {@link #currentState}. + */ + private int currentStateConsecutiveVisits; + + /** + * The bitrate when we entered the current state {@link #currentState}. + */ + private long currentStateStartBitrateBps; + + /** + * Computes the min/max/avg/sd of the bitrate while in + * {@link #currentState}. + */ + private LongSummaryStatistics currentStateBitrateStatistics + = new LongSummaryStatistics(); + + /** + * Computes the min/max/avg/sd of the loss while in + * {@link #currentState}. + */ + private IntSummaryStatistics currentStateLossStatistics + = new IntSummaryStatistics(); + + /** + * True when the fields of this class have changed from their default + * value. The purpose is to avoid creating new IntSummaryStatistics and + * LongSummaryStatistics when it's not needed. + */ + private boolean isDirty = false; + + /** + * Computes the sum of the duration of the different states. + */ + private LongSummaryStatistics + lossFreeMsStats = new LongSummaryStatistics(), + lossDegradedMsStats = new LongSummaryStatistics(), + lossLimitedMsStats = new LongSummaryStatistics(); + + public void update(long nowMs) + { + synchronized (SendSideBandwidthEstimation.this) + { + long time_since_packet_report_ms + = nowMs - last_packet_report_ms_; + + boolean currentStateHasTimedOut = time_since_packet_report_ms + < kPacketReportTimeoutIntervals * kFeedbackIntervalMs; + + update(nowMs, currentStateHasTimedOut, null); + } + } + + /** + * Records a state transition and updates the statistics information. + * + * @param nowMs the time (in millis) of the transition. + * @param currentStateHasTimedOut true if the current state has timed + * out, i.e. we haven't received receiver reports "in a while". + * @param nextState the that the bwe is transitioning to. + */ + void update( + long nowMs, boolean currentStateHasTimedOut, LossRegion nextState) + { + synchronized (SendSideBandwidthEstimation.this) + { + if (lastTransitionTimestampMs > -1 && !currentStateHasTimedOut) + { + isDirty = true; + currentStateCumulativeDurationMs + += nowMs - lastTransitionTimestampMs; + } + + lastTransitionTimestampMs = nowMs; + if (!currentStateHasTimedOut) + { + isDirty = true; + // If the current state has not timed out, then update the + // stats that we gather. + currentStateLossStatistics.accept(last_fraction_loss_); + currentStateConsecutiveVisits++; // we start counting from 0. + if (this.currentState == nextState) + { + currentStateBitrateStatistics.accept(bitrate_); + return; + } + } + + if (this.currentState != null) + { + // This is not a loop, we're transitioning to another state. + // Record how much time we've spent on this state, how many + // times we've looped through it and what was the impact on + // the bitrate. + switch (this.currentState) + { + case LossDegraded: + lossDegradedMsStats.accept( + currentStateCumulativeDurationMs); + break; + case LossFree: + lossFreeMsStats.accept(currentStateCumulativeDurationMs); + break; + case LossLimited: + lossLimitedMsStats.accept( + currentStateCumulativeDurationMs); + break; + } + + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("loss_estimate") + .addField("state", currentState.name()) + .addField("max_loss", + currentStateLossStatistics.getMax() / 256.0f) + .addField("min_loss", + currentStateLossStatistics.getMin() / 256.0f) + .addField("avg_loss", + currentStateLossStatistics.getAverage()/256.0f) + .addField("max_bps", + currentStateBitrateStatistics.getMax()) + .addField("min_bps", + currentStateBitrateStatistics.getMin()) + .addField("avg_bps", + currentStateBitrateStatistics.getAverage()) + .addField("duration_ms", + currentStateCumulativeDurationMs) + .addField("consecutive_visits", + currentStateConsecutiveVisits) + .addField("bitrate_threshold", + bitrate_threshold_bps_) + .addField("low_loss_threshold", + low_loss_threshold_) + .addField("high_loss_threshold", + high_loss_threshold_) + .addField("delta_bps", + bitrate_ - currentStateStartBitrateBps) + .addField("bitrate", + bitrate_) + .addField("bwe_incoming", + bwe_incoming_) + .addField("rtt_ms", + rttMs)); + } + } + + currentState = nextState; + currentStateStartBitrateBps = bitrate_; + + if (isDirty) + { + currentStateLossStatistics = new IntSummaryStatistics(); + currentStateBitrateStatistics = new LongSummaryStatistics(); + currentStateConsecutiveVisits = 0; + currentStateCumulativeDurationMs = 0; + isDirty = false; + } + + currentStateBitrateStatistics.accept(bitrate_); + } + } + + /** Return to state immediately after construction. */ + void reset() + { + synchronized (SendSideBandwidthEstimation.this) + { + currentState = null; + + lastTransitionTimestampMs = -1; + + lossFreeMsStats = new LongSummaryStatistics(); + lossDegradedMsStats = new LongSummaryStatistics(); + lossLimitedMsStats = new LongSummaryStatistics(); + + if (isDirty) + { + currentStateLossStatistics = new IntSummaryStatistics(); + currentStateBitrateStatistics = new LongSummaryStatistics(); + currentStateConsecutiveVisits = 0; + currentStateCumulativeDurationMs = 0; + isDirty = false; + } + } + } + + /** + * @return the number of millis spent in the loss-limited state. + */ + public long getLossLimitedMs() + { + synchronized (SendSideBandwidthEstimation.this) + { + return lossLimitedMsStats.getSum(); + } + } + + /** + * @return the number of millis spent in the loss-degraded state. + */ + public long getLossDegradedMs() + { + synchronized (SendSideBandwidthEstimation.this) + { + return lossDegradedMsStats.getSum(); + } + } + + /** + * @return the number of millis spent in the loss-free state. + */ + public long getLossFreeMs() + { + synchronized (SendSideBandwidthEstimation.this) + { + return lossFreeMsStats.getSum(); + } + } + } + + /** + * Represents the loss-based controller states. + */ + private enum LossRegion + { + /** + * Loss is between 2% and 10%. + */ + LossLimited, + + /** + * Loss is above 10%. + */ + LossDegraded, + + /** + * Loss is bellow 2%. + */ + LossFree + } +} diff --git a/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/transform/PacketTransformer.java b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/transform/PacketTransformer.java new file mode 100644 index 0000000..f67f5aa --- /dev/null +++ b/jitsi-media-transform/src/main/java/org/jitsi_modified/impl/neomedia/transform/PacketTransformer.java @@ -0,0 +1,53 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.transform; + +import org.jitsi.rtp.*; + +/** + * Encapsulate the concept of packet transformation. Given an array of packets, + * PacketTransformer can either "transform" each one of them, or + * "reverse transform" (e.g. restore) each one of them. + * + * @author Bing SU (nova.su@gmail.com) + * @author Boris Grozev + */ +public interface PacketTransformer +{ + /** + * Closes this PacketTransformer i.e. releases the resources + * allocated by it and prepares it for garbage collection. + */ + public void close(); + + /** + * Reverse-transforms each packet in an array of packets. Null values + * must be ignored. + * + * @param pkts the transformed packets to be restored. + * @return the restored packets. + */ + public Packet[] reverseTransform(Packet[] pkts); + + /** + * Transforms each packet in an array of packets. Null values must be + * ignored. + * + * @param pkts the packets to be transformed + * @return the transformed packets + */ + public Packet[] transform(Packet[] pkts); +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/AudioLevelListener.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/AudioLevelListener.kt new file mode 100644 index 0000000..807c1fb --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/AudioLevelListener.kt @@ -0,0 +1,29 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj + +interface AudioLevelListener { + /** + * Process the audio level from a received audio RTP packet. + * + * @param sourceSsrc The SSRC identifying the source of the audio stream. + * @param level The audio level. + * @return A boolean set to {@code true} if this packet should be + * discarded without being forwarded to other endpoints. + */ + fun onLevelReceived(sourceSsrc: Long, level: Long): Boolean +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/DebugStateMode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/DebugStateMode.kt new file mode 100644 index 0000000..5a90977 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/DebugStateMode.kt @@ -0,0 +1,27 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +enum class DebugStateMode { + /** Include full details, mostly for manually debugging */ + FULL, + + /** Include all information needed for stats/metric collection (e.g. rtcstats) */ + STATS, + + /** Include only the most important information, e.g. listing conferences and endpoints. */ + SHORT +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Event.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Event.kt new file mode 100644 index 0000000..5da7448 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Event.kt @@ -0,0 +1,27 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import org.jitsi.utils.MediaType + +interface Event + +class SetMediaSourcesEvent( + val mediaSourceDescs: Array, + val signaledMediaSourceDescs: Array +) : Event + +class SetLocalSsrcEvent(val mediaType: MediaType, val ssrc: Long) : Event diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/EventHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/EventHandler.kt new file mode 100644 index 0000000..77fb759 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/EventHandler.kt @@ -0,0 +1,20 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +interface EventHandler { + fun handleEvent(event: Event) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Features.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Features.kt new file mode 100644 index 0000000..40d94d7 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Features.kt @@ -0,0 +1,21 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj + +enum class Features { + TRANSCEIVER_PCAP_DUMP +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt new file mode 100644 index 0000000..7d6dddb --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSourceDesc.kt @@ -0,0 +1,222 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.ArrayUtils +import java.util.Collections +import java.util.NavigableMap +import java.util.TreeMap + +/** + * Represents a collection of [RtpLayerDesc]s that encode the same + * media source. This specific implementation provides webrtc simulcast stream + * suspension detection. + * + * We take the definition of "Media Source" from RFC 7656. It takes a single + * logical source of media, which might be represented by multiple encoded streams. + * + * @author George Politis + */ +class MediaSourceDesc +@JvmOverloads constructor( + /** + * The [RtpEncodingDesc]s that this [MediaSourceDesc] + * possesses, ordered by their subjective quality from low to high. + */ + val rtpEncodings: Array, + /** + * A string which identifies the owner of this source (e.g. the endpoint + * which is the sender of the source). + */ + val owner: String, + /** + * A string which identifies this source. + */ + val sourceName: String, + /** + * The {@link VideoType} signaled for this media source (defaulting to {@code CAMERA} if nothing has been signaled). + */ + var videoType: VideoType = VideoType.CAMERA, + /** + * Whether this is a synthetic source, i.e. one generated by the bridge rather than sent by an endpoint. + */ + val synthetic: Boolean = false, +) { + /** + * Current single-list view of all the encodings' layers. + */ + private lateinit var layers: List + + /** + * Allow the lookup of a layer by the encoding id of a received packet. + */ + private val layersById: MutableMap = HashMap() + + /** + * Allow the lookup of a layer by index. + */ + private val layersByIndex: NavigableMap = TreeMap() + + /** + * Get a view of the source's RTP layers, in quality order. + */ + val rtpLayers: List + @Synchronized + get() = layers + + /** + * Update the layer cache. Should be synchronized on [this]. + */ + private fun updateLayerCache() { + layersById.clear() + layersByIndex.clear() + val tempLayers = ArrayList() + + for (encoding in rtpEncodings) { + for (layer in encoding.layers) { + layersById[encoding.encodingId(layer)] = layer + layersByIndex[layer.index] = layer + tempLayers.add(layer) + } + } + layers = Collections.unmodifiableList(tempLayers) + } + + init { + updateLayerCache() + } + + /** + * Gets the last "stable" bitrate (in bps) of the encoding of the specified + * index. The "stable" bitrate is measured on every new frame and with a + * 5000ms window. + * + * If the bitrate for the specified index is 0, return bitrate of the highest- + * index layer less than the index with a non-zero bitrate. + * + * @return the last "stable" bitrate (bps) of the encoding with a non-zero rate + * at or below the specified index. + */ + @Synchronized + fun getBitrate(nowMs: Long, idx: Int): Bandwidth { + for (entry in layersByIndex.headMap(idx, true).descendingMap()) { + val bitrate = entry.value.getBitrate(nowMs) + if (bitrate.bps > 0) { + return bitrate + } + } + return 0.bps + } + + @Synchronized + fun hasRtpLayers(): Boolean = layers.isNotEmpty() + + @Synchronized + fun numRtpLayers(): Int = layersByIndex.size + + val primarySSRC: Long + get() = rtpEncodings[0].primarySSRC + + @Synchronized + fun getRtpLayerByQualityIdx(idx: Int): RtpLayerDesc? = layersByIndex[idx] + + @Synchronized + fun findRtpLayerDescs(videoRtpPacket: VideoRtpPacket): Collection { + if (ArrayUtils.isNullOrEmpty(rtpEncodings)) { + return emptyList() + } + return videoRtpPacket.getEncodingIds().mapNotNull { layersById[it] } + } + + @Synchronized + fun findRtpEncodingDesc(ssrc: Long): RtpEncodingDesc? = rtpEncodings.find { it.hasSsrc(ssrc) } + + @Synchronized + fun getEncodingLayers(ssrc: Long): Array { + val enc = findRtpEncodingDesc(ssrc) ?: return emptyArray() + return Array(enc.layers.size) { i -> + enc.layers[i].copy() + } + } + + @Synchronized + fun setEncodingLayers(layers: Array, ssrc: Long) { + val enc = findRtpEncodingDesc(ssrc) ?: return + enc.layers = layers + updateLayerCache() + } + + /** + * Clone an existing media source desc, inheriting layer descs' statistics. + */ + @Synchronized + fun copy() = MediaSourceDesc( + Array(this.rtpEncodings.size) { i -> this.rtpEncodings[i].copy() }, + this.owner, + this.sourceName, + this.videoType, + this.synthetic + ) + + override fun toString(): String = "MediaSourceDesc[name=$sourceName owner=$owner, videoType=$videoType, " + + "synthetic=$synthetic, encodings=${rtpEncodings.joinToString(",")}]" + + /** + * Checks whether the given SSRC matches this source's [primarySSRC]. + * This is mostly useful only for determining quickly whether two source + * descriptions describe the same source; other functions (probably [hasSsrc]) should be used + * to match received media packets. + * + * @param ssrc the SSRC to match. + * @return `true` if the specified `ssrc` is the primary SSRC + * for this source. + */ + fun matches(ssrc: Long) = rtpEncodings.getOrNull(0)?.primarySSRC == ssrc + + /** + * Checks whether any encoding of this source has this [ssrc] + */ + @Synchronized + fun hasSsrc(ssrc: Long) = rtpEncodings.any { it.hasSsrc(ssrc) } +} + +/** + * Clone an array of media source descriptions. + */ +fun Array.copy() = Array(this.size) { i -> this[i].copy() } + +fun Array.findRtpLayerDescs(packet: VideoRtpPacket): Collection = this.flatMap { + it.findRtpLayerDescs(packet) +} + +fun Array.findRtpSourceByPrimary(ssrc: Long): MediaSourceDesc? = this.find { it.matches(ssrc) } + +fun Array.findRtpSource(ssrc: Long): MediaSourceDesc? = this.find { it.hasSsrc(ssrc) } + +fun Array.findRtpSource(packet: RtpPacket): MediaSourceDesc? = findRtpSource(packet.ssrc) + +fun Array.findRtpEncodingId(packet: VideoRtpPacket): Int? { + for (source in this) { + source.findRtpEncodingDesc(packet.ssrc)?.let { + return it.eid + } + } + return null +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt new file mode 100644 index 0000000..df7f533 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/MediaSources.kt @@ -0,0 +1,75 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode + +/** + * Maintains an array of [MediaSourceDesc]. The set method preserves the existing sources that match one of the new + * sources, because [MediaSourceDesc] holds some local state (rate statistics) that we would like to keep. Ideally + * this state should be out of the source descriptor, in which case this class would be obsolete. + * + * @author Boris Grozev + */ +class MediaSources { + private var sources: Array = arrayOf() + + fun setMediaSources(newSources: Array): Boolean { + val oldSources = sources + + if (oldSources.isEmpty() || newSources.isEmpty()) { + sources = newSources + return oldSources.size != newSources.size + } + + var cntMatched = 0 + val mergedSources: Array = Array(newSources.size) { i -> + val newPrimarySSRC = newSources[i].primarySSRC + for (j in 0 until oldSources.size) { + if (oldSources[j].matches(newPrimarySSRC)) { + cntMatched++ + // NOTE: we deliberately do not update the old source instance + // with the encodings of the new one. Values set on the + // source are more likely to be correct than ones generated + // from signaling about the source. (Revisit this if we + // encounter a scenario where this isn't true...) + return@Array oldSources[j] + } + } + newSources[i] + } + + sources = mergedSources + return oldSources.size != newSources.size || cntMatched != oldSources.size + } + + fun getMediaSources(): Array = sources + + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + sources.forEach { source -> + val sourceNode = JsonNodeFactory.instance.objectNode().apply { + put("owner", source.owner) + put("video_type", source.videoType.toString()) + source.rtpEncodings.forEach { + set("rtp_encoding_${it.primarySSRC}", it.debugState()) + } + } + set(source.sourceName, sourceNode) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketHandler.kt new file mode 100644 index 0000000..da14dfb --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketHandler.kt @@ -0,0 +1,81 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.NodeStatsProducer +import org.jitsi.nlj.util.addMbps + +interface PacketHandler { + /** + * Process the given packets + */ + fun processPacket(packetInfo: PacketInfo) +} + +/** + * A [PacketHandler] which tracks statistics about the packets it handles + */ +abstract class StatsKeepingPacketHandler : PacketHandler, NodeStatsProducer { + private val statistics = Statistics() + + final override fun processPacket(packetInfo: PacketInfo) { + val now = System.currentTimeMillis() + statistics.packetsReceived++ + statistics.bytesReceived += packetInfo.packet.length + if (statistics.firstPacketReceivedTime == 0L) { + statistics.firstPacketReceivedTime = now + } + statistics.lastPacketReceivedTime = now + + doProcessPacket(packetInfo) + } + + /** + * We override [processPacket] above and prevent subclasses from doing so + * to make sure this class sees all packets that pass through it and it + * can track stats correctly. This method should be implemented by + * subclasses as a stand-in for [processPacket], as it is called once + * all the stat-tracking has been done. + */ + protected abstract fun doProcessPacket(packetInfo: PacketInfo) + + override fun getNodeStats(): NodeStatsBlock = statistics.getNodeStats() + + /** + * Statistics common to [PacketHandler]s which are interesting to track + */ + data class Statistics( + var firstPacketReceivedTime: Long = 0, + var lastPacketReceivedTime: Long = 0, + var bytesReceived: Long = 0, + var packetsReceived: Long = 0 + ) : NodeStatsProducer { + override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("Packet handler stats").apply { + addNumber(RECEIVED_PACKETS, packetsReceived) + addNumber(RECEIVED_BYTES, bytesReceived) + addNumber(RECEIVED_DURATION_MS, lastPacketReceivedTime - firstPacketReceivedTime) + addMbps(RECEIVED_BITRATE_MBPS, RECEIVED_BYTES, RECEIVED_DURATION_MS) + } + + companion object { + private const val RECEIVED_PACKETS = "received_packets" + private const val RECEIVED_BYTES = "received_bytes" + private const val RECEIVED_DURATION_MS = "received_duration_ms" + private const val RECEIVED_BITRATE_MBPS = "received_bitrate_mbps" + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt new file mode 100644 index 0000000..8519cfc --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/PacketInfo.kt @@ -0,0 +1,298 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.format.PayloadType +import org.jitsi.rtp.Packet +import org.jitsi.utils.logging2.createLogger +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.Collections + +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class EventTimeline( + /** + * We want this thread safe, because while PacketInfo objects are only handled by a single thread at a time, + * [StatsKeepingNode] may add an "exit" event after the packets has been added to another queue potentially handled + * by a different thread. This is not critical as it only affects the timeline and the result is just some "exit" + * events missing from the trace logs. + */ + timelineArg: MutableList> = mutableListOf(), + private val clock: Clock = Clock.systemUTC() +) : Iterable> { + + private val timeline = Collections.synchronizedList(timelineArg) + + /** + * The [referenceTime] refers to the first timestamp we have + * in the timeline. In the timeline this is used as time "0" and + * all other times are represented as deltas from this 0. + */ + var referenceTime: Instant? = null + + val size: Int + get() = timeline.size + + fun addEvent(desc: String) { + val now = clock.instant() + if (referenceTime == null) { + referenceTime = now + } + timeline.add(desc to Duration.between(referenceTime, now)) + } + + fun clone(): EventTimeline { + val clone = EventTimeline(timeline.toMutableList(), clock) + clone.referenceTime = referenceTime + return clone + } + + override fun iterator(): Iterator> = timeline.iterator() + + /** + * Return the total time between this packet's first event and last event + * or -1 if there is no reference time + */ + fun totalDelay(): Duration { + return referenceTime?.let { + return timeline.last().second + } ?: Duration.ofMillis(-1) + } + + override fun toString(): String = with(StringBuffer()) { + referenceTime?.let { + append("Reference time: $referenceTime; ") + synchronized(timeline) { + append(timeline.joinToString(separator = "; ")) + } + } ?: run { + append("[No timeline]") + } + toString() + } +} + +/** + * The origin of a packet in the system; used to track outgoing packets in [OutgoingStatisticsTracker] + * to measure the bitrates of each type of data. + * + * Currently only used for RTP, so RTCP, SCTP, and Datachannel will all be either Routed or Misc. + */ +enum class PacketOrigin { + Routed, + Retransmission, + Probing, + Padding, + Synthesized, + Misc + /* TODO: Add RTCP, SCTP, and datachannel if needed */ +} + +/** + * [PacketInfo] is a wrapper around a [Packet] instance to be passed through + * a pipeline. Since the [Packet] can change as it moves through the pipeline + * (as it is parsed into different types), the wrapping [PacketInfo] stays consistent + * and allows for metadata to be passed along with a packet. + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +open class PacketInfo @JvmOverloads constructor( + var packet: Packet, + /** The original length of the packet, i.e. before decryption. Stays unchanged even if the packet is updated. */ + val originalLength: Int = packet.length, + val timeline: EventTimeline? = if (enableTimeline) EventTimeline() else null +) { + /** + * An explicit tag for when this packet was originally received (assuming it + * was an incoming packet and not one created by jvb itself). + */ + var receivedTime: Instant? = null + set(value) { + field = value + if (timeline != null && timeline.referenceTime == null) { + timeline.referenceTime = value + } + } + + /** Whether the packet originally had cryptex RTP header extensions. */ + var originalHadCryptex: Boolean = false + + /** + * Whether this packet has been recognized to contain only shouldDiscard. + */ + var shouldDiscard: Boolean = false + + /** + * The ID of the endpoint associated with this packet (i.e. the source endpoint). + */ + var endpointId: String? = null + + /** + * Whether this packet indicates a point in which its stream's layering changed, in + * a way that indicates that bitrate allocation may need to be recomputed. + */ + var layeringChanged = false + + var payloadType: PayloadType? = null + + /** + * The RFC 6464 audio level (0-127, i.e. -dBov: 0 is loudest, 127 is silence) reported by the sender in the + * ssrc-audio-level RTP header extension, or null if the packet had no such extension. Populated on the incoming + * path by [org.jitsi.nlj.transform.node.incoming.AudioLevelReader]. + */ + var audioLevel: Int? = null + + /** + * The RFC 6464 Voice Activity Detection flag reported by the sender in the ssrc-audio-level RTP header extension, + * or null if the packet had no such extension. Populated alongside [audioLevel]. + */ + var vad: Boolean? = null + + /** + * The payload verification string for the packet, or 'null' if payload verification is disabled. Calculating the + * it is expensive, thus we only do it when the flag is enabled. + */ + var payloadVerification = if (enablePayloadVerification) packet.payloadVerification else null + + /** + * Information about whether this packet is used for probing by the transport-cc engine. + * The type is internal to that object. + */ + var probingInfo: Any? = null + + /** + * The origin of the packet, used for tracking the sources of media being routed. + */ + var packetOrigin: PacketOrigin = PacketOrigin.Misc + + /** + * Re-calculates the expected payload verification string. This should be called any time that the code + * intentionally modifies the packet in a way that could change the verification string (for example, re-creates + * it with a new type (parsing), or intentionally modifies the payload (SRTP)). + */ + fun resetPayloadVerification() { + payloadVerification = if (enablePayloadVerification) packet.payloadVerification else null + } + + /** + * Get the contained packet cast to [ExpectedPacketType] + */ + @Suppress("UNCHECKED_CAST") + fun packetAs(): ExpectedPacketType = packet as ExpectedPacketType + + /** + * Create a deep clone of this PacketInfo (both the contained packet and the metadata map + * will be copied for the cloned PacketInfo). + */ + fun clone(): PacketInfo { + val clone = PacketInfo(packet.clone(), originalLength, timeline?.clone()) + clone.receivedTime = receivedTime + clone.originalHadCryptex = originalHadCryptex + clone.shouldDiscard = shouldDiscard + clone.endpointId = endpointId + clone.payloadType = payloadType + clone.audioLevel = audioLevel + clone.vad = vad + clone.layeringChanged = layeringChanged + clone.payloadVerification = payloadVerification + clone.probingInfo = probingInfo + clone.packetOrigin = packetOrigin + @Suppress("UNCHECKED_CAST") // ArrayList.clone() really does return ArrayList, not Object. + clone.onSentActions = onSentActions?.clone() as ArrayList<(PacketInfo) -> Unit>? + return clone + } + + fun addEvent(desc: String) = timeline?.addEvent(desc) + + /** + * The list of pending actions, or [null] if none. + */ + private var onSentActions: ArrayList<(PacketInfo) -> Unit>? = null + + /** + * Add an action to be performed when the packet is sent (i.e. when this packet's + * [sent] method is called). + * + * If this [PacketInfo] object is cloned, the action will be called for every + * cloned instance. If packet is dropped (i.e. [sent] is never called), the + * action will not be called. + */ + fun onSent(action: (PacketInfo) -> Unit) { + synchronized(this) { + if (onSentActions == null) { + onSentActions = ArrayList(1) + } + onSentActions!!.add(action) + } + } + + /** + * Invoke any actions previously registered with this [PacketInfo]'s [onSent] + * method. This should be called just before, or after, this packet is sent. + */ + fun sent() { + var actions: List<(PacketInfo) -> Unit> = Collections.emptyList() + synchronized(this) { + onSentActions?.let { + actions = it + onSentActions = null + } ?: run { return@sent } + } + for (action in actions) { + action.invoke(this) + } + } + + companion object { + private val enableTimeline: Boolean by config { + "jmt.debug.packet-timeline.enabled".from(JitsiConfig.newConfig) + } + + private val enablePayloadVerificationDefault: Boolean by config { + "jmt.debug.payload-verification.enabled".from(JitsiConfig.newConfig) + } + + /** + * If this is enabled all [Node]s will verify that the payload didn't unexpectedly change. This is expensive. + */ + @field:Suppress("ktlint:standard:property-naming") + var enablePayloadVerification = enablePayloadVerificationDefault + + init { + if (enableTimeline) { + createLogger().info("Packet timeline is enabled.") + } + if (enablePayloadVerification) { + createLogger().info("Payload verification is enabled.") + } + } + } +} + +/** + * This is a specialization of [org.jitsi.nlj.util.forEachAs] method which makes it easier + * to operate on lists of [PacketInfo] when the caller wants to treat the contained [Packet] + * as a specific packet type. This method iterates over the iterable of [PacketInfo]s and calls + * the given lambda with the [PacketInfo] instance and the contained [Packet] instance, cast + * as [ExpectedPacketType]. This will throw if the cast attempt is unsuccessful. + */ +@Suppress("UNCHECKED_CAST") +inline fun Iterable.forEachAs(action: (PacketInfo, ExpectedPacketType) -> Unit) { + for (element in this) action(element, element.packet as ExpectedPacketType) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt new file mode 100644 index 0000000..2d062dc --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpEncodingDesc.kt @@ -0,0 +1,203 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.stats.NodeStatsBlock + +/** + * Keeps track of information specific to an RTP encoded stream + * (and its associated secondary sources). + * + * @author Jonathan Lennox + */ +class RtpEncodingDesc +@JvmOverloads +constructor( + /** + * The primary SSRC for this encoding. + */ + val primarySSRC: Long, + /** + * The [RtpLayerDesc]s describing the encoding's layers. + */ + initialLayers: Array, + /** + * The ID of this encoding. + */ + val eid: Int = requireNotNull(initialLayers.getOrNull(0)?.eid) { + "initialLayers may not be empty if no explicit EID is provided" + } +) { + constructor(primarySSRC: Long, eid: Int) : this(primarySSRC, arrayOf(), eid) + + /** + * The ssrcs associated with this encoding (for example, RTX or FLEXFEC) + * Maps ssrc -> type [SsrcAssociationType] (rtx, etc.) + */ + private val secondarySsrcs: MutableMap = HashMap() + + fun addSecondarySsrc(ssrc: Long, type: SsrcAssociationType) { + secondarySsrcs[ssrc] = type + } + + /** + * All SSRCs (primary and secondary) associated with this encoding. + */ + val ssrcs: Collection + get() = HashSet().also { set -> + set.add(primarySSRC) + set.addAll(secondarySsrcs.keys) + } + + private fun validateLayerEids(layers: Array) { + for (layer in layers) { + require(layer.eid == eid) { "Cannot add layer with EID ${layer.eid} to encoding with EID $eid" } + } + } + init { + validateLayerEids(initialLayers) + } + + private var nominalHeight = initialLayers.getNominalHeight() + + internal var layers = initialLayers + set(newLayers) { + validateLayerEids(newLayers) + /* Check if the new layer set is a single spatial layer that doesn't specify a height - if so, we + * want to apply the nominal height to them. + */ + val useNominalHeight = nominalHeight != RtpLayerDesc.NO_HEIGHT && + newLayers.all { it.sid == 0 } && + newLayers.all { it.height == RtpLayerDesc.NO_HEIGHT } + /* Copy the rate statistics objects from the old layers to the new layers + * with matching layer IDs. + */ + /* Note: because layer arrays are sorted by ID we could avoid creating this + * intermediate map object, and do this in a single pass in O(1) space. + * The number of layers is small enough that this more complicated code + * is probably unnecessary, though. + */ + val oldLayerMap = field.associateBy { it.layerId } + for (newLayer in newLayers) { + oldLayerMap[newLayer.layerId]?.let { + newLayer.inheritFrom(it) + } + if (useNominalHeight) { + newLayer.height = nominalHeight + } + } + if (!useNominalHeight) { + val newNominalHeight = newLayers.getNominalHeight() + if (newNominalHeight != RtpLayerDesc.NO_HEIGHT) { + nominalHeight = newNominalHeight + } + } + field = newLayers + } + + /** + * @return the "id" of a layer within this source, across all encodings. This is a server-side id and should + * not be confused with any encoding id defined in the client (such as the + * rid). This server-side id is used in the layer lookup table that is + * maintained in [MediaSourceDesc]. + */ + fun encodingId(layer: RtpLayerDesc): Long = calcEncodingId(primarySSRC, layer.layerId) + + /** + * Get the secondary ssrc for this encoding that corresponds to the given + * type + * @param type the type of the secondary ssrc (e.g. RTX) + * @return the ssrc for the encoding that corresponds to the given type, + * if it exists; otherwise -1 + */ + fun getSecondarySsrc(type: SsrcAssociationType): Long { + for ((key, value) in secondarySsrcs) { + if (value == type) { + return key + } + } + return -1 + } + + /** + * Clone an existing encoding desc, inheriting layer descs' statistics, + * modifying only specific values. + */ + fun copy( + primarySSRC: Long = this.primarySSRC, + layers: Array = Array(this.layers.size) { i -> this.layers[i].copy() } + ) = RtpEncodingDesc(primarySSRC, layers, eid).also { + this.secondarySsrcs.forEach { (ssrc, type) -> it.addSecondarySsrc(ssrc, type) } + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = "primary_ssrc=$primarySSRC,secondary_ssrcs=$secondarySsrcs," + + "layers=\n ${layers.joinToString(separator = "\n ")}" + + /** + * Gets a boolean indicating whether the SSRC specified in the + * arguments is used by this encoding. + * + * @param ssrc the SSRC to match. + */ + fun hasSsrc(ssrc: Long): Boolean = if (primarySSRC == ssrc) { + true + } else { + secondarySsrcs.containsKey(ssrc) + } + + /** + * Extracts a [NodeStatsBlock] from an [RtpEncodingDesc]. + */ + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("rtx_ssrc", getSecondarySsrc(SsrcAssociationType.RTX)) + put("fec_ssrc", getSecondarySsrc(SsrcAssociationType.FEC)) + put("eid", eid) + put("nominal_height", nominalHeight) + for (layer in layers) { + set(layer.indexString(), layer.debugState()) + } + } + + companion object { + fun calcEncodingId(ssrc: Long, layerId: Int) = ssrc or (layerId.toLong() shl 32) + } +} + +fun VideoRtpPacket.getEncodingIds(): Collection = this.layerIds.map { RtpEncodingDesc.calcEncodingId(ssrc, it) } + +/** + * Get the "nominal" height of a set of layers - if they all indicate the same spatial layer and same height. + */ +private fun Array.getNominalHeight(): Int { + if (isEmpty()) { + return RtpLayerDesc.NO_HEIGHT + } + val firstHeight = first().height + if (!(all { it.sid == 0 } || all { it.sid == -1 })) { + return RtpLayerDesc.NO_HEIGHT + } + if (any { it.height != firstHeight }) { + return RtpLayerDesc.NO_HEIGHT + } + return firstHeight +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt new file mode 100644 index 0000000..b75f9c4 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpLayerDesc.kt @@ -0,0 +1,280 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.transform.node.incoming.BitrateCalculator +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.DataSize + +/** + * Keeps track of its subjective quality index, + * its last stable bitrate and other useful things for adaptivity/routing. + * + * @author George Politis + */ +abstract class RtpLayerDesc( + /** + * The index of this instance's encoding in the source encoding array. + */ + val eid: Int, + /** + * The temporal layer ID of this instance. + */ + val tid: Int, + /** + * The spatial layer ID of this instance. + */ + val sid: Int, + /** + * The max "height" of the bitstream that this instance represents. The actual + * height may be less due to bad network or system load. [NO_HEIGHT] for unknown. + * + * In order to handle portrait-mode video correctly, this should actually be the smaller of + * the bitstream's height or width. + * + * Where possible we sniff the actual height from the RTP packets. + */ + var height: Int, + /** + * The max frame rate (in fps) of the bitstream that this instance + * represents. The actual frame rate may be less due to bad network or + * system load. [NO_FRAME_RATE] for unknown. + */ + var frameRate: Double, +) { + abstract fun copy(height: Int = this.height, tid: Int = this.tid, inherit: Boolean = true): RtpLayerDesc + + /** + * The [BitrateTracker] instance used to calculate the receiving bitrate of this RTP layer. + */ + protected var bitrateTracker = BitrateCalculator.createBitrateTracker() + + /** + * A second [BitrateTracker] with a longer window, for sources whose bitrate is bursty. It is only created once + * [enableSmoothedBitrate] is called, because it costs memory per layer and an update per packet per layer, and + * only screen sharing sources use it. See [BitrateCalculator.createSmoothedBitrateTracker]. + * + * Volatile because it is enabled from the threads which run bandwidth allocation (one per receiver, so possibly + * several at once), but read and updated from the thread which reads packets. + */ + @Volatile + private var smoothedBitrateTracker: BitrateTracker? = null + + /** + * The [clock] time at which [smoothedBitrateTracker] was created. Only meaningful while the tracker is non-null. + * Writes to it are made visible by the subsequent volatile write to [smoothedBitrateTracker]. + */ + private var smoothedBitrateTrackerCreatedMs: Long = -1 + + /** + * Start tracking a smoothed bitrate for this layer. Idempotent. Until the tracker created here has a full window + * of history, [getSmoothedBitrate] falls back to [getBitrate], so that a source is not priced at close to zero + * while the longer window fills up. + */ + fun enableSmoothedBitrate(nowMs: Long) { + if (smoothedBitrateTracker == null) { + synchronized(this) { + if (smoothedBitrateTracker == null) { + smoothedBitrateTrackerCreatedMs = nowMs + smoothedBitrateTracker = BitrateCalculator.createSmoothedBitrateTracker() + } + } + } + } + + /** + * Stop tracking a smoothed bitrate for this layer, dropping the tracker and its history. Called when the source + * stops being bursty (e.g. a screen share reverting to camera), so that the layer stops paying for the tracker, + * and so that a later re-enable does not price the source from the previous mode's traffic. + */ + fun disableSmoothedBitrate() { + if (smoothedBitrateTracker != null) { + synchronized(this) { + smoothedBitrateTracker = null + } + } + } + + /** + * The rate of this layer alone, either over the estimator-matched window or over the longer window used for + * bursty sources. The smoothed rate falls back to the other one while it is not being tracked, and while its + * tracker does not yet have a full window of history (since it always divides by the full window, a young tracker + * would under-report the rate, in the extreme pricing the source at zero at the moment it is enabled). + */ + protected fun layerRate(nowMs: Long, smoothed: Boolean): Bandwidth { + val smoothedTracker = if (smoothed) smoothedBitrateTracker else null + return if (smoothedTracker != null && nowMs - smoothedBitrateTrackerCreatedMs >= smoothedWindowMs) { + smoothedTracker.getRateOverFullWindow(nowMs) + } else { + bitrateTracker.getRate(nowMs) + } + } + + var targetBitrate: Bandwidth? = null + + /** + * @return the "id" of this layer within this encoding. This is a server-side id and should + * not be confused with any encoding id defined in the client (such as the + * rid). + */ + abstract val layerId: Int + + /** + * A local index of this track. + */ + abstract val index: Int + + /** + * Inherit a [BitrateTracker] object + */ + internal fun inheritStatistics(tracker: BitrateTracker) { + bitrateTracker = tracker + } + + /** + * Inherit another layer description's [BitrateTracker] object. + */ + internal open fun inheritFrom(other: RtpLayerDesc) { + inheritStatistics(other.bitrateTracker) + // Only take the other layer's smoothed tracker if it has one: layer arrays are replaced whenever the + // scalability structure is re-signaled, and dropping an established tracker would restart its window. + other.smoothedBitrateTracker?.let { + smoothedBitrateTrackerCreatedMs = other.smoothedBitrateTrackerCreatedMs + smoothedBitrateTracker = it + } + targetBitrate = other.targetBitrate + } + + /** + * + * @param packetSize + * @param nowMs + * @return true if this caused the rate being tracked to transition from zero to nonzero + */ + fun updateBitrate(packetSize: DataSize, nowMs: Long): Boolean { + val wasInactive = hasZeroBitrate(nowMs) + // Update rate stats (this should run after padding termination). + bitrateTracker.update(packetSize, nowMs) + smoothedBitrateTracker?.update(packetSize, nowMs) + return wasInactive && packetSize.bits > 0L + } + + /** + * Gets the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. + * + * @param nowMs + * @return the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. + */ + abstract fun getBitrate(nowMs: Long): Bandwidth + + /** + * Like [getBitrate], but measured over the longer window of [smoothedBitrateTracker]. Used for sources whose + * bitrate is bursty, where the instantaneous rate over-states the bandwidth the source needs. + */ + abstract fun getSmoothedBitrate(nowMs: Long): Bandwidth + + /** + * Recursively checks this layer and its dependencies to see if the bitrate is zero. + * Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead + * of the hash table is usually more than the cost of any double-visits. + */ + abstract fun hasZeroBitrate(nowMs: Long): Boolean + + open fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("frameRate", frameRate) + put("height", height) + put("index", index) + put("bitrate_bps", getBitrate(System.currentTimeMillis()).bps) + put("smoothed_bitrate_bps", getSmoothedBitrate(System.currentTimeMillis()).bps) + put("target_bitrate", targetBitrate?.bps ?: 0) + put("indexString", indexString()) + } + + abstract fun indexString(): String + + companion object { + /** The length of the smoothed trackers' window, cached (it is fixed by configuration). */ + private val smoothedWindowMs: Long by lazy { BitrateCalculator.smoothedWindowSize.toMillis() } + + /** + * The index value that is used to represent that forwarding is suspended. + */ + const val SUSPENDED_INDEX = -1 + + /** + * The encoding ID value that is used to represent that forwarding is suspended. + */ + const val SUSPENDED_ENCODING_ID = -1 + + /** + * A value used to designate the absence of height information. + */ + const val NO_HEIGHT = -1 + + /** + * A value used to designate the absence of frame rate information. + */ + const val NO_FRAME_RATE = -1.0 + + /** + * Calculate the "index" of a layer based on its encoding, spatial, and temporal ID. + * This is a server-side id and should not be confused with any encoding id defined + * in the client (such as the rid) or the encodingId. This is used by the videobridge's + * adaptive source projection for filtering. + */ + @JvmStatic + fun getIndex(eid: Int, sid: Int, tid: Int): Int { + val e = if (eid < 0) 0 else eid + val s = if (sid < 0) 0 else sid + val t = if (tid < 0) 0 else tid + + return (e shl 6) or (s shl 3) or t + } + + /** + * Get an encoding ID from a layer index. If the index is [SUSPENDED_INDEX], + * [SUSPENDED_ENCODING_ID] will be returned. + */ + @JvmStatic + fun getEidFromIndex(index: Int) = index shr 6 + + /** + * Get a spatial ID from a layer index. If the index is [SUSPENDED_INDEX], + * the value is unspecified. + */ + @JvmStatic + fun getSidFromIndex(index: Int) = (index and 0x38) shr 3 + + /** + * Get a temporal ID from a layer index. If the index is [SUSPENDED_INDEX], + * the value is unspecified. + */ + @JvmStatic + fun getTidFromIndex(index: Int) = index and 0x7 + + /** + * Get a string description of a layer index. + */ + fun indexString(index: Int): String = if (index == SUSPENDED_INDEX) { + "SUSP" + } else { + "E${getEidFromIndex(index)}S${getSidFromIndex(index)}T${getTidFromIndex(index)}" + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt new file mode 100644 index 0000000..944a77c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiver.kt @@ -0,0 +1,79 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.stats.EndpointConnectionStats +import org.jitsi.nlj.stats.RtpReceiverStats +import org.jitsi.nlj.util.Bandwidth + +abstract class RtpReceiver : + StatsKeepingPacketHandler(), + EventHandler, + Stoppable, + EndpointConnectionStats.EndpointConnectionStatsListener { + /** + * The handler which will be invoked for each RTP/RTCP packet received + * by this receiver (after it has gone through the receiver's + * input chain). + */ + abstract var packetHandler: PacketHandler? + + /** + * Enqueue an incoming packet to be processed + */ + abstract fun enqueuePacket(p: PacketInfo) + + abstract fun debugState(mode: DebugStateMode): ObjectNode + + /** + * Set the SRTP transformers to be used for RTP/RTCP encryption and decryption + */ + abstract fun setSrtpTransformers(srtpTransformers: SrtpTransformers) + + abstract fun getStats(): RtpReceiverStats + + abstract fun tearDown() + + abstract fun isReceivingAudio(): Boolean + abstract fun isReceivingVideo(): Boolean + + abstract fun addLossListener(lossListener: LossListener) + + abstract fun setFeature(feature: Features, enabled: Boolean) + abstract fun isFeatureEnabled(feature: Features): Boolean + + /** + * Forcibly mute or unmute the incoming audio stream + */ + abstract fun forceMuteAudio(shouldMute: Boolean) + + abstract fun forceMuteVideo(shouldMute: Boolean) +} + +interface RtpReceiverEventHandler { + /** + * We received an audio level indication from the remote endpoint. + */ + fun audioLevelReceived(sourceSsrc: Long, level: Long): Boolean = false + + /** + * The estimation of the available send bandwidth changed. + */ + fun bandwidthEstimationChanged(newValue: Bandwidth) {} +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt new file mode 100644 index 0000000..5309efe --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpReceiverImpl.kt @@ -0,0 +1,355 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.nlj.rtcp.CompoundRtcpParser +import org.jitsi.nlj.rtcp.RembHandler +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtcp.RtcpRrGenerator +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.stats.RtpReceiverStats +import org.jitsi.nlj.transform.NodeDebugStateVisitor +import org.jitsi.nlj.transform.NodeEventVisitor +import org.jitsi.nlj.transform.NodeTeardownVisitor +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.transform.node.PacketLossConfig +import org.jitsi.nlj.transform.node.PacketLossNode +import org.jitsi.nlj.transform.node.PacketStreamStatsNode +import org.jitsi.nlj.transform.node.RtpParser +import org.jitsi.nlj.transform.node.SrtcpDecryptNode +import org.jitsi.nlj.transform.node.SrtpDecryptNode +import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.node.incoming.AudioLevelReader +import org.jitsi.nlj.transform.node.incoming.BitrateCalculator +import org.jitsi.nlj.transform.node.incoming.DiscardableDiscarder +import org.jitsi.nlj.transform.node.incoming.DuplicateTermination +import org.jitsi.nlj.transform.node.incoming.IncomingStatisticsTracker +import org.jitsi.nlj.transform.node.incoming.PaddingTermination +import org.jitsi.nlj.transform.node.incoming.RemoteBandwidthEstimator +import org.jitsi.nlj.transform.node.incoming.RetransmissionRequesterNode +import org.jitsi.nlj.transform.node.incoming.RtcpTermination +import org.jitsi.nlj.transform.node.incoming.RtxHandler +import org.jitsi.nlj.transform.node.incoming.TccGeneratorNode +import org.jitsi.nlj.transform.node.incoming.VideoBitrateCalculator +import org.jitsi.nlj.transform.node.incoming.VideoMuteNode +import org.jitsi.nlj.transform.node.incoming.VideoParser +import org.jitsi.nlj.transform.node.incoming.VideoQualityLayerLookup +import org.jitsi.nlj.transform.node.incoming.VlaReaderNode +import org.jitsi.nlj.transform.packetPath +import org.jitsi.nlj.transform.pipeline +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.PacketInfoQueue +import org.jitsi.nlj.util.PacketPredicate +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.Packet +import org.jitsi.rtp.extensions.looksLikeRtcp +import org.jitsi.rtp.extensions.looksLikeRtp +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.queue.CountingErrorHandler +import java.util.concurrent.ExecutorService +import java.util.concurrent.ScheduledExecutorService + +class RtpReceiverImpl @JvmOverloads constructor( + val id: String, + /** + * A function to be used when the receiver wants to send RTCP packets to the + * participant it's receiving data from (NACK packets, for example) + */ + private val rtcpSender: (RtcpPacket) -> Unit = {}, + rtcpEventNotifier: RtcpEventNotifier, + /** + * The executor this class will use for its primary work (i.e. critical path + * packet processing). This [RtpReceiver] will execute a blocking queue read + * on this executor. + */ + private val executor: ExecutorService, + /** + * A [ScheduledExecutorService] which can be used for less important + * background tasks, or tasks that need to execute at some fixed delay/rate + */ + private val backgroundExecutor: ScheduledExecutorService, + streamInformationStore: ReadOnlyStreamInformationStore, + private val eventHandler: RtpReceiverEventHandler, + parentLogger: Logger, + diagnosticContext: DiagnosticContext = DiagnosticContext() +) : RtpReceiver() { + private val logger = createChildLogger(parentLogger) + private var running: Boolean = true + private val inputTreeRoot: Node + private val incomingPacketQueue = PacketInfoQueue( + "rtp-receiver-incoming-packet-queue", + executor, + this::handleIncomingPacket, + queueSize + ) + private val srtpDecryptWrapper = SrtpDecryptNode() + private val srtcpDecryptWrapper = SrtcpDecryptNode() + private val tccGenerator = TccGeneratorNode(rtcpSender, streamInformationStore, logger) + private val remoteBandwidthEstimator = RemoteBandwidthEstimator(streamInformationStore, logger, diagnosticContext) + private val audioLevelReader = AudioLevelReader(streamInformationStore).apply { + audioLevelListener = object : AudioLevelListener { + override fun onLevelReceived(sourceSsrc: Long, level: Long): Boolean = + eventHandler.audioLevelReceived(sourceSsrc, level) + } + } + + private val videoMuteNode = VideoMuteNode() + + private val silenceDiscarder = DiscardableDiscarder("Silence discarder", false) + private val paddingOnlyDiscarder = DiscardableDiscarder("Padding-only discarder", true) + private val statsTracker = IncomingStatisticsTracker(streamInformationStore) + private val packetStreamStats = PacketStreamStatsNode(diagnosticContext, "receive") + private val rtcpRrGenerator = RtcpRrGenerator(backgroundExecutor, rtcpSender, statsTracker) { + remoteBandwidthEstimator.createRemb()?.let { + listOf(it) + } ?: emptyList() + } + private val rtcpTermination = RtcpTermination(rtcpEventNotifier, logger) + private val retransmissionRequester = RetransmissionRequesterNode(rtcpSender, backgroundExecutor, logger) + private val rembHandler = RembHandler(streamInformationStore, logger).apply { + addListener(object : TransportCcEngine.BandwidthListener { + override fun bandwidthEstimationChanged(newValue: Bandwidth) { + eventHandler.bandwidthEstimationChanged(newValue) + } + }) + } + private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-rx") + private val videoBitrateCalculator = VideoBitrateCalculator(parentLogger) + private val audioBitrateCalculator = BitrateCalculator("Audio bitrate calculator") + + private val videoParser = VideoParser(streamInformationStore, logger, diagnosticContext) + + override fun isReceivingAudio() = audioBitrateCalculator.active + override fun isReceivingVideo() = videoBitrateCalculator.active + override fun addLossListener(lossListener: LossListener) { + tccGenerator.addLossListener(lossListener) + } + + companion object { + var queueErrorCounter = CountingErrorHandler() + + private const val PACKET_QUEUE_ENTRY_EVENT = "Entered RTP receiver incoming queue" + private const val PACKET_QUEUE_EXIT_EVENT = "Exited RTP receiver incoming queue" + + private val queueSize: Int by config("jmt.transceiver.recv.queue-size".from(JitsiConfig.newConfig)) + + /** + * Configuration for the packet loss to introduce in the receive pipeline (for debugging/testing purposes). + */ + private val packetLossConfig = PacketLossConfig("jmt.debug.packet-loss.incoming") + } + + /** + * [packetHandler] will be invoked with RTP packets that have made + * it through the entire receive pipeline. Some external entity should + * assign it to a [PacketHandler] with appropriate logic. + */ + override var packetHandler: PacketHandler? = null + + /** + * The [packetHandler] can be re-assigned at any time, but it should maintain + * its place in the receive pipeline. To support both keeping it in the same + * place and allowing it to be re-assigned, we wrap it with this. + */ + private val packetHandlerWrapper = object : ConsumerNode("Input pipeline termination node") { + override fun consume(packetInfo: PacketInfo) { + // When there's no handler set we're effectively dropping packets, so their buffers + // should be returned. + packetHandler?.processPacket(packetInfo) ?: packetDiscarded(packetInfo) + } + + override val aggregationKey: String = name + + override fun trace(f: () -> Unit) = f.invoke() + } + + init { + logger.cdebug { "using executor ${executor.hashCode()}" } + + if (packetLossConfig.enabled) { + logger.warn("Will simulate packet loss: $packetLossConfig") + } + + rtcpEventNotifier.addRtcpEventListener(rtcpRrGenerator) + rtcpEventNotifier.addRtcpEventListener(rembHandler) + + incomingPacketQueue.setErrorHandler(queueErrorCounter) + + inputTreeRoot = pipeline { + node(packetStreamStats) + demux("SRTP/SRTCP") { + packetPath { + name = "SRTP" + predicate = PacketPredicate(Packet::looksLikeRtp) + path = pipeline { + node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled }) + node(RtpParser(streamInformationStore, logger)) + // TODO: temporarily putting the audioLevelReader node here such that we can determine whether + // or not a packet should be discarded before doing SRTP. audioLevelReader has been moved here + // (instead of introducing a different class to read audio levels) to avoid parsing the RTP + // header extensions twice (which is expensive). In the future we will parse and cache the + // header extensions to make this lookup more efficient, at which time we could move + // audioLevelReader back to where it was (in the audio path) and add a new node here which would + // check for different discard conditions (i.e. checking the audio level for silence) + node(audioLevelReader.preDecryptNode) + node(videoMuteNode) + node(srtpDecryptWrapper) + node(tccGenerator) + node(remoteBandwidthEstimator) + // This reads audio levels from packets that use cryptex. TODO: should it go in the Audio path? + node(audioLevelReader.postDecryptNode) + node(toggleablePcapWriter.newObserverNode(outbound = false, suffix = "rx_rtp")) + node(statsTracker) + node(PaddingTermination(logger)) + demux("Media Type") { + packetPath { + name = "Audio" + predicate = PacketPredicate { it is AudioRtpPacket } + path = pipeline { + node(silenceDiscarder) + node(audioBitrateCalculator) + node(packetHandlerWrapper) + } + } + packetPath { + name = "Video" + predicate = PacketPredicate { it is VideoRtpPacket } + path = pipeline { + node(RtxHandler(streamInformationStore, logger)) + node(DuplicateTermination()) + node(retransmissionRequester) + node(paddingOnlyDiscarder) + node(videoParser) + node(VideoQualityLayerLookup(logger)) + node(videoBitrateCalculator) + node(VlaReaderNode(streamInformationStore, logger)) + node(packetHandlerWrapper) + } + } + } + } + } + packetPath { + name = "SRTCP" + predicate = PacketPredicate(Packet::looksLikeRtcp) + path = pipeline { + node(srtcpDecryptWrapper) + node(toggleablePcapWriter.newObserverNode(outbound = false, suffix = "rx_rtcp")) + node(CompoundRtcpParser(logger)) + node(rtcpTermination) + node(packetHandlerWrapper) + } + } + } + } + } + + private fun handleIncomingPacket(packet: PacketInfo): Boolean = if (running) { + packet.addEvent(PACKET_QUEUE_EXIT_EVENT) + processPacket(packet) + true + } else { + BufferPool.returnBuffer(packet.packet.buffer) + false + } + + override fun doProcessPacket(packetInfo: PacketInfo) = inputTreeRoot.processPacket(packetInfo) + + override fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + NodeDebugStateVisitor(this, mode).visit(inputTreeRoot) + } + + override fun enqueuePacket(p: PacketInfo) { +// logger.cinfo { "Receiver $id enqueing data" } + if (running) { + p.addEvent(PACKET_QUEUE_ENTRY_EVENT) + incomingPacketQueue.add(p) + } else { + BufferPool.returnBuffer(p.packet.buffer) + } + } + + override fun setSrtpTransformers(srtpTransformers: SrtpTransformers) { + srtpDecryptWrapper.transformer = srtpTransformers.srtpDecryptTransformer + srtcpDecryptWrapper.transformer = srtpTransformers.srtcpDecryptTransformer + } + + override fun handleEvent(event: Event) { + NodeEventVisitor(event).visit(inputTreeRoot) + } + + override fun getStats(): RtpReceiverStats = RtpReceiverStats( + incomingStats = statsTracker.getSnapshot(), + packetStreamStats = packetStreamStats.snapshot(), + videoParserStats = videoParser.getStats() + ) + + override fun forceMuteAudio(shouldMute: Boolean) { + audioLevelReader.forceMute = shouldMute + } + + override fun forceMuteVideo(shouldMute: Boolean) { + videoMuteNode.forceMute = shouldMute + } + + override fun setFeature(feature: Features, enabled: Boolean) { + when (feature) { + Features.TRANSCEIVER_PCAP_DUMP -> { + if (enabled) { + toggleablePcapWriter.enable() + } else { + toggleablePcapWriter.disable() + } + } + } + } + + override fun isFeatureEnabled(feature: Features): Boolean = when (feature) { + Features.TRANSCEIVER_PCAP_DUMP -> toggleablePcapWriter.isEnabled() + } + + override fun stop() { + running = false + rtcpRrGenerator.running = false + retransmissionRequester.stop() + } + + override fun tearDown() { + logger.debug("Tearing down") + NodeTeardownVisitor().visit(inputTreeRoot) + incomingPacketQueue.close() + toggleablePcapWriter.disable() + } + + override fun onRttUpdate(newRttMs: Double) { + remoteBandwidthEstimator.onRttUpdate(newRttMs) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt new file mode 100644 index 0000000..606f84e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSender.kt @@ -0,0 +1,59 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.stats.EndpointConnectionStats +import org.jitsi.nlj.stats.PacketStreamStats +import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsSnapshot + +/** + * Not an 'RtpSender' in the sense that it sends only RTP (and not + * RTCP) but in the sense of a webrtc 'RTCRTPSender' which handles + * all RTP and RTP control packets. + */ +abstract class RtpSender : + StatsKeepingPacketHandler(), + EventHandler, + Stoppable, + EndpointConnectionStats.EndpointConnectionStatsListener { + + abstract fun sendProbing(mediaSsrcs: Collection, numBytes: Int): Int + abstract fun onOutgoingPacket(handler: PacketHandler) + abstract fun setSrtpTransformers(srtpTransformers: SrtpTransformers) + abstract fun getStreamStats(): OutgoingStatisticsSnapshot + abstract fun getPacketStreamStats(): PacketStreamStats.Snapshot + abstract fun addBandwidthListener(listener: TransportCcEngine.BandwidthListener) + abstract fun removeBandwidthListener(listener: TransportCcEngine.BandwidthListener) + abstract fun getTransportCcEngineStats(): TransportCcEngine.StatisticsSnapshot + abstract fun requestKeyframe(requesterID: String?, mediaSsrc: Long? = null) + abstract fun addLossListener(lossListener: LossListener) + abstract fun setFeature(feature: Features, enabled: Boolean) + abstract fun isFeatureEnabled(feature: Features): Boolean + abstract fun tearDown() + abstract fun addRtpExtensionToRetain(extensionType: RtpExtensionType) + + /** + * An optional function to be executed for each RTP packet, as the first step of the send pipeline. + */ + var preProcesor: ((PacketInfo) -> PacketInfo?)? = null + + abstract fun debugState(mode: DebugStateMode): ObjectNode +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt new file mode 100644 index 0000000..d8ecba8 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/RtpSenderImpl.kt @@ -0,0 +1,384 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.nlj.rtcp.KeyframeRequester +import org.jitsi.nlj.rtcp.NackHandler +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtcp.RtcpSrUpdater +import org.jitsi.nlj.rtp.ClassicTransportCcEngine +import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorConfig +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorEngine +import org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator +import org.jitsi.nlj.rtp.bandwidthestimation2.GoogCcTransportCcEngine +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.transform.NodeDebugStateVisitor +import org.jitsi.nlj.transform.NodeEventVisitor +import org.jitsi.nlj.transform.NodeTeardownVisitor +import org.jitsi.nlj.transform.node.AudioRedHandler +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.transform.node.PacketCacher +import org.jitsi.nlj.transform.node.PacketLossConfig +import org.jitsi.nlj.transform.node.PacketLossNode +import org.jitsi.nlj.transform.node.PacketStreamStatsNode +import org.jitsi.nlj.transform.node.PluggableTransformerNode +import org.jitsi.nlj.transform.node.SrtcpEncryptNode +import org.jitsi.nlj.transform.node.SrtpEncryptNode +import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.node.outgoing.AbsSendTime +import org.jitsi.nlj.transform.node.outgoing.HeaderExtEncoder +import org.jitsi.nlj.transform.node.outgoing.HeaderExtStripper +import org.jitsi.nlj.transform.node.outgoing.MidStamper +import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsTracker +import org.jitsi.nlj.transform.node.outgoing.ProbingDataSender +import org.jitsi.nlj.transform.node.outgoing.RetransmissionSender +import org.jitsi.nlj.transform.node.outgoing.SentRtcpStats +import org.jitsi.nlj.transform.node.outgoing.TccSeqNumTagger +import org.jitsi.nlj.transform.pipeline +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.PacketInfoQueue +import org.jitsi.nlj.util.StreamInformationStore +import org.jitsi.nlj.util.appendAll +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.queue.CountingErrorHandler +import java.time.Duration +import java.util.concurrent.ExecutorService +import java.util.concurrent.ScheduledExecutorService + +class RtpSenderImpl( + val id: String, + private val rtcpEventNotifier: RtcpEventNotifier, + /** + * The executor this class will use for its primary work (i.e. critical path + * packet processing). This [RtpSender] will execute a blocking queue read + * on this executor. + */ + val executor: ExecutorService, + /** + * A [ScheduledExecutorService] which can be used for less important + * background tasks, or tasks that need to execute at some fixed delay/rate + */ + val backgroundExecutor: ScheduledExecutorService, + private val streamInformationStore: StreamInformationStore, + parentLogger: Logger, + diagnosticContext: DiagnosticContext = DiagnosticContext(), + /** + * Resolves the sdes:mid to stamp on an outgoing packet given its (rewritten) SSRC, or null to not stamp a mid. + * Used under SSRC rewriting for mid-based demuxing on the receiving client. + */ + private val getMidBySsrc: (Long) -> String? = { null } +) : RtpSender() { + private val logger = createChildLogger(parentLogger) + private val outgoingRtpRoot: Node + private val outgoingRtxRoot: Node + private val outgoingRtcpRoot: Node + private val incomingPacketQueue = PacketInfoQueue( + "rtp-sender-incoming-packet-queue", + executor, + this::handlePacket, + queueSize + ) + var running = true + private var localVideoSsrc: Long? = null + private var localAudioSsrc: Long? = null + + // TODO(brian): this is changed to a handler instead of a queue because we want to use + // a PacketQueue, and the handler for a PacketQueue must be set at the time of creation. + // since we want the handler to be another entity (something in jvb) we just use + // a generic handler here and then the bridge can put it into its PacketQueue and have + // its handler (likely in another thread) grab the packet and send it out + private var outgoingPacketHandler: PacketHandler? = null + + private val transportCcEngine: TransportCcEngine = + when (BandwidthEstimatorConfig.engine) { + BandwidthEstimatorEngine.GoogleCc -> + ClassicTransportCcEngine(GoogleCcEstimator(diagnosticContext, logger), logger) + + BandwidthEstimatorEngine.GoogleCc2 -> + GoogCcTransportCcEngine(diagnosticContext, logger, backgroundExecutor, { dataSize, probingData -> + return@GoogCcTransportCcEngine probingDataSender.sendProbing( + null, + dataSize.bytes.toInt(), + probingData + ) + }) + }.also { it.start() } + + private val srtpEncryptWrapper = SrtpEncryptNode() + private val srtcpEncryptWrapper = SrtcpEncryptNode() + private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-tx") + private val outgoingPacketCache = PacketCacher() + private val headerExtensionStripper = HeaderExtStripper(streamInformationStore) + private val absSendTime = AbsSendTime(streamInformationStore) + private val statsTracker = OutgoingStatisticsTracker(diagnosticContext) + private val packetStreamStats = PacketStreamStatsNode(diagnosticContext, "send") + private val rtcpSrUpdater = RtcpSrUpdater(statsTracker) + private val keyframeRequester = KeyframeRequester(streamInformationStore, logger) + private val probingDataSender: ProbingDataSender + + private val nackHandler: NackHandler + + private val outputPipelineTerminationNode = object : ConsumerNode("Output pipeline termination node") { + override fun consume(packetInfo: PacketInfo) { + // While there's no handler set we're effectively dropping packets, so their buffers + // should be returned. + outgoingPacketHandler?.processPacket(packetInfo) ?: packetDiscarded(packetInfo) + } + + override val aggregationKey = name + + override fun trace(f: () -> Unit) = f.invoke() + } + + init { + logger.cdebug { "Sender $id using executor ${executor.hashCode()}" } + + if (packetLossConfig.enabled) { + logger.warn("Will simulate packet loss: $packetLossConfig") + } + + incomingPacketQueue.setErrorHandler(queueErrorCounter) + + outgoingRtpRoot = pipeline { + node(PluggableTransformerNode("RTP pre-processor") { preProcesor }) + node(AudioRedHandler(streamInformationStore, logger)) + node(headerExtensionStripper) + node(outgoingPacketCache) + node(absSendTime) + // Placed right after absSendTime, which is where the RTX pipeline joins the main pipeline, so that both + // media packets and their retransmissions get the mid stamped. + node(MidStamper(streamInformationStore, getMidBySsrc)) + node(statsTracker) + node(TccSeqNumTagger(transportCcEngine, streamInformationStore)) + node(HeaderExtEncoder(streamInformationStore, logger)) + node(toggleablePcapWriter.newObserverNode(outbound = true, suffix = "tx_rtp")) + node(srtpEncryptWrapper) + node(packetStreamStats.createNewNode()) + node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled }) + node(outputPipelineTerminationNode) + } + + outgoingRtxRoot = pipeline { + node(RetransmissionSender(streamInformationStore, logger)) + // We want RTX packets to hook into the main RTP pipeline starting at AbsSendTime + node(absSendTime) + } + + nackHandler = NackHandler(outgoingPacketCache.getPacketCache(), outgoingRtxRoot, logger) + rtcpEventNotifier.addRtcpEventListener(nackHandler) + rtcpEventNotifier.addRtcpEventListener(transportCcEngine) + + // TODO: are we setting outgoing rtcp sequence numbers correctly? just add a simple node here to rewrite them + outgoingRtcpRoot = pipeline { + node(PluggableTransformerNode("RTCP pre-processor") { preProcesor }) + node(keyframeRequester) + node(SentRtcpStats()) + // TODO(brian): not sure this is a great idea. it works as a catch-call but can also be error-prone + // (i found i was accidentally clobbering the sender ssrc for SRs which caused issues). I think + // it'd be better to notify everything creating RTCP the bridge SSRCs and then everything should be + // responsible for setting it themselves + simpleNode("RTCP sender ssrc setter") { packetInfo -> + val senderSsrc = localVideoSsrc ?: return@simpleNode packetInfo + val rtcpPacket = packetInfo.packetAs() + if (rtcpPacket.senderSsrc == 0L) { + rtcpPacket.senderSsrc = senderSsrc + } + packetInfo + } + node(rtcpSrUpdater) + node(toggleablePcapWriter.newObserverNode(outbound = true, suffix = "tx_rtcp")) + node(object : ObserverNode("RTCP sent notifier") { + override fun observe(packetInfo: PacketInfo) { + val packet = packetInfo.packet + if (packet is RtcpPacket) { + rtcpEventNotifier.notifyRtcpSent(packet) + } + } + + override fun trace(f: () -> Unit) {} + }) + node(srtcpEncryptWrapper) + node(packetStreamStats.createNewNode()) + node(PacketLossNode(packetLossConfig), condition = { packetLossConfig.enabled }) + node(outputPipelineTerminationNode) + } + + probingDataSender = ProbingDataSender( + packetCache = outgoingPacketCache.getPacketCache(), + rtxDataSender = outgoingRtxRoot, + garbageDataSender = absSendTime, + diagnosticContext = diagnosticContext, + streamInformationStore = streamInformationStore, + parentLogger = logger + ) + } + + override fun onRttUpdate(newRttMs: Double) { + nackHandler.onRttUpdate(newRttMs) + keyframeRequester.onRttUpdate(newRttMs) + transportCcEngine.onRttUpdate(Duration.ofNanos((newRttMs * 1e6).toLong())) + } + + /** + * Insert packets into the incoming packet queue + */ + override fun doProcessPacket(packetInfo: PacketInfo) { + if (running) { + packetInfo.addEvent(PACKET_QUEUE_ENTRY_EVENT) + incomingPacketQueue.add(packetInfo) + } else { + BufferPool.returnBuffer(packetInfo.packet.buffer) + } + } + + override fun sendProbing(mediaSsrcs: Collection, numBytes: Int): Int = + probingDataSender.sendProbing(mediaSsrcs, numBytes, null) + + override fun onOutgoingPacket(handler: PacketHandler) { + outgoingPacketHandler = handler + } + + override fun setSrtpTransformers(srtpTransformers: SrtpTransformers) { + srtpEncryptWrapper.transformer = srtpTransformers.srtpEncryptTransformer + srtcpEncryptWrapper.transformer = srtpTransformers.srtcpEncryptTransformer + } + + override fun requestKeyframe(requesterID: String?, mediaSsrc: Long?) { + keyframeRequester.requestKeyframe(requesterID, mediaSsrc) + } + + override fun addLossListener(lossListener: LossListener) { + transportCcEngine.addLossListener(lossListener) + } + + override fun setFeature(feature: Features, enabled: Boolean) { + when (feature) { + Features.TRANSCEIVER_PCAP_DUMP -> { + if (enabled) { + toggleablePcapWriter.enable() + } else { + toggleablePcapWriter.disable() + } + } + } + } + + override fun isFeatureEnabled(feature: Features): Boolean = when (feature) { + Features.TRANSCEIVER_PCAP_DUMP -> toggleablePcapWriter.isEnabled() + } + + /** + * Handles packets that have gone through the incoming queue and sends them + * through the sender pipeline + */ + private fun handlePacket(packetInfo: PacketInfo): Boolean = if (running) { + packetInfo.addEvent(PACKET_QUEUE_EXIT_EVENT) + + val root = when (packetInfo.packet) { + is RtcpPacket -> outgoingRtcpRoot + else -> outgoingRtpRoot + } + root.processPacket(packetInfo) + true + } else { + BufferPool.returnBuffer(packetInfo.packet.buffer) + false + } + + override fun getStreamStats() = statsTracker.getSnapshot() + + override fun getPacketStreamStats() = packetStreamStats.snapshot() + + override fun getTransportCcEngineStats() = transportCcEngine.getStatistics() + + override fun addBandwidthListener(listener: TransportCcEngine.BandwidthListener) = + transportCcEngine.addBandwidthListener(listener) + + override fun removeBandwidthListener(listener: TransportCcEngine.BandwidthListener) = + transportCcEngine.removeBandwidthListener(listener) + + override fun handleEvent(event: Event) { + when (event) { + is SetLocalSsrcEvent -> { + when (event.mediaType) { + MediaType.VIDEO -> localVideoSsrc = event.ssrc + MediaType.AUDIO -> localAudioSsrc = event.ssrc + else -> {} + } + } + } + NodeEventVisitor(event).reverseVisit(outputPipelineTerminationNode) + probingDataSender.handleEvent(event) + } + + override fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + if (mode == DebugStateMode.FULL) { + appendAll(super.getNodeStats().toJson()) + set("packet_queue", incomingPacketQueue.debugState) + put("running", running) + } + appendAll(nackHandler.getNodeStats().toJson()) + set("probing_data_sender", probingDataSender.debugState(mode)) + put("local_video_ssrc", localVideoSsrc?.toString() ?: "null") + put("local_audio_ssrc", localAudioSsrc?.toString() ?: "null") + set("transport_cc_engine", transportCcEngine.getStatistics().toJson()) + NodeDebugStateVisitor(this, mode).reverseVisit(outputPipelineTerminationNode) + } + + override fun stop() { + running = false + } + + override fun tearDown() { + logger.debug("Tearing down") + NodeTeardownVisitor().reverseVisit(outputPipelineTerminationNode) + incomingPacketQueue.close() + toggleablePcapWriter.disable() + } + + override fun addRtpExtensionToRetain(extensionType: RtpExtensionType) { + headerExtensionStripper.addRtpExtensionToRetain(extensionType) + } + + companion object { + var queueErrorCounter = CountingErrorHandler() + + private const val PACKET_QUEUE_ENTRY_EVENT = "Entered RTP sender incoming queue" + private const val PACKET_QUEUE_EXIT_EVENT = "Exited RTP sender incoming queue" + + private val queueSize: Int by config("jmt.transceiver.send.queue-size".from(JitsiConfig.newConfig)) + + /** + * Configuration for the packet loss to introduce in the send pipeline (for debugging/testing purposes). + */ + private val packetLossConfig = PacketLossConfig("jmt.debug.packet-loss.outgoing") + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Stoppable.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Stoppable.kt new file mode 100644 index 0000000..465453a --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Stoppable.kt @@ -0,0 +1,20 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +interface Stoppable { + fun stop() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt new file mode 100644 index 0000000..f6d7bef --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/Transceiver.kt @@ -0,0 +1,397 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.nlj.stats.EndpointConnectionStats +import org.jitsi.nlj.stats.PacketIOActivity +import org.jitsi.nlj.stats.TransceiverStats +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.LocalSsrcAssociation +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.SsrcAssociation +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cinfo +import org.jitsi.utils.logging2.createChildLogger +import java.time.Clock +import java.util.concurrent.ExecutorService +import java.util.concurrent.ScheduledExecutorService + +// This is an API class, so its usages will largely be outside of this library +/** + * Handles all packets (incoming and outgoing) for a particular stream. + * (TODO: 'stream' defined as what, exactly, here?) + * Handles the DTLS negotiation + * + * Incoming packets should be written via [handleIncomingPacket]. Outgoing + * packets are put in [outgoingQueue] (and should be read by something) + * TODO: maybe we want to have this 'push' the outgoing packets somewhere + * else instead (then we could have all senders push to a single queue and + * have the one thread just read from the queue and send, rather than that thread + * having to read from a bunch of individual queues) + */ +@Suppress("unused") +class Transceiver( + private val id: String, + receiverExecutor: ExecutorService, + senderExecutor: ExecutorService, + /** + * A [ScheduledExecutorService] which can be used for less important + * background tasks, or tasks that need to execute at some fixed delay/rate + */ + backgroundExecutor: ScheduledExecutorService, + diagnosticContext: DiagnosticContext, + parentLogger: Logger, + /** + * The handler for events coming out of this [Transceiver]. Note that these events are fired synchronously, + * potentially in one of the threads in the CPU pool, and it is up to the user of the library to handle the + * transition to another thread if necessary. + */ + private val eventHandler: TransceiverEventHandler, + private val clock: Clock = Clock.systemUTC(), + /** + * Resolves the sdes:mid to stamp on an outgoing packet given its (rewritten) SSRC, or null to not stamp a mid. + */ + getMidBySsrc: (Long) -> String? = { null } +) : Stoppable { + private val logger = createChildLogger(parentLogger) + val packetIOActivity = PacketIOActivity() + private val endpointConnectionStats = EndpointConnectionStats(logger) + private val streamInformationStore = StreamInformationStoreImpl() + val readOnlyStreamInformationStore: ReadOnlyStreamInformationStore = streamInformationStore + + /** + * A central place to subscribe to be notified on the reception or transmission of RTCP packets for + * this transceiver. This is intended to be used by internal entities: mainly logic for things like generating + * SRs and RRs and calculating RTT. Since it is used for both send and receive, it is held here and passed to + * the sender and receive so each can push or subscribe to updates. + */ + val rtcpEventNotifier = RtcpEventNotifier() + + private var mediaSources = MediaSources() + + var srtpTransformers: SrtpTransformers? = null + + /** Whether the srtpTransformers were created inside this object, or passed in externally. + * For external transformers it's the external owner's responsibility to close them. + */ + var internalTransformers = false + + /** + * Whether this [Transceiver] is receiving audio from the remote endpoint. + */ + fun isReceivingAudio(): Boolean = rtpReceiver.isReceivingAudio() + + /** + * Whether this [Transceiver] is receiving video from the remote endpoint. + */ + fun isReceivingVideo(): Boolean = rtpReceiver.isReceivingVideo() + + val rtpSender: RtpSender = RtpSenderImpl( + id, + rtcpEventNotifier, + senderExecutor, + backgroundExecutor, + streamInformationStore, + logger, + diagnosticContext, + getMidBySsrc + ) + private val rtpReceiver: RtpReceiver = + RtpReceiverImpl( + id, + { rtcpPacket -> + if (rtcpPacket.length >= 1500) { + logger.warn( + "Sending large locally-generated RTCP packet of size ${rtcpPacket.length}, " + + "first packet of type ${rtcpPacket.packetType} rc ${rtcpPacket.reportCount}." + ) + } + rtpSender.processPacket(PacketInfo(rtcpPacket)) + }, + rtcpEventNotifier, + receiverExecutor, + backgroundExecutor, + streamInformationStore, + eventHandler, + logger, + diagnosticContext + ) + + init { + rtpSender.addBandwidthListener( + object : TransportCcEngine.BandwidthListener { + override fun bandwidthEstimationChanged(newValue: Bandwidth) { + eventHandler.bandwidthEstimationChanged(newValue) + } + } + ) + + rtpReceiver.addLossListener(endpointConnectionStats.incomingLossTracker) + rtpSender.addLossListener(endpointConnectionStats.outgoingLossTracker) + + rtcpEventNotifier.addRtcpEventListener(endpointConnectionStats) + + endpointConnectionStats.addListener(rtpSender) + endpointConnectionStats.addListener(rtpReceiver) + } + + /** + * Handle an incoming [PacketInfo] (that is, a packet received by the endpoint + * this transceiver is associated with) to be processed by the receiver pipeline. + */ + fun handleIncomingPacket(p: PacketInfo) { + packetIOActivity.lastRtpPacketReceivedInstant = clock.instant() + rtpReceiver.enqueuePacket(p) + } + + /** + * Send packets to the endpoint this transceiver is associated with by + * passing them out the sender's outgoing pipeline + */ + fun sendPacket(packetInfo: PacketInfo) { + packetIOActivity.lastRtpPacketSentInstant = clock.instant() + rtpSender.processPacket(packetInfo) + } + + fun sendProbing(mediaSsrcs: Collection, numBytes: Int): Int = rtpSender.sendProbing(mediaSsrcs, numBytes) + + /** + * Set a handler to be invoked when incoming RTP packets have finished + * being processed. + */ + fun setIncomingPacketHandler(rtpHandler: PacketHandler) { + rtpReceiver.packetHandler = rtpHandler + } + + /** + * Set a handler to be invoked when outgoing packets have finished + * being processed (and are ready to be sent) + */ + fun setOutgoingPacketHandler(outgoingPacketHandler: PacketHandler) { + rtpSender.onOutgoingPacket(outgoingPacketHandler) + } + + fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) { + logger.cdebug { "${hashCode()} adding receive ssrc $ssrc of type $mediaType" } + streamInformationStore.addReceiveSsrc(ssrc, mediaType) + } + + fun removeReceiveSsrc(ssrc: Long) { + logger.cinfo { "Transceiver ${hashCode()} removing receive ssrc $ssrc" } + streamInformationStore.removeReceiveSsrc(ssrc) + } + + /** + * Set the 'local' bridge SSRC to [ssrc] for [mediaType] + */ + fun setLocalSsrc(mediaType: MediaType, ssrc: Long) { + val localSsrcSetEvent = SetLocalSsrcEvent(mediaType, ssrc) + rtpSender.handleEvent(localSsrcSetEvent) + rtpReceiver.handleEvent(localSsrcSetEvent) + } + + fun addRtpExtensionToRetain(extensionType: RtpExtensionType) { + rtpSender.addRtpExtensionToRetain(extensionType) + } + + fun receivesSsrc(ssrc: Long): Boolean = streamInformationStore.receiveSsrcs.contains(ssrc) + + val receiveSsrcs: Set + get() = streamInformationStore.receiveSsrcs + + fun setMediaSources(mediaSources: Array): Boolean { + logger.cdebug { "$id setting media sources: ${mediaSources.joinToString()}" } + val ret = this.mediaSources.setMediaSources(mediaSources) + val mergedMediaSources = this.mediaSources.getMediaSources() + val signaledMediaSources = mediaSources.copy() + rtpReceiver.handleEvent(SetMediaSourcesEvent(mergedMediaSources, signaledMediaSources)) + return ret + } + + // TODO(brian): we should only expose an immutable version of this, but Array doesn't have that. Go in + // and change all the storage of the media sources to use a list + fun getMediaSources(): Array = mediaSources.getMediaSources() + + @JvmOverloads + fun requestKeyFrame(requesterID: String? = null, mediaSsrc: Long? = null) = + rtpSender.requestKeyframe(requesterID, mediaSsrc) + + fun addPayloadType(payloadType: PayloadType) { + logger.cdebug { "Payload type added: $payloadType" } + streamInformationStore.addRtpPayloadType(payloadType) + } + + fun clearPayloadTypes() { + logger.cinfo { "All payload types being cleared" } + streamInformationStore.clearRtpPayloadTypes() + } + + fun addRtpExtension(rtpExtension: RtpExtension) { + logger.cdebug { "Adding RTP extension: $rtpExtension" } + streamInformationStore.addRtpExtensionMapping(rtpExtension) + } + + fun clearRtpExtensions() { + logger.cinfo { "Clearing all RTP extensions" } + // TODO: ignoring this for now, since we'll have conflicts from each channel calling it +// val rtpExtensionClearEvent = RtpExtensionClearEvent() +// rtpReceiver.handleEvent(rtpExtensionClearEvent) +// rtpSender.handleEvent(rtpExtensionClearEvent) +// rtpExtensions.clear() + } + + fun setExtmapAllowMixed(allow: Boolean) { + streamInformationStore.setExtmapAllowMixed(allow) + } + + // TODO(brian): we may want to handle local and remote ssrc associations differently, as different parts of the + // code care about one or the other, but currently there is no issue treating them the same. + fun addSsrcAssociation(ssrcAssociation: SsrcAssociation) { + logger.cdebug { + val location = if (ssrcAssociation is LocalSsrcAssociation) "local" else "remote" + "Adding $location SSRC association: $ssrcAssociation" + } + streamInformationStore.addSsrcAssociation(ssrcAssociation) + } + + fun setSrtpInformation( + chosenSrtpProtectionProfile: Int, + tlsRole: TlsRole, + keyingMaterial: ByteArray, + cryptex: Boolean + ) { + val srtpProfileInfo = + SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile) + logger.cdebug { + "Transceiver $id creating transformers with:\n" + + "profile info:\n$srtpProfileInfo\n" + + "tls role: $tlsRole\n" + + "cryptex: $cryptex" + } + srtpTransformers = SrtpUtil.initializeTransformer( + srtpProfileInfo, + keyingMaterial, + tlsRole, + cryptex, + logger + ).also { setSrtpInformationInternal(it, true) } + } + + private fun setSrtpInformationInternal(srtpTransformers: SrtpTransformers, internal: Boolean) { + rtpReceiver.setSrtpTransformers(srtpTransformers) + rtpSender.setSrtpTransformers(srtpTransformers) + internalTransformers = internal + } + + fun setSrtpInformation(srtpTransformers: SrtpTransformers) = setSrtpInformationInternal(srtpTransformers, false) + + /** + * Forcibly mute or unmute the incoming audio stream + */ + fun forceMuteAudio(shouldMute: Boolean) { + when (shouldMute) { + true -> logger.info("Muting incoming audio") + false -> logger.info("Unmuting incoming audio") + } + rtpReceiver.forceMuteAudio(shouldMute) + } + + fun forceMuteVideo(shouldMute: Boolean) { + when (shouldMute) { + true -> logger.info("Muting incoming video") + false -> logger.info("Unmuting incoming video") + } + rtpReceiver.forceMuteVideo(shouldMute) + } + + fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("stream_information_store", streamInformationStore.debugState(mode)) + set("media_sources", mediaSources.debugState()) + set("endpoint_connection_stats", endpointConnectionStats.getSnapshot().toJson()) + set("receiver", rtpReceiver.debugState(mode)) + set("sender", rtpSender.debugState(mode)) + } + + /** + * Get various media and network stats + */ + fun getTransceiverStats(): TransceiverStats = TransceiverStats( + endpointConnectionStats.getSnapshot(), + rtpReceiver.getStats(), + rtpSender.getStreamStats(), + rtpSender.getPacketStreamStats(), + rtpSender.getTransportCcEngineStats() + ) + + fun addEndpointConnectionStatsListener(listener: EndpointConnectionStats.EndpointConnectionStatsListener) = + endpointConnectionStats.addListener(listener) + fun removeEndpointConnectionStatsListener(listener: EndpointConnectionStats.EndpointConnectionStatsListener) = + endpointConnectionStats.removeListener(listener) + + override fun stop() { + rtpReceiver.stop() + rtpSender.stop() + if (internalTransformers) { + srtpTransformers?.close() + } + } + + fun teardown() { + logger.debug("Tearing down") + rtpReceiver.tearDown() + rtpSender.tearDown() + } + + fun setFeature(feature: Features, enabled: Boolean) { + rtpReceiver.setFeature(feature, enabled) + rtpSender.setFeature(feature, enabled) + } + + fun isFeatureEnabled(feature: Features): Boolean { + // As of now, the only feature we have (pcap) is always enabled on both + // the RTP sender and RTP receiver at the same time, so returning + // the state of one of them is sufficient. If that were to change + // in the future we'd have to rethink this API + return rtpReceiver.isFeatureEnabled(feature) + } + + companion object { + init { +// Node.plugins.add(BufferTracePlugin) +// Node.PLUGINS_ENABLED = true + } + } +} + +/** + * Interface for handling events coming from a [Transceiver] + * The intention is to extend if needed (e.g. merge with EndpointConnectionStats or a potential RtpSenderEventHandler). + */ +interface TransceiverEventHandler : RtpReceiverEventHandler diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt new file mode 100644 index 0000000..b4a8d11 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/VideoType.kt @@ -0,0 +1,37 @@ +/* + * Copyright @ 2021-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +enum class VideoType { + CAMERA, + DESKTOP, + DESKTOP_HIGH_FPS, + DISABLED, + + // NONE was used in the context where an Endpoint has always one media source description. It used to cover both + // lack of the actual source and the source being temporarily disabled. With the support for multiple sources per + // endpoint DISABLED means a source is turned off. Lack of a MediaSourceDesc is equivalent to NONE. + @Deprecated("Use DISABLED instead", ReplaceWith("DISABLED")) + NONE; + + fun isEnabled() = this != NONE && this != DISABLED + + /** + * Whether this is a screen sharing type. Note that among other things screen sharing sources have bursty bitrates, + * because the encoder may leave long gaps between frames while the content is static. + */ + fun isScreenshare() = this == DESKTOP || this == DESKTOP_HIGH_FPS +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt new file mode 100644 index 0000000..1f189ef --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vp8/Vp8Utils.kt @@ -0,0 +1,94 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.codec.vp8 + +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi_modified.impl.neomedia.codec.video.vp8.DePacketizer +import java.nio.ByteBuffer + +class Vp8Utils { + companion object { + private const val VP8_PAYLOAD_HEADER_LEN = 3 + + // TODO(brian): should move these elsewhere probably + private const val MIN_HD_HEIGHT = 540 + private const val MIN_SD_HEIGHT = 360 + private const val HD_LAYER_ID = 2 + private const val SD_LAYER_ID = 1 + private const val LD_LAYER_ID = 0 + private const val SUSPENDED_LAYER_ID = -1 + + fun isKeyFrame(vp8Payload: ByteBuffer): Boolean = + DePacketizer.isKeyFrame(vp8Payload.array(), vp8Payload.arrayOffset(), vp8Payload.limit()) + + fun getSpatialLayerIndexFromKeyFrame(vp8Payload: ByteBuffer): Int { + // Copied from VP8QualityFilter#getSpatialLayerIndexFromKeyframe + val payloadDescriptorLen = + DePacketizer.VP8PayloadDescriptor.getSize( + vp8Payload.array(), + vp8Payload.arrayOffset(), + vp8Payload.limit() + ) + val height = DePacketizer.VP8KeyframeHeader.getHeight( + vp8Payload.array(), + vp8Payload.arrayOffset() + payloadDescriptorLen + VP8_PAYLOAD_HEADER_LEN + ) + return when { + height >= MIN_HD_HEIGHT -> HD_LAYER_ID + height >= MIN_SD_HEIGHT -> SD_LAYER_ID + height > -1 -> LD_LAYER_ID + else -> -1 + } + } + + fun getHeightFromKeyFrame(vp8Packet: RtpPacket): Int { + val payloadDescriptorLen = + DePacketizer.VP8PayloadDescriptor.getSize( + vp8Packet.buffer, + vp8Packet.payloadOffset, + vp8Packet.payloadLength + ) + return DePacketizer.VP8KeyframeHeader.getHeight( + vp8Packet.buffer, + vp8Packet.payloadOffset + payloadDescriptorLen + VP8_PAYLOAD_HEADER_LEN + ) + } + + fun getSpatialLayerIndexFromKeyFrame(vp8Packet: RtpPacket): Int { + // Copied from VP8QUalityFilter#getSpatialLayerIndexFromKeyframe + val height = getHeightFromKeyFrame(vp8Packet) + return when { + height >= MIN_HD_HEIGHT -> HD_LAYER_ID + height >= MIN_SD_HEIGHT -> SD_LAYER_ID + height > -1 -> LD_LAYER_ID + else -> -1 + } + } + + fun getTemporalLayerIdOfFrame(vp8Payload: ByteBuffer) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( + vp8Payload.array(), + vp8Payload.arrayOffset(), + vp8Payload.limit() + ) + + fun getTemporalLayerIdOfFrame(vp8Packet: RtpPacket) = DePacketizer.VP8PayloadDescriptor.getTemporalLayerIndex( + vp8Packet.buffer, + vp8Packet.payloadOffset, + vp8Packet.payloadLength + ) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt new file mode 100644 index 0000000..bac5d7c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/codec/vpx/VpxUtils.kt @@ -0,0 +1,90 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.codec.vpx + +class VpxUtils { + companion object { + /** + * The bitmask for the TL0PICIDX field. + */ + const val TL0PICIDX_MASK = 0xff + + /** + * The bitmask for the extended picture id field. + */ + const val EXTENDED_PICTURE_ID_MASK = 0x7fff + + /** + * Returns the delta between two VP8/VP9 extended picture IDs, taking into account + * rollover. This will return the 'shortest' delta between the two + * picture IDs in the form of the number you'd add to b to get a. e.g.: + * getExtendedPictureIdDelta(1, 10) -> -9 (10 + -9 = 1) + * getExtendedPictureIdDelta(1, 32760) -> 9 (32760 + 9 = 1) + * @return the delta between two extended picture IDs (modulo 2^15). + */ + @JvmStatic + fun getExtendedPictureIdDelta(a: Int, b: Int): Int { + val diff = a - b + return when { + diff < -(1 shl 14) -> diff + (1 shl 15) + diff > (1 shl 14) -> diff - (1 shl 15) + else -> diff + } + } + + /** + * Apply a delta to a given extended picture ID and return the result (taking + * rollover into account) + * @param start the starting extended picture ID + * @param delta the delta to be applied + * @return the extended picture ID resulting from doing "start + delta" + */ + @JvmStatic + fun applyExtendedPictureIdDelta(start: Int, delta: Int): Int = (start + delta) and EXTENDED_PICTURE_ID_MASK + + /** + * Returns the delta between two VP8/VP9 Tl0PicIdx values, taking into account + * rollover. This will return the 'shortest' delta between the two + * picture IDs in the form of the number you'd add to b to get a. e.g.: + * getTl0PicIdxDelta(1, 10) -> -9 (10 + -9 = 1) + * getTl0PicIdxDelta(1, 250) -> 7 (250 + 7 = 1) + * + * If either value is -1 (meaning tl0picidx not found) return 0. + * @return the delta between two extended picture IDs (modulo 2^8). + */ + @JvmStatic + fun getTl0PicIdxDelta(a: Int, b: Int): Int { + if (a < 0 || b < 0) return 0 + val diff = a - b + return when { + diff < -(1 shl 7) -> diff + (1 shl 8) + diff > (1 shl 7) -> diff - (1 shl 8) + else -> diff + } + } + + /** + * Apply a delta to a given Tl0PidIcx and return the result (taking + * rollover into account) + * @param start the starting Tl0PicIdx + * @param delta the delta to be applied + * @return the Tl0PicIdx resulting from doing "start + delta" + */ + @JvmStatic + fun applyTl0PicIdxDelta(start: Int, delta: Int): Int = (start + delta) and TL0PICIDX_MASK + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/CertificateInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/CertificateInfo.kt new file mode 100644 index 0000000..a15b179 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/CertificateInfo.kt @@ -0,0 +1,34 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.dtls + +import java.security.KeyPair + +/** + * Store various information about a generated Certificate, including: + * The [KeyPair] used to build and sign it, as well as the fingerprint + * hash function and the calculated fingerprint we use (which is transmitted + * over the signaling channel so the certificate can be verified). We also + * store the time at which this certificate was created so we can refresh it + * appropriately. + */ +data class CertificateInfo( + val keyPair: KeyPair, + val certificate: org.bouncycastle.tls.Certificate, + val localFingerprintHashFunction: String, + val localFingerprint: String, + val creationTimestampMs: Long +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsClient.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsClient.kt new file mode 100644 index 0000000..43117e1 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsClient.kt @@ -0,0 +1,58 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import org.bouncycastle.tls.Certificate +import org.bouncycastle.tls.DTLSClientProtocol +import org.bouncycastle.tls.DTLSTransport +import org.bouncycastle.tls.DatagramTransport +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cerror +import org.jitsi.utils.logging2.createChildLogger + +class DtlsClient( + private val datagramTransport: DatagramTransport, + certificateInfo: CertificateInfo, + private val handshakeCompleteHandler: (Int, TlsRole, ByteArray) -> Unit = { _, _, _ -> }, + verifyAndValidateRemoteCertificate: (Certificate?) -> Unit = {}, + parentLogger: Logger, + private val dtlsClientProtocol: DTLSClientProtocol = DTLSClientProtocol() +) : DtlsRole { + private val logger = createChildLogger(parentLogger) + + private val tlsClient: TlsClientImpl = TlsClientImpl(certificateInfo, verifyAndValidateRemoteCertificate, logger) + + override fun start(): DTLSTransport = connect() + + fun connect(): DTLSTransport { + try { + return dtlsClientProtocol.connect(this.tlsClient, datagramTransport).also { + logger.cdebug { "DTLS handshake finished" } + handshakeCompleteHandler( + tlsClient.chosenSrtpProtectionProfile, + TlsRole.CLIENT, + tlsClient.srtpKeyingMaterial + ) + } + } catch (e: Exception) { + logger.cerror { "Error during DTLS connection: $e" } + throw e + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt new file mode 100644 index 0000000..ce20bd6 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsConfig.kt @@ -0,0 +1,76 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.dtls + +import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder +import org.bouncycastle.tls.CipherSuite +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.ConfigException +import org.jitsi.metaconfig.config +import java.time.Duration + +class DtlsConfig private constructor() { + val handshakeTimeout: Duration by config { + "jmt.dtls.handshake-timeout".from(JitsiConfig.newConfig) + } + + val cipherSuites: List by config { + "jmt.dtls.cipher-suites".from(JitsiConfig.newConfig).convertFrom> { list -> + val ciphers = list.map { it.toBcCipherSuite() } + if (ciphers.isEmpty()) { + throw ConfigException.UnableToRetrieve.ConditionNotMet("cipher-suites must not be empty") + } + ciphers + } + } + + val localFingerprintHashFunction: String by config { + "jmt.dtls.local-fingerprint-hash-function".from(JitsiConfig.newConfig).transformedBy { + validateHashFunction(it) + } + } + + val acceptedFingerprintHashFunctions: List by config { + "jmt.dtls.accepted-fingerprint-hash-functions".from(JitsiConfig.newConfig).convertFrom> { list -> + if (list.isEmpty()) { + throw ConfigException.UnableToRetrieve.ConditionNotMet( + "accepted-fingerprint-hash-functions must not be empty" + ) + } + list.map { validateHashFunction(it) } + } + } + + companion object { + val config = DtlsConfig() + } +} + +private fun validateHashFunction(func: String): String { + val ucFunc = func.uppercase() + DefaultDigestAlgorithmIdentifierFinder().find(ucFunc) + ?: throw ConfigException.UnableToRetrieve.WrongType("Unknown hash function $func") + if (ucFunc == "MD5" || ucFunc == "MD2") { + throw ConfigException.UnableToRetrieve.WrongType("Forbidden hash function $func") + } + return func.lowercase() +} + +private fun String.toBcCipherSuite(): Int = try { + CipherSuite::class.java.getDeclaredField(this).getInt(null) +} catch (e: Exception) { + throw ConfigException.UnableToRetrieve.ConditionNotMet("Value is not a valid BouncyCastle cipher suite name: $this") +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsProtocolPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsProtocolPacket.kt new file mode 100644 index 0000000..414fb5e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsProtocolPacket.kt @@ -0,0 +1,28 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import org.jitsi.rtp.Packet + +class DtlsProtocolPacket( + buf: ByteArray, + offset: Int, + length: Int +) : Packet(buf, offset, length) { + + override fun clone(): DtlsProtocolPacket = DtlsProtocolPacket(cloneBuffer(0), 0, length) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsRole.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsRole.kt new file mode 100644 index 0000000..6a673a3 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsRole.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import org.bouncycastle.tls.DTLSTransport + +interface DtlsRole { + /** + * 'Starts' a DTLS connection. Blocks until the connection + * has been successfully established (an exception is thrown + * upon failure). Returns the resulting [DTLSTransport]. + * + * The definition of 'start' varies between roles. For a client + * this means 'connect' to a remote server, for a server it means + * 'accept' and wait for an incoming connection. + */ + fun start(): DTLSTransport +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt new file mode 100644 index 0000000..2d8d7c6 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsServer.kt @@ -0,0 +1,58 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import org.bouncycastle.tls.Certificate +import org.bouncycastle.tls.DTLSServerProtocol +import org.bouncycastle.tls.DTLSTransport +import org.bouncycastle.tls.DatagramTransport +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cerror +import org.jitsi.utils.logging2.createChildLogger + +class DtlsServer( + private val datagramTransport: DatagramTransport, + certificateInfo: CertificateInfo, + private val handshakeCompleteHandler: (Int, TlsRole, ByteArray) -> Unit = { _, _, _ -> }, + verifyAndValidateRemoteCertificate: (Certificate?) -> Unit = {}, + parentLogger: Logger, + private val dtlsServerProtocol: DTLSServerProtocol = DTLSServerProtocol() +) : DtlsRole { + private val logger = createChildLogger(parentLogger) + + private val tlsServer: TlsServerImpl = TlsServerImpl(certificateInfo, verifyAndValidateRemoteCertificate, logger) + + override fun start(): DTLSTransport = accept() + + fun accept(): DTLSTransport { + try { + return dtlsServerProtocol.accept(tlsServer, datagramTransport).also { + logger.cdebug { "DTLS handshake finished" } + handshakeCompleteHandler( + tlsServer.chosenSrtpProtectionProfile, + TlsRole.SERVER, + tlsServer.srtpKeyingMaterial + ) + } + } catch (t: Throwable) { + logger.cerror { "Error during DTLS connection: $t" } + throw t + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt new file mode 100644 index 0000000..9bd37f5 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsStack.kt @@ -0,0 +1,355 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.bouncycastle.tls.Certificate +import org.bouncycastle.tls.DTLSTransport +import org.bouncycastle.tls.DatagramTransport +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.nlj.util.BufferPool +import org.jitsi.utils.concurrent.ArrayBlockingQueueWithShutdown +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import java.nio.ByteBuffer +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread +import kotlin.math.min + +/** + * A DTLS stack implementation, which can act as either the DTLS server or DTLS client, and can be used + * for negotiating the DTLS connection and sending and receiving data over that connection. + * + * This class does not directly communicate with the network in any way. Raw DTLS packets must be fed into + * this stack via the [processIncomingProtocolData] method. [incomingDataHandler] will be invoked with the + * decrypted, application data sent over the connection. + * + * Data can be sent through the DTLS connection via [sendApplicationData]; it will be encrypted by the stack and + * then sent out via the [outgoingDataHandler], which must be set to have the data go anywhere interesting :) + * + * [eventHandler] will be invoked when any events occur. + * + * After wiring up all the handlers, to start a connection the stack must be told which role to fullfil: client + * or server. This can be done by calling either the [actAsClient] or [actAsServer] methods. Once the role has + * been set, [start] can be called to start the negotiation. + */ +class DtlsStack( + parentLogger: Logger +) { + private val logger = createChildLogger(parentLogger) + private val roleSet = CountDownLatch(1) + + /** + * The certificate info for this particular [DtlsStack] instance. We save it in a local val because the global one + * might be refreshed. + */ + private val certificateInfo = DtlsStack.certificateInfo + + val localFingerprintHashFunction: String + get() = certificateInfo.localFingerprintHashFunction + + val localFingerprint: String + get() = certificateInfo.localFingerprint + + /** + * The remote fingerprints sent to us over the signaling path. + */ + var remoteFingerprints: Map> = mapOf() + + /** + * A handler which will be invoked when DTLS application data is received + */ + var incomingDataHandler: IncomingDataHandler? = null + + /** + * The method [DtlsStack] will invoke when it wants to send DTLS data out onto the network. + */ + var outgoingDataHandler: OutgoingDataHandler? = null + + /** + * Handle to be invoked when events occur + */ + var eventHandler: EventHandler? = null + + private val incomingProtocolData = ArrayBlockingQueueWithShutdown(QUEUE_SIZE, true) + private var numPacketDropsQueueFull = 0 + + /** + * The [DtlsRole] 'plugin' that will determine how this stack operates (as a client + * or a server). A call to [actAsClient] or [actAsServer] must be made to fill out + * this role and successfully call [start] + */ + var role: DtlsRole? = null + private set + + /** + * A buffer we'll use to receive data from [dtlsTransport]. + */ + private val dtlsAppDataBuf = ByteArray(1500) + + /** + * The negotiated DTLS transport. This is used to read and write DTLS application data. + */ + private var dtlsTransport: DTLSTransport? = null + + /** + * The [DatagramTransport] implementation we use for this stack. + */ + private val datagramTransport: DatagramTransport = DatagramTransportImpl(logger) + + /** + * Set the role to server, unless it already is one. + * + * The role is re-applied whenever the peer signals its `setup` attribute again, which it does on every + * transport update and not only when something about DTLS changed. Building a new [DtlsRole] each time would + * replace the one that has already negotiated the connection with one that never will, so repeating the role + * we already have is a no-op. + * + * @return true if the role changed. + */ + fun actAsServer(): Boolean { + if (role is DtlsServer) { + return false + } + warnIfRoleChangesAfterStart("server") + role = DtlsServer( + datagramTransport, + certificateInfo, + { chosenSrtpProfile, tlsRole, keyingMaterial -> + eventHandler?.handshakeComplete(chosenSrtpProfile, tlsRole, keyingMaterial) + }, + this::verifyAndValidateRemoteCertificate, + logger + ) + roleSet.countDown() + return true + } + + /** Set the role to client, unless it already is one. See [actAsServer]. */ + fun actAsClient(): Boolean { + if (role is DtlsClient) { + return false + } + warnIfRoleChangesAfterStart("client") + role = DtlsClient( + datagramTransport, + certificateInfo, + { chosenSrtpProfile, tlsRole, keyingMaterial -> + eventHandler?.handshakeComplete(chosenSrtpProfile, tlsRole, keyingMaterial) + }, + this::verifyAndValidateRemoteCertificate, + logger + ) + roleSet.countDown() + return true + } + + /** + * The stack has been started with the role it has now, so replacing that role leaves it with one that was + * never started while [dtlsTransport] keeps running the old one. Nothing in the stack does this today, and + * the peer is not supposed to change its `setup` after the handshake, so log loudly if it happens. + */ + private fun warnIfRoleChangesAfterStart(newRole: String) { + if (dtlsTransport != null) { + logger.warn("Changing the DTLS role to $newRole after the stack was started with ${role?.javaClass}.") + } + } + + /** + * 'start' this stack, in whatever role it has been told to operate (client or server). If a role + * has not yet been yet (via [actAsServer] or [actAsClient]), then it will block until the role + * has been set. + */ + fun start() { + roleSet.await() + + dtlsTransport = role?.start() + // There is a bit of a race here: It's technically possible the + // far side could finish the handshake and send a message before + // this side assigns dtlsTransport here. If so, that message + // would be passed to #processIncomingProtocolData and put in + // incomingProtocolData, but, since dtlsTransport won't be set + // yet, we won't 'receive' it yet. Check for any incoming packets + // here, to handle this case. + processIncomingProtocolData() + } + + fun close() { + datagramTransport.close() + + incomingProtocolData.shutdown() + incomingProtocolData.forEach { + BufferPool.returnBuffer(it.array()) + } + incomingProtocolData.clear() + } + + /** + * Checks that a specific [Certificate] matches the remote fingerprints sent to us over the signaling path. + */ + private fun verifyAndValidateRemoteCertificate(remoteCertificate: Certificate?) { + remoteCertificate?.let { + DtlsUtils.verifyAndValidateCertificate(it, remoteFingerprints) + // The above throws an exception if the checks fail. + logger.cdebug { "Fingerprints verified." } + } ?: run { + throw DtlsUtils.DtlsException("Remote certificate was null") + } + } + + fun sendApplicationData(data: ByteArray, off: Int, len: Int) { + dtlsTransport?.send(data, off, len) + } + + private fun processIncomingProtocolData() { + var bytesReceived: Int + do { + val bufCopy2: ByteArray? = synchronized(dtlsAppDataBuf) { + bytesReceived = dtlsTransport?.receive(dtlsAppDataBuf, 0, 1500, 1) ?: -1 + + if (bytesReceived > 0) { + // Copy again to copy out of dtlsAppDataBuf, which we re-use. + BufferPool.getBuffer(bytesReceived).apply { + System.arraycopy(dtlsAppDataBuf, 0, this, 0, bytesReceived) + } + } else { + null + } + } + if (bufCopy2 != null) { + incomingDataHandler?.dataReceived(bufCopy2, 0, bytesReceived) + } + } while (bytesReceived > 0) + } + + /** + * We get 'pushed' the data from a lower transport layer, but bouncycastle wants to 'pull' the data + * itself. To mimic this, we put the received data into a queue, and then 'pull' it through ourselves by + * calling 'receive' on the negotiated [DTLSTransport]. + * + * Note: the data we get here may be a DTLS protocol packet and therefore won't generate anything + * to be received by [dtlsTransport] or a DTLS app packet (data sent over DTLS after the handshake has + * completed) and will result in data being received through [dtlsTransport]. It's possible, though, + * that the handshake has finished and the far end has sent application data but we've not yet set + * [dtlsTransport], so we "miss" it. We don't lose this data, but it will sit inside of [incomingProtocolData] + * until the next packet comes through. This means that we must make a copy of the buffer we receive, as we + * won't necessarily be done with it by the time this method completes. + */ + fun processIncomingProtocolData(data: ByteArray, off: Int, len: Int) { + val bufCopy = BufferPool.getBuffer(len).apply { + System.arraycopy(data, off, this, 0, len) + } + if (!incomingProtocolData.offer(ByteBuffer.wrap(bufCopy, 0, len))) { + BufferPool.returnBuffer(bufCopy) + if (!incomingProtocolData.isShutdown) { + logger.warn("DTLS stack queue full, dropping packet") + numPacketDropsQueueFull++ + } + } + + processIncomingProtocolData() + } + + fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("localFingerprintHashFunction", certificateInfo.localFingerprint) + put("remoteFingerprints", remoteFingerprints.map { (hash, fp) -> "$hash: $fp" }.joinToString()) + put("role", (role?.javaClass ?: "null").toString()) + put("num_packet_drops_queue_full", numPacketDropsQueueFull) + } + + companion object { + private const val QUEUE_SIZE = 50 + + /** + * Because generating the certificateInfo can be expensive, we generate a single + * one to be used everywhere which expires in 24 hours (when we'll generate + * another one). + */ + private val syncRoot: Any = Any() + private var certificateInfo: CertificateInfo = DtlsUtils.generateCertificateInfo() + get() = synchronized(syncRoot) { + val expirationPeriodMs = Duration.ofDays(1).toMillis() + if (field.creationTimestampMs + expirationPeriodMs < System.currentTimeMillis()) { + // TODO: avoid creating our own thread + thread { field = DtlsUtils.generateCertificateInfo() } + } + return field + } + } + + /** + * An implementation of [DatagramTransport] which 'receives' from the queue in [DtlsStack] and sends out + * via [DtlsStack]'s [outgoingDataHandler] + */ + inner class DatagramTransportImpl(parentLogger: Logger) : DatagramTransport { + private val logger = createChildLogger(parentLogger) + + override fun receive(buf: ByteArray, off: Int, len: Int, waitMillis: Int): Int { + val data = try { + incomingProtocolData.poll(waitMillis.toLong(), TimeUnit.MILLISECONDS) ?: return -1 + } catch (ie: InterruptedException) { + return -1 + } + val length = min(len, data.limit()) + if (length < data.limit()) { + logger.warn( + "Passed buffer size ($len) was too small to hold incoming data size (${data.limit()}); " + + "data was truncated" + ) + } + System.arraycopy(data.array(), data.arrayOffset(), buf, off, length) + BufferPool.returnBuffer(data.array()) + return length + } + + override fun send(buf: ByteArray, off: Int, len: Int) { + outgoingDataHandler?.sendData(buf, off, len) + } + + /** + * Receive limit computation copied from [org.bouncycastle.tls.UDPTransport] + */ + override fun getReceiveLimit(): Int = 1500 - 20 - 8 + + /** + * Send limit computation copied from [org.bouncycastle.tls.UDPTransport] + */ + override fun getSendLimit(): Int = 1500 - 84 - 8 + + override fun close() {} + } + + interface IncomingDataHandler { + /** + * Notify the handler that data has been received. The handler takes ownership of the passed + * buffer, and it should be returned to the buffer pool when done with it. + */ + fun dataReceived(data: ByteArray, off: Int, len: Int) + } + + interface OutgoingDataHandler { + fun sendData(data: ByteArray, off: Int, len: Int) + } + + interface EventHandler { + fun handshakeComplete(chosenSrtpProtectionProfile: Int, tlsRole: TlsRole, keyingMaterial: ByteArray) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt new file mode 100644 index 0000000..1c8eb0a --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/DtlsUtils.kt @@ -0,0 +1,323 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.dtls + +import org.bouncycastle.asn1.ASN1Encoding +import org.bouncycastle.asn1.x500.X500Name +import org.bouncycastle.asn1.x500.X500NameBuilder +import org.bouncycastle.asn1.x500.style.BCStyle +import org.bouncycastle.asn1.x509.Certificate +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder +import org.bouncycastle.jce.ECNamedCurveTable +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.operator.DefaultDigestAlgorithmIdentifierFinder +import org.bouncycastle.operator.bc.BcDefaultDigestProvider +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder +import org.bouncycastle.tls.AlertDescription +import org.bouncycastle.tls.TlsContext +import org.bouncycastle.tls.TlsUtils +import org.bouncycastle.tls.crypto.TlsSecret +import org.bouncycastle.tls.crypto.impl.bc.BcTlsCertificate +import org.bouncycastle.tls.crypto.impl.bc.BcTlsCrypto +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cerror +import org.jitsi.utils.logging2.cinfo +import java.math.BigInteger +import java.security.KeyPair +import java.security.KeyPairGenerator +import java.security.SecureRandom +import java.security.Security +import java.time.Duration +import java.util.Date +import java.util.NoSuchElementException + +private val SECURE_RANDOM = SecureRandom() +val BC_TLS_CRYPTO = BcTlsCrypto(SECURE_RANDOM) + +/** + * Various helper utilities for DTLS + * + * https://tools.ietf.org/html/draft-ietf-rtcweb-security-arch-18 + */ +class DtlsUtils { + companion object { + init { + Security.addProvider(BouncyCastleProvider()) + } + + fun generateCertificateInfo(): CertificateInfo { + val cn = generateCN("TODO-APP-NAME", "TODO-APP-VERSION") + val keyPair = generateEcKeyPair() + val x509certificate = generateCertificate(cn, keyPair) + val localFingerprintHashFunction = DtlsConfig.config.localFingerprintHashFunction + val localFingerprint = x509certificate.getFingerprint(localFingerprintHashFunction) + + val certificate = org.bouncycastle.tls.Certificate( + arrayOf(BcTlsCertificate(BC_TLS_CRYPTO, x509certificate)) + ) + return CertificateInfo( + keyPair, + certificate, + localFingerprintHashFunction, + localFingerprint, + System.currentTimeMillis() + ) + } + + /** + * A helper which finds an SRTP protection profile present in both + * [ours] and [theirs]. Throws [DtlsException] if no common profile is found. + */ + fun chooseSrtpProtectionProfile(ours: Iterable, theirs: Iterable): Int = try { + ours.first(theirs::contains) + } catch (e: NoSuchElementException) { + throw DtlsException( + "No common SRTP protection profile found. Ours: ${ours.joinToString()} " + + "Theirs: ${theirs.joinToString()}" + ) + } + + /** + * Generate an x509 certificate valid from 1 day ago until 7 days from now. + * + * TODO: make the algorithm dynamic (passed in) to support older dtls versions/clients + */ + private fun generateCertificate(subject: X500Name, keyPair: KeyPair): Certificate { + val now = System.currentTimeMillis() + val startDate = Date(now - Duration.ofDays(1).toMillis()) + val expiryDate = Date(now + Duration.ofDays(7).toMillis()) + val serialNumber = BigInteger.valueOf(now) + + val certBuilder = JcaX509v3CertificateBuilder( + subject, + serialNumber, + startDate, + expiryDate, + subject, + keyPair.public + ) + val signer = JcaContentSignerBuilder("SHA256withECDSA").build(keyPair.private) + + return certBuilder.build(signer).toASN1Structure() + } + + /** + * Generate an eliptic-curve keypair using the secp256r1 named curve: + * "All Implementations MUST implement DTLS 1.2 with the + * TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 cipher suite and the P-256 + * curve" + * + * --https://tools.ietf.org/html/draft-ietf-rtcweb-security-arch-18#section-6.5 + * + * NOTE(brian): I used 'secp256r1' specifically because it's what I saw in wireshark traces from chrome + */ + private fun generateEcKeyPair(): KeyPair { + val keyGen = KeyPairGenerator.getInstance("EC", "BC") + val ecCurveSpec = ECNamedCurveTable.getParameterSpec("secp256r1") + + keyGen.initialize(ecCurveSpec) + + return keyGen.generateKeyPair() + } + + /** + * Generate an [X500Name] using the given [appName] and [appVersion] + */ + private fun generateCN(appName: String, appVersion: String): X500Name { + val builder = X500NameBuilder(BCStyle.INSTANCE) + val rdn = "$appName $appVersion" + builder.addRDN(BCStyle.CN, rdn) + return builder.build() + } + + /** + * Verifies and validates a specific certificate against the fingerprints + * presented by the remote endpoint via the signaling path. + * + * @param certificateInfo the certificate to be verified and validated against + * the fingerprints presented by the remote endpoint via the signaling path + * @throws [DtlsException] if [certificateInfo] fails validation + */ + fun verifyAndValidateCertificate( + certificateInfo: org.bouncycastle.tls.Certificate, + remoteFingerprints: Map> + ) { + if (certificateInfo.certificateList.isEmpty()) { + throw DtlsException("No remote fingerprints.") + } + for (currCertificate in certificateInfo.certificateList) { + val x509Cert = Certificate.getInstance(currCertificate.encoded) + verifyAndValidateCertificate(x509Cert, remoteFingerprints) + } + } + + /** + * Verifies and validates a specific certificate against the fingerprints + * presented by the remote endpoint via the signaling path. + * + * @param certificate the certificate to be verified and validated against + * the fingerprints presented by the remote endpoint via the signaling path. + * @throws DtlsException if the specified [certificate] failed to verify + * and validate against the fingerprints presented by the remote endpoint + * via the signaling path. + */ + private fun verifyAndValidateCertificate( + certificate: Certificate, + remoteFingerprints: Map> + ) { + /* RFC 8122: + * An endpoint MUST select the set of fingerprints that use its most + * preferred hash function (out of those offered by the peer) and verify + * that each certificate used matches one fingerprint out of that set. + */ + DtlsConfig.config.acceptedFingerprintHashFunctions.forEach { hashFunction -> + val fingerprints = remoteFingerprints[hashFunction] ?: return@forEach + + val certificateFingerprint = certificate.getFingerprint(hashFunction) + + if (fingerprints.none { it == certificateFingerprint }) { + throw DtlsException( + "None of the fingerprints ${fingerprints.joinToString()} match the $hashFunction-hashed " + + "certificate $certificateFingerprint" + ) + } + return@verifyAndValidateCertificate + } + /* If we got here none of our accepted fingerprint functions were listed. */ + throw DtlsException( + "No fingerprint declared over the signaling path with any of the accepted hash functions: " + + DtlsConfig.config.acceptedFingerprintHashFunctions.joinToString() + ) + } + + /** + * Computes the fingerprint of a [org.bouncycastle.asn1.x509.Certificate] using [hashFunction] and returns it + * as a [String] + */ + private fun Certificate.getFingerprint(hashFunction: String): String { + val digAlgId = DefaultDigestAlgorithmIdentifierFinder().find(hashFunction.uppercase()) + val digest = BcDefaultDigestProvider.INSTANCE.get(digAlgId) + val input: ByteArray = getEncoded(ASN1Encoding.DER) + val output = ByteArray(digest.digestSize) + + digest.update(input, 0, input.size) + digest.doFinal(output, 0) + + return output.toFingerprint() + } + + private val HEX_CHARS = "0123456789ABCDEF".toCharArray() + + /** + * Helper function to convert a [ByteArray] to a colon-delimited hex string + */ + private fun ByteArray.toFingerprint(): String { + val buf = StringBuffer() + for (i in 0 until size) { + val octet = get(i).toInt() + val firstIndex = (octet and 0xF0).ushr(4) + val secondIndex = octet and 0x0F + buf.append(HEX_CHARS[firstIndex]) + buf.append(HEX_CHARS[secondIndex]) + if (i < size - 1) { + buf.append(":") + } + } + return buf.toString() + } + + /* + * Copied from TlsContext#exportKeyingMaterial and modified to work with + * an externally provided masterSecret value. + */ + fun exportKeyingMaterial( + context: TlsContext, + asciiLabel: String, + context_value: ByteArray?, + length: Int, + masterSecret: TlsSecret + ): ByteArray { + if (context_value != null && !TlsUtils.isValidUint16(context_value.size)) { + throw IllegalArgumentException("'context_value' must have a length less than 2^16 (or be null)") + } + val sp = context.securityParameters + val cr = sp.clientRandom + val sr = sp.serverRandom + + var seedLength = cr.size + sr.size + if (context_value != null) { + seedLength += (2 + context_value.size) + } + + val seed = ByteArray(seedLength) + var seedPos = 0 + + System.arraycopy(cr, 0, seed, seedPos, cr.size) + seedPos += cr.size + System.arraycopy(sr, 0, seed, seedPos, sr.size) + seedPos += sr.size + + if (context_value != null) { + TlsUtils.writeUint16(context_value.size, seed, seedPos) + seedPos += 2 + System.arraycopy(context_value, 0, seed, seedPos, context_value.size) + seedPos += context_value.size + } + + if (seedPos != seedLength) { + throw IllegalStateException("error in calculation of seed for export") + } + + return TlsUtils.PRF(sp, masterSecret, asciiLabel, seed, length).extract() + } + } + + class DtlsException(msg: String) : Exception(msg) +} + +@Suppress("NOTHING_TO_INLINE") // Avoid adding to the trace in the log file +inline fun Logger.notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) { + when (alertDescription) { + AlertDescription.close_notify -> cdebug { "close_notify raised, connection closing" } + + else -> { + val stack = with(StringBuffer()) { + val e = Exception() + for (el in e.stackTrace) { + appendLine(el.toString()) + } + toString() + } + cinfo { + "Alert raised: level=$alertLevel, description=$alertDescription, message=$message " + + "cause=$cause $stack" + } + } + } +} + +@Suppress("NOTHING_TO_INLINE") // Avoid adding to the trace in the log file +inline fun Logger.notifyAlertReceived(alertLevel: Short, alertDescription: Short) { + when (alertDescription) { + AlertDescription.close_notify -> cinfo { "close_notify received, connection closing" } + + else -> cerror { + "Alert received: level=$alertLevel, description=$alertDescription " + + "(${AlertDescription.getName(alertDescription)})" + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt new file mode 100644 index 0000000..0ac70ca --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsClientImpl.kt @@ -0,0 +1,170 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.dtls + +import org.bouncycastle.crypto.util.PrivateKeyFactory +import org.bouncycastle.tls.Certificate +import org.bouncycastle.tls.CertificateRequest +import org.bouncycastle.tls.DefaultTlsClient +import org.bouncycastle.tls.ExporterLabel +import org.bouncycastle.tls.ExtensionType +import org.bouncycastle.tls.HashAlgorithm +import org.bouncycastle.tls.ProtocolVersion +import org.bouncycastle.tls.SignatureAlgorithm +import org.bouncycastle.tls.SignatureAndHashAlgorithm +import org.bouncycastle.tls.TlsAuthentication +import org.bouncycastle.tls.TlsCredentials +import org.bouncycastle.tls.TlsSRTPUtils +import org.bouncycastle.tls.TlsServerCertificate +import org.bouncycastle.tls.TlsSession +import org.bouncycastle.tls.TlsUtils +import org.bouncycastle.tls.UseSRTPData +import org.bouncycastle.tls.crypto.TlsCryptoParameters +import org.bouncycastle.tls.crypto.impl.bc.BcDefaultTlsCredentialedSigner +import org.bouncycastle.tls.crypto.impl.bc.BcTlsCrypto +import org.jitsi.nlj.srtp.SrtpConfig +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.rtp.extensions.toHex +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cinfo +import org.jitsi.utils.logging2.createChildLogger +import java.nio.ByteBuffer +import java.util.Hashtable + +/** + * Implementation of [DefaultTlsClient]. + */ +class TlsClientImpl( + private val certificateInfo: CertificateInfo, + /** + * The function to call when the server certificateInfo is available. + */ + private val notifyServerCertificate: (Certificate?) -> Unit, + parentLogger: Logger +) : DefaultTlsClient(BC_TLS_CRYPTO) { + + private val logger = createChildLogger(parentLogger) + + private var session: TlsSession? = null + + private var clientCredentials: TlsCredentials? = null + + /** + * Only set after a handshake has completed + */ + lateinit var srtpKeyingMaterial: ByteArray + private set + + var chosenSrtpProtectionProfile: Int = 0 + + override fun getSessionToResume(): TlsSession? = session + + override fun getAuthentication(): TlsAuthentication { + return object : TlsAuthentication { + override fun getClientCredentials(certificateRequest: CertificateRequest): TlsCredentials { + // NOTE: can't set clientCredentials when it is declared because 'context' won't be set yet + if (clientCredentials == null) { + clientCredentials = BcDefaultTlsCredentialedSigner( + TlsCryptoParameters(context), + (context.crypto as BcTlsCrypto), + PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded), + certificateInfo.certificate, + if (TlsUtils.isSignatureAlgorithmsExtensionAllowed(context.serverVersion)) { + SignatureAndHashAlgorithm( + HashAlgorithm.sha256, + SignatureAlgorithm.ecdsa + ) + } else { + null + } + ) + } + return clientCredentials!! + } + + override fun notifyServerCertificate(serverCertificate: TlsServerCertificate) { + this@TlsClientImpl.notifyServerCertificate(serverCertificate.certificate) + } + } + } + + override fun getClientExtensions(): Hashtable<*, *> { + var clientExtensions = super.getClientExtensions() + if (TlsSRTPUtils.getUseSRTPExtension(clientExtensions) == null) { + if (clientExtensions == null) { + clientExtensions = Hashtable() + } + + TlsSRTPUtils.addUseSRTPExtension( + clientExtensions, + UseSRTPData(SrtpConfig.protectionProfiles.toIntArray(), TlsUtils.EMPTY_BYTES) + ) + } + clientExtensions.put(ExtensionType.renegotiation_info, byteArrayOf(0)) + + return clientExtensions + } + + override fun processServerExtensions(serverExtensions: Hashtable<*, *>?) { + // TODO: a few cases we should be throwing alerts for in here. see old TlsClientImpl + val useSRTPData = TlsSRTPUtils.getUseSRTPExtension(serverExtensions) + val protectionProfiles = useSRTPData.protectionProfiles + chosenSrtpProtectionProfile = + DtlsUtils.chooseSrtpProtectionProfile(SrtpConfig.protectionProfiles, protectionProfiles.asIterable()) + } + + override fun getCipherSuites() = DtlsConfig.config.cipherSuites.toIntArray() + + override fun getHandshakeTimeoutMillis(): Int = DtlsConfig.config.handshakeTimeout.toMillis().toInt() + + override fun notifyHandshakeComplete() { + super.notifyHandshakeComplete() + logger.cinfo { "Negotiated DTLS version ${context.securityParameters.negotiatedVersion}" } + context.resumableSession?.let { newSession -> + + session?.let { existingSession -> + if (existingSession.sessionID?.contentEquals(newSession.sessionID) == true) { + logger.cdebug { + val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex() + "Resumed DTLS session $newSessionIdHex" + } + } + } ?: run { + logger.cdebug { + val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex() + "Established DTLS session $newSessionIdHex" + } + this.session = newSession + } + } + val srtpProfileInformation = + SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile) + srtpKeyingMaterial = context.exportKeyingMaterial( + ExporterLabel.dtls_srtp, + null, + 2 * (srtpProfileInformation.cipherKeyLength + srtpProfileInformation.cipherSaltLength) + ) + } + + override fun getSupportedVersions(): Array = arrayOf(ProtocolVersion.DTLSv12) + + override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) = + logger.notifyAlertRaised(alertLevel, alertDescription, message, cause) + + override fun notifyAlertReceived(alertLevel: Short, alertDescription: Short) = + logger.notifyAlertReceived(alertLevel, alertDescription) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt new file mode 100644 index 0000000..97da039 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/dtls/TlsServerImpl.kt @@ -0,0 +1,168 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import org.bouncycastle.crypto.util.PrivateKeyFactory +import org.bouncycastle.tls.Certificate +import org.bouncycastle.tls.CertificateRequest +import org.bouncycastle.tls.ClientCertificateType +import org.bouncycastle.tls.DefaultTlsServer +import org.bouncycastle.tls.ExporterLabel +import org.bouncycastle.tls.HashAlgorithm +import org.bouncycastle.tls.ProtocolVersion +import org.bouncycastle.tls.SignatureAlgorithm +import org.bouncycastle.tls.SignatureAndHashAlgorithm +import org.bouncycastle.tls.TlsCredentialedDecryptor +import org.bouncycastle.tls.TlsCredentialedSigner +import org.bouncycastle.tls.TlsSRTPUtils +import org.bouncycastle.tls.TlsSession +import org.bouncycastle.tls.TlsUtils +import org.bouncycastle.tls.UseSRTPData +import org.bouncycastle.tls.crypto.TlsCryptoParameters +import org.bouncycastle.tls.crypto.impl.bc.BcDefaultTlsCredentialedDecryptor +import org.bouncycastle.tls.crypto.impl.bc.BcDefaultTlsCredentialedSigner +import org.bouncycastle.tls.crypto.impl.bc.BcTlsCrypto +import org.jitsi.nlj.srtp.SrtpConfig +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.rtp.extensions.toHex +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cinfo +import org.jitsi.utils.logging2.createChildLogger +import java.nio.ByteBuffer +import java.util.Hashtable +import java.util.Vector + +class TlsServerImpl( + private val certificateInfo: CertificateInfo, + /** + * The function to call when the client certificateInfo is available. + */ + private val notifyClientCertificateReceived: (Certificate?) -> Unit, + parentLogger: Logger +) : DefaultTlsServer(BC_TLS_CRYPTO) { + + private val logger = createChildLogger(parentLogger) + + private var session: TlsSession? = null + + /** + * Only set after a handshake has completed + */ + lateinit var srtpKeyingMaterial: ByteArray + private set + + var chosenSrtpProtectionProfile: Int = 0 + + override fun getSessionToResume(sessionID: ByteArray?): TlsSession? { + return session + // TODO: do we need to map multiple sessions (per sessionID?) +// return super.getSessionToResume(sessionID) + } + + override fun getServerExtensions(): Hashtable<*, *> { + val extensions = super.getServerExtensions() + ?: Hashtable() + return extensions.also { + if (TlsSRTPUtils.getUseSRTPExtension(it) == null) { + TlsSRTPUtils.addUseSRTPExtension( + it, + UseSRTPData(intArrayOf(chosenSrtpProtectionProfile), TlsUtils.EMPTY_BYTES) + ) + } + } + } + + override fun processClientExtensions(clientExtensions: Hashtable<*, *>?) { + super.processClientExtensions(clientExtensions) + val useSRTPData = TlsSRTPUtils.getUseSRTPExtension(clientExtensions) + val protectionProfiles = useSRTPData.protectionProfiles + chosenSrtpProtectionProfile = + DtlsUtils.chooseSrtpProtectionProfile(SrtpConfig.protectionProfiles, protectionProfiles.asIterable()) + } + + override fun getCipherSuites() = DtlsConfig.config.cipherSuites.toIntArray() + + override fun getRSAEncryptionCredentials(): TlsCredentialedDecryptor = BcDefaultTlsCredentialedDecryptor( + (context.crypto as BcTlsCrypto), + certificateInfo.certificate, + PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded) + ) + + override fun getECDSASignerCredentials(): TlsCredentialedSigner = BcDefaultTlsCredentialedSigner( + TlsCryptoParameters(context), + (context.crypto as BcTlsCrypto), + PrivateKeyFactory.createKey(certificateInfo.keyPair.private.encoded), + certificateInfo.certificate, + SignatureAndHashAlgorithm(HashAlgorithm.sha256, SignatureAlgorithm.ecdsa) + ) + + override fun getCertificateRequest(): CertificateRequest { + val signatureAlgorithms = Vector(1) + signatureAlgorithms.add(SignatureAndHashAlgorithm(HashAlgorithm.sha256, SignatureAlgorithm.ecdsa)) + return CertificateRequest(shortArrayOf(ClientCertificateType.ecdsa_sign), signatureAlgorithms, null) + } + + override fun getHandshakeTimeoutMillis(): Int = DtlsConfig.config.handshakeTimeout.toMillis().toInt() + + override fun notifyHandshakeComplete() { + super.notifyHandshakeComplete() + logger.cinfo { "Negotiated DTLS version ${context.securityParameters.negotiatedVersion}" } + context.resumableSession?.let { newSession -> + val newSessionIdHex = ByteBuffer.wrap(newSession.sessionID).toHex() + + session?.let { existingSession -> + if (existingSession.sessionID?.contentEquals(newSession.sessionID) == true) { + logger.cinfo { "Resumed DTLS session $newSessionIdHex" } + } + } ?: run { + logger.cinfo { "Established DTLS session $newSessionIdHex" } + this.session = newSession + } + } + val srtpProfileInformation = + SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile) + if (!context.securityParameters.isExtendedMasterSecret) { + context.session?.exportSessionParameters()?.masterSecret?.let { + srtpKeyingMaterial = DtlsUtils.exportKeyingMaterial( + context, + ExporterLabel.dtls_srtp, + null, + 2 * (srtpProfileInformation.cipherKeyLength + srtpProfileInformation.cipherSaltLength), + it + ) + } + } else { + srtpKeyingMaterial = context.exportKeyingMaterial( + ExporterLabel.dtls_srtp, + null, + 2 * (srtpProfileInformation.cipherKeyLength + srtpProfileInformation.cipherSaltLength) + ) + } + } + + override fun notifyClientCertificate(clientCertificate: Certificate?) { + notifyClientCertificateReceived(clientCertificate) + } + + override fun notifyAlertRaised(alertLevel: Short, alertDescription: Short, message: String?, cause: Throwable?) = + logger.notifyAlertRaised(alertLevel, alertDescription, message, cause) + + override fun notifyAlertReceived(alertLevel: Short, alertDescription: Short) = + logger.notifyAlertReceived(alertLevel, alertDescription) + + override fun getSupportedVersions(): Array = arrayOf(ProtocolVersion.DTLSv12) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt new file mode 100644 index 0000000..468988c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/format/PayloadType.kt @@ -0,0 +1,218 @@ +/* + * Copyright @ 2019-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.format + +import org.jitsi.utils.MediaType +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArraySet + +typealias PayloadTypeParams = Map + +typealias RtcpFeedbackSet = Set + +fun RtcpFeedbackSet.supportsPli(): Boolean = this.contains("nack pli") +fun RtcpFeedbackSet.supportsFir(): Boolean = this.contains("ccm fir") +fun RtcpFeedbackSet.supportsRemb(): Boolean = this.contains("goog-remb") +fun RtcpFeedbackSet.supportsTcc(): Boolean = this.contains("transport-cc") + +/** + * Represents an RTP payload type. + * + * @author Boris Grozev + * @author Brian Baldino + */ +abstract class PayloadType( + /** + * The 7-bit RTP payload type number. + */ + val pt: Byte, + /** + * The encoding name. + */ + val encoding: PayloadTypeEncoding, + /** + * The media type (audio or video). + */ + val mediaType: MediaType, + /** + * The RTP clock rate. + */ + val clockRate: Int, + /** + * Additional parameters associated with the payload type (e.g. the "apt" used for RTX). + */ + val parameters: PayloadTypeParams = ConcurrentHashMap(), + + /** + * The rtcp feedback messages associated with the payload type (e.g. nack, nack pli, transport-cc, goog-remb, ccm fir, etc). + */ + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) { + val rtcpFeedbackSet = CopyOnWriteArraySet(rtcpFeedbackSet) + + override fun toString(): String = with(StringBuffer()) { + append(pt).append(" -> ").append(encodingName()) + .append(" (").append(clockRate) + .append(channelsString()) + .append("): ").append(parameters) + + toString() + } + + open fun encodingName() = encoding.name.lowercase() + + open fun channelsString() = "" +} + +enum class PayloadTypeEncoding { + OTHER, + VP8, + VP9, + AV1, + H264, + RED, + RTX, + OPUS, + TELEPHONE_EVENT; + + companion object { + private const val TEL_EVENT_TEXT = "telephone-event" + + /** + * [valueOf] does not allow for case-insensitivity and can't be overridden, so this + * method should be used when creating an instance of this enum from a string + */ + fun createFrom(value: String): PayloadTypeEncoding { + return try { + if (value.lowercase() == TEL_EVENT_TEXT) { + return TELEPHONE_EVENT + } + valueOf(value.uppercase()) + } catch (e: IllegalArgumentException) { + return OTHER + } + } + } + + override fun toString(): String = with(StringBuffer()) { + if (super.equals(TELEPHONE_EVENT)) { + append(TEL_EVENT_TEXT) + } else { + append(super.toString()) + } + toString() + } +} + +abstract class VideoPayloadType( + pt: Byte, + encoding: PayloadTypeEncoding, + clockRate: Int = 90000, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : PayloadType(pt, encoding, MediaType.VIDEO, clockRate, parameters, rtcpFeedbackSet) + +class Vp8PayloadType( + pt: Byte, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : VideoPayloadType(pt, PayloadTypeEncoding.VP8, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet) + +class Vp9PayloadType( + pt: Byte, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : VideoPayloadType(pt, PayloadTypeEncoding.VP9, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet) + +class Av1PayloadType( + pt: Byte, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : VideoPayloadType(pt, PayloadTypeEncoding.AV1, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet) + +class H264PayloadType( + pt: Byte, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : VideoPayloadType(pt, PayloadTypeEncoding.H264, parameters = parameters, rtcpFeedbackSet = rtcpFeedbackSet) + +class RtxPayloadType( + pt: Byte, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : VideoPayloadType(pt, PayloadTypeEncoding.RTX, parameters = parameters) { + val associatedPayloadType: Int = + parameters["apt"]?.toInt() ?: error("RtxPayloadType must contain 'apt'") +} + +abstract class AudioPayloadType( + pt: Byte, + encoding: PayloadTypeEncoding, + clockRate: Int = 48000, + /** + * The number of channels + */ + val channels: Int = 1, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : PayloadType(pt, encoding, MediaType.AUDIO, clockRate, parameters) { + override fun channelsString(): String = if (channels > 1) "/$channels" else "" +} + +class OpusPayloadType( + pt: Byte, + clockRate: Int = 48000, + channels: Int = 2, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : AudioPayloadType(pt, PayloadTypeEncoding.OPUS, clockRate, channels, parameters = parameters) + +class TelephoneEventPayloadType( + pt: Byte, + clockRate: Int, + channels: Int = 1, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : AudioPayloadType(pt, PayloadTypeEncoding.TELEPHONE_EVENT, clockRate, channels, parameters) + +class AudioRedPayloadType( + pt: Byte, + clockRate: Int = 48000, + channels: Int = 1, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : AudioPayloadType(pt, PayloadTypeEncoding.RED, clockRate, channels, parameters) + +class VideoRedPayloadType( + pt: Byte, + clockRate: Int = 90000, + parameters: PayloadTypeParams = ConcurrentHashMap(), + rtcpFeedbackSet: RtcpFeedbackSet = emptySet() +) : VideoPayloadType(pt, PayloadTypeEncoding.RED, clockRate, parameters, rtcpFeedbackSet) + +class OtherAudioPayloadType( + pt: Byte, + private val encodingName: String, + clockRate: Int, + channels: Int = 1, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : AudioPayloadType(pt, PayloadTypeEncoding.OTHER, clockRate, channels, parameters) { + override fun encodingName() = encodingName.lowercase() +} + +class OtherVideoPayloadType( + pt: Byte, + private val encodingName: String, + clockRate: Int, + parameters: PayloadTypeParams = ConcurrentHashMap() +) : VideoPayloadType(pt, PayloadTypeEncoding.OTHER, clockRate, parameters) { + override fun encodingName() = encodingName.lowercase() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt new file mode 100644 index 0000000..741614d --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/KeyframeRequester.kt @@ -0,0 +1,310 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.Event +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetLocalSsrcEvent +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacketBuilder +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacketBuilder +import org.jitsi.utils.MediaType +import org.jitsi.utils.RateLimit +import org.jitsi.utils.durationOfDoubleSeconds +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.min +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.atomic.AtomicInteger + +/** + * [KeyframeRequester] handles a few things around keyframes: + * 1) The bridge requesting a keyframe (e.g. in order to switch) via the [KeyframeRequester#requestKeyframe] + * method which will create a new keyframe request and forward it + * 2) PLI/FIR translation. If a PLI or FIR packet is forwarded through here, this class may translate it depending + * on what the client supports + * 3) Aggregation. This class will pace outgoing requests such that we don't spam the sender + */ +class KeyframeRequester @JvmOverloads constructor( + private val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger, + private val clock: Clock = Clock.systemDefaultZone() +) : TransformerNode("Keyframe Requester") { + private val logger = createChildLogger(parentLogger) + + // Map the tuple of requester and SSRC to a rate limiter + private val perReceiverKeyframeLimiter = mutableMapOf>() + private val perSourceKeyframeLimiter = mutableMapOf() + private val keyframeLimiterSyncRoot = Any() + private val firCommandSequenceNumber: AtomicInteger = AtomicInteger(0) + private var localSsrc: Long? = null + private var waitInterval = minInterval + + // Stats + + // Number of PLI/FIRs received and forwarded to the endpoint. + private var numPlisForwarded: Int = 0 + private var numFirsForwarded: Int = 0 + + // Number of PLI/FIRs received but dropped due to throttling. + private var numPlisDropped: Int = 0 + private var numFirsDropped: Int = 0 + + // Number of PLI/FIRs generated as a result of an API request or due to translation between PLI/FIR. + private var numPlisGenerated: Int = 0 + private var numFirsGenerated: Int = 0 + + // Number of calls to requestKeyframe + private var numApiRequests: Int = 0 + + // Number of calls to requestKeyframe ignored due to throttling + private var numApiRequestsDropped: Int = 0 + + // Number of requests dropped by each limiter, to show which one is binding. + private var numRequestsDroppedPerReceiverLimit: Int = 0 + private var numRequestsDroppedSourceWideLimit: Int = 0 + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val pliOrFirPacket = packetInfo.getPliOrFirPacket() ?: return packetInfo + + val now = clock.instant() + val sourceSsrc: Long + val canSend: Boolean + val forward: Boolean + when (pliOrFirPacket) { + is RtcpFbPliPacket -> { + sourceSsrc = pliOrFirPacket.mediaSourceSsrc + canSend = canSendKeyframeRequest(packetInfo.endpointId, sourceSsrc, now) + forward = canSend && streamInformationStore.supportsPli + if (forward) numPlisForwarded++ + if (!canSend) numPlisDropped++ + } + + is RtcpFbFirPacket -> { + sourceSsrc = pliOrFirPacket.mediaSenderSsrc + canSend = canSendKeyframeRequest(packetInfo.endpointId, sourceSsrc, now) + // When both are supported, we favor generating a PLI rather than forwarding a FIR + forward = canSend && streamInformationStore.supportsFir && !streamInformationStore.supportsPli + if (forward) { + // When we forward a FIR we need to update the seq num. + pliOrFirPacket.seqNum = firCommandSequenceNumber.incrementAndGet() + // We manage the seq num space, so we should use the same SSRC + localSsrc?.let { pliOrFirPacket.mediaSenderSsrc = it } + numFirsForwarded++ + } + if (!canSend) numFirsDropped++ + } + + // This is not possible, but the compiler doesn't know it. + else -> throw IllegalStateException("Packet is neither PLI nor FIR") + } + + if (!forward && canSend) { + doRequestKeyframe(sourceSsrc) + } + + return if (forward) packetInfo else null + } + + /** + * Returns 'true' when at least one method is supported, AND this requester hasn't sent a request very recently. + */ + private fun canSendKeyframeRequest(requesterID: String?, mediaSsrc: Long, now: Instant): Boolean { + if (!streamInformationStore.supportsPli && !streamInformationStore.supportsFir) { + return false + } + synchronized(keyframeLimiterSyncRoot) { + /* A null requesterID is a dominant speaker switch, or a request relayed from another bridge (relayed + * RTCP carries no endpoint id). There is no receiver to attribute it to, so skip only the per-receiver + * limit; the source-wide limit still applies, since it is what protects the sender's encoder and this + * is the only bridge that sees every requester for the source. */ + val perReceiverLimiter = requesterID?.let { requester -> + perReceiverKeyframeLimiter.computeIfAbsent(requester) { mutableMapOf() } + .computeIfAbsent(mediaSsrc) { + RateLimit( + defaultMinInterval = minInterval, + maxRequests = maxRequests, + interval = maxRequestInterval + ) + } + } + if (perReceiverLimiter != null && !perReceiverLimiter.wouldAccept(now, waitInterval)) { + numRequestsDroppedPerReceiverLimit++ + logger.cdebug { + "Ignoring keyframe request for $mediaSsrc from $requesterID, per-receiver rate limited" + } + return false + } + + val perSourceLimiter = perSourceKeyframeLimiter.computeIfAbsent(mediaSsrc) { + RateLimit( + defaultMinInterval = sourceWideMinInterval, + maxRequests = sourceWideMaxRequests, + interval = sourceWideMaxRequestInterval + ) + } + if (!perSourceLimiter.wouldAccept(now, sourceWideInterval())) { + numRequestsDroppedSourceWideLimit++ + logger.cdebug { "Ignoring keyframe request for $mediaSsrc from $requesterID, per-source rate limited" } + return false + } + + /* Both limits accept, so record the request with both only now. A receiver waiting for a keyframe + * re-requests on every packet, so if requests dropped by the source-wide limit counted against its + * per-receiver limit it would exhaust that limit while the source-wide one is closed, and then be unable + * to request again for max-request-interval after the source-wide limit reopens. */ + perReceiverLimiter?.record(now) + perSourceLimiter.record(now) + + logger.cdebug { "Keyframe requester requesting keyframe for $mediaSsrc, requested by $requesterID" } + return true + } + } + + /** + * The minimum interval to enforce between keyframe requests for a source, from any receiver. [waitInterval] is + * derived from the per-receiver min-interval and the RTT, so it is never longer than the per-receiver interval; + * take the larger of the two so that source-wide-min-interval is actually enforced. + */ + private fun sourceWideInterval(): Duration = maxOf(waitInterval, sourceWideMinInterval) + + fun requestKeyframe(requesterID: String?, mediaSsrc: Long? = null) { + val ssrc = mediaSsrc ?: streamInformationStore.primaryMediaSsrcs.firstOrNull() ?: run { + numApiRequestsDropped++ + logger.cdebug { "No video SSRC found to request keyframe" } + return + } + numApiRequests++ + if (!canSendKeyframeRequest(requesterID, ssrc, clock.instant())) { + numApiRequestsDropped++ + return + } + + doRequestKeyframe(ssrc) + } + + private fun doRequestKeyframe(mediaSsrc: Long) { + val pkt = when { + streamInformationStore.supportsPli -> { + numPlisGenerated++ + RtcpFbPliPacketBuilder(mediaSourceSsrc = mediaSsrc).build() + } + + streamInformationStore.supportsFir -> { + numFirsGenerated++ + RtcpFbFirPacketBuilder( + mediaSenderSsrc = mediaSsrc, + firCommandSeqNum = firCommandSequenceNumber.incrementAndGet() + ).build() + } + + else -> { + logger.warn("Can not send neither PLI nor FIR") + return + } + } + + next(PacketInfo(pkt)) + } + + override fun handleEvent(event: Event) { + when (event) { + is SetLocalSsrcEvent -> { + if (event.mediaType == MediaType.VIDEO) { + localSsrc = event.ssrc + } + } + } + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("wait_interval_ms", waitInterval.toMillis()) + addNumber("num_api_requests", numApiRequests) + addNumber("num_api_requests_dropped", numApiRequestsDropped) + addNumber("num_firs_dropped", numFirsDropped) + addNumber("num_firs_generated", numFirsGenerated) + addNumber("num_firs_forwarded", numFirsForwarded) + addNumber("num_plis_dropped", numPlisDropped) + addNumber("num_plis_generated", numPlisGenerated) + addNumber("num_plis_forwarded", numPlisForwarded) + addNumber("num_requests_dropped_per_receiver_limit", numRequestsDroppedPerReceiverLimit) + addNumber("num_requests_dropped_source_wide_limit", numRequestsDroppedSourceWideLimit) + } + + override fun statsJson() = super.statsJson().apply { + put("num_api_requests", numApiRequests) + put("num_api_requests_dropped", numApiRequestsDropped) + put("num_firs_dropped", numFirsDropped) + put("num_firs_generated", numFirsGenerated) + put("num_firs_forwarded", numFirsForwarded) + put("num_plis_dropped", numPlisDropped) + put("num_plis_generated", numPlisGenerated) + put("num_plis_forwarded", numPlisForwarded) + put("num_requests_dropped_per_receiver_limit", numRequestsDroppedPerReceiverLimit) + put("num_requests_dropped_source_wide_limit", numRequestsDroppedSourceWideLimit) + } + + fun onRttUpdate(newRtt: Double) { + // avg(rtt) + stddev(rtt) would be more accurate than rtt + 10. + waitInterval = min(minInterval, durationOfDoubleSeconds((newRtt + 10) / 1e3)) + } + + companion object { + private val minInterval: Duration by config { + "jmt.keyframe.min-interval".from(JitsiConfig.newConfig) + } + private val maxRequests: Int by config { + "jmt.keyframe.max-requests".from(JitsiConfig.newConfig) + } + private val maxRequestInterval: Duration by config { + "jmt.keyframe.max-request-interval".from(JitsiConfig.newConfig) + } + + private val sourceWideMinInterval: Duration by config { + "jmt.keyframe.source-wide-min-interval".from(JitsiConfig.newConfig) + } + private val sourceWideMaxRequests: Int by config { + "jmt.keyframe.source-wide-max-requests".from(JitsiConfig.newConfig) + } + private val sourceWideMaxRequestInterval: Duration by config { + "jmt.keyframe.source-wide-max-request-interval".from(JitsiConfig.newConfig) + } + } +} + +private fun PacketInfo.getPliOrFirPacket(): RtcpFbPacket? = when (val pkt = packet) { + // We intentionally ignore compound RTCP packets in order to avoid unnecessary parsing. We can do this because: + // 1. Compound packets coming from remote endpoint are terminated in RtcpTermination + // 2. Whenever a PLI or FIR is generated in our code, it is not part of a compound packet. + is RtcpFbFirPacket -> pkt + + is RtcpFbPliPacket -> pkt + + else -> null +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/NackHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/NackHandler.kt new file mode 100644 index 0000000..1782bc6 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/NackHandler.kt @@ -0,0 +1,104 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtcp + +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.stats.EndpointConnectionStats +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.NodeStatsProducer +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.PacketCache +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import java.time.Instant + +/** + * When a nack packet is received, the [NackHandler] will try to retrieve the + * nacked packets from the cache and then send them to the RTX output pipeline. + */ +class NackHandler( + private val packetCache: PacketCache, + private val onNackedPacketsReady: PacketHandler, + parentLogger: Logger +) : NodeStatsProducer, RtcpListener, EndpointConnectionStats.EndpointConnectionStatsListener { + private var numNacksReceived = 0 + private var numNackedPackets = 0 + private var numRetransmittedPackets = 0 + private var numPacketsNotResentDueToDelay = 0 + private var numCacheMisses = 0 + private val logger = createChildLogger(parentLogger) + private var currRtt: Double = -1.0 + + override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) { + if (packet is RtcpFbNackPacket) { + onNackPacket(packet) + } + } + + private fun onNackPacket(nackPacket: RtcpFbNackPacket) { + logger.cdebug { "Nack received for ${nackPacket.mediaSourceSsrc} ${nackPacket.missingSeqNums}" } + val now = System.currentTimeMillis() + + numNacksReceived++ + val nackedPackets = mutableListOf() + val ssrc = nackPacket.mediaSourceSsrc + numNackedPackets += nackPacket.missingSeqNums.size + nackPacket.missingSeqNums.forEach { missingSeqNum -> + packetCache.get(ssrc, missingSeqNum)?.let { container -> + val delay = now - container.timeAdded + val shouldResendPacket = + (currRtt == -1.0) || + (delay >= Math.min(currRtt * .9, currRtt - 5)) + if (shouldResendPacket) { + // The cache returns a null container on failure, never a container with a null packet. Maybe we + // can refactor to make it explicit. + nackedPackets.add(container.item!!) + packetCache.updateTimestamp(ssrc, missingSeqNum, now) + numRetransmittedPackets++ + } else { + BufferPool.returnBuffer(container.item!!.buffer) + numPacketsNotResentDueToDelay++ + } + } ?: run { + logger.cdebug { "Nack'd packet $ssrc $missingSeqNum wasn't in cache, unable to retransmit" } + numCacheMisses++ + } + } + nackedPackets.forEach { + val packetInfo = PacketInfo(it) + packetInfo.packetOrigin = PacketOrigin.Retransmission + onNackedPacketsReady.processPacket(packetInfo) + } + } + + override fun onRttUpdate(newRttMs: Double) { + currRtt = newRttMs + } + + override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("Nack handler").apply { + addNumber("num_nack_packets_received", numNackedPackets) + addNumber("num_nacked_packets", numNackedPackets) + addNumber("num_retransmitted_packets", numRetransmittedPackets) + addNumber("num_packets_not_retransmitted", numPacketsNotResentDueToDelay) + addNumber("num_cache_misses", numCacheMisses) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RembHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RembHandler.kt new file mode 100644 index 0000000..6b0f23e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RembHandler.kt @@ -0,0 +1,72 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.bps +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import java.time.Instant +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicInteger + +class RembHandler( + val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : RtcpListener { + private val logger = createChildLogger(parentLogger) + private var sawSpuriousRemb = false + + private val bweUpdateListeners: MutableList = + CopyOnWriteArrayList() + + override fun rtcpPacketReceived(packet: RtcpPacket?, receivedTime: Instant?) { + if (packet is RtcpFbRembPacket) { + logger.debug { "Received REMB packet" } + onRembPacket(packet) + } + } + + fun addListener(bweUpdateListener: TransportCcEngine.BandwidthListener) { + bweUpdateListeners.add(bweUpdateListener) + } + + private fun onRembPacket(rembPacket: RtcpFbRembPacket) { + if (streamInformationStore.supportsTcc) { + if (!sawSpuriousRemb) { + logger.warn { + "Ignoring unexpected REMB, when using TCC (for ${rembPacket.bitrate.bps}). Will " + + "suppress future logs for this endpoint." + } + endpointsWithSpuriousRemb.incrementAndGet() + sawSpuriousRemb = true + } + return + } + logger.debug { "Updating bandwidth to ${rembPacket.bitrate.bps}" } + bweUpdateListeners.forEach { it.bandwidthEstimationChanged(rembPacket.bitrate.bps) } + } + + companion object { + private val endpointsWithSpuriousRemb = AtomicInteger() + + fun endpointsWithSpuriousRemb() = endpointsWithSpuriousRemb.get() + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RetransmissionRequester.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RetransmissionRequester.kt new file mode 100644 index 0000000..48a7e0b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RetransmissionRequester.kt @@ -0,0 +1,260 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacketBuilder +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNextAfter +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.rtp.util.numPacketsTo +import org.jitsi.utils.LRUCache +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.logging2.cwarn +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.SortedSet +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +class RetransmissionRequester( + private val rtcpSender: (RtcpPacket) -> Unit, + private val scheduler: ScheduledExecutorService, + parentLogger: Logger, + private val clock: Clock = Clock.systemUTC() +) { + private val logger = createChildLogger(parentLogger) + + /** + * Per-SSRC requesters, keyed on the RTP SSRC. Bounded with an LRU to limit the number of SSRCs tracked; evicted + * requesters are stopped to cancel their scheduled tasks. Access is synchronized on the map. + */ + private val streamPacketRequesters: MutableMap = + object : LRUCache(MAX_SSRCS, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry): Boolean = + super.removeEldestEntry(eldest).also { if (it) eldest.value.stop() } + } + + fun packetReceived(ssrc: Long, seqNum: Int) { + val streamPacketRequester = synchronized(streamPacketRequesters) { + streamPacketRequesters.computeIfAbsent(ssrc) { key -> + StreamPacketRequester(key, scheduler, clock, rtcpSender, logger) + } + } + streamPacketRequester.packetReceived(seqNum) + } + + fun stop() { + synchronized(streamPacketRequesters) { + streamPacketRequesters.values.forEach(StreamPacketRequester::stop) + streamPacketRequesters.clear() + } + } + + /** + * Manages retransmission requests for all packets for a specific SSRC + */ + class StreamPacketRequester( + val ssrc: Long, + private val scheduler: ScheduledExecutorService, + private val clock: Clock, + private val rtcpSender: (RtcpPacket) -> Unit, + parentLogger: Logger, + private val maxMissingSeqNums: Int = 100 + ) { + companion object { + val NO_REQUEST_DUE: Instant = Instant.MAX + } + private var running: AtomicBoolean = AtomicBoolean(true) + private val logger = + createChildLogger(parentLogger, mapOf("ssrc" to ssrc.toString())) + private var highestReceivedSeqNum = -1 + private val requests: MutableMap = HashMap() + private val taskHandleLock = Any() + private var currentTaskHandle: ScheduledFuture<*>? = null + + fun packetReceived(seqNum: Int) { + if (highestReceivedSeqNum == -1) { + highestReceivedSeqNum = seqNum + return + } + synchronized(requests) { + when { + seqNum == highestReceivedSeqNum -> { + logger.cdebug { "$ssrc packet $seqNum was received, currently missing ${getMissingSeqNums()}" } + // By definition we've already received highestReceivedSeqNum, so nothing needs to be done. + } + + seqNum isOlderThan highestReceivedSeqNum -> { + logger.cdebug { "$ssrc packet $seqNum was received, currently missing ${getMissingSeqNums()}" } + // An older packet, possibly already requested + requests.remove(seqNum) + if (requests.isEmpty()) { + logger.cdebug { "$ssrc no more missing seq nums, cancelling pending work" } + updateWorkDueTime(NO_REQUEST_DUE) + } + } + + seqNum isNextAfter highestReceivedSeqNum -> { + highestReceivedSeqNum = seqNum + } + + highestReceivedSeqNum numPacketsTo seqNum < maxMissingSeqNums -> { + logger.cdebug { + "$ssrc missing packet detected! Just received " + + "$seqNum, last received was $highestReceivedSeqNum" + } + RtpUtils.sequenceNumbersBetween(highestReceivedSeqNum, seqNum).forEach { missingSeqNum -> + val request = PacketRetransmissionRequest(missingSeqNum) + requests[missingSeqNum] = request + updateWorkDueTime(clock.instant()) + } + highestReceivedSeqNum = seqNum + } + + else -> { // diff > maxMissingSeqNums + logger.cwarn { + "$ssrc large jump in sequence numbers detected (highest received was " + + "$highestReceivedSeqNum, current is $seqNum, jump of " + + "${highestReceivedSeqNum numPacketsTo seqNum}) , not requesting retransmissions" + } + highestReceivedSeqNum = seqNum + // Reset and clear any pending work to do for this source + requests.clear() + logger.cdebug { "$ssrc large packet gap, resetting and clearing all work" } + updateWorkDueTime(NO_REQUEST_DUE) + } + } + } + } + + fun stop() { + running.set(false) + synchronized(taskHandleLock) { + currentTaskHandle?.cancel(false) + } + synchronized(requests) { + requests.clear() + } + } + + private fun updateWorkDueTime(newWorkDueTs: Instant) { + logger.cdebug { "$ssrc updating next work due time to $newWorkDueTs" } + synchronized(taskHandleLock) { + if (!running.get()) { + logger.cdebug { "$ssrc is stopped, not rescheduling task" } + } + when (newWorkDueTs) { + NO_REQUEST_DUE -> { + logger.cdebug { "$ssrc no more work to do, cancelling job handle" } + currentTaskHandle?.cancel(false) + } + + else -> { + // TODO(brian): only re-schedule if the change is larger than X ms? + // The work is now due either sooner or later than we previously thought, so + // re-schedule the task + currentTaskHandle?.cancel(false) + currentTaskHandle = scheduler.schedule( + ::doWork, + Duration.between(clock.instant(), newWorkDueTs).toMillis(), + TimeUnit.MILLISECONDS + ) + } + } + } + } + + private fun doWork() { + logger.cdebug { "$ssrc doing work at ${clock.instant()}" } + val now = clock.instant() + val missingSeqNums = getMissingSeqNums() + if (missingSeqNums.size >= maxMissingSeqNums) { + logger.warn { "$ssrc sending NACK for ${missingSeqNums.size} missing packets" } + } + val nackPacket = RtcpFbNackPacketBuilder( + mediaSourceSsrc = ssrc, + missingSeqNums = missingSeqNums + ).build() + notifyNackSent(now, missingSeqNums) + rtcpSender(nackPacket) + } + + private fun notifyNackSent(timestamp: Instant, nackedSeqNums: Collection) { + synchronized(requests) { + nackedSeqNums.forEach { nackedSeqNum -> + // It's possible that in between sending the nack and calling this method the packet + // was received and is no longer in the requests map + requests[nackedSeqNum]?.let { request -> + request.requested(timestamp) + if (request.numTimesRequested == MAX_REQUESTS) { + logger.cdebug { + "$ssrc generated the last NACK for seq num ${request.seqNum}, " + + "time since the first request = " + + "${Duration.between(request.firstRequestTimestamp, timestamp)}" + } + + requests.remove(nackedSeqNum) + } + } ?: run { + logger.cdebug { + "$ssrc packet $nackedSeqNum must have just been received, it was " + + "no longer in the requests map" + } + } + } + val nextDueTime = if (requests.isNotEmpty()) timestamp.plus(REQUEST_INTERVAL) else NO_REQUEST_DUE + logger.cdebug { "$ssrc nack sent at $timestamp, next one will be sent at $nextDueTime" } + updateWorkDueTime(nextDueTime) + } + } + + private fun getMissingSeqNums(): SortedSet = synchronized(requests) { requests.keys.toSortedSet() } + } + + /** + * Tracks a request for retransmission of a specific RTP packet. + */ + private class PacketRetransmissionRequest( + val seqNum: Int + ) { + var numTimesRequested = 0 + private set + var firstRequestTimestamp: Instant = Instant.MIN + private set + + fun requested(timestamp: Instant) { + if (firstRequestTimestamp == Instant.MIN) { + firstRequestTimestamp = timestamp + } + numTimesRequested++ + } + } + + companion object { + private const val MAX_REQUESTS = 10 + private val REQUEST_INTERVAL = Duration.ofMillis(150) + + /** The maximum number of SSRCs to track retransmission requesters for. */ + const val MAX_SSRCS = 1024 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpEventNotifier.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpEventNotifier.kt new file mode 100644 index 0000000..7a72ff0 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpEventNotifier.kt @@ -0,0 +1,51 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import org.jitsi.rtp.rtcp.RtcpPacket +import java.time.Instant +import java.util.concurrent.CopyOnWriteArrayList + +/** + * A central place to allow the publishing of when RTCP packets are received or sent. We're + * interested in both of these scenarios for things like SRs, RRs and for RTT calculations + */ +// TODO(brian): maybe post the notifications to another pool, so we don't hold up the caller? +class RtcpEventNotifier { + private val rtcpListeners: MutableList> = CopyOnWriteArrayList() + + /** + * Add an [RtcpListener]. An [external] listener will not receive notifications that are themselves + * marked as external. This allows notifications to be passed between multiple listeners without + * creating an infinite loop. + */ + fun addRtcpEventListener(listener: RtcpListener, external: Boolean = false) { + rtcpListeners.add(Pair(listener, external)) + } + + fun notifyRtcpReceived(packet: RtcpPacket, receivedTime: Instant?, external: Boolean = false) { + rtcpListeners.forEach { (listener, listenerIsExternal) -> + if (!external || !listenerIsExternal) listener.rtcpPacketReceived(packet, receivedTime) + } + } + + fun notifyRtcpSent(rtcpPacket: RtcpPacket, external: Boolean = false) { + rtcpListeners.forEach { (listener, listenerIsExternal) -> + if (!external || !listenerIsExternal) listener.rtcpPacketSent(rtcpPacket) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpListener.java b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpListener.java new file mode 100644 index 0000000..6ef062a --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpListener.java @@ -0,0 +1,34 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp; + +import org.jetbrains.annotations.*; +import org.jitsi.rtp.rtcp.*; + +import java.time.*; + +/** + * NOTE(brian): Java doesn't see default implementation interface methods when + * written from kotlin, so this is implemented in java (which works for both + * kotlin and java) because I got tired of adding no-op methods for one of + * these methods as most listeners care about one but not the other + */ +public interface RtcpListener { + default void rtcpPacketReceived(RtcpPacket packet, @Nullable Instant receivedTime) {} + default void rtcpPacketSent(RtcpPacket packet) {} +} + diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt new file mode 100644 index 0000000..8d75a79 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpParsers.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2019 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import org.jitsi.nlj.transform.node.PacketParser +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.utils.logging2.Logger + +class CompoundRtcpParser(parentLogger: Logger) : PacketParser( + "Compound RTCP parser", + parentLogger, + { + CompoundRtcpPacket(it.buffer, it.offset, it.length).also { compoundPacket -> + // Force packets to be evaluated to trigger any parsing errors + compoundPacket.packets + } + } +) { + override fun trace(f: () -> Unit) = f.invoke() +} + +class SingleRtcpParser(parentLogger: Logger) : PacketParser( + "Single RTCP parser", + parentLogger, + { + RtcpPacket.parse(it.buffer, it.offset, it.length) + } +) { + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt new file mode 100644 index 0000000..36413c7 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpRrGenerator.kt @@ -0,0 +1,152 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtcp + +import org.jitsi.nlj.transform.node.incoming.IncomingSsrcStats +import org.jitsi.nlj.transform.node.incoming.IncomingStatisticsTracker +import org.jitsi.nlj.util.schedule +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpReportBlock +import org.jitsi.rtp.rtcp.RtcpRrPacketBuilder +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.utils.LRUCache +import org.jitsi.utils.MediaType +import org.jitsi.utils.ms +import org.jitsi.utils.times +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.Collections +import java.util.concurrent.ScheduledExecutorService + +/** + * Information about a sender that is used in the generation of RTCP report blocks. NOTE that this does NOT correspond + * to the Sender Info block of an SR + * TODO: rename to not be confused with Sender Info in SR? + */ +private data class SenderInfo( + var lastSrCompactedTimestamp: Long = 0, + var lastSrReceivedTime: Instant? = null, + // The media type doesn't affect RTCP RR/SR generation. Initialize with a dummy value. + var statsSnapshot: IncomingSsrcStats.Snapshot = IncomingSsrcStats.Snapshot(mediaType = MediaType.VIDEO) +) { + private fun hasReceivedSr(): Boolean = lastSrReceivedTime != null + + fun getDelaySinceLastSr(now: Instant): Long = if (hasReceivedSr()) { + // This value is in 1/65536 of a second, so multiplying by 65536 gives us the value + Duration.between(lastSrReceivedTime, now).times(65536).seconds + } else { + 0 + } +} + +/** + * Retrieves statistics about incoming streams and creates RTCP RR packets. Since RR packets are created based on + * time (and not on a number of incoming packets received, etc.) it does not live within the packet pipelines. + * + * @param additionalPacketSupplier A function which supplies additional RTCP packets (such as REMB) to be sent together + * with RRs. + */ +class RtcpRrGenerator( + private val backgroundExecutor: ScheduledExecutorService, + private val rtcpSender: (RtcpPacket) -> Unit = {}, + private val incomingStatisticsTracker: IncomingStatisticsTracker, + private val clock: Clock = Clock.systemUTC(), + private val additionalPacketSupplier: () -> List +) : RtcpListener { + var running: Boolean = true + + /** + * Per-SSRC sender info, keyed on the RTCP sender SSRC. Bounded with an LRU to limit the number of SSRCs tracked. + */ + private val senderInfos: MutableMap = + Collections.synchronizedMap(LRUCache(MAX_SSRCS, true)) + + init { + doWork() + } + + override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) { + when (packet) { + is RtcpSrPacket -> { + // Note the time we received an SR so that it can be used when creating RtcpReportBlocks + // TODO: we have a concurrency issue here: we could be halfway through updating the senderinfo when + // the doWork context thread runs + val senderInfo = senderInfos.computeIfAbsent(packet.senderSsrc) { SenderInfo() } + senderInfo.lastSrCompactedTimestamp = packet.senderInfo.compactedNtpTimestamp + senderInfo.lastSrReceivedTime = receivedTime + } + } + } + + private fun doWork() { + if (running) { + val streamStats = incomingStatisticsTracker.getSnapshotOfActiveSsrcs() + val now = clock.instant() + val reportBlocks = mutableListOf() + streamStats.ssrcStats.forEach { (ssrc, statsSnapshot) -> + val senderInfo = senderInfos.computeIfAbsent(ssrc) { + SenderInfo() + } + val fractionLost = statsSnapshot.computeFractionLost(senderInfo.statsSnapshot) + senderInfo.statsSnapshot = statsSnapshot + + reportBlocks.add( + RtcpReportBlock( + ssrc, + fractionLost, + statsSnapshot.cumulativePacketsLost, + statsSnapshot.seqNumCycles, + statsSnapshot.maxSeqNum, + statsSnapshot.jitter.toLong(), + senderInfo.lastSrCompactedTimestamp, + senderInfo.getDelaySinceLastSr(now) + ) + ) + } + + val packets = mutableListOf() + if (reportBlocks.isNotEmpty()) { + for (blocks in reportBlocks.chunked(31)) { + packets.add(RtcpRrPacketBuilder(reportBlocks = blocks).build()) + } + } + packets.addAll(additionalPacketSupplier()) + + when (packets.size) { + 0 -> {} + + 1 -> rtcpSender(packets.first()) + + else -> for (packet in CompoundRtcpPacket.createWithMtu(packets)) { + rtcpSender(packet) + } + } + backgroundExecutor.schedule(this::doWork, reportingInterval) + } + } + + companion object { + /** + * The interval at which RRs and REMBs will be sent. + */ + val reportingInterval = 500.ms + + /** The maximum number of SSRCs to track sender info for. */ + const val MAX_SSRCS = 1024 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpSrUpdater.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpSrUpdater.kt new file mode 100644 index 0000000..cf2a9d3 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtcp/RtcpSrUpdater.kt @@ -0,0 +1,44 @@ +/* + * Copyright @ 2019 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtcp + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsTracker +import org.jitsi.rtp.rtcp.RtcpSrPacket + +/** + * Updates RTCP Sender Reports with the current octet and packet count. + */ +class RtcpSrUpdater( + val statsTracker: OutgoingStatisticsTracker +) : TransformerNode("RtcpSrUpdater") { + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + // TODO support compound packets + val rtcpSrPacket = packetInfo.packet as? RtcpSrPacket ?: return packetInfo + + // If we haven't sent RTP for this SSRC, drop the SR + val ssrcStats = statsTracker.getSsrcSnapshot(rtcpSrPacket.senderSsrc) ?: return null + + rtcpSrPacket.senderInfo.sendersOctetCount = ssrcStats.octetCount.toLong() + rtcpSrPacket.senderInfo.sendersPacketCount = ssrcStats.packetCount.toLong() + + return packetInfo + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt new file mode 100644 index 0000000..175f165 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/AudioRtpPacket.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import org.jitsi.rtp.rtp.RtpPacket + +open class AudioRtpPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtpPacket(buffer, offset, length) { + + override fun clone(): AudioRtpPacket = AudioRtpPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ).also { postClone(it) } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ClassicTransportCcEngine.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ClassicTransportCcEngine.kt new file mode 100644 index 0000000..7bbc866 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ClassicTransportCcEngine.kt @@ -0,0 +1,340 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator +import org.jitsi.nlj.util.ArrayCache +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.PacketReport +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.ReceivedPacketReport +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.UnreceivedPacketReport +import org.jitsi.utils.NEVER +import org.jitsi.utils.formatMilli +import org.jitsi.utils.joinToRangedString +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.atomic.LongAdder + +/** + * Implements transport-cc functionality. + * + * See https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01 + * + * @author Boris Grozev + * @author Julian Chukwu + * @author George Politis + */ +class ClassicTransportCcEngine( + private val bandwidthEstimator: BandwidthEstimator, + parentLogger: Logger, + private val clock: Clock = Clock.systemUTC() +) : TransportCcEngine() { + + /** + * The [Logger] used by this instance for logging output. + */ + private val logger: Logger = createChildLogger(parentLogger) + + val numPacketsReported = LongAdder() + val numPacketsReportedLost = LongAdder() + val numDuplicateReports = LongAdder() + val numPacketsReportedAfterLost = LongAdder() + val numPacketsUnreported = LongAdder() + val numMissingPacketReports = LongAdder() + + /** + * The reference time of the remote clock. This is used to rebase the + * arrival times in the TCC packets to a meaningful time base (that of the + * sender). This is technically not necessary and it's done for convenience. + */ + private var remoteReferenceTime: Instant = NEVER + + /** + * Local time to map to the reference time of the remote clock. This is used + * to rebase the arrival times in the TCC packets to a meaningful time base + * (that of the sender). This is technically not necessary and it's done for + * convenience. + */ + private var localReferenceTime: Instant = NEVER + + /** + * Holds a key value pair of the packet sequence number and an object made + * up of the packet send time and the packet size. + */ + private val sentPacketDetails = PacketDetailTracker(clock) + + private val missingPacketDetailSeqNums = mutableListOf() + + private var lastRtt: Duration? = null + + /** + * Called when an RTP sender has a new round-trip time estimate. + */ + override fun onRttUpdate(rtt: Duration) { + val now = clock.instant() + bandwidthEstimator.onRttUpdate(now, rtt) + lastRtt = rtt + } + + override fun rtcpPacketReceived(rtcpPacket: RtcpPacket, receivedTime: Instant?) { + if (rtcpPacket is RtcpFbTccPacket) { + tccReceived(rtcpPacket) + } + } + + private fun tccReceived(tccPacket: RtcpFbTccPacket) { + val now = clock.instant() + var currArrivalTimestamp = tccPacket.BaseTime() + if (remoteReferenceTime == NEVER) { + remoteReferenceTime = currArrivalTimestamp + localReferenceTime = now + } + + for (packetReport in tccPacket) { + val tccSeqNum = packetReport.seqNum + val packetDetail = sentPacketDetails.get(tccSeqNum) + + if (packetDetail == null) { + if (packetReport is ReceivedPacketReport) { + currArrivalTimestamp += packetReport.deltaDuration + missingPacketDetailSeqNums.add(tccSeqNum) + numMissingPacketReports.increment() + } + continue + } + + when (packetReport) { + is UnreceivedPacketReport -> { + if (packetDetail.state == PacketDetailState.Unreported) { + bandwidthEstimator.processPacketLoss(now, packetDetail.packetSendTime, tccSeqNum) + packetDetail.state = PacketDetailState.ReportedLost + numPacketsReported.increment() + numPacketsReportedLost.increment() + synchronized(this) { + lossListeners.forEach { + it.packetLost(1) + } + } + } + } + + is ReceivedPacketReport -> { + currArrivalTimestamp += packetReport.deltaDuration + + when (packetDetail.state) { + PacketDetailState.Unreported, PacketDetailState.ReportedLost -> { + val previouslyReportedLost = packetDetail.state == PacketDetailState.ReportedLost + if (previouslyReportedLost) { + numPacketsReportedAfterLost.increment() + numPacketsReportedLost.decrement() + /* Packet has already been counted in numPacketsReported */ + } else { + numPacketsReported.increment() + } + + val arrivalTimeInLocalClock = + currArrivalTimestamp - Duration.between(localReferenceTime, remoteReferenceTime) + + bandwidthEstimator.processPacketArrival( + now, + packetDetail.packetSendTime, + arrivalTimeInLocalClock, + tccSeqNum, + packetDetail.packetLength, + previouslyReportedLost = previouslyReportedLost + ) + synchronized(this) { + lossListeners.forEach { + it.packetReceived(previouslyReportedLost) + } + } + packetDetail.state = PacketDetailState.ReportedReceived + } + + PacketDetailState.ReportedReceived -> + numDuplicateReports.increment() + } + } + } + } + bandwidthEstimator.feedbackComplete(now) + + if (missingPacketDetailSeqNums.isNotEmpty()) { + logger.warn( + "TCC packet contained received sequence numbers: " + + "${tccPacket.iterator().asSequence() + .filterIsInstance() + .map(PacketReport::seqNum) + .joinToRangedString()}. " + + "Couldn't find packet detail for the seq nums: " + + "${missingPacketDetailSeqNums.joinToRangedString()}. " + + if (sentPacketDetails.empty) { + "Sent packet details map was empty." + } else { + "Latest seqNum was ${sentPacketDetails.lastSequence}, size is ${sentPacketDetails.size}." + } + + ( + lastRtt?.let { + " Latest RTT is ${it.formatMilli()} ms." + } ?: run { + "" + } + ) + ) + missingPacketDetailSeqNums.clear() + } + } + + override fun mediaPacketTagged(packetInfo: PacketInfo, tccSeqNum: Long) { + /* Nothing needs to be done */ + } + + override fun mediaPacketSent(packetInfo: PacketInfo, tccSeqNum: Long) { + val now = clock.instant() + val seq = (tccSeqNum and 0xFFFF).toInt() + if (!sentPacketDetails.insert(seq, PacketDetail(packetInfo.packet.length.bytes, now))) { + /* Very old seq? Something odd is happening with whatever is + * generating tccSeqNum values. + */ + logger.warn( + "Not inserting very old TCC seq num $seq ($tccSeqNum), latest is " + + "${sentPacketDetails.lastSequence}, size is ${sentPacketDetails.size}" + ) + return + } + } + + override fun getStatistics(): StatisticsSnapshot = StatisticsSnapshot( + numPacketsReported.sum(), + numPacketsReportedLost.sum(), + numDuplicateReports.sum(), + numPacketsReportedAfterLost.sum(), + numPacketsUnreported.sum(), + numMissingPacketReports.sum(), + bandwidthEstimator.getStats() + ) + + override fun addBandwidthListener(listener: BandwidthListener) = bandwidthEstimator.addListener(listener) + + override fun removeBandwidthListener(listener: BandwidthListener) = bandwidthEstimator.removeListener(listener) + + override fun start() { } + + override fun stop() { } + + /** + * [PacketDetailState] is the state of a [PacketDetail] + */ + private enum class PacketDetailState { + Unreported, + ReportedLost, + ReportedReceived + } + + /** + * [PacketDetail] is an object that holds the + * length(size) of the packet in [packetLength] + * and the time stamps of the outgoing packet + * in [packetSendTime] + */ + @ConsistentCopyVisibility + private data class PacketDetail internal constructor( + val packetLength: DataSize, + val packetSendTime: Instant + ) { + /** + * [state] represents the state of this packet detail with regards to the + * reception of a TCC feedback from the remote side. [PacketDetail]s start out + * as [unreported]: once we receive a TCC feedback from the remote side referring + * to this packet, the state will transition to either [reportedLost] or [reportedReceived]. + */ + var state = PacketDetailState.Unreported + } + + class StatisticsSnapshot( + val numPacketsReported: Long, + val numPacketsReportedLost: Long, + val numDuplicateReports: Long, + val numPacketsReportedAfterLost: Long, + val numPacketsUnreported: Long, + val numMissingPacketReports: Long, + val bandwidthEstimatorStats: BandwidthEstimator.StatisticsSnapshot + ) : TransportCcEngine.StatisticsSnapshot() { + override fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().also { + it.put("name", ClassicTransportCcEngine::class.java.simpleName) + it.put("numPacketsReported", numPacketsReported) + it.put("numPacketsReportedLost", numPacketsReportedLost) + it.put("numDuplicateReports", numDuplicateReports) + it.put("numPacketsReportedAfterLost", numPacketsReportedAfterLost) + it.put("numPacketsUnreported", numPacketsUnreported) + it.put("numMissingPacketReports", numMissingPacketReports) + it.set("bandwidth_estimator_stats", bandwidthEstimatorStats.toJson()) + } + } + + private inner class PacketDetailTracker(clock: Clock) : ArrayCache( + MAX_OUTGOING_PACKETS_HISTORY, + /* We don't want to clone [PacketDetail] objects that get put in the tracker. */ + { it }, + clock = clock + ) { + override fun discardItem(item: PacketDetail) { + if (item.state == PacketDetailState.Unreported) { + numPacketsUnreported.increment() + } + } + + private val rtpSequenceIndexTracker = RtpSequenceIndexTracker() + + /** + * Gets a packet with a given RTP sequence number from the cache. + */ + fun get(sequenceNumber: Int): PacketDetail? { + // Note that we use [interpret] because we don't want the ROC to get out of sync because of funny requests + // (TCCs) + val index = rtpSequenceIndexTracker.interpret(sequenceNumber) + return super.getContainer(index, shouldCloneItem = true)?.item + } + + fun insert(seq: Int, packetDetail: PacketDetail): Boolean { + val index = rtpSequenceIndexTracker.update(seq) + return super.insertItem(packetDetail, index, packetDetail.packetSendTime.toEpochMilli()) + } + + val lastSequence: Int + get() = if (lastIndex == -1L) -1 else (lastIndex and 0xFFFF).toInt() + } + + companion object { + /** + * The maximum number of received packets and their timestamps to save. + * + * NOTE rtt + minimum amount + * XXX this is an uninformed value. + */ + private const val MAX_OUTGOING_PACKETS_HISTORY = 1000 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/LossListener.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/LossListener.kt new file mode 100644 index 0000000..d837232 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/LossListener.kt @@ -0,0 +1,65 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import org.jitsi.utils.secs +import org.jitsi.utils.stats.RateTracker + +class LossTracker : LossListener { + private val lostPackets = RateTracker(60.secs, 1.secs) + private val receivedPackets = RateTracker(60.secs, 1.secs) + + @Synchronized + override fun packetReceived(previouslyReportedLost: Boolean) { + receivedPackets.update(1) + if (previouslyReportedLost) { + lostPackets.update(-1) + } + } + + @Synchronized + override fun packetLost(numLost: Int) { + lostPackets.update(numLost.toLong()) + } + + @Synchronized + fun getSnapshot(): Snapshot = Snapshot( + lostPackets.getAccumulatedCount(), + receivedPackets.getAccumulatedCount() + ) + + data class Snapshot( + val packetsLost: Long, + val packetsReceived: Long + ) { + fun toJson() = JsonNodeFactory.instance.objectNode().apply { + put("packets_lost", packetsLost) + put("packets_received", packetsReceived) + } + } +} + +/** + * An interface to report when a packet is received, or is observed to be lost. + */ +/* TODO? This kind of overlaps with BandwidthEstimator? But it can be used in cases where we +* don't have all the information the BandwidthEstimator API needs. */ +interface LossListener { + fun packetReceived(previouslyReportedLost: Boolean) + fun packetLost(numLost: Int) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt new file mode 100644 index 0000000..12a4dbf --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacket.kt @@ -0,0 +1,54 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +import org.jitsi.nlj.util.BufferPool +import org.jitsi.rtp.rtp.RtpHeader + +class PaddingVideoPacket private constructor( + buffer: ByteArray, + offset: Int, + length: Int +) : VideoRtpPacket(buffer, offset, length) { + + override fun clone(): PaddingVideoPacket = throw NotImplementedError("clone() not supported for padding packets.") + + companion object { + /** + * Creating a PaddingVideoPacket by directly grabbing a buffer in its + * ctor is problematic because we cannot clear the buffer we retrieve + * before calling the parent class' constructor. Because the buffer + * may contain invalid data, any attempts to parse it by parent class(es) + * could fail, so we use a helper here instead + */ + fun create(length: Int): PaddingVideoPacket { + val buf = BufferPool.getBuffer(length) + // It's possible we the buffer we pulled from the pool already has + // data in it, and we won't be overwriting it with anything so clear + // out the data + buf.fill(0, 0, length) + + return PaddingVideoPacket(buf, 0, length).apply { + // Recalculate the header length now that we've zero'd everything out + // and set the fields + version = RtpHeader.VERSION + headerLength = RtpHeader.getTotalLength(buffer, offset) + paddingSize = payloadLength + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt new file mode 100644 index 0000000..5614e6f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ParsedVideoPacket.kt @@ -0,0 +1,40 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +/** + * A [VideoRtpPacket] which has been parsed and therefore + * contains more information, such as whether or not it's + * a keyframe + */ +abstract class ParsedVideoPacket( + buffer: ByteArray, + offset: Int, + length: Int, + encodingId: Int +) : VideoRtpPacket(buffer, offset, length, encodingId) { + + abstract val isKeyframe: Boolean + abstract val isStartOfFrame: Boolean + abstract val isEndOfFrame: Boolean + + /** Whether the packet meets the needs of the routing infrastructure. + * If a packet could be parsed more than one way (e.g. it is VP8 or VP9 but also has an AV1 DD) + * this will let us choose which parse to prefer. + */ + abstract fun meetsRoutingNeeds(): Boolean +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt new file mode 100644 index 0000000..4992df3 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RedAudioRtpPacket.kt @@ -0,0 +1,52 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import org.jitsi.rtp.rtp.RedPacketBuilder +import org.jitsi.rtp.rtp.RedPacketParser +import java.lang.IllegalStateException + +class RedAudioRtpPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : AudioRtpPacket(buffer, offset, length) { + + private var removed = false + + fun removeRed() { + if (removed) throw IllegalStateException("RED encapsulation already removed.") + parser.decapsulate(this, parseRedundancy = false) + removed = true + } + + fun removeRedAndGetRedundancyPackets(): List = if (removed) { + throw IllegalStateException("RED encapsulation already removed.") + } else { + parser.decapsulate(this, parseRedundancy = true).also { removed = true } + } + + override fun clone(): RedAudioRtpPacket = RedAudioRtpPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ).also { postClone(it) } + + companion object { + val parser = RedPacketParser(::AudioRtpPacket) + val builder = RedPacketBuilder(::RedAudioRtpPacket) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt new file mode 100644 index 0000000..9aee532 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/ResumableStreamRewriter.kt @@ -0,0 +1,259 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import org.jitsi.nlj.util.ArrayCache +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.isNewerThan + +/** + * Rewrites sequence numbers for RTP streams by hiding any gaps caused by + * dropped packets. Rewriters are not thread-safe. If multiple threads access a + * rewriter concurrently, it must be synchronized externally. + * + * Port of the class in libjitsi. + * + * @author Maryam Daneshi + * @author George Politis + * @author Boris Grozev + * @author Jonathan Lennox + */ +class ResumableStreamRewriter(val keepHistory: Boolean = false) { + /** + * The sequence number delta between what's been accepted and what's been + * received, mod 2^16. + */ + var seqnumDelta = 0 + private set + + private var history: StreamRewriteHistory? = null + + /** + * The highest sequence number that got accepted, mod 2^16. + */ + var highestSequenceNumberSent = -1 + private set + + /** + * Rewrites the sequence number of the given RTP packet hiding any gaps caused by drops. + * + * @param accept true if the packet is accepted, false otherwise + */ + fun rewriteRtp(accept: Boolean, rtpPacket: RtpPacket) { + val sequenceNumber = rtpPacket.sequenceNumber + val newSequenceNumber = rewriteSequenceNumber(accept, sequenceNumber) + if (sequenceNumber != newSequenceNumber) { + rtpPacket.sequenceNumber = newSequenceNumber + } + } + + /** + * Rewrites the sequence number passed as a parameter, hiding any gaps + * caused by drops. + * + * @param accept true if the packet is accepted, false otherwise + * @param sequenceNumber the sequence number to rewrite + * @return a rewritten sequence number that hides any gaps caused by drops. + */ + fun rewriteSequenceNumber(accept: Boolean, sequenceNumber: Int): Int { + if (keepHistory && !accept && history == null) { + /* Don't instantiate history until it's needed; many streams never discard. */ + history = StreamRewriteHistory(highestSequenceNumberSent) + } + + history?.let { + return it.rewriteSequenceNumber(accept, sequenceNumber) + } + + if (accept) { + // overwrite the sequence number (if needed) + val newSequenceNumber = (sequenceNumber - seqnumDelta) and 0xffff + + // init or update the highest sent sequence number (if needed) + if (highestSequenceNumberSent == -1 || newSequenceNumber isNewerThan highestSequenceNumberSent) { + highestSequenceNumberSent = newSequenceNumber + } + + return newSequenceNumber + } else { + // update the sequence number delta (if needed) + if (highestSequenceNumberSent != -1) { + val newDelta = (sequenceNumber - highestSequenceNumberSent) and 0xffff + + if (newDelta isNewerThan seqnumDelta) { + seqnumDelta = newDelta + } + } + + return sequenceNumber + } + } + + val gapsLeft: Int + get() = history?.gapsLeft ?: 0 + + private class RewriteHistoryItem( + var accept: Boolean?, + val newIndex: Long + ) + + private class StreamRewriteHistory(highestSeqSent: Int) : ArrayCache( + MAX_REWRITE_HISTORY, + /* We don't want to clone objects that get put in the tracker. */ + { it }, + /* Caller should have this object synchronized if needed. */ + synchronize = false + ) { + var firstIndex: Long = -1L + + var gapsLeft = 0 + private set + + private val rtpSequenceIndexTracker = RtpSequenceIndexTracker() + + private fun fillBetween(start: Long, end: Long, firstNewIndex: Long) { + if (end <= lastIndex - size + 1) { + return + } + val actualStart = if (start <= lastIndex - size) { + lastIndex - size + 1 + } else { + start + } + + var newIndex = firstNewIndex + + for (i in actualStart..end) { + insertItem(RewriteHistoryItem(null, newIndex), i) + newIndex++ + } + } + + fun rewriteSequenceNumber(accept: Boolean, sequenceNumber: Int): Int { + val index = rtpSequenceIndexTracker.update(sequenceNumber) + + val newIndex: Long + + when { + (firstIndex == -1L) -> { + /* First index seen. */ + insertItem(RewriteHistoryItem(accept, index), index) + + firstIndex = index + + newIndex = index + } + + (index > lastIndex) -> { + /* New. Roll forward, filling in gap if necessary. */ + + val newestIndex = lastIndex + + val newest = getContainer(newestIndex) + ?: throw IllegalStateException("No newest container found") + + val newestNewIndex = newest.item!!.newIndex + + val indexGap = index - newestIndex + + val newGap = indexGap - if (accept) 0 else 1 + + newIndex = newestNewIndex + newGap + + insertItem(RewriteHistoryItem(accept, newIndex), index) + + fillBetween(newestIndex + 1, index - 1, newestNewIndex + 1) + } + + (index > lastIndex - size && index >= firstIndex) -> { + /* In history. Retrieve. */ + + val container = getContainer(index) + ?: throw IllegalStateException("No container found for index $index in history") + val item = container.item!! + + if (item.accept == null) { + item.accept = accept + if (!accept) { + gapsLeft++ + } + } + + newIndex = item.newIndex + } + + (index > lastIndex - size && index < firstIndex) -> { + /* Older than previous oldest, but still in range of the map. + Project map backwards. */ + + val oldestIndex = Math.max(lastIndex - size + 1, firstIndex) + val oldest = getContainer(oldestIndex) + ?: throw IllegalStateException("No oldest container found") + + val oldestNewIndex = oldest.item!!.newIndex + + val indexGap = index - oldestIndex // Negative + + val newGap = indexGap + if (accept) 0 else 1 + + newIndex = oldestNewIndex + newGap + + insertItem(RewriteHistoryItem(accept, newIndex), index) + + fillBetween(index + 1, oldestIndex - 1, newIndex + 1) + + firstIndex = index + } + + else -> { + /* Older than the map. */ + + val oldestIndex = Math.max(lastIndex - size + 1, firstIndex) + val oldest = getContainer(oldestIndex) + ?: throw IllegalStateException("No oldest container found") + val oldestDelta = oldestIndex - oldest.item!!.newIndex + + newIndex = index - oldestDelta + } + } + + if (accept) return toSequenceNumber(newIndex) + /* Don't care about sequence numbers for non-accepted packets, + so make sure rewriteRtp does nothing. */ + return sequenceNumber + } + + init { + if (highestSeqSent != -1) { + rewriteSequenceNumber(true, highestSeqSent) + } + } + + companion object { + /** + * The maximum number of packets to save history for. + * + * NOTE rtt + minimum amount + * XXX this is an uninformed value. + */ + private const val MAX_REWRITE_HISTORY = 1000 + + /** Map an index back to a sequence number. */ + private fun toSequenceNumber(index: Long): Int = (index and 0xffff).toInt() + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt new file mode 100644 index 0000000..c1fb067 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtpExtensions.kt @@ -0,0 +1,123 @@ +/* + * Copyright @ 2019-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +/** + * Represents a signalled RTP header extension. + * + * @author Boris Grozev + */ +data class RtpExtension( + /** + * The ID that was signalled. + */ + val id: Byte, + /** + * The type. + */ + val type: RtpExtensionType +) + +/** + * Represents an RTP header extension type. + * + * @author Boris Grozev + */ +enum class RtpExtensionType(val uri: String) { + /** + * The URN identifying the RTP extension that allows mixers to send to + * conference participants the audio levels of all contributing sources. + * Defined in RFC6465. + */ + CSRC_AUDIO_LEVEL("urn:ietf:params:rtp-hdrext:csrc-audio-level"), + + /** + * The URN identifying the RTP extension that allows clients to send to + * conference mixers the audio level of their packet payload. Defined in + * RFC6464. + */ + SSRC_AUDIO_LEVEL("urn:ietf:params:rtp-hdrext:ssrc-audio-level"), + + /** + * The URN identifying the abs-send-time RTP extension. + * Defined at + * {@link "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"} + */ + ABS_SEND_TIME("http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"), + + /** + * The URN which identifies the framemarking RTP extension defined at + * {@link "https://tools.ietf.org/html/draft-ietf-avtext-framemarking-03"} + */ + FRAME_MARKING("http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"), + + /** + * The URN which identifies the Original Header Block RTP extension defined + * in {@link "https://tools.ietf.org/html/draft-ietf-perc-double-02"}. + */ + ORIGINAL_HEADER_BLOCK("urn:ietf:params:rtp-hdrext:ohb"), + + /** + * The URN which identifies the Transport-Wide Congestion Control RTP + * extension. + */ + TRANSPORT_CC("http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"), + + /** + * The URN which identifies the rtp-stream-id extensions + * in {@link "https://tools.ietf.org/html/draft-ietf-mmusic-rid-10"}. + */ + RTP_STREAM_ID("urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"), + + /** + * The URN which identifies the transmission time-offset extensions + * in {@link "https://tools.ietf.org/html/rfc5450"}. + */ + TOF("urn:ietf:params:rtp-hdrext:toffset"), + + /** + * The URN which identifies the RTP Header Extension for Video Content Type. + */ + VIDEO_CONTENT_TYPE("http://www.webrtc.org/experiments/rtp-hdrext/video-content-type"), + + /** + * The URN which identifies the RTP Header Extension for Video Orientation. + */ + VIDEO_ORIENTATION("urn:3gpp:video-orientation"), + + /** + * The URN which identifies the AV1 Dependency Descriptor RTP Header Extension + */ + AV1_DEPENDENCY_DESCRIPTOR( + "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension" + ), + + /** + * Video Layers Allocation + * https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/video-layers-allocation00 + */ + VLA("http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00"), + + /** + * The URN which identifies the sdes:mid (media identification) RTP header extension defined in RFC 8843/9143. + */ + MID("urn:ietf:params:rtp-hdrext:sdes:mid"); + + companion object { + private val uriMap = RtpExtensionType.values().associateBy(RtpExtensionType::uri) + fun createFromUri(uri: String): RtpExtensionType? = uriMap.getOrDefault(uri, null) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtxPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtxPacket.kt new file mode 100644 index 0000000..03609f4 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/RtxPacket.kt @@ -0,0 +1,62 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +import org.jitsi.nlj.util.shiftPayloadRight +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.getShortAsInt + +/** + * https://tools.ietf.org/html/rfc4588#section-4 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | RTP Header | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | OSN | | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | + * | Original RTP Packet Payload | + * | | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class RtxPacket { + companion object { + fun getOriginalSequenceNumber(rtxPacket: RtpPacket): Int = + rtxPacket.buffer.getShortAsInt(rtxPacket.offset + rtxPacket.headerLength) + + /** + * Removes the original sequence number by shifting the header 2 + * bytes to the right + */ + fun removeOriginalSequenceNumber(rtxPacket: RtpPacket) = rtxPacket.apply { + // Remove the original sequence number by moving the RTP header 2 bytes to the right + // Note this changes the buffer underlying the RtpPacket -- this is safe (but fragile) + // because we leave header values (which are cached) unchanged. + System.arraycopy(buffer, offset, buffer, offset + 2, headerLength) + offset += 2 + length -= 2 + } + + fun addOriginalSequenceNumber(rtpPacket: RtpPacket) = rtpPacket.apply { + // TODO: possible optimization to try to shift the header left instead + shiftPayloadRight(2) + buffer.putShort(offset + headerLength, sequenceNumber.toShort()) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/SsrcAssociationType.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/SsrcAssociationType.kt new file mode 100644 index 0000000..6bf50be --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/SsrcAssociationType.kt @@ -0,0 +1,23 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +enum class SsrcAssociationType { + RTX, + SIM, + FEC +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt new file mode 100644 index 0000000..1bbbf0e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/TransportCcEngine.kt @@ -0,0 +1,83 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtcp.RtcpListener +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.rtp.rtcp.RtcpPacket +import java.time.Duration +import java.time.Instant + +abstract class TransportCcEngine : RtcpListener { + protected val lossListeners = mutableListOf() + + /** + * Called when an RTP sender has a new round-trip time estimate. + */ + abstract fun onRttUpdate(rtt: Duration) + + abstract override fun rtcpPacketReceived(rtcpPacket: RtcpPacket, receivedTime: Instant?) + + /** This is called when a tccSeqNum is first assigned to a packet, i.e. the soonest the packet can meaningfully + * be described to the engine. */ + abstract fun mediaPacketTagged(packetInfo: PacketInfo, tccSeqNum: Long) + + abstract fun mediaPacketSent(packetInfo: PacketInfo, tccSeqNum: Long) + + abstract fun getStatistics(): StatisticsSnapshot + + /** + * Adds a loss listener to be notified about packet arrival and loss reports. + * @param listener + */ + @Synchronized + fun addLossListener(listener: LossListener) { + lossListeners.add(listener) + } + + /** + * Removes a loss listener. + * @param listener + */ + @Synchronized + fun removeLossListener(listener: LossListener) { + lossListeners.remove(listener) + } + + abstract fun addBandwidthListener(listener: BandwidthListener) + + abstract fun removeBandwidthListener(listener: BandwidthListener) + + abstract fun start() + + abstract fun stop() + + abstract class StatisticsSnapshot { + abstract fun toJson(): ObjectNode + + /** Cumulative count of transport-cc feedback reports that could not be matched against + * the send-time history. A sustained flood of these means the estimator has lost both + * its loss and delay signals (unmatched reports are dropped entirely) and is the + * earliest observable marker of a feedback-starvation saturation lock. */ + open val unmatchedFeedback: Long get() = 0 + } + + interface BandwidthListener { + fun bandwidthEstimationChanged(newValue: Bandwidth) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt new file mode 100644 index 0000000..6a8e121 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/VideoRtpPacket.kt @@ -0,0 +1,44 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.rtp.rtp.RtpPacket + +/** + * A packet which we know contains video, but we've not yet + * parsed (i.e. we don't know information gained from + * parsing codec-specific data). + */ +open class VideoRtpPacket @JvmOverloads constructor( + buffer: ByteArray, + offset: Int, + length: Int, + /** The encoding ID of this packet. */ + var encodingId: Int = RtpLayerDesc.SUSPENDED_ENCODING_ID +) : RtpPacket(buffer, offset, length) { + + open val layerIds: Collection = listOf(0) + + override fun toString(): String = super.toString() + ", EncID=$encodingId" + + override fun clone(): VideoRtpPacket = VideoRtpPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length, + encodingId = encodingId + ).also { postClone(it) } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimator.kt new file mode 100644 index 0000000..c70ead4 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimator.kt @@ -0,0 +1,353 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.bandwidthestimation + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.utils.NEVER +import org.jitsi.utils.formatMilli +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.LinkedList + +/** + * An abstract interface to a bandwidth estimation algorithm. + * + * The invoker of the algorithm will periodically call [processPacketArrival] + * and/or [processPacketLoss] as it learns information about packets + * that have traversed the network. + * + * All bandwidths/bitrates are in bits per second. + */ +abstract class BandwidthEstimator( + protected val diagnosticContext: DiagnosticContext +) { + /** + * The [TimeSeriesLogger] to be used by this instance to print time + * series. + */ + protected val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass) + + /** The name of the algorithm implemented by this [BandwidthEstimator]. */ + abstract val algorithmName: String + + /** The initial bandwidth estimate. */ + abstract var initBw: Bandwidth + + /** The minimum bandwidth the estimator will return. */ + abstract var minBw: Bandwidth + + /** The maximum bandwidth the estimator will return. */ + abstract var maxBw: Bandwidth + + /** + * Inform the bandwidth estimator about a packet that has arrived at its + * destination. + * + * This function will be called at most once for any value of [seq] (up to cycles); + * however, it may be called after a call to [processPacketLoss] for the + * same [seq] value, if a packet is delayed. + * + * It is possible (e.g., if feedback was lost) that neither + * [processPacketArrival] nor [processPacketLoss] is called for a given [seq]. + * + * The clocks reported by [now], [sendTime], and [recvTime] do not + * necessarily have any relationship to each other, but must be consistent + * within themselves across all calls to functions of this [BandwidthEstimator]. + * + * All arrival and loss reports based on a single feedback message should have the + * same [now] value. [feedbackComplete] should be called once all feedback reports + * based on a single feedback message have been processed. + * + * @param[now] The current time, when this function is called. + * @param[sendTime] The time the packet was sent, if known, or null. + * @param[recvTime] The time the packet was received, if known, or null. + * @param[seq] A 16-bit sequence number of packets processed by this + * [BandwidthEstimator]. + * @param[size] The size of the packet. + * @param[ecn] The ECN markings with which the packet was received. + * @param[previouslyReportedLost] Whether [processPacketLoss] was previously + * called for this packet. + */ + fun processPacketArrival( + now: Instant, + sendTime: Instant?, + recvTime: Instant?, + seq: Int, + size: DataSize, + ecn: Byte = 0, + previouslyReportedLost: Boolean = false + ) { + if (timeSeriesLogger.isTraceEnabled) { + val point = diagnosticContext.makeTimeSeriesPoint("bwe_packet_arrival", now) + if (sendTime != null) { + point.addField("sendTime", sendTime.formatMilli()) + } + if (recvTime != null) { + point.addField("recvTime", recvTime.formatMilli()) + } + point.addField("seq", seq) + point.addField("size", size.bytes) + if (ecn != 0.toByte()) { + point.addField("ecn", ecn) + } + point.addField("previouslyReportedLost", previouslyReportedLost) + timeSeriesLogger.trace(point) + } + + doProcessPacketArrival(now, sendTime, recvTime, seq, size, ecn, previouslyReportedLost) + } + + /** + * A subclass's implementation of [processPacketArrival]. + * + * See that function for parameter details. + */ + protected abstract fun doProcessPacketArrival( + now: Instant, + sendTime: Instant?, + recvTime: Instant?, + seq: Int, + size: DataSize, + ecn: Byte = 0, + previouslyReportedLost: Boolean = false + ) + + /** + * Inform the bandwidth estimator that a packet was lost. + * + * All arrival and loss reports based on a single feedback message should have the + * same [now] value. [feedbackComplete] should be called once all feedback reports + * based on a single feedback message have been processed. + * + * @param[now] The current time, when this function is called. + * @param[sendTime] The time the packet was sent, if known, or null. + * @param[seq] A 16-bit sequence number of packets processed by this + * [BandwidthEstimator]. + */ + fun processPacketLoss(now: Instant, sendTime: Instant?, seq: Int) { + if (timeSeriesLogger.isTraceEnabled) { + val point = diagnosticContext.makeTimeSeriesPoint("bwe_packet_loss", now) + if (sendTime != null) { + point.addField("sendTime", sendTime.formatMilli()) + } + point.addField("seq", seq) + timeSeriesLogger.trace(point) + } + + doProcessPacketLoss(now, sendTime, seq) + } + + /** + * A subclass's implementation of [processPacketLoss]. + * + * See that function for parameter details. + */ + protected abstract fun doProcessPacketLoss(now: Instant, sendTime: Instant?, seq: Int) + + /** + * Inform the bandwidth estimator that a block of feedback is complete. + * + * @param[now] The current time, when this function is called. This should match + * the value passed to [processPacketArrival] and [processPacketLoss]. + */ + fun feedbackComplete(now: Instant) { + if (timeSeriesLogger.isTraceEnabled) { + val point = diagnosticContext.makeTimeSeriesPoint("bwe_feedback_complete", now) + timeSeriesLogger.trace(point) + } + + doFeedbackComplete(now) + } + + /** + * A subclass's implementation of [feedbackComplete]. + * + * See that function for parameter details. + */ + protected abstract fun doFeedbackComplete(now: Instant) + + /** + * Inform the bandwidth estimator about a new round-trip time value + */ + fun onRttUpdate(now: Instant, newRtt: Duration) { + if (timeSeriesLogger.isTraceEnabled) { + val point = diagnosticContext.makeTimeSeriesPoint("bwe_rtt", now) + point.addField("rtt", newRtt.formatMilli()) + timeSeriesLogger.trace(point) + } + + doRttUpdate(now, newRtt) + } + + /** + * A subclass's implementation of [onRttUpdate]. + * + * See that function for parameter details. + */ + protected abstract fun doRttUpdate(now: Instant, newRtt: Duration) + + /** Get the estimator's current estimate of the available bandwidth. + * + * @param[now] The current time, when this function is called. + */ + abstract fun getCurrentBw(now: Instant): Bandwidth + + /** Get the current statistics related to this estimator. + * + * @param[now] The current time, when this function is called. + */ + abstract fun getStats(now: Instant = Clock.systemUTC().instant()): StatisticsSnapshot + + /** Reset the estimator to its initial state. */ + abstract fun reset() + + private val listeners = LinkedList() + private var curBandwidth = (-1).bps + + private var lastBweLogTime = NEVER + private val minBweLogInterval = Duration.ofMillis(500) + + /** + * Notifies registered listeners that the estimate of the available + * bandwidth has changed. + */ + @Synchronized + protected fun reportBandwidthEstimate(now: Instant, newValue: Bandwidth) { + if (timeSeriesLogger.isTraceEnabled) { + if (newValue != curBandwidth || + lastBweLogTime == NEVER || + Duration.between(lastBweLogTime, now) >= minBweLogInterval + ) { + val point = diagnosticContext.makeTimeSeriesPoint("bwe_estimate", now) + point.addField("bw", newValue.bps) + timeSeriesLogger.trace(point) + lastBweLogTime = now + } + } + + if (newValue == curBandwidth) { + return + } + for (listener in listeners) { + listener.bandwidthEstimationChanged(newValue) + } + curBandwidth = newValue + } + + /** + * Adds a listener to be notified about changes to the bandwidth estimation. + * @param listener + */ + @Synchronized + fun addListener(listener: TransportCcEngine.BandwidthListener) { + listeners.add(listener) + } + + /** + * Removes a listener. + * @param listener + */ + @Synchronized + fun removeListener(listener: TransportCcEngine.BandwidthListener) { + listeners.remove(listener) + } + + /** + * Holds a snapshot of stats specific to the bandwidth estimator. + */ + class StatisticsSnapshot(algorithmName: String, currentEstimate: Bandwidth) { + private val stats = mutableMapOf() + var algorithmName: String by stats + var currentEstimate: Bandwidth by stats + + init { + addString("algorithmName", algorithmName) + addBandwidth("currentEstimate", currentEstimate) + } + + fun getValue(name: String): Any? = stats[name] + + /** + * Gets the value of a stat with a given name, if this [StatisticsSnapshot] has it and it is a [Number]. + * Otherwise returns 'null'. + */ + fun getNumber(name: String): Number? = stats[name] as? Number + + /** + * Promotes integer values to [Long] and floating point values to [Double]. Returns a + * [Long], [Double], or null. + */ + private fun promote(n: Number): Number? = when (n) { + is Byte, is Short, is Int, is Long -> n.toLong() + is Float, is Double -> n.toDouble() + else -> null + } + + /** + * Adds a stat with a number value. Integral values are promoted to [Long], while floating point values are + * promoted to [Double]. + */ + fun addNumber(name: String, value: Number) { + promote(value)?.let { stats[name] = it } + } + + /** + * Adds a stat with a string value. + */ + fun addString(name: String, value: String) { + stats[name] = value + } + + /** + * Adds a stat with a boolean value. + */ + fun addBoolean(name: String, value: Boolean) { + stats[name] = value + } + + /** + * Adds a stat with a bandwidth value. + */ + fun addBandwidth(name: String, value: Bandwidth) { + stats[name] = value + } + + /** + * Returns a JSON representation of this [StatisticsSnapshot] object. + */ + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + stats.forEach { (name, value) -> + when (value) { + is Bandwidth -> put(name, value.bps) + is Long -> put(name, value) + is Int -> put(name, value) + is Double -> put(name, value) + is Boolean -> put(name, value) + else -> put(name, value.toString()) + } + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimatorConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimatorConfig.kt new file mode 100644 index 0000000..19fb4ad --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimatorConfig.kt @@ -0,0 +1,38 @@ +package org.jitsi.nlj.rtp.bandwidthestimation + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.util.Bandwidth + +class BandwidthEstimatorConfig { + companion object { + val engine: BandwidthEstimatorEngine by config { + "jmt.bwe.estimator.engine".from(JitsiConfig.newConfig) + .convertFrom { BandwidthEstimatorEngine.valueOf(it) } + } + + val initBw: Bandwidth by config { + "jmt.bwe.estimator.initial-bw".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + + val minBw: Bandwidth by config { + "jmt.bwe.google-cc.min-bw".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) }.softDeprecated("use jmt.bwe.estimator.min-bw") + "jmt.bwe.estimator.min-bw".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + + val maxBw: Bandwidth by config { + "jmt.bwe.google-cc.max-bw".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) }.softDeprecated("use jmt.bwe.estimator.max-bw") + "jmt.bwe.estimator.max-bw".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + } +} + +enum class BandwidthEstimatorEngine { + GoogleCc, + GoogleCc2 +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt new file mode 100644 index 0000000..6733f2b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/GoogleCcEstimator.kt @@ -0,0 +1,146 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.RemoteBitrateEstimatorAbsSendTime +import org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation.SendSideBandwidthEstimation +import java.time.Duration +import java.time.Instant +import kotlin.properties.Delegates + +class GoogleCcEstimator(diagnosticContext: DiagnosticContext, parentLogger: Logger) : + BandwidthEstimator(diagnosticContext) { + override val algorithmName = "Google CC" + + /* TODO: Use configuration service to set this default value. */ + override var initBw: Bandwidth = BandwidthEstimatorConfig.initBw + /* TODO: observable which sets the components' values if we're in initial state. */ + + override var minBw: Bandwidth by Delegates.observable(BandwidthEstimatorConfig.minBw) { _, _, newValue -> + bitrateEstimatorAbsSendTime.setMinBitrate(newValue.bps.toInt()) + sendSideBandwidthEstimation.setMinMaxBitrate(newValue.bps.toInt(), maxBw.bps.toInt()) + } + + override var maxBw: Bandwidth by Delegates.observable(BandwidthEstimatorConfig.maxBw) { _, _, newValue -> + sendSideBandwidthEstimation.setMinMaxBitrate(minBw.bps.toInt(), newValue.bps.toInt()) + } + + private val logger = createChildLogger(parentLogger) + + /** + * Implements the delay-based part of Google CC. + */ + private val bitrateEstimatorAbsSendTime = RemoteBitrateEstimatorAbsSendTime(diagnosticContext, logger).also { + it.setMinBitrate(minBw.bps.toInt()) + } + + /** + * Implements the loss-based part of Google CC. + */ + private val sendSideBandwidthEstimation = + SendSideBandwidthEstimation(diagnosticContext, initBw.bps, logger).also { + it.setMinMaxBitrate(minBw.bps.toInt(), maxBw.bps.toInt()) + } + + override fun doProcessPacketArrival( + now: Instant, + sendTime: Instant?, + recvTime: Instant?, + seq: Int, + size: DataSize, + ecn: Byte, + previouslyReportedLost: Boolean + ) { + if (sendTime != null && recvTime != null) { + bitrateEstimatorAbsSendTime.incomingPacketInfo( + now.toEpochMilli(), + sendTime.toEpochMilli(), + recvTime.toEpochMilli(), + size.bytes.toInt() + ) + } + sendSideBandwidthEstimation.updateReceiverEstimate(bitrateEstimatorAbsSendTime.latestEstimate) + sendSideBandwidthEstimation.reportPacketArrived(now.toEpochMilli(), previouslyReportedLost) + } + + override fun doProcessPacketLoss(now: Instant, sendTime: Instant?, seq: Int) { + sendSideBandwidthEstimation.reportPacketLost(now.toEpochMilli()) + } + + override fun doFeedbackComplete(now: Instant) { + /* TODO: rate-limit how often we call updateEstimate? */ + sendSideBandwidthEstimation.updateEstimate(now.toEpochMilli()) + reportBandwidthEstimate(now, sendSideBandwidthEstimation.latestEstimate.bps) + } + + override fun doRttUpdate(now: Instant, newRtt: Duration) { + bitrateEstimatorAbsSendTime.onRttUpdate(now.toEpochMilli(), newRtt.toMillis()) + sendSideBandwidthEstimation.onRttUpdate(newRtt) + } + + override fun getCurrentBw(now: Instant): Bandwidth = sendSideBandwidthEstimation.latestEstimate.bps + + override fun getStats(now: Instant): StatisticsSnapshot = StatisticsSnapshot( + "GoogleCcEstimator", + getCurrentBw(now) + ).apply { + bitrateEstimatorAbsSendTime.statistics?.run { + addNumber("delayBasedEstimatorOffset", offset) + addNumber("delayBasedEstimatorThreshold", threshold) + addNumber("delayBasedEstimatorHypothesis", hypothesis.value) + } + addNumber("latestDelayBasedEstimate", sendSideBandwidthEstimation.latestREMB) + addNumber("latestLossFraction", sendSideBandwidthEstimation.latestFractionLoss / 256.0) + with(sendSideBandwidthEstimation.statistics) { + update(now.toEpochMilli()) + addNumber("lossDegradedMs", lossDegradedMs) + addNumber("lossFreeMs", lossFreeMs) + addNumber("lossLimitedMs", lossLimitedMs) + } + } + + override fun reset() { + initBw = BandwidthEstimatorConfig.initBw + minBw = BandwidthEstimatorConfig.minBw + maxBw = BandwidthEstimatorConfig.maxBw + + bitrateEstimatorAbsSendTime.reset() + sendSideBandwidthEstimation.reset(initBw.bps) + + sendSideBandwidthEstimation.setMinMaxBitrate(minBw.bps.toInt(), maxBw.bps.toInt()) + } + + companion object { + /* Default config settings to use when the classic Google CC estimator engine is used. */ + val defaultRateTrackerWindowSize: Duration by config { + "jmt.bwe.estimator.GoogleCc.default-window-size".from(JitsiConfig.newConfig) + } + val defaultRateTrackerBucketSize: Duration by config { + "jmt.bwe.estimator.GoogleCc.default-bucket-size".from(JitsiConfig.newConfig) + } + val defaultInitialIgnoreBwePeriod: Duration by config { + "jmt.bwe.estimator.GoogleCc.default-initial-ignore-bwe-period".from(JitsiConfig.newConfig) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimator.kt new file mode 100644 index 0000000..475c645 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimator.kt @@ -0,0 +1,62 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import java.time.Instant + +/** + * Basic implementation to estimate acknowledged bitrate. + * + * Based on WebRTC modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class AcknowledgedBitrateEstimator( + val bitrateEstimator: BitrateEstimator +) : AcknowledgedBitrateEstimatorInterface { + private var alrEndedTime: Instant? = null + private var inAlr = false + + constructor() : this(BitrateEstimator()) + + override fun incomingPacketFeedbackVector(packetFeedbackVector: List) { + check(packetFeedbackVector.isSortedByReceiveTime()) + for (packet in packetFeedbackVector) { + if (alrEndedTime != null && packet.sentPacket.sendTime > alrEndedTime) { + bitrateEstimator.expectFastRateChange() + alrEndedTime = null + } + val acknowledgedEstimate = packet.sentPacket.size + packet.sentPacket.priorUnackedData + bitrateEstimator.update(packet.receiveTime, acknowledgedEstimate, inAlr) + } + } + + override fun bitrate(): Bandwidth? = bitrateEstimator.bitrate() + + override fun peekRate(): Bandwidth? = bitrateEstimator.peekRate() + + override fun setAlr(inAlr: Boolean) { + this.inAlr = inAlr + } + + override fun setAlrEndedTime(alrEndedTime: Instant) { + this.alrEndedTime = alrEndedTime + } +} + +private fun List.isSortedByReceiveTime(): Boolean = this.asSequence().zipWithNext { a, b -> + a.receiveTime <= b.receiveTime +}.all { it } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorInterface.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorInterface.kt new file mode 100644 index 0000000..e869110 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorInterface.kt @@ -0,0 +1,82 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +/** + * Interface to estimate acknowledged bitrate. + * * + * Based on WebRTC modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_interface.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +interface AcknowledgedBitrateEstimatorInterface { + fun incomingPacketFeedbackVector(packetFeedbackVector: List) + fun bitrate(): Bandwidth? + fun peekRate(): Bandwidth? + fun setAlr(inAlr: Boolean) + fun setAlrEndedTime(alrEndedTime: Instant) + + companion object { + fun create( + settings: RobustThroughputEstimatorSettings = RobustThroughputEstimatorSettings() + ): AcknowledgedBitrateEstimatorInterface { + if (settings.enabled) { + return RobustThroughputEstimator(settings) + } else { + return AcknowledgedBitrateEstimator() + } + } + } +} + +data class RobustThroughputEstimatorSettings( + // Set `enabled` to true to use the RobustThroughputEstimator, false to use + // the AcknowledgedBitrateEstimator. + val enabled: Boolean = true, + + // The estimator keeps the smallest window containing at least + // `window_packets` and at least the packets received during the last + // `min_window_duration` milliseconds. + // (This means that it may store more than `window_packets` at high bitrates, + // and a longer duration than `min_window_duration` at low bitrates.) + // However, if will never store more than kMaxPackets (for performance + // reasons), and never longer than max_window_duration (to avoid very old + // packets influencing the estimate for example when sending is paused). + val windowPackets: Int = 20, + val maxWindowPackets: Int = 500, + val minWindowDuration: Duration = 1.secs, + val maxWindowDuration: Duration = 5.secs, + + // The estimator window requires at least `required_packets` packets + // to produce an estimate. + val requiredPackets: Int = 10, + + // If audio packets aren't included in allocation (i.e. the + // estimated available bandwidth is divided only among the video + // streams), then `unacked_weight` should be set to 0. + // If audio packets are included in allocation, but not in bandwidth + // estimation (i.e. they don't have transport-wide sequence numbers, + // but we nevertheless divide the estimated available bandwidth among + // both audio and video streams), then `unacked_weight` should be set to 1. + // If all packets have transport-wide sequence numbers, then the value + // of `unacked_weight` doesn't matter. + val unackedWeight: Double = 1.0 +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControl.kt new file mode 100644 index 0000000..972f41f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControl.kt @@ -0,0 +1,361 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This file uses WebRTC's naming style for enums +@file:Suppress("ktlint:standard:enum-entry-name-case", "ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.div +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.max +import org.jitsi.nlj.util.per +import org.jitsi.nlj.util.times +import org.jitsi.utils.NEVER +import org.jitsi.utils.coerceIn +import org.jitsi.utils.div +import org.jitsi.utils.durationOfDoubleSeconds +import org.jitsi.utils.max +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.times +import org.jitsi.utils.toDouble +import java.time.Duration +import java.time.Instant +import kotlin.math.ceil +import kotlin.math.min +import kotlin.math.pow + +/** + * A rate control implementation based on additive increases of bitrate when no + * over-use is detected and multiplicative decreases when over-uses are + * detected. When we think the available bandwidth has changes or is unknown, we + * will switch to a "slow-start mode" where we increase multiplicatively. + * + * Based on WebRTC modules/remote_bitrate_estimator/aimd_rate_control.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class AimdRateControl(private val sendSide: Boolean = false) { + + private var minConfiguredBitrate: Bandwidth = kCongestionControllerMinBitrate + private var maxConfiguredBitrate: Bandwidth = 30000.kbps + private var currentBitrate: Bandwidth = maxConfiguredBitrate + private var latestEstimatedThroughput: Bandwidth = currentBitrate + + val linkCapacity = LinkCapacityEstimator() + + // Omitted: + // val networkEstimate: NetworkStateEstimate? = null + // As far as I can tell it is not ever set in current Chromium? + + var rateControlState: RateControlState = RateControlState.kRcHold + private set + + private var timeLastBitrateChange: Instant = NEVER + + @SuppressFBWarnings("URF_UNREAD_FIELD") + private var timeLastBitrateDecrease: Instant = NEVER + private var timeFirstThroughputEstimate: Instant = NEVER + + private var bitrateIsInitialized: Boolean = false + private var beta: Double = kDefaultBackoffFactor + var inAlr: Boolean = false + var rtt: Duration = kDefaultRtt + + // TODO: field trials: remove code that checks these + + // Allow the delay based estimate to only increase as long as application + // limited region (alr) is not detected. + private val noBitrateIncreaseInAlr: Boolean = false + + // Only tested when networkEstimate != null + // private val disableEstimateBoundedIncrease: Boolean = false + // private val useCurrentEstimateAsMinUpperBound = true + + private var lastDecrease: Bandwidth? = null + + fun setStartBitrate(startBitrate: Bandwidth) { + currentBitrate = startBitrate + latestEstimatedThroughput = startBitrate + bitrateIsInitialized = true + } + + fun setMinBitrate(minBitrate: Bandwidth) { + minConfiguredBitrate = minBitrate + currentBitrate = max(minBitrate, currentBitrate) + } + + fun validEstimate(): Boolean = bitrateIsInitialized + + fun getFeedbackInterval(): Duration { + // Estimate how often we can send RTCP if we allocate up to 5% of bandwidth + // to feedback. + val kRtcpSize = 80.bytes + val rtcpBitrate = currentBitrate * 0.05 + val interval = kRtcpSize / rtcpBitrate + val kMinFeedbackInterval = 200.ms + val kMaxFeedbackInterval = 1000.ms + return interval.coerceIn(kMinFeedbackInterval, kMaxFeedbackInterval) + } + + fun timeToReduceFurther(atTime: Instant, estimatedThroughput: Bandwidth): Boolean { + val bitrateReductionInterval = rtt.coerceIn(10.ms, 200.ms) + if (Duration.between(timeLastBitrateChange, atTime) >= bitrateReductionInterval) { + return true + } + if (validEstimate()) { + // TODO(terelius/holmer): Investigate consequences of increasing + // the threshold to 0.95 * LatestEstimate(). + val threshold = 0.5 * latestEstimate() + return estimatedThroughput < threshold + } + return false + } + + fun initialTimeToReduceFurther(atTime: Instant): Boolean = validEstimate() && + timeToReduceFurther(atTime, latestEstimate() / 2 - 1.bps) + + fun latestEstimate() = currentBitrate + + fun update(input: RateControlInput, atTime: Instant): Bandwidth { + // Set the initial bit rate value to what we're receiving the first half + // second. + // TODO(bugs.webrtc.org/9379): The comment above doesn't match to the code. + if (!bitrateIsInitialized) { + val kInitializionTime = 5.secs + check(kBitrateWindow <= kInitializionTime) + if (timeFirstThroughputEstimate == NEVER) { + if (input.estimatedThroughput != null) { + timeFirstThroughputEstimate = atTime + } + } else if (Duration.between(timeFirstThroughputEstimate, atTime) > kInitializionTime) { + input.estimatedThroughput?.let { + currentBitrate = it + bitrateIsInitialized = true + } + } + } + changeBitrate(input, atTime) + return currentBitrate + } + + fun setEstimate(bitrate: Bandwidth, atTime: Instant) { + bitrateIsInitialized = true + val prevBitrate = currentBitrate + currentBitrate = clampBitrate(bitrate) + timeLastBitrateChange = atTime + if (currentBitrate < prevBitrate) { + timeLastBitrateDecrease = atTime + } + } + + fun getNearMaxIncreaseRateBpsPerSecond(): Double { + check(currentBitrate != 0.bps) + val kFrameInterval = 1.secs / 30.0 + val frameSize = (currentBitrate * kFrameInterval).toWholeBytes() + val kPacketSize = 1200.bytes + val packetsPerFrame = ceil(frameSize / kPacketSize) + val avgPacketSize = (frameSize / packetsPerFrame).toWholeBytes() + + // Approximate the over-use estimator delay to 100 ms. + var responseTime = rtt + 100.ms + + responseTime = responseTime * 2 + val increaseRateBpsPerSecond = (avgPacketSize.per(responseTime)).bps.toDouble() + val kMinIncreaseRateBpsPerSecond = 4000.0 + return increaseRateBpsPerSecond.coerceAtLeast(kMinIncreaseRateBpsPerSecond) + } + + fun getExpectedBandwidthPeriod(): Duration { + val kMinPeriod = 2.secs + val kDefaultPeriod = 3.secs + val kMaxPeriod = 50.secs + + val increaseRateBpsPerSecond = getNearMaxIncreaseRateBpsPerSecond() + if (lastDecrease == null) { + return kDefaultPeriod + } + + val timeToRecoverDecreaseSeconds = + lastDecrease!!.bps / increaseRateBpsPerSecond + + val period = durationOfDoubleSeconds(timeToRecoverDecreaseSeconds) + return period.coerceIn(kMinPeriod, kMaxPeriod) + } + + private fun changeBitrate(input: RateControlInput, atTime: Instant) { + var newBitrate: Bandwidth? = null + val estimatedThroughput = input.estimatedThroughput ?: latestEstimatedThroughput + input.estimatedThroughput?.let { latestEstimatedThroughput = it } + + // An over-use should always trigger us to reduce the bitrate, even though + // we have not yet established our first estimate. By acting on the over-use, + // we will end up with a valid estimate. + if (!bitrateIsInitialized && input.bwState != BandwidthUsage.kBwOverusing) { + return + } + + changeState(input, atTime) + + when (rateControlState) { + RateControlState.kRcHold -> Unit + + RateControlState.kRcIncrease -> { + if (estimatedThroughput > linkCapacity.upperBound()) { + linkCapacity.reset() + } + + // We limit the new bitrate based on the troughput to avoid unlimited + // bitrate increases. We allow a bit more lag at very low rates to not too + // easily get stuck if the encoder produces uneven outputs. + + var increaseLimit = 1.5 * estimatedThroughput + 10.kbps + if (sendSide && inAlr && noBitrateIncreaseInAlr) { + // TODO this is dead code because we're not using the noBitrateIncreaseInAlr field trial + // Do not increase the delay based estimate in alr since the estimator + // will not be able to get transport feedback necessary to detect if + // the new estimate is correct. + // If we have previously increased above the limit (for instance due to + // probing), we don't allow further changes. + increaseLimit = currentBitrate + } + + if (currentBitrate < increaseLimit) { + var increasedBitrate = Bandwidth.MINUS_INFINITY + if (linkCapacity.hasEstimate()) { + // The link_capacity estimate is reset if the measured throughput + // is too far from the estimate. We can therefore assume that our + // target rate is reasonably close to link capacity and use additive + // increase. + val additiveIncrease = + additiveRateIncrease(atTime, timeLastBitrateChange) + increasedBitrate = currentBitrate + additiveIncrease + } else { + // If we don't have an estimate of the link capacity, use faster ramp + // up to discover the capacity. + val multiplicativeIncrease = multiplicativeRateIncrease( + atTime, + timeLastBitrateChange, + currentBitrate + ) + increasedBitrate = currentBitrate + multiplicativeIncrease + } + newBitrate = increasedBitrate.coerceAtMost(increaseLimit) + } + timeLastBitrateChange = atTime + } + + RateControlState.kRcDecrease -> { + var decreasedBitrate = Bandwidth.INFINITY + + // Set bit rate to something slightly lower than the measured throughput + // to get rid of any self-induced delay. + decreasedBitrate = estimatedThroughput * beta + if (decreasedBitrate > 5.kbps) { + decreasedBitrate -= 5.kbps + } + if (decreasedBitrate > currentBitrate) { + // TODO(terelius): The link_capacity estimate may be based on old + // throughput measurements. Relying on them may lead to unnecessary + // BWE drops. + if (linkCapacity.hasEstimate()) { + decreasedBitrate = linkCapacity.estimate!! * beta + } + } + // Avoid increasing the rate when over-using. + if (decreasedBitrate < currentBitrate) { + newBitrate = decreasedBitrate + } + + if (bitrateIsInitialized && estimatedThroughput < currentBitrate) { + if (newBitrate == null) { + lastDecrease = Bandwidth.ZERO + } else { + lastDecrease = currentBitrate - newBitrate + } + } + if (estimatedThroughput < linkCapacity.lowerBound()) { + // The current throughput is far from the estimated link capacity. Clear + // the estimate to allow an immediate update in onOveruseDetected. + linkCapacity.reset() + } + + bitrateIsInitialized = true + linkCapacity.onOveruseDetected(estimatedThroughput) + // Stay on hold until the pipes are cleared. + rateControlState = RateControlState.kRcHold + timeLastBitrateChange = atTime + timeLastBitrateDecrease = atTime + } + } + currentBitrate = clampBitrate(newBitrate ?: currentBitrate) + } + + private fun clampBitrate(newBitrate: Bandwidth): Bandwidth { + /* Skipping some conditions related to networkEstimate != null */ + + return newBitrate.coerceAtLeast(minConfiguredBitrate) + } + + private fun multiplicativeRateIncrease(atTime: Instant, lastTime: Instant, currentBitrate: Bandwidth): Bandwidth { + var alpha = 1.08 + if (lastTime != NEVER) { + val timeSinceLastUpdate = Duration.between(lastTime, atTime) + alpha = alpha.pow(min(timeSinceLastUpdate.toDouble(), 1.0)) + } + val multiplicativeIncrease = + max(currentBitrate * (alpha - 1.0), 1000.bps) + return multiplicativeIncrease + } + + private fun additiveRateIncrease(atTime: Instant, lastTime: Instant): Bandwidth { + val timePeriodSeconds = Duration.between(lastTime, atTime).toDouble() + val dataRateIncreaseBps = getNearMaxIncreaseRateBpsPerSecond() * timePeriodSeconds + return dataRateIncreaseBps.bps + } + + private fun changeState(input: RateControlInput, atTime: Instant) { + when (input.bwState) { + BandwidthUsage.kBwNormal -> + if (rateControlState == RateControlState.kRcHold) { + timeLastBitrateChange = atTime + rateControlState = RateControlState.kRcIncrease + } + + BandwidthUsage.kBwOverusing -> + if (rateControlState != RateControlState.kRcDecrease) { + rateControlState = RateControlState.kRcDecrease + } + + BandwidthUsage.kBwUnderusing -> + rateControlState = RateControlState.kRcHold + } + } + + enum class RateControlState { + kRcHold, + kRcIncrease, + kRcDecrease + } + + companion object { + private val kDefaultRtt = 200.ms + private const val kDefaultBackoffFactor = 0.85 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetector.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetector.kt new file mode 100644 index 0000000..8f23aff --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetector.kt @@ -0,0 +1,101 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +/** + * An Application-limited region detector. + * + * Based on WebRTC modules/congestion_controller/goog_cc/alr_detector.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings. + */ + +data class AlrDetectorConfig( + // Sent traffic ratio as a function of network capacity used to determine + // application-limited region. ALR region start when bandwidth usage drops + // below kAlrStartUsageRatio and ends when it raises above + // kAlrEndUsageRatio. NOTE: This is intentionally conservative at the moment + // until BW adjustments of application limited region is fine tuned. + val bandwidthUsageRatio: Double = 0.65, + val startBudgetLevelRatio: Double = 0.80, + val stopBudgetLevelRatio: Double = 0.50 +) + +/** Application limited region detector is a class that utilizes signals of +elapsed time and bytes sent to estimate whether network traffic is +currently limited by the application's ability to generate traffic. + +AlrDetector provides a signal that can be utilized to adjust +estimate bandwidth. +Note: This class is not thread-safe. +*/ +class AlrDetector { + fun onBytesSent(bytesSent: Long, sendTimeMs: Long) { + if (lastSendTimeMs == null) { + lastSendTimeMs = sendTimeMs + // Since the duration for sending the bytes is unknwon, return without + // updating alr state. + return + } + val deltaTimeMs = sendTimeMs - lastSendTimeMs!! + lastSendTimeMs = sendTimeMs + + alrBudget.useBudget(bytesSent) + alrBudget.increaseBudget(deltaTimeMs) + var stateChanged = false + if (alrBudget.budgetRatio() > conf.startBudgetLevelRatio && + alrStartedTimeMs == null + ) { + alrStartedTimeMs = System.currentTimeMillis() + stateChanged = true + } else if (alrBudget.budgetRatio() < conf.startBudgetLevelRatio && + alrStartedTimeMs != null + ) { + stateChanged = true + alrStartedTimeMs = null + } + if (stateChanged) { + // TODO? Log event + } + } + + // Set current estimated bandwidth. + fun setEstimatedBitrate(bitrateBps: Int) { + val targetRateKbps = bitrateBps.toDouble() * conf.bandwidthUsageRatio / 1000 + alrBudget.setTargetRateKbps(targetRateKbps.toInt()) + } + + // Returns time in milliseconds when the current application-limited region + // started or empty result if the sender is currently not application-limited. + fun getApplicationLimitedRegionStartTime(): Long? = alrStartedTimeMs + + private val conf = kDefaultProbingScreenshareBweSettings + + private var lastSendTimeMs: Long? = null + + private val alrBudget = IntervalBudget(0, true) + private var alrStartedTimeMs: Long? = null + + companion object { + /* From rtc_base/experiments/alr_experiment.cc */ + val kDefaultProbingScreenshareBweSettings = AlrDetectorConfig( + bandwidthUsageRatio = 0.80, + startBudgetLevelRatio = 0.40, + stopBudgetLevelRatio = -0.60 + ) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateEstimator.kt new file mode 100644 index 0000000..7628df4 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateEstimator.kt @@ -0,0 +1,154 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.per +import org.jitsi.utils.ms +import org.jitsi.utils.toRoundedEpochMilli +import java.time.Instant +import kotlin.math.abs +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +/** + * Computes a bayesian estimate of the throughput given acks containing + * the arrival time and payload size. Samples which are far from the current + * estimate or are based on few packets are given a smaller weight, as they + * are considered to be more likely to have been caused by, e.g., delay spikes + * unrelated to congestion. + * + * Based on WebRTC modules/congestion_controller/goog_cc/bitrate_estimator.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +open class BitrateEstimator { + private var sum = 0 + private val initialWindowMs = kInitialRateWindowMs + private val noninitialWindowMs = kRateWindowMs + private val uncertaintyScale = 10.0f + private val uncertaintyScaleInAlr = uncertaintyScale + private val smallSampleUncertaintyScale = uncertaintyScale + private val smallSampleThreshold = DataSize.ZERO + private val uncertaintySymmetryCap = Bandwidth.ZERO + private val estimateFloor = Bandwidth.ZERO + + private var currentWindowMs = 0L + private var prevTimeMs = -1L + private var bitrateEstimateKbps = -1.0f + private var bitrateEstimateVar = 50.0f + + open fun update(atTime: Instant, amount: DataSize, inAlr: Boolean) { + val rateWindowMs = if (bitrateEstimateKbps < 0.0f) initialWindowMs else noninitialWindowMs + val (bitrateSampleKbps, isSmallSample) = + updateWindow(atTime.toRoundedEpochMilli(), amount.bytes.roundToInt(), rateWindowMs) + if (bitrateSampleKbps < 0.0f) { + return + } + if (bitrateEstimateKbps < 0.0f) { + // This is the very first sample we get. Use it to initialize the estimate. + bitrateEstimateKbps = bitrateSampleKbps + return + } + // Optionally use higher uncertainty for very small samples to avoid dropping + // estimate and for samples obtained in ALR. + val scale = if (isSmallSample && bitrateSampleKbps < bitrateEstimateKbps) { + smallSampleUncertaintyScale + } else if (inAlr && bitrateSampleKbps < bitrateEstimateKbps) { + uncertaintyScaleInAlr + } else { + uncertaintyScale + } + // Define the sample uncertainty as a function of how far away it is from the + // current estimate. With low values of uncertaintySymmetryCap we add more + // uncertainty to increases than to decreases. For higher values we approach + // symmetry. + val sampleUncertainty = + scale * abs(bitrateEstimateKbps - bitrateSampleKbps) / + ( + bitrateEstimateKbps + + min(bitrateSampleKbps, uncertaintySymmetryCap.kbps.toFloat()) + ) + + val sampleVar = sampleUncertainty * sampleUncertainty + // Update a bayesian estimate of the rate, weighting it lower if the sample + // uncertainty is large. + // The bitrate estimate uncertainty is increased with each update to model + // that the bitrate changes over time. + val predBitrateEstimateVar = bitrateEstimateVar + 5.0f + bitrateEstimateKbps = ( + sampleVar * bitrateEstimateKbps + + predBitrateEstimateVar * bitrateSampleKbps + ) / + (sampleVar + predBitrateEstimateVar) + bitrateEstimateKbps = max(bitrateEstimateKbps, estimateFloor.kbps.toFloat()) + bitrateEstimateVar = sampleVar * predBitrateEstimateVar / + (sampleVar + predBitrateEstimateVar) + } + + private fun updateWindow(nowMs: Long, bytes: Int, rateWindowMs: Int): Pair { + // Reset if time moves backwards. + if (nowMs < prevTimeMs) { + prevTimeMs = -1 + sum = 0 + currentWindowMs = 0 + } + if (prevTimeMs >= 0) { + currentWindowMs += nowMs - prevTimeMs + // Reset if nothing has been received for more than a full window. + if (nowMs - prevTimeMs > rateWindowMs) { + sum = 0 + currentWindowMs %= rateWindowMs + } + } + prevTimeMs = nowMs + val bitrateSample: Float + val isSmallSample: Boolean + if (currentWindowMs >= rateWindowMs) { + isSmallSample = sum < smallSampleThreshold.bytes + bitrateSample = 8.0f * sum / rateWindowMs.toFloat() + currentWindowMs -= rateWindowMs + sum = 0 + } else { + isSmallSample = false + bitrateSample = -1.0f + } + sum += bytes + return Pair(bitrateSample, isSmallSample) + } + + open fun bitrate(): Bandwidth? = if (bitrateEstimateKbps < 0.0f) null else bitrateEstimateKbps.kbps + + fun peekRate(): Bandwidth? = if (currentWindowMs > 0) sum.bytes.per(currentWindowMs.ms) else null + + open fun expectFastRateChange() { + // By setting the bitrate-estimate variance to a higher value we allow the + // bitrate to change fast for the next few samples. + bitrateEstimateVar += 200 + } + + companion object { + const val kInitialRateWindowMs = 500 + const val kRateWindowMs = 150 + const val kMinRateWindowMs = 150 + const val kMaxRateWindowMs = 1000 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProber.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProber.kt new file mode 100644 index 0000000..8dab451 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProber.kt @@ -0,0 +1,274 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:enum-entry-name-case") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.div +import org.jitsi.nlj.util.min +import org.jitsi.nlj.util.times +import org.jitsi.utils.isFinite +import org.jitsi.utils.isInfinite +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.min +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +/** Bitrate prober, + * based on WebRTC modules/pacing/bitrate_prober.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +private val kProbeClusterTimeout = 5.secs +private val kMaxPendingProbeClusters = 5 + +data class BitrateProberConfig( + var maxProbeDelay: Duration = 20.ms, + var minPacketSize: DataSize = 200.bytes, + var allowStartProbingImmediately: Boolean = false +) + +class BitrateProber( + parentLogger: Logger, + configIn: BitrateProberConfig = BitrateProberConfig() +) { + private val logger = createChildLogger(parentLogger) + + fun setEnabled(enable: Boolean) { + if (enable) { + if (probingState == ProbingState.kDisabled) { + probingState = ProbingState.kInactive + logger.info("Bandwidth probing enabled, set to inactive") + } + } else { + probingState = ProbingState.kDisabled + logger.info("Bandwidth probing disabled") + } + } + + fun setAllowProbeWithoutMediaPacket(allow: Boolean) { + config.allowStartProbingImmediately = allow + maybeSetActiveState(DataSize.ZERO) + } + + // Returns true if the prober is in a probing session, i.e., it currently + // wants packets to be sent out according to the time returned by + // TimeUntilNextProbe(). + fun isProbing(): Boolean = probingState == ProbingState.kActive + + // Initializes a new probing session if the prober is allowed to probe. Does + // not initialize the prober unless the packet size is large enough to probe + // with. + fun onIncomingPacket(packetSize: DataSize) { + maybeSetActiveState(packetSize) + } + + // Create a cluster used to probe. + fun createProbeCluster(clusterConfig: ProbeClusterConfig) { + check(probingState != ProbingState.kDisabled) + check(clusterConfig.minProbeDelta > Duration.ZERO) + + while (clusters.isNotEmpty() && ( + Duration.between(clusters.first().requestedAt, clusterConfig.atTime) > kProbeClusterTimeout || + clusters.size > kMaxPendingProbeClusters + ) + ) { + clusters.removeFirst() + } + + val cluster = ProbeCluster( + requestedAt = clusterConfig.atTime, + paceInfo = PacedPacketInfo( + probeClusterMinProbes = clusterConfig.targetProbeCount, + probeClusterMinBytes = (clusterConfig.targetDataRate * clusterConfig.targetDuration).bytes.toInt(), + probeClusterId = clusterConfig.id + ), + minProbeDelta = clusterConfig.minProbeDelta + ) + check(cluster.paceInfo.probeClusterMinBytes >= 0) + cluster.paceInfo.sendBitrate = clusterConfig.targetDataRate + clusters.addLast(cluster) + + maybeSetActiveState(packetSize = DataSize.ZERO) + + check(probingState == ProbingState.kActive || probingState == ProbingState.kInactive) + + logger.info { + "Probe cluster (bitrate_bps:min bytes:min packets): (" + + "${cluster.paceInfo.sendBitrate}:${cluster.paceInfo.probeClusterMinBytes}:" + + "${cluster.paceInfo.probeClusterMinProbes}, $probingState)" + } + } + + // Returns the time at which the next probe should be sent to get accurate + // probing. If probing is not desired at this time, [Instant.MAX] + // will be returned. + // TODO(bugs.webrtc.org/11780): Remove `now` argument when old mode is gone. + fun nextProbeTime(now: Instant): Instant { + // Probing is not active or probing is already complete. + if (probingState != ProbingState.kActive || clusters.isEmpty()) { + return Instant.MAX + } + return nextProbeTime + } + + // Information about the current probing cluster. + fun currentCluster(now: Instant): PacedPacketInfo? { + if (clusters.isEmpty() || probingState != ProbingState.kActive) { + return null + } + + if (nextProbeTime.isFinite() && + Duration.between(nextProbeTime, now) > config.maxProbeDelay + ) { + logger.warn( + "Probe delay too high (next:$nextProbeTime, now:$now)" + + "discarding probe cluster." + ) + clusters.removeFirst() + if (clusters.isEmpty()) { + probingState = ProbingState.kInactive + return null + } + } + + val info = clusters.first().paceInfo.copy() + info.probeClusterBytesSent = clusters.first().sentBytes + return info + } + + // Returns the minimum number of bytes that the prober recommends for + // the next probe, or zero if not probing. A probe can consist of multiple + // packets that are sent back to back. + fun recommendedMinProbeSize(): DataSize { + if (clusters.isEmpty()) { + return DataSize.ZERO + } + val sendRate = clusters.first().paceInfo.sendBitrate + return sendRate * clusters.first().minProbeDelta + } + + // Called to report to the prober that a probe has been sent. In case of + // multiple packets per probe, this call would be made at the end of sending + // the last packet in probe. `size` is the total size of all packets in probe. + fun probeSent(now: Instant, size: DataSize) { + check(probingState == ProbingState.kActive) + check(size != DataSize.ZERO) + + if (clusters.isNotEmpty()) { + val cluster = clusters.first() + if (cluster.sentProbes == 0) { + check(cluster.startedAt.isInfinite()) + cluster.startedAt = now + } + cluster.sentBytes += size.bytes.toInt() + cluster.sentProbes += 1 + nextProbeTime = calculateNextProbeTime(cluster) + if (cluster.sentBytes >= cluster.paceInfo.probeClusterMinBytes && + cluster.sentProbes >= cluster.paceInfo.probeClusterMinProbes + ) { + clusters.removeFirst() + } + if (clusters.isEmpty()) { + probingState = ProbingState.kInactive + } + } + } + + private enum class ProbingState { + // Probing will not be triggered in this state at all times. + kDisabled, + + // Probing is enabled and ready to trigger on the first packet arrival if + // there is a probe cluster. + kInactive, + + // Probe cluster is filled with the set of data rates to be probed and + // probes are being sent. + kActive, + } + + // A probe cluster consists of a set of probes. Each probe in turn can be + // divided into a number of packets to accommodate the MTU on the network. + private data class ProbeCluster( + val paceInfo: PacedPacketInfo, + var sentProbes: Int = 0, + var sentBytes: Int = 0, + var minProbeDelta: Duration = Duration.ZERO, + var requestedAt: Instant = Instant.MIN, + var startedAt: Instant = Instant.MIN + ) + + private fun calculateNextProbeTime(cluster: ProbeCluster): Instant { + check(cluster.paceInfo.sendBitrate >= 0.bps) + check(cluster.startedAt.isFinite()) + + // Compute the time delta from the cluster start to ensure probe bitrate stays + // close to the target bitrate. Result is in milliseconds. + val sentBytes = cluster.sentBytes.bytes + val sendBitrate = cluster.paceInfo.sendBitrate + + val delta = sentBytes / sendBitrate + return cluster.startedAt + delta + } + + private fun maybeSetActiveState(packetSize: DataSize) { + if (readyToSetActiveState(packetSize)) { + nextProbeTime = Instant.MIN + probingState = ProbingState.kActive + } + } + + private fun readyToSetActiveState(packetSize: DataSize): Boolean { + if (clusters.isEmpty()) { + check(probingState == ProbingState.kDisabled || probingState == ProbingState.kInactive) + return false + } + when (probingState) { + ProbingState.kDisabled, ProbingState.kActive -> { + return false + } + + ProbingState.kInactive -> { + if (config.allowStartProbingImmediately) { + return true + } + // If config_.min_packet_size > 0, a "large enough" packet must be + // sent first, before a probe can be generated and sent. Otherwise, + // send the probe asap. + return packetSize >= min(recommendedMinProbeSize(), config.minPacketSize) + } + } + } + + private var probingState: ProbingState = ProbingState.kInactive + + // Probe bitrate per packet. These are used to compute the delta relative to + // the previous probe packet based on the size and time when that packet was + // sent. + private val clusters = ArrayDeque() + + // Time that the next probe should be sent when in kActive state + private var nextProbeTime: Instant = Instant.MAX + + private var config = configIn.copy() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BweDefines.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BweDefines.kt new file mode 100644 index 0000000..3785fe1 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BweDefines.kt @@ -0,0 +1,64 @@ +/* + * Copyright @ 2015 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file uses WebRTC's naming style for enums +@file:Suppress("ktlint:standard:enum-entry-name-case") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.utils.secs + +/** Common defines for bandwidth estimation, + * based on WebRTC modules/remote_bitrate_estimator/{include/bwe_defines.h,bwe_defines.cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +val kCongestionControllerMinBitrate = 5000.bps + +val kBitrateWindow = 1.secs + +/** + * Input to Rate Control. + */ +data class RateControlInput( + var bwState: BandwidthUsage, + var estimatedThroughput: Bandwidth? +) { + /** + * Assigns the values of the fields of [source] to the respective + * fields of this [RateControlInput]. + * + * @param source the [RateControlInput] the values of the fields of + * which are to be assigned to the respective fields of this + * [RateControlInput] + */ + fun copy(source: RateControlInput) { + bwState = source.bwState + estimatedThroughput = source.estimatedThroughput + } +} + +/** + * This is from WebRTC api/transport/bandwidth_usage.h in the same WebRTC revision. + */ +enum class BandwidthUsage { + kBwNormal, + kBwUnderusing, + kBwOverusing + // kLast not needed in Kotlin +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackController.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackController.kt new file mode 100644 index 0000000..a0a984a --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackController.kt @@ -0,0 +1,86 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.DataSize + +/** Congestion window pushback controller, + * based on WebRTC modules/congestion_controller/goog_cc/congestion_window_pushback_controller.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings. + */ +class CongestionWindowPushbackController { + fun updateOutstandingData(outstandingBytes: Long) { + this.outstandingBytes = outstandingBytes + } + + fun updatePacingQueue(pacingBytes: Long) { + this.pacingBytes = pacingBytes + } + + fun updateTargetBitrate(bitrateBps: Int): Int { + if (currentDataWindow == null || currentDataWindow == DataSize.ZERO) { + return bitrateBps + } + var totalBytes = outstandingBytes + if (addPacing) { + totalBytes += pacingBytes + } + val fillRatio = totalBytes / currentDataWindow!!.bytes.toLong() + if (fillRatio > 1.5) { + encodingRateRatio *= 0.9 + } else if (fillRatio > 1.0) { + encodingRateRatio *= 0.95 + } else if (fillRatio < 0.1) { + encodingRateRatio = 1.0 + } else { + encodingRateRatio *= 1.05 + encodingRateRatio = Math.min(encodingRateRatio, 1.0) + } + val adjustedTargetBitrateBps = (bitrateBps * encodingRateRatio).toInt() + + // Do not adjust below the minimum pushback bitrate but do obey if the + // original estimate is below it. + return if (adjustedTargetBitrateBps < minPushbackTargetBitrateBps) { + Math.min(bitrateBps, minPushbackTargetBitrateBps) + } else { + adjustedTargetBitrateBps + } + } + + fun setDataWindow(dataWindow: DataSize) { + currentDataWindow = dataWindow + } + + private val addPacing = false + + private val minPushbackTargetBitrateBps: Int = kDefaultMinPushbackTargetBitrateBps + + private var currentDataWindow: DataSize? = null + + private var outstandingBytes = 0L + + private var pacingBytes = 0L + + private var encodingRateRatio = 1.0 +} + +/** From rtc_base/experiments/rate_control_settings.cc */ + +private const val kDefaultMinPushbackTargetBitrateBps = 30000 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBwe.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBwe.kt new file mode 100644 index 0000000..fb97223 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBwe.kt @@ -0,0 +1,250 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.utils.NEVER +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.toDoubleMillis +import org.jitsi.utils.toRoundedEpochMilli +import java.time.Duration +import java.time.Instant + +/** Delay-based bandwidth estimation, + * based on WebRTC modules/congestion_controller/goog_cc/delay_based_bwe.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings, and APIs that aren't + * used by Chrome have also been removed. + */ +class DelayBasedBwe( + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) { + private val logger = parentLogger.createChildLogger(javaClass.name) + + private var interArrivalDelta: InterArrivalDelta? = null + val delayDetector: DelayIncreaseDetectorInterface = TrendlineEstimator(logger, diagnosticContext) + + private var lastSeenPacket: Instant = NEVER + + /* private var umaRecorded: Boolean = false */ + + val rateControl = AimdRateControl(sendSide = true) + private var prevBitrate: Bandwidth = Bandwidth.ZERO + private var prevState: BandwidthUsage = BandwidthUsage.kBwNormal + + fun incomingPacketFeedbackVector( + msg: TransportPacketsFeedback, + ackedBitrate: Bandwidth?, + probeBitrate: Bandwidth?, + /* networkEstimate: NetworkStateEstimate?, */ + inAlr: Boolean + ): Result { + val packetFeedbackVector = msg.sortedByReceiveTime() + if (packetFeedbackVector.isEmpty()) { + // TODO(holmer): An empty feedback vector here likely means that + // all acks were too late and that the send time history had + // timed out. We should reduce the rate when this occurs. + logger.warn("Very late feedback received.") + return Result() + } + + /* Skipping uma_recorded, it's just for histograms. */ + var delayedFeedback = true + var recoveredFromOveruse = false + + var prevDetectorState = delayDetector.state() + for (packetFeedback in packetFeedbackVector) { + delayedFeedback = false + incomingPacketFeedback(packetFeedback, msg.feedbackTime) + if (prevDetectorState == BandwidthUsage.kBwUnderusing && + delayDetector.state() == BandwidthUsage.kBwNormal + ) { + recoveredFromOveruse = true + } + prevDetectorState = delayDetector.state() + } + + if (delayedFeedback) { + // TODO(bugs.webrtc.org/10125): Design a better mechanism to safe-guard + // against building very large network queues. + return Result() + } + + rateControl.inAlr = inAlr + // rateControl.networkStateEstimate = networkStateEstimate + return maybeUpdateEstimate( + ackedBitrate, + probeBitrate, + /* networkEstimate, */ + recoveredFromOveruse, + inAlr, + msg.feedbackTime + ) + } + + private fun incomingPacketFeedback(packetFeedback: PacketResult, atTime: Instant) { + // Reset if the stream has timed out. + if (lastSeenPacket == NEVER || Duration.between(lastSeenPacket, atTime) > kStreamTimeOut) { + interArrivalDelta = InterArrivalDelta(kSendTimeGroupLength) + } + lastSeenPacket = atTime + val packetSize = packetFeedback.sentPacket.size + + val calculatedDeltas = interArrivalDelta!!.computeDeltas( + packetFeedback.sentPacket.sendTime, + packetFeedback.receiveTime, + atTime, + packetSize.bytes.toLong() + ) + + delayDetector.update( + calculatedDeltas.arrivalTimeDelta.toDoubleMillis(), + calculatedDeltas.sendTimeDelta.toDoubleMillis(), + packetFeedback.sentPacket.sendTime.toRoundedEpochMilli(), + packetFeedback.receiveTime.toRoundedEpochMilli(), + packetSize.bytes.toLong(), + calculatedDeltas.computed + ) + } + + fun triggerOveruse(atTime: Instant, linkCapacity: Bandwidth?): Bandwidth { + val input = RateControlInput(BandwidthUsage.kBwOverusing, linkCapacity) + return rateControl.update(input, atTime) + } + + private fun maybeUpdateEstimate( + ackedBitrate: Bandwidth?, + probeBitrate: Bandwidth?, + /* networkEstimate: NetworkStateEstimate?, */ + recoveredFromOveruse: Boolean, + inAlr: Boolean, + atTime: Instant + ): Result { + val result = + // Currently overusing the bandwidth. + if (delayDetector.state() == BandwidthUsage.kBwOverusing) { + if (ackedBitrate != null && rateControl.timeToReduceFurther(atTime, ackedBitrate)) { + val targetBitrate = updateEstimate(atTime, ackedBitrate) + if (targetBitrate != null) { + Result(updated = true, targetBitrate = targetBitrate) + } else { + Result(updated = false) + } + } else if (ackedBitrate == null && rateControl.validEstimate() && + rateControl.initialTimeToReduceFurther(atTime) + ) { + rateControl.setEstimate(rateControl.latestEstimate() / 2, atTime) + Result(updated = true, probe = false, targetBitrate = rateControl.latestEstimate()) + } else { + Result() + } + } else { + if (probeBitrate != null) { + rateControl.setEstimate(probeBitrate, atTime) + Result(probe = true, updated = true, targetBitrate = rateControl.latestEstimate()) + } else { + val targetBitrate = updateEstimate(atTime, ackedBitrate) + if (targetBitrate != null) { + Result( + updated = true, + targetBitrate = targetBitrate, + recoveredFromOveruse = recoveredFromOveruse + ) + } else { + Result(updated = false, recoveredFromOveruse = recoveredFromOveruse) + } + } + } + val detectorState = delayDetector.state() + if ((result.updated && prevBitrate != result.targetBitrate) || + detectorState != prevState + ) { + val bitrate = if (result.updated) { + result.targetBitrate + } else { + prevBitrate + } + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("bwe_update_delay_based", atTime) + .addField("bitrate_bps", bitrate.bps) + .addField("detector_state", detectorState.name) + } + + prevBitrate = bitrate + prevState = detectorState + } + result.delayDetectorState = detectorState + return result + } + + private fun updateEstimate(atTime: Instant, ackedBitrate: Bandwidth?): Bandwidth? { + val input = RateControlInput(delayDetector.state(), ackedBitrate) + val targetRate = rateControl.update(input, atTime) + return if (rateControl.validEstimate()) { + targetRate + } else { + null + } + } + + fun onRttUpdate(avgRtt: Duration) { + rateControl.rtt = avgRtt + } + + fun latestEstimate(): Bandwidth? { + if (!rateControl.validEstimate()) { + return null + } + return rateControl.latestEstimate() + } + + fun setStartBitrate(startBitrate: Bandwidth) { + logger.info("BWE setting start bitrate to $startBitrate") + rateControl.setStartBitrate(startBitrate) + } + + fun setMinBitrate(minBitrate: Bandwidth) { + rateControl.setMinBitrate(minBitrate) + } + + fun getExpectedBwePeriod(): Duration = rateControl.getExpectedBandwidthPeriod() + + fun lastEstimate(): Bandwidth = prevBitrate + fun lastState(): BandwidthUsage = prevState + + companion object { + val kStreamTimeOut = 2.secs + val kSendTimeGroupLength = 5.ms + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(DelayBasedBwe::class.java) + } + + data class Result( + val updated: Boolean = false, + val probe: Boolean = false, + val targetBitrate: Bandwidth = 0.bps, + val recoveredFromOveruse: Boolean = false, + var delayDetectorState: BandwidthUsage = BandwidthUsage.kBwNormal + ) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayIncreaseDetectorInterface.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayIncreaseDetectorInterface.kt new file mode 100644 index 0000000..29dc2c6 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayIncreaseDetectorInterface.kt @@ -0,0 +1,36 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +/** + * Interface to detect delay increase. + * * + * Based on WebRTC modules/congestion_controller/goog_cc/delay_increase_detector_interface.h in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +interface DelayIncreaseDetectorInterface { + fun update( + recvDeltaMs: Double, + sendDeltaMs: Double, + sendTimeMs: Long, + arrivalTimeMs: Long, + packetSize: Long, + calculatedDeltas: Boolean + ) + + fun state(): BandwidthUsage +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcFactory.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcFactory.kt new file mode 100644 index 0000000..51831f8 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcFactory.kt @@ -0,0 +1,47 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.utils.ms +import java.time.Duration + +/** Google CC Network Controller Factory, + * based on WebRTC api/transport/goog_cc_factory.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings, and features not used in + * Chrome have been removed. + */ + +class GoogCcFactoryConfig( + val feedbackOnly: Boolean = false, + /* Not in this object in WebRTC: This is a field trial parameter there */ + val rateControlSettings: CongestionWindowConfig = CongestionWindowConfig() +) + +class GoogCcNetworkControllerFactory( + private val factoryConfig: GoogCcFactoryConfig = GoogCcFactoryConfig() +) : NetworkControllerFactoryInterface { + override fun create(config: NetworkControllerConfig): NetworkControllerInterface { + val googCcConfig = GoogCcConfig(feedbackOnly = factoryConfig.feedbackOnly) + return GoogCcNetworkController(config, googCcConfig) + } + + override fun getProcessInterval(): Duration { + val kUpdateIntervalMs = 25L + return kUpdateIntervalMs.ms + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkController.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkController.kt new file mode 100644 index 0000000..10c3b22 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkController.kt @@ -0,0 +1,817 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.max +import org.jitsi.nlj.util.min +import org.jitsi.nlj.util.times +import org.jitsi.utils.MAX_DURATION +import org.jitsi.utils.MIN_DURATION +import org.jitsi.utils.isFinite +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.max +import org.jitsi.utils.min +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.toDouble +import org.jitsi.utils.toRoundedMillis +import java.time.Duration +import java.time.Instant +import kotlin.math.roundToLong + +/** Top-level Google CC Network Controller, + * based on WebRTC modules/congestion_controller/goog_cc/goog_cc_network_control.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings, + * except where needed by unit tests. + */ +class GoogCcConfig( + val feedbackOnly: Boolean = false, + /* Not in this object in WebRTC: This is a field trial parameter there */ + val rateControlSettings: CongestionWindowConfig = CongestionWindowConfig() +) + +class GoogCcNetworkController( + config: NetworkControllerConfig, + googCcConfig: GoogCcConfig +) : NetworkControllerInterface { + private val logger = config.parentLogger.createChildLogger(javaClass.name) + private val diagnosticContext = config.diagnosticContext + + override fun onNetworkAvailability(msg: NetworkAvailability): NetworkControlUpdate = NetworkControlUpdate( + probeClusterConfigs = probeController.onNetworkAvailability(msg) + ) + + override fun onNetworkRouteChange(msg: NetworkRouteChange): NetworkControlUpdate { + if (safeResetOnRouteChange) { + var estimatedBitrate: Bandwidth? + if (safeResetAcknowledgedRate) { + estimatedBitrate = acknowledgedBitrateEstimator.bitrate() + if (estimatedBitrate == null) { + estimatedBitrate = acknowledgedBitrateEstimator.peekRate() + } + } else { + estimatedBitrate = bandwidthEstimation.targetRate() + } + if (estimatedBitrate != null) { + if (msg.constraints.startingRate != null) { + msg.constraints.startingRate = min(msg.constraints.startingRate!!, estimatedBitrate) + } else { + msg.constraints.startingRate = estimatedBitrate + } + } + } + + acknowledgedBitrateEstimator = AcknowledgedBitrateEstimatorInterface.create() + probeBitrateEstimator = ProbeBitrateEstimator(logger, diagnosticContext) + delayBasedBwe = DelayBasedBwe(logger, diagnosticContext) + + bandwidthEstimation.onRouteChange() + probeController.reset(msg.atTime) + val update = MutableNetworkControlUpdate( + probeClusterConfigs = resetConstraints(msg.constraints) + ) + maybeTriggerOnNetworkChanged(update, msg.atTime) + + return update + } + + override fun onProcessInterval(msg: ProcessInterval): NetworkControlUpdate { + val update = MutableNetworkControlUpdate() + if (initialConfig != null) { + update.probeClusterConfigs = resetConstraints(initialConfig!!.constraints) + update.pacerConfig = getPacingRates(msg.atTime) + + if (initialConfig!!.streamBasedConfig.requestsAlrProbing != null) { + probeController.enablePeriodicAlrProbing(initialConfig!!.streamBasedConfig.requestsAlrProbing!!) + } + if (initialConfig!!.streamBasedConfig.enableRepeatedInitialProbing != null) { + probeController.enableRepeatedInitialProbing( + initialConfig!!.streamBasedConfig.enableRepeatedInitialProbing!! + ) + } + val totalBitrate = initialConfig!!.streamBasedConfig.maxTotalAllocatedBitrate + if (totalBitrate != null) { + val probes = probeController.onMaxTotalAllocatedBitrate(totalBitrate, msg.atTime) + update.probeClusterConfigs.addAll(probes) + } + initialConfig = null + } + if (congestionWindowPushbackController != null && msg.pacerQueue != null) { + congestionWindowPushbackController.updatePacingQueue(msg.pacerQueue.bytes.roundToLong()) + } + bandwidthEstimation.updateEstimate(msg.atTime) + val startTimeMs = alrDetector.getApplicationLimitedRegionStartTime() + probeController.setAlrStartTimeMs(startTimeMs) + + val probes = probeController.process(msg.atTime) + update.probeClusterConfigs.addAll(probes) + + if (rateControlSettings.useCongestionWindow() && + !feedbackMaxRtts.isEmpty() + ) { + updateCongestionWindowSize() + } + if (congestionWindowPushbackController != null && currentDataWindow != null) { + congestionWindowPushbackController.setDataWindow(currentDataWindow!!) + } else { + update.congestionWindow = currentDataWindow + } + maybeTriggerOnNetworkChanged(update, msg.atTime) + return update + } + + override fun onRemoteBitrateReport(msg: RemoteBitrateReport): NetworkControlUpdate { + if (packetFeedbackOnly) { + logger.error("Received REMB for packet feedback only GoogCC") + return NetworkControlUpdate() + } + bandwidthEstimation.updateReceiverEstimate(msg.receiveTime, msg.bandwidth) + + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("REMB_BW", msg.receiveTime) + .addField("REMB_kbps", msg.bandwidth.kbps) + } + return NetworkControlUpdate() + } + + override fun onRoundTripTimeUpdate(msg: RoundTripTimeUpdate): NetworkControlUpdate { + if (packetFeedbackOnly || msg.smoothed) { + return NetworkControlUpdate() + } + assert(msg.roundTripTime != Duration.ZERO) + delayBasedBwe.onRttUpdate(msg.roundTripTime) + bandwidthEstimation.updateRtt(msg.roundTripTime, msg.receiveTime) + return NetworkControlUpdate() + } + + override fun onSentPacket(sentPacket: SentPacket): NetworkControlUpdate { + alrDetector.onBytesSent(sentPacket.size.bytes.roundToLong(), sentPacket.sendTime.toEpochMilli()) + acknowledgedBitrateEstimator.setAlr(alrDetector.getApplicationLimitedRegionStartTime() != null) + + if (!firstPacketSent) { + firstPacketSent = true + // Initialize feedback time to send time to allow estimation of RTT until + // first feedback is received. + bandwidthEstimation.updatePropagationRtt(sentPacket.sendTime, Duration.ZERO) + } + bandwidthEstimation.onSentPacket(sentPacket) + + if (congestionWindowPushbackController != null) { + congestionWindowPushbackController.updateOutstandingData(sentPacket.dataInFlight.bytes.roundToLong()) + val update = MutableNetworkControlUpdate() + maybeTriggerOnNetworkChanged(update, sentPacket.sendTime) + return update + } else { + return NetworkControlUpdate() + } + } + + override fun onStreamsConfig(msg: StreamsConfig): NetworkControlUpdate { + val update = MutableNetworkControlUpdate() + if (msg.requestsAlrProbing != null) { + probeController.enablePeriodicAlrProbing(msg.requestsAlrProbing) + } + if (msg.maxTotalAllocatedBitrate != null) { + update.probeClusterConfigs = + probeController.onMaxTotalAllocatedBitrate(msg.maxTotalAllocatedBitrate, msg.atTime) + } + + var pacingChanged = false + if (msg.pacingFactor != null && msg.pacingFactor != pacingFactor) { + pacingFactor = msg.pacingFactor + pacingChanged = true + } + if (msg.minTotalAllocatedBitrate != null && + minTotalAllocatedBitrate != msg.minTotalAllocatedBitrate + ) { + minTotalAllocatedBitrate = msg.minTotalAllocatedBitrate + pacingChanged = true + + if (useMinAllocatableAsLowerBound) { + clampConstraints() + delayBasedBwe.setMinBitrate(minDataRate) + bandwidthEstimation.setMinMaxBitrate(minDataRate, maxDataRate) + } + } + if (msg.maxPaddingRate != null && msg.maxPaddingRate != maxPaddingRate) { + maxPaddingRate = msg.maxPaddingRate + pacingChanged = true + } + + if (pacingChanged) { + update.pacerConfig = getPacingRates(msg.atTime) + } + return update + } + + override fun onTargetRateConstraints(constraints: TargetRateConstraints): NetworkControlUpdate { + val update = MutableNetworkControlUpdate() + update.probeClusterConfigs = resetConstraints(constraints) + maybeTriggerOnNetworkChanged(update, constraints.atTime) + return update + } + + private fun clampConstraints() { + // TODO(holmer): We should make sure the default bitrates are set to 10 kbps, + // and that we don't try to set the min bitrate to 0 from any applications. + // The congestion controller should allow a min bitrate of 0. + minDataRate = max(minTargetRate, kCongestionControllerMinBitrate) + if (useMinAllocatableAsLowerBound) { + minDataRate = max(minDataRate, minTotalAllocatedBitrate) + } + if (maxDataRate < minDataRate) { + logger.warn("max bitrate $maxDataRate smaller than min bitrate $minDataRate") + maxDataRate = minDataRate + } + if (startingRate != null && startingRate!! < minDataRate) { + logger.warn("start bitrate ${startingRate!!} smaller than min bitrate $minDataRate") + startingRate = minDataRate + } + } + + private fun resetConstraints(newConstraints: TargetRateConstraints): MutableList { + minTargetRate = newConstraints.minDataRate ?: Bandwidth.ZERO + maxDataRate = newConstraints.maxDataRate ?: Bandwidth.INFINITY + startingRate = newConstraints.startingRate + clampConstraints() + + bandwidthEstimation.setBitrates(startingRate, minDataRate, maxDataRate, newConstraints.atTime) + if (startingRate != null) { + delayBasedBwe.setStartBitrate(startingRate!!) + } + delayBasedBwe.setMinBitrate(minDataRate) + + return probeController.setBitrates( + minDataRate, + startingRate ?: Bandwidth.ZERO, + maxDataRate, + newConstraints.atTime + ) + } + + override fun onTransportLossReport(msg: TransportLossReport): NetworkControlUpdate { + if (packetFeedbackOnly) { + return NetworkControlUpdate() + } + val totalPacketsDelta = msg.packetsReceivedDelta + msg.packetsLostDelta + bandwidthEstimation.updatePacketsLost(msg.packetsLostDelta, totalPacketsDelta, msg.receiveTime) + return NetworkControlUpdate() + } + + private fun updateCongestionWindowSize() { + val minFeedbackMaxRtt = feedbackMaxRtts.min().ms + + val kMinCwnd = (2 * 1500).bytes + val timeWindow = minFeedbackMaxRtt + rateControlSettings.getCongestionWindowAdditionalTimeMs().ms + + var dataWindow = lastLossBasedTargetRate * timeWindow + if (currentDataWindow != null) { + dataWindow = max(kMinCwnd, (dataWindow + currentDataWindow!!) / 2.0) + } else { + dataWindow = max(kMinCwnd, dataWindow) + } + currentDataWindow = dataWindow + } + + override fun onTransportPacketsFeedback(report: TransportPacketsFeedback): NetworkControlUpdate { + if (report.packetFeedbacks.isEmpty()) { + // TODO(bugs.webrtc.org/10125): Design a better mechanism to safe-guard + // against building very large network queues. + return NetworkControlUpdate() + } + if (congestionWindowPushbackController != null) { + congestionWindowPushbackController.updateOutstandingData(report.dataInFlight.bytes.roundToLong()) + } + var maxFeedbackRtt = MIN_DURATION + var minPropagationRtt = MAX_DURATION + var maxRecvTime = Instant.MIN + + val feedbacks = report.receivedWithSendInfo() + for (feedback in feedbacks) { + maxRecvTime = max(maxRecvTime, feedback.receiveTime) + } + for (feedback in feedbacks) { + val feedbackRtt = Duration.between(feedback.sentPacket.sendTime, report.feedbackTime) + val minPendingTime = Duration.between(feedback.receiveTime, maxRecvTime) + val propagationRtt = feedbackRtt - minPendingTime + maxFeedbackRtt = max(maxFeedbackRtt, feedbackRtt) + minPropagationRtt = min(minPropagationRtt, propagationRtt) + } + + if (maxFeedbackRtt.isFinite()) { + feedbackMaxRtts.add(maxFeedbackRtt.toRoundedMillis()) + val kMaxFeedbackRttWindow = 32 + if (feedbackMaxRtts.size > kMaxFeedbackRttWindow) { + feedbackMaxRtts.removeFirst() + } + // TODO(srte): Use time since last unacknowledged packet. + bandwidthEstimation.updatePropagationRtt(report.feedbackTime, minPropagationRtt) + } + if (packetFeedbackOnly) { + if (!feedbackMaxRtts.isEmpty()) { + val sumRttMs = feedbackMaxRtts.sum() + val meanRttMs = sumRttMs / feedbackMaxRtts.size + delayBasedBwe.onRttUpdate(meanRttMs.ms) + } + + var feedbackMinRtt = MAX_DURATION + for (packetFeedback in feedbacks) { + val pendingTime = Duration.between(packetFeedback.receiveTime, maxRecvTime) + val rtt = Duration.between(packetFeedback.sentPacket.sendTime, report.feedbackTime) - pendingTime + // Value used for predicting NACK round trip time in FEC controller. + feedbackMinRtt = min(rtt, feedbackMinRtt) + } + if (feedbackMinRtt.isFinite()) { + bandwidthEstimation.updateRtt(feedbackMinRtt, report.feedbackTime) + } + + expectedPacketsSinceLastLossUpdate += report.packetsWithFeedback().size + for (packetFeedback in report.packetsWithFeedback()) { + if (!packetFeedback.isReceived()) { + lostPacketsSinceLastLossUpdate += 1 + } + } + if (report.feedbackTime > nextLossUpdate) { + nextLossUpdate = report.feedbackTime + kLossUpdateInterval + bandwidthEstimation.updatePacketsLost( + lostPacketsSinceLastLossUpdate.toLong(), + expectedPacketsSinceLastLossUpdate.toLong(), + report.feedbackTime + ) + expectedPacketsSinceLastLossUpdate = 0 + lostPacketsSinceLastLossUpdate = 0 + } + } + val alrStartTime = alrDetector.getApplicationLimitedRegionStartTime() + + if (previouslyInAlr && alrStartTime == null) { + val nowMs = report.feedbackTime.toEpochMilli() + acknowledgedBitrateEstimator.setAlrEndedTime(report.feedbackTime) + probeController.setAlrEndedTimeMs(nowMs) + } + previouslyInAlr = alrStartTime != null + acknowledgedBitrateEstimator.incomingPacketFeedbackVector(report.sortedByReceiveTime()) + val acknowledgedBitrate = acknowledgedBitrateEstimator.bitrate() + bandwidthEstimation.setAcknowledgedRate(acknowledgedBitrate, report.feedbackTime) + for (feedback in report.sortedByReceiveTime()) { + if (feedback.sentPacket.pacingInfo.probeClusterId != PacedPacketInfo.kNotAProbe) { + probeBitrateEstimator.handleProbeAndEstimateBitrate(feedback) + } + } + + /* Skipped network_estimator code */ + + var probeBitrate = probeBitrateEstimator.fetchAndResetLastEstimatedBitrate() + + /* Skipped network_estimator code */ + + if (limitProbesLowerThanThroughputEstimate && probeBitrate != null && acknowledgedBitrate != null) { + // Limit the backoff to something slightly below the acknowledged + // bitrate. ("Slightly below" because we want to drain the queues + // if we are actually overusing.) + // The acknowledged bitrate shouldn't normally be higher than the delay + // based estimate, but it could happen e.g. due to packet bursts or + // encoder overshoot. We use std::min to ensure that a probe result + // below the current BWE never causes an increase. + val limit = min(delayBasedBwe.lastEstimate(), acknowledgedBitrate * kProbeDropThroughputFraction) + probeBitrate = max(probeBitrate, limit) + } + + val update = MutableNetworkControlUpdate() + var recoveredFromOveruse = false + + val result = delayBasedBwe.incomingPacketFeedbackVector( + report, + acknowledgedBitrate, + probeBitrate, + alrStartTime != null + ) + + if (result.updated) { + if (result.probe) { + bandwidthEstimation.setSendBitrate(result.targetBitrate, report.feedbackTime) + } + // Since SetSendBitrate now resets the delay-based estimate, we have to + // call UpdateDelayBasedEstimate after SetSendBitrate. + bandwidthEstimation.updateDelayBasedEstimate(report.feedbackTime, result.targetBitrate) + } + bandwidthEstimation.updateLossBasedEstimator( + report, + result.delayDetectorState, + probeBitrate, + alrStartTime != null + ) + if (result.updated) { + // Update the estimate in the ProbeController, in case we want to probe. + maybeTriggerOnNetworkChanged(update, report.feedbackTime) + } + + recoveredFromOveruse = result.recoveredFromOveruse + + if (recoveredFromOveruse) { + probeController.setAlrStartTimeMs(alrStartTime) + val probes = probeController.requestProbe(report.feedbackTime) + update.probeClusterConfigs.addAll(probes) + } + + // No valid RTT could be because send-side BWE isn't used, in which case + // we don't try to limit the outstanding packets. + if (rateControlSettings.useCongestionWindow() && maxFeedbackRtt.isFinite()) { + updateCongestionWindowSize() + } + if (congestionWindowPushbackController != null && currentDataWindow != null) { + congestionWindowPushbackController.setDataWindow(currentDataWindow!!) + } else { + update.congestionWindow = currentDataWindow + } + + return update + } + + fun getNetworkState(atTime: Instant): NetworkControlUpdate { + val update = MutableNetworkControlUpdate() + update.targetRate = TargetTransferRate().apply { + networkEstimate.atTime = atTime + networkEstimate.lossRateRatio = (lastEstimatedFractionLoss ?: 0u).toFloat() / 255.0f + networkEstimate.roundTripTime = lastEstimatedRoundTripTime + networkEstimate.bwePeriod = delayBasedBwe.getExpectedBwePeriod() + + this.atTime = atTime + if (rateControlSettings.useCongestionWindowDropFrameOnly()) { + targetRate = lastLossBasedTargetRate + } else { + targetRate = lastPushbackTargetRate + } + + targetRate = lastPushbackTargetRate + stableTargetRate = bandwidthEstimation.getEstimatedLinkCapacity() + } + update.pacerConfig = getPacingRates(atTime) + update.congestionWindow = currentDataWindow + return update + } + + private fun maybeTriggerOnNetworkChanged(update: MutableNetworkControlUpdate, atTime: Instant) { + val fractionLoss = bandwidthEstimation.fractionLoss() + val roundTripTime = bandwidthEstimation.roundTripTime() + val lossBasedTargetRate = bandwidthEstimation.targetRate() + val lossBasedState = bandwidthEstimation.lossBasedState() + var pushbackTargetRate = lossBasedTargetRate + + /* TODO: plotting */ + + var cwndReduceRatio = 0.0 + if (congestionWindowPushbackController != null) { + var pushbackRate = congestionWindowPushbackController.updateTargetBitrate(lossBasedTargetRate.bps.toInt()) + pushbackRate = Math.max(bandwidthEstimation.getMinBitrate(), pushbackRate) + pushbackTargetRate = pushbackRate.bps + if (rateControlSettings.useCongestionWindowDropFrameOnly()) { + cwndReduceRatio = (lossBasedTargetRate.bps - pushbackTargetRate.bps).toDouble() / + lossBasedTargetRate.bps + } + } + var stableTargetRate = bandwidthEstimation.getEstimatedLinkCapacity() + stableTargetRate = min(stableTargetRate, pushbackTargetRate) + + if (lossBasedTargetRate != lastLossBasedTargetRate || + lossBasedState != lastLossBasedState || + fractionLoss != lastEstimatedFractionLoss || + roundTripTime != lastEstimatedRoundTripTime || + pushbackTargetRate != lastPushbackTargetRate || + stableTargetRate != lastStableTargetRate + ) { + lastLossBasedTargetRate = lossBasedTargetRate + lastPushbackTargetRate = pushbackTargetRate + lastEstimatedFractionLoss = fractionLoss + lastEstimatedRoundTripTime = roundTripTime + lastStableTargetRate = stableTargetRate + lastLossBasedState = lossBasedState + + alrDetector.setEstimatedBitrate(lossBasedTargetRate.bps.toInt()) + + val bwePeriod = delayBasedBwe.getExpectedBwePeriod() + + val targetRateMsg = TargetTransferRate() + targetRateMsg.atTime = atTime + if (rateControlSettings.useCongestionWindowDropFrameOnly()) { + targetRateMsg.targetRate = lossBasedTargetRate + targetRateMsg.cwndReduceRatio = cwndReduceRatio + } else { + targetRateMsg.targetRate = pushbackTargetRate + } + targetRateMsg.stableTargetRate = stableTargetRate + targetRateMsg.networkEstimate.atTime = atTime + targetRateMsg.networkEstimate.roundTripTime = roundTripTime + targetRateMsg.networkEstimate.lossRateRatio = fractionLoss.toFloat() / 255.0f + targetRateMsg.networkEstimate.bwePeriod = bwePeriod + + update.targetRate = targetRateMsg + + val probes = probeController.setEstimatedBitrate( + lossBasedTargetRate, + getBandwidthLimitedCause( + bandwidthEstimation.lossBasedState(), + bandwidthEstimation.isRttAboveLimit(), + delayBasedBwe.lastState() + ), + atTime + ) + update.probeClusterConfigs.addAll(probes) + update.pacerConfig = getPacingRates(atTime) + logger.debug( + "bwe $atTime: pushback_target_bps=${lastPushbackTargetRate.bps} " + + "estimate_bps=${lossBasedTargetRate.bps}" + ) + } + } + + private fun getPacingRates(atTime: Instant): PacerConfig { + // Pacing rate is based on target rate before congestion window pushback, + // because we don't want to build queues in the pacer when pushback occurs. + val pacingRate = max(minTotalAllocatedBitrate, lastLossBasedTargetRate) * pacingFactor + var paddingRate = if (lastLossBasedState == LossBasedState.kIncreaseUsingPadding) { + max(maxPaddingRate, lastLossBasedTargetRate) + } else { + maxPaddingRate + } + paddingRate = min(paddingRate, lastPushbackTargetRate) + val msg = PacerConfig() + msg.atTime = atTime + msg.timeWindow = 1.secs + msg.dataWindow = pacingRate * msg.timeWindow + msg.padWindow = paddingRate * msg.timeWindow + return msg + } + + /** Jitsi local addition. + * Fields based on WebRTC modules/congestion_controller/goog_cc/test/goog_cc_printer.{cc,h} + */ + class StatisticsSnapshot( + val time: Instant, + val rtt: Duration, + val target: Bandwidth, + val stableTarget: Bandwidth, + val pacing: Bandwidth?, + val padding: Bandwidth?, + val window: DataSize, + val rateControlState: AimdRateControl.RateControlState, + val stableEstimate: Bandwidth?, + val trendline: Double, + val trendlineModifiedOffset: Double, + val trendlineOffsetThreshold: Double, + val acknowledgedRate: Bandwidth?, + /* Skipped, based on NetworkStateEstimate + * estCapacity + * estCapacityDev + * estCapacityMin + * estCrossDelay + * estSpikeDelay + * estPreBuffer + * estPostBuffer + * estPropagation + */ + + val lossRatio: Float, + + /* Fields where data from LossBasedBweV1 are printed, even though LossBasedBweV2 is the default. + TODO: print state out of LossBasedBweV2. + val lossAverage: Double, + val lossAverageMax: Double, + val lossThresInc: Double, + val lossThresDec: Double, + val lossBasedRate: Bandwidth, + val lossAckRate: Bandwidth, + */ + /* SendSideBandwidthEstimator populates itself from LossBasedBwe's estimate. */ + val sendSideTarget: Bandwidth, + val lossBasedState: LossBasedState, + val dataWindow: DataSize?, + val pushbackTarget: Bandwidth, + /* Additions to the fields from goog_cc_printer */ + val inAlr: Boolean + ) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("time", time.toEpochMilli()) + put("rtt", rtt.toDouble()) + put("target", target.bps) + put("stable_target", stableTarget.bps) + put("pacing", pacing?.bps?.toDouble() ?: Double.NaN) + put("padding", padding?.bps?.toDouble() ?: Double.NaN) + put("window", window.bytes) + put("rate_control_state", rateControlState.name) + put("stable_estimate", stableEstimate?.bps?.toDouble() ?: Double.NaN) + put("trendline", trendline) + put("trendline_modified_offset", trendlineModifiedOffset) + put("trendline_modified_threshold", trendlineOffsetThreshold) + put("acknowledged_rate", acknowledgedRate?.bps?.toDouble() ?: Double.NaN) + put("loss_ratio", lossRatio) + put("send_side_target", sendSideTarget.bps) + put("last_loss_based_state", lossBasedState.name) + put("data_window", dataWindow?.bytes ?: Double.NaN) + put("pushback_target", pushbackTarget.bps) + put("in_alr", inAlr) + } + + fun addToTimeSeriesPoint(point: DiagnosticContext.TimeSeriesPoint) { + point.apply { + addField("rtt", rtt.toDouble()) + addField("target", target.bps) + addField("stable_target", stableTarget.bps) + addField("pacing", pacing?.bps ?: Double.NaN) + addField("padding", padding?.bps ?: Double.NaN) + addField("window", window.bytes) + addField("rate_control_state", rateControlState.name) + addField("stable_estimate", stableEstimate?.bps ?: Double.NaN) + addField("trendline", trendline) + addField("trendline_modified_offset", trendlineModifiedOffset) + addField("trendline_modified_threshold", trendlineOffsetThreshold) + addField("acknowledged_rate", acknowledgedRate?.bps ?: Double.NaN) + addField("loss_ratio", lossRatio) + addField("send_side_target", sendSideTarget.bps) + addField("last_loss_based_state", lossBasedState.name) + addField("data_window", dataWindow?.bytes ?: Double.NaN) + addField("pushback_target", pushbackTarget.bps) + addField("in_alr", inAlr) + } + } + } + + private fun trend() = delayBasedBwe.delayDetector as TrendlineEstimator + + fun getStatistics(now: Instant): StatisticsSnapshot { + val stateUpdate = getNetworkState(now) + val target = stateUpdate.targetRate!! + val pacing = stateUpdate.pacerConfig + val congestionWindow = stateUpdate.congestionWindow ?: DataSize.INFINITY + return StatisticsSnapshot( + time = target.atTime, + rtt = target.networkEstimate.roundTripTime, + target = target.targetRate, + stableTarget = target.stableTargetRate, + pacing = pacing?.dataRate(), + padding = pacing?.padRate(), + window = congestionWindow, + rateControlState = delayBasedBwe.rateControl.rateControlState, + stableEstimate = delayBasedBwe.rateControl.linkCapacity.estimate, + trendline = trend().prevTrend, + trendlineModifiedOffset = trend().prevModifiedTrend, + trendlineOffsetThreshold = trend().threshold, + acknowledgedRate = acknowledgedBitrateEstimator.bitrate(), + lossRatio = target.networkEstimate.lossRateRatio, + sendSideTarget = bandwidthEstimation.targetRate(), + lossBasedState = lastLossBasedState, + dataWindow = currentDataWindow, + pushbackTarget = lastPushbackTargetRate, + inAlr = previouslyInAlr + ) + } + + private val packetFeedbackOnly = googCcConfig.feedbackOnly + private val safeResetOnRouteChange = false + private val safeResetAcknowledgedRate = false + private val useMinAllocatableAsLowerBound = true + private val limitProbesLowerThanThroughputEstimate = true + private val rateControlSettings = googCcConfig.rateControlSettings + + private val probeController = ProbeController(logger, diagnosticContext) + private val congestionWindowPushbackController = if (rateControlSettings.useCongestionWindowPushback()) { + CongestionWindowPushbackController() + } else { + null + } + + private val bandwidthEstimation = SendSideBandwidthEstimation(logger, diagnosticContext) + private val alrDetector = AlrDetector() + private var probeBitrateEstimator = ProbeBitrateEstimator(logger, diagnosticContext) + private var delayBasedBwe = DelayBasedBwe(logger, diagnosticContext).also { + it.setMinBitrate(kCongestionControllerMinBitrate) + } + private var acknowledgedBitrateEstimator = AcknowledgedBitrateEstimatorInterface.create() + + private var initialConfig: NetworkControllerConfig? = config + + private var minTargetRate = Bandwidth.ZERO + private var minDataRate = Bandwidth.ZERO + private var maxDataRate = Bandwidth.ZERO + private var startingRate: Bandwidth? = null + + private var firstPacketSent = false + + /* Skipping NetworkStateEstimate */ + + private var nextLossUpdate = Instant.MIN + private var lostPacketsSinceLastLossUpdate = 0 + private var expectedPacketsSinceLastLossUpdate = 0 + + private val feedbackMaxRtts = ArrayDeque() + + private var lastLossBasedTargetRate = config.constraints.startingRate!! + private var lastPushbackTargetRate = lastLossBasedTargetRate + private var lastStableTargetRate = lastLossBasedTargetRate + private var lastLossBasedState: LossBasedState = LossBasedState.kDelayBasedEstimate + + private var lastEstimatedFractionLoss: UByte? = 0u + private var lastEstimatedRoundTripTime = MAX_DURATION + + private var pacingFactor = config.streamBasedConfig.pacingFactor ?: kDefaultPaceMultiplier + private var minTotalAllocatedBitrate = config.streamBasedConfig.minTotalAllocatedBitrate ?: Bandwidth.ZERO + private var maxPaddingRate = config.streamBasedConfig.maxPaddingRate ?: Bandwidth.ZERO + + private var previouslyInAlr = false + private var currentDataWindow: DataSize? = null + + companion object { + // From RTCPSender video report interval. + private val kLossUpdateInterval = 1000.ms + + // Pacing-rate relative to our target send rate. + // Multiplicative factor that is applied to the target bitrate to calculate + // the number of bytes that can be transmitted per interval. + // Increasing this factor will result in lower delays in cases of bitrate + // overshoots from the encoder. + private const val kDefaultPaceMultiplier = 2.5 + + // If the probe result is far below the current throughput estimate + // it's unlikely that the probe is accurate, so we don't want to drop too far. + // However, if we actually are overusing, we want to drop to something slightly + // below the current throughput estimate to drain the network queues. + private const val kProbeDropThroughputFraction = 0.85 + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(GoogCcNetworkController::class.java) + + private fun getBandwidthLimitedCause( + lossBasedState: LossBasedState, + isRttAboveLimit: Boolean, + bandwidthUsage: BandwidthUsage + ): BandwidthLimitedCause { + if (bandwidthUsage == BandwidthUsage.kBwOverusing || + bandwidthUsage == BandwidthUsage.kBwUnderusing + ) { + return BandwidthLimitedCause.kDelayBasedLimitedDelayIncreased + } else if (isRttAboveLimit) { + return BandwidthLimitedCause.kRttBasedBackOffHighRtt + } + + return when (lossBasedState) { + LossBasedState.kDecreasing -> + // Probes may not be sent in this state. + BandwidthLimitedCause.kLossLimitedBwe + + LossBasedState.kIncreaseUsingPadding -> + // Probes may not be sent in this state. + BandwidthLimitedCause.kLossLimitedBwe + + LossBasedState.kIncreasing -> + // Probes may be sent in this state. + BandwidthLimitedCause.kLossLimitedBweIncreasing + + LossBasedState.kDelayBasedEstimate -> + BandwidthLimitedCause.kDelayBasedLimited + } + } + } +} + +/* Congestion window config, from WebRTC rtc_base/experiments/rate_control_settings.{h,cc} + * stripped down to used fields only. */ +class CongestionWindowConfig( + private val queueSizeMs: Int? = 350, + private val minBitrateBps: Int? = 30000, + private val dropFrameOnly: Boolean = true +) { + fun useCongestionWindow() = queueSizeMs != null + + fun getCongestionWindowAdditionalTimeMs() = queueSizeMs ?: kDefaultAcceptedQueueMs + + fun useCongestionWindowPushback() = queueSizeMs != null && minBitrateBps != null + + fun useCongestionWindowDropFrameOnly() = dropFrameOnly + + companion object { + private const val kDefaultAcceptedQueueMs = 350 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcTransportCcEngine.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcTransportCcEngine.kt new file mode 100644 index 0000000..a32456b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcTransportCcEngine.kt @@ -0,0 +1,371 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorConfig +import org.jitsi.nlj.rtp.bandwidthestimation2.PacedPacketInfo.Companion.kNotAProbe +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpReportBlock +import org.jitsi.rtp.rtcp.RtcpRrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.* +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + +/** + * Transport CC engine invoking GoogCc NetworkController. Contains some code based loosely on + * WebRTC call/rtp_transport_controller_send.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138) + * + */ +class GoogCcTransportCcEngine( + val diagnosticContext: DiagnosticContext, + parentLogger: Logger, + private val scheduledExecutor: ScheduledExecutorService, + private val sendProbing: (DataSize, Any?) -> Int, + val clock: Clock = Clock.systemUTC(), +) : TransportCcEngine() { + private val logger = createChildLogger(parentLogger) + + private val feedbackAdapter = TransportFeedbackAdapter(logger) + private val networkController = factory.create( + NetworkControllerConfig( + logger, + diagnosticContext, + constraints = TargetRateConstraints( + atTime = clock.instant(), + startingRate = BandwidthEstimatorConfig.initBw, + maxDataRate = BandwidthEstimatorConfig.maxBw, + minDataRate = BandwidthEstimatorConfig.minBw + ) + ) + ) + private val bitrateProber = BitrateProber(logger) + private var probeTask: ScheduledFuture<*>? = null + + private var processTask: ScheduledFuture<*>? = null + + private val listeners = LinkedList() + + @Synchronized + override fun onRttUpdate(rtt: Duration) { + val update = + networkController.onRoundTripTimeUpdate( + RoundTripTimeUpdate(receiveTime = clock.instant(), roundTripTime = rtt) + ) + processUpdate(update) + } + + @Synchronized + override fun rtcpPacketReceived(rtcpPacket: RtcpPacket, receivedTime: Instant?) { + when (rtcpPacket) { + is RtcpFbTccPacket -> { + val time = receivedTime ?: clock.instant() + val feedback = feedbackAdapter.processTransportFeedback(rtcpPacket, time) + if (feedback != null) { + val update = networkController.onTransportPacketsFeedback(feedback) + processUpdate(update) + + feedback.packetFeedbacks.forEach { fb -> + lossListeners.forEach { l -> + if (fb.isReceived()) { + l.packetReceived(fb.previouslyReportedLost) + } else if (!fb.previouslyReportedLost) { + l.packetLost(1) + } + } + } + } + } + + is RtcpFbRembPacket -> { + /* Ignore REMB packets - if we're supposed to be receiving them they'll be handled by [RembHandler], + * and if we're not we're getting mysterious spurious REMB messages which we want to ignore. + */ + + /* + val time = receivedTime ?: clock.instant() + val msg = RemoteBitrateReport(receiveTime = time, bandwidth = rtcpPacket.bitrate.bps) + val update = networkController.onRemoteBitrateReport(msg) + processUpdate(update) + */ + } + + is RtcpSrPacket -> { + val time = receivedTime ?: clock.instant() + onReport(time, rtcpPacket.reportBlocks) + } + + is RtcpRrPacket -> { + val time = receivedTime ?: clock.instant() + onReport(time, rtcpPacket.reportBlocks) + } + } + } + + private class LossReport { + var extendedHighestSequenceNumber: Long = 0 + var cumulativeLost = 0 + } + + private val lastReportBlocks = HashMap() + private var lastReportBlockTime = clock.instant() + + private fun onReport(receiveTime: Instant, reportBlocks: List) { + if (reportBlocks.isEmpty()) { + return + } + + var totalPacketsLostDelta = 0L + var totalPacketsDelta = 0L + + reportBlocks.forEach { reportBlock -> + val newLossReport = LossReport() + val lastLossReport = lastReportBlocks.putIfAbsent(reportBlock.ssrc, newLossReport) + if (lastLossReport != null) { + totalPacketsDelta += reportBlock.extendedHighestSeqNum - lastLossReport.extendedHighestSequenceNumber + totalPacketsLostDelta += reportBlock.cumulativePacketsLost - lastLossReport.cumulativeLost + } + val lossReport = lastLossReport ?: newLossReport + lossReport.extendedHighestSequenceNumber = reportBlock.extendedHighestSeqNum + lossReport.cumulativeLost = reportBlock.cumulativePacketsLost + } + // Can only compute delta if there has been previous blocks to compare to. If + // not, total_packets_delta will be unchanged and there's nothing more to do. + if (totalPacketsDelta == 0L) { + return + } + val packetsReceivedDelta = totalPacketsDelta - totalPacketsLostDelta + // To detect lost packets, at least one packet has to be received. + if (packetsReceivedDelta < 1) { + return + } + val msg = TransportLossReport( + packetsLostDelta = totalPacketsLostDelta, + packetsReceivedDelta = packetsReceivedDelta, + receiveTime = receiveTime, + startTime = lastReportBlockTime, + endTime = receiveTime + ) + val update = networkController.onTransportLossReport(msg) + processUpdate(update) + lastReportBlockTime = receiveTime + } + + @Synchronized + override fun mediaPacketTagged(packetInfo: org.jitsi.nlj.PacketInfo, tccSeqNum: Long) { + val now = clock.instant() + val length = packetInfo.packet.length.bytes + val pacedPacketInfo = packetInfo.probingInfo as? PacedPacketInfo + feedbackAdapter.addPacket( + packetInfo, + tccSeqNum, + DataSize.ZERO, // TODO: network overhead + creationTime = now + ) + if (pacedPacketInfo == null) { + bitrateProber.onIncomingPacket(length) + } + maybeScheduleProbing(now) + } + + @Synchronized + override fun mediaPacketSent(packetInfo: org.jitsi.nlj.PacketInfo, tccSeqNum: Long) { + val now = clock.instant() + val length = packetInfo.packet.length.toLong() + val sentPacketInfo = SentPacketInfo( + packetId = tccSeqNum, + sendTime = now, + info = PacketInfo( + // TODO I think these should always be true when tccSeqNum is defined? + includedInAllocation = true, + includedInFeedback = true, + packetSizeBytes = length + ) + ) + val sentPacket = feedbackAdapter.processSentPacket(sentPacketInfo) + if (sentPacket != null) { + val update = networkController.onSentPacket(sentPacket) + processUpdate(update) + } + } + + @Synchronized + override fun getStatistics(): StatisticsSnapshot { + val now = clock.instant() + return StatisticsSnapshot( + feedbackAdapter.getStatisitics(), + (networkController as GoogCcNetworkController).getStatistics(now) + ) + } + + @Synchronized + override fun addBandwidthListener(listener: BandwidthListener) { + listeners.add(listener) + } + + @Synchronized + override fun removeBandwidthListener(listener: BandwidthListener) { + listeners.remove(listener) + } + + @Synchronized + override fun start() { + val startTime = clock.instant() + var update = + networkController.onNetworkAvailability(NetworkAvailability(atTime = startTime, networkAvailable = true)) + processUpdate(update) // Does this make sense to do during init? + + update = networkController.onProcessInterval(ProcessInterval(atTime = startTime)) + processUpdate(update) + + val processInterval = factory.getProcessInterval().toMillis() + + processTask = scheduledExecutor.scheduleAtFixedRate({ + synchronized(this@GoogCcTransportCcEngine) { + val now = clock.instant() + update = networkController.onProcessInterval(ProcessInterval(atTime = now)) + processUpdate(update) + } + }, processInterval, processInterval, TimeUnit.MILLISECONDS) + } + + @Synchronized + override fun stop() { + // Stop bitrateProber from initiating any new probes + bitrateProber.setEnabled(false) + probeTask?.cancel(false) + processTask?.cancel(false) + } + + private fun processUpdate(update: NetworkControlUpdate) { + update.targetRate?.let { targetRate -> + logger.debug { "GoogleCcEstimator setting TargetRate to $targetRate" } + listeners.forEach { it.bandwidthEstimationChanged(targetRate.targetRate) } + } + update.congestionWindow?.let { congestionWindow -> + /* We don't use a congestion window */ + /* TODO: does this do anything bad to the estimator? */ + logger.trace { "GoogleCcEstimator wants to set CongestionWindow to $congestionWindow" } + } + update.pacerConfig?.let { pacerConfig -> + /* We don't use a pacer */ + /* TODO: does this do anything bad to the estimator? */ + logger.trace { "GoogleCcEstimator wants to set PacerConfig to $pacerConfig" } + } + update.probeClusterConfigs.let { configs -> + if (configs.isNotEmpty()) { + logger.debug { "GoogleCcEstimator creating ${configs.size} ProbeClusterConfigs: $configs" } + configs.forEach { config -> + bitrateProber.createProbeCluster(config) + } + maybeScheduleProbing(clock.instant()) + } + } + + if (timeSeriesLogger.isTraceEnabled && update.isNotEmpty()) { + val now = update.atTime ?: clock.instant() + val stats = (networkController as GoogCcNetworkController).getStatistics(now) + val statsPoint = diagnosticContext.makeTimeSeriesPoint("goog_cc_stats", now) + stats.addToTimeSeriesPoint(statsPoint) + timeSeriesLogger.trace(statsPoint) + + val updatePoint = diagnosticContext.makeTimeSeriesPoint("goog_cc_update", now) + update.addToTimeSeriesPoint(updatePoint) + timeSeriesLogger.trace(updatePoint) + } + } + + /** Schedule bitrate probing if needed and not current scheduled. + * Should be synchronized on this@GoogCcTransportCcEngine. */ + private fun maybeScheduleProbing(now: Instant) { + if (bitrateProber.isProbing() && probeTask == null) { + val nextProbeTime = bitrateProber.nextProbeTime(now) + if (nextProbeTime == Instant.MAX) { + return + } + val delay = if (nextProbeTime == Instant.MIN) { + 0 + } else { + Duration.between(now, nextProbeTime).toMillis().coerceAtLeast(0) + } + + probeTask = scheduledExecutor.schedule({ + val cluster: PacedPacketInfo + val probeSize: DataSize + synchronized(this@GoogCcTransportCcEngine) { + probeTask = null + val now = clock.instant() + cluster = bitrateProber.currentCluster(now) ?: PacedPacketInfo() + if (cluster.probeClusterId == kNotAProbe) { + return@schedule + } + probeSize = bitrateProber.recommendedMinProbeSize() + val probeSent = sendProbing(probeSize, cluster) + bitrateProber.probeSent(now, probeSent.bytes) + maybeScheduleProbing(now) + } + }, delay, TimeUnit.MILLISECONDS) + } + } + + companion object { + private val factory = GoogCcNetworkControllerFactory() + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(GoogCcTransportCcEngine::class.java) + + /* Default config settings to use when this version of the Google transport CC estimator engine is used. */ + val defaultRateTrackerWindowSize: Duration by config { + "jmt.bwe.estimator.GoogleCc2.default-window-size".from(JitsiConfig.newConfig) + } + val defaultRateTrackerBucketSize: Duration by config { + "jmt.bwe.estimator.GoogleCc2.default-bucket-size".from(JitsiConfig.newConfig) + } + val defaultInitialIgnoreBwePeriod: Duration by config { + "jmt.bwe.estimator.GoogleCc2.default-initial-ignore-bwe-period".from(JitsiConfig.newConfig) + } + } + + class StatisticsSnapshot( + val transportAdapterState: TransportFeedbackAdapter.StatisticsSnapshot, + val networkControllerState: GoogCcNetworkController.StatisticsSnapshot + ) : TransportCcEngine.StatisticsSnapshot() { + override val unmatchedFeedback: Long get() = transportAdapterState.totalUnmatchedReports + + override fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("name", GoogCcTransportCcEngine::class.java.simpleName) + set("transport_adapter", transportAdapterState.toJson()) + set("network_controller", networkControllerState.toJson()) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/InterArrivalDelta.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/InterArrivalDelta.kt new file mode 100644 index 0000000..b55b57c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/InterArrivalDelta.kt @@ -0,0 +1,181 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.utils.NEVER +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.max +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +/** + * Helper class to compute the inter-arrival time delta and the size delta + * between two timestamp groups. This code is branched from + * modules/remote_bitrate_estimator/inter_arrival. + * * + * Based on WebRTC modules/congestion_controller/goog_cc/inter_arrival_delta.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class InterArrivalDelta( + private val sendTimeGroupLength: Duration +) { + // TODO: pass parent logger in so we have log contexts + private val logger = createLogger() + + private var currentTimestampGroup: SendTimeGroup = SendTimeGroup() + private var prevTimestampGroup: SendTimeGroup = SendTimeGroup() + private var numConsecutiveReorderedPackets: Int = 0 + + data class ComputeDeltasResult( + val computed: Boolean, + val sendTimeDelta: Duration = Duration.ZERO, + val arrivalTimeDelta: Duration = Duration.ZERO, + val packetSizeDelta: Int = 0 + ) + + fun computeDeltas( + sendTime: Instant, + arrivalTime: Instant, + systemTime: Instant, + packetSize: Long + ): ComputeDeltasResult { + var sendTimeDelta = Duration.ZERO + var arrivalTimeDelta = Duration.ZERO + var packetSizeDelta = 0 + + var calculatedDeltas = false + + if (currentTimestampGroup.isFirstPacket()) { + // We don't have enough data to update the filter, so we store it until we + // have two frames of data to process. + currentTimestampGroup.sendTime = sendTime + currentTimestampGroup.firstSendTime = sendTime + currentTimestampGroup.firstArrival = arrivalTime + } else if (currentTimestampGroup.firstSendTime > sendTime) { + // Reordered packet + return ComputeDeltasResult(false) + } else if (newTimestampGroup(arrivalTime, sendTime)) { + // First packet of a later frame, the previous frame sample is ready. + if (prevTimestampGroup.completeTime != NEVER) { + sendTimeDelta = Duration.between(prevTimestampGroup.sendTime, currentTimestampGroup.sendTime) + arrivalTimeDelta = Duration.between(prevTimestampGroup.completeTime, currentTimestampGroup.completeTime) + val systemTimeDelta = + Duration.between(prevTimestampGroup.lastSystemTime, currentTimestampGroup.lastSystemTime) + + if (arrivalTimeDelta - systemTimeDelta >= kArrivalTimeOffsetThreshold) { + logger.warn( + "The arrival clock offset has changed (diff = " + + "${arrivalTimeDelta - systemTimeDelta}), resetting" + ) + reset() + return ComputeDeltasResult(false) + } + if (arrivalTimeDelta < Duration.ZERO) { + // The group of packets has been reordered since receiving its local + // arrival timestamp. + ++numConsecutiveReorderedPackets + if (numConsecutiveReorderedPackets >= kReorderedResetThreshold) { + logger.warn( + "Packets between send burst arrived out of order, restting: " + + "arrivalTimeDelta=$arrivalTimeDelta, sendTimeDelta=$sendTimeDelta" + ) + reset() + } + return ComputeDeltasResult(false) + } else { + numConsecutiveReorderedPackets = 0 + } + packetSizeDelta = currentTimestampGroup.size.toInt() - prevTimestampGroup.size.toInt() + calculatedDeltas = true + } + prevTimestampGroup = currentTimestampGroup.copy() + // The new timestamp is now the current frame + currentTimestampGroup.firstSendTime = sendTime + currentTimestampGroup.sendTime = sendTime + currentTimestampGroup.firstArrival = arrivalTime + currentTimestampGroup.size = 0 + } else { + currentTimestampGroup.sendTime = max(currentTimestampGroup.sendTime, sendTime) + } + // Accumulate the frame size. + currentTimestampGroup.size += packetSize + currentTimestampGroup.completeTime = arrivalTime + currentTimestampGroup.lastSystemTime = systemTime + + return ComputeDeltasResult(calculatedDeltas, sendTimeDelta, arrivalTimeDelta, packetSizeDelta) + } + + private fun newTimestampGroup(arrivalTime: Instant, sendTime: Instant): Boolean { + if (currentTimestampGroup.isFirstPacket()) { + return false + } else if (belongsToBurst(arrivalTime, sendTime)) { + return false + } else { + return Duration.between(currentTimestampGroup.firstSendTime, sendTime) > sendTimeGroupLength + } + } + + private fun belongsToBurst(arrivalTime: Instant, sendTime: Instant): Boolean { + check(currentTimestampGroup.completeTime != NEVER) + val arrivalTimeDelta = Duration.between(currentTimestampGroup.completeTime, arrivalTime) + val sendTimeDelta = Duration.between(currentTimestampGroup.sendTime, sendTime) + if (sendTimeDelta == Duration.ZERO) { + return true + } + val propagationDelta = arrivalTimeDelta - sendTimeDelta + if (propagationDelta < Duration.ZERO && + arrivalTimeDelta <= kBurstDeltaThreshold && + Duration.between(currentTimestampGroup.firstArrival, arrivalTime) < kMaxBurstDuration + ) { + return true + } + return false + } + + private fun reset() { + numConsecutiveReorderedPackets = 0 + currentTimestampGroup = SendTimeGroup() + prevTimestampGroup = SendTimeGroup() + } + + data class SendTimeGroup( + var size: Long = 0, + var firstSendTime: Instant = NEVER, + var sendTime: Instant = NEVER, + var firstArrival: Instant = NEVER, + var completeTime: Instant = NEVER, + var lastSystemTime: Instant = NEVER + ) { + fun isFirstPacket(): Boolean = completeTime == NEVER + } + + companion object { + /** + * After this many packet groups received out of order InterArrival will + * reset, assuming that clocks have made a jump. + */ + const val kReorderedResetThreshold = 3 + + val kArrivalTimeOffsetThreshold = 3.secs + + val kBurstDeltaThreshold = 5.ms + val kMaxBurstDuration = 100.ms + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudget.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudget.kt new file mode 100644 index 0000000..960083e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudget.kt @@ -0,0 +1,77 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This file uses WebRTC's naming style for constants +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import kotlin.math.max +import kotlin.math.min + +/** Interval budget, + * based on WebRTC modules/pacing/interval_budget.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +class IntervalBudget( + initialTargetRateKbps: Int, + private val canBuildUpUnderuse: Boolean = false +) { + private var targetRateKbps: Int = 0 + private var maxBytesInBudget: Long = 0 + private var bytesRemaining: Long = 0 + + init { + setTargetRateKbps(initialTargetRateKbps) + } + + fun setTargetRateKbps(targetRateKbps: Int) { + this.targetRateKbps = targetRateKbps + maxBytesInBudget = (kWindowMs * targetRateKbps) / 8 + bytesRemaining = min(max(-maxBytesInBudget, bytesRemaining), maxBytesInBudget) + } + + // TODO(tschumim): Unify IncreaseBudget and UseBudget to one function. + fun increaseBudget(deltaTimeMs: Long) { + val bytes = targetRateKbps * deltaTimeMs / 8 + if (bytesRemaining < 0 || canBuildUpUnderuse) { + // We overused last interval, compensate this interval. + bytesRemaining = min(bytesRemaining + bytes, maxBytesInBudget) + } else { + // If we underused last interval we can't use it this interval. + bytesRemaining = min(bytes, maxBytesInBudget) + } + } + + fun useBudget(bytes: Long) { + bytesRemaining = max(bytesRemaining - bytes, -maxBytesInBudget) + } + + fun bytesRemaining(): Long = max(0, bytesRemaining) + + fun budgetRatio(): Double { + if (maxBytesInBudget == 0L) { + return 0.0 + } + return bytesRemaining.toDouble() / maxBytesInBudget + } + + fun targetRateKbps(): Int = targetRateKbps + + companion object { + private const val kWindowMs = 500L + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LinkCapacityEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LinkCapacityEstimator.kt new file mode 100644 index 0000000..69ae9bc --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LinkCapacityEstimator.kt @@ -0,0 +1,75 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.kbps +import kotlin.math.max +import kotlin.math.sqrt + +/** Link capacity estimator, + * based on WebRTC modules/congestion_controller/goog_cc/link_capacity_estimator.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class LinkCapacityEstimator { + private var estimateKbps: Double? = null + private var deviationKbps: Double = 0.4 + + fun upperBound(): Bandwidth = estimateKbps?.let { + (it + 3 * deviationEstimateKbps()).kbps + } ?: Bandwidth.INFINITY + + fun lowerBound(): Bandwidth = estimateKbps?.let { + (max(0.0, it - 3 * deviationEstimateKbps())).kbps + } ?: Bandwidth.ZERO + + fun reset() { + estimateKbps = null + } + + fun onOveruseDetected(acknowledgedRate: Bandwidth) = update(acknowledgedRate, 0.05) + + fun onProbeRate(probeRate: Bandwidth) = update(probeRate, 0.5) + + fun hasEstimate(): Boolean = estimateKbps != null + + val estimate: Bandwidth? + get() = estimateKbps?.kbps + + private fun update(capacitySample: Bandwidth, alpha: Double) { + // This double-cast is probably a bug in the Google CC code (it calls `capacity_sample.kbps()` rather than + // `capacity_sample.kbps()`) but emulate it to be bit-exact. + val sampleKbps = capacitySample.kbps.toLong().toDouble() + estimateKbps = if (estimateKbps == null) { + sampleKbps + } else { + (1 - alpha) * estimateKbps!! + alpha * sampleKbps + } + // Estimate the variance of the link capacity estimate and normalize the + // variance with the link capacity estimate. + val norm = max(estimateKbps!!, 1.0) + val errorKbps = estimateKbps!! - sampleKbps + deviationKbps = (1 - alpha) * deviationKbps + alpha * errorKbps * errorKbps / norm + // 0.4 ~= 14 kbit/s at 500 kbit/s + // 2.5 ~= 35 kbit/s at 500 kbit/s + deviationKbps = deviationKbps.coerceIn(0.4, 2.5) + } + + private fun deviationEstimateKbps(): Double = // Calculate the max bit rate std dev given the normalized + // variance and the current throughput bitrate. The standard deviation will + // only be used if estimateKbps has a value. + sqrt(deviationKbps * estimateKbps!!) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2.kt new file mode 100644 index 0000000..b5bba9d --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2.kt @@ -0,0 +1,1169 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This file uses WebRTC's naming style for enums +@file:Suppress("ktlint:standard:enum-entry-name-case") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.max +import org.jitsi.nlj.util.min +import org.jitsi.nlj.util.per +import org.jitsi.nlj.util.times +import org.jitsi.utils.div +import org.jitsi.utils.isFinite +import org.jitsi.utils.isInfinite +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.max +import org.jitsi.utils.min +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.times +import java.time.Duration +import java.time.Instant +import kotlin.math.abs +import kotlin.math.ln +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow + +/** Loss-based bandwidth estimation, + * based on WebRTC modules/congestion_controller/goog_cc/loss_based_bwe_v2.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings are settable through the API, but not currently as config options. + */ +enum class LossBasedState { + kIncreasing, + + // TODO(bugs.webrtc.org/12707): Remove one of the increasing states once we + // have decided if padding is usefull for ramping up when BWE is loss + // limited. + kIncreaseUsingPadding, + kDecreasing, + kDelayBasedEstimate +} + +private val kInitHoldDuration = 300.ms +private val kMaxHoldDuration = 60.secs + +private fun isValid(datarate: Bandwidth?): Boolean = datarate?.isFinite() ?: false + +private fun isValid(timestamp: Instant) = timestamp.isFinite() + +private fun getLossProbability(inherentLoss_: Double, lossLimitedBandwidth: Bandwidth, sendingRate: Bandwidth): Double { + if (inherentLoss_ < 0.0 || inherentLoss_ > 1.0) { + LossBasedBweV2.logger.warn("The inherent loss must be in [0,1]: $inherentLoss_") + } + val inherentLoss = inherentLoss_.coerceIn(0.0, 1.0) + if (!sendingRate.isFinite()) { + LossBasedBweV2.logger.warn("The sending rate must be finite: $sendingRate") + } + if (!lossLimitedBandwidth.isFinite()) { + LossBasedBweV2.logger.warn("The loss limited bandwidth must be finite: $lossLimitedBandwidth") + } + var lossProbability = inherentLoss + + if (isValid(sendingRate) && isValid(lossLimitedBandwidth) && + (sendingRate > lossLimitedBandwidth) + ) { + lossProbability += (1 - inherentLoss) * + ((sendingRate - lossLimitedBandwidth) / sendingRate) + } + return lossProbability.coerceIn(1.0e-6, 1.0 - 1.0e-6) +} + +class LossBasedBweV2(configIn: Config = defaultConfig) { + private val config = configIn.copy() + private var acknowledgedBitrate: Bandwidth? = null + private var currentBestEstimate: ChannelParameters = + ChannelParameters(inherentLoss = config.initialInherentLossEstimate) + private var numObservations = 0 + private val observations = MutableList(config.observationWindowSize) { Observation() } + private var partialObservation = PartialObservation() + private var lastSendTimeMostRecentObservation = Instant.MAX + private var lastTimeEstimateReduced = Instant.MIN + private var cachedInstantUpperBound: Bandwidth? = null + private var cachedInstantLowerBound: Bandwidth? = null + private val instantUpperBoundTemporalWeights = MutableList(config.observationWindowSize) { 0.0 } + private val temporalWeights = MutableList(config.observationWindowSize) { 0.0 } + private var recoveringAfterLossTimestamp = Instant.MIN + private var bandwidthLimitInCurrentWindow = Bandwidth.INFINITY + private var minBitrate = 1.kbps + private var maxBitrate = Bandwidth.INFINITY + private var delayBasedEstimate = Bandwidth.INFINITY + private var lossBasedResult = Result() + private var lastHoldInfo = HoldInfo(duration = kInitHoldDuration) + private var lastPaddingInfo = PaddingInfo() + private var averageReportedLossRatio = 0.0 + + init { + if (!config.enabled) { + logger.debug( + "The configuration does not specify that the " + + "estimator should be enabled, disabling it." + ) + } + if (!config.isValid()) { + logger.warn("The configuration is not valid, disabling the estimator.") + config.enabled = false + } + calculateTemporalWeights() + } + + data class Result( + var bandwidthEstimate: Bandwidth = Bandwidth.ZERO, + var state: LossBasedState = LossBasedState.kDelayBasedEstimate + ) + + fun isEnabled(): Boolean = config.enabled + + /** Returns true iff a BWE can be calculated, i.e., the estimator has been + initialized with a BWE and then has received enough `PacketResult`s. + */ + fun isReady(): Boolean = isEnabled() && + isValid(currentBestEstimate.lossLimitedBandwidth) && + numObservations >= config.minNumObservations + + fun readyToUseInStartPhase(): Boolean = isReady() && config.useInStartPhase + + fun useInStartPhase(): Boolean = config.useInStartPhase + + /** Returns [Bandwidth.INFINITY] if no BWE can be calculated. */ + fun getLossBasedResult(): Result { + if (!isReady()) { + if (!isEnabled()) { + logger.warn("The estimator must be enabled before it can be used.") + } else { + if (!isValid(currentBestEstimate.lossLimitedBandwidth)) { + logger.warn("The estimator must be initialized before it can be used.") + } + if (numObservations <= config.minNumObservations) { + logger.warn("The estimator must receive enough loss statistics before it can be used.") + } + } + + return Result( + bandwidthEstimate = if (isValid(delayBasedEstimate)) { + delayBasedEstimate + } else { + Bandwidth.INFINITY + }, + state = LossBasedState.kDelayBasedEstimate + ) + } + return lossBasedResult.copy() + } + + fun setAcknowledgedBitrate(acknowledgedBitrate: Bandwidth) { + if (isValid(acknowledgedBitrate)) { + this.acknowledgedBitrate = acknowledgedBitrate + calculateInstantLowerBound() + } else { + logger.warn("The acknowledged bitrate must be finite: $acknowledgedBitrate") + } + } + + fun setMinMaxBitrate(minBitrate: Bandwidth, maxBitrate: Bandwidth) { + if (isValid(minBitrate)) { + this.minBitrate = minBitrate + calculateInstantLowerBound() + } else { + logger.warn("The min bitrate must be finite: $minBitrate") + } + + if (isValid(maxBitrate)) { + this.maxBitrate = maxBitrate + } else { + logger.warn("The max bitrate must be finite: $maxBitrate") + } + } + + fun updateBandwidthEstimate(packetResults: List, delayBasedEstimate: Bandwidth, inAlr: Boolean) { + this.delayBasedEstimate = delayBasedEstimate + if (!isEnabled()) { + logger.warn("The estimator must be enabled before it can be used.") + return + } + + if (packetResults.isEmpty()) { + logger.debug("The estimate cannot be updated without any loss statistics.") + return + } + + if (!pushBackObservation(packetResults)) { + return + } + + if (!isValid(currentBestEstimate.lossLimitedBandwidth)) { + if (!isValid(delayBasedEstimate)) { + logger.warn("The delay based estimate must be finite: $delayBasedEstimate.") + return + } + currentBestEstimate.lossLimitedBandwidth = delayBasedEstimate + lossBasedResult = Result( + bandwidthEstimate = delayBasedEstimate, + state = LossBasedState.kDelayBasedEstimate + ) + } + + var bestCandidate = currentBestEstimate + var objectiveMax = -Double.MAX_VALUE + for (candidate in getCandidates(inAlr)) { + newtonsMethodUpdate(candidate) + + val candidateObjective = getObjective(candidate) + if (candidateObjective > objectiveMax) { + objectiveMax = candidateObjective + bestCandidate = candidate + } + } + + if (bestCandidate.lossLimitedBandwidth < currentBestEstimate.lossLimitedBandwidth) { + lastTimeEstimateReduced = lastSendTimeMostRecentObservation + } + + // Do not increase the estimate if the average loss is greater than current + // inherent loss. + if (averageReportedLossRatio > bestCandidate.inherentLoss && + config.notIncreaseIfInherentLossLessThanAverageLoss && + currentBestEstimate.lossLimitedBandwidth < bestCandidate.lossLimitedBandwidth + ) { + bestCandidate.lossLimitedBandwidth = currentBestEstimate.lossLimitedBandwidth + } + + if (isInLossLimitedState()) { + // Bound the estimate increase if: + // 1. The estimate has been increased for less than + // `delayed_increase_window` ago, and + // 2. The best candidate is greater than bandwidth_limit_in_current_window. + if (recoveringAfterLossTimestamp.isFinite() && + recoveringAfterLossTimestamp + config.delayedIncreaseWindow > lastSendTimeMostRecentObservation && + bestCandidate.lossLimitedBandwidth > bandwidthLimitInCurrentWindow + ) { + bestCandidate.lossLimitedBandwidth = bandwidthLimitInCurrentWindow + } + + val increasingWhenLossLimited = + isEstimateIncreasingWhenLossLimited( + oldEstimate = currentBestEstimate.lossLimitedBandwidth, + newEstimate = bestCandidate.lossLimitedBandwidth + ) + // Bound the best candidate by the acked bitrate. + if (increasingWhenLossLimited && isValid(acknowledgedBitrate)) { + val rampupFactor = if (isValid(lastHoldInfo.rate) && + acknowledgedBitrate!! < config.bandwidthRampupHoldThreshold * lastHoldInfo.rate + ) { + config.bandwidthRampupUpperBoundFactorInHold + } else { + config.bandwidthRampupUpperBoundFactor + } + bestCandidate.lossLimitedBandwidth = + max( + currentBestEstimate.lossLimitedBandwidth, + min( + bestCandidate.lossLimitedBandwidth, + rampupFactor * acknowledgedBitrate!! + ) + ) + // Increase current estimate by at least 1kbps to make sure that the state + // will be switched to kIncreasing, thus padding is triggered. + if (lossBasedResult.state == LossBasedState.kDecreasing && + bestCandidate.lossLimitedBandwidth == currentBestEstimate.lossLimitedBandwidth + ) { + bestCandidate.lossLimitedBandwidth = + currentBestEstimate.lossLimitedBandwidth + 1.bps + } + } + } + + var boundedBandwidthEstimate = Bandwidth.INFINITY + boundedBandwidthEstimate = if (isValid(delayBasedEstimate)) { + max( + getInstantLowerBound(), + minOf( + bestCandidate.lossLimitedBandwidth, + getInstantUpperBound(), + delayBasedEstimate + ) + ) + } else { + max( + getInstantLowerBound(), + min(bestCandidate.lossLimitedBandwidth, getInstantUpperBound()) + ) + } + if (config.boundBestCandidate && + boundedBandwidthEstimate < bestCandidate.lossLimitedBandwidth + ) { + // If network is lossy, cap the best estimate by the instant upper bound, + // e.g. 450kbps if loss rate is 50%. + // Otherwise, cap the estimate by the delay-based estimate or max_bitrate. + currentBestEstimate.lossLimitedBandwidth = boundedBandwidthEstimate + currentBestEstimate.inherentLoss = 0.0 + } else { + currentBestEstimate = bestCandidate + if (config.lowerBoundByAckedRateFactor > 0.0) { + currentBestEstimate.lossLimitedBandwidth = + max(currentBestEstimate.lossLimitedBandwidth, getInstantLowerBound()) + } + } + + if (lossBasedResult.state == LossBasedState.kDecreasing && + lastHoldInfo.timestamp > lastSendTimeMostRecentObservation && + boundedBandwidthEstimate < delayBasedEstimate + ) { + // Ensure that acked rate is the lower bound of HOLD rate + if (config.lowerBoundByAckedRateFactor > 0.0) { + lastHoldInfo.rate = + max(getInstantLowerBound(), lastHoldInfo.rate) + } + // BWE is not allowed to increase above the HOLD rate. The purpose of + // HOLD is to not immediately ramp up BWE to a rate that may cause loss. + lossBasedResult.bandwidthEstimate = + min(lastHoldInfo.rate, boundedBandwidthEstimate) + return + } + + if (isEstimateIncreasingWhenLossLimited( + oldEstimate = lossBasedResult.bandwidthEstimate, + newEstimate = boundedBandwidthEstimate + ) && + canKeepIncreasingState(boundedBandwidthEstimate) && + boundedBandwidthEstimate < delayBasedEstimate && + boundedBandwidthEstimate < maxBitrate + ) { + if (config.paddingDuration > Duration.ZERO && + boundedBandwidthEstimate > lastPaddingInfo.paddingRate + ) { + // Start a new padding duration. + lastPaddingInfo.paddingRate = boundedBandwidthEstimate + lastPaddingInfo.paddingTimestamp = lastSendTimeMostRecentObservation + } + lossBasedResult.state = if (config.paddingDuration > Duration.ZERO) { + LossBasedState.kIncreaseUsingPadding + } else { + LossBasedState.kIncreasing + } + } else if (boundedBandwidthEstimate < delayBasedEstimate && + boundedBandwidthEstimate < maxBitrate + ) { + if (lossBasedResult.state != LossBasedState.kDecreasing && + config.holdDurationFactor > 0 + ) { + logger.info { + "Switch to HOLD. Bounded BWE: $boundedBandwidthEstimate, duration: ${lastHoldInfo.duration}" + } + lastHoldInfo = HoldInfo( + timestamp = lastSendTimeMostRecentObservation + lastHoldInfo.duration, + duration = min(kMaxHoldDuration, lastHoldInfo.duration * config.holdDurationFactor), + rate = boundedBandwidthEstimate + ) + } + lossBasedResult.state = LossBasedState.kDecreasing + } else { + // Reset the HOLD info if delay based estimate works to avoid getting + // stuck in low bitrate. + lastHoldInfo = HoldInfo( + timestamp = Instant.MIN, + duration = kInitHoldDuration, + rate = Bandwidth.INFINITY + ) + lossBasedResult.state = LossBasedState.kDelayBasedEstimate + } + lossBasedResult.bandwidthEstimate = boundedBandwidthEstimate + + if (isInLossLimitedState() && + ( + recoveringAfterLossTimestamp.isInfinite() || + recoveringAfterLossTimestamp + config.delayedIncreaseWindow < + lastSendTimeMostRecentObservation + ) + ) { + bandwidthLimitInCurrentWindow = + max( + kCongestionControllerMinBitrate, + currentBestEstimate.lossLimitedBandwidth * config.maxIncreaseFactor + ) + recoveringAfterLossTimestamp = lastSendTimeMostRecentObservation + } + } + + fun getMedianSendingRate(): Bandwidth { + val sendingRates = mutableListOf() + for (observation in observations) { + if (!observation.isInitialized() || + !isValid(observation.sendingRate) || + observation.sendingRate == Bandwidth.ZERO + ) { + continue + } + sendingRates.add(observation.sendingRate) + } + if (sendingRates.isEmpty()) { + return Bandwidth.ZERO + } + sendingRates.sort() + if (sendingRates.size % 2 == 0) { + return ( + sendingRates[sendingRates.size / 2 - 1] + sendingRates[sendingRates.size / 2] + ) / 2 + } + return sendingRates[sendingRates.size / 2] + } + + // For unit testing only + fun setBandwidthEstimate(bandwidthEstimate: Bandwidth) { + if (isValid(bandwidthEstimate)) { + currentBestEstimate.lossLimitedBandwidth = bandwidthEstimate + lossBasedResult = Result( + bandwidthEstimate = bandwidthEstimate, + state = LossBasedState.kDelayBasedEstimate + ) + } else { + logger.warn("The bandwidth estimate must be finite: $bandwidthEstimate") + } + } + + private data class ChannelParameters( + var inherentLoss: Double = 0.0, + var lossLimitedBandwidth: Bandwidth = Bandwidth.MINUS_INFINITY + ) + + /** These are the parameters set by field trial parameters in libwebrtc. They are initialized to their + * default values. + */ + data class Config( + var enabled: Boolean = true, + val bandwidthRampupUpperBoundFactor: Double = 1.5, + val bandwidthRampupUpperBoundFactorInHold: Double = 1.2, + val bandwidthRampupHoldThreshold: Double = 1.3, + val rampupAccelerationMaxFactor: Double = 0.0, + val rampupAccelerationMaxoutTime: Duration = 60.secs, + val candidateFactors: DoubleArray = doubleArrayOf(1.02, 1.0, 0.95), + val higherBandwidthBiasFactor: Double = 0.0002, + val higherLogBandwidthBiasFactor: Double = 0.02, + val inherentLossLowerBound: Double = 1.0e-3, + val lossThresholdOfHighBandwidthPreference: Double = 0.2, + val bandwidthPreferenceSmoothingFactor: Double = 0.002, + val inherentLossUpperBoundBandwidthBalance: Bandwidth = 100.kbps, + val inherentLossUpperBoundOffset: Double = 0.05, + val initialInherentLossEstimate: Double = 0.01, + val newtonIterations: Int = 1, + val newtonStepSize: Double = 0.75, + val appendAcknowledgedRateCandidate: Boolean = true, + val appendDelayBasedEstimateCandidate: Boolean = true, + val appendUpperBoundCandidateInAlr: Boolean = false, + val observationDurationLowerBound: Duration = 250.ms, + val observationWindowSize: Int = 15, + val sendingRateSmoothingFactor: Double = 0.0, + val instantUpperBoundTemporalWeightFactor: Double = 0.9, + val instantUpperBoundBandwidthBalance: Bandwidth = 100.kbps, + val instantUpperBoundLossOffset: Double = 0.05, + val temporalWeightFactor: Double = 0.9, + val bandwidthBackoffLowerBoundFactor: Double = 1.0, + val maxIncreaseFactor: Double = 1.3, + val delayedIncreaseWindow: Duration = 300.ms, + val notIncreaseIfInherentLossLessThanAverageLoss: Boolean = true, + val notUseAckedRateInAlr: Boolean = true, + val useInStartPhase: Boolean = true, + val minNumObservations: Int = 3, + val lowerBoundByAckedRateFactor: Double = 1.0, + val holdDurationFactor: Double = 2.0, + val useByteLossRate: Boolean = true, + val paddingDuration: Duration = 2.secs, + val boundBestCandidate: Boolean = true, + val medianSendingRateFactor: Double = 2.0 + ) { + fun isValid(): Boolean { + if (!enabled) { + return false + } + + var valid = true + + if (bandwidthRampupUpperBoundFactor <= 1.0) { + logger.warn( + "The bandwidth rampup upper bound factor must be greater than 1: " + + bandwidthRampupUpperBoundFactor + ) + valid = false + } + if (bandwidthRampupUpperBoundFactorInHold <= 1.0) { + logger.warn( + "The bandwidth rampup upper bound factor in hold must be greater than 1: " + + bandwidthRampupUpperBoundFactorInHold + ) + valid = false + } + if (bandwidthRampupHoldThreshold < 0.0) { + logger.warn( + "The bandwidth rampup hold threshold must be non-negative.: " + + bandwidthRampupHoldThreshold + ) + valid = false + } + + if (rampupAccelerationMaxFactor < 0.0) { + logger.warn("The rampup acceleration max factor must be non-negative.: $rampupAccelerationMaxFactor") + valid = false + } + if (rampupAccelerationMaxoutTime <= Duration.ZERO) { + logger.warn("The rampup acceleration maxout time must be above zero: $rampupAccelerationMaxoutTime") + valid = false + } + for (candidateFactor in candidateFactors) { + if (candidateFactor <= 0.0) { + logger.warn("All candidate factors must be greater than zero: $candidateFactor") + valid = false + } + } + + // Ensure that the configuration allows generation of at least one candidate + // other than the current estimate. + if (!appendAcknowledgedRateCandidate && !appendDelayBasedEstimateCandidate && + !candidateFactors.any { it != 1.0 } + ) { + logger.warn( + "The configuration does not allow generating candidates. Specify " + + "a candidate factor other than 1.0, allow the acknowledged rate " + + "to be a candidate, and/or allow the delay based estimate to be a " + + "candidate." + ) + valid = false + } + if (higherBandwidthBiasFactor < 0.0) { + logger.warn("The higher bandwidth bias factor must be non-negative: $higherBandwidthBiasFactor") + valid = false + } + if (inherentLossLowerBound < 0.0 || inherentLossLowerBound >= 1.0) { + logger.warn("The inherent loss lower bound must be in [0, 1): $inherentLossLowerBound") + valid = false + } + if (lossThresholdOfHighBandwidthPreference < 0.0 || lossThresholdOfHighBandwidthPreference >= 1.0) { + logger.warn( + "The loss threshold of high bandwidth preference must be in [0, 1): " + + lossThresholdOfHighBandwidthPreference + ) + valid = false + } + if (bandwidthPreferenceSmoothingFactor <= 0.0 || bandwidthPreferenceSmoothingFactor > 1.0) { + logger.warn( + "The bandwidth preference smoothing factor must be in (0, 1]: " + + bandwidthPreferenceSmoothingFactor + ) + valid = false + } + if (inherentLossUpperBoundBandwidthBalance <= Bandwidth.ZERO) { + logger.warn( + "The inherent loss upper bound bandwidth balance must be positive: " + + inherentLossUpperBoundBandwidthBalance + ) + valid = false + } + if (inherentLossUpperBoundOffset < inherentLossLowerBound || inherentLossUpperBoundOffset >= 1.0) { + logger.warn( + "The inherent loss upper bound must be greater than or equal to the inherent loss " + + "lower bound, which is $inherentLossLowerBound, and less than 1: $inherentLossUpperBoundOffset" + ) + valid = false + } + if (initialInherentLossEstimate < 0.0 || initialInherentLossEstimate >= 1.0) { + logger.warn("The initial inherent loss estimate must be in [0, 1): $initialInherentLossEstimate") + valid = false + } + if (newtonIterations <= 0) { + logger.warn("The number of Newton iterations must be positive: $newtonIterations") + valid = false + } + if (newtonStepSize <= 0.0) { + logger.warn("The Newton step size must be positive: $newtonStepSize") + valid = false + } + if (observationDurationLowerBound <= Duration.ZERO) { + logger.warn("The observation duration lower bound must be positive: $observationDurationLowerBound") + valid = false + } + if (observationWindowSize < 2) { + logger.warn("The observation window size must be at least 2: $observationWindowSize") + valid = false + } + if (sendingRateSmoothingFactor < 0.0 || sendingRateSmoothingFactor >= 1.0) { + logger.warn("The sending rate smoothing factor must be in [0, 1): $sendingRateSmoothingFactor") + valid = false + } + if (instantUpperBoundTemporalWeightFactor <= 0.0 || instantUpperBoundTemporalWeightFactor > 1.0) { + logger.warn( + "The instant upper bound temporal weight factor must be in (0, 1]: " + + instantUpperBoundTemporalWeightFactor + ) + valid = false + } + if (instantUpperBoundBandwidthBalance <= Bandwidth.ZERO) { + logger.warn( + "The instant upper bound bandwidth balance must be positive: " + + instantUpperBoundBandwidthBalance + ) + valid = false + } + if (instantUpperBoundLossOffset < 0.0 || instantUpperBoundLossOffset >= 1.0) { + logger.warn("The instant upper bound loss offset must be in [0, 1): $instantUpperBoundLossOffset") + valid = false + } + if (temporalWeightFactor <= 0.0 || temporalWeightFactor > 1.0) { + logger.warn("The temporal weight factor must be in (0, 1]: $temporalWeightFactor") + valid = false + } + if (bandwidthBackoffLowerBoundFactor > 1.0) { + logger.warn( + "The bandwidth backoff lower bound factor must not be greater than 1: " + + bandwidthBackoffLowerBoundFactor + ) + valid = false + } + if (maxIncreaseFactor <= 0.0) { + logger.warn("The maximum increase factor must be positive: $maxIncreaseFactor") + valid = false + } + if (delayedIncreaseWindow <= Duration.ZERO) { + logger.warn("The delayed increase window must be positive: $delayedIncreaseWindow") + valid = false + } + if (minNumObservations <= 0) { + logger.warn("The min number of observations must be positive: $minNumObservations") + valid = false + } + if (lowerBoundByAckedRateFactor < 0.0) { + logger.warn( + "The estimate lower bound by acknowledged rate factor must be non-negative: " + + lowerBoundByAckedRateFactor + ) + valid = false + } + + return valid + } + } + + private data class Derivatives( + var first: Double = 0.0, + var second: Double = 0.0 + ) + + private class Observation { + var numPackets = 0 + var numLostPackets = 0 + var numReceivedPackets = 0 + var sendingRate = Bandwidth.MINUS_INFINITY + var size = DataSize.ZERO + var lostSize = DataSize.ZERO + var id = -1 + + fun isInitialized() = id != -1 + } + + private class PartialObservation { + var numPackets = 0 + val lostPackets = HashMap() + var size = DataSize.ZERO + } + + private class PaddingInfo { + var paddingRate = Bandwidth.MINUS_INFINITY + var paddingTimestamp = Instant.MIN + } + + private class HoldInfo( + var timestamp: Instant = Instant.MIN, + var duration: Duration = Duration.ZERO, + var rate: Bandwidth = Bandwidth.INFINITY + ) + + /** Returns `0.0` if not enough loss statistics have been received. */ + private fun updateAverageReportedLossRatio() { + averageReportedLossRatio = if (config.useByteLossRate) { + calculateAverageReportedByteLossRatio() + } else { + calculateAverageReportedPacketLossRatio() + } + } + + private fun calculateAverageReportedPacketLossRatio(): Double { + if (numObservations <= 0) { + return 0.0 + } + + var numPackets = 0.0 + var numLostPackets = 0.0 + + for (observation in observations) { + if (!observation.isInitialized()) { + continue + } + + val instantTemporalWeight = + instantUpperBoundTemporalWeights[ + (numObservations - 1) - + observation.id + ] + numPackets += instantTemporalWeight * observation.numPackets + numLostPackets += instantTemporalWeight * observation.numLostPackets + } + + return numLostPackets / numPackets + } + + // Calculates the average loss ratio over the last `observation_window_size` + // observations but skips the observation with min and max loss ratio in order + // to filter out loss spikes. + private fun calculateAverageReportedByteLossRatio(): Double { + if (numObservations <= 0) { + return 0.0 + } + + var totalBytes = DataSize.ZERO + var lostBytes = DataSize.ZERO + var minLossRate = 1.0 + var maxLossRate = 0.0 + var minLostBytes = DataSize.ZERO + var maxLostBytes = DataSize.ZERO + var minBytesReceived = DataSize.ZERO + var maxBytesReceived = DataSize.ZERO + + var sendRateOfMaxLossObservation = Bandwidth.ZERO + for (observation in observations) { + if (!observation.isInitialized()) { + continue + } + + val instantTemporalWeight = + instantUpperBoundTemporalWeights[(numObservations - 1) - observation.id] + totalBytes += observation.size * instantTemporalWeight + lostBytes += observation.lostSize * instantTemporalWeight + + val lossRate = if (observation.size != DataSize.ZERO) { + observation.lostSize / observation.size + } else { + 0.0 + } + if (numObservations > 3) { + if (lossRate > maxLossRate) { + maxLossRate = lossRate + maxLostBytes = instantTemporalWeight * observation.lostSize + maxBytesReceived = instantTemporalWeight * observation.size + sendRateOfMaxLossObservation = observation.sendingRate + } + if (lossRate < minLossRate) { + minLossRate = lossRate + minLostBytes = instantTemporalWeight * observation.lostSize + minBytesReceived = instantTemporalWeight * observation.size + } + } + } + if (getMedianSendingRate() * config.medianSendingRateFactor <= + sendRateOfMaxLossObservation + ) { + // If the median sending rate is less than half of the sending rate of the + // observation with max loss rate, i.e. we suddenly send a lot of data, then + // the loss rate might not be due to a spike. + return lostBytes / totalBytes + } + + if (totalBytes == maxBytesReceived + minBytesReceived) { + // It could happen if the observation window was 2. + return lostBytes / totalBytes + } + + return (lostBytes - minLostBytes - maxLostBytes) / + (totalBytes - maxBytesReceived - minBytesReceived) + } + + private fun getCandidateBandwidthUpperBound(): Bandwidth { + var candidateBandwidthUpperBound = maxBitrate + if (isInLossLimitedState() && isValid(bandwidthLimitInCurrentWindow)) { + candidateBandwidthUpperBound = bandwidthLimitInCurrentWindow + } + + if (acknowledgedBitrate == null) { + return candidateBandwidthUpperBound + } + + if (config.rampupAccelerationMaxFactor > 0.0) { + val timeSinceBandwidthReduced = min( + config.rampupAccelerationMaxoutTime, + max(Duration.ZERO, Duration.between(lastTimeEstimateReduced, lastSendTimeMostRecentObservation)) + ) + val rampupAcceleration = config.rampupAccelerationMaxFactor * + (timeSinceBandwidthReduced / config.rampupAccelerationMaxoutTime) + + candidateBandwidthUpperBound += rampupAcceleration * acknowledgedBitrate!! + } + + return candidateBandwidthUpperBound + } + + private fun getCandidates(inAlr: Boolean): List { + val bestEstimate = currentBestEstimate.copy() + val bandwidths = mutableListOf() + for (candidateFactor in config.candidateFactors) { + bandwidths.add(candidateFactor * currentBestEstimate.lossLimitedBandwidth) + } + + if (acknowledgedBitrate != null && + config.appendAcknowledgedRateCandidate + ) { + if (!(config.notUseAckedRateInAlr && inAlr) || + ( + config.paddingDuration > Duration.ZERO && + lastPaddingInfo.paddingTimestamp + config.paddingDuration >= lastSendTimeMostRecentObservation + ) + ) { + bandwidths.add(acknowledgedBitrate!! * config.bandwidthBackoffLowerBoundFactor) + } + } + + if (isValid(delayBasedEstimate) && config.appendDelayBasedEstimateCandidate) { + if (delayBasedEstimate > bestEstimate.lossLimitedBandwidth) { + bandwidths.add(delayBasedEstimate) + } + } + + if (inAlr && config.appendUpperBoundCandidateInAlr && + bestEstimate.lossLimitedBandwidth > getInstantUpperBound() + ) { + bandwidths.add(getInstantUpperBound()) + } + + val candidateBandwidthUpperBound = getCandidateBandwidthUpperBound() + + val candidates = mutableListOf() + for (i in bandwidths.indices) { + val candidate = bestEstimate.copy() + candidate.lossLimitedBandwidth = min( + bandwidths[i], + max(bestEstimate.lossLimitedBandwidth, candidateBandwidthUpperBound) + ) + candidate.inherentLoss = getFeasibleInherentLoss(candidate) + candidates.add(candidate) + } + check(candidates.size == bandwidths.size) + return candidates + } + + private fun getDerivatives(channelParameters: ChannelParameters): Derivatives { + val derivatives = Derivatives() + + for (observation in observations) { + if (!observation.isInitialized()) { + continue + } + + val lossProbability = getLossProbability( + channelParameters.inherentLoss, + channelParameters.lossLimitedBandwidth, + observation.sendingRate + ) + + val temporalWeight = temporalWeights[(numObservations - 1) - observation.id] + + if (config.useByteLossRate) { + derivatives.first += + temporalWeight * + ( + ( + observation.lostSize.kiloBytes / lossProbability - + ((observation.size - observation.lostSize).kiloBytes) / (1.0 - lossProbability) + ) + ) + derivatives.second -= + temporalWeight * + ( + ( + observation.lostSize.kiloBytes / lossProbability.pow(2) + + (observation.size - observation.lostSize).kiloBytes / (1.0 - lossProbability).pow(2) + ) + ) + } else { + derivatives.first += + temporalWeight * + ( + (observation.numLostPackets / lossProbability) - + (observation.numReceivedPackets / (1.0 - lossProbability)) + ) + derivatives.second -= + temporalWeight * + ( + (observation.numLostPackets / lossProbability.pow(2)) + + ( + observation.numReceivedPackets / + (1.0 - lossProbability).pow(2) + ) + ) + } + } + + if (derivatives.second >= 0.0) { + logger.error( + "The second derivative is mathematically guaranteed " + + "to be negative but is ${derivatives.second}." + ) + derivatives.second = -1.0e6 + } + + return derivatives + } + + private fun getFeasibleInherentLoss(channelParameters: ChannelParameters): Double = min( + max(channelParameters.inherentLoss, config.inherentLossLowerBound), + getInherentLossUpperBound(channelParameters.lossLimitedBandwidth) + ) + + private fun getInherentLossUpperBound(bandwidth: Bandwidth): Double { + if (bandwidth == Bandwidth.ZERO) { + return 1.0 + } + + val inherentLossUpperBound = + config.inherentLossUpperBoundOffset + + config.inherentLossUpperBoundBandwidthBalance / bandwidth + + return min(inherentLossUpperBound, 1.0) + } + + private fun adjustBiasFactor(lossRate: Double, biasFactor: Double): Double = biasFactor * + (config.lossThresholdOfHighBandwidthPreference - lossRate) / + ( + config.bandwidthPreferenceSmoothingFactor + + abs(config.lossThresholdOfHighBandwidthPreference - lossRate) + ) + + private fun getHighBandwidthBias(bandwidth: Bandwidth): Double { + if (isValid(bandwidth)) { + return adjustBiasFactor(averageReportedLossRatio, config.higherBandwidthBiasFactor) * + bandwidth.kbps + + adjustBiasFactor(averageReportedLossRatio, config.higherLogBandwidthBiasFactor) * + ln(1.0 + bandwidth.kbps) + } + return 0.0 + } + + private fun getObjective(channelParameters: ChannelParameters): Double { + var objective = 0.0 + + val highBandwidthBias = getHighBandwidthBias(channelParameters.lossLimitedBandwidth) + + for (observation in observations) { + if (!observation.isInitialized()) { + continue + } + + val lossProbability = getLossProbability( + channelParameters.inherentLoss, + channelParameters.lossLimitedBandwidth, + observation.sendingRate + ) + + val temporalWeight = temporalWeights[(numObservations - 1) - observation.id] + + if (config.useByteLossRate) { + objective += + temporalWeight * + ( + (observation.lostSize.kiloBytes * ln(lossProbability)) + + ( + (observation.size - observation.lostSize).kiloBytes * ln(1.0 - lossProbability) + ) + ) + objective += + temporalWeight * highBandwidthBias * observation.size.kiloBytes + } else { + objective += + temporalWeight * + ( + (observation.numLostPackets * ln(lossProbability)) + + (observation.numReceivedPackets * ln(1.0 - lossProbability)) + ) + objective += + temporalWeight * highBandwidthBias * observation.numPackets + } + } + + return objective + } + + private fun getSendingRate(instantaneousSendingRate: Bandwidth): Bandwidth { + if (numObservations <= 0) { + return instantaneousSendingRate + } + + val mostRecentObservationIdx = + (numObservations - 1) % config.observationWindowSize + val mostRecentObservation = observations[mostRecentObservationIdx] + val sendingRatePreviousObservation = + mostRecentObservation.sendingRate + + return config.sendingRateSmoothingFactor * sendingRatePreviousObservation + + (1.0 - config.sendingRateSmoothingFactor) * instantaneousSendingRate + } + + private fun getInstantUpperBound(): Bandwidth = cachedInstantUpperBound ?: maxBitrate + + private fun calculateInstantUpperBound() { + var instantLimit = maxBitrate + if (averageReportedLossRatio > config.instantUpperBoundLossOffset) { + instantLimit = config.instantUpperBoundBandwidthBalance / + ( + averageReportedLossRatio - + config.instantUpperBoundLossOffset + ) + } + + cachedInstantUpperBound = instantLimit + } + + private fun getInstantLowerBound(): Bandwidth = cachedInstantLowerBound ?: Bandwidth.ZERO + + private fun calculateInstantLowerBound() { + var instanceLowerBound = Bandwidth.ZERO + if (isValid(acknowledgedBitrate) && + config.lowerBoundByAckedRateFactor > 0.0 + ) { + instanceLowerBound = config.lowerBoundByAckedRateFactor * acknowledgedBitrate!! + } + if (isValid(minBitrate)) { + instanceLowerBound = max(instanceLowerBound, minBitrate) + } + cachedInstantLowerBound = instanceLowerBound + } + + private fun calculateTemporalWeights() { + for (i in 0 until config.observationWindowSize) { + temporalWeights[i] = config.temporalWeightFactor.pow(i) + instantUpperBoundTemporalWeights[i] = config.instantUpperBoundTemporalWeightFactor.pow(i) + } + } + + private fun newtonsMethodUpdate(channelParameters: ChannelParameters) { + if (numObservations <= 0) { + return + } + + for (i in 0 until config.newtonIterations) { + val derivatives = getDerivatives(channelParameters) + channelParameters.inherentLoss -= + config.newtonStepSize * derivatives.first / derivatives.second + channelParameters.inherentLoss = + getFeasibleInherentLoss(channelParameters) + } + } + + /** Returns false if no observation was created. */ + private fun pushBackObservation(packetResults: List): Boolean { + if (packetResults.isEmpty()) { + return false + } + + partialObservation.numPackets += packetResults.size + var lastSendTime = Instant.MIN + var firstSendTime = Instant.MAX + for (packet in packetResults) { + if (packet.isReceived()) { + partialObservation.lostPackets.remove(packet.sentPacket.sequenceNumber) + } else { + partialObservation.lostPackets[packet.sentPacket.sequenceNumber] = packet.sentPacket.size + } + partialObservation.size += packet.sentPacket.size + lastSendTime = max(lastSendTime, packet.sentPacket.sendTime) + firstSendTime = min(firstSendTime, packet.sentPacket.sendTime) + } + + // This is the first packet report we have received. + if (!isValid(lastSendTimeMostRecentObservation)) { + lastSendTimeMostRecentObservation = firstSendTime + } + + val observationDuration = Duration.between(lastSendTimeMostRecentObservation, lastSendTime) + // Too small to be meaningful. + // To consider: what if it is too long?, i.e. we did not receive any packets + // for a long time, then all the packets we received are too old. + if (observationDuration <= Duration.ZERO || + observationDuration < config.observationDurationLowerBound + ) { + return false + } + + lastSendTimeMostRecentObservation = lastSendTime + + val observation = Observation() + observation.numPackets = partialObservation.numPackets + observation.numLostPackets = partialObservation.lostPackets.size + observation.numReceivedPackets = + observation.numPackets - observation.numLostPackets + observation.sendingRate = + getSendingRate(partialObservation.size.per(observationDuration)) + for ((_, packetSize) in partialObservation.lostPackets) { + observation.lostSize += packetSize + } + observation.size = partialObservation.size + observation.id = numObservations++ + observations[observation.id % config.observationWindowSize] = + observation + + partialObservation = PartialObservation() + + updateAverageReportedLossRatio() + calculateInstantUpperBound() + return true + } + + private fun isEstimateIncreasingWhenLossLimited(oldEstimate: Bandwidth, newEstimate: Bandwidth): Boolean = ( + oldEstimate < newEstimate || + ( + oldEstimate == newEstimate && + ( + lossBasedResult.state == LossBasedState.kIncreasing || + lossBasedResult.state == LossBasedState.kIncreaseUsingPadding + ) + ) + ) && + isInLossLimitedState() + + private fun isInLossLimitedState(): Boolean = lossBasedResult.state != LossBasedState.kDelayBasedEstimate + + private fun canKeepIncreasingState(estimate: Bandwidth): Boolean { + if (config.paddingDuration == Duration.ZERO || + lossBasedResult.state != LossBasedState.kIncreaseUsingPadding + ) { + return true + } + + // Keep using the kIncreaseUsingPadding if either the state has been + // kIncreaseUsingPadding for less than kPaddingDuration or the estimate + // increases. + return lastPaddingInfo.paddingTimestamp + config.paddingDuration >= + lastSendTimeMostRecentObservation || + lastPaddingInfo.paddingRate < estimate + } + + companion object { + private val defaultConfig = Config() + + val logger = createLogger() + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkController.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkController.kt new file mode 100644 index 0000000..29b0785 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkController.kt @@ -0,0 +1,102 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import java.time.Duration + +/* Base type for network controller, + * based on WebRTC api/transport/network_control.h in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +/** Configuration sent to factory create function. The parameters here are + * optional to use for a network controller implementation. */ +class NetworkControllerConfig( + val parentLogger: Logger, + val diagnosticContext: DiagnosticContext, + + /** The initial constraints to start with, these can be changed at any later + * time by calls to OnTargetRateConstraints. Note that the starting rate + * has to be set initially to provide a starting state for the network + * controller, even though the field is marked as nullable. + */ + val constraints: TargetRateConstraints = TargetRateConstraints(), + + /** Initial stream specific configuration, these are changed at any later time + * by calls to OnStreamsConfig. + */ + val streamBasedConfig: StreamsConfig = StreamsConfig() +) + +/** NetworkControllerInterface is implemented by network controllers. A network + * controller is a class that uses information about network state and traffic + * to estimate network parameters such as round trip time and bandwidth. Network + * controllers does not guarantee thread safety, the interface must be used in a + * non-concurrent fashion. + */ +interface NetworkControllerInterface { + /** Called when network availabilty changes */ + fun onNetworkAvailability(msg: NetworkAvailability): NetworkControlUpdate + + /** Called when the receiving or sending endpoint changes address.*/ + fun onNetworkRouteChange(msg: NetworkRouteChange): NetworkControlUpdate + + /** Called periodically with a periodicy as specified by + * [NetworkControllerFactoryInterface.getProcessInterval]. + */ + fun onProcessInterval(msg: ProcessInterval): NetworkControlUpdate + + /** Called when remotely calculated bitrate is received. */ + fun onRemoteBitrateReport(msg: RemoteBitrateReport): NetworkControlUpdate + + /** Called round trip time has been calculated by protocol specific mechanisms. */ + fun onRoundTripTimeUpdate(msg: RoundTripTimeUpdate): NetworkControlUpdate + + /** Called when a packet is sent on the network. */ + fun onSentPacket(sentPacket: SentPacket): NetworkControlUpdate + + // Omitted: onReceivedPacket: Not used + + /** Called when the stream specific configuration has been updated. */ + fun onStreamsConfig(msg: StreamsConfig): NetworkControlUpdate + + /** Called when target transfer rate constraints has been changed. */ + fun onTargetRateConstraints(constraints: TargetRateConstraints): NetworkControlUpdate + + /** Called when a protocol specific calculation of packet loss has been made. */ + fun onTransportLossReport(msg: TransportLossReport): NetworkControlUpdate + + /** Called with per packet feedback regarding receive time. */ + fun onTransportPacketsFeedback(report: TransportPacketsFeedback): NetworkControlUpdate + + // Omitted: onNetworkStateUpdate: Not used +} + +/** NetworkControllerFactoryInterface is an interface for creating a network + * controller. */ +interface NetworkControllerFactoryInterface { + /** Used to create a new network controller, requires an observer to be + * provided to handle callbacks. + */ + fun create(config: NetworkControllerConfig): NetworkControllerInterface + + /** Returns the interval by which the network controller expects + * OnProcessInterval calls. + */ + fun getProcessInterval(): Duration +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkTypes.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkTypes.kt new file mode 100644 index 0000000..20a55bf --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/NetworkTypes.kt @@ -0,0 +1,322 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.per +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.EcnMarking +import org.jitsi.utils.MAX_DURATION +import org.jitsi.utils.NEVER +import org.jitsi.utils.isFinite +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.ms +import org.jitsi.utils.toDoubleMillis +import java.time.Duration +import java.time.Instant + +/* Common network types used for bandwidth estimation, + * based on WebRTC api/transport/network_types.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +/* Configuration */ +/** Represents constraints and rates related to the currently enabled streams. + * This is used as input to the congestion controller via the StreamsConfig + * struct. + */ +class BitrateAllocationLimits( + /** The total minimum send bitrate required by all sending streams. */ + val minAllocatableRate: Bandwidth = Bandwidth.ZERO, + /** The total maximum allocatable bitrate for all currently available streams. */ + val maxAllocatableRate: Bandwidth = Bandwidth.ZERO, + /** The max bitrate to use for padding. The sum of the per-stream max padding + * rate. */ + val maxPaddingRate: Bandwidth = Bandwidth.ZERO +) + +/** + * Use StreamsConfig for information about streams that is required for specific + * adjustments to the algorithms in network controllers. Especially useful + * for experiments. + */ +class StreamsConfig( + val atTime: Instant = Instant.MAX, + val requestsAlrProbing: Boolean? = null, + // If `enable_repeated_initial_probing` is set to true, Probes are sent + // periodically every 1s during the first 5s after the network becomes + // available. The probes ignores max_total_allocated_bitrate. + val enableRepeatedInitialProbing: Boolean? = null, + + val pacingFactor: Double? = null, + + // TODO(srte): Use BitrateAllocationLimits here. + val minTotalAllocatedBitrate: Bandwidth? = null, + val maxPaddingRate: Bandwidth? = null, + val maxTotalAllocatedBitrate: Bandwidth? = null +) + +class TargetRateConstraints( + val atTime: Instant = Instant.MAX, + val minDataRate: Bandwidth? = null, + val maxDataRate: Bandwidth? = null, + // The initial bandwidth estimate to base target rate on. This should be used + // as the basis for initial OnTargetTransferRate and OnPacerConfig callbacks. + var startingRate: Bandwidth? = null +) + +/** Send side information */ + +class NetworkAvailability( + val atTime: Instant = Instant.MAX, + val networkAvailable: Boolean = false +) + +class NetworkRouteChange( + val atTime: Instant = Instant.MAX, + // The TargetRateConstraints are set here so they can be changed synchronously + // when network route changes. + val constraints: TargetRateConstraints = TargetRateConstraints() +) + +/** + * Information about a paced packet + */ +data class PacedPacketInfo( + val probeClusterId: Int = kNotAProbe, + val probeClusterMinProbes: Int = -1, + val probeClusterMinBytes: Int = -1, + var sendBitrate: Bandwidth = 0.bps +) { + // TODO(srte): Move probing info to a separate, optional struct. + var probeClusterBytesSent = 0 + + companion object { + const val kNotAProbe = -1 + } +} + +/** + * A sent packet + */ +data class SentPacket( + var sendTime: Instant = NEVER, + /** Size of packet with overhead up to IP layer. */ + var size: DataSize = DataSize.ZERO, + /** Size of preceeding packets that are not part of feedback */ + var priorUnackedData: DataSize = DataSize.ZERO, + /** Probe cluster id and parameters including bitrate, number of packets and + number of bytes. */ + var pacingInfo: PacedPacketInfo = PacedPacketInfo(), + /** True if the packet is an audio packet, false for video, padding, RTX, etc. */ + var audio: Boolean = false, + /** Transport independent sequence number, any tracked packet should have a + sequence number that is unique over the whole call and increasing by 1 for + each packet. */ + var sequenceNumber: Long = 0, + /** Tracked data in flight when the packet was sent, excluding unacked data. */ + var dataInFlight: DataSize = DataSize.ZERO +) + +/** Transport level feedback */ + +class RemoteBitrateReport( + val receiveTime: Instant = Instant.MAX, + val bandwidth: Bandwidth = Bandwidth.INFINITY +) + +class RoundTripTimeUpdate( + val receiveTime: Instant = Instant.MAX, + val roundTripTime: Duration = MAX_DURATION, + val smoothed: Boolean = false +) + +class TransportLossReport( + val receiveTime: Instant = Instant.MAX, + val startTime: Instant = Instant.MAX, + val endTime: Instant = Instant.MAX, + val packetsLostDelta: Long = 0, + val packetsReceivedDelta: Long = 0 +) + +/** + * The result of packet feedback + */ +class PacketResult( + var sentPacket: SentPacket = SentPacket(), + var receiveTime: Instant = NEVER, + var ecn: EcnMarking = EcnMarking.kNotEct, + // `rtp_packet_info` is only set if the feedback is related to a RTP packet. + var rtpPacketInfo: RtpPacketInfo? = null, + // Jitsi extension: Whether a packet was previously reported lost + var previouslyReportedLost: Boolean = false +) { + data class RtpPacketInfo( + val ssrc: Long = 0, + val rtpSequenceNumber: Int = 0, + val isRetransmission: Boolean = false + ) + + fun isReceived() = receiveTime.isFinite() + + fun copy(): PacketResult = PacketResult(sentPacket.copy(), receiveTime, ecn, rtpPacketInfo, previouslyReportedLost) +} + +/** + * Summary of transport packets feedback + */ +class TransportPacketsFeedback { + var feedbackTime: Instant = NEVER + var dataInFlight: DataSize = DataSize.ZERO + var packetFeedbacks: MutableList = ArrayList() + var transportSupportsEcn: Boolean = false + + /** Arrival times for messages without send times information */ + val sendlessArrivalTimes = ArrayList() + + fun receivedWithSendInfo(): List = packetFeedbacks.filter { it.isReceived() } + + fun lostWithSendInfo(): List = packetFeedbacks.filterNot { it.isReceived() } + + fun packetsWithFeedback(): List = packetFeedbacks + + fun sortedByReceiveTime(): List = receivedWithSendInfo().sortedBy { it.receiveTime } +} + +// Network estimation +class NetworkEstimate { + var atTime = Instant.MAX + + // Deprecated, use TargetTransferRate::target_rate instead. + var bandwidth = Bandwidth.INFINITY + var roundTripTime = MAX_DURATION + var bwePeriod = MAX_DURATION + + var lossRateRatio = 0.0f + + /* Jitsi local */ + override fun toString(): String = "atTime $atTime: " + + "bandwidth $bandwidth, rtt $roundTripTime, bwePeriod $bwePeriod, lossRateRatio $lossRateRatio" +} + +class PacerConfig { + var atTime = Instant.MAX + + // Pacer should send at most data_window data over time_window duration. + var dataWindow = DataSize.INFINITY + var timeWindow = MAX_DURATION + + // Pacer should send at least pad_window data over time_window duration. + var padWindow = DataSize.ZERO + + fun dataRate() = dataWindow.per(timeWindow) + fun padRate() = padWindow.per(timeWindow) + + /* Jitsi Local */ + override fun toString(): String = "Data rate ${dataRate()} ($dataWindow per $timeWindow), " + + "pad rate ${padRate()} ($padWindow per $timeWindow)" +} + +data class ProbeClusterConfig( + var atTime: Instant = Instant.MAX, + var targetDataRate: Bandwidth = Bandwidth.ZERO, + + // Duration of a probe. + var targetDuration: Duration = Duration.ZERO, + + // Delta time between sent bursts of packets during probe. + var minProbeDelta: Duration = 2.ms, + var targetProbeCount: Int = 0, + var id: Int = 0 +) { + /* Jitsi local */ + override fun toString(): String = "atTime $atTime: ID=$id: DataRate $targetDataRate Duration $targetDuration " + + "ProbeDelta $minProbeDelta ProbeCount $targetProbeCount" +} + +class TargetTransferRate { + var atTime = Instant.MAX + + // The estimate on which the target rate is based on. + var networkEstimate = NetworkEstimate() + var targetRate = Bandwidth.ZERO + var stableTargetRate = Bandwidth.ZERO + var cwndReduceRatio = 0.0 + + /* Jitsi local */ + override fun toString(): String = "atTime $atTime: networkEstimate {$networkEstimate}, " + + "targetRate $targetRate, stableTargetRate $stableTargetRate, cwndReduceRatio $cwndReduceRatio" +} + +// Contains updates of network controller comand state. Using nullables to +// indicate whether a member has been updated. The array of probe clusters +// should be used to send out probes if not empty. +open class NetworkControlUpdate( + open val congestionWindow: DataSize? = null, + open val pacerConfig: PacerConfig? = null, + open val probeClusterConfigs: List = listOf(), + open val targetRate: TargetTransferRate? = null +) { + /* Jitsi local */ + fun isEmpty(): Boolean = + congestionWindow == null && pacerConfig == null && probeClusterConfigs.isEmpty() && targetRate == null + + fun isNotEmpty() = !isEmpty() + + val atTime: Instant? + get() = targetRate?.atTime ?: probeClusterConfigs.firstOrNull()?.atTime ?: pacerConfig?.atTime + + fun addToTimeSeriesPoint(point: DiagnosticContext.TimeSeriesPoint) { + targetRate?.let { + point.addField("target_rate_bps", it.targetRate.bps) + point.addField("stable_target_rate_bps", it.stableTargetRate.bps) + point.addField("cwnd_reduce_ratio", it.cwndReduceRatio) + point.addField("rtt_ms", it.networkEstimate.roundTripTime.toDoubleMillis()) + point.addField("bwe_period_ms", it.networkEstimate.bwePeriod.toDoubleMillis()) + point.addField("loss_rate_ratio", it.networkEstimate.lossRateRatio) + } + congestionWindow?.let { + point.addField("congestion_window_bytes", it.bytes) + } + pacerConfig?.let { + point.addField("pacer_data_rate_bps", it.dataRate().bps) + point.addField("pacer_pad_rate_bps", it.padRate().bps) + } + probeClusterConfigs.forEachIndexed { i, it -> + point.addField("probe_cluster_" + i + "_id", it.id) + point.addField("probe_cluster_" + i + "_data_rate_bps", it.targetDataRate.bps) + point.addField("probe_cluster_" + i + "_duration_ms", it.targetDuration.toDoubleMillis()) + point.addField("probe_cluster_" + i + "_delta_ms", it.minProbeDelta.toDoubleMillis()) + point.addField("probe_cluster_" + i + "_count", it.targetProbeCount) + } + } +} + +class MutableNetworkControlUpdate( + override var congestionWindow: DataSize? = null, + override var pacerConfig: PacerConfig? = null, + override var probeClusterConfigs: MutableList = mutableListOf(), + override var targetRate: TargetTransferRate? = null +) : NetworkControlUpdate() + +/** Process control */ +class ProcessInterval( + val atTime: Instant = Instant.MAX, + val pacerQueue: DataSize? = null +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimator.kt new file mode 100644 index 0000000..e13f09e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimator.kt @@ -0,0 +1,204 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.min +import org.jitsi.nlj.util.per +import org.jitsi.nlj.util.times +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant +import java.util.* + +/** + * Basic implementation to estimate bitrate of probes. + * + * Based on WebRTC modules/congestion_controller/goog_cc/probe_bitrate_estimator.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class ProbeBitrateEstimator( + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) { + // TODO: pass parent logger in so we have log contexts + private val logger = createChildLogger(parentLogger) + + private val clusters = TreeMap() + + private var estimatedDataRate: Bandwidth? = null + + fun handleProbeAndEstimateBitrate(packetFeedback: PacketResult): Bandwidth? { + val clusterId = packetFeedback.sentPacket.pacingInfo.probeClusterId + check(clusterId != PacedPacketInfo.kNotAProbe) + + eraseOldClusters(packetFeedback.receiveTime) + + val cluster = clusters.getOrPut(clusterId) { AggregatedCluster() } + + if (packetFeedback.sentPacket.sendTime < cluster.firstSend) { + cluster.firstSend = packetFeedback.sentPacket.sendTime + } + if (packetFeedback.sentPacket.sendTime > cluster.lastSend) { + cluster.lastSend = packetFeedback.sentPacket.sendTime + cluster.sizeLastSend = packetFeedback.sentPacket.size + } + if (packetFeedback.receiveTime < cluster.firstReceive) { + cluster.firstReceive = packetFeedback.receiveTime + cluster.sizeFirstReceive = packetFeedback.sentPacket.size + } + if (packetFeedback.receiveTime > cluster.lastReceive) { + cluster.lastReceive = packetFeedback.receiveTime + } + cluster.sizeTotal += packetFeedback.sentPacket.size + cluster.numProbes += 1 + + check(packetFeedback.sentPacket.pacingInfo.probeClusterMinProbes > 0) + check(packetFeedback.sentPacket.pacingInfo.probeClusterMinBytes > 0) + + val minProbes = packetFeedback.sentPacket.pacingInfo.probeClusterMinProbes * + kMinReceivedProbesRatio + val minSize = packetFeedback.sentPacket.pacingInfo.probeClusterMinBytes.bytes * + kMinReceivedBytesRatio + if (cluster.numProbes < minProbes || cluster.sizeTotal < minSize) { + return null + } + + val sendInterval = Duration.between(cluster.firstSend, cluster.lastSend) + val receiveInterval = Duration.between(cluster.firstReceive, cluster.lastReceive) + if (sendInterval <= Duration.ZERO || sendInterval > kMaxProbeInterval || + receiveInterval <= Duration.ZERO || + receiveInterval > kMaxProbeInterval + ) { + logger.info( + "Probing unsuccessful, invalid send/receive interval " + + "[cluster id: $clusterId] [sendInterval: $sendInterval] " + + "[receive interval: $receiveInterval]" + ) + return null + } + // Since the `send_interval` does not include the time it takes to actually + // send the last packet the size of the last sent packet should not be + // included when calculating the send bitrate. + check(cluster.sizeTotal >= cluster.sizeLastSend) + val sendSize = cluster.sizeTotal - cluster.sizeLastSend + val sendRate = sendSize.per(sendInterval) + + // Since the `receive_interval` does not include the time it takes to + // actually receive the first packet the size of the first received packet + // should not be included when calculating the receive bitrate. + check(cluster.sizeTotal >= cluster.sizeFirstReceive) + val receiveSize = cluster.sizeTotal - cluster.sizeFirstReceive + val receiveRate = receiveSize.per(receiveInterval) + + val ratio = receiveRate / sendRate + if (ratio > kMaxValidRatio) { + logger.info( + "Probing unsuccessful, receive/send ratio too high " + + "[cluster id: $clusterId] [send: $sendSize/$sendInterval = $sendRate] " + + "[receive: $receiveSize/$receiveInterval = $receiveRate] " + + "[ratio: $receiveRate / $sendRate = $ratio > kMaxValidRatio ($kMaxValidRatio)]" + ) + return null + } + logger.info( + "Probing successful [cluster id: $clusterId] " + + "[send: $sendSize / $sendInterval = $sendRate]" + + "[receive: $receiveSize / $receiveInterval = $receiveRate]" + ) + + var res = min(sendRate, receiveRate) + // If we're receiving at significantly lower bitrate than we were sending at, + // it suggests that we've found the true capacity of the link. In this case, + // set the target bitrate slightly lower to not immediately overuse. + if (receiveRate < kMinRatioForUnsaturatedLink * sendRate) { + check(sendRate > receiveRate) + res = kTargetUtilizationFraction * receiveRate + } + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("probe_result_success") + .addField("id", clusterId) + .addField("bitrate_bps", res.bps) + } + estimatedDataRate = res + return estimatedDataRate + } + + fun fetchAndResetLastEstimatedBitrate(): Bandwidth? { + val estimatedDataRate = this.estimatedDataRate + this.estimatedDataRate = null + return estimatedDataRate + } + + private fun eraseOldClusters(timestamp: Instant) { + clusters.entries.removeIf { + it.value.lastReceive + kMaxClusterHistory < timestamp + } + } + + private data class AggregatedCluster( + var numProbes: Int = 0, + var firstSend: Instant = Instant.MAX, + var lastSend: Instant = Instant.MIN, + var firstReceive: Instant = Instant.MAX, + var lastReceive: Instant = Instant.MIN, + var sizeLastSend: DataSize = DataSize.ZERO, + var sizeFirstReceive: DataSize = DataSize.ZERO, + var sizeTotal: DataSize = DataSize.ZERO + ) + + companion object { + /** The minumum number of probes we need to receive feedback about in percent + * in order to have a valid estimate. */ + const val kMinReceivedProbesRatio = 0.80 + + /** The minumum number of bytes we need to receive feedback about in percent + * in order to have a valid estimate. */ + const val kMinReceivedBytesRatio = 0.80 + + /** The maximum |receive rate| / |send rate| ratio for a valid estimate. */ + const val kMaxValidRatio = 2.0f + + /** The minimum |receive rate| / |send rate| ratio assuming that the link is + * not saturated, i.e. we assume that we will receive at least + * kMinRatioForUnsaturatedLink * |send rate| if |send rate| is less than the + * link capacity. */ + const val kMinRatioForUnsaturatedLink = 0.9 + + /* The target utilization of the link. If we know true link capacity + * we'd like to send at 95% of that rate. */ + const val kTargetUtilizationFraction = 0.95 + + /* The maximum time period over which the cluster history is retained. + * This is also the maximum time period beyond which a probing burst is not + * expected to last. */ + val kMaxClusterHistory = 1.secs + + /* The maximum time interval between first and the last probe on a cluster + * on the sender side as well as the receive side. */ + val kMaxProbeInterval = 1.secs + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(ProbeBitrateEstimator::class.java) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeController.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeController.kt new file mode 100644 index 0000000..5a8b452 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeController.kt @@ -0,0 +1,607 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:enum-entry-name-case") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.min +import org.jitsi.nlj.util.times +import org.jitsi.utils.MAX_DURATION +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.max +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +/** Probe controller, + * based on WebRTC modules/congestion_controller/goog_cc/probe_controller.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings. + */ + +// Maximum waiting time from the time of initiating probing to getting +// the measured results back. +private val kMaxWaitingTimeForProbingResult = 1.secs + +// Default probing bitrate limit. Applied only when the application didn't +// specify max bitrate. +private val kDefaultMaxProbingBitrate = 5000.kbps + +// If the bitrate drops to a factor `kBitrateDropThreshold` or lower +// and we recover within `kBitrateDropTimeoutMs`, then we'll send +// a probe at a fraction `kProbeFractionAfterDrop` of the original bitrate. +private const val kBitrateDropThreshold = 0.66 +private val kBitrateDropTimeout = 5.secs +private const val kProbeFractionAfterDrop = 0.85 + +// Timeout for probing after leaving ALR. If the bitrate drops significantly, +// (as determined by the delay based estimator) and we leave ALR, then we will +// send a probe if we recover within `kLeftAlrTimeoutMs` ms. +private val kAlrEndedTimeout = 3.secs + +// This is a limit on how often probing can be done when there is a BW +// drop detected in ALR. +private val kMinTimeBetweenAlrProbes = 5.secs + +// The expected uncertainty of probe result (as a fraction of the target probe +// bitrate). Used to avoid probing if the probe bitrate is close to our current +// estimate. +private const val kProbeUncertainty = 0.05 + +class ProbeControllerConfig( + // These parameters configure the initial probes. First we send one or two + // probes of sizes p1 * start_bitrate_ and p2 * start_bitrate_. + // Then whenever we get a bitrate estimate of at least further_probe_threshold + // times the size of the last sent probe we'll send another one of size + // step_size times the new estimate. + val firstExponentialProbeScale: Double = 3.0, + val secondExponentialProbeScale: Double? = 6.0, + val furtherExponentialProbeScale: Double = 2.0, + val furtherProbeThreshold: Double = 0.7, + val abortFurtherProbeIfMaxLowerThanCurrent: Boolean = false, + + val repeatedInitialProbingTimePeriod: Duration = 5.secs, + // The minimum probing duration of an individual probe during + // the repeated_initial_probing_time_period. + val initialProbeDuration: Duration = 100.ms, + // Delta time between sent bursts of packets in a probe during + // the repeated_initial_probing_time_period. + val initialMinProbeDelta: Duration = 20.ms, + // Configures how often we send ALR probes and how big they are. + val alrProbingInterval: Duration = 5.secs, + val alrProbeScale: Double = 2.0, + // Configures how often we send probes if NetworkStateEstimate is available. + val networkStateEstimateProbingInterval: Duration = MAX_DURATION, + // Periodically probe as long as the ratio between current estimate and + // NetworkStateEstimate is lower then this. + val probeIfEstimateLowerThanNetworkStateEstimateRatio: Double = 0.0, + val estimateLowerThanNetworkStateProbingInterval: Duration = 3.secs, + val networkStateProbeScale: Double = 1.0, + // Overrides min_probe_duration if network_state_estimate_probing_interval + // is set and a network state estimate is known. + val networkStateProbeDuration: Duration = 15.ms, + // Overrides min_probe_delta if network_state_estimate_probing_interval + // is set and a network state estimate is known and equal or higher than the + // probe target. + val networkStateMinProbeDelta: Duration = 20.ms, + + // Configures the probes emitted by changed to the allocated bitrate. + val probeOnMaxAllocatedBitrateChange: Boolean = true, + val firstAllocationProbeScale: Double? = 1.0, + val secondAllocationProbeScale: Double? = 2.0, + val allocationProbeLimitByCurrentScale: Double = 2.0, + + // The minimum number probing packets used. + val minProbePacketsSent: Int = 5, + // The minimum probing duration. + val minProbeDuration: Duration = 15.ms, + // Delta time between sent bursts of packets in a probe. + val minProbeDelta: Duration = 2.ms, + val lossLimitedProbeScale: Double = 1.5, + // Don't send a probe if min(estimate, network state estimate) is larger than + // this fraction of the set max or max allocated bitrate. + val skipIfEstimateLargerThanFractionOfMax: Double = 0.0, + // Scale factor of the max allocated bitrate. Used when deciding if a probe + // can be skiped due to that the estimate is already high enough. + val skipProbeMaxAllocatedScale: Double = 1.0, +) + +/* Reason that bandwidth estimate is limited. Bandwidth estimate can be limited + * by either delay based bwe, or loss based bwe when it increases/decreases the + * estimate. + */ +enum class BandwidthLimitedCause { + kLossLimitedBweIncreasing, + kLossLimitedBwe, + kDelayBasedLimited, + kDelayBasedLimitedDelayIncreased, + kRttBasedBackOffHighRtt +} + +/* This class controls initiation of probing to estimate initial channel + * capacity. There is also support for probing during a session when max + * bitrate is adjusted by an application. + */ +class ProbeController( + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext, + private val config: ProbeControllerConfig = ProbeControllerConfig() +) { + private val logger = createChildLogger(parentLogger) + + private var networkAvailable = false + private var repeatedInitialProbingEnabled = false + private var lastAllowedRepeatedInitialProbe = Instant.MIN + private var bandwidthLimitedCause = BandwidthLimitedCause.kDelayBasedLimited + private var state = State.kInit + private var minBitrateToProbeFurther = Bandwidth.INFINITY + private var timeLastProbingInitiated = Instant.MIN + private var estimatedBitrate = Bandwidth.ZERO + + /* Skipping network_estimate */ + + private var startBitrate = Bandwidth.ZERO + private var maxBitrate = Bandwidth.INFINITY + private var lastBweDropProbingTime = Instant.MIN + private var alrStartTime: Instant? = null + private var alrEndTime: Instant? = null + private var enablePeriodicAlrProbing = false + private var timeOfLastLargeDrop = Instant.MIN + private var bitrateBeforeLastLargeDrop = Bandwidth.ZERO + private var maxTotalAllocatedBitrate = Bandwidth.ZERO + + private val inRapidRecoveryExperiment = false + + private var nextProbeClusterId = 1 + + fun setBitrates( + minBitrate: Bandwidth, + startBitrate: Bandwidth, + maxBitrate: Bandwidth, + atTime: Instant + ): MutableList { + if (startBitrate > Bandwidth.ZERO) { + this.startBitrate = startBitrate + estimatedBitrate = startBitrate + } else if (this.startBitrate == Bandwidth.ZERO) { + this.startBitrate = minBitrate + } + + // The reason we use the variable `old_max_bitrate_pbs` is because we + // need to set `max_bitrate_` before we call InitiateProbing. + val oldMaxBitrate = this.maxBitrate + this.maxBitrate = if (maxBitrate.isFinite()) { + maxBitrate + } else { + kDefaultMaxProbingBitrate + } + + when (state) { + State.kInit -> + if (networkAvailable) { + return initiateExponentialProbing(atTime) + } + + State.kWaitingForProbingResult -> + Unit + + State.kProbingComplete -> + // If the new max bitrate is higher than both the old max bitrate and the + // estimate then initiate probing. + if (estimatedBitrate != Bandwidth.ZERO && oldMaxBitrate < this.maxBitrate && + estimatedBitrate < this.maxBitrate + ) { + return initiateProbing(atTime, listOf(maxBitrate), false) + } + } + + return mutableListOf() + } + + // The total bitrate, as opposed to the max bitrate, is the sum of the + // configured bitrates for all active streams. + fun onMaxTotalAllocatedBitrate( + maxTotalAllocatedBitrate: Bandwidth, + atTime: Instant + ): MutableList { + val inAlr = alrStartTime != null + val allowAllocationProbe = inAlr + + if (config.probeOnMaxAllocatedBitrateChange && + state == State.kProbingComplete && + maxTotalAllocatedBitrate != this.maxTotalAllocatedBitrate && + estimatedBitrate < maxBitrate && + estimatedBitrate < maxTotalAllocatedBitrate && + allowAllocationProbe + ) { + this.maxTotalAllocatedBitrate = maxTotalAllocatedBitrate + + if (config.firstAllocationProbeScale == null) { + return mutableListOf() + } + var firstProbeRate = maxTotalAllocatedBitrate * config.firstAllocationProbeScale + val currentBweLimit = + config.allocationProbeLimitByCurrentScale * + estimatedBitrate + var limitedByCurrentBwe = currentBweLimit < firstProbeRate + if (limitedByCurrentBwe) { + firstProbeRate = currentBweLimit + } + + val probes = mutableListOf(firstProbeRate) + if (!limitedByCurrentBwe && config.secondAllocationProbeScale != null) { + var secondProbeRate = maxTotalAllocatedBitrate * config.secondAllocationProbeScale + limitedByCurrentBwe = currentBweLimit < secondProbeRate + if (limitedByCurrentBwe) { + secondProbeRate = currentBweLimit + } + if (secondProbeRate > firstProbeRate) { + probes.add(secondProbeRate) + } + } + val allowFurtherProbing = limitedByCurrentBwe + return initiateProbing(atTime, probes, allowFurtherProbing) + } + if (maxTotalAllocatedBitrate != Bandwidth.ZERO) { + lastAllowedRepeatedInitialProbe = atTime + } + + this.maxTotalAllocatedBitrate = maxTotalAllocatedBitrate + return mutableListOf() + } + + fun onNetworkAvailability(msg: NetworkAvailability): MutableList { + networkAvailable = msg.networkAvailable + + if (!networkAvailable && state == State.kWaitingForProbingResult) { + state = State.kProbingComplete + minBitrateToProbeFurther = Bandwidth.INFINITY + } + + if (networkAvailable && state == State.kInit && startBitrate != Bandwidth.ZERO) { + return initiateExponentialProbing(msg.atTime) + } + return mutableListOf() + } + + private fun updateState(newState: State) { + when (newState) { + State.kInit -> + state = State.kInit + + State.kWaitingForProbingResult -> + state = State.kWaitingForProbingResult + + State.kProbingComplete -> { + state = State.kProbingComplete + minBitrateToProbeFurther = Bandwidth.INFINITY + } + } + } + + private fun initiateExponentialProbing(atTime: Instant): MutableList { + assert(networkAvailable) + assert(state == State.kInit) + assert(startBitrate > Bandwidth.ZERO) + + // When probing at 1.8 Mbps ( 6x 300), this represents a threshold of + // 1.2 Mbps to continue probing. + val probes = mutableListOf(config.firstExponentialProbeScale * startBitrate) + if (config.secondExponentialProbeScale != null && config.secondExponentialProbeScale > 0.0) { + probes.add(config.secondExponentialProbeScale * startBitrate) + } + if (repeatedInitialProbingEnabled && maxTotalAllocatedBitrate == Bandwidth.ZERO) { + lastAllowedRepeatedInitialProbe = + atTime + config.repeatedInitialProbingTimePeriod + logger.info { + "Repeated initial probing enabled, last allowed probe: $lastAllowedRepeatedInitialProbe now: $atTime" + } + } + + return initiateProbing(atTime, probes, true) + } + + fun setEstimatedBitrate( + bitrate: Bandwidth, + bandwidthLimitedCause: BandwidthLimitedCause, + atTime: Instant + ): MutableList { + this.bandwidthLimitedCause = bandwidthLimitedCause + if (bitrate < kBitrateDropThreshold * estimatedBitrate) { + timeOfLastLargeDrop = atTime + bitrateBeforeLastLargeDrop = estimatedBitrate + } + estimatedBitrate = bitrate + + if (state == State.kWaitingForProbingResult) { + // Continue probing if probing results indicate channel has greater + // capacity unless we already reached the needed bitrate. + if (config.abortFurtherProbeIfMaxLowerThanCurrent && ( + bitrate > maxBitrate || ( + maxTotalAllocatedBitrate != Bandwidth.ZERO && + bitrate > 2 * maxTotalAllocatedBitrate + ) + ) + ) { + // No need to continue probing + minBitrateToProbeFurther = Bandwidth.INFINITY + } + val networkStateEstimateProbeFurtherLimit = + /* Skipping networkEstimate */ + Bandwidth.INFINITY + logger.info( + "Measured bitrate: $bitrate Minimum to probe further: $minBitrateToProbeFurther " + + "upper limit: $networkStateEstimateProbeFurtherLimit" + ) + + if (bitrate > minBitrateToProbeFurther && + bitrate <= networkStateEstimateProbeFurtherLimit + ) { + return initiateProbing(atTime, listOf(config.furtherExponentialProbeScale * bitrate), true) + } + } + return mutableListOf() + } + + fun enablePeriodicAlrProbing(enable: Boolean) { + enablePeriodicAlrProbing = enable + } + + // Probes are sent periodically every 1s during the first 5s after the network + // becomes available or until OnMaxTotalAllocatedBitrate is invoked with a + // none zero max_total_allocated_bitrate (there are active streams being + // sent.) Probe rate is up to max configured bitrate configured via + // SetBitrates. + fun enableRepeatedInitialProbing(enable: Boolean) { + repeatedInitialProbingEnabled = enable + } + + fun setAlrStartTimeMs(alrStartTimeMs: Long?) { + if (alrStartTimeMs != null) { + alrStartTime = Instant.ofEpochMilli(alrStartTimeMs) + } else { + alrStartTime = null + } + } + + fun setAlrEndedTimeMs(alrEndTimeMs: Long) { + alrEndTime = Instant.ofEpochMilli(alrEndTimeMs) + } + + fun requestProbe(atTime: Instant): MutableList { + // Called once we have returned to normal state after a large drop in + // estimated bandwidth. The current response is to initiate a single probe + // session (if not already probing) at the previous bitrate. + // + // If the probe session fails, the assumption is that this drop was a + // real one from a competing flow or a network change. + val inAlr = alrStartTime != null + val alrEndedRecently = ( + alrEndTime != null && + Duration.between(alrEndTime, atTime) < kAlrEndedTimeout + ) + if (inAlr || alrEndedRecently || inRapidRecoveryExperiment) { + if (state == State.kProbingComplete) { + val suggestedProbe = kProbeFractionAfterDrop * bitrateBeforeLastLargeDrop + val minExpectedProbeResult = (1 - kProbeUncertainty) * suggestedProbe + val timeSinceDrop = Duration.between(timeOfLastLargeDrop, atTime) + val timeSinceProbe = Duration.between(lastBweDropProbingTime, atTime) + if (minExpectedProbeResult > estimatedBitrate && + timeSinceDrop < kBitrateDropTimeout && + timeSinceProbe > kMinTimeBetweenAlrProbes + ) { + logger.info("Detected big bandwidth drop, start probing") + // Track how often we probe in response to bandwidth drop in ALR. + // TODO: histogram + lastBweDropProbingTime = atTime + return initiateProbing(atTime, listOf(suggestedProbe), false) + } + } + } + return mutableListOf() + } + + /* Skipping setNetworkStateEstimate */ + + /** + * Resets the ProbeController to a state equivalent to as if it was just + * created EXCEPT for configuration settings like + * `enable_periodic_alr_probing_` `network_available_` and + * `max_total_allocated_bitrate_`. + */ + fun reset(atTime: Instant) { + bandwidthLimitedCause = BandwidthLimitedCause.kDelayBasedLimited + state = State.kInit + minBitrateToProbeFurther = Bandwidth.INFINITY + timeLastProbingInitiated = Instant.MIN + estimatedBitrate = Bandwidth.ZERO + startBitrate = Bandwidth.ZERO + maxBitrate = kDefaultMaxProbingBitrate + val now = atTime + lastBweDropProbingTime = now + alrEndTime = null + timeOfLastLargeDrop = now + bitrateBeforeLastLargeDrop = Bandwidth.ZERO + } + + private fun timeForAlrProbe(atTime: Instant): Boolean { + if (enablePeriodicAlrProbing && alrStartTime != null) { + val nextProbeTime = + max(alrStartTime!!, timeLastProbingInitiated) + + config.alrProbingInterval + return atTime >= nextProbeTime + } + return false + } + + private fun timeForNetworkStateProbe(atTime: Instant): Boolean { + /* Not using network_estimate */ + return false + } + + private fun timeForNextRepeatedInitialProbe(atTime: Instant): Boolean { + if (state != State.kWaitingForProbingResult && + lastAllowedRepeatedInitialProbe > atTime + ) { + val nextProbeTime = timeLastProbingInitiated + kMaxWaitingTimeForProbingResult + if (atTime >= nextProbeTime) { + return true + } + } + return false + } + + private fun createProbeClusterConfig(atTime: Instant, bitrate: Bandwidth): ProbeClusterConfig { + val config = ProbeClusterConfig() + config.atTime = atTime + config.targetDataRate = bitrate + if (atTime < lastAllowedRepeatedInitialProbe) { + config.targetDuration = this.config.initialProbeDuration + config.minProbeDelta = this.config.initialMinProbeDelta + } else { + config.targetDuration = this.config.minProbeDuration + config.minProbeDelta = this.config.minProbeDelta + } + config.targetProbeCount = this.config.minProbePacketsSent + config.id = nextProbeClusterId + nextProbeClusterId++ + maybeLogProbeClusterCreated(diagnosticContext, config) + return config + } + + fun process(atTime: Instant): MutableList { + if (Duration.between(timeLastProbingInitiated, atTime) > kMaxWaitingTimeForProbingResult) { + if (state == State.kWaitingForProbingResult) { + logger.info("kWaitingForProbingResult: timeout") + updateState(State.kProbingComplete) + } + } + if (estimatedBitrate == Bandwidth.ZERO || state != State.kProbingComplete) { + return mutableListOf() + } + if (timeForNextRepeatedInitialProbe(atTime)) { + return initiateProbing(atTime, listOf(estimatedBitrate * config.firstExponentialProbeScale), true) + } + if (timeForAlrProbe(atTime) || timeForNetworkStateProbe(atTime)) { + return initiateProbing(atTime, listOf(estimatedBitrate * config.alrProbeScale), true) + } + return mutableListOf() + } + + private enum class State { + /** Initial state where no probing has been triggrered yet */ + kInit, + + /** Waiting for probing results to continue further probing. */ + kWaitingForProbingResult, + + /** Probing is complete. */ + kProbingComplete + } + + private fun initiateProbing( + now: Instant, + bitratesToProbe: List, + probeFurtherIn: Boolean + ): MutableList { + var probeFurther = probeFurtherIn + if (config.skipIfEstimateLargerThanFractionOfMax > 0.0) { + val networkEstimate = Bandwidth.INFINITY + val maxProbeRate = if (maxTotalAllocatedBitrate == Bandwidth.ZERO) { + maxBitrate + } else { + min(config.skipProbeMaxAllocatedScale * maxTotalAllocatedBitrate, maxBitrate) + } + if (min(networkEstimate, estimatedBitrate) > config.skipIfEstimateLargerThanFractionOfMax * maxProbeRate) { + updateState(State.kProbingComplete) + return mutableListOf() + } + } + + var maxProbeBitrate = maxBitrate + if (maxTotalAllocatedBitrate > Bandwidth.ZERO) { + // If a max allocated bitrate has been configured, allow probing up to 2x + // that rate. This allows some overhead to account for bursty streams, + // which otherwise would have to ramp up when the overshoot is already in + // progress. + // It also avoids minor quality reduction caused by probes often being + // received at slightly less than the target probe bitrate. + maxProbeBitrate = min(maxProbeBitrate, maxTotalAllocatedBitrate * 2) + } + + when (bandwidthLimitedCause) { + BandwidthLimitedCause.kRttBasedBackOffHighRtt, + BandwidthLimitedCause.kDelayBasedLimitedDelayIncreased, + BandwidthLimitedCause.kLossLimitedBwe -> { + logger.info { "Not sending probe in bandwidth limited state. $bandwidthLimitedCause" } + return mutableListOf() + } + + BandwidthLimitedCause.kLossLimitedBweIncreasing -> + maxProbeBitrate = + min(maxProbeBitrate, estimatedBitrate * config.lossLimitedProbeScale) + + BandwidthLimitedCause.kDelayBasedLimited -> + Unit + } + + /* Skipping use of networkEstimate */ + + val pendingProbes = mutableListOf() + for (b in bitratesToProbe) { + assert(b != Bandwidth.ZERO) + var bitrate = b + if (bitrate >= maxProbeBitrate) { + bitrate = maxProbeBitrate + probeFurther = false + } + + pendingProbes.add(createProbeClusterConfig(now, bitrate)) + } + timeLastProbingInitiated = now + if (probeFurther) { + updateState(State.kWaitingForProbingResult) + // Dont expect probe results to be larger than a fraction of the actual + // probe rate. + minBitrateToProbeFurther = pendingProbes.last().targetDataRate * config.furtherProbeThreshold + } else { + updateState(State.kProbingComplete) + } + return pendingProbes + } + + companion object { + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(ProbeController::class.java) + + private fun maybeLogProbeClusterCreated(diagnosticContext: DiagnosticContext, probe: ProbeClusterConfig) { + val minDataSize = probe.targetDataRate * probe.targetDuration + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("ProbeClusterCreated") + .addField("probe_id", probe.id) + .addField("probe_target_data_rate_bps", probe.targetDataRate.bps) + .addField("probe_target_probe_count", probe.targetProbeCount) + .addField("probe_min_data_size", minDataSize.bytes) + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimator.kt new file mode 100644 index 0000000..b2c48cb --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimator.kt @@ -0,0 +1,233 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.min +import org.jitsi.nlj.util.per +import org.jitsi.utils.isFinite +import org.jitsi.utils.isInfinite +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.max +import org.jitsi.utils.min +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant +import java.util.* +import kotlin.Comparator +import kotlin.collections.ArrayDeque + +/** + * Robust throughput estimator. + * + * Based on WebRTC modules/congestion_controller/goog_cc/robust_throughput_estimator.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138) + */ +class RobustThroughputEstimator(val settings: RobustThroughputEstimatorSettings) : + AcknowledgedBitrateEstimatorInterface { + // TODO: pass parent logger in so we have log contexts + private val logger = createLogger() + + init { + require(settings.enabled) + } + + override fun incomingPacketFeedbackVector(packetFeedbackVector: List) { + require(packetFeedbackVector.isSorted(compareBy { it.receiveTime })) + packetFeedbackVector.forEach { packet -> + // Ignore packets without valid send or receive times. + // (This should not happen in production since lost packets are filtered + // out before passing the feedback vector to the throughput estimator. + // However, explicitly handling this case makes the estimator more robust + // and avoids a hard-to-detect bad state.) + if (packet.receiveTime.isInfinite() || + packet.sentPacket.sendTime.isInfinite() + ) { + return@forEach + } + // Insert the new packet. + window.addLast(packet) + window.last().sentPacket.priorUnackedData = + window.last().sentPacket.priorUnackedData * + settings.unackedWeight + // In most cases, receive timestamps should already be in order, but in the + // rare case where feedback packets have been reordered, we do some swaps to + // ensure that the window is sorted. + var i = window.size - 1 + while (i > 0 && window[i].receiveTime < window[i - 1].receiveTime) { + Collections.swap(window, i, i - 1) + i-- + } + val kMaxReorderingTime = 1.secs + val receiveDelta = Duration.between(packet.receiveTime, window.last().receiveTime) + + if (receiveDelta > kMaxReorderingTime) { + logger.warn("Severe packet re-ordering or timestamps offset changed: $receiveDelta") + window.clear() + latestDiscardedSendTime = Instant.MIN + } + } + + // Remove old packets. + while (firstPacketOutsideWindow()) { + latestDiscardedSendTime = max(latestDiscardedSendTime, window.first().sentPacket.sendTime) + window.removeFirst() + } + } + + override fun bitrate(): Bandwidth? { + if (window.isEmpty() || window.size < settings.requiredPackets) { + return null + } + + var largestRecvGap = Duration.ZERO + var secondLargestRecvGap = Duration.ZERO + for (i in 1 until window.size) { + // Find receive time gaps. + val gap = Duration.between(window[i - 1].receiveTime, window[i].receiveTime) + if (gap > largestRecvGap) { + secondLargestRecvGap = largestRecvGap + largestRecvGap = gap + } else if (gap > secondLargestRecvGap) { + secondLargestRecvGap = gap + } + } + + var firstSendTime = Instant.MAX + var lastSendTime = Instant.MIN + var firstRecvTime = Instant.MAX + var lastRecvTime = Instant.MIN + var recvSize = 0.bytes + var sendSize = 0.bytes + var firstRecvSize = 0.bytes + var lastSendSize = 0.bytes + var numSentPacketsInWindow = 0 + window.forEach { packet -> + if (packet.receiveTime < firstRecvTime) { + firstRecvTime = packet.receiveTime + firstRecvSize = + packet.sentPacket.size + packet.sentPacket.priorUnackedData + } + lastRecvTime = max(lastRecvTime, packet.receiveTime) + recvSize += packet.sentPacket.size + recvSize += packet.sentPacket.priorUnackedData + + if (packet.sentPacket.sendTime < latestDiscardedSendTime) { + // If we have dropped packets from the window that were sent after + // this packet, then this packet was reordered. Ignore it from + // the send rate computation (since the send time may be very far + // in the past, leading to underestimation of the send rate.) + // However, ignoring packets creates a risk that we end up without + // any packets left to compute a send rate. + return@forEach + } + if (packet.sentPacket.sendTime > lastSendTime) { + lastSendTime = packet.sentPacket.sendTime + lastSendSize = + packet.sentPacket.size + packet.sentPacket.priorUnackedData + } + firstSendTime = min(firstSendTime, packet.sentPacket.sendTime) + + sendSize += packet.sentPacket.size + sendSize += packet.sentPacket.priorUnackedData + ++numSentPacketsInWindow + } + + // Suppose a packet of size S is sent every T milliseconds. + // A window of N packets would contain N*S bytes, but the time difference + // between the first and the last packet would only be (N-1)*T. Thus, we + // need to remove the size of one packet to get the correct rate of S/T. + // Which packet to remove (if the packets have varying sizes), + // depends on the network model. + // Suppose that 2 packets with sizes s1 and s2, are received at times t1 + // and t2, respectively. If the packets were transmitted back to back over + // a bottleneck with rate capacity r, then we'd expect t2 = t1 + r * s2. + // Thus, r = (t2-t1) / s2, so the size of the first packet doesn't affect + // the difference between t1 and t2. + // Analoguously, if the first packet is sent at time t1 and the sender + // paces the packets at rate r, then the second packet can be sent at time + // t2 = t1 + r * s1. Thus, the send rate estimate r = (t2-t1) / s1 doesn't + // depend on the size of the last packet. + recvSize -= firstRecvSize + sendSize -= lastSendSize + + // Remove the largest gap by replacing it by the second largest gap. + // This is to ensure that spurious "delay spikes" (i.e. when the + // network stops transmitting packets for a short period, followed + // by a burst of delayed packets), don't cause the estimate to drop. + // This could cause an overestimation, which we guard against by + // never returning an estimate above the send rate. + check(firstRecvTime.isFinite()) + check(lastRecvTime.isFinite()) + val recvDuration = ( + Duration.between(firstRecvTime, lastRecvTime) - + largestRecvGap + secondLargestRecvGap + ).coerceAtLeast(1.ms) + + if (numSentPacketsInWindow < settings.requiredPackets) { + // Too few send times to calculate a reliable send rate. + return recvSize.per(recvDuration) + } + + check(firstSendTime.isFinite()) + check(lastSendTime.isFinite()) + val sendDuration = Duration.between(firstSendTime, lastSendTime).coerceAtLeast(1.ms) + + return min(sendSize.per(sendDuration), recvSize.per(recvDuration)) + } + + override fun peekRate(): Bandwidth? = bitrate() + + override fun setAlr(inAlr: Boolean) {} + + override fun setAlrEndedTime(alrEndedTime: Instant) {} + + private fun firstPacketOutsideWindow(): Boolean { + if (window.isEmpty()) { + return false + } + if (window.size > settings.maxWindowPackets) { + return true + } + val currentWindowDuration = Duration.between(window.first().receiveTime, window.last().receiveTime) + if (currentWindowDuration > settings.maxWindowDuration) { + return true + } + if (window.size > settings.windowPackets && + currentWindowDuration > settings.minWindowDuration + ) { + return true + } + return false + } + + private val window = ArrayDeque() + + private var latestDiscardedSendTime: Instant = Instant.MIN +} + +/* TODO: move this to a util somewhere */ +fun Collection.isSorted(comparator: Comparator): Boolean { + this.asSequence().windowed(2).forEach { (a, b) -> + if (comparator.compare(a, b) > 0) { + return@isSorted false + } + } + return true +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimation.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimation.kt new file mode 100644 index 0000000..f625dc6 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimation.kt @@ -0,0 +1,595 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This file uses WebRTC's naming style for enums and constants +@file:Suppress("ktlint:standard:enum-entry-name-case", "ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.max +import org.jitsi.nlj.util.min +import org.jitsi.utils.div +import org.jitsi.utils.isFinite +import org.jitsi.utils.isInfinite +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.max +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.times +import java.time.Duration +import java.time.Instant +import java.util.* +import kotlin.math.exp +import kotlin.math.min + +/** Send-side bandwidth estimation, + * based on WebRTC modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings. + */ +class LinkCapacityTracker { + private var capacityEstimateBps = 0.0 + private var lastLinkCapcityUpdate = Instant.MIN + private var lastDelayBasedEstimate = Bandwidth.INFINITY + + fun updateDelayBasedEstimate(atTime: Instant, delayBasedEstimate: Bandwidth) { + if (delayBasedEstimate < lastDelayBasedEstimate) { + capacityEstimateBps = min(capacityEstimateBps, delayBasedEstimate.bps.toDouble()) + lastLinkCapcityUpdate = atTime + } + lastDelayBasedEstimate = delayBasedEstimate + } + + fun onStartingRate(startRate: Bandwidth) { + if (lastLinkCapcityUpdate.isInfinite()) { + capacityEstimateBps = startRate.bps.toDouble() + } + } + + fun onRateUpdate(acknowledged: Bandwidth?, target: Bandwidth, atTime: Instant) { + if (acknowledged == null) { + return + } + val acknowledgedTarget = min(acknowledged, target) + if (acknowledgedTarget.bps > capacityEstimateBps) { + val alpha = if (lastLinkCapcityUpdate.isFinite() && atTime.isFinite()) { + val delta = Duration.between(lastLinkCapcityUpdate, atTime) + exp(-(delta / 10.secs)) + } else { + 0.0 + } + capacityEstimateBps = alpha * capacityEstimateBps * alpha + (1 - alpha) * acknowledgedTarget.bps + } + lastLinkCapcityUpdate = atTime + } + + fun onRttBackoff(backoffRate: Bandwidth, atTime: Instant) { + capacityEstimateBps = min(capacityEstimateBps, backoffRate.bps.toDouble()) + lastLinkCapcityUpdate = atTime + } + + fun estimate(): Bandwidth = capacityEstimateBps.bps +} + +class RttBasedBackoff { + val disabled = false + val configuredLimit = 3.secs + val dropFraction = 0.8 + val dropInterval = 1.secs + val bandwidthFloor = 5.kbps + + var rttLimit = configuredLimit + var lastPropagationRttUpdate = Instant.MAX + var lastPropagationRtt = Duration.ZERO + var lastPacketSent = Instant.MIN + + fun updatePropagationRtt(atTime: Instant, propagationRtt: Duration) { + lastPropagationRttUpdate = atTime + lastPropagationRtt = propagationRtt + } + + fun isRttAboveLimit(): Boolean = correctedRtt() > rttLimit + + private fun correctedRtt(): Duration { + // Avoid timeout when no packets are being sent. + val timeoutCorrection = max(Duration.between(lastPropagationRttUpdate, lastPacketSent), Duration.ZERO) + return timeoutCorrection + lastPropagationRtt + } +} + +private val kBweIncreaseInterval = 1000.ms +private val kBweDecreaseInterval = 300.ms +private val kStartPhase = 2000.ms +private val kBweConverganceTime = 20000.ms +private const val kLimitNumPackets = 20 +private val kDefaultMaxBitrate = 1000000000.bps +private val kLowBitrateLogPeriod = 10000.ms +private val kRtcEventLogPeriod = 5000.ms + +// Expecting that RTCP feedback is sent uniformly within [0.5, 1.5]s intervals. +private val kMaxRtcpFeedbackInterval = 5000.ms + +private const val kDefaultLowLossThreshold = 0.02f +private const val kDefaultHighLossThreshold = 0.1f +private val kDefaultBitrateThreshold = Bandwidth.ZERO + +class SendSideBandwidthEstimation( + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) { + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass) + private val logger = parentLogger.createChildLogger(javaClass.name) + + private val rttBackoff = RttBasedBackoff() + private val linkCapacity = LinkCapacityTracker() + + private val minBitrateHistory = ArrayDeque>() + + // incoming filters + private var lostPacketsSinceLastLossUpdate = 0L + private var expectedPacketsSinceLastLossUpdate = 0L + + private var acknowledgedRate: Bandwidth? = null + private var currentTarget: Bandwidth = Bandwidth.ZERO + private var lastLoggedTarget: Bandwidth = Bandwidth.ZERO + private var minBitrateConfigured: Bandwidth = kCongestionControllerMinBitrate + private var maxBitrateConfigured: Bandwidth = kDefaultMaxBitrate + private var lastLowBitrateLog: Instant = Instant.MIN + + private var hasDecreasedSinceLastFractionLoss: Boolean = false + + @SuppressFBWarnings("URF_UNREAD_FIELD") + private var lastLossFeedback: Instant = Instant.MIN + private var lastLossPacketReport: Instant = Instant.MIN + private var lastFractionLoss: UByte = 0u + private var lastLoggedFractionLoss: UByte = 0u + private var lastRoundTripTime: Duration = Duration.ZERO + + // The max bitrate as set by the receiver in the call. This is typically + // signalled using the REMB RTCP message and is used when we don't have any + // send side delay based estimate. + private var receiverLimit: Bandwidth = Bandwidth.INFINITY + private var delayBasedLimit: Bandwidth = Bandwidth.INFINITY + private var timeLastDecrease: Instant = Instant.MIN + private var firstReportTime: Instant = Instant.MIN + private var initiallyLostPackets: Int = 0 + private var bitrateAt2Seconds: Bandwidth = Bandwidth.ZERO + private var umaUpdateState: UmaState = UmaState.kNoUpdate + private var umaRttState: UmaState = UmaState.kNoUpdate + private val rampupUmaStatsUpdated = ArrayList(kUmaRampupMetrics.size) + private var lastRtcEventLog: Instant = Instant.MIN + private val lowLossThreshold: Float = kDefaultLowLossThreshold + private val highLossThreshold: Float = kDefaultHighLossThreshold + private val bitrateThreshold: Bandwidth = kDefaultBitrateThreshold + var lossBasedBandwidthEstimatorV2 = LossBasedBweV2().apply { + setMinMaxBitrate(minBitrateConfigured, maxBitrateConfigured) + } + private set + private var lossBasedState: LossBasedState = LossBasedState.kDelayBasedEstimate + + fun onRouteChange() { + lostPacketsSinceLastLossUpdate = 0 + expectedPacketsSinceLastLossUpdate = 0 + currentTarget = Bandwidth.ZERO + minBitrateConfigured = kCongestionControllerMinBitrate + maxBitrateConfigured = kDefaultMaxBitrate + lastLowBitrateLog = Instant.MIN + hasDecreasedSinceLastFractionLoss = false + lastLossFeedback = Instant.MIN + lastLossPacketReport = Instant.MIN + lastFractionLoss = 0u + lastLoggedFractionLoss = 0u + lastRoundTripTime = Duration.ZERO + receiverLimit = Bandwidth.INFINITY + delayBasedLimit = Bandwidth.INFINITY + timeLastDecrease = Instant.MIN + firstReportTime = Instant.MIN + initiallyLostPackets = 0 + bitrateAt2Seconds = Bandwidth.ZERO + umaUpdateState = UmaState.kNoUpdate + umaRttState = UmaState.kNoUpdate + lastRtcEventLog = Instant.MIN + if (lossBasedBandwidthEstimatorV2.useInStartPhase()) { + lossBasedBandwidthEstimatorV2 = LossBasedBweV2() + } + } + + fun targetRate(): Bandwidth { + var target = currentTarget + if (!disableReceiverLimitCapsOnly) { + target = min(target, receiverLimit) + } + return max(minBitrateConfigured, target) + } + + fun lossBasedState(): LossBasedState = lossBasedState + + // Return whether the current rtt is higher than the rtt limited configured in + // RttBasedBackoff. + fun isRttAboveLimit(): Boolean = rttBackoff.isRttAboveLimit() + + fun fractionLoss() = lastFractionLoss + + fun roundTripTime() = lastRoundTripTime + + fun getEstimatedLinkCapacity(): Bandwidth = linkCapacity.estimate() + + // Call periodically to update estimate + fun updateEstimate(atTime: Instant) { + if (rttBackoff.isRttAboveLimit()) { + if (Duration.between(timeLastDecrease, atTime) >= rttBackoff.dropInterval && + currentTarget > rttBackoff.bandwidthFloor + ) { + timeLastDecrease = atTime + val newBitrate = (currentTarget * rttBackoff.dropFraction).coerceAtLeast(rttBackoff.bandwidthFloor) + linkCapacity.onRttBackoff(newBitrate, atTime) + updateTargetBitrate(newBitrate, atTime) + return + } + // TODO(srte): This is likely redundant in most cases. + applyTargetLimits(atTime) + return + } + + // We trust the REMB and/or delay-based estimate during the first 2 seconds if + // we haven't had any packet loss reported, to allow startup bitrate probing. + if (lastFractionLoss.toUInt() == 0u && isInStartPhase(atTime) && + !lossBasedBandwidthEstimatorV2.readyToUseInStartPhase() + ) { + var newBitrate = currentTarget + // TODO(srte): We should not allow the new_bitrate to be larger than the + // receiver limit here. + if (receiverLimit.isFinite()) { + newBitrate = max(receiverLimit, newBitrate) + } + if (delayBasedLimit.isFinite()) { + newBitrate = max(delayBasedLimit, newBitrate) + } + + if (newBitrate != currentTarget) { + minBitrateHistory.clear() + minBitrateHistory.add(Pair(atTime, currentTarget)) + updateTargetBitrate(newBitrate, atTime) + return + } + } + updateMinHistory(atTime) + if (lastLossPacketReport.isInfinite()) { + // No feedback received. + // TODO(srte): This is likely redundant in most cases. + applyTargetLimits(atTime) + return + } + if (lossBasedBandwidthEstimatorV2.isReady()) { + val result = lossBasedBandwidthEstimatorV2.getLossBasedResult() + lossBasedState = result.state + updateTargetBitrate(result.bandwidthEstimate, atTime) + return + } + + val timeSinceLossPacketReport = Duration.between(lastLossPacketReport, atTime) + if (timeSinceLossPacketReport < 1.2 * kMaxRtcpFeedbackInterval) { + // We only care about loss above a given bitrate threshold. + val loss = lastFractionLoss.toFloat() / 256f + // We only make decisions based on loss when the bitrate is above a + // threshold. This is a crude way of handling loss which is uncorrelated + // to congestion. + if (currentTarget < bitrateThreshold || loss <= lowLossThreshold) { + // Loss < 2%: Increase rate by 8% of the min bitrate in the last + // kBweIncreaseInterval. + // Note that by remembering the bitrate over the last second one can + // rampup up one second faster than if only allowed to start ramping + // at 8% per second rate now. E.g.: + // If sending a constant 100kbps it can rampup immediately to 108kbps + // whenever a receiver report is received with lower packet loss. + // If instead one would do: current_bitrate_ *= 1.08^(delta time), + // it would take over one second since the lower packet loss to achieve + // 108kbps. + var newBitrate = (minBitrateHistory.first().second.bps * 1.08 + 0.5).bps + + // Add 1 kbps extra, just to make sure that we do not get stuck + // (gives a little extra increase at low rates, negligible at higher + // rates). + newBitrate += 1000.bps + updateTargetBitrate(newBitrate, atTime) + return + } else if (currentTarget > bitrateThreshold) { + if (loss <= highLossThreshold) { + // Loss between 2% - 10%: Do nothing. + } else { + // Loss > 10%: Limit the rate decreases to once a kBweDecreaseInterval + // + rtt. + if (!hasDecreasedSinceLastFractionLoss && + Duration.between(timeLastDecrease, atTime) >= (kBweDecreaseInterval + lastRoundTripTime) + ) { + timeLastDecrease = atTime + + // Reduce rate: + // newRate = rate * (1 - 0.5*lossRate); + // where packetLoss = 256*lossRate; + val newBitrate = (currentTarget.bps * (512u - lastFractionLoss.toUInt()).toDouble() / 512.0).bps + hasDecreasedSinceLastFractionLoss = true + updateTargetBitrate(newBitrate, atTime) + return + } + } + } + } + // TODO(srte): This is likely redundant in most cases. + applyTargetLimits(atTime) + } + + fun onSentPacket(sentPacket: SentPacket) { + // Only feedback-triggering packets will be reported here. + rttBackoff.lastPacketSent = sentPacket.sendTime + } + + fun updatePropagationRtt(atTime: Instant, propagationRtt: Duration) { + rttBackoff.updatePropagationRtt(atTime, propagationRtt) + } + + // Call when we receive a RTCP message with TMMBR or REMB. + fun updateReceiverEstimate(atTime: Instant, bandwidth: Bandwidth) { + // TODO(srte): Ensure caller passes PlusInfinity, not zero, to represent no + // limitation. + receiverLimit = if (bandwidth == Bandwidth.ZERO) { + Bandwidth.INFINITY + } else { + bandwidth + } + applyTargetLimits(atTime) + } + + // Call when a new delay-based estimate is available. + fun updateDelayBasedEstimate(atTime: Instant, bitrate: Bandwidth) { + linkCapacity.updateDelayBasedEstimate(atTime, bitrate) + // TODO(srte): Ensure caller passes PlusInfinity, not zero, to represent no + // limitation. + delayBasedLimit = if (bitrate == Bandwidth.ZERO) { + Bandwidth.INFINITY + } else { + bitrate + } + applyTargetLimits(atTime) + } + + // Call when we receive a RTCP message with a ReceiveBlock. + fun updatePacketsLost(packetsLost: Long, numberOfPackets: Long, atTime: Instant) { + lastLossFeedback = atTime + if (firstReportTime.isInfinite()) { + firstReportTime = atTime + } + + // Check sequence number diff and weight loss report + if (numberOfPackets > 0) { + val expected = expectedPacketsSinceLastLossUpdate + numberOfPackets + + // Don't generate a loss rate until it can be based on enough packets. + if (expected < kLimitNumPackets) { + // Accumulate reports + expectedPacketsSinceLastLossUpdate = expected + lostPacketsSinceLastLossUpdate = packetsLost + return + } + + hasDecreasedSinceLastFractionLoss = false + val lostQ8 = (lostPacketsSinceLastLossUpdate + packetsLost).coerceAtLeast(0L) shl 8 + lastFractionLoss = (lostQ8 / expected).coerceAtMost(255).toUByte() + + // Reset accumulators + lostPacketsSinceLastLossUpdate = 0 + expectedPacketsSinceLastLossUpdate = 0 + lastLossPacketReport = atTime + updateEstimate(atTime) + } + } + + // Call when we receive a RTCP message with a ReceiveBlock. + fun updateRtt(rtt: Duration, atTime: Instant) { + // Update RTT if we were able to compute an RTT based on this RTCP. + // FlexFEC doesn't send RTCP SR, which means we won't be able to compute RTT. + if (rtt > Duration.ZERO) { + lastRoundTripTime = rtt + } + + if (!isInStartPhase(atTime) && umaRttState == UmaState.kNoUpdate) { + umaRttState = UmaState.kDone + /* TODO: histograms */ + } + } + + fun setBitrates(sendBitrate: Bandwidth?, minBitrate: Bandwidth, maxBitrate: Bandwidth, atTime: Instant) { + setMinMaxBitrate(minBitrate, maxBitrate) + if (sendBitrate != null) { + linkCapacity.onStartingRate(sendBitrate) + setSendBitrate(sendBitrate, atTime) + } + } + + fun setSendBitrate(bitrate: Bandwidth, atTime: Instant) { + assert(bitrate >= Bandwidth.ZERO) + // Reset to avoid being capped by the estimate. + delayBasedLimit = Bandwidth.INFINITY + updateTargetBitrate(bitrate, atTime) + minBitrateHistory.clear() + } + + fun setMinMaxBitrate(minBitrate: Bandwidth, maxBitrate: Bandwidth) { + minBitrateConfigured = max(minBitrate, kCongestionControllerMinBitrate) + if (maxBitrate > Bandwidth.ZERO && maxBitrate.isFinite()) { + maxBitrateConfigured = max(minBitrateConfigured, maxBitrate) + } else { + maxBitrateConfigured = kDefaultMaxBitrate + } + lossBasedBandwidthEstimatorV2.setMinMaxBitrate(minBitrateConfigured, maxBitrateConfigured) + } + + fun getMinBitrate(): Int = minBitrateConfigured.bps.toInt() + + fun setAcknowledgedRate(acknowledgedRate: Bandwidth?, atTime: Instant) { + this.acknowledgedRate = acknowledgedRate + if (acknowledgedRate == null) { + return + } + lossBasedBandwidthEstimatorV2.setAcknowledgedBitrate(acknowledgedRate) + } + + fun updateLossBasedEstimator( + report: TransportPacketsFeedback, + delayDetectorState: BandwidthUsage, + probeBitrate: Bandwidth?, + inAlr: Boolean + ) { + lossBasedBandwidthEstimatorV2.updateBandwidthEstimate( + report.packetFeedbacks, + delayBasedLimit, + inAlr + ) + updateEstimate(report.feedbackTime) + } + + private fun isInStartPhase(atTime: Instant): Boolean = firstReportTime.isInfinite() || + Duration.between(firstReportTime, atTime) <= kStartPhase + + private fun updateUmaStatsPacketsLost(atTime: Instant, packetsLost: Int) { + val bitrateKbps = ((currentTarget.bps + 500) / 1000).kbps + for (i in kUmaRampupMetrics.indices) { + if (!rampupUmaStatsUpdated[i] && + bitrateKbps.kbps >= kUmaRampupMetrics[i].bitrateKbps + ) { + /* TODO: histograms */ + rampupUmaStatsUpdated[i] = true + } + } + if (isInStartPhase(atTime)) { + initiallyLostPackets += packetsLost + } else if (umaUpdateState == UmaState.kNoUpdate) { + umaUpdateState = UmaState.kFirstDone + bitrateAt2Seconds = bitrateKbps + /* TODO: histograms */ + } else if (umaUpdateState == UmaState.kFirstDone && + Duration.between(firstReportTime, atTime) >= kBweConverganceTime + ) { + umaUpdateState = UmaState.kDone + val bitrateDiffKbps = bitrateAt2Seconds.kbps.toInt().coerceAtLeast(0) + /* TODO: histograms */ + } + } + + // Updates history of min bitrates. + // After this method returns min_bitrate_history_.front().second contains the + // min bitrate used during last kBweIncreaseIntervalMs. + private fun updateMinHistory(atTime: Instant) { + // Remove old data points from history. + // Since history precision is in ms, add one so it is able to increase + // bitrate if it is off by as little as 0.5ms. + while (!minBitrateHistory.isEmpty() && + (Duration.between(minBitrateHistory.first().first, atTime) + 1.ms) > kBweIncreaseInterval + ) { + minBitrateHistory.removeFirst() + } + + // Typical minimum sliding-window algorithm: Pop values higher than current + // bitrate before pushing it. + while (!minBitrateHistory.isEmpty() && + currentTarget <= minBitrateHistory.last().second + ) { + minBitrateHistory.removeLast() + } + + minBitrateHistory.add(Pair(atTime, currentTarget)) + } + + // Gets the upper limit for the target bitrate. This is the minimum of the + // delay based limit, the receiver limit and the loss based controller limit. + private fun getUpperLimit(): Bandwidth { + var upperLimit = delayBasedLimit + if (disableReceiverLimitCapsOnly) { + upperLimit = min(upperLimit, receiverLimit) + } + return min(upperLimit, maxBitrateConfigured) + } + + // Prints a warning if `bitrate` if sufficiently long time has past since last + // warning. + private fun maybeLogLowBitrateWarning(bitrate: Bandwidth, atTime: Instant) { + if (Duration.between(lastLowBitrateLog, atTime) > kLowBitrateLogPeriod) { + logger.warn("Estimated available bandwidth $bitrate is below configured min bitrate $minBitrateConfigured.") + lastLowBitrateLog = atTime + } + } + + // Stores an update to the event log if the loss rate has changed, the target + // has changed, or sufficient time has passed since last stored event. + private fun maybeLogLossBasedEvent(atTime: Instant) { + if (currentTarget != lastLoggedTarget || + lastFractionLoss != lastLoggedFractionLoss || + Duration.between(lastRtcEventLog, atTime) > kRtcEventLogPeriod + ) { + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("RtcEventBweUpdateLossBased", atTime) + .addField("currentTarget", currentTarget.bps) + .addField("lastFractionLoss", lastFractionLoss) + .addField("expectedPacketsSinceLastLossUpdate", expectedPacketsSinceLastLossUpdate) + } + lastLoggedFractionLoss = lastFractionLoss + lastLoggedTarget = currentTarget + lastRtcEventLog = atTime + } + } + + // Cap `bitrate` to [min_bitrate_configured_, max_bitrate_configured_] and + // set `current_bitrate_` to the capped value and updates the event log. + private fun updateTargetBitrate(bitrate: Bandwidth, atTime: Instant) { + val newBitrate = min(bitrate, getUpperLimit()) + if (newBitrate < minBitrateConfigured) { + maybeLogLowBitrateWarning(newBitrate, atTime) + } + currentTarget = newBitrate + maybeLogLossBasedEvent(atTime) + linkCapacity.onRateUpdate(acknowledgedRate, currentTarget, atTime) + } + + // Applies lower and upper bounds to the current target rate. + // TODO(srte): This seems to be called even when limits haven't changed, that + // should be cleaned up. + private fun applyTargetLimits(atTime: Instant) { + updateTargetBitrate(currentTarget, atTime) + } + + private enum class UmaState { kNoUpdate, kFirstDone, kDone } + + companion object { + const val disableReceiverLimitCapsOnly = false + } +} + +private data class UmaRampUpMetric( + val metricName: String, + val bitrateKbps: Int +) + +private val kUmaRampupMetrics = arrayOf( + UmaRampUpMetric("WebRTC.BWE.RampUpTimeTo500kbpsInMs", 500), + UmaRampUpMetric("WebRTC.BWE.RampUpTimeTo1000kbpsInMs", 1000), + UmaRampUpMetric("WebRTC.BWE.RampUpTimeTo2000kbpsInMs", 2000) +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SentPacketInfo.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SentPacketInfo.kt new file mode 100644 index 0000000..6a78569 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SentPacketInfo.kt @@ -0,0 +1,37 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.utils.NEVER +import java.time.Instant + +/** Sent packet info, + * based loosely on WebRTC rtc_base/network/sent_packet.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138) + * stripped down to only the fields needed. + */ + +class PacketInfo( + var includedInFeedback: Boolean = false, + var includedInAllocation: Boolean = false, + var packetSizeBytes: Long = 0 +) + +class SentPacketInfo( + var packetId: Long = -1, + var sendTime: Instant = NEVER, + val info: PacketInfo = PacketInfo(), +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TransportFeedbackAdapter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TransportFeedbackAdapter.kt new file mode 100644 index 0000000..fb1d45e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TransportFeedbackAdapter.kt @@ -0,0 +1,454 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:enum-entry-name-case") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.EcnMarking +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.ReceivedPacketInfo +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.RtcpFbCcfbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.ReceivedPacketReport +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.TimeUtils +import org.jitsi.utils.isFinite +import org.jitsi.utils.isInfinite +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.max +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant +import java.util.* + +private val kSendTimeHistoryWindow = 60.secs + +/** Transport feedback adapter, + * based loosely on WebRTC modules/congestion_controller/rtp/transport_feedback_adapter.{h,cc} in + * WebRTC tag branch-heads/6613 (Chromium 128) + * modified to use Jitsi types for objects outside the congestion controller, + * and not using Network Routes. + */ +private data class PacketFeedback( + // Time corresponding to when this object was created. + val creationTime: Instant = Instant.MIN, + val sent: SentPacket, + val ssrc: Long = 0, + val rtpSequenceNumber: Int = 0, + val isRetransmission: Boolean = false +) { + // Jitsi extension: whether the packet was previously reported lost + var reportedLost: Boolean = false +} + +private class InFlightBytesTracker { + fun addInFlightPacketBytes(packet: PacketFeedback) { + require(packet.sent.sendTime.isFinite()) + inFlightData += packet.sent.size + } + + fun removeInFlightPacketBytes(packet: PacketFeedback) { + if (packet.sent.sendTime.isInfinite()) { + return + } + check(packet.sent.size <= inFlightData) + inFlightData -= packet.sent.size + } + + fun getOutstandingData(): DataSize = inFlightData + + var inFlightData: DataSize = DataSize.ZERO +} + +/** TransportFeedbackAdapter converts RTCP feedback packets to RTCP agnostic per + * packet send/receive information. + * It supports [RtcpFbCcfbPacket] according to RFC 8888 and + * [RtcpFbTccPacket] according to + * https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01 + */ +class TransportFeedbackAdapter( + parentLogger: Logger +) { + val logger = parentLogger.createChildLogger(javaClass.name) + + fun addPacket(packet: PacketInfo, tccSeqNum: Long, overheadBytes: DataSize, creationTime: Instant) { + val rtpPacket = packet.packetAs() + // Update seqNumUnwrapper's notion of the ROC, and verify that it's in sync + val truncatedSeqNum = (tccSeqNum and 0xFFFF).toInt() + val unwrappedSeqNum = seqNumUnwrapper.update(truncatedSeqNum) + check(unwrappedSeqNum == tccSeqNum) + val feedback = PacketFeedback( + creationTime = creationTime, + sent = SentPacket( + sequenceNumber = tccSeqNum, + size = packet.packet.length.bytes + overheadBytes, + pacingInfo = packet.probingInfo as? PacedPacketInfo ?: PacedPacketInfo() + ), + ssrc = rtpPacket.ssrc, + rtpSequenceNumber = rtpPacket.sequenceNumber, + isRetransmission = packet.packetOrigin == PacketOrigin.Retransmission + ) + while (history.isNotEmpty() && + Duration.between(history.firstEntry().value.creationTime, creationTime) > kSendTimeHistoryWindow + ) { + // TODO(sprang): Warn if erasing (too many) old items? + if (history.firstEntry().value.sent.sequenceNumber > lastAckSeqNum) { + inFlight.removeInFlightPacketBytes(history.firstEntry().value) + } + val packet = history.firstEntry().value + rtpToTransportSequenceNumber.remove( + SsrcAndRtpSequenceNumber(packet.ssrc, packet.rtpSequenceNumber) + ) + history.remove(history.firstEntry().key) + } + // Note that it can happen that the same SSRC and sequence number is sent + // again. e.g, audio retransmission. + rtpToTransportSequenceNumber[SsrcAndRtpSequenceNumber(feedback.ssrc, feedback.rtpSequenceNumber)] = + feedback.sent.sequenceNumber + history[feedback.sent.sequenceNumber] = feedback + } + + fun processSentPacket(sentPacket: SentPacketInfo): SentPacket? { + val sendTime = sentPacket.sendTime + // TODO(srte): Only use one way to indicate that packet feedback is used. + if (sentPacket.info.includedInFeedback || sentPacket.packetId != -1L) { + val seqNum = sentPacket.packetId + val it = history[seqNum] + if (it != null) { + val packetRetransmit = it.sent.sendTime.isFinite() + it.sent.sendTime = sendTime + lastSendTime = max(lastSendTime, sendTime) + // TODO(srte): Don't do this on retransmit. + if (pendingUntrackedSize != DataSize.ZERO) { + if (sendTime < lastUntrackedSendTime) { + logger.warn( + "appending acknowledged data for out of order packet. (Diff: " + + "${Duration.between(sendTime, lastUntrackedSendTime)}.)" + ) + } + pendingUntrackedSize += sentPacket.info.packetSizeBytes.bytes + } + if (!packetRetransmit) { + if (it.sent.sequenceNumber > lastAckSeqNum) { + inFlight.addInFlightPacketBytes(it) + it.sent.dataInFlight = inFlight.getOutstandingData() + return it.sent + } + } + } + } else if (sentPacket.info.includedInAllocation) { + if (sendTime < lastSendTime) { + logger.warn("ignoring untracked data for out of order packet") + } + pendingUntrackedSize += sentPacket.info.packetSizeBytes.bytes + lastUntrackedSendTime = max(lastUntrackedSendTime, sendTime) + } + return null + } + + /** Cumulative count of feedback reports that could not be matched against the send-time + * history (see [TransportCcEngine.StatisticsSnapshot.unmatchedFeedback]). */ + var totalUnmatchedReports: Long = 0 + private set + + fun processTransportFeedback(feedback: RtcpFbTccPacket, feedbackReceiveTime: Instant): TransportPacketsFeedback? { + if (feedback.GetPacketStatusCount() == 0) { + logger.info { "Empty transport feedback packet received" } + return null + } + + // Add timestamp deltas to a local time base selected on first packet arrival. + // This won't be the true time base, but makes it easier to manually inspect + // time stamps. + if (lastTransportFeedbackBaseTime.isInfinite()) { + currentOffset = feedbackReceiveTime + } else { + val delta = feedback.GetBaseDelta(lastTransportFeedbackBaseTime) + // Protect against assigning current_offset_ negative value. + if (delta < Duration.between(currentOffset, Instant.EPOCH)) { + logger.warn("Unexpected feedback timestamp received: $feedbackReceiveTime") + currentOffset = feedbackReceiveTime + } else { + currentOffset += delta + } + } + lastTransportFeedbackBaseTime = feedback.BaseTime() + + val packetResultVector = ArrayList() + packetResultVector.ensureCapacity(feedback.GetPacketStatusCount()) + + var failedLookups = 0 + + var deltaSinceBase = Duration.ZERO + + feedback.forEach { report -> + val seqNum = seqNumUnwrapper.interpret(report.seqNum) + + val packetFeedback = retrievePacketFeedback(seqNum, received = report is ReceivedPacketReport) + if (packetFeedback == null) { + ++failedLookups + return@forEach + } + + val previouslyReportedLost = packetFeedback.reportedLost + + val result = PacketResult() + result.sentPacket = packetFeedback.sent + + if (report is ReceivedPacketReport) { + deltaSinceBase += report.deltaDuration + result.receiveTime = currentOffset + deltaSinceBase + history.remove(seqNum) + } else { + packetFeedback.reportedLost = true + } + result.rtpPacketInfo = PacketResult.RtpPacketInfo( + packetFeedback.ssrc, + packetFeedback.rtpSequenceNumber, + packetFeedback.isRetransmission + ) + result.previouslyReportedLost = previouslyReportedLost + packetResultVector.add(result) + } + + totalUnmatchedReports += failedLookups + if (failedLookups > 0) { + logger.info( + "Failed to lookup send time for $failedLookups packet${if (failedLookups > 1) "s" else ""}. " + + "Packets reordered or send time history too small?" + ) + } + + return toTransportFeedback(packetResultVector, feedbackReceiveTime, supportsEcn = false) + } + + fun processCongestionControlFeedback( + feedback: RtcpFbCcfbPacket, + feedbackReceiveTime: Instant + ): TransportPacketsFeedback? { + if (feedback.packets.isEmpty()) { + logger.info { "Empty congestion control feedback packet received" } + return null + } + if (currentOffset.isInfinite()) { + currentOffset = feedbackReceiveTime + } + val feedbackDelta = lastFeedbackCompactNtpTime?.let { + Duration.ofMillis( + TimeUtils.ntpShortToMs(feedback.reportTimestampCompactNtp) - + TimeUtils.ntpShortToMs(it) + ) + } ?: Duration.ZERO + lastFeedbackCompactNtpTime = feedback.reportTimestampCompactNtp + if (feedbackDelta < Duration.ZERO) { + logger.warn("Unexpected feedback ntp time delta $feedbackDelta") + currentOffset = feedbackReceiveTime + } else { + currentOffset += feedbackDelta + } + + var failedLookups = 0 + var supportsEcn = true + val packetResultVector = mutableListOf() + feedback.packets.forEach { packetInfo -> + val packetFeedback = retrievePacketFeedback( + SsrcAndRtpSequenceNumber(ssrc = packetInfo.ssrc, packetInfo.sequenceNumber), + received = packetInfo is ReceivedPacketInfo + ) + if (packetFeedback == null) { + ++failedLookups + return@forEach + } + val result = PacketResult() + result.sentPacket = packetFeedback.sent + if (packetInfo is ReceivedPacketInfo) { + result.receiveTime = currentOffset - packetInfo.arrivalTimeOffset + result.ecn = packetInfo.ecn + supportsEcn = supportsEcn && packetInfo.ecn != EcnMarking.kNotEct + } + result.rtpPacketInfo = PacketResult.RtpPacketInfo( + ssrc = packetFeedback.ssrc, + rtpSequenceNumber = packetFeedback.rtpSequenceNumber, + isRetransmission = packetFeedback.isRetransmission, + ) + packetResultVector.add(result) + } + + totalUnmatchedReports += failedLookups + if (failedLookups > 0) { + logger.warn { + "Failed to lookup send time for $failedLookups packets. " + + "Packets reordered or send time history too small?" + } + } + + // Feedback is expected to be sorted in send order. + packetResultVector.sortWith { lhs, rhs -> + compareValuesBy(lhs, rhs) { it.sentPacket.sequenceNumber } + } + + return toTransportFeedback(packetResultVector, feedbackReceiveTime, supportsEcn) + } + + fun getOutstandingData() = inFlight.getOutstandingData() + + private data class SsrcAndRtpSequenceNumber( + val ssrc: Long, + val rtpSequenceNumber: Int + ) : Comparable { + override fun compareTo(other: SsrcAndRtpSequenceNumber): Int = + compareValuesBy(this, other, { it.ssrc }, { it.rtpSequenceNumber }) + } + + private fun toTransportFeedback( + packetResults: MutableList, + feedbackReceiveTime: Instant, + supportsEcn: Boolean + ): TransportPacketsFeedback? { + val msg = TransportPacketsFeedback() + msg.feedbackTime = feedbackReceiveTime + if (packetResults.isEmpty()) { + return null + } + msg.packetFeedbacks = packetResults + msg.dataInFlight = inFlight.getOutstandingData() + msg.transportSupportsEcn = supportsEcn + + return msg + } + + private fun retrievePacketFeedback(transportSeqNum: Long, received: Boolean): PacketFeedback? { + if (transportSeqNum > lastAckSeqNum) { + // Starts at history_.begin() if last_ack_seq_num_ < 0, since any + // valid sequence number is >= 0. + for (it in history.subMap(lastAckSeqNum, false, transportSeqNum, true)) { + inFlight.removeInFlightPacketBytes(it.value) + } + lastAckSeqNum = transportSeqNum + } + + val packetFeedback = history[transportSeqNum] + + if (packetFeedback == null) { + logger.debug { + "No history entry found for seqNum $transportSeqNum" + } + return null + } + + if (packetFeedback.sent.sendTime.isInfinite()) { + // TODO(srte): Fix the tests that makes this happen and make this a + // DCHECK. + logger.error( + "Received feedback before packet with seqNum $transportSeqNum was indicated as sent" + ) + return null + } + + if (received) { + // Note: Lost packets are not removed from history because they might + // be reported as received by a later feedback. + rtpToTransportSequenceNumber.remove( + SsrcAndRtpSequenceNumber(packetFeedback.ssrc, packetFeedback.rtpSequenceNumber) + ) + history.remove(transportSeqNum) + } + + return packetFeedback + } + + private fun retrievePacketFeedback(key: SsrcAndRtpSequenceNumber, received: Boolean): PacketFeedback? { + val transportSeqNum = rtpToTransportSequenceNumber[key] + if (transportSeqNum == null) { + return null + } + return retrievePacketFeedback(transportSeqNum, received) + } + + private var pendingUntrackedSize = DataSize.ZERO + private var lastSendTime = Instant.MIN + private var lastUntrackedSendTime = Instant.MIN + private val seqNumUnwrapper = RtpSequenceIndexTracker() + + private var lastAckSeqNum = -1L + private val inFlight = InFlightBytesTracker() + + private var currentOffset = Instant.MIN + + // `last_transport_feedback_base_time` is only used for transport feedback to + // track base time. + private var lastTransportFeedbackBaseTime = Instant.MIN + + // Used by RFC 8888 congestion control feedback to track base time. + private var lastFeedbackCompactNtpTime: Long? = null + + // Map SSRC and RTP sequence number to transport sequence number. + private val rtpToTransportSequenceNumber = TreeMap() + private val history = TreeMap() + + /** Jitsi local */ + fun getStatisitics(): StatisticsSnapshot = StatisticsSnapshot( + inFlight.inFlightData, + pendingUntrackedSize, + lastSendTime, + lastUntrackedSendTime, + lastAckSeqNum, + history.size, + currentOffset, + lastTransportFeedbackBaseTime, + totalUnmatchedReports, + ) + + class StatisticsSnapshot( + val inFlight: DataSize, + val pendingUntrackedSize: DataSize, + val lastSendTime: Instant, + val lastUntrackedSendTime: Instant, + val lastAckSeqNum: Long, + val historySize: Int, + val currentOffset: Instant, + val lastTransportFeedbackBaseTime: Instant, + val totalUnmatchedReports: Long + ) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("in_flight_bytes", inFlight.bytes) + put("pending_untracked_size", pendingUntrackedSize.bytes) + put("last_send_time", lastSendTime.toEpochMilliOrInf().toString()) + put("last_untracked_send_time", lastUntrackedSendTime.toEpochMilliOrInf().toString()) + put("last_ack_seq_num", lastAckSeqNum) + put("history_size", historySize) + put("current_offset", currentOffset.toEpochMilliOrInf().toString()) + put("last_transport_feedback_base_time", lastTransportFeedbackBaseTime.toEpochMilliOrInf().toString()) + put("total_unmatched_reports", totalUnmatchedReports) + } + } +} + +private fun Instant.toEpochMilliOrInf(): Number = try { + this.toEpochMilli() +} catch (e: ArithmeticException) { + if (this < Instant.EPOCH) { + Double.NEGATIVE_INFINITY + } else { + Double.POSITIVE_INFINITY + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TrendlineEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TrendlineEstimator.kt new file mode 100644 index 0000000..c215081 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TrendlineEstimator.kt @@ -0,0 +1,311 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import kotlin.collections.ArrayDeque +import kotlin.math.abs +import kotlin.math.min + +/** + * Trendline-based delay increase detector + * * + * Based on WebRTC modules/congestion_controller/goog_cc/trendline_estimator.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class TrendlineEstimator( + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) : DelayIncreaseDetectorInterface { + private val settings = TrendlineEstimatorSettings() + + // Parameters + private val smoothingCoef = kDefaultTrendlineSmoothingCoeff + private val thresholdGain = kDefaultTrendlineThresholdGain + + // Used by the existing threshold. + private var numOfDeltas: Int = 0 + + // Keep the arrival times small by using the change from the first packet + private var firstArrivalTimeMs: Long = -1 + + // Exponential backoff filtering. + private var accumulatedDelay: Double = 0.0 + private var smoothedDelay: Double = 0.0 + + // Linear least squares regression + private val delayHist = ArrayDeque() + + private val kUp = 0.0087 + private val kDown = 0.039 + private val overusingTimeThreshold = kOverusingTimeThreshold + var threshold = 12.5 + private set + + var prevModifiedTrend = Double.NaN + private set + + private var lastUpdateMs = -1L + var prevTrend = 0.0 + private set + private var timeOverUsing = -1.0 + private var overuseCounter = 0 + private var hypothesis = BandwidthUsage.kBwNormal + + // Only used with networkStatePredictor + // private var hypothesisPredicted = BandwidthUsage.kBwNormal + + // private var networkStatePredictor: NetworkStatePredictor? = null + + fun updateTrendline( + recvDeltaMs: Double, + sendDeltaMs: Double, + sendTimeMs: Long, + arrivalTimeMs: Long, + packetSize: Long + ) { + val deltaMs = recvDeltaMs - sendDeltaMs + ++numOfDeltas + numOfDeltas = min(numOfDeltas, kDeltaCounterMax) + if (firstArrivalTimeMs == -1L) { + firstArrivalTimeMs = arrivalTimeMs + } + + // Exponential backoff filter. + accumulatedDelay += deltaMs + smoothedDelay = smoothingCoef * smoothedDelay + (1 - smoothingCoef) * accumulatedDelay + + // Maintain packet window + delayHist.addLast( + PacketTiming((arrivalTimeMs - firstArrivalTimeMs).toDouble(), smoothedDelay, accumulatedDelay) + ) + if (settings.enableSort) { + // TODO - not currently enabled + } + if (delayHist.size > settings.windowSize) { + delayHist.removeFirst() + } + + // Simple linear regression. + var trend = prevTrend + if (delayHist.size == settings.windowSize) { + // Update trend_ if it is possible to fit a line to the data. The delay + // trend can be seen as an estimate of (send_rate - capacity)/capacity. + // 0 < trend < 1 -> the delay increases, queues are filling up + // trend == 0 -> the delay does not change + // trend < 0 -> the delay decreases, queues are being emptied + trend = linearFitSlope(delayHist) ?: trend + if (settings.enableCap) { + val cap = computeSlopeCap(delayHist, settings) + // We only use the cap to filter out overuse detections, not + // to detect additional underuses. + if (trend >= 0 && cap != null && trend > cap) { + trend = cap + } + } + } + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("trendline_updated") + .addField("accumulated_delay_ms", accumulatedDelay) + .addField("smoothed_delay_ms", smoothedDelay) + } + + detect(trend, sendDeltaMs, arrivalTimeMs) + } + + override fun update( + recvDeltaMs: Double, + sendDeltaMs: Double, + sendTimeMs: Long, + arrivalTimeMs: Long, + packetSize: Long, + calculatedDeltas: Boolean + ) { + if (calculatedDeltas) { + updateTrendline(recvDeltaMs, sendDeltaMs, sendTimeMs, arrivalTimeMs, packetSize) + } + /* if (networkStatePredictor != null) { + hypothesisPredicted = networkStatePredictor.update(sendTimeMs, arrivalTimeMs, hypothesis) + } + */ + } + + override fun state(): BandwidthUsage { + /* if (networkStatePredictor != null) { + return hypothesisPredicted + */ + return hypothesis + } + + private fun detect(trend: Double, tsDelta: Double, nowMs: Long) { + if (numOfDeltas < 2) { + hypothesis = BandwidthUsage.kBwNormal + return + } + val modifiedTrend = min(numOfDeltas, kMinNumDeltas) * trend * thresholdGain + prevModifiedTrend = modifiedTrend + timeSeriesLogger.trace { + diagnosticContext.makeTimeSeriesPoint("trendline_detect", nowMs) + .addField("trend", modifiedTrend) + .addField("threshold", threshold) + } + if (modifiedTrend > threshold) { + if (timeOverUsing == -1.0) { + // Initialize the timer. Assume that we've been + // over-using half of the time since the previous + // sample. + timeOverUsing = tsDelta / 2 + } else { + // Increment timer + timeOverUsing += tsDelta + } + overuseCounter++ + if (timeOverUsing > overusingTimeThreshold && overuseCounter > 1) { + if (trend >= prevTrend) { + timeOverUsing = 0.0 + overuseCounter = 0 + hypothesis = BandwidthUsage.kBwOverusing + } + } + } else if (modifiedTrend < -threshold) { + timeOverUsing = -1.0 + overuseCounter = 0 + hypothesis = BandwidthUsage.kBwUnderusing + } else { + timeOverUsing = -1.0 + overuseCounter = 0 + hypothesis = BandwidthUsage.kBwNormal + } + prevTrend = trend + updateThreshold(modifiedTrend, nowMs) + } + + private fun updateThreshold(modifiedTrend: Double, nowMs: Long) { + if (lastUpdateMs == -1L) { + lastUpdateMs = nowMs + } + + if (abs(modifiedTrend) > threshold + kMaxAdaptOffsetMs) { + // Avoid adapting the threshold to big latency spikes, caused e.g., + // by a sudden capacity drop. + lastUpdateMs = nowMs + return + } + + val k = if (abs(modifiedTrend) < threshold) { + kDown + } else { + kUp + } + val kMaxTimeDeltaMs = 100L + val timeDeltaMs = min(nowMs - lastUpdateMs, kMaxTimeDeltaMs) + threshold += k * (abs(modifiedTrend) - threshold) * timeDeltaMs + threshold = threshold.coerceIn(6.0, 600.0) + lastUpdateMs = nowMs + } + + companion object { + const val kDefaultTrendlineSmoothingCoeff = 0.9 + const val kDefaultTrendlineThresholdGain = 4.0 + + const val kMaxAdaptOffsetMs = 15.0 + const val kOverusingTimeThreshold = 10.0 + const val kMinNumDeltas = 60 + const val kDeltaCounterMax = 1000 + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(TrendlineEstimator::class.java) + } +} + +/* TODO: this class is redundant if we don't have field trial settings - remove it? */ +private class TrendlineEstimatorSettings() { + // Sort the packets in the window. Should be redundant, + // but then almost no cost. + val enableSort = false + + // Cap the trendline slope based on the minimum delay seen + // in the beginning_packets and end_packets respectively. + val enableCap = false + + val beginningPackets = 7 + val endPackets = 7 + val capUncertainty = 0.0 + + val windowSize = kDefaultTrendlineWindowSize + + companion object { + const val kDefaultTrendlineWindowSize = 20 + } +} + +private data class PacketTiming( + val arrivalTimeMs: Double, + val smoothedDelayMs: Double, + val rawDelayMs: Double +) + +private fun linearFitSlope(packets: ArrayDeque): Double? { + check(packets.size >= 2) + // Compute the "center of mass" + var sumX = 0.0 + var sumY = 0.0 + for (packet in packets) { + sumX += packet.arrivalTimeMs + sumY += packet.smoothedDelayMs + } + val xAvg = sumX / packets.size + val yAvg = sumY / packets.size + // Compute the slope k = \sum (x_i-x_avg)(y_i-y_avg) / \sum (x_i-x_avg)^2 + var numerator = 0.0 + var denominator = 0.0 + for (packet in packets) { + val x = packet.arrivalTimeMs + val y = packet.smoothedDelayMs + numerator += (x - xAvg) * (y - yAvg) + denominator += (x - xAvg) * (x - xAvg) + } + if (denominator == 0.0) { + return null + } + return numerator / denominator +} + +private fun computeSlopeCap(packets: ArrayDeque, settings: TrendlineEstimatorSettings): Double? { + check(1 <= settings.beginningPackets && settings.beginningPackets < packets.size) + check(1 <= settings.endPackets && settings.endPackets < packets.size) + check(settings.beginningPackets + settings.endPackets <= packets.size) + var early = packets.first() + for (i in 1 until settings.beginningPackets) { + if (packets[i].rawDelayMs < early.rawDelayMs) { + early = packets[i] + } + } + val lateStart = packets.size - settings.endPackets + var late = packets[lateStart] + for (i in lateStart + 1 until packets.size) { + if (packets[i].rawDelayMs < late.rawDelayMs) { + late = packets[i] + } + } + if (late.arrivalTimeMs - early.arrivalTimeMs < 1) { + return null + } + return (late.rawDelayMs - early.rawDelayMs) / (late.arrivalTimeMs - early.arrivalTimeMs) + settings.capUncertainty +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt new file mode 100644 index 0000000..0f83e7e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/VideoCodecParser.kt @@ -0,0 +1,43 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.rtp.VideoRtpPacket + +/** + * Abstract interface to the codec-specific parsing of a video source. + * + * Typically this will set layer information as interpreted from received packets, + * and verify stream consistency. + */ +abstract class VideoCodecParser( + var source: MediaSourceDesc +) { + abstract fun parse(packetInfo: PacketInfo) + + protected fun findRtpEncodingDesc(packet: VideoRtpPacket): RtpEncodingDesc? { + source.findRtpEncodingDesc(packet.ssrc)?.let { + return it + } + return null + } + + protected fun findRtpLayerDescs(packet: VideoRtpPacket) = source.findRtpLayerDescs(packet) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt new file mode 100644 index 0000000..b7ed67e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacket.kt @@ -0,0 +1,231 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorHeaderExtension +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorStatelessSubset +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyException +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.rtp.rtp.header_extensions.FrameInfo +import org.jitsi.utils.logging2.Logger +import kotlin.math.min + +/** A video packet carrying an AV1 Dependency Descriptor. Note that this may or may not be an actual AV1 packet; + * other video codecs can also carry the AV1 DD. + */ +class Av1DDPacket : ParsedVideoPacket { + var descriptor: Av1DependencyDescriptorHeaderExtension? + val statelessDescriptor: Av1DependencyDescriptorStatelessSubset + val frameInfo: FrameInfo? + val av1DDHeaderExtensionId: Int + + private constructor( + buffer: ByteArray, + offset: Int, + length: Int, + av1DDHeaderExtensionId: Int, + encodingId: Int, + descriptor: Av1DependencyDescriptorHeaderExtension?, + statelessDescriptor: Av1DependencyDescriptorStatelessSubset, + frameInfo: FrameInfo? + ) : super(buffer, offset, length, encodingId) { + this.descriptor = descriptor + this.statelessDescriptor = statelessDescriptor + this.frameInfo = frameInfo + this.av1DDHeaderExtensionId = av1DDHeaderExtensionId + } + + constructor( + packet: RtpPacket, + av1DDHeaderExtensionId: Int, + templateDependencyStructure: Av1TemplateDependencyStructure?, + logger: Logger + ) : super(packet.buffer, packet.offset, packet.length, RtpLayerDesc.SUSPENDED_ENCODING_ID) { + this.av1DDHeaderExtensionId = av1DDHeaderExtensionId + val ddExt = packet.getHeaderExtension(av1DDHeaderExtensionId) + requireNotNull(ddExt) { + "Packet did not have Dependency Descriptor" + } + val parser = Av1DependencyDescriptorReader(ddExt) + descriptor = try { + parser.parse(templateDependencyStructure) + } catch (e: Av1DependencyException) { + logger.warn( + "Could not parse AV1 Dependency Descriptor for ssrc ${packet.ssrc} seq ${packet.sequenceNumber}: " + + e.message + ) + null + } + statelessDescriptor = descriptor ?: parser.parseStateless() + frameInfo = try { + descriptor?.frameInfo + } catch (e: Av1DependencyException) { + logger.warn( + "Could not extract frame info from AV1 Dependency Descriptor for " + + "ssrc ${packet.ssrc} seq ${packet.sequenceNumber}: ${e.message}" + ) + null + } + } + + /* "template_dependency_structure_present_flag MUST be set to 1 for the first packet of a coded video sequence, + * and MUST be set to 0 otherwise" + */ + override val isKeyframe: Boolean + get() = statelessDescriptor.newTemplateDependencyStructure != null + + override val isStartOfFrame: Boolean + get() = statelessDescriptor.startOfFrame + + override val isEndOfFrame: Boolean + get() = statelessDescriptor.endOfFrame + + override fun meetsRoutingNeeds(): Boolean = + true // If it didn't parse as AV1 we would have failed in the constructor + + override val layerIds: Collection + get() = frameInfo?.dtisPresent + ?: run { super.layerIds } + + val frameNumber + get() = statelessDescriptor.frameNumber + + val activeDecodeTargets + get() = descriptor?.activeDecodeTargetsBitmask + + override fun toString(): String = buildString { + append(super.toString()) + append(", DTIs=${frameInfo?.dtisPresent}") + activeDecodeTargets?.let { append(", ActiveTargets=$it") } + } + + override fun clone(): Av1DDPacket { + val descriptor = descriptor?.clone() + val statelessDescriptor = descriptor ?: statelessDescriptor.clone() + return Av1DDPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length, + av1DDHeaderExtensionId = av1DDHeaderExtensionId, + encodingId = encodingId, + descriptor = descriptor, + statelessDescriptor = statelessDescriptor, + frameInfo = frameInfo + ).also { postClone(it) } + } + + fun getScalabilityStructure(eid: Int = 0, baseFrameRate: Double = 30.0): RtpEncodingDesc? { + val descriptor = this.descriptor + requireNotNull(descriptor) { + "Can't get scalability structure from packet without a descriptor" + } + return descriptor.getScalabilityStructure(ssrc, eid, baseFrameRate) + } + + /** Re-encode the current descriptor to the header extension. For use after modifying it. */ + fun reencodeDdExt() { + val descriptor = this.descriptor + requireNotNull(descriptor) { + "Can't re-encode extension from a packet without a descriptor" + } + + var ext = getHeaderExtension(av1DDHeaderExtensionId) + if (ext == null || ext.dataLengthBytes != descriptor.encodedLength) { + removeHeaderExtension(av1DDHeaderExtensionId) + ext = addHeaderExtension(av1DDHeaderExtensionId, descriptor.encodedLength) + } + descriptor.write(ext) + } +} + +fun Av1DependencyDescriptorHeaderExtension.getScalabilityStructure( + ssrc: Long, + eid: Int = 0, + baseFrameRate: Double = 30.0 +): RtpEncodingDesc? { + val activeDecodeTargetsBitmask = this.activeDecodeTargetsBitmask + ?: // Can't get scalability structure from dependency descriptor that doesn't specify decode targets + return null + + val layerCounts = Array(structure.maxSpatialId + 1) { + IntArray(structure.maxTemporalId + 1) + } + + // Figure out the frame rates per spatial/temporal layer. + structure.templateInfo.forEach { t -> + if (!t.hasInterPictureDependency()) { + // This is a template that doesn't reference any previous frames, so is probably a key frame or + // part of the same temporal picture with one, i.e. not part of the regular structure. + return@forEach + } + layerCounts[t.spatialId][t.temporalId]++ + } + + // Sum up counts per spatial layer + layerCounts.forEach { a -> + var total = 0 + for (i in a.indices) { + val entry = a[i] + a[i] += total + total += entry + } + } + + val maxFrameGroup = layerCounts.maxOf { it.maxOrNull()!! } + + val layers = ArrayList() + + structure.decodeTargetLayers.forEachIndexed { i, dt -> + if (!activeDecodeTargetsBitmask.containsDecodeTarget(i)) { + return@forEachIndexed + } + // Treat the lesser of width and height as the height in order to handle portrait-mode video correctly + val height = structure.maxRenderResolutions.getOrNull(dt.spatialId)?.let { min(it.width, it.height) } ?: -1 + + // Calculate the fraction of this spatial layer's framerate this DT comprises. + val frameRate = baseFrameRate * layerCounts[dt.spatialId][dt.temporalId] / maxFrameGroup + + layers.add(Av1DDRtpLayerDesc(eid, i, dt.temporalId, dt.spatialId, height, frameRate)) + } + return RtpEncodingDesc(ssrc, layers.toArray(arrayOf()), eid) +} + +/** Check whether an activeDecodeTargetsBitmask contains a specific decode target. */ +fun Int.containsDecodeTarget(dt: Int) = ((1 shl dt) and this) != 0 + +/** + * Returns the delta between two AV1 templateID values, taking into account + * rollover. This will return the 'positive' delta between the two + * picture IDs in the form of the number you'd add to b to get a. e.g.: + * getTl0PicIdxDelta(1, 10) -> 55 (10 + 55 = 1) + * getTl0PicIdxDelta(1, 58) -> 7 (58 + 7 = 1) + */ +fun getTemplateIdDelta(a: Int, b: Int): Int = (a - b + 64) % 64 + +/** + * Apply a delta to a given templateID and return the result (taking + * rollover into account) + * @param start the starting templateID + * @param delta the delta to be applied + * @return the templateID resulting from doing "start + delta" + */ +fun applyTemplateIdDelta(start: Int, delta: Int): Int = (start + delta) % 64 diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt new file mode 100644 index 0000000..d93b4ae --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDParser.kt @@ -0,0 +1,179 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.nlj.util.TreeCache +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.utils.LRUCache +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +/** + * Some [Av1DDPacket] fields are not able to be determined by looking at a single packet with an AV1 DD + * (for example the template dependency structure is only carried in keyframes). This class updates the layer + * descriptions with information from frames, and also diagnoses packet format variants that the Jitsi videobridge + * won't be able to route. + */ +class Av1DDParser( + source: MediaSourceDesc, + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) : VideoCodecParser(source) { + private val logger = createChildLogger(parentLogger) + + /** History of AV1 templates. */ + private val ddStateHistory = LRUCache(STATE_HISTORY_SIZE, true) + + fun createFrom(packet: RtpPacket, av1DdExtId: Int): Av1DDPacket { + val history = ddStateHistory.getOrPut(packet.ssrc) { + TemplateHistory(TEMPLATE_HISTORY_SIZE) + } + + val priorEntry = history.get(packet.sequenceNumber) + + val priorStructure = priorEntry?.value?.structure?.clone() + + val av1Packet = Av1DDPacket(packet, av1DdExtId, priorStructure, logger) + + val newStructure = av1Packet.descriptor?.newTemplateDependencyStructure + if (newStructure != null) { + val structureChanged = newStructure.templateIdOffset != priorStructure?.templateIdOffset + history.insert(packet.sequenceNumber, Av1DdInfo(newStructure.clone(), structureChanged)) + logger.debug { + "Inserting new structure with templates ${newStructure.templateIdOffset} .. " + + "${(newStructure.templateIdOffset + newStructure.templateCount - 1) % 64} " + + "for RTP packet ssrc ${packet.ssrc} seq ${packet.sequenceNumber}. " + + "Changed from previous: $structureChanged." + } + } + + if (timeSeriesLogger.isTraceEnabled) { + val point = diagnosticContext + .makeTimeSeriesPoint("av1_parser") + .addField("rtp.ssrc", packet.ssrc) + .addField("rtp.seq", packet.sequenceNumber) + .addField("rtp.timestamp", packet.timestamp) + .addField("av1_parser.key", priorEntry?.key) + .addField("av1.startOfFrame", av1Packet.statelessDescriptor.startOfFrame) + .addField("av1.endOfFrame", av1Packet.statelessDescriptor.endOfFrame) + .addField("av1.templateId", av1Packet.statelessDescriptor.frameDependencyTemplateId) + .addField("av1.frameNum", av1Packet.statelessDescriptor.frameNumber) + .addField("av1.frameInfo", av1Packet.frameInfo?.toString()) + .addField("av1.structure", newStructure != null) + .addField("av1.activeTargets", av1Packet.descriptor?.activeDecodeTargetsBitmask) + val packetStructure = av1Packet.descriptor?.structure + if (packetStructure != null) { + point.addField("av1.structureIdOffset", packetStructure.templateIdOffset) + .addField("av1.templateCount", packetStructure.templateCount) + .addField("av1.structureId", System.identityHashCode(packetStructure)) + } + if (newStructure != null) { + point.addField("av1.newStructureIdOffset", newStructure.templateIdOffset) + .addField("av1.newTemplateCount", newStructure.templateCount) + .addField("av1.newStructureId", System.identityHashCode(newStructure)) + } + timeSeriesLogger.trace(point) + } + + return av1Packet + } + + override fun parse(packetInfo: PacketInfo) { + val av1Packet = packetInfo.packetAs() + val history = ddStateHistory[av1Packet.ssrc] + + if (history == null) { + /* Probably getting spammed with SSRCs? */ + logger.warn("History for ${av1Packet.ssrc} disappeared between createFrom and parse!") + return + } + + val activeDecodeTargets = av1Packet.activeDecodeTargets + + if (activeDecodeTargets != null) { + val changed = history.updateDecodeTargets(av1Packet.sequenceNumber, activeDecodeTargets) + + if (changed) { + packetInfo.layeringChanged = true + logger.debug { + "Decode targets for ${av1Packet.ssrc} changed in seq ${av1Packet.sequenceNumber}: " + + "now 0x${Integer.toHexString(activeDecodeTargets)}. Updating layering." + } + + findRtpEncodingDesc(av1Packet)?.let { enc -> + av1Packet.getScalabilityStructure(eid = enc.eid)?.let { + source.setEncodingLayers(it.layers, av1Packet.ssrc) + } + for (otherEnc in source.rtpEncodings) { + if (!ddStateHistory.keys.contains(otherEnc.primarySSRC)) { + source.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) + } + } + } + } + } + } + + companion object { + const val STATE_HISTORY_SIZE = 500 + const val TEMPLATE_HISTORY_SIZE = 500 + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(Av1DDParser::class.java) + } +} + +class TemplateHistory(minHistory: Int) { + private val indexTracker = RtpSequenceIndexTracker() + private val history = TreeCache(minHistory) + private var latestDecodeTargets = -1 + private var latestDecodeTargetIndex = -1L + + fun get(seqNo: Int): Map.Entry? { + val index = indexTracker.update(seqNo) + return history.getEntryBefore(index) + } + + fun insert(seqNo: Int, value: Av1DdInfo) { + val index = indexTracker.update(seqNo) + return history.insert(index, value) + } + + /** Update the current decode targets. + * Return true if the decode target set or the template structure has changed. */ + fun updateDecodeTargets(seqNo: Int, decodeTargets: Int): Boolean { + val index = indexTracker.update(seqNo) + if (index < latestDecodeTargetIndex) { + return false + } + val changed = decodeTargets != latestDecodeTargets || history.get(index)?.changed == true + latestDecodeTargetIndex = index + latestDecodeTargets = decodeTargets + return changed + } +} + +data class Av1DdInfo( + val structure: Av1TemplateDependencyStructure, + val changed: Boolean +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt new file mode 100644 index 0000000..b61c51c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt @@ -0,0 +1,123 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.codec.av1 + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.stats.NodeStatsBlock + +/** + * An RtpLayerDesc of the type needed to describe AV1 DD scalability. + */ + +class Av1DDRtpLayerDesc( + /** + * The index of this instance's encoding in the source encoding array. + */ + eid: Int, + /** + * The decoding target of this instance, or negative for unknown. + */ + val dt: Int, + /** + * The temporal layer ID of this instance,. + */ + tid: Int, + /** + * The spatial layer ID of this instance. + */ + sid: Int, + /** + * The max height of the bitstream that this instance represents. The actual + * height may be less due to bad network or system load. + */ + + height: Int, + /** + * The max frame rate (in fps) of the bitstream that this instance + * represents. The actual frame rate may be less due to bad network or + * system load. + */ + frameRate: Double, +) : RtpLayerDesc(eid, tid, sid, height, frameRate) { + override fun copy(height: Int, tid: Int, inherit: Boolean): RtpLayerDesc = + Av1DDRtpLayerDesc(eid, dt, tid, sid, height, frameRate).also { + if (inherit) { + it.inheritFrom(this) + } + } + + override val layerId = dt + override val index = getIndex(eid, dt) + + override fun getBitrate(nowMs: Long) = layerRate(nowMs, smoothed = false) + + override fun getSmoothedBitrate(nowMs: Long) = layerRate(nowMs, smoothed = true) + + override fun hasZeroBitrate(nowMs: Long) = bitrateTracker.getAccumulatedSize(nowMs).bits == 0L + + /** + * Extracts a [NodeStatsBlock] from an [RtpLayerDesc]. + */ + override fun debugState() = super.debugState().apply { + put("dt", dt) + } + + override fun indexString(): String = indexString(index) + + /** + * {@inheritDoc} + */ + override fun toString(): String = "subjective_quality=$index,DT=$dt,height=$height,frameRate=$frameRate" + + companion object { + /** + * The index value that is used to represent that forwarding is suspended. + */ + const val SUSPENDED_INDEX = -1 + + const val SUSPENDED_DT = -1 + + /** + * Calculate the "index" of a layer based on its encoding and decode target. + * This is a server-side id and should not be confused with any encoding id defined + * in the client (such as the rid) or the encodingId. This is used by the videobridge's + * adaptive source projection for filtering. + */ + @JvmStatic + fun getIndex(eid: Int, dt: Int): Int { + val e = if (eid < 0) 0 else eid + val d = if (dt < 0) 0 else dt + + return (e shl 6) or d + } + + /** + * Get a decode target ID from a layer index. If the index is [SUSPENDED_INDEX], + * the value is unspecified. + */ + @JvmStatic + fun getDtFromIndex(index: Int) = if (index == SUSPENDED_INDEX) SUSPENDED_DT else index and 0x3f + + /** + * Get a string description of a layer index. + */ + fun indexString(index: Int): String = if (index == SUSPENDED_INDEX) { + "SUSP" + } else { + "E${getEidFromIndex(index)}DT${getDtFromIndex(index)}" + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt new file mode 100644 index 0000000..7f3c28c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Packet.kt @@ -0,0 +1,169 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.vp8 + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.codec.vp8.Vp8Utils +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.logging2.cwarn +import org.jitsi_modified.impl.neomedia.codec.video.vp8.DePacketizer + +/** + * If this [Vp8Packet] instance is being created via a clone, + * we've already parsed the packet itself and determined whether + * or not its a keyframe and what its spatial layer index is, + * so the constructor allows passing in those values if + * they're already known. If they're null, this instance + * will do the parsing itself. + */ +class Vp8Packet private constructor( + buffer: ByteArray, + offset: Int, + length: Int, + isKeyframe: Boolean?, + isStartOfFrame: Boolean?, + encodingId: Int, + height: Int?, + pictureId: Int?, + TL0PICIDX: Int? +) : ParsedVideoPacket(buffer, offset, length, encodingId) { + + constructor( + buffer: ByteArray, + offset: Int, + length: Int + ) : this( + buffer, offset, length, + isKeyframe = null, + isStartOfFrame = null, + encodingId = RtpLayerDesc.SUSPENDED_ENCODING_ID, + height = null, + pictureId = null, + TL0PICIDX = null + ) + + /** Due to the format of the VP8 payload, this value is only reliable for packets where [isStartOfFrame] is true. */ + override val isKeyframe: Boolean = isKeyframe ?: DePacketizer.isKeyFrame(this.buffer, payloadOffset, payloadLength) + + override val isStartOfFrame: Boolean = + isStartOfFrame ?: DePacketizer.VP8PayloadDescriptor.isStartOfFrame(buffer, payloadOffset) + + /** End of VP8 frame is the marker bit. */ + override val isEndOfFrame: Boolean + /* This uses [get] rather than initialization because [isMarked] is a var. */ + get() = isMarked + + override fun meetsRoutingNeeds(): Boolean = hasPictureId && hasTemporalLayerIndex + + val hasTemporalLayerIndex = + DePacketizer.VP8PayloadDescriptor.hasTemporalLayerIndex(buffer, payloadOffset, payloadLength) + + val hasPictureId = DePacketizer.VP8PayloadDescriptor.hasPictureId(buffer, payloadOffset, payloadLength) + + val hasExtendedPictureId = + DePacketizer.VP8PayloadDescriptor.hasExtendedPictureId(buffer, payloadOffset, payloadLength) + + val hasTL0PICIDX = DePacketizer.VP8PayloadDescriptor.hasTL0PICIDX(buffer, payloadOffset, payloadLength) + + @field:Suppress("ktlint:standard:property-naming", "ktlint:standard:backing-property-naming") + private var _TL0PICIDX = TL0PICIDX + ?: DePacketizer.VP8PayloadDescriptor.getTL0PICIDX(buffer, payloadOffset, payloadLength) + + var TL0PICIDX: Int + get() = _TL0PICIDX + set(newValue) { + _TL0PICIDX = newValue + if (newValue != -1 && !DePacketizer.VP8PayloadDescriptor.setTL0PICIDX( + buffer, + payloadOffset, + payloadLength, + newValue + ) + ) { + logger.cwarn { "Failed to set the TL0PICIDX of a VP8 packet." } + } + } + + private var _pictureId = pictureId ?: DePacketizer.VP8PayloadDescriptor.getPictureId(buffer, payloadOffset) + var pictureId: Int + get() = _pictureId + set(newValue) { + _pictureId = newValue + if (!DePacketizer.VP8PayloadDescriptor.setExtendedPictureId( + buffer, + payloadOffset, + payloadLength, + newValue + ) + ) { + logger.cwarn { "Failed to set the picture id of a VP8 packet." } + } + } + + val temporalLayerIndex: Int = Vp8Utils.getTemporalLayerIdOfFrame(this) + + override val layerIds: Collection + get() = if (hasTemporalLayerIndex) { + listOf(RtpLayerDesc.getIndex(0, 0, temporalLayerIndex)) + } else { + super.layerIds + } + + /** + * This is currently used as an overall spatial index, not an in-band spatial quality index a la vp9. That is, + * this index will correspond to an overall simulcast layer index across multiple simulcast stream. e.g. + * 180p stream packets will have 0, 360p -> 1, 720p -> 2 + */ + val height: Int = height ?: if (this.isKeyframe) { + Vp8Utils.getHeightFromKeyFrame(this) + } else { + -1 + } + + /** + * For [Vp8Packet] the payload excludes the VP8 Payload Descriptor. + */ + override val payloadVerification: String + get() { + val rtpPayloadLength = payloadLength + val rtpPayloadOffset = payloadOffset + val vp8pdSize = DePacketizer.VP8PayloadDescriptor.getSize(buffer, rtpPayloadOffset, rtpPayloadLength) + val vp8PayloadLength = rtpPayloadLength - vp8pdSize + val hashCode = buffer.hashCodeOfSegment(payloadOffset + vp8pdSize, rtpPayloadOffset + rtpPayloadLength) + return "type=Vp8Packet len=$vp8PayloadLength hashCode=$hashCode" + } + + override fun toString(): String = super.toString() + ", TID=$temporalLayerIndex" + + override fun clone(): Vp8Packet = Vp8Packet( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length, + isKeyframe = isKeyframe, + isStartOfFrame = isStartOfFrame, + encodingId = encodingId, + height = height, + pictureId = pictureId, + TL0PICIDX = TL0PICIDX + ).also { postClone(it) } + + companion object { + private val logger = createLogger() + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt new file mode 100644 index 0000000..86f92e9 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp8/Vp8Parser.kt @@ -0,0 +1,67 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.vp8 + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.util.StateChangeLogger +import org.jitsi.rtp.extensions.toHex +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +/** + * Some [Vp8Packet] fields are not able to be determined by looking at a single VP8 packet (for example the frame + * height can only be acquired from keyframes). This class updates the layer descriptions with information + * from frames, and also diagnoses packet format variants that the Jitsi videobridge won't be able to route. + */ +class Vp8Parser( + source: MediaSourceDesc, + parentLogger: Logger +) : VideoCodecParser(source) { + private val logger = createChildLogger(parentLogger) + + // Consistency + private val pictureIdState = StateChangeLogger("missing picture id", logger) + private val extendedPictureIdState = StateChangeLogger("missing extended picture ID", logger) + private val tidWithoutTl0PicIdxState = StateChangeLogger("TID with missing TL0PICIDX", logger) + + override fun parse(packetInfo: PacketInfo) { + val vp8Packet = packetInfo.packetAs() + if (vp8Packet.height > -1) { + // TODO: handle case where new height is from a packet older than the + // latest height we've seen. + findRtpEncodingDesc(vp8Packet)?.let { enc -> + val newLayers = enc.layers.map { layer -> layer.copy(height = vp8Packet.height) } + enc.layers = newLayers.toTypedArray() + } + } + + pictureIdState.setState(vp8Packet.hasPictureId, vp8Packet) { + "Packet Data: ${vp8Packet.toHex(80)}" + } + extendedPictureIdState.setState(vp8Packet.hasExtendedPictureId, vp8Packet) { + "Packet Data: ${vp8Packet.toHex(80)}" + } + tidWithoutTl0PicIdxState.setState( + vp8Packet.hasTL0PICIDX || !vp8Packet.hasTemporalLayerIndex, + vp8Packet + ) { + "Packet Data: ${vp8Packet.toHex(80)}" + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt new file mode 100644 index 0000000..2a38984 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Packet.kt @@ -0,0 +1,353 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.vp9 + +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc +import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.logging2.cwarn +import org.jitsi_modified.impl.neomedia.codec.video.vp9.DePacketizer +import kotlin.math.min + +/** + * If this [Vp9Packet] instance is being created via a clone, + * we've already parsed the packet itself and determined whether + * or not its a keyframe and what its spatial layer index is, + * so the constructor allows passing in those values if + * they're already known. If they're null, this instance + * will do the parsing itself. + */ +class Vp9Packet private constructor( + buffer: ByteArray, + offset: Int, + length: Int, + isKeyframe: Boolean?, + isStartOfFrame: Boolean?, + isEndOfFrame: Boolean?, + encodingId: Int, + pictureId: Int?, + TL0PICIDX: Int? +) : ParsedVideoPacket(buffer, offset, length, encodingId) { + + constructor( + buffer: ByteArray, + offset: Int, + length: Int + ) : this( + buffer, offset, length, + isKeyframe = null, + isStartOfFrame = null, + isEndOfFrame = null, + encodingId = RtpLayerDesc.SUSPENDED_ENCODING_ID, + pictureId = null, + TL0PICIDX = null + ) + + override val isKeyframe: Boolean = + isKeyframe ?: DePacketizer.VP9PayloadDescriptor.isKeyFrame(this.buffer, payloadOffset, payloadLength) + + override val isStartOfFrame: Boolean = + isStartOfFrame ?: DePacketizer.VP9PayloadDescriptor.isStartOfFrame(buffer, payloadOffset, payloadLength) + + override val isEndOfFrame: Boolean = + isEndOfFrame ?: DePacketizer.VP9PayloadDescriptor.isEndOfFrame(buffer, payloadOffset, payloadLength) + + override fun meetsRoutingNeeds(): Boolean { + // Question: should we include hasLayerIndices here? I.e. if we get a VP9 packet with an AV1 DD and + // a VP9 picture ID, but no VP9 layer indices, are we better off parsing it as VP9 or AV1? + return hasPictureId + } + + override val layerIds: Collection + get() = if (hasLayerIndices) { + listOf(RtpLayerDesc.getIndex(0, spatialLayerIndex, temporalLayerIndex)) + } else { + super.layerIds + } + + /** End of VP9 picture is the marker bit. Note frame/picture distinction. */ + /* TODO: not sure this should be the override from [ParsedVideoPacket] */ + val isEndOfPicture: Boolean + /* This uses [get] rather than initialization because [isMarked] is a var. */ + get() = isMarked + + val hasLayerIndices = + DePacketizer.VP9PayloadDescriptor.hasLayerIndices(buffer, payloadOffset, payloadLength) + + val hasPictureId = + DePacketizer.VP9PayloadDescriptor.hasPictureId(buffer, payloadOffset, payloadLength) + + val hasExtendedPictureId = + DePacketizer.VP9PayloadDescriptor.hasExtendedPictureId(buffer, payloadOffset, payloadLength) + + val isFlexibleMode = + DePacketizer.VP9PayloadDescriptor.isFlexibleMode(buffer, payloadOffset, payloadLength) + + val hasScalabilityStructure = + DePacketizer.VP9PayloadDescriptor.hasScalabilityStructure(buffer, payloadOffset, payloadLength) + + val isUpperLevelReference = + DePacketizer.VP9PayloadDescriptor.isUpperLevelReference(buffer, payloadOffset, payloadLength) + + val isInterPicturePredicted = + DePacketizer.VP9PayloadDescriptor.isInterPicturePredicted(buffer, payloadOffset, payloadLength) + + @field:Suppress("ktlint:standard:property-naming", "ktlint:standard:backing-property-naming") + private var _TL0PICIDX = + TL0PICIDX ?: DePacketizer.VP9PayloadDescriptor.getTL0PICIDX(buffer, payloadOffset, payloadLength) + + var TL0PICIDX: Int + get() = _TL0PICIDX + set(newValue) { + if (newValue != -1 && !DePacketizer.VP9PayloadDescriptor.setTL0PICIDX( + buffer, + payloadOffset, + payloadLength, + newValue + ) + ) { + logger.cwarn { "Failed to set the TL0PICIDX of a VP9 packet." } + } + _TL0PICIDX = newValue + } + + val hasTL0PICIDX: Boolean + get() = _TL0PICIDX != -1 + + private var _pictureId = + pictureId ?: DePacketizer.VP9PayloadDescriptor.getPictureId(buffer, payloadOffset, payloadLength) + + var pictureId: Int + get() = _pictureId + set(newValue) { + if (!DePacketizer.VP9PayloadDescriptor.setExtendedPictureId( + buffer, + payloadOffset, + payloadLength, + newValue + ) + ) { + logger.cwarn { "Failed to set the picture id of a VP9 packet." } + } + _pictureId = newValue + } + + /* TODO: avoid recomputing these on clone */ + + val temporalLayerIndex: Int = + DePacketizer.VP9PayloadDescriptor.getTemporalLayerIndex(buffer, payloadOffset, payloadLength) + + val spatialLayerIndex: Int = + DePacketizer.VP9PayloadDescriptor.getSpatialLayerIndex(buffer, payloadOffset, payloadLength) + + val effectiveTemporalLayerIndex: Int = + if (hasLayerIndices) temporalLayerIndex else 0 + + val effectiveSpatialLayerIndex: Int = + if (hasLayerIndices) spatialLayerIndex else 0 + + val isSwitchingUpPoint: Boolean = + DePacketizer.VP9PayloadDescriptor.isSwitchingUpPoint(buffer, payloadOffset, payloadLength) + + val usesInterLayerDependency: Boolean = + DePacketizer.VP9PayloadDescriptor.usesInterLayerDependency(buffer, payloadOffset, payloadLength) + + fun getScalabilityStructure(eid: Int = 0, baseFrameRate: Double = 30.0) = + Companion.getScalabilityStructure(buffer, payloadOffset, payloadLength, ssrc, eid, baseFrameRate) + + val scalabilityStructureNumSpatial: Int + get() { + val off = + DePacketizer.VP9PayloadDescriptor.getScalabilityStructureOffset(buffer, payloadOffset, payloadLength) + if (off == -1) { + return -1 + } + val ssHeader = buffer[off].toInt() + + return ((ssHeader and 0xE0) shr 5) + 1 + } + + /** + * For [Vp9Packet] the payload excludes the VP9 Payload Descriptor. + */ + override val payloadVerification: String + get() { + val rtpPayloadLength = payloadLength + val rtpPayloadOffset = payloadOffset + val vp9pdSize = DePacketizer.VP9PayloadDescriptor.getSize(buffer, rtpPayloadOffset, rtpPayloadLength) + val vp9PayloadLength = rtpPayloadLength - vp9pdSize + val hashCode = buffer.hashCodeOfSegment(rtpPayloadOffset + vp9pdSize, rtpPayloadOffset + rtpPayloadLength) + return "type=VP9Packet len=$vp9PayloadLength hashCode=$hashCode" + } + + override fun toString(): String = super.toString() + ", SID=$spatialLayerIndex, TID=$temporalLayerIndex" + + override fun clone(): Vp9Packet = Vp9Packet( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length, + isKeyframe = isKeyframe, + isStartOfFrame = isStartOfFrame, + isEndOfFrame = isEndOfFrame, + encodingId = encodingId, + pictureId = pictureId, + TL0PICIDX = TL0PICIDX + ).also { postClone(it) } + + companion object { + private val logger = createLogger() + + private val Y_BIT = 1 shl 4 + private val G_BIT = 1 shl 3 + + private val MAX_NUM_TLAYERS = 8 + + /* In theory this would fit better in vp9.DePacketizer, but I don't feel like translating this code + * to Java, or translating that file to Kotlin. + */ + fun getScalabilityStructure( + buffer: ByteArray, + payloadOffset: Int, + payloadLength: Int, + ssrc: Long, + eid: Int, + baseFrameRate: Double + ): RtpEncodingDesc? { + /* + * VP9 Scalability structure: + * + * +-+-+-+-+-+-+-+-+ + * V: | N_S |Y|G|-|-|-| + * +-+-+-+-+-+-+-+-+ -\ + * Y: | WIDTH | (OPTIONAL) . + * + + . + * | | (OPTIONAL) . + * +-+-+-+-+-+-+-+-+ . - N_S + 1 times + * | HEIGHT | (OPTIONAL) . + * + + . + * | | (OPTIONAL) . + * +-+-+-+-+-+-+-+-+ -/ + * G: | N_G | (OPTIONAL) + * +-+-+-+-+-+-+-+-+ -\ + * N_G: | TID |U| R |-|-| (OPTIONAL) . + * +-+-+-+-+-+-+-+-+ -\ . - N_G times + * | P_DIFF | (OPTIONAL) . - R times . + * +-+-+-+-+-+-+-+-+ -/ -/ + */ + + var off = DePacketizer.VP9PayloadDescriptor.getScalabilityStructureOffset( + buffer, + payloadOffset, + payloadLength + ) + if (off == -1) { + return null + } + val ssHeader = buffer[off].toInt() + + val numSpatial = ((ssHeader and 0xE0) shr 5) + 1 + val hasResolution = (ssHeader and Y_BIT) != 0 + val hasPictureGroup = (ssHeader and G_BIT) != 0 + + off++ + + val heights = if (hasResolution) { + Array(numSpatial) { + val width = ((buffer[off].toInt() and 0xff) shl 8) or + (buffer[off + 1].toInt() and 0xff) + off += 2 + + val height = ((buffer[off].toInt() and 0xff) shl 8) or + (buffer[off + 1].toInt() and 0xff) + off += 2 + + // Treat the lesser of width or height as the height in order to handle + // portrait-mode video correctly + min(width, height) + } + } else { + Array(numSpatial) { RtpLayerDesc.NO_HEIGHT } + } + + val tlCounts = Array(MAX_NUM_TLAYERS) { 0 } + val groupSize = if (hasPictureGroup) { + val groupSize = buffer[off].toInt() + off++ + + for (i in 0 until groupSize) { + val descByte = buffer[off].toInt() + val tid = (descByte and 0xE0) shr 5 + val numRefs = (descByte and 0x0C) shr 2 + off++ + + tlCounts[tid]++ + + /* TODO: do something with U, R, P_DIFF? */ + for (j in 0 until numRefs) { + off++ + } + } + groupSize + } else { + tlCounts[0] = 1 + 1 + } + + val numTemporal = tlCounts.indexOfLast { it > 0 } + 1 + + for (t in 1 until numTemporal) { + /* Sum up frames per picture group */ + tlCounts[t] += tlCounts[t - 1] + } + + val layers = ArrayList() + + for (s in 0 until numSpatial) { + for (t in 0 until numTemporal) { + val dependencies = ArrayList() + val softDependencies = ArrayList() + if (s > 0) { + /* Because of K-SVC, spatial layer dependencies are soft */ + layers.find { it.sid == s - 1 && it.tid == t }?.let { softDependencies.add(it) } + } + if (t > 0) { + layers.find { it.sid == s && it.tid == t - 1 }?.let { dependencies.add(it) } + } + val layerDesc = VpxRtpLayerDesc( + eid = eid, + tid = t, + sid = s, + height = heights[s], + frameRate = if (hasPictureGroup) { + baseFrameRate * tlCounts[t] / groupSize + } else { + RtpLayerDesc.NO_FRAME_RATE + }, + dependencyLayers = dependencies.toArray(arrayOf()), + softDependencyLayers = softDependencies.toArray(arrayOf()) + ) + layers.add(layerDesc) + } + } + + return RtpEncodingDesc(ssrc, layers.toArray(arrayOf()), eid) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt new file mode 100644 index 0000000..05300ff --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9Parser.kt @@ -0,0 +1,156 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.vp9 + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc +import org.jitsi.nlj.util.StateChangeLogger +import org.jitsi.rtp.extensions.toHex +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import kotlin.math.max + +/** + * Some [Vp9Packet] fields are not able to be determined by looking at a single VP9 packet (for example the scalability + * structure is only carried in keyframes). This class updates the layer descriptions with information + * from frames, and also diagnoses packet format variants that the Jitsi videobridge won't be able to route. + */ +class Vp9Parser( + source: MediaSourceDesc, + parentLogger: Logger +) : VideoCodecParser(source) { + private val logger = createChildLogger(parentLogger) + + private val pictureIdState = StateChangeLogger("missing picture id", logger) + private val extendedPictureIdState = StateChangeLogger("missing extended picture ID", logger) + private var numSpatialLayers = -1 + + /** Encodings we've actually seen, and the layers seen for each one. + * Used to clear out inferred-from-signaling encoding information, and to synthesize temporal layers + * for flexible-mode encodings. */ + private val ssrcsInfo = HashMap>() + + override fun parse(packetInfo: PacketInfo) { + val vp9Packet = packetInfo.packetAs() + + val layerMap = ssrcsInfo.getOrPut(vp9Packet.ssrc) { + HashMap() + } + + layerMap[vp9Packet.spatialLayerIndex]?.let { + layerMap[vp9Packet.spatialLayerIndex] = max(it, vp9Packet.temporalLayerIndex) + } ?: run { + layerMap[vp9Packet.spatialLayerIndex] = vp9Packet.temporalLayerIndex + } + + if (vp9Packet.hasScalabilityStructure) { + // TODO: handle case where new SS is from a packet older than the + // latest SS we've seen. + val packetSpatialLayers = vp9Packet.scalabilityStructureNumSpatial + if (packetSpatialLayers != -1) { + if (numSpatialLayers != -1 && numSpatialLayers != packetSpatialLayers) { + packetInfo.layeringChanged = true + } + numSpatialLayers = packetSpatialLayers + } + val ss = findRtpEncodingDesc(vp9Packet)?.let { enc -> + vp9Packet.getScalabilityStructure(eid = enc.eid) + } + + if (ss != null) { + val layers = + if (vp9Packet.isFlexibleMode) { + /* In flexible mode, the number of temporal layers isn't announced in the keyframe. + * Thus, add temporal layer information to the source's encoding layers based on the temporal + * layers we've seen previously. + */ + val layersList = ss.layers.toMutableList() + + for ((sid, maxTid) in layerMap) { + addTemporalLayers(layersList, sid, maxTid) + } + layersList.toTypedArray() + } else { + ss.layers + } + + source.setEncodingLayers(layers, vp9Packet.ssrc) + + for (otherEnc in source.rtpEncodings) { + if (!ssrcsInfo.contains(otherEnc.primarySSRC)) { + source.setEncodingLayers(emptyArray(), otherEnc.primarySSRC) + } + } + } + } + if (vp9Packet.spatialLayerIndex > 0 && vp9Packet.isInterPicturePredicted) { + /* Check if this layer is using K-SVC. */ + /* Note: In K-SVC mode, this entirely ignores the bitrate of lower-layer keyframes + * when calculating layers' bitrates. These values are small enough this is probably + * fine, but revisit this if it turns out to be a problem. + */ + findRtpLayerDescs(vp9Packet).forEach { + if (it is VpxRtpLayerDesc) { + it.useSoftDependencies = vp9Packet.usesInterLayerDependency + } + } + } + + if (vp9Packet.isFlexibleMode && findRtpLayerDescs(vp9Packet).isEmpty()) { + val layers = source.getEncodingLayers(vp9Packet.ssrc).toMutableList() + /* In flexible mode, the number of temporal layers isn't announced in the keyframe. + * Thus, add temporal layer information to the source's encoding layers as we see packets with + * temporal layers. + */ + val changed = addTemporalLayers(layers, vp9Packet.spatialLayerIndex, vp9Packet.temporalLayerIndex) + if (changed) { + source.setEncodingLayers(layers.toTypedArray(), vp9Packet.ssrc) + packetInfo.layeringChanged = true + } + } + + pictureIdState.setState(vp9Packet.hasPictureId, vp9Packet) { + "Packet Data: ${vp9Packet.toHex(80)}" + } + extendedPictureIdState.setState(vp9Packet.hasExtendedPictureId, vp9Packet) { + "Packet Data: ${vp9Packet.toHex(80)}" + } + } + + /** Add temporal layers to the list of layers. Needed if VP9 is encoded in flexible mode, because + * in flexible mode the scalability structure doesn't describe the temporal layers. + */ + private fun addTemporalLayers(layers: MutableList, sid: Int, maxTid: Int): Boolean { + var changed = false + + for (tid in 1..maxTid) { + val layer = layers.find { it.sid == sid && it.tid == tid } + if (layer == null) { + val prevLayer = layers.find { it.sid == sid && it.tid == tid - 1 } + if (prevLayer != null) { + val newLayer = prevLayer.copy(tid = tid, inherit = false) + layers.add(newLayer) + changed = true + } + } + } + return changed + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt new file mode 100644 index 0000000..c5f2cde --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt @@ -0,0 +1,197 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp.codec.vpx + +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.sum + +/** + * An RtpLayerDesc of the type needed to describe VP8 and VP9 scalability. + */ +class VpxRtpLayerDesc +@JvmOverloads +constructor( + /** + * The index of this instance's encoding in the source encoding array. + */ + eid: Int, + /** + * The temporal layer ID of this instance, or negative for unknown. + */ + tid: Int, + /** + * The spatial layer ID of this instance, or negative for unknown. + */ + sid: Int, + /** + * The max height of the bitstream that this instance represents. The actual + * height may be less due to bad network or system load. [RtpLayerDesc.NO_HEIGHT] for unknown. + * + * XXX we should be able to sniff the actual height from the RTP + * packets. + */ + height: Int, + /** + * The max frame rate (in fps) of the bitstream that this instance + * represents. The actual frame rate may be less due to bad network or + * system load. [RtpLayerDesc.NO_FRAME_RATE] for unknown. + */ + frameRate: Double, + /** + * The [RtpLayerDesc]s on which this layer definitely depends. + */ + val dependencyLayers: Array = emptyArray(), + /** + * The [RtpLayerDesc]s on which this layer possibly depends. + * (The intended use case is K-SVC mode.) + */ + val softDependencyLayers: Array = emptyArray() +) : RtpLayerDesc(eid, tid, sid, height, frameRate) { + init { + require(tid < 8) { "Invalid temporal ID $tid" } + require(sid < 8) { "Invalid spatial ID $sid" } + } + + /** + * Clone an existing layer desc, inheriting its statistics if [inherit], + * modifying only specific values. + */ + override fun copy(height: Int, tid: Int, inherit: Boolean) = VpxRtpLayerDesc( + eid = this.eid, + tid = tid, + sid = this.sid, + height = height, + frameRate = this.frameRate, + dependencyLayers = this.dependencyLayers, + softDependencyLayers = this.softDependencyLayers + ).also { + if (inherit) { + it.inheritFrom(this) + } + } + + /** + * Whether softDependencyLayers are to be used. + */ + var useSoftDependencies = true + + /** + * @return the "id" of this layer within this encoding. This is a server-side id and should + * not be confused with any encoding id defined in the client (such as the + * rid). + */ + override val layerId = getIndex(0, sid, tid) + + /** + * A local index of this track. + */ + override val index = getIndex(eid, sid, tid) + + /** + * Inherit another layer description's [BitrateTracker] object. + */ + override fun inheritFrom(other: RtpLayerDesc) { + super.inheritFrom(other) + if (other is VpxRtpLayerDesc) { + useSoftDependencies = other.useSoftDependencies + } + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = + "subjective_quality=$index,temporal_id=$tid,spatial_id=$sid,height=$height,frameRate=$frameRate" + + /** + * Gets the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. + * + * This is left open for use in testing. + * + * @param nowMs + * @return the cumulative bitrate (in bps) of this [RtpLayerDesc] and its dependencies. + */ + override fun getBitrate(nowMs: Long): Bandwidth = calcBitrate(nowMs, smoothed = false).values.sum() + + override fun getSmoothedBitrate(nowMs: Long): Bandwidth = calcBitrate(nowMs, smoothed = true).values.sum() + + /** + * Recursively adds the bitrate (in bps) of this [RtpLayerDesc] and + * its dependencies in the map passed in as an argument. + * + * This is necessary to ensure we don't double-count layers in cases + * of multiple dependencies. + * + * @param nowMs + */ + private fun calcBitrate( + nowMs: Long, + smoothed: Boolean, + rates: MutableMap = HashMap() + ): MutableMap { + if (rates.containsKey(index)) { + return rates + } + rates[index] = layerRate(nowMs, smoothed) + + dependencyLayers.forEach { it.calcBitrate(nowMs, smoothed, rates) } + + if (useSoftDependencies) { + softDependencyLayers.forEach { it.calcBitrate(nowMs, smoothed, rates) } + } + + return rates + } + + /** + * Returns true if this layer, alone, has a zero bitrate. + */ + private fun layerHasZeroBitrate(nowMs: Long) = bitrateTracker.getAccumulatedSize(nowMs).bits == 0L + + /** + * Recursively checks this layer and its dependencies to see if the bitrate is zero. + * Note that unlike [calcBitrate] this does not avoid double-visiting layers; the overhead + * of the hash table is usually more than the cost of any double-visits. + * + * This is left open for use in testing. + */ + override fun hasZeroBitrate(nowMs: Long): Boolean { + if (!layerHasZeroBitrate(nowMs)) { + return false + } + if (dependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) { + return false + } + if (useSoftDependencies && softDependencyLayers.any { !it.layerHasZeroBitrate(nowMs) }) { + return false + } + return true + } + + /** + * Extracts a [NodeStatsBlock] from an [RtpLayerDesc]. + */ + override fun debugState() = super.debugState().apply { + put("tid", tid) + put("sid", sid) + } + + override fun indexString(): String = indexString(index) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpConfig.kt new file mode 100644 index 0000000..bac9dc7 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpConfig.kt @@ -0,0 +1,38 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.srtp + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.optionalconfig +import org.jitsi.nlj.srtp.SrtpUtil.Companion.getSrtpProtectionProfileFromName + +class SrtpConfig { + companion object { + val maxConsecutivePacketsDiscardedEarly: Int by config { + "jmt.srtp.max-consecutive-packets-discarded-early".from(JitsiConfig.newConfig) + } + + val protectionProfiles: List by config { + "jmt.srtp.protection-profiles".from(JitsiConfig.newConfig) + .convertFrom> { list -> list.map { getSrtpProtectionProfileFromName(it) } } + } + + val factoryClass: String? by optionalconfig { + "jmt.srtp.factory-class".from(JitsiConfig.newConfig) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpProfileInformation.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpProfileInformation.kt new file mode 100644 index 0000000..908ecae --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpProfileInformation.kt @@ -0,0 +1,26 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.srtp + +data class SrtpProfileInformation( + val cipherKeyLength: Int, + val cipherSaltLength: Int, + val cipherName: Int, + val authFunctionName: Int, + val authKeyLength: Int, + val rtcpAuthTagLength: Int, + val rtpAuthTagLength: Int +) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt new file mode 100644 index 0000000..f3ead9f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt @@ -0,0 +1,296 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.srtp + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.srtp.SrtpConfig.Companion.maxConsecutivePacketsDiscardedEarly +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.rtcp.RtcpHeader +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.srtp.BaseSrtpCryptoContext +import org.jitsi.srtp.SrtcpCryptoContext +import org.jitsi.srtp.SrtpContextFactory +import org.jitsi.srtp.SrtpCryptoContext +import org.jitsi.srtp.SrtpErrorStatus +import org.jitsi.utils.LRUCache +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.logging2.cwarn + +/** + * Implements the methods common to all 4 transformer implementation (encrypt/decrypt for SRTP/SRTCP) + */ +abstract class AbstractSrtpTransformer( + protected val contextFactory: SrtpContextFactory, + parentLogger: Logger +) { + protected val logger = createChildLogger(parentLogger) + + /** + * All the known SSRC's corresponding SrtpCryptoContexts. Bounded with an LRU to limit the number of SSRCs tracked; + * evicted contexts are simply dropped (their native cipher resources are freed by a [java.lang.ref.Cleaner], and + * the contexts hold no other state that needs explicit cleanup). All access is guarded by synchronizing on the map. + * Note that eviction resets the ROC and replay state for an SSRC, so the cap must be high enough that + * legitimately-active SSRCs are never evicted. + */ + private val contexts: MutableMap = LRUCache(MAX_SSRCS, true) + + /** + * The number of SSRCs for which a crypto context is currently cached. + */ + val numCachedContexts: Int + get() = synchronized(contexts) { contexts.size } + + fun close() { + synchronized(contexts) { + contextFactory.close() + } + } + + /** + * Whether a context that is derived on demand (because none was cached for the SSRC yet) should only be added to + * [contexts] after the first transform operation using it succeeds. This is used for decryption, so that we do not + * retain a context for an SSRC whose packets never successfully authenticate. + */ + protected open val cacheContextOnlyOnSuccessfulTransform: Boolean = false + + /** + * Derives a new context for a specific SSRC and index. + */ + protected abstract fun deriveContext(ssrc: Long, index: Long): CryptoContextType? + + /** + * Gets the SSRC and index (sequence number) to use for a specific packet, or null if it can not be handled. + */ + protected abstract fun getSsrcAndIndex(packetInfo: PacketInfo): SsrcAndIndex? + + /** + * Does the actual transformation of a packet, with a specific context. [isNewContext] indicates that the context was + * just derived and is not (yet) cached, which a transformer may use to force a full transform (e.g. to bypass the + * "skip decryption" optimization) so that the returned status reliably reflects whether the context is valid. + */ + protected abstract fun transform( + packetInfo: PacketInfo, + context: CryptoContextType, + isNewContext: Boolean + ): SrtpErrorStatus + + /** + * Transforms a packet, returns [SrtpErrorStatus.OK] on success or another [SrtpErrorStatus] on failure. + */ + fun transform(packetInfo: PacketInfo): SrtpErrorStatus { + val ssrcAndIndex = getSsrcAndIndex(packetInfo) ?: return SrtpErrorStatus.FAIL + val ssrc = ssrcAndIndex.ssrc + + // Look up (or derive) the context under the lock. We must synchronize even the lookup because the map is not + // necessarily thread-safe (and an access-order map is structurally modified on read). The lock is only held + // for the lookup/derivation, not for the transform itself. + var isNewContext = false + val context = synchronized(contexts) { + contexts[ssrc] ?: run { + val derivedContext = deriveContext(ssrc, ssrcAndIndex.index) ?: run { + logger.warn("Failed to derive context for $ssrc ${ssrcAndIndex.index}") + return SrtpErrorStatus.FAIL + } + isNewContext = true + if (!cacheContextOnlyOnSuccessfulTransform) { + contexts[ssrc] = derivedContext + } + derivedContext + } + } + + val status = transform(packetInfo, context, isNewContext) + if (isNewContext && cacheContextOnlyOnSuccessfulTransform && status == SrtpErrorStatus.OK) { + synchronized(contexts) { + contexts.putIfAbsent(ssrc, context) + } + } + return status + } + + /** + * The SSRC and index (sequence number) identifying the context to use for a packet. + */ + protected data class SsrcAndIndex(val ssrc: Long, val index: Long) + + companion object { + /** The maximum number of SSRCs to keep crypto contexts for. */ + const val MAX_SSRCS = 1024 + } +} + +/** + * Implements methods common for the two SRTP transformer implementations. + */ +abstract class SrtpTransformer( + contextFactory: SrtpContextFactory, + logger: Logger +) : AbstractSrtpTransformer(contextFactory, logger) { + + override fun deriveContext(ssrc: Long, index: Long): SrtpCryptoContext? = + contextFactory.deriveContext(ssrc.toInt(), 0) + + override fun getSsrcAndIndex(packetInfo: PacketInfo): SsrcAndIndex? { + val rtpPacket: RtpPacket = packetInfo.packet as? RtpPacket ?: run { + logger.cwarn { "Can not handle non-RTP packet: ${packetInfo.packet.javaClass}" } + return null + } + return SsrcAndIndex(rtpPacket.ssrc, rtpPacket.sequenceNumber.toLong()) + } +} + +/** + * Implements methods common for the two SRTCP transformer implementations. + */ +abstract class SrtcpTransformer( + contextFactory: SrtpContextFactory, + logger: Logger +) : AbstractSrtpTransformer(contextFactory, logger) { + + override fun deriveContext(ssrc: Long, index: Long): SrtcpCryptoContext? = + contextFactory.deriveControlContext(ssrc.toInt()) + + override fun getSsrcAndIndex(packetInfo: PacketInfo): SsrcAndIndex? { + // Contrary to RTP packets, RTCP packets do not get parsed before they are + // decrypted. So (if this is a decrypting transformer) we are working with + // an UnparsedPacket here and need to read the SSRC manually. + val senderSsrc = RtcpHeader.getSenderSsrc(packetInfo.packet.getBuffer(), packetInfo.packet.getOffset()) + return SsrcAndIndex(senderSsrc, 0) + } +} + +/** + * A transformer which decrypts SRTCP packets. + */ +class SrtcpDecryptTransformer( + contextFactory: SrtpContextFactory, + parentLogger: Logger +) : SrtcpTransformer(contextFactory, parentLogger) { + + // Only cache a derived context once it has successfully authenticated a packet. + override val cacheContextOnlyOnSuccessfulTransform = true + + override fun transform( + packetInfo: PacketInfo, + context: SrtcpCryptoContext, + isNewContext: Boolean + ): SrtpErrorStatus = context.reverseTransformPacket(packetInfo.packet).apply { + packetInfo.resetPayloadVerification() + } +} + +/** + * A transformer which encrypts RTCP packets (producing SRTCP packets). Note that as opposed to the other transformers, + * this one replaces the [Packet]. + */ +class SrtcpEncryptTransformer( + contextFactory: SrtpContextFactory, + parentLogger: Logger +) : SrtcpTransformer(contextFactory, parentLogger) { + + override fun transform( + packetInfo: PacketInfo, + context: SrtcpCryptoContext, + isNewContext: Boolean + ): SrtpErrorStatus = context.transformPacket(packetInfo.packet).apply { + // We convert the encrypted RTCP packet to an UnparsedPacket because + // we don't want any of the RTCP fields trying to parse the data + // (since it's now encrypted) + // TODO: better way we can do this? it's not typically a problem + // in the pipeline's usage, but it's a bit of a landmine since by + // accessing the packet it can try and parse the fields. + packetInfo.packet = packetInfo.packet.toOtherType(::UnparsedPacket) + packetInfo.resetPayloadVerification() + } +} + +/** + * A transformer which decrypts SRTP packets. Note that it expects the [Packet] to have already been parsed as + * [RtpPacket]. + */ +class SrtpDecryptTransformer( + contextFactory: SrtpContextFactory, + parentLogger: Logger +) : SrtpTransformer(contextFactory, parentLogger) { + var earlyDiscardedPacketsSinceLastSuccess = 0 + private val alwaysProcess = maxConsecutivePacketsDiscardedEarly <= 0 + + // Only cache a derived context once it has successfully authenticated a packet. + override val cacheContextOnlyOnSuccessfulTransform = true + + override fun transform(packetInfo: PacketInfo, context: SrtpCryptoContext, isNewContext: Boolean): SrtpErrorStatus { + // We want to avoid authenticating and decrypting packets that we are going to discarded (e.g. silence). We + // can not just discard them without passing them to the SRTP stack, because this will eventually break the ROC. + // Here we bypass the SRTP stack for packets marked to be discarded, but make sure that we haven't dropped too + // many consecutive packets. We never bypass for a newly-derived context: it must actually decrypt (and thus + // authenticate) a packet before it is cached, so the returned status must reflect whether the context is valid. + if (packetInfo.shouldDiscard) { + return if (isNewContext || + alwaysProcess || + earlyDiscardedPacketsSinceLastSuccess++ > maxConsecutivePacketsDiscardedEarly + ) { + doTransform(packetInfo, context) + } else { + // Bypass the SRTP stack. The packet is already marked to be discarded, so there's no error condition. + SrtpErrorStatus.OK + } + } + + return doTransform(packetInfo, context) + } + + private fun doTransform(packetInfo: PacketInfo, context: SrtpCryptoContext): SrtpErrorStatus = + context.reverseTransformPacket(packetInfo.packetAs(), packetInfo.shouldDiscard).also { + packetInfo.resetPayloadVerification() + if (it == SrtpErrorStatus.OK) { + earlyDiscardedPacketsSinceLastSuccess = 0 + } + } +} + +/** + * A transformer which encrypts RTP packets (producing SRTP packets). + */ +class SrtpEncryptTransformer( + contextFactory: SrtpContextFactory, + parentLogger: Logger +) : SrtpTransformer(contextFactory, parentLogger) { + + override fun transform(packetInfo: PacketInfo, context: SrtpCryptoContext, isNewContext: Boolean): SrtpErrorStatus = + context.transformPacket(packetInfo.packetAs()).apply { + packetInfo.packet = packetInfo.packet.toOtherType(::UnparsedPacket) + packetInfo.resetPayloadVerification() + } +} + +/** + * Holds the four SRTP-related transformers for a session. + */ +data class SrtpTransformers( + val srtpDecryptTransformer: SrtpDecryptTransformer, + val srtpEncryptTransformer: SrtpEncryptTransformer, + val srtcpDecryptTransformer: SrtcpDecryptTransformer, + val srtcpEncryptTransformer: SrtcpEncryptTransformer +) { + fun close() { + srtpDecryptTransformer.close() + srtpEncryptTransformer.close() + srtcpDecryptTransformer.close() + srtcpEncryptTransformer.close() + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt new file mode 100644 index 0000000..0570d55 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/srtp/SrtpUtil.kt @@ -0,0 +1,235 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.srtp + +import org.bouncycastle.tls.SRTPProtectionProfile +import org.jitsi.srtp.SrtpContextFactory +import org.jitsi.srtp.SrtpPolicy +import org.jitsi.srtp.crypto.Aes +import org.jitsi.utils.logging2.Logger + +enum class TlsRole { + CLIENT, + SERVER +} + +class SrtpUtil { + companion object { + init { + SrtpConfig.factoryClass?.let { Aes.setFactoryClassName(it) } + } + + fun getSrtpProtectionProfileFromName(profileName: String): Int = when (profileName) { + "SRTP_AES128_CM_HMAC_SHA1_80" -> { + SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_80 + } + + "SRTP_AES128_CM_HMAC_SHA1_32" -> { + SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_32 + } + + "SRTP_NULL_HMAC_SHA1_32" -> { + SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_32 + } + + "SRTP_NULL_HMAC_SHA1_80" -> { + SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_80 + } + + "SRTP_AEAD_AES_128_GCM" -> { + SRTPProtectionProfile.SRTP_AEAD_AES_128_GCM + } + + "SRTP_AEAD_AES_256_GCM" -> { + SRTPProtectionProfile.SRTP_AEAD_AES_256_GCM + } + + else -> throw IllegalArgumentException("Unsupported SRTP protection profile: $profileName") + } + + fun getSrtpProfileInformationFromSrtpProtectionProfile(srtpProtectionProfile: Int): SrtpProfileInformation = + when (srtpProtectionProfile) { + SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_32 -> { + SrtpProfileInformation( + cipherKeyLength = 128 / 8, + cipherSaltLength = 112 / 8, + cipherName = SrtpPolicy.AESCM_ENCRYPTION, + authFunctionName = SrtpPolicy.HMACSHA1_AUTHENTICATION, + authKeyLength = 160 / 8, + rtcpAuthTagLength = 80 / 8, + rtpAuthTagLength = 32 / 8 + ) + } + + SRTPProtectionProfile.SRTP_AES128_CM_HMAC_SHA1_80 -> { + SrtpProfileInformation( + cipherKeyLength = 128 / 8, + cipherSaltLength = 112 / 8, + cipherName = SrtpPolicy.AESCM_ENCRYPTION, + authFunctionName = SrtpPolicy.HMACSHA1_AUTHENTICATION, + authKeyLength = 160 / 8, + rtcpAuthTagLength = 80 / 8, + rtpAuthTagLength = 80 / 8 + ) + } + + SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_32 -> { + SrtpProfileInformation( + cipherKeyLength = 0, + cipherSaltLength = 0, + cipherName = SrtpPolicy.NULL_ENCRYPTION, + authFunctionName = SrtpPolicy.HMACSHA1_AUTHENTICATION, + authKeyLength = 160 / 8, + rtcpAuthTagLength = 80 / 8, + rtpAuthTagLength = 32 / 8 + ) + } + + SRTPProtectionProfile.SRTP_NULL_HMAC_SHA1_80 -> { + SrtpProfileInformation( + cipherKeyLength = 0, + cipherSaltLength = 0, + cipherName = SrtpPolicy.NULL_ENCRYPTION, + authFunctionName = SrtpPolicy.HMACSHA1_AUTHENTICATION, + authKeyLength = 160 / 8, + rtcpAuthTagLength = 80 / 8, + rtpAuthTagLength = 80 / 8 + ) + } + + SRTPProtectionProfile.SRTP_AEAD_AES_128_GCM -> { + SrtpProfileInformation( + cipherKeyLength = 128 / 8, + cipherSaltLength = 96 / 8, + cipherName = SrtpPolicy.AESGCM_ENCRYPTION, + authFunctionName = SrtpPolicy.NULL_AUTHENTICATION, + authKeyLength = 0, + rtcpAuthTagLength = 128 / 8, + rtpAuthTagLength = 128 / 8 + ) + } + + SRTPProtectionProfile.SRTP_AEAD_AES_256_GCM -> { + SrtpProfileInformation( + cipherKeyLength = 256 / 8, + cipherSaltLength = 96 / 8, + cipherName = SrtpPolicy.AESGCM_ENCRYPTION, + authFunctionName = SrtpPolicy.NULL_AUTHENTICATION, + authKeyLength = 0, + rtcpAuthTagLength = 128 / 8, + rtpAuthTagLength = 128 / 8 + ) + } + + else -> throw IllegalArgumentException("Unsupported SRTP protection profile: $srtpProtectionProfile") + } + + fun initializeTransformer( + srtpProfileInformation: SrtpProfileInformation, + keyingMaterial: ByteArray, + tlsRole: TlsRole, + cryptex: Boolean, + parentLogger: Logger + ): SrtpTransformers { + val clientWriteSrtpMasterKey = ByteArray(srtpProfileInformation.cipherKeyLength) + val serverWriteSrtpMasterKey = ByteArray(srtpProfileInformation.cipherKeyLength) + val clientWriterSrtpMasterSalt = ByteArray(srtpProfileInformation.cipherSaltLength) + val serverWriterSrtpMasterSalt = ByteArray(srtpProfileInformation.cipherSaltLength) + val keyingMaterialValues = listOf( + clientWriteSrtpMasterKey, + serverWriteSrtpMasterKey, + clientWriterSrtpMasterSalt, + serverWriterSrtpMasterSalt + ) + + var keyingMaterialOffset = 0 + for (i in 0 until keyingMaterialValues.size) { + val keyingMaterialValue = keyingMaterialValues[i] + + System.arraycopy( + keyingMaterial, + keyingMaterialOffset, + keyingMaterialValue, + 0, + keyingMaterialValue.size + ) + keyingMaterialOffset += keyingMaterialValue.size + } + + val srtcpPolicy = SrtpPolicy( + srtpProfileInformation.cipherName, + srtpProfileInformation.cipherKeyLength, + srtpProfileInformation.authFunctionName, + srtpProfileInformation.authKeyLength, + srtpProfileInformation.rtcpAuthTagLength, + srtpProfileInformation.cipherSaltLength + ) + val srtpPolicy = SrtpPolicy( + srtpProfileInformation.cipherName, + srtpProfileInformation.cipherKeyLength, + srtpProfileInformation.authFunctionName, + srtpProfileInformation.authKeyLength, + srtpProfileInformation.rtpAuthTagLength, + srtpProfileInformation.cipherSaltLength + ) + + /* To support RetransmissionSender.retransmitPlain, we need to disable + send-side SRTP replay protection. */ + /* TODO: disable this only in cases where we actually need to use retransmitPlain? */ + srtpPolicy.isSendReplayEnabled = false + + srtpPolicy.isCryptexEnabled = cryptex + + val clientSrtpContextFactory = SrtpContextFactory( + tlsRole == TlsRole.CLIENT, + clientWriteSrtpMasterKey, + clientWriterSrtpMasterSalt, + srtpPolicy, + srtcpPolicy, + parentLogger + ) + val serverSrtpContextFactory = SrtpContextFactory( + tlsRole == TlsRole.SERVER, + serverWriteSrtpMasterKey, + serverWriterSrtpMasterSalt, + srtpPolicy, + srtcpPolicy, + parentLogger + ) + val forwardSrtpContextFactory: SrtpContextFactory + val reverseSrtpContextFactory: SrtpContextFactory + + when (tlsRole) { + TlsRole.CLIENT -> { + forwardSrtpContextFactory = clientSrtpContextFactory + reverseSrtpContextFactory = serverSrtpContextFactory + } + + TlsRole.SERVER -> { + forwardSrtpContextFactory = serverSrtpContextFactory + reverseSrtpContextFactory = clientSrtpContextFactory + } + } + + return SrtpTransformers( + SrtpDecryptTransformer(reverseSrtpContextFactory, parentLogger), + SrtpEncryptTransformer(forwardSrtpContextFactory, parentLogger), + SrtcpDecryptTransformer(reverseSrtpContextFactory, parentLogger), + SrtcpEncryptTransformer(forwardSrtpContextFactory, parentLogger) + ) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt new file mode 100644 index 0000000..a42a526 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/DelayStats.kt @@ -0,0 +1,41 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.utils.stats.BucketStats +import java.util.concurrent.atomic.LongAdder + +open class DelayStats(thresholds: List = defaultThresholds) : + BucketStats(thresholds, "_delay_ms", "_ms") { + + fun addDelay(delayMs: Long) = addValue(delayMs) + + companion object { + val defaultThresholds = listOf(0, 5, 50, 500, Long.MAX_VALUE) + } +} + +class PacketDelayStats(thresholds: List = defaultThresholds) : DelayStats(thresholds) { + private val numPacketsWithoutTimestamps = LongAdder() + + fun addUnknown() = numPacketsWithoutTimestamps.increment() + + override fun toJson(format: Format): ObjectNode = super.toJson(format).apply { + put("packets_without_timestamps", numPacketsWithoutTimestamps.sum()) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt new file mode 100644 index 0000000..89ada08 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/EndpointConnectionStats.kt @@ -0,0 +1,181 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtcp.RtcpListener +import org.jitsi.nlj.rtp.LossTracker +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpReportBlock +import org.jitsi.rtp.rtcp.RtcpRrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.utils.LRUCache +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.secs +import org.jitsi.utils.toDoubleMillis +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.Collections +import java.util.concurrent.CopyOnWriteArrayList + +/** + * The maximum number of SR packets and their timestamps to save. + */ +private const val MAX_SR_TIMESTAMP_HISTORY = 200 + +private data class SsrcAndTimestamp(val ssrc: Long, val timestamp: Long) + +/** + * Tracks stats which are not necessarily tied to send or receive but the endpoint overall + */ +class EndpointConnectionStats( + parentLogger: Logger, + private val clock: Clock = Clock.systemUTC() +) : RtcpListener { + interface EndpointConnectionStatsListener { + fun onRttUpdate(newRttMs: Double) + } + data class Snapshot( + val rtt: Double, + val incomingLossStats: LossTracker.Snapshot, + val outgoingLossStats: LossTracker.Snapshot + ) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("rtt", rtt) + set("incoming_loss_stats", incomingLossStats.toJson()) + set("outgoing_loss_stats", outgoingLossStats.toJson()) + } + } + + private val endpointConnectionStatsListeners: MutableList = CopyOnWriteArrayList() + + // Per-SSRC, maps the compacted NTP timestamp found in an SR SenderInfo to + // the clock time at which it was transmitted + private val srSentTimes: MutableMap = + Collections.synchronizedMap(LRUCache(MAX_SR_TIMESTAMP_HISTORY)) + private val logger = createChildLogger(parentLogger) + + private val lock = Any() + + /** + * The calculated RTT, in milliseconds, between the bridge and the endpoint + */ + private var rtt: Double = 0.0 + + val incomingLossTracker = LossTracker() + val outgoingLossTracker = LossTracker() + + fun addListener(listener: EndpointConnectionStatsListener) { + endpointConnectionStatsListeners.add(listener) + } + + fun removeListener(listener: EndpointConnectionStatsListener) { + endpointConnectionStatsListeners.remove(listener) + } + + fun getSnapshot(): Snapshot = synchronized(lock) { + Snapshot( + rtt = rtt, + incomingLossStats = incomingLossTracker.getSnapshot(), + outgoingLossStats = outgoingLossTracker.getSnapshot() + ) + } + + override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) { + when (packet) { + is RtcpSrPacket -> { + logger.cdebug { "Received SR packet with ${packet.reportBlocks.size} report blocks" } + packet.reportBlocks.forEach { reportBlock -> processReportBlock(receivedTime, reportBlock) } + } + + is RtcpRrPacket -> { + logger.cdebug { "Received RR packet with ${packet.reportBlocks.size} report blocks" } + packet.reportBlocks.forEach { reportBlock -> processReportBlock(receivedTime, reportBlock) } + } + } + } + + override fun rtcpPacketSent(packet: RtcpPacket) { + when (packet) { + is RtcpSrPacket -> { + logger.cdebug { + "Tracking sent SR packet with compacted timestamp " + + "${packet.senderInfo.compactedNtpTimestamp}" + } + val entry = SsrcAndTimestamp(packet.senderSsrc, packet.senderInfo.compactedNtpTimestamp) + srSentTimes[entry] = clock.instant() + } + } + } + + private fun processReportBlock(receivedTime: Instant?, reportBlock: RtcpReportBlock): Unit = synchronized(lock) { + if (reportBlock.lastSrTimestamp == 0L && reportBlock.delaySinceLastSr == 0L) { + logger.cdebug { + "Report block for ssrc ${reportBlock.ssrc} didn't have SR data: " + + "lastSrTimestamp was ${reportBlock.lastSrTimestamp}, " + + "delaySinceLastSr was ${reportBlock.delaySinceLastSr}" + } + return + } + if (receivedTime == null) { + logger.cdebug { + "Arrival time of report block is null, cannot calculate RTT" + } + return + } + // We need to know when we sent the last SR + srSentTimes[SsrcAndTimestamp(reportBlock.ssrc, reportBlock.lastSrTimestamp)]?.let { srSentTime -> + // The delaySinceLastSr value is given in 1/65536ths of a second, so divide it by .000065536 to get it + // in nanoseconds + val remoteProcessingDelay = Duration.ofNanos((reportBlock.delaySinceLastSr / .000065536).toLong()) + if (remoteProcessingDelay > Duration.ofMinutes(5)) { + logger.warn("Ignoring report block with suspiciously long DLSR: $remoteProcessingDelay") + return@let + } + + val newRtt = (Duration.between(srSentTime, receivedTime) - remoteProcessingDelay).toDoubleMillis() + if (newRtt > 7.secs.toMillis()) { + logger.warn( + "Ignoring suspiciously high rtt value: $newRtt ms, remote processing delay was " + + "$remoteProcessingDelay (${reportBlock.delaySinceLastSr}), srSentTime was $srSentTime, " + + "received time was $receivedTime" + ) + return@let + } + if (newRtt < 0) { + logger.warn( + "Negative rtt value: $newRtt ms, remote processing delay was " + + "$remoteProcessingDelay (${reportBlock.delaySinceLastSr}), srSentTime was $srSentTime, " + + "received time was $receivedTime" + ) + return@let + } + + rtt = newRtt + endpointConnectionStatsListeners.forEach { it.onRttUpdate(rtt) } + } ?: run { + logger.cdebug { + "No sent SR found for SSRC ${reportBlock.ssrc} and SR " + + "timestamp ${reportBlock.lastSrTimestamp}" + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/JitterStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/JitterStats.kt new file mode 100644 index 0000000..00e1503 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/JitterStats.kt @@ -0,0 +1,97 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import org.jitsi.nlj.PacketInfo +import java.time.Clock +import java.time.Duration +import java.time.Instant +import kotlin.math.abs + +open class JitterStats { + var jitter: Double = 0.0 + private set + + /** + * The timestamp of the previously received packet, converted to a millisecond timestamp based on the received + * RTP timestamp and the clock rate for that stream. + * 'previously received packet' here is as defined by the order in which the packets were received by this code, + * which may be different than the order according to sequence number. + */ + private var previousPacketReceivedTimestamp: Instant? = null + private var previousPacketSentTimestamp: Instant? = null + + fun addPacket(currentPacketSentTimestamp: Instant, currentPacketReceivedTimestamp: Instant) { + if (previousPacketReceivedTimestamp != null) { + jitter = calculateJitter( + jitter, + previousPacketSentTimestamp!!, + previousPacketReceivedTimestamp!!, + currentPacketSentTimestamp, + currentPacketReceivedTimestamp + ) + } + previousPacketSentTimestamp = currentPacketSentTimestamp + previousPacketReceivedTimestamp = currentPacketReceivedTimestamp + } + + companion object { + fun calculateJitter( + currentJitter: Double, + previousPacketSentTimestamp: Instant, + previousPacketReceivedTimestamp: Instant, + currentPacketSentTimestamp: Instant, + currentPacketReceivedTimestamp: Instant + ): Double { + /** + * If Si is the RTP timestamp from packet i, and Ri is the time of + * arrival in RTP timestamp units for packet i, then for two packets + * i and j, D may be expressed as + * + * D(i,j) = (Rj - Ri) - (Sj - Si) = (Rj - Sj) - (Ri - Si) + */ + // TODO(boris) take wraps into account + val delta = Duration.between(previousPacketSentTimestamp, previousPacketReceivedTimestamp) - + Duration.between(currentPacketSentTimestamp, currentPacketReceivedTimestamp) + + /* + * The interarrival jitter SHOULD be calculated continuously as each + * data packet i is received from source SSRC_n, using this + * difference D for that packet and the previous packet i-1 in order + * of arrival (not necessarily in sequence), according to the formula + * + * J(i) = J(i-1) + (|D(i-1,i)| - J(i-1))/16 + * + * Whenever a reception report is issued, the current value of J is + * sampled. + */ + return currentJitter + (abs(delta.toMillis()) - currentJitter) / 16.0 + } + } +} + +/** + * Tracks the jitter of packets *within* the bridge (not over the network) + */ +class BridgeJitterStats( + private val clock: Clock = Clock.systemUTC() +) : JitterStats() { + + fun packetSent(packetInfo: PacketInfo) { + packetInfo.receivedTime?.let { super.addPacket(it, clock.instant()) } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/NodeStatsBlock.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/NodeStatsBlock.kt new file mode 100644 index 0000000..a5a6f8f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/NodeStatsBlock.kt @@ -0,0 +1,220 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.util.appendLnIndent + +class NodeStatsBlock(val name: String) { + private val stats = mutableMapOf() + + /** + * Holds stats that are computed based on other values in the map (to e.g. calculate the + * ratio of two values). Restricted to [Number] because this makes it easier to implement and + * we don't need other values right now. + */ + private val compoundStats = mutableMapOf Number>() + + /** + * Adds a stat with a number value. Integral values are promoted to [Long], while floating point values are + * promoted to [Double]. + */ + fun addNumber(name: String, value: Number) { + promote(value)?.let { stats[name] = it } + } + + /** + * Adds a stat with a string value. + */ + fun addString(name: String, value: String) { + stats[name] = value + } + + /** + * Adds a stat with a boolean value. + */ + fun addBoolean(name: String, value: Boolean) { + stats[name] = value + } + + /** + * Adds another [NodeStatsBlock] as a child. + */ + fun addBlock(otherBlock: NodeStatsBlock) { + stats[otherBlock.name] = otherBlock + } + + /** + * Adds a block with a given name from a from a JSON-ish key-value object map. + */ + fun addJson(name: String, json: Map<*, *>) { + addBlock(fromJson(name, json)) + } + + fun addJson(name: String, json: ObjectNode) { + addBlock(fromJson(name, json)) + } + + /** + * Adds a named value to this [NodeStatsBlock] which is derived from other values in the block. + * The value will be calculated (by invoking the given function) when it is needed (e.g. in [getValue] or + * when exporting this block to another format (printing or JSON). + */ + fun addCompoundValue(name: String, compoundValue: (NodeStatsBlock) -> Number) { + compoundStats[name] = compoundValue + } + + fun getValue(name: String): Any? = when { + stats.containsKey(name) -> stats[name] + compoundStats.containsKey(name) -> compoundStats[name]?.invoke(this) + else -> null + } + + /** + * Gets the value of a stat with a given name, if this [NodeStatsBlock] has it and it is a [Number]. + * Otherwise returns 'null'. + */ + fun getNumber(name: String): Number? = when { + stats[name] is Number -> stats[name] as Number + compoundStats.containsKey(name) -> compoundStats[name]?.invoke(this) + else -> null + } + + fun getNumberOrDefault(name: String, default: Number): Number = getNumber(name) ?: default + + /** + * Aggregates another block into this one. That is, takes any stats with number values from the + * other block and updates the current block with the sum of the current and other value. + */ + fun aggregate(otherBlock: NodeStatsBlock) { + otherBlock.stats.forEach { (name, value) -> + val existingValue = stats[name] + // We only aggregate numbers, and we "only" handle Long and Double because we've already + // promoted them when adding. For other value types, we override with the new one. + when { + existingValue == null && (value is Long || value is Double) -> + stats[name] = value + + existingValue is Long && value is Long -> + stats[name] = existingValue + value + + existingValue is Double && value is Double -> + stats[name] = existingValue + value + + existingValue is Long && value is Double -> + stats[name] = existingValue + value + + existingValue is Double && value is Long -> + stats[name] = existingValue + value + + else -> stats[name] = value + } + } + otherBlock.compoundStats.forEach { (name, function) -> + addCompoundValue(name, function) + } + stats[AGGREGATES] = (stats.getOrDefault(AGGREGATES, 0L) as Long) + 1 + } + + /** + * Promotes integer values to [Long] and floating point values to [Double]. Returns a + * [Long], [Double], or null. + */ + private fun promote(n: Number): Number? = when (n) { + is Byte, is Short, is Int, is Long -> n.toLong() + is Float, is Double -> n.toDouble() + else -> null + } + + fun prettyPrint(indentLevel: Int = 0): String = with(StringBuffer()) { + appendLnIndent(indentLevel, name) + stats.forEach { (statName, statValue) -> + when (statValue) { + is NodeStatsBlock -> { + appendLine(statValue.prettyPrint(indentLevel + 2)) + } + + else -> { + // statValue is Any, so we know it's non-null + appendLnIndent(indentLevel + 2, "$statName: $statValue") + } + } + } + compoundStats.forEach { (statName, function) -> + val statValue = function.invoke(this@NodeStatsBlock) + appendLnIndent(indentLevel + 2, "$statName: $statValue") + } + toString() + } + + /** + * Returns a JSON representation of this [NodeStatsBlock]. + */ + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + stats.forEach { (name, value) -> + when (value) { + is NodeStatsBlock -> set(name, value.toJson()) + is Long -> put(name, value) + is Double -> put(name, value) + is Boolean -> put(name, value) + is String -> put(name, value) + else -> put(name, value.toString()) + } + } + compoundStats.forEach { (name, function) -> + val num = function.invoke(this@NodeStatsBlock) + when (num) { + is Long -> put(name, num) + is Double -> put(name, num) + else -> put(name, num.toLong()) + } + } + } + + companion object { + /** + * The stat name that we use to could the number of other block aggregated in this one. + */ + private const val AGGREGATES = "_aggregates" + + /** + * Creates a [NodeStatsBlock] from a JSON-ish key-value object map. It recognizes recursively included objects. + */ + fun fromJson(name: String, json: Map<*, *>): NodeStatsBlock = NodeStatsBlock(name).apply { + json.entries.forEach { (key, value) -> + when (value) { + is Map<*, *> -> addBlock(fromJson(key.toString(), value)) + is Number -> addNumber(key.toString(), value) + is Boolean -> addBoolean(key.toString(), value) + else -> addString(key.toString(), value.toString()) + } + } + } + + fun fromJson(name: String, json: ObjectNode): NodeStatsBlock = NodeStatsBlock(name).apply { + json.fields().forEach { (key, value) -> + when { + value.isObject -> addBlock(fromJson(key, value as ObjectNode)) + value.isNumber -> addNumber(key, value.numberValue()) + value.isBoolean -> addBoolean(key, value.booleanValue()) + else -> addString(key, value.asText()) + } + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt new file mode 100644 index 0000000..5bed72f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketIOActivity.kt @@ -0,0 +1,64 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import org.jitsi.nlj.util.latest +import org.jitsi.nlj.util.threadSafeVetoable +import org.jitsi.utils.NEVER +import java.time.Instant + +@Suppress("unused") +class PacketIOActivity { + /** + * The last time an RTP or RTCP packet was received. + */ + var lastRtpPacketReceivedInstant: Instant by threadSafeVetoable(NEVER) { _, oldValue, newValue -> + newValue.isAfter(oldValue) + } + + /** + * The last time an RTP or RTCP packet was received. + */ + var lastRtpPacketSentInstant: Instant by threadSafeVetoable(NEVER) { _, oldValue, newValue -> + newValue.isAfter(oldValue) + } + + /** + * The last time ICE consent was refreshed. + */ + var lastIceActivityInstant: Instant by threadSafeVetoable(NEVER) { _, oldValue, newValue -> + newValue.isAfter(oldValue) + } + + /** + * The last time an RTP or RTCP packet was sent or received. + */ + val lastRtpActivityInstant: Instant + get() = latest(lastRtpPacketReceivedInstant, lastRtpPacketSentInstant) + + /** + * The last time a packet was received (RTP, RTCP or ICE consent). + */ + val lastIncomingActivityInstant: Instant + get() = latest(lastRtpPacketReceivedInstant, lastIceActivityInstant) + + /** + * The last time a packet was sent or received. + */ + val lastActivityInstant: Instant + get() = latest(lastRtpPacketReceivedInstant, lastRtpPacketSentInstant, lastIceActivityInstant) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketStreamStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketStreamStats.kt new file mode 100644 index 0000000..1c0d1f9 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/PacketStreamStats.kt @@ -0,0 +1,96 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import org.jitsi.nlj.transform.node.incoming.BitrateCalculator +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bytes +import java.util.concurrent.atomic.AtomicLong + +/** + * Keeps tracks of the basic statistics for a stream of packets. + * + * @author Boris Grozev + */ +class PacketStreamStats { + /** + * The bitrate in bits per second. + */ + private val bitrate = BitrateCalculator.createBitrateTracker() + + /** + * The packet rate in packets per second. + */ + private val packetRate = BitrateCalculator.createRateTracker() + + /** + * Total total number of bytes. + */ + private val bytes = AtomicLong() + + /** + * Total total number of packets. + */ + private val packets = AtomicLong() + + fun update(lengthInBytes: Int) { + val now = System.currentTimeMillis() + + bitrate.update(lengthInBytes.bytes, now) + bytes.addAndGet(lengthInBytes.toLong()) + + packetRate.update(1, now) + packets.incrementAndGet() + } + + fun snapshot(): Snapshot { + val now = System.currentTimeMillis() + return Snapshot(bitrate.getRate(now), packetRate.getRate(now), bytes.get(), packets.get()) + } + + class Snapshot( + /** + * The current bitrate in bits per second. + */ + val bitrate: Bandwidth, + /** + * The current packet rate in packets per second. + */ + val packetRate: Long, + /** + * Total number of bytes. + */ + val bytes: Long, + /** + * Total number of packets. + */ + val packets: Long + ) { + fun toJson() = JsonNodeFactory.instance.objectNode().apply { + put("bitrate_bps", bitrate.bps) + put("packetrate", packetRate) + put("total_bytes", bytes) + put("total_packets", packets) + } + + /** + * Expose [bitrate] as a [Long] in order to make it accessible from java (since [Bandwidth] is an inline + * class). + */ + fun getBitrateBps(): Long = bitrate.bps + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/TransceiverStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/TransceiverStats.kt new file mode 100644 index 0000000..9a52625 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/stats/TransceiverStats.kt @@ -0,0 +1,52 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.transform.node.incoming.IncomingStatisticsSnapshot +import org.jitsi.nlj.transform.node.incoming.VideoParser +import org.jitsi.nlj.transform.node.outgoing.OutgoingStatisticsSnapshot + +data class TransceiverStats( + val endpointConnectionStats: EndpointConnectionStats.Snapshot, + val rtpReceiverStats: RtpReceiverStats, + val outgoingStats: OutgoingStatisticsSnapshot, + val outgoingPacketStreamStats: PacketStreamStats.Snapshot, + val tccEngineStats: TransportCcEngine.StatisticsSnapshot +) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("endpoint_connection_stats", endpointConnectionStats.toJson()) + set("rtp_receiver_stats", rtpReceiverStats.toJson()) + set("outgoing_stats", outgoingStats.toJson()) + set("outgoing_packet_stream_stats", outgoingPacketStreamStats.toJson()) + set("tcc_engine_stats", tccEngineStats.toJson()) + } +} + +data class RtpReceiverStats( + val incomingStats: IncomingStatisticsSnapshot, + val packetStreamStats: PacketStreamStats.Snapshot, + val videoParserStats: VideoParser.Stats.Snapshot +) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("incoming_stats", incomingStats.toJson()) + set("packet_stream_stats", packetStreamStats.toJson()) + set("video_parser_stats", videoParserStats.toJson()) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/NodeStatsProducer.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/NodeStatsProducer.kt new file mode 100644 index 0000000..3bc9a7a --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/NodeStatsProducer.kt @@ -0,0 +1,22 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform + +import org.jitsi.nlj.stats.NodeStatsBlock + +interface NodeStatsProducer { + fun getNodeStats(): NodeStatsBlock +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/NodeVisitor.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/NodeVisitor.kt new file mode 100644 index 0000000..1726ed2 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/NodeVisitor.kt @@ -0,0 +1,112 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.Event +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.DemuxerNode +import org.jitsi.nlj.transform.node.Node + +abstract class NodeVisitor { + open fun visit(node: Node) { + doWork(node) + node.getChildren().forEach { this.visit(it) } + } + + /** + * [reverseVisit] is used for traversing an 'outgoing'-style tree which + * has many input paths but terminates at a single point (as opposed to an + * 'incoming'-style tree which starts at a single point and then branches + * into multiple paths. With reverseVisit, we start at the single terminating + * point and traverse backwards through the tree. It should be noted, however, + * that reverseVisit is done in a 'postorder' traversal style (meaning that a [Node]'s + * 'inputNodes' are visited before that Node itself. + * TODO: protect against visiting the same node twice in the event of a cycle (which + * we should do for 'visit' as well) + */ + open fun reverseVisit(node: Node) { + // NOTE(brian): although we're doing a reverse visit here, we still visit the nodes + // in 'forward' order by going up through all the parents first and then calling + // doWork on our way back 'down', as it's usually more useful to do things like + // view the stats in the 'normal' order and just use the 'reverse' visit as a + // way to handle the 'outgoing' tree style (as mentioned in the comment above) + node.getParents().forEach { this.reverseVisit(it) } + doWork(node) + } + protected abstract fun doWork(node: Node) +} + +class NodeStatsVisitor(val nodeStatsBlock: NodeStatsBlock) : NodeVisitor() { + override fun doWork(node: Node) { + val block = node.getNodeStats() + nodeStatsBlock.addBlock(block) + } +} + +class NodeDebugStateVisitor(val o: ObjectNode, val mode: DebugStateMode) : NodeVisitor() { + override fun doWork(node: Node) { + val debugState = when (mode) { + DebugStateMode.FULL -> node.getNodeStats().toJson() + else -> node.statsJson() + } + if (!debugState.isEmpty) { + o.set(node.name, debugState) + } + } +} + +class NodeEventVisitor(val event: Event) : NodeVisitor() { + override fun doWork(node: Node) { + node.handleEvent(event) + } +} + +/** + * Produces a set of all notes in the tree. + */ +class NodeSetVisitor(val nodeSet: MutableSet = mutableSetOf()) : NodeVisitor() { + override fun doWork(node: Node) { + nodeSet.add(node) + } +} + +class NodeTeardownVisitor : NodeVisitor() { + override fun doWork(node: Node) { + node.stop() + when (node) { + is DemuxerNode -> node.removePacketPaths() + else -> node.detachNext() + } + } + + override fun visit(node: Node) { + node.getChildren().forEach { this.visit(it) } + doWork(node) + } + + override fun reverseVisit(node: Node) { + // We can't use the default reverseVisit method, because in doWork + // we modify the parents list, so we'll get a ConcurrentModificationException. + // So instead we override the method here and make a copy of the parents + // and use that to iterate over so we can modify the real one + val parentsCopy = node.getParents().toList() + parentsCopy.forEach { this.reverseVisit(it) } + doWork(node) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt new file mode 100644 index 0000000..c5aec68 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/PipelineDsl.kt @@ -0,0 +1,79 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ConditionalPacketPath +import org.jitsi.nlj.transform.node.DemuxerNode +import org.jitsi.nlj.transform.node.ExclusivePathDemuxer +import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.rtp.Packet + +// TODO: look into @DslMarker to prevent inner dsl builders from accidentally setting parent +// member variables when they overlap + +fun DemuxerNode.packetPath(b: ConditionalPacketPath.() -> Unit) { + this.addPacketPath(ConditionalPacketPath().apply(b)) +} + +class PipelineBuilder { + private var head: Node? = null + private var tail: Node? = null + + private fun addNode(node: Node) { + if (head == null) { + head = node + } + if (tail is DemuxerNode) { + // In the future we could separate 'input/output' nodes from purely + // input nodes and use that here? + throw Exception("Cannot attach node to a DemuxerNode") + } + tail?.attach(node) + tail = node + } + + fun node(node: Node, condition: () -> Boolean = { true }) { + if (condition()) { + addNode(node) + } + } + + /** + * simpleNode allows the caller to pass in a block of code which takes a list of input + * [Packet]s and returns a list of output [Packet]s to be forwarded to the next + * [Node] + */ + fun simpleNode(name: String, packetHandler: (PacketInfo) -> PacketInfo?) { + val node = object : TransformerNode(name) { + override fun transform(packetInfo: PacketInfo): PacketInfo? = packetHandler.invoke(packetInfo) + + override val aggregationKey = this.name + override fun trace(f: () -> Unit) = f.invoke() + } + addNode(node) + } + + fun demux(name: String, block: DemuxerNode.() -> Unit) { + val demuxer = ExclusivePathDemuxer(name).apply(block) + addNode(demuxer) + } + + fun build(): Node = head!! +} + +fun pipeline(block: PipelineBuilder.() -> Unit): Node = PipelineBuilder().apply(block).build() diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt new file mode 100644 index 0000000..a08a713 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/AudioRedHandler.kt @@ -0,0 +1,356 @@ +/* + * Copyright @ 2019 - present 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.format.AudioRedPayloadType +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.RedAudioRtpPacket +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.RtpPacketCache +import org.jitsi.rtp.extensions.toHex +import org.jitsi.rtp.rtp.RedundancyBlockHeader +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.AudioLevelHeaderExtension +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.RtpUtils.Companion.getTimestampDiffAsInt +import org.jitsi.utils.LRUCache +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +class AudioRedHandler( + streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : MultipleOutputTransformerNode("RedHandler") { + + private val logger = createChildLogger(parentLogger) + private val stats = Stats() + + var audioLevelExtId: Int? = null + var redPayloadType: Int? = null + + /** + * Per-SSRC RED handlers, keyed on the RTP SSRC. Bounded with an LRU to limit the number of SSRCs tracked; evicted + * handlers are stopped to release their cached buffers. + */ + private val ssrcRedHandlers: MutableMap = + object : LRUCache(MAX_SSRCS, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry): Boolean = + super.removeEldestEntry(eldest).also { if (it) eldest.value.stop() } + } + + init { + streamInformationStore.onRtpPayloadTypesChanged { payloadTypes -> + redPayloadType = payloadTypes.values.find { it is AudioRedPayloadType }?.pt?.toInt() + } + streamInformationStore.onRtpExtensionMapping(RtpExtensionType.SSRC_AUDIO_LEVEL) { + audioLevelExtId = it + } + } + + override fun transform(packetInfo: PacketInfo): List { + val audioPacket = packetInfo.packet as? AudioRtpPacket ?: return listOf(packetInfo) + + val ssrcHandler = ssrcRedHandlers.computeIfAbsent(audioPacket.ssrc) { SsrcRedHandler() } + + return when (audioPacket) { + is RedAudioRtpPacket -> ssrcHandler.transformRed(packetInfo) + else -> listOf(ssrcHandler.transformAudio(packetInfo)) + } + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addString("red_payload_type", redPayloadType?.toString() ?: "null") + addString("audio_level_ext_id", audioLevelExtId?.toString() ?: "null") + addString("policy", config.policy.toString()) + addString("distance", config.distance.toString()) + addBoolean("vad_only", config.vadOnly) + + addNumber("red_packets_decapsulated", stats.redPacketsDecapsulated) + addNumber("red_packets_forwarded", stats.redPacketsForwarded) + addNumber("audio_packets_encapsulated", stats.audioPacketsEncapsulated) + addNumber("audio_packets_forwarded", stats.audioPacketsForwarded) + addNumber("lost_packets_recovered", stats.lostPacketsRecovered) + addNumber("redundancy_packets_added", stats.redundancyPacketsAdded) + addNumber("invalid_red_packets_dropped", stats.invalidRedPacketsDropped) + } + + override fun statsJson() = super.statsJson().apply { + put("red_packets_decapsulated", stats.redPacketsDecapsulated) + put("red_packets_forwarded", stats.redPacketsForwarded) + put("audio_packets_encapsulated", stats.audioPacketsEncapsulated) + put("audio_packets_forwarded", stats.audioPacketsForwarded) + put("lost_packets_recovered", stats.lostPacketsRecovered) + put("redundancy_packets_added", stats.redundancyPacketsAdded) + put("invalid_red_packets_dropped", stats.invalidRedPacketsDropped) + } + + override fun stop() = super.stop().also { + ssrcRedHandlers.values.forEach { it.stop() } + ssrcRedHandlers.clear() + } + + /** + * Handler for a specific stream (SSRC) + */ + private inner class SsrcRedHandler { + /** + * Saves audio (non-RED) packets that we've sent. Packets are cloned on insert and the inserted copies are owned + * by the cache, meaning that the cache is responsible for returning their buffers to the pool. + * It needs to be synchronized because [stop] and [transformAudio] may run in different threads. + */ + val sentAudioCache = RtpPacketCache(20, synchronize = true) + + /** + * Process an incoming audio packet. It is either forwarded as it is, or encapsulated in RED with previous + * packets optionally added as redundancy. + */ + fun transformAudio(packetInfo: PacketInfo): PacketInfo { + // Whether we need to add RED encapsulation + val encapsulate = when (config.policy) { + RedPolicy.NOOP, RedPolicy.STRIP -> false + RedPolicy.PROTECT_ALL -> true + // RedPolicy.PROTECT_DOMINANT -> =isDominant + } + + val redPayloadType = redPayloadType + if (redPayloadType == null || !encapsulate) { + stats.audioPacketForwarded() + return packetInfo + } + + val audioRtpPacket = packetInfo.packetAs() + sentAudioCache.insert(audioRtpPacket) + + val redundancy = mutableListOf() + val seq = audioRtpPacket.sequenceNumber + + when (config.distance) { + RedDistance.ONE -> { + getPacketToProtect( + applySequenceNumberDelta(seq, -1), + audioRtpPacket.timestamp, + config.vadOnly + )?.also { secondary -> + redundancy.add(secondary) + stats.redundancyPacketAdded() + } + } + + RedDistance.TWO -> { + getPacketToProtect( + applySequenceNumberDelta(seq, -1), + audioRtpPacket.timestamp, + false + )?.also { secondary -> + // With distance 2 we only add the tertiary packet when there is a secondary available + // (regardless of secondary's VAD). This guarantees that the sequence numbers of the + // redundancy packets always directly proceed the primary packet, i.e. that we don't encode + // a packet with primary seq=N and a single redundancy with seq=N-2. This is be important + // when the receiver of the RED stream is another jitsi-videobridge instance (via Octo), + // which makes that assumption about the stream it receives. + val tertiary = getPacketToProtect( + applySequenceNumberDelta(seq, -2), + audioRtpPacket.timestamp, + config.vadOnly + ) + if (tertiary != null) { + redundancy.add(tertiary) + stats.redundancyPacketAdded() + redundancy.add(secondary) + stats.redundancyPacketAdded() + } else if (!config.vadOnly || secondary.hasVad()) { + // If there's no tertiary encode the secondary alone, but this time check its VAD. + redundancy.add(secondary) + stats.redundancyPacketAdded() + } + } + } + } + + val redPacket = RedAudioRtpPacket.builder.build(redPayloadType, audioRtpPacket, redundancy) + packetInfo.packet = redPacket + + // We replaced packetInfo.packet with our newly allocated packet, so the original can now be returned. + // We do not return the redundancy packets, because we only peek()ed at them from the cache. + BufferPool.returnBuffer(audioRtpPacket.buffer) + + stats.audioPacketEncapsulated() + return packetInfo + } + + private fun getPacketToProtect(seq: Int, primaryTimestamp: Long, vadOnly: Boolean): RtpPacket? { + // All of the transform pipeline runs in a single thread, and we only use the packet momentarily to make a + // copy into a new RED packet, so it's safe to just peek() at it. + sentAudioCache.peek(seq)?.item?.let { + // In vad-only mode, we only add redundancy for packets that have an audio level extension with the VAD + // bit set. + if (!vadOnly || it.hasVad()) { + // Don't attempt to encode packets with timestamp diff that's too large to encode (happens with + // 400ms opus frames e.g. when DTX is used) + if (getTimestampDiffAsInt(primaryTimestamp, it.timestamp) <= + RedundancyBlockHeader.MAX_TIMESTAMP_OFFSET + ) { + return it + } + } + } + return null + } + + private fun RtpPacket.hasVad(): Boolean = audioLevelExtId?.let { extId -> + getHeaderExtension(extId)?.let { AudioLevelHeaderExtension.getVad(it) } ?: false + } ?: false + + fun stop() = sentAudioCache.flush() + + /** + * Process an incoming RED packet. Depending on the configured policy and whether the receiver supports the + * RED format, it is either forwarded as it is or it is "stripped" to its primary encoding, with redundancy + * blocks being read if there are non-received packets. + */ + fun transformRed(packetInfo: PacketInfo): List { + // Whether we need to strip the RED encapsulation + val strip = when (redPayloadType) { + null -> true + + else -> when (config.policy) { + RedPolicy.STRIP -> true + + // RedPolicy.PROTECT_DOMINANT -> !isDominant + RedPolicy.NOOP, RedPolicy.PROTECT_ALL -> false + } + } + + return if (strip) { + buildList { + val redPacket = packetInfo.packetAs() + + val seq = redPacket.sequenceNumber + val prev = applySequenceNumberDelta(seq, -1) + val prev2 = applySequenceNumberDelta(seq, -2) + val prevMissing = !sentAudioCache.contains(prev) + val prev2Missing = !sentAudioCache.contains(prev2) + + try { + if (prevMissing || prev2Missing) { + redPacket.removeRedAndGetRedundancyPackets().forEach { + if ((it.sequenceNumber == prev && prevMissing) || + (it.sequenceNumber == prev2 && prev2Missing) + ) { + val packetInfo = PacketInfo(it) + packetInfo.packetOrigin = PacketOrigin.Synthesized + add(packetInfo) + stats.lostPacketRecovered() + } + sentAudioCache.insert(it) + } + } else { + redPacket.removeRed() + } + } catch (e: IllegalArgumentException) { + logger.warn( + "Dropping invalid RED packet from ep=${packetInfo.endpointId} (${e.message}): " + + "$redPacket. Contents (50B): ${redPacket.toHex(50)}" + ) + stats.invalidRedPacketDropped() + return@buildList + } + + stats.redPacketDecapsulated() + packetInfo.packet = redPacket.toOtherType(::AudioRtpPacket) + + // It's possible we already forwarded the primary packet if we recovered it from a previously received + // packet. + if (!sentAudioCache.contains(seq)) { + sentAudioCache.insert(packetInfo.packetAs()) + add(packetInfo) + } + } + } else { + stats.redPacketForwarded() + listOf(packetInfo) + } + } + } + + companion object { + val config = Config() + + /** The maximum number of SSRCs to track RED handlers for. */ + const val MAX_SSRCS = 1024 + } +} + +enum class RedPolicy { + /** + * No change. + */ + NOOP, + + /** + * Always strip. + */ + STRIP, + + /** + * Add RED for all endpoints. + */ + PROTECT_ALL, + // TODO + // /** + // * Add redundancy for the dominant (if missing), strip it for everyone else (if present). + // */ + // PROTECT_DOMINANT +} + +enum class RedDistance { + ONE, + TWO +} + +data class Stats( + var redPacketsDecapsulated: Int = 0, + var redPacketsForwarded: Int = 0, + var invalidRedPacketsDropped: Int = 0, + var audioPacketsEncapsulated: Int = 0, + var audioPacketsForwarded: Int = 0, + var lostPacketsRecovered: Int = 0, + var redundancyPacketsAdded: Int = 0 +) { + fun redPacketDecapsulated() = redPacketsDecapsulated++ + fun redPacketForwarded() = redPacketsForwarded++ + fun invalidRedPacketDropped() = invalidRedPacketsDropped++ + fun audioPacketEncapsulated() = audioPacketsEncapsulated++ + fun audioPacketForwarded() = audioPacketsForwarded++ + fun lostPacketRecovered() = lostPacketsRecovered++ + fun redundancyPacketAdded() = redundancyPacketsAdded++ +} + +class Config { + val policy: RedPolicy by config { "jmt.audio.red.policy".from(JitsiConfig.newConfig) } + val distance: RedDistance by config { "jmt.audio.red.distance".from(JitsiConfig.newConfig) } + val vadOnly: Boolean by config { "jmt.audio.red.vad-only".from(JitsiConfig.newConfig) } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt new file mode 100644 index 0000000..f92d12c --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/Node.kt @@ -0,0 +1,576 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.Event +import org.jitsi.nlj.EventHandler +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.Stoppable +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.NodeStatsProducer +import org.jitsi.nlj.transform.NodeVisitor +import org.jitsi.nlj.transform.node.debug.PayloadVerificationPlugin +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.PacketPredicate +import org.jitsi.nlj.util.addMbps +import org.jitsi.nlj.util.addRatio +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArraySet +import java.util.function.Predicate +import kotlin.properties.Delegates + +/** + * An abstract base class for all [Node] subclasses. This class + * takes care of the following behaviors: + * 1) Attaching the next node in the chain + * 2) Adding and removing parent nodes + * 3) Propagating [visit] calls + */ +sealed class Node( + var name: String +) : PacketHandler, EventHandler, NodeStatsProducer, Stoppable { + + private var nextNode: Node? = null + private val inputNodes: MutableList by lazy { mutableListOf() } + + // Create these once here so we don't allocate a new string every time + protected val nodeEntryString = "Entered node $name" + protected val nodeExitString = "Exited node $name" + + open fun statsJson(): ObjectNode = JsonNodeFactory.instance.objectNode() + + open fun visit(visitor: NodeVisitor) { + visitor.visit(this) + } + + /** + * Marking this as open since [DemuxerNode] wants to throw an exception + * if attach is called. + */ + open fun attach(node: Node): Node { + // Remove ourselves as an input from the node we're currently connected to + if (nextNode != null) { + throw Exception("Attempt to replace a Node's child. If this is intentional, use detachNext first.") + } + nextNode = node + node.addParent(this) + + return node + } + + open fun detachNext() { + nextNode?.removeParent(this) + nextNode = null + } + + fun addParent(newParent: Node) { + inputNodes.add(newParent) + } + + fun removeParent(parent: Node) { + inputNodes.remove(parent) + } + + open fun getChildren(): Collection { + val actualNextNode = nextNode ?: return listOf() + return listOf(actualNextNode) + } + + open fun getParents(): Collection = inputNodes + + override fun handleEvent(event: Event) { + // No-op by default + } + + override fun stop() { + // No-op by default + } + + protected fun next(packetInfo: PacketInfo) { + if (PLUGINS_ENABLED) { + plugins.forEach { it.observe(this, packetInfo) } + } + nextNode?.processPacket(packetInfo) + } + + protected fun next(packetInfos: List) { + packetInfos.forEach { packetInfo -> + if (PLUGINS_ENABLED) { + plugins.forEach { it.observe(this, packetInfo) } + } + nextNode?.processPacket(packetInfo) + } + } + + /** + * This function must be implemented by leaf nodes, as + * ``` + * override fun trace(f: () -> Unit) = f.invoke() + * ``` + * or the Java equivalent. When [NODE_TRACING] is + * turned on, this ensures that call stacks always include an method from + * the derived class, rather than one of the parent classes. This can greatly + * aid debugging and profiling. + */ + + abstract fun trace(f: () -> Unit) + + companion object { + @field:Suppress("ktlint:standard:property-naming") + var TRACE_ENABLED = false + + @field:Suppress("ktlint:standard:property-naming") + var PLUGINS_ENABLED = false + + // 'Plugins' are observers which, when enabled, will be passed every packet that passes through + // every node + val plugins: MutableSet = mutableSetOf() + + fun enablePayloadVerification(enable: Boolean) { + if (enable) { + PLUGINS_ENABLED = true + plugins.add(PayloadVerificationPlugin) + PacketInfo.enablePayloadVerification = true + } else { + plugins.remove(PayloadVerificationPlugin) + PLUGINS_ENABLED = plugins.isNotEmpty() + PacketInfo.enablePayloadVerification = false + } + } + + fun isPayloadVerificationEnabled(): Boolean = PacketInfo.enablePayloadVerification + + fun enableNodeTracing(enable: Boolean) { + TRACE_ENABLED = enable + } + + fun isNodeTracingEnabled(): Boolean = TRACE_ENABLED + } +} + +/** + * A [Node] which keeps track of some basic statistics, such as number of packets and bytes that passed through + * it, and the processing time. In order to accurately compute the processing time, this class depends on its + * subclasses calling [doneProcessing] when they finish their own processing of the packet, but before they pass the + * packet to any children. The intention is for this class to not be subclassed directly, except from classes defined + * in this file (but making it 'private' doesn't seem possible). + */ +sealed class StatsKeepingNode(name: String) : Node(name) { + /** + * The time at which processing of the currently processed packet started (in nanos). + */ + private var startTime: Long = 0 + + /** + * The time (in nanos) that [processPacket] was first called. + */ + private var firstPacketTime: Long = -1 + + /** + * The time (in nanos) that [processPacket] was last called. + */ + private var lastPacketTime: Long = -1 + + /** + * Keeps stats for this [Node] + */ + private val stats = NodeStats() + + /** + * Avoid stopping more than once. + */ + private var stopped = false + + /** + * The function that all subclasses should implement to do the actual + * packet processing. A protected method is used for this so we can + * guarantee all packets pass through this base for stat-tracking + * purposes. + */ + protected abstract fun doProcessPacket(packetInfo: PacketInfo) + + override fun processPacket(packetInfo: PacketInfo) { + onEntry(packetInfo) + if (TRACE_ENABLED) { + trace { doProcessPacket(packetInfo) } + } else { + doProcessPacket(packetInfo) + } + } + + override fun getNodeStats() = NodeStatsBlock("Node $name ${hashCode()}").apply { + this@StatsKeepingNode.stats.appendTo(this) + val numBytes = this@StatsKeepingNode.stats.numInputBytes + + val duration = Duration.ofNanos(lastPacketTime - firstPacketTime) + addNumber("num_input_bytes", numBytes) + addNumber("duration_ms", duration.toMillis()) + addMbps("throughput_mbps", "num_input_bytes", "duration_ms") + } + + /** + * Get the stats that should be aggregated per-class. + */ + protected open fun getNodeStatsToAggregate() = getNodeStats() + protected open val aggregationKey: String = javaClass.name.split(".").last() + + private fun onEntry(packetInfo: PacketInfo) { + if (enableStatistics) { + startTime = System.nanoTime() + if (firstPacketTime == -1L) { + firstPacketTime = startTime + } + + stats.numInputPackets++ + stats.numInputBytes += packetInfo.packet.length + + packetInfo.addEvent(nodeEntryString) + lastPacketTime = startTime + } + } + + /** + * Should be called by sub classes when they finish processing of the input packet, but before they call into any + * other nodes, so that [Node] can keep track of its statistics. + */ + protected fun doneProcessing(packetInfo: PacketInfo?) { + if (enableStatistics) { + val processingDuration = System.nanoTime() - startTime + stats.totalProcessingDurationNs += processingDuration + stats.maxProcessingDurationNs = Math.max(stats.maxProcessingDurationNs, processingDuration) + + packetInfo?.let { + stats.numOutputPackets++ + it.addEvent(nodeExitString) + } + } + } + + /** + * Should be called by sub classes when they finish processing of the input packet, but before they call into any + * other nodes, so that [Node] can keep track of its statistics. + */ + protected fun doneProcessing(packetInfos: List) { + if (enableStatistics) { + val processingDuration = System.nanoTime() - startTime + stats.totalProcessingDurationNs += processingDuration + stats.maxProcessingDurationNs = Math.max(stats.maxProcessingDurationNs, processingDuration) + + stats.numOutputPackets += packetInfos.size + packetInfos.forEach { + it.addEvent(nodeExitString) + } + } + } + + protected open fun packetDiscarded(packetInfo: PacketInfo) { + stats.numDiscardedPackets++ + BufferPool.returnBuffer(packetInfo.packet.buffer) + } + + override fun stop() { + if (stopped) { + return + } + stopped = true + + if (enableStatistics && stats.numInputPackets > 0) { + synchronized(globalStats) { + val classStats = globalStats.computeIfAbsent(aggregationKey) { NodeStatsBlock(aggregationKey) } + classStats.aggregate(getNodeStatsToAggregate()) + } + } + } + + companion object { + /** + * Maps a [Node]'s class name to a [NodeStats] object with aggregated stats for all instances of that class. + */ + private val globalStats: MutableMap = ConcurrentHashMap() + + var enableStatistics = true + + /** + * Gets the aggregated statistics for all classes as a JSON map. + */ + fun getStatsJson(): ObjectNode { + val jsonObject = JsonNodeFactory.instance.objectNode() + globalStats.forEach { (className, stats) -> + jsonObject.set(className, stats.toJson()) + } + jsonObject.put("num_payload_verification_failures", PayloadVerificationPlugin.numFailures.get()) + return jsonObject + } + } + + /** + * This just holds the stats kept by [StatsKeepingNode] itself. + */ + data class NodeStats( + /** + * Total nanoseconds spent processing packets in this node. + */ + var totalProcessingDurationNs: Long = 0, + var numInputPackets: Long = 0, + var numOutputPackets: Long = 0, + var numInputBytes: Long = 0, + var numDiscardedPackets: Long = 0, + /** + * The longest time it took to process a single packet. + */ + var maxProcessingDurationNs: Long = 0 + ) { + private val maxProcessingDurationMs: Double + get() = maxProcessingDurationNs / 1_000_000.0 + + fun appendTo(block: NodeStatsBlock) { + block.apply { + addNumber("num_input_packets", numInputPackets) + addNumber("num_output_packets", numOutputPackets) + addNumber("num_discarded_packets", numDiscardedPackets) + addNumber("total_time_spent_ns", totalProcessingDurationNs) + addCompoundValue("total_time_spent_ms") { + Duration.ofNanos(it.getNumberOrDefault("total_time_spent_ns", 0).toLong()).toMillis() + } + addRatio("average_time_per_packet_ns", "total_time_spent_ns", "num_input_packets") + addMbps("processing_throughput_mbps", "num_input_bytes", "total_time_spent_ms") + addNumber("max_packet_process_time_ms", maxProcessingDurationMs) + } + } + fun appendTo(json: ObjectNode) { + json.put("num_input_packets", numInputPackets) + json.put("num_output_packets", numOutputPackets) + json.put("num_discarded_packets", numDiscardedPackets) + json.put("total_time_spent_ns", totalProcessingDurationNs) + json.put("max_packet_process_time_ms", maxProcessingDurationMs) + } + } +} + +/** + * A node which will always forward the [PacketInfo] it is given. + * NOTE that the [PacketInfo] instance may be modified, but only the original + * [PacketInfo] instance will be forwarded. + */ +abstract class NeverDiscardNode(name: String) : StatsKeepingNode(name) { + protected abstract fun handlePacket(packetInfo: PacketInfo) + + final override fun doProcessPacket(packetInfo: PacketInfo) { + handlePacket(packetInfo) + doneProcessing(packetInfo) + next(packetInfo) + } +} + +/** + * A [Node] which transforms a single packet, possibly dropping it (by returning null). + * If null is returned, the [PacketInfo] instance given to [transform] will be + * discarded. + */ +abstract class TransformerNode(name: String) : StatsKeepingNode(name) { + protected abstract fun transform(packetInfo: PacketInfo): PacketInfo? + + override fun doProcessPacket(packetInfo: PacketInfo) { + val transformedPacket = transform(packetInfo) + doneProcessing(transformedPacket) + when (transformedPacket) { + null -> super.packetDiscarded(packetInfo) + else -> next(transformedPacket) + } + } + + final override fun packetDiscarded(packetInfo: PacketInfo): Unit = throw Exception( + "No subclass of TransformerNode should call packetDiscarded, return null from 'transform' instead" + ) +} + +/** A [TransformerNode] which gets its transformation function dynamically. */ +class PluggableTransformerNode( + name: String, + val transform: () -> ((PacketInfo) -> PacketInfo?)? +) : TransformerNode(name) { + override fun transform(packetInfo: PacketInfo): PacketInfo? { + transform()?.let { return it.invoke(packetInfo) } + return packetInfo + } + override fun trace(f: () -> Unit) {} + override val aggregationKey = this.name +} + +/** + * Unlike a [TransformerNode], [ModifierNode] modifies a packet in-place and never + * outright 'fails', meaning the original [PacketInfo] will *always* be forwarded. + */ +abstract class ModifierNode(name: String) : NeverDiscardNode(name) { + protected abstract fun modify(packetInfo: PacketInfo): PacketInfo + + override fun handlePacket(packetInfo: PacketInfo) { + modify(packetInfo) + } +} + +/** + * A [Node] which drops some of the packets (the ones which are not accepted). + */ +abstract class FilterNode(name: String) : TransformerNode(name) { + protected abstract fun accept(packetInfo: PacketInfo): Boolean + + override fun transform(packetInfo: PacketInfo): PacketInfo? = if (accept(packetInfo)) { + packetInfo + } else { + null + } +} + +typealias PacketInfoPredicate = Predicate +abstract class PredicateFilterNode( + name: String, + val predicate: PacketInfoPredicate +) : FilterNode(name) { + override fun accept(packetInfo: PacketInfo): Boolean = predicate.test(packetInfo) +} + +/** + * A [Node] which observes packets, but makes no modifications. + */ +abstract class ObserverNode(name: String) : NeverDiscardNode(name) { + + protected abstract fun observe(packetInfo: PacketInfo) + + override fun handlePacket(packetInfo: PacketInfo) { + observe(packetInfo) + } +} + +/** + * A node which consumes all packets (i.e. does something with them, but does not forward them to another node). + */ +abstract class ConsumerNode(name: String) : StatsKeepingNode(name) { + + protected abstract fun consume(packetInfo: PacketInfo) + + override fun doProcessPacket(packetInfo: PacketInfo) { + consume(packetInfo) + doneProcessing(packetInfo) + } + + // Consumer nodes shouldn't have children, because they don't forward + // any packets anyway. + override fun attach(node: Node): Node = + throw Exception("ConsumerNode must be a terminal and should not have child nodes attached.") +} + +/** + * A [Node] which transforms a single packet into a list of packets. + */ +abstract class MultipleOutputTransformerNode(name: String) : StatsKeepingNode(name) { + + protected abstract fun transform(packetInfo: PacketInfo): List + + override fun doProcessPacket(packetInfo: PacketInfo) { + val outputPacketInfos = transform(packetInfo) + doneProcessing(outputPacketInfos) + next(outputPacketInfos) + } +} + +class ConditionalPacketPath() { + var name: String by Delegates.notNull() + var predicate: PacketPredicate by Delegates.notNull() + var path: Node by Delegates.notNull() + var packetsAccepted: Int = 0 + + constructor(name: String) : this() { + this.name = name + } +} + +abstract class DemuxerNode(name: String) : StatsKeepingNode("$name demuxer") { + protected var transformPaths: MutableSet = CopyOnWriteArraySet() + + fun addPacketPath(packetPath: ConditionalPacketPath): DemuxerNode { + transformPaths.add(packetPath) + // We want to make sure the paths correctly see this Demuxer in their 'inputNodes' so that we can traverse the + // reverse tree correctly. + packetPath.path.addParent(this) + + return this + } + + fun addPacketPath(name: String, predicate: PacketPredicate, root: Node): DemuxerNode { + val path = ConditionalPacketPath(name) + path.predicate = predicate + path.path = root + + return addPacketPath(path) + } + + fun removePacketPaths() { + transformPaths.forEach { it.path.removeParent(this) } + transformPaths.clear() + } + + override fun attach(node: Node) = throw Exception() + override fun detachNext() = throw Exception() + + override fun visit(visitor: NodeVisitor) { + visitor.visit(this) + transformPaths.forEach { conditionalPath -> + conditionalPath.path.visit(visitor) + } + } + + override fun getChildren(): Collection = transformPaths.map { it.path } + + override fun getNodeStats(): NodeStatsBlock { + val superStats = super.getNodeStats() + + transformPaths.forEach { path -> + superStats.addNumber("packets_accepted_${path.name}", path.packetsAccepted) + } + return superStats + } + + override fun statsJson() = super.statsJson().apply { + transformPaths.forEach { path -> + put("packets_accepted_${path.name}", path.packetsAccepted) + } + } +} + +/** + * Packets are passed only to the first path which accepts them + */ +class ExclusivePathDemuxer(name: String) : DemuxerNode(name) { + override fun doProcessPacket(packetInfo: PacketInfo) { + transformPaths.forEach { conditionalPath -> + if (conditionalPath.predicate.test(packetInfo.packet)) { + doneProcessing(packetInfo) + conditionalPath.packetsAccepted++ + conditionalPath.path.processPacket(packetInfo) + return + } + } + packetDiscarded(packetInfo) + } + override val aggregationKey = this.name + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/NodePlugin.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/NodePlugin.kt new file mode 100644 index 0000000..7b0bd10 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/NodePlugin.kt @@ -0,0 +1,34 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node + +import org.jitsi.nlj.PacketInfo + +/** + * A [NodePlugin] can be statically added to [Node] so that it + * can function as an observer between every node created. This + * is useful for debugging things like when the backing array of + * a packet changes throughout the pipeline. + */ +interface NodePlugin { + /** + * Invoked in between every node in any pipeline. + * [after] is the [Node] which just processed the packet. + * [packetInfo] is the [PacketInfo] being passed from one node to the next. + */ + fun observe(after: Node, packetInfo: PacketInfo) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt new file mode 100644 index 0000000..9307602 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketCacher.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.util.PacketCache + +class PacketCacher : ObserverNode("Packet cache") { + private val packetCache = PacketCache() + + override fun observe(packetInfo: PacketInfo) { + packetCache.insert(packetInfo.packetAs()) + } + + override fun stop() { + packetCache.stop() + super.stop() + } + + override fun trace(f: () -> Unit) = f.invoke() + + fun getPacketCache(): PacketCache = packetCache + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + // Put the values directly in the node stats and not inside a block to make aggregation work correctly. + aggregate(packetCache.getNodeStats()) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt new file mode 100644 index 0000000..28afe92 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketLossNode.kt @@ -0,0 +1,85 @@ +/* + * Copyright @ 2020 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.util.addRatio +import kotlin.random.Random + +/** + * A [Node] which drops packets randomly with a certain probability and a uniform distribution. + */ +class PacketLossNode(val config: PacketLossConfig) : FilterNode("PacketLossNode($config)") { + private val random = Random(System.currentTimeMillis()) + private var inBurst = false + private var packetsSeen = 0 + private var currentBurstPacketsDropped = 0 + + override fun accept(packetInfo: PacketInfo) = acceptBurst() && random.nextDouble() >= config.uniformRate + + private fun acceptBurst(): Boolean { + if (!config.burstEnabled) return true + + packetsSeen++ + if (packetsSeen % config.burstInterval == 0) { + inBurst = true + } + + return if (inBurst) { + currentBurstPacketsDropped++ + if (currentBurstPacketsDropped == config.burstSize) { + inBurst = false + currentBurstPacketsDropped = 0 + } + false + } else { + true + } + } + + override fun trace(f: () -> Unit) { } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("configured_uniform_rate", config.uniformRate) + addNumber("configured_burst_size", config.burstSize) + addNumber("configured_burst_interval", config.burstInterval) + addRatio("actual_drop_rate", "num_discarded_packets", "num_input_packets") + } + + override fun getNodeStatsToAggregate(): NodeStatsBlock = super.getNodeStatsToAggregate().apply { + addRatio("actual_drop_rate", "num_discarded_packets", "num_input_packets") + } +} + +class PacketLossConfig(val base: String) { + val uniformRate: Double by config { + "$base.uniform-rate".from(JitsiConfig.newConfig) + } + val burstSize: Int by config { + "$base.burst-size".from(JitsiConfig.newConfig) + } + val burstInterval: Int by config { + "$base.burst-interval".from(JitsiConfig.newConfig) + } + + val burstEnabled = burstSize > 0 && burstInterval > 0 + val enabled = burstEnabled || uniformRate > 0 + + override fun toString() = "uniform rate ${uniformRate * 100}%, burst size $burstSize, burst interval $burstInterval" +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketParser.kt new file mode 100644 index 0000000..3c02752 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketParser.kt @@ -0,0 +1,40 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.rtp.Packet +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +open class PacketParser( + name: String, + parentLogger: Logger, + private val action: (Packet) -> Packet +) : TransformerNode(name) { + private val logger = createChildLogger(parentLogger) + + override fun transform(packetInfo: PacketInfo): PacketInfo? = try { + packetInfo.packet = action(packetInfo.packet) + packetInfo.resetPayloadVerification() + packetInfo + } catch (e: Exception) { + logger.warn("Error parsing packet: $e") + null + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketStreamStatsNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketStreamStatsNode.kt new file mode 100644 index 0000000..669aaf8 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PacketStreamStatsNode.kt @@ -0,0 +1,74 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.PacketStreamStats +import org.jitsi.nlj.util.appendAll +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger + +/** + * A [Node] which keeps track of the basic statistics for a stream of packets (packet and bit rates) + * + * @author Boris Grozev + */ +class PacketStreamStatsNode( + private val diagnosticContext: DiagnosticContext, + private val direction: String, + private val packetStreamStats: PacketStreamStats = PacketStreamStats() +) : ObserverNode("PacketStreamStats") { + + override fun observe(packetInfo: PacketInfo) { + packetStreamStats.update(packetInfo.packet.length) + } + + override fun trace(f: () -> Unit) = f.invoke() + + fun snapshot() = packetStreamStats.snapshot().also { + if (timeseriesLogger.isTraceEnabled) { + timeseriesLogger.trace( + diagnosticContext.makeTimeSeriesPoint("${direction}_packet_stream_stats") + .addField("bitrate_bps", it.bitrate.bps) + .addField("packet_rate", it.packetRate) + ) + } + } + + fun getBitrate() = snapshot().bitrate + + override fun statsJson() = super.statsJson().appendAll( + packetStreamStats.snapshot().toJson() + ) + + override fun getNodeStats() = super.getNodeStats().apply { + val snapshot = packetStreamStats.snapshot() + addNumber("bitrate_bps", snapshot.bitrate.bps) + addNumber("packet_rate", snapshot.packetRate) + addNumber("bytes_sent", snapshot.bytes) + addNumber("packets_sent", snapshot.packets) + } + + /** + * Creates a new [Node] instance which shares the same [packetStreamStats]. Useful when we want to add nodes to + * different branches of a [Node] tree. + */ + fun createNewNode() = PacketStreamStatsNode(diagnosticContext, direction, packetStreamStats) + + companion object { + private val timeseriesLogger = TimeSeriesLogger.getTimeSeriesLogger(PacketStreamStatsNode::class.java) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt new file mode 100644 index 0000000..133f821 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/PcapWriter.kt @@ -0,0 +1,135 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.nlj.PacketInfo +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cinfo +import org.jitsi.utils.logging2.createChildLogger +import org.pcap4j.core.Pcaps +import org.pcap4j.packet.EthernetPacket +import org.pcap4j.packet.IpV4Packet +import org.pcap4j.packet.IpV4Rfc1349Tos +import org.pcap4j.packet.UdpPacket +import org.pcap4j.packet.UnknownPacket +import org.pcap4j.packet.namednumber.DataLinkType +import org.pcap4j.packet.namednumber.EtherType +import org.pcap4j.packet.namednumber.IpNumber +import org.pcap4j.packet.namednumber.IpVersion +import org.pcap4j.packet.namednumber.UdpPort +import org.pcap4j.util.MacAddress +import java.net.Inet4Address +import java.nio.file.Path +import java.util.Random +import kotlin.io.path.Path + +class PcapWriter( + parentLogger: Logger, + filePath: Path = Path(directory, "${Random().nextLong()}.pcap") +) { + constructor(parentLogger: Logger, filePath: String) : this(parentLogger, Path(filePath)) + + private val logger = createChildLogger(parentLogger) + private val lazyHandle = lazy { + Pcaps.openDead(DataLinkType.EN10MB, 65536) + } + private val handle by lazyHandle + + private val lazyWriter = lazy { + logger.cinfo { "Pcap writer writing to file $filePath" } + handle.dumpOpen(filePath.toString()) + } + + private val writer by lazyWriter + + companion object { + private val localhost = Inet4Address.getByName("127.0.0.1") as Inet4Address + private val remotehost = Inet4Address.getByName("192.0.2.0") as Inet4Address + + private val localport = UdpPort(123, "blah") + private val remoteport = UdpPort(456, "blah") + + val directory: String by config("jmt.debug.pcap.directory".from(JitsiConfig.newConfig)) + } + + fun observe(packetInfo: PacketInfo, outbound: Boolean) = + observe(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length, outbound) + + fun observe(buffer: ByteArray, offset: Int, length: Int, outbound: Boolean) { + val udpPayload = UnknownPacket.Builder() + // We can't pass offset/limit values to udpPayload.rawData, so we need to create an array that contains + // only exactly what we want to write + val subBuf = ByteArray(length) + System.arraycopy(buffer, offset, subBuf, 0, length) + udpPayload.rawData(subBuf) + val srchost: Inet4Address + val dsthost: Inet4Address + val srcport: UdpPort + val dstport: UdpPort + if (outbound) { + srchost = localhost + srcport = localport + dsthost = remotehost + dstport = remoteport + } else { + srchost = remotehost + srcport = remoteport + dsthost = localhost + dstport = localport + } + + val udp = UdpPacket.Builder() + .srcPort(srcport) + .dstPort(dstport) + .srcAddr(srchost) + .dstAddr(dsthost) + .correctChecksumAtBuild(true) + .correctLengthAtBuild(true) + .payloadBuilder(udpPayload) + + val ipPacket = IpV4Packet.Builder() + .srcAddr(srchost) + .dstAddr(dsthost) + .protocol(IpNumber.UDP) + .version(IpVersion.IPV4) + .tos(IpV4Rfc1349Tos.newInstance(0)) + .correctLengthAtBuild(true) + .payloadBuilder(udp) + + val eth = EthernetPacket.Builder() + .srcAddr(MacAddress.ETHER_BROADCAST_ADDRESS) + .dstAddr(MacAddress.ETHER_BROADCAST_ADDRESS) + .type(EtherType.IPV4) + .paddingAtBuild(true) + .payloadBuilder(ipPacket) + .build() + + writer.dump(eth) + } + + fun close() { + if (lazyWriter.isInitialized() && writer.isOpen) { + writer.close() + } + + if (lazyHandle.isInitialized() && handle.isOpen) { + handle.close() + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt new file mode 100644 index 0000000..46d7733 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/RtpParser.kt @@ -0,0 +1,79 @@ +/* + * Copyright @ 2019 - present 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.PayloadTypeEncoding.RED +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.RedAudioRtpPacket +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpHeader +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger + +class RtpParser( + private val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : TransformerNode("RTP Parser") { + private val logger = createChildLogger(parentLogger) + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val packet = packetInfo.packet + val payloadTypeNumber = RtpHeader.getPayloadType(packet.buffer, packet.offset).toByte() + + val payloadType = streamInformationStore.rtpPayloadTypes[payloadTypeNumber] ?: run { + logger.cdebug { "Unknown payload type: $payloadTypeNumber" } + return null + } + packetInfo.payloadType = payloadType + + val rtpPacket = when (payloadType.mediaType) { + MediaType.AUDIO -> try { + when (payloadType.encoding) { + RED -> packet.toOtherType(::RedAudioRtpPacket) + else -> packet.toOtherType(::AudioRtpPacket) + } + } catch (e: Exception) { + logger.info("Dropping audio packet due to parse failure: ${e.message}") + return null + } + + MediaType.VIDEO -> try { + packet.toOtherType(::VideoRtpPacket) + } catch (e: Exception) { + logger.info("Dropping video packet due to parse failure: ${e.message}") + return null + } + + else -> { + logger.info("Dropping packet with unrecognized media type: '${payloadType.mediaType}'") + return null + } + } + packetInfo.packet = rtpPacket + if (rtpPacket.extensionsProfileType == 0xC0DE || rtpPacket.extensionsProfileType == 0xC2DE) { + packetInfo.originalHadCryptex = true + } + + packetInfo.resetPayloadVerification() + return packetInfo + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt new file mode 100644 index 0000000..0470006 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/SrtpTransformerNode.kt @@ -0,0 +1,174 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.srtp.AbstractSrtpTransformer +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.srtp.SrtpErrorStatus + +abstract class SrtpTransformerNode(name: String) : MultipleOutputTransformerNode(name) { + /** + * The function to use to use protect or unprotect a single SRT(C)P packet. + */ + var transformer: AbstractSrtpTransformer<*>? = null + + /** + * We'll cache all packets that come through before [transformer] + * gets set so that we don't lose any packets at the beginning + * (likely a keyframe) + */ + private var cachedPackets = mutableListOf() + + /** + * Transforms a list of packets using [#transformer] + * + * We pass it as an arg (rather than referencing it from the object) so Kotlin + * knows it's non-null here. + */ + private fun transformList( + packetInfos: List, + transformer: AbstractSrtpTransformer<*> + ): List { + val transformedPackets = mutableListOf() + packetInfos.forEach { packetInfo -> + val err = transformer.transform(packetInfo) + if (err == SrtpErrorStatus.OK) { + transformedPackets.add(packetInfo) + } else { + packetDiscarded(packetInfo) + } + countErrorStatus(err) + } + return transformedPackets + } + + private var firstPacketReceivedTimestamp = -1L + private var firstPacketForwardedTimestamp = -1L + + /** + * How many packets, total, we put into the cache while waiting for the transformer + * (this includes packets which may have been dropped due to the cache filling up) + */ + private var numCachedPackets = 0 + + override fun transform(packetInfo: PacketInfo): List { + if (firstPacketReceivedTimestamp == -1L) { + firstPacketReceivedTimestamp = System.currentTimeMillis() + } + transformer?.let { transformer -> + if (firstPacketForwardedTimestamp == -1L) { + firstPacketForwardedTimestamp = System.currentTimeMillis() + } + val outPackets: List + synchronized(cachedPackets) { + if (cachedPackets.isNotEmpty()) { + cachedPackets.add(packetInfo) + outPackets = transformList(cachedPackets, transformer) + cachedPackets.clear() + } else { + val err = transformer.transform(packetInfo) + countErrorStatus(err) + outPackets = if (err == SrtpErrorStatus.OK) { + listOf(packetInfo) + } else { + packetDiscarded(packetInfo) + emptyList() + } + } + } + return outPackets + } ?: run { + numCachedPackets++ + synchronized(cachedPackets) { + cachedPackets.add(packetInfo) + while (cachedPackets.size > 1024) { + packetDiscarded(cachedPackets.removeAt(0)) + } + } + return emptyList() + } + } + + private var numSrtpProcessed = 0 + private var numSrtpFail = 0 + private var numSrtpAuthFail = 0 + private var numSrtpReplayFail = 0 + private var numSrtpReplayOld = 0 + private var numSrtpInvalidPacket = 0 + + private fun countErrorStatus(err: SrtpErrorStatus) { + when (err) { + SrtpErrorStatus.OK -> numSrtpProcessed++ + SrtpErrorStatus.FAIL -> numSrtpFail++ + SrtpErrorStatus.AUTH_FAIL -> numSrtpAuthFail++ + SrtpErrorStatus.REPLAY_FAIL -> numSrtpReplayFail++ + SrtpErrorStatus.REPLAY_OLD -> numSrtpReplayOld++ + SrtpErrorStatus.INVALID_PACKET -> numSrtpInvalidPacket++ + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_cached_packets", numCachedPackets) + if (firstPacketReceivedTimestamp != -1L && firstPacketForwardedTimestamp != -1L) { + val timeBetweenReceivedAndForwarded = firstPacketForwardedTimestamp - firstPacketReceivedTimestamp + addNumber("time_initial_hold_ms", timeBetweenReceivedAndForwarded) + } else { + addString("state", "hold_for_transformer") + } + addNumber("num_srtp_processed", numSrtpProcessed) + addNumber("num_srtp_fail", numSrtpFail) + addNumber("num_srtp_auth_fail", numSrtpAuthFail) + addNumber("num_srtp_replay_fail", numSrtpReplayFail) + addNumber("num_srtp_replay_old", numSrtpReplayOld) + addNumber("num_srtp_invalid_packet", numSrtpInvalidPacket) + addNumber("num_cached_contexts", transformer?.numCachedContexts ?: 0) + } + + override fun statsJson() = super.statsJson().apply { + put("num_srtp_processed", numSrtpProcessed) + put("num_srtp_fail", numSrtpFail) + put("num_srtp_auth_fail", numSrtpAuthFail) + put("num_srtp_replay_fail", numSrtpReplayFail) + put("num_srtp_replay_old", numSrtpReplayOld) + put("num_srtp_invalid_packet", numSrtpInvalidPacket) + put("num_cached_contexts", transformer?.numCachedContexts ?: 0) + } + + override fun stop() { + super.stop() + synchronized(cachedPackets) { + cachedPackets.forEach { packetDiscarded(it) } + cachedPackets.clear() + } + } +} + +class SrtcpDecryptNode : SrtpTransformerNode("SRTCP Decrypt Node") { + override fun trace(f: () -> Unit) = f.invoke() +} + +class SrtcpEncryptNode : SrtpTransformerNode("SRTCP Encrypt Node") { + override fun trace(f: () -> Unit) = f.invoke() +} + +class SrtpDecryptNode : SrtpTransformerNode("SRTP Decrypt Node") { + override fun trace(f: () -> Unit) = f.invoke() +} + +class SrtpEncryptNode : SrtpTransformerNode("SRTP Encrypt Node") { + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt new file mode 100644 index 0000000..2bb0f62 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/ToggleablePcapWriter.kt @@ -0,0 +1,78 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.nlj.PacketInfo +import org.jitsi.utils.logging2.Logger +import java.util.Date +import kotlin.io.path.Path + +class ToggleablePcapWriter( + private val parentLogger: Logger, + private val prefix: String +) { + private var pcapWriter: PcapWriter? = null + private val pcapLock = Any() + + fun enable() { + if (!allowed) { + throw IllegalStateException("PCAP capture is disabled in configuration") + } + + synchronized(pcapLock) { + if (pcapWriter == null) { + pcapWriter = PcapWriter(parentLogger, Path(PcapWriter.directory, "$prefix-${Date().toInstant()}.pcap")) + } + } + } + + fun disable() { + synchronized(pcapLock) { + pcapWriter?.close() + pcapWriter = null + } + } + + fun isEnabled(): Boolean = pcapWriter != null + + fun newObserverNode(outbound: Boolean, suffix: String) = PcapWriterNode("ToggleablePcapWriter_$suffix", outbound) + + inner class PcapWriterNode(name: String, val outbound: Boolean) : ObserverNode(name) { + override fun observe(packetInfo: PacketInfo) { + pcapWriter?.observe(packetInfo, outbound) + } + + fun observe(buffer: ByteArray, offset: Int, length: Int) { + pcapWriter?.observe(buffer, offset, length, outbound) + } + + override fun trace(f: () -> Unit) = f.invoke() + } + + init { + if (startEnabled) { + enable() + } + } + + companion object { + private val allowed: Boolean by config("jmt.debug.pcap.enabled".from(JitsiConfig.newConfig)) + private val startEnabled: Boolean by config("jmt.debug.pcap.start-enabled".from(JitsiConfig.newConfig)) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/BufferTracePlugin.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/BufferTracePlugin.kt new file mode 100644 index 0000000..414af29 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/BufferTracePlugin.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.debug + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.transform.node.NodePlugin + +class BufferTracePlugin { + companion object : NodePlugin { + override fun observe(after: Node, packetInfo: PacketInfo) { + println( + "array trace @${after.name}: ${System.identityHashCode(packetInfo.packet.buffer)} " + + "offset ${packetInfo.packet.offset} length ${packetInfo.packet.length}" + ) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt new file mode 100644 index 0000000..4585bcb --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/PayloadVerificationPlugin.kt @@ -0,0 +1,57 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.debug + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.transform.node.NodePlugin +import org.jitsi.utils.logging2.createLogger +import java.util.concurrent.atomic.AtomicInteger + +/** + * Verifies that the payload verification string of the packet hasn't changed. + * + * @author Boris Grozev + */ +class PayloadVerificationPlugin { + companion object : NodePlugin { + private val logger = createLogger() + + val numFailures = AtomicInteger() + + @JvmStatic + fun getStatsJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("num_payload_verification_failures", numFailures.get()) + } + + override fun observe(after: Node, packetInfo: PacketInfo) { + if (PacketInfo.enablePayloadVerification && + packetInfo.payloadVerification != null + ) { + val expected = packetInfo.payloadVerification + val actual = packetInfo.packet.payloadVerification + if (expected != actual) { + logger.warn("Payload unexpectedly modified by ${after.name}! Expected: $expected, actual: $actual") + numFailures.incrementAndGet() + packetInfo.resetPayloadVerification() + } + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/RtpPacketTraceNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/RtpPacketTraceNode.kt new file mode 100644 index 0000000..414bd9e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/debug/RtpPacketTraceNode.kt @@ -0,0 +1,30 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.debug + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.rtp.rtp.RtpPacket + +class RtpPacketTraceNode(val where: String) : ObserverNode("RtpPacketTraceNode@$where") { + override fun observe(packetInfo: PacketInfo) { + val rtpPacket = packetInfo.packetAs() + println("$where: RTP packet ${rtpPacket.ssrc} ${rtpPacket.sequenceNumber}") + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/AudioLevelReader.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/AudioLevelReader.kt new file mode 100644 index 0000000..bf3030b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/AudioLevelReader.kt @@ -0,0 +1,159 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.AudioLevelListener +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.RtpExtensionType.SSRC_AUDIO_LEVEL +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.rtp.rtp.header_extensions.AudioLevelHeaderExtension + +/** + * https://tools.ietf.org/html/rfc6464#section-3 + */ +class AudioLevelReader( + streamInformationStore: ReadOnlyStreamInformationStore +) { + /** + * Process packets without cryptex pre-SRTP to allow the "skip decryption" optimization if they are to be dropped. + */ + val preDecryptNode = AudioLevelReaderNode("AudioLevelReader_pre_srtp") { !it.originalHadCryptex } + val postDecryptNode = AudioLevelReaderNode("AudioLevelReader_post_srtp") { it.originalHadCryptex } + + private var audioLevelExtId: Int? = null + var audioLevelListener: AudioLevelListener? = null + var forwardedSilencePackets: Int = 0 + private val stats = Stats() + + /** + * Whether we should forcibly mute this audio stream (by setting shouldDiscard to true). + */ + var forceMute: Boolean = false + + init { + streamInformationStore.onRtpExtensionMapping(SSRC_AUDIO_LEVEL) { + audioLevelExtId = it + } + } + + inner class AudioLevelReaderNode( + name: String, + val shouldProcess: (PacketInfo) -> Boolean + ) : ObserverNode(name) { + + override fun observe(packetInfo: PacketInfo) { + if (!shouldProcess(packetInfo)) return + + val audioRtpPacket = packetInfo.packet as? AudioRtpPacket ?: return + + audioLevelExtId?.let { audioLevelId -> + audioRtpPacket.getHeaderExtension(audioLevelId)?.let { ext -> + stats.audioLevel() + + val level = AudioLevelHeaderExtension.getAudioLevel(ext) + val vad = AudioLevelHeaderExtension.getVad(ext) + val silence = level == MUTED_LEVEL + + // Preserve the sender-reported level and VAD flag on the PacketInfo so downstream consumers (e.g. + // the mediajson Exporter) can forward them without re-parsing the header extension (they don't have + // the extension ID mapping). + packetInfo.audioLevel = level + packetInfo.vad = vad + + if (!silence) stats.nonSilence(vad) + if (silence && discardSilence && forwardedSilencePackets > forwardedSilencePacketsLimit) { + packetInfo.shouldDiscard = true + stats.discardedSilence() + } else if (this@AudioLevelReader.forceMute) { + packetInfo.shouldDiscard = true + stats.discardedForceMute() + } else { + forwardedSilencePackets = if (silence) forwardedSilencePackets + 1 else 0 + audioLevelListener?.let { listener -> + if (listener.onLevelReceived(audioRtpPacket.ssrc, (127 - level).toPositiveLong())) { + packetInfo.shouldDiscard = true + stats.discardedRanking() + } + } + } + } + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addString("audio_level_ext_id", audioLevelExtId.toString()) + addNumber("num_audio_levels", stats.numAudioLevels) + addNumber("num_silence_packets_discarded", stats.numDiscardedSilence) + addNumber("num_force_mute_discarded", stats.numDiscardedForceMute) + addNumber("num_ranking_discarded", stats.numDiscardedRanking) + addNumber("num_non_silence", stats.numNonSilence) + addNumber("num_non_silence_with_vad", stats.numNonSilenceWithVad) + addBoolean("force_mute", forceMute) + } + + override fun statsJson() = super.statsJson().apply { + put("num_audio_levels", stats.numAudioLevels) + put("num_silence_packets_discarded", stats.numDiscardedSilence) + put("num_force_mute_discarded", stats.numDiscardedForceMute) + put("num_ranking_discarded", stats.numDiscardedRanking) + } + + override fun trace(f: () -> Unit) = f.invoke() + } + + companion object { + const val MUTED_LEVEL = 127 + private val forwardedSilencePacketsLimit: Int by config { + "jmt.audio.level.forwarded-silence-packets-limit".from(JitsiConfig.newConfig) + } + private val discardSilence: Boolean by config { + "jmt.audio.level.discard-silence".from(JitsiConfig.newConfig) + } + } +} + +private class Stats( + var numAudioLevels: Long = 0, + var numDiscardedSilence: Long = 0, + var numDiscardedForceMute: Long = 0, + var numDiscardedRanking: Long = 0, + var numNonSilence: Long = 0, + var numNonSilenceWithVad: Long = 0 +) { + /** A packet contained an audio level header */ + fun audioLevel() = numAudioLevels++ + + /** A packet was discarded because it was silence. */ + fun discardedSilence() = numDiscardedSilence++ + + /** A packet was discarded because it was force-muted. */ + fun discardedForceMute() = numDiscardedForceMute++ + + /** A packet was discarded due to insufficient energy ranking or active speaker status. */ + fun discardedRanking() = numDiscardedRanking++ + + /** A non-silence packet was received (with or without the Voice Activity Detection flag). */ + fun nonSilence(hasVad: Boolean) { + numNonSilence++ + if (hasVad) numNonSilenceWithVad++ + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt new file mode 100644 index 0000000..0fc8272 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/BitrateCalculator.kt @@ -0,0 +1,201 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.metaconfig.optionalconfig +import org.jitsi.nlj.Event +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.findRtpLayerDescs +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorConfig +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorEngine +import org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator +import org.jitsi.nlj.rtp.bandwidthestimation2.GoogCcTransportCcEngine +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.bytes +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.secs +import org.jitsi.utils.stats.RateTracker +import java.time.Clock +import java.time.Duration + +/** + * When deciding what can be forwarded, we want to know the bitrate of a stream so we can fill the receiver's + * available bandwidth as much as possible without going over. This node tracks the incoming bitrate per each + * individual layer (that is, each forwardable stream taking into account spatial and temporal scalability) and + * tags the [VideoRtpPacket] with a snapshot of the current estimated bitrate for the encoding to which it belongs + */ +class VideoBitrateCalculator( + parentLogger: Logger, + // Screen sharing static content can result in very low packet/bit rates, hence the low threshold. + activePacketRateThreshold: Int = 1 +) : BitrateCalculator("Video bitrate calculator", activePacketRateThreshold) { + private val logger = createChildLogger(parentLogger) + private var mediaSourceDescs: Array = arrayOf() + + override fun observe(packetInfo: PacketInfo) { + super.observe(packetInfo) + + val videoRtpPacket: VideoRtpPacket = packetInfo.packet as VideoRtpPacket + val now = clock.millis() + val layerDescs = mediaSourceDescs.findRtpLayerDescs(videoRtpPacket) + + if (layerDescs.isEmpty()) { + logger.warn("No layer found for packet $videoRtpPacket") + } + + layerDescs.forEach { + if (it.updateBitrate(videoRtpPacket.length.bytes, now)) { + /* When a layer is started when it was previously inactive, + * we want to recalculate bandwidth allocation. + */ + packetInfo.layeringChanged = true + } + } + } + + override fun handleEvent(event: Event) { + when (event) { + is SetMediaSourcesEvent -> { + mediaSourceDescs = event.mediaSourceDescs.copyOf() + logger.cdebug { "Video bitrate calculator got media sources:\n${mediaSourceDescs.joinToString()}" } + } + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} + +open class BitrateCalculator( + name: String = "Bitrate calculator", + /** + * At what threshold the stream is considered active. + */ + private val activePacketRateThreshold: Int = 5, + protected val clock: Clock = Clock.systemUTC() +) : ObserverNode(name) { + private val bitrateTracker = createBitrateTracker() + private val packetRateTracker = createRateTracker() + val bitrate: Bandwidth + get() = bitrateTracker.rate + val packetRatePps: Long + get() = packetRateTracker.rate + private val start = clock.instant() + + /** + * Keep track of whether the stream is active (has packets at at least [activePacketRateThreshold]) + */ + val active: Boolean + get() = if (Duration.between(start, clock.instant()) <= GRACE_PERIOD) { + // In the grace period any received data counts, and we check the bitrate because we can only access the + // packet rate rounded to an Int. + bitrate.bps > 0 + } else { + packetRatePps >= activePacketRateThreshold + } + + override fun observe(packetInfo: PacketInfo) { + val now = clock.millis() + bitrateTracker.update(packetInfo.packet.length.bytes, now) + packetRateTracker.update(1, now) + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("bitrate_bps", bitrate.bps) + addNumber("packet_rate_pps", packetRatePps) + addBoolean("active", active) + } + + override fun getNodeStatsToAggregate(): NodeStatsBlock = super.getNodeStats() + + companion object { + /** + * The initial period in which we consider the stream active regardless of packet rate. + */ + val GRACE_PERIOD = 10.secs + + private val _windowSize: Duration? by optionalconfig { + "jmt.rtp.bitrate-calculator.window-size".from(JitsiConfig.newConfig) + } + + /** The default bitrate calculator window size if not set in jvb.conf, based on the BWE algorithm in use.*/ + private fun defaultWindowSize(): Duration = when (BandwidthEstimatorConfig.engine) { + BandwidthEstimatorEngine.GoogleCc -> GoogleCcEstimator.defaultRateTrackerWindowSize + BandwidthEstimatorEngine.GoogleCc2 -> GoogCcTransportCcEngine.defaultRateTrackerWindowSize + } + + /** + * The size of the window over which to calculate average rates. + */ + val windowSize: Duration + get() = _windowSize ?: defaultWindowSize() + + private val _bucketSize: Duration? by optionalconfig { + "jmt.rtp.bitrate-calculator.bucket-size".from(JitsiConfig.newConfig) + } + + /** The default bitrate calculator bucket size if not set in jvb.conf, based on the BWE algorithm in use.*/ + private fun defaultBucketSize(): Duration = when (BandwidthEstimatorConfig.engine) { + BandwidthEstimatorEngine.GoogleCc -> GoogleCcEstimator.defaultRateTrackerBucketSize + BandwidthEstimatorEngine.GoogleCc2 -> GoogCcTransportCcEngine.defaultRateTrackerBucketSize + } + + /** + * The size of the buckets to use when calculating average rates. + */ + val bucketSize + get() = _bucketSize ?: defaultBucketSize() + + /** + * The size of the window over which to calculate average rates for sources whose bitrate is bursty, i.e. + * screen sharing. See [createSmoothedBitrateTracker]. + */ + val smoothedWindowSize: Duration by config( + "jmt.rtp.bitrate-calculator.smoothed-window-size".from(JitsiConfig.newConfig) + ) + + /** The size of the buckets to use with [smoothedWindowSize]. This must divide it evenly. */ + val smoothedBucketSize: Duration by config( + "jmt.rtp.bitrate-calculator.smoothed-bucket-size".from(JitsiConfig.newConfig) + ) + + fun createBitrateTracker() = BitrateTracker(windowSize, bucketSize) + + /** + * Creates a tracker with a longer window than [createBitrateTracker], for use with sources whose bitrate is + * bursty. A screen sharing encoder is allowed to accumulate about a second of its target bitrate while the + * screen is static and then spend it on a single frame, and libwebrtc's screen sharing rate control tolerates + * intervals of up to 2.75 seconds between frames, so a rate measured over [windowSize] (which matches the + * bandwidth estimator's window) is not representative of the bandwidth such a source actually needs. + */ + fun createSmoothedBitrateTracker() = BitrateTracker(smoothedWindowSize, smoothedBucketSize) + + fun createRateTracker() = RateTracker(windowSize, bucketSize) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/DiscardableDiscarder.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/DiscardableDiscarder.kt new file mode 100644 index 0000000..24b2587 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/DiscardableDiscarder.kt @@ -0,0 +1,54 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.ResumableStreamRewriter +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.LRUCache +import java.util.Collections + +/** + * Discards RTP packets which have shouldDiscard set to true, masking their loss + * in the RTP sequence numbers of RTP packets. + */ +class DiscardableDiscarder(name: String, val keepHistory: Boolean) : TransformerNode(name) { + /** + * Per-SSRC rewriters, keyed on the RTP SSRC. Bounded with an LRU to limit the number of SSRCs tracked. + */ + val rewriters: MutableMap = + Collections.synchronizedMap(LRUCache(MAX_SSRCS, true)) + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val packet = packetInfo.packet as? RtpPacket ?: return packetInfo + rewriters.computeIfAbsent(packet.ssrc) { ResumableStreamRewriter(keepHistory) } + .rewriteRtp(!packetInfo.shouldDiscard, packet) + + return if (packetInfo.shouldDiscard) { + null + } else { + packetInfo + } + } + + override fun trace(f: () -> Unit) = f.invoke() + + companion object { + /** The maximum number of SSRCs to track rewriters for. */ + const val MAX_SSRCS = 1024 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/DuplicateTermination.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/DuplicateTermination.kt new file mode 100644 index 0000000..af15b28 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/DuplicateTermination.kt @@ -0,0 +1,66 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.LRUCache + +/** + * A node which drops packets with SSRC and sequence number pairs identical to ones + * that have previously been seen. + * + * (Since SRTP also has anti-replay protection, the normal case where duplicates + * will occur is after the [RtxHandler], since duplicate packets are sent over RTX for probing.) + */ +class DuplicateTermination() : TransformerNode("Duplicate termination") { + /** + * Per-SSRC replay context, keyed on the RTP SSRC. Bounded with an LRU to limit the number of SSRCs tracked. + */ + private val replayContexts: MutableMap> = LRUCache(MAX_SSRCS, true) + private var numDuplicatePacketsDropped = 0 + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val rtpPacket = packetInfo.packetAs() + val replayContext = replayContexts.computeIfAbsent(rtpPacket.ssrc) { + LRUCache.lruSet(1500, true) + } + + if (!replayContext.add(rtpPacket.sequenceNumber)) { + numDuplicatePacketsDropped++ + return null + } + + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_duplicate_packets_dropped", numDuplicatePacketsDropped) + } + + override fun statsJson() = super.statsJson().apply { + put("num_duplicate_packets_dropped", numDuplicatePacketsDropped) + } + + override fun trace(f: () -> Unit) = f.invoke() + + companion object { + /** The maximum number of SSRCs to track replay contexts for. */ + const val MAX_SSRCS = 1024 + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt new file mode 100644 index 0000000..d0d31de --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingStatisticsTracker.kt @@ -0,0 +1,356 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.stats.JitterStats +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.RtpUtils.Companion.convertRtpTimestampToInstant +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isNextAfter +import org.jitsi.rtp.util.numPacketsTo +import org.jitsi.rtp.util.rolledOverTo +import org.jitsi.utils.LRUCache +import org.jitsi.utils.MediaType +import java.time.Duration +import java.time.Instant +import java.util.Collections + +/** + * Track various statistics about received RTP streams to be used in SR/RR report blocks + */ +class IncomingStatisticsTracker( + private val streamInformationStore: ReadOnlyStreamInformationStore +) : ObserverNode("Incoming statistics tracker") { + /** + * Per-SSRC stats, keyed on the RTP SSRC. Bounded with an LRU to limit the number of SSRCs tracked for a single + * stream. + */ + private val ssrcStats: MutableMap = + Collections.synchronizedMap(LRUCache(MAX_SSRCS, true)) + + override fun observe(packetInfo: PacketInfo) { + val rtpPacket = packetInfo.packetAs() + streamInformationStore.rtpPayloadTypes[rtpPacket.payloadType.toByte()]?.let { payloadType -> + // We don't want to track jitter, etc. for RTX streams + if (payloadType !is RtxPayloadType) { + val stats = ssrcStats.computeIfAbsent(rtpPacket.ssrc) { + IncomingSsrcStats(rtpPacket.ssrc, rtpPacket.sequenceNumber, payloadType.mediaType) + } + val packetSentTimestamp = convertRtpTimestampToInstant( + rtpPacket.timestamp.toInt(), + payloadType.clockRate + ) + packetInfo.receivedTime?.let { + stats.packetReceived(rtpPacket, packetSentTimestamp, it) + } + } + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + val stats = getSnapshot() + stats.ssrcStats.forEach { (ssrc, streamStats) -> + addJson(ssrc.toString(), streamStats.toJson()) + } + } + + /** + * Don't aggregate the per-SSRC stats. + */ + override fun getNodeStatsToAggregate() = super.getNodeStats() + + override fun trace(f: () -> Unit) = f.invoke() + + /** + * Gets a snapshot of the SSRCs which have received a packet since the last call to this method. There is a single + * flag keeping track of activity, so this should not be used in more than one place. Currently, it is used for RR + * generation. Other code should use [getSnapshot]. + */ + fun getSnapshotOfActiveSsrcs() = IncomingStatisticsSnapshot( + synchronized(ssrcStats) { + ssrcStats.mapNotNull { (ssrc, stats) -> + stats.getSnapshotIfActive()?.let { Pair(ssrc, it) } + }.toMap() + } + ) + + fun getSnapshot() = IncomingStatisticsSnapshot( + synchronized(ssrcStats) { + ssrcStats.map { (ssrc, stats) -> Pair(ssrc, stats.getSnapshot()) }.toMap() + } + ) + + companion object { + /** The maximum number of SSRCs to track stats for. */ + const val MAX_SSRCS = 1024 + } +} + +class IncomingStatisticsSnapshot( + /** + * Per-ssrc stats. + */ + val ssrcStats: Map +) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + ssrcStats.forEach { (ssrc, snapshot) -> + set(ssrc.toString(), snapshot.toJson()) + } + } +} + +/** + * Tracks various statistics for the stream using ssrc [ssrc]. Some statistics are tracked only + * over a specific interval (in between calls to [reset]) and others persist across calls to [reset]. This + * is tuned for use in generating an RR packet. + * TODO: max dropout/max misorder/probation handling according to appendix A.1 + */ +class IncomingSsrcStats( + private val ssrc: Long, + private var baseSeqNum: Int, + private val mediaType: MediaType +) { + // TODO: for now we'll synchronize access to all the stats so we can create a consistent snapshot when it's + // requested from another context. it'd be great to be able to avoid this (coroutines make it easy to switch + // contexts and could be a nice option here in the future). another option would be doing the RR generation + // in the same context as where we receive packets: this would work fine as long as that sort of interval is + // acceptable for RRs? + private var statsLock = Any() + // Start variables protected by statsLock + + /** + * This will be initialized to the first sequence number we process + */ + private var maxSeqNum: Int = baseSeqNum + private var seqNumCycles: Int = 0 + private val numExpectedPackets: Int + get() = calculateExpectedPacketCount(0, baseSeqNum, seqNumCycles, maxSeqNum) + private var cumulativePacketsLost: Int = 0 + private var outOfOrderPacketCount: Int = 0 + private val jitterStats = JitterStats() + private var numReceivedPackets: Int = 0 + private var numReceivedBytes: Int = 0 + + /** How long this SSRC has been active. */ + private var durationActive = Duration.ZERO + + /** The receiveTime of the last packet */ + private var lastPacketReceivedTime: Instant? = null + + /** + * Whether there has been any activity (packets received) since the last time the stats were queried with + * onlyActive = true. + */ + private var activitySinceLastSnapshot: Boolean = false + // End variables protected by statsLock + + private var probation: Int = INITIAL_MIN_SEQUENTIAL + + companion object { + /** + * The maximum 'out-of-order' amount, meaning a packet whose sequence number + * difference from the current highest sequence number larger than this amount + * will not be treated as a received out-of-order packet (and therefore subtract + * from the cumulative loss amount) + */ + const val MAX_OOO_AMOUNT = 100 + + /** + * https://tools.ietf.org/html/rfc3550#appendix-A.1 + * "...a source is declared valid only after MIN_SEQUENTIAL packets have been received in + * sequence." + */ + const val INITIAL_MIN_SEQUENTIAL = 2 + + const val MAX_DROPOUT = 3000 + + /** + * The maximum delay between two packets that counts as activity (as opposed to the stream going inactive and + * back to active). + */ + val ACTIVITY_TIMEOUT: Duration = Duration.ofMillis(1000) + + /** + * Find how many packets we would expected to have received in the range [[baseSeqNum], [currSeqNum]], taking + * into account the amount of cycles present when we received [baseSeqNum] ([baseSeqNumCycles]) and the + * amount of cycles when we received [currSeqNum] ([currCycles]) + */ + fun calculateExpectedPacketCount( + baseSeqNumCycles: Int, + baseSeqNum: Int, + currCycles: Int, + currSeqNum: Int + ): Int { + val baseExtended = (baseSeqNumCycles shl 16) + baseSeqNum + val maxExtended = (currCycles shl 16) + currSeqNum + + return maxExtended - baseExtended + 1 + } + } + + private fun createSnapshot() = Snapshot( + numReceivedPackets = numReceivedPackets, + numReceivedBytes = numReceivedBytes, + maxSeqNum = maxSeqNum, + seqNumCycles = seqNumCycles, + numExpectedPackets = numExpectedPackets, + cumulativePacketsLost = cumulativePacketsLost, + jitter = jitterStats.jitter, + durationActive = durationActive, + mediaType = mediaType + ) + + fun getSnapshotIfActive(): Snapshot? { + synchronized(statsLock) { + if (!activitySinceLastSnapshot) return null + activitySinceLastSnapshot = false + return createSnapshot() + } + } + + fun getSnapshot(): Snapshot = synchronized(statsLock) { createSnapshot() } + + /* + * Resets this [IncomingSsrcStats]'s tracking variables such that: + * 1) A new base sequence number (the given [newBaseSeqNum]) will be used to start new loss calculations. + * 2) Any lost packet counters will be reset + * 3) Jitter calculations will NOT be reset + */ +// fun reset(newBaseSeqNum: Int) { +// baseSeqNum = newBaseSeqNum +// maxSeqNum = baseSeqNum - 1 +// cumulativePacketsLost = 0 +// } + + /** + * Notify this [IncomingSsrcStats] instance that an RTP packet [packet] for the stream it is + * tracking has been received and that it: was sent at [packetSentTimestamp] (note this is NOT the + * raw RTP timestamp, but the 'translated' timestamp which is a function of the RTP timestamp and the clockrate) + * and was received at [packetReceivedTime] + */ + fun packetReceived(packet: RtpPacket, packetSentTimestamp: Instant, packetReceivedTime: Instant) { + val packetSequenceNumber = packet.sequenceNumber + synchronized(statsLock) { + activitySinceLastSnapshot = true + numReceivedPackets++ + numReceivedBytes += packet.length + if (lastPacketReceivedTime != null) { + val timeSincePreviousPacket = Duration.between(lastPacketReceivedTime, packetReceivedTime) + if (timeSincePreviousPacket < ACTIVITY_TIMEOUT) { + durationActive += timeSincePreviousPacket + } + } + lastPacketReceivedTime = packetReceivedTime + + if (packetSequenceNumber isNewerThan maxSeqNum) { + if (packetSequenceNumber isNextAfter maxSeqNum) { + if (probation > 0) { + probation-- + // TODO: do we want to 'reset' the sequence after the probation period is finished? + } + } else if (maxSeqNum numPacketsTo packetSequenceNumber in 0..MAX_DROPOUT) { + // In order with a gap, but gap is within acceptable range + cumulativePacketsLost += maxSeqNum numPacketsTo packetSequenceNumber + maybeResetProbation() + } else { + // Very large jump + // TODO + } + if (maxSeqNum rolledOverTo packetSequenceNumber) { + seqNumCycles++ + } + maxSeqNum = packetSequenceNumber + } else { + // Older packet + if (packetSequenceNumber numPacketsTo maxSeqNum in 0..MAX_OOO_AMOUNT) { + // This packet would've already been counted as lost, so subtract from the loss count + // NOTE: this is susceptible to inaccuracy in the case of duplicate, out-of-order packets + // but for now we'll assume those are rare enough not to cause issues. + cumulativePacketsLost-- + } else { + // Older packet which is too old to be counted as out-of-order + // TODO + } + outOfOrderPacketCount++ + maybeResetProbation() + } + + jitterStats.addPacket( + packetSentTimestamp, + packetReceivedTime + ) + } + } + + /** + * Reset the probation period if it's currently active + */ + private fun maybeResetProbation() { + if (probation > 0) { + // When we reset, we subtract 1 since we've already started receiving packets so we already have + // a 'starting point' for our sequential check (unlike when this instance is initialized, when we + // haven't processing any incoming packets yet) + probation = INITIAL_MIN_SEQUENTIAL - 1 + } + } + + /** + * A class to export a consistent snapshot of the data held inside [IncomingSsrcStats] + * TODO: these really need to be documented! + */ + data class Snapshot( + val numReceivedPackets: Int = 0, + val numReceivedBytes: Int = 0, + val maxSeqNum: Int = 0, + val seqNumCycles: Int = 0, + val numExpectedPackets: Int = 0, + val cumulativePacketsLost: Int = 0, + val jitter: Double = 0.0, + val durationActive: Duration = Duration.ZERO, + val mediaType: MediaType + ) { + fun computeFractionLost(previousSnapshot: Snapshot): Int { + val numExpectedPacketsInterval = numExpectedPackets - previousSnapshot.numExpectedPackets + val numReceivedPacketsInterval = numReceivedPackets - previousSnapshot.numReceivedPackets + + val numLostPacketsInterval = numExpectedPacketsInterval - numReceivedPacketsInterval + return if (numExpectedPacketsInterval == 0 || numLostPacketsInterval <= 0) { + 0 + } else { + (((numLostPacketsInterval shl 8) / numExpectedPacketsInterval.toDouble())).toInt() + } + } + + fun toJson() = JsonNodeFactory.instance.objectNode().apply { + put("num_received_packets", numReceivedPackets) + put("num_received_bytes", numReceivedBytes) + put("max_seq_num", maxSeqNum) + put("seq_num_cycles", seqNumCycles) + put("num_expected_packets", numExpectedPackets) + put("cumulative_packets_lost", cumulativePacketsLost) + put("jitter", jitter) + put("duration_active_ms", durationActive.toMillis()) + put("media_type", mediaType.toString()) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/PaddingTermination.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/PaddingTermination.kt new file mode 100644 index 0000000..06d3f70 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/PaddingTermination.kt @@ -0,0 +1,68 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import kotlin.math.max + +/** + * A node which removes padding from RTP packets. + * Padding-only packets are marked as shouldDiscard in their PacketInfo. + */ +class PaddingTermination(parentLogger: Logger) : TransformerNode("Padding termination") { + private val logger = createChildLogger(parentLogger) + private var numPaddedPacketsSeen = 0 + private var numPaddingOnlyPacketsSeen = 0 + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val rtpPacket = packetInfo.packetAs() + + if (rtpPacket.hasPadding) { + rtpPacket.removePadding() + packetInfo.resetPayloadVerification() + numPaddedPacketsSeen++ + if (rtpPacket.payloadLength == 0) { + numPaddingOnlyPacketsSeen++ + packetInfo.shouldDiscard = true + } + } + + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_padded_packets_seen", numPaddedPacketsSeen) + addNumber("num_padding_only_packets_seen", numPaddingOnlyPacketsSeen) + } + + override fun statsJson() = super.statsJson().apply { + put("num_padded_packets_seen", numPaddedPacketsSeen) + put("num_padding_only_packets_seen", numPaddingOnlyPacketsSeen) + } + + override fun trace(f: () -> Unit) = f.invoke() +} + +fun RtpPacket.removePadding() { + val paddingSize = paddingSize + length = max(length - paddingSize, headerLength) + hasPadding = false +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt new file mode 100644 index 0000000..c4395ad --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimator.kt @@ -0,0 +1,149 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.Event +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetLocalSsrcEvent +import org.jitsi.nlj.rtp.RtpExtensionType.ABS_SEND_TIME +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator +import org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.AbsSendTimeHeaderExtension +import org.jitsi.utils.LRUCache +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.observableWhenChanged +import java.time.Clock +import java.time.Duration +import java.util.Collections + +/** + * Estimates the available bandwidth for the incoming stream using the abs-send-time extension. + */ +class RemoteBandwidthEstimator( + private val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger, + diagnosticContext: DiagnosticContext = DiagnosticContext(), + private val clock: Clock = Clock.systemUTC() +) : ObserverNode("Remote Bandwidth Estimator") { + private val logger = createChildLogger(parentLogger) + + /** + * The remote bandwidth estimation is enabled when REMB support is signaled, but TCC is not signaled. + */ + private var enabled: Boolean by observableWhenChanged(false) { _, _, newValue -> + logger.debug { "Setting enabled=$newValue." } + } + private var astExtId: Int? = null + + /** + * We use the full [GoogleCcEstimator] here, but we don't notify it of packet loss, effectively using only the + * delay-based part. + */ + private val bwe: BandwidthEstimator by lazy { GoogleCcEstimator(diagnosticContext, logger) } + private val ssrcs: MutableSet = + Collections.synchronizedSet( + LRUCache.lruSet( + MAX_SSRCS, + // accessOrder + true + ) + ) + private var numRembsCreated = 0 + private var numPacketsWithoutAbsSendTime = 0 + private var localSsrc = 0L + + init { + streamInformationStore.onRtpExtensionMapping(ABS_SEND_TIME) { + astExtId = it + logger.debug { "Setting abs-send-time extension ID to $astExtId" } + } + streamInformationStore.onRtpPayloadTypesChanged { + enabled = streamInformationStore.supportsRemb && !streamInformationStore.supportsTcc + } + } + + companion object { + private const val MAX_SSRCS: Int = 8 + } + + override fun handleEvent(event: Event) { + if (event is SetLocalSsrcEvent && event.mediaType == MediaType.VIDEO) { + localSsrc = event.ssrc + } + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun observe(packetInfo: PacketInfo) { + if (!enabled) return + + astExtId?.let { + val rtpPacket = packetInfo.packetAs() + rtpPacket.getHeaderExtension(it)?.let { ext -> + val now = clock.instant() + bwe.processPacketArrival( + now, + AbsSendTimeHeaderExtension.getTime(ext), + packetInfo.receivedTime, + rtpPacket.sequenceNumber, + packetInfo.originalLength.bytes + ) + /* With receiver-side bwe we need to treat each received packet as separate feedback */ + bwe.feedbackComplete(now) + ssrcs.add(rtpPacket.ssrc) + } + } ?: numPacketsWithoutAbsSendTime++ + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addString("ast_ext_id", astExtId.toString()) + addBoolean("enabled", enabled) + addNumber("num_rembs_created", numRembsCreated) + addNumber("num_packets_without_ast", numPacketsWithoutAbsSendTime) + } + + fun createRemb(): RtcpFbRembPacket? { + // REMB based BWE is not configured. + if (!enabled || astExtId == null) return null + + val currentBw = bwe.getCurrentBw(clock.instant()) + // The estimator does not yet have a valid value. + if (currentBw < 0.bps) return null + + numRembsCreated++ + return RtcpFbRembPacketBuilder( + rtcpHeader = RtcpHeaderBuilder(senderSsrc = localSsrc), + brBps = currentBw.bps, + ssrcs = ssrcs.toList() + ).build() + } + + fun onRttUpdate(newRttMs: Double) { + bwe.onRttUpdate(clock.instant(), Duration.ofNanos((newRttMs * 1000_000).toLong())) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RetransmissionRequesterNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RetransmissionRequesterNode.kt new file mode 100644 index 0000000..8680afd --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RetransmissionRequesterNode.kt @@ -0,0 +1,44 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtcp.RetransmissionRequester +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.Logger +import java.util.concurrent.ScheduledExecutorService + +class RetransmissionRequesterNode( + rtcpSender: (RtcpPacket) -> Unit, + scheduler: ScheduledExecutorService, + parentLogger: Logger +) : ObserverNode("Retransmission requester") { + private val retransmissionRequester = RetransmissionRequester(rtcpSender, scheduler, parentLogger) + + override fun observe(packetInfo: PacketInfo) { + val rtpPacket = packetInfo.packetAs() + retransmissionRequester.packetReceived(rtpPacket.ssrc, rtpPacket.sequenceNumber) + } + + override fun stop() { + super.stop() + retransmissionRequester.stop() + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt new file mode 100644 index 0000000..2a62a13 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTermination.kt @@ -0,0 +1,147 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.nlj.util.BufferPool +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtcp.RtcpByePacket +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpRrPacket +import org.jitsi.rtp.rtcp.RtcpSdesPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.RtcpXrPacket +import org.jitsi.rtp.rtcp.rtcpfb.UnsupportedRtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cinfo +import org.jitsi.utils.logging2.createChildLogger + +class RtcpTermination( + private val rtcpEventNotifier: RtcpEventNotifier, + parentLogger: Logger +) : TransformerNode("RTCP termination") { + private val logger = createChildLogger(parentLogger) + private var packetReceiveCounts = mutableMapOf() + + /** + * Number of packets we failed to forward because a compound packet contained more than one + * packet we wanted to forward. Ideally this shouldn't happen. + */ + private var numFailedToForward = 0 + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val compoundRtcp = packetInfo.packetAs() + var forwardedRtcp: RtcpPacket? = null + + compoundRtcp.packets.forEach { rtcpPacket -> + when (rtcpPacket) { + is RtcpFbPliPacket, is RtcpFbFirPacket, is RtcpSrPacket -> { + // We'll let these pass through and be forwarded to the conference (where they will be + // routed to the other endpoint(s)) + // NOTE(brian): this should work fine as long as we can't receive 2 RTCP packets + // we want to forward in the same compound packet. If we can, then we may need + // to turn this into a MultipleOutputNode + forwardedRtcp?.let { + logger.cinfo { + "Failed to forward a packet of type ${it::class.simpleName} " + + ". Replaced by ${rtcpPacket::class.simpleName}." + } + numFailedToForward++ + } + forwardedRtcp = rtcpPacket + } + + is RtcpSdesPacket, is RtcpRrPacket, is RtcpFbNackPacket, + is RtcpByePacket, is RtcpFbTccPacket, is RtcpFbRembPacket -> { + // Supported, but no special handling here (any special handling will be in + // notifyRtcpReceived below + } + + is RtcpXrPacket -> { + // Unsupported, but we get them when chrome does screenshare and the + // message below clouds up the logs. They are still tracked as part + // of the packetReceiveCount + } + + is UnsupportedRtcpFbPacket -> { + logger.cinfo { + "TODO: not yet handling RTCP packet of type ${rtcpPacket.packetType} fmt " + + "${rtcpPacket.reportCount} ${rtcpPacket.javaClass}" + } + } + + else -> { + logger.cinfo { + "TODO: not yet handling RTCP packet of type ${rtcpPacket.packetType} " + + "${rtcpPacket.javaClass}" + } + } + } + // TODO: keep an eye on if anything in here takes a while it could slow the packet pipeline down + packetReceiveCounts.merge(rtcpPacket::class.simpleName!!, 1, Int::plus) + rtcpEventNotifier.notifyRtcpReceived(rtcpPacket, packetInfo.receivedTime) + + (forwardedRtcp as? RtcpSrPacket)?.let { + logger.cdebug { + "Saw an sr from ssrc=${rtcpPacket.senderSsrc}, " + + "timestamp=${it.senderInfo.rtpTimestamp}" + } + forwardedRtcp = if (it.reportCount > 0) { + // Eliminates any report blocks as we don't want to relay those + it.cloneWithoutReportBlocks() + } else { + it + } + } + } + + return forwardedRtcp?.let { + if (it.buffer !== packetInfo.packet.buffer) { + // We're not using the original packet's buffer, so we can return it to the pool + BufferPool.returnBuffer(packetInfo.packet.buffer) + } + packetInfo.packet = it + packetInfo + } ?: run { + null + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + packetReceiveCounts.forEach { (type, count) -> + addNumber("num_${type}_rx", count) + } + addNumber("num_failed_to_forward", numFailedToForward) + } + + override fun statsJson() = super.statsJson().apply { + put("num_failed_to_forward", numFailedToForward) + packetReceiveCounts.forEach { (type, count) -> + put("num_${type}_rx", count) + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt new file mode 100644 index 0000000..b47a567 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/RtxHandler.kt @@ -0,0 +1,108 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.rtp.RtxPacket +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import java.util.concurrent.ConcurrentHashMap + +/** + * Handle incoming RTX packets to strip the RTX information and make them + * look like their original packets. + * https://tools.ietf.org/html/rfc4588 + */ +class RtxHandler( + private val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : TransformerNode("RTX handler") { + private val logger = createChildLogger(parentLogger) + private var numPaddingPacketsReceived = 0 + private var numRtxPacketsReceived = 0 + + /** + * Maps the Integer payload type of RTX to the [RtxPayloadType] instance. We do this + * so we can look up the associated (original) payload type from the [RtxPayloadType] + * instance quickly via the Int RTX payload type in an incoming RTX packet. + */ + private val rtxPtToRtxPayloadType = ConcurrentHashMap() + + init { + streamInformationStore.onRtpPayloadTypesChanged { currentPayloadTypes -> + rtxPtToRtxPayloadType.clear() + currentPayloadTypes.values + .filterIsInstance() + .map { + rtxPtToRtxPayloadType[it.pt.toPositiveInt()] = it + } + } + } + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val rtpPacket = packetInfo.packetAs() + val rtxPayloadType = rtxPtToRtxPayloadType[rtpPacket.payloadType.toPositiveInt()] ?: return packetInfo + val originalSsrc = streamInformationStore.getLocalPrimarySsrc(rtpPacket.ssrc) ?: return packetInfo + if (packetInfo.shouldDiscard) { + /* Don't try to interpret an rtx [shouldDiscard] packet - SrtpTransformer may have skipped decrypting + * it, which means its originalSeqNum is gibberish. + * This doesn't need to wait until [DiscardableDiscarder], because we don't care about continuity of + * RTX sequence numbers. + */ + return null + } + // We do this check only after verifying we determine it's an RTX packet by finding + // the associated payload type and SSRC above + if (rtpPacket.payloadLength - rtpPacket.paddingSize < 2) { + logger.cdebug { "RTX packet is padding, ignore" } + numPaddingPacketsReceived++ + return null + } + val originalSeqNum = RtxPacket.getOriginalSequenceNumber(rtpPacket) + val originalPt = rtxPayloadType.associatedPayloadType + + // Move the payload 2 bytes to the left + RtxPacket.removeOriginalSequenceNumber(rtpPacket) + rtpPacket.sequenceNumber = originalSeqNum + rtpPacket.payloadType = originalPt + rtpPacket.ssrc = originalSsrc + + logger.cdebug { "Recovered RTX packet. Original packet: $originalSsrc $originalSeqNum" } + numRtxPacketsReceived++ + packetInfo.resetPayloadVerification() + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_rtx_packets_received", numRtxPacketsReceived) + addNumber("num_padding_packets_received", numPaddingPacketsReceived) + addString("rtx_payload_types", rtxPtToRtxPayloadType.values.toString()) + } + + override fun statsJson() = super.statsJson().apply { + put("num_rtx_packets_received", numRtxPacketsReceived) + put("num_padding_packets_received", numPaddingPacketsReceived) + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt new file mode 100644 index 0000000..de6e2ad --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNode.kt @@ -0,0 +1,257 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.RtpExtensionType.TRANSPORT_CC +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.TccHeaderExtension +import org.jitsi.utils.NEVER +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.ms +import org.jitsi.utils.observableWhenChanged +import org.jitsi.utils.secs +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.TreeMap + +/** + * Extract the TCC sequence numbers from each passing packet and generate + * a TCC packet to send transmit to the sender. + */ +class TccGeneratorNode( + private val onTccPacketReady: (RtcpPacket) -> Unit = {}, + private val streamInformation: ReadOnlyStreamInformationStore, + parentLogger: Logger, + private val clock: Clock = Clock.systemDefaultZone() +) : ObserverNode("TCC generator") { + private val logger = createChildLogger(parentLogger) + private var tccExtensionId: Int? = null + private var currTccSeqNum: Int = 0 + private var lastTccSentTime: Instant = NEVER + private val lock = Any() + + // Tcc seq num -> arrival time in ms + private val packetArrivalTimes = TreeMap() + + // The first sequence number of the current tcc feedback packet + private var windowStartSeq: Long = -1 + private val tccFeedbackBitrate = BitrateTracker(1.secs, 10.ms) + private var numTccSent: Int = 0 + private var numMultipleTccPackets = 0 + private var enabled: Boolean by observableWhenChanged(false) { _, _, newValue -> + logger.debug("Setting enabled=$newValue") + } + private val rtpSequenceIndexTracker = RtpSequenceIndexTracker() + + private val lossListeners = mutableListOf() + + init { + streamInformation.onRtpExtensionMapping(TRANSPORT_CC) { + tccExtensionId = it + } + streamInformation.onRtpPayloadTypesChanged { + enabled = streamInformation.supportsTcc + } + } + + override fun observe(packetInfo: PacketInfo) { + if (!enabled) return + + tccExtensionId?.let { tccExtId -> + val rtpPacket = packetInfo.packetAs() + rtpPacket.getHeaderExtension(tccExtId)?.let { ext -> + val tccSeqNum = rtpSequenceIndexTracker.update(TccHeaderExtension.getSequenceNumber(ext)) + addPacket(tccSeqNum, packetInfo.receivedTime, rtpPacket.isMarked, rtpPacket.ssrc) + } + } + } + + /** + * Adds a loss listener to be notified about packet arrival and loss reports. + * @param listener + */ + fun addLossListener(listener: LossListener) { + synchronized(lock) { + lossListeners.add(listener) + } + } + + /** + * Removes a loss listener. + * @param listener + */ + @Synchronized + fun removeLossListener(listener: LossListener) { + synchronized(lock) { + lossListeners.remove(listener) + } + } + + /** + * @param tccSeqNum the extended sequence number. + */ + private fun addPacket(tccSeqNum: Long, timestamp: Instant?, isMarked: Boolean, ssrc: Long) { + synchronized(lock) { + if (packetArrivalTimes.ceilingKey(windowStartSeq) == null) { + // Packets in map are all older than the start of the next tcc feedback packet, + // remove them + // TODO: Chrome does something more advanced, keeping older sequences to replay on packet reordering. + packetArrivalTimes.clear() + } + + timestamp?.run { + if (packetArrivalTimes.isEmpty() && windowStartSeq == -1L) { + lossListeners.forEach { + it.packetReceived(false) + } + } else { + val oldMax = if (packetArrivalTimes.isNotEmpty()) { + packetArrivalTimes.lastKey() + } else { + windowStartSeq - 1 + } + if (tccSeqNum > oldMax) { + val numLost = tccSeqNum - oldMax - 1 + /* TODO: should we squelch for large tcc jumps? */ + lossListeners.forEach { + if (numLost > 0) { + it.packetLost(numLost.toInt()) + } + it.packetReceived(false) + } + } else if (tccSeqNum < windowStartSeq || !packetArrivalTimes.containsKey(tccSeqNum)) { + /* If we've already cleared the arrival info about this packet, assume it was previously + * reported as lost - there are some corner cases where this isn't true, but they should be rare. + */ + lossListeners.forEach { + it.packetReceived(true) + } + } + } + + if (windowStartSeq == -1L || tccSeqNum < windowStartSeq) { + windowStartSeq = tccSeqNum + } + + packetArrivalTimes.putIfAbsent(tccSeqNum, timestamp) + } + if (isTccReadyToSend(isMarked)) { + buildFeedback(ssrc).forEach { sendTcc(it) } + } + } + } + + private fun buildFeedback(mediaSsrc: Long): List { + synchronized(lock) { + // windowStartSeq is the first sequence number to include in the current feedback, but we may not have + // received it so the base time shall be the time of the first received packet which will be included + // in this feedback + val firstEntry = packetArrivalTimes.ceilingEntry(windowStartSeq) ?: return emptyList() + + val tccPackets = mutableListOf() + var currentTccPacket = RtcpFbTccPacketBuilder( + mediaSourceSsrc = mediaSsrc, + feedbackPacketSeqNum = currTccSeqNum++ + ) + currentTccPacket.SetBase(windowStartSeq.toInt(), firstEntry.value) + + var nextSequenceNumber = windowStartSeq + val feedbackBlockPackets = packetArrivalTimes.tailMap(windowStartSeq) + feedbackBlockPackets.forEach { (seq, timestamp) -> + if (!currentTccPacket.AddReceivedPacket(seq.toInt(), timestamp)) { + tccPackets.add(currentTccPacket.build()) + currentTccPacket = RtcpFbTccPacketBuilder( + mediaSourceSsrc = mediaSsrc, + feedbackPacketSeqNum = currTccSeqNum++ + ).apply { + SetBase(seq.toInt(), timestamp) + AddReceivedPacket(seq.toInt(), timestamp) + } + } + nextSequenceNumber = seq + 1 + } + + tccPackets.add(currentTccPacket.build()) + if (tccPackets.size > 1) { + numMultipleTccPackets++ + logger.info( + "Sending TCC feedback in ${tccPackets.size} packets " + + "(${feedbackBlockPackets.size} media packets)" + ) + } + // The next window will start with the sequence number after the last one we included in the previous + // feedback + windowStartSeq = nextSequenceNumber + + return tccPackets + } + } + + private fun sendTcc(tccPacket: RtcpFbTccPacket) { + onTccPacketReady(tccPacket) + logger.cdebug { "sent TCC packet with seq num ${tccPacket.feedbackSeqNum}" } + numTccSent++ + lastTccSentTime = clock.instant() + tccFeedbackBitrate.update(tccPacket.length.bytes, clock.millis()) + } + + private fun isTccReadyToSend(currentPacketMarked: Boolean): Boolean { + val now = clock.instant() + // We don't want to send TCC the very first time we check (which would + // be after the first packet was added). So the first time we check, + // set the last sent time to now to delay sending TCC by at least one 'interval' + if (lastTccSentTime == NEVER) { + lastTccSentTime = now + return false + } + + val timeSinceLastTcc = Duration.between(lastTccSentTime, now) + return timeSinceLastTcc >= 100.ms || + ((timeSinceLastTcc >= 20.ms) && currentPacketMarked) + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_tcc_packets_sent", numTccSent) + addNumber("tcc_feedback_bitrate_bps", tccFeedbackBitrate.rate.bps) + addString("tcc_extension_id", tccExtensionId.toString()) + addNumber("num_multiple_tcc_packets", numMultipleTccPackets) + addBoolean("enabled", enabled) + } + + override fun statsJson() = super.statsJson().apply { + put("num_tcc_packets_sent", numTccSent) + put("tcc_feedback_bitrate_bps", tccFeedbackBitrate.rate.bps) + put("tcc_extension_id", tccExtensionId.toString()) + put("num_multiple_tcc_packets", numMultipleTccPackets) + put("enabled", enabled) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoMuteNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoMuteNode.kt new file mode 100644 index 0000000..f32c46b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoMuteNode.kt @@ -0,0 +1,32 @@ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode + +class VideoMuteNode : ObserverNode("VideoMuteNode") { + + private var numMutedPackets = 0 + var forceMute: Boolean = false + + override fun observe(packetInfo: PacketInfo) { + if (packetInfo.packet !is VideoRtpPacket) return + if (this.forceMute) { + packetInfo.shouldDiscard = true + numMutedPackets++ + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_video_packets_discarded", numMutedPackets) + addBoolean("force_mute", forceMute) + } + + override fun statsJson() = super.statsJson().apply { + put("num_video_packets_discarded", numMutedPackets) + put("force_mute", forceMute) + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt new file mode 100644 index 0000000..1484672 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoParser.kt @@ -0,0 +1,284 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.Event +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.findRtpSource +import org.jitsi.nlj.findRtpSourceByPrimary +import org.jitsi.nlj.format.Vp8PayloadType +import org.jitsi.nlj.format.Vp9PayloadType +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.codec.VideoCodecParser +import org.jitsi.nlj.rtp.codec.av1.Av1DDParser +import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet +import org.jitsi.nlj.rtp.codec.vp8.Vp8Parser +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.nlj.rtp.codec.vp9.Vp9Parser +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger + +/** + * Parse video packets at a codec level + */ +class VideoParser( + private val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext +) : TransformerNode("Video parser") { + private val logger = createChildLogger(parentLogger) + private val stats = Stats() + + private var sources: Array = arrayOf() + private var signaledSources: Array = sources + + private var av1DDExtId: Int? = null + + private val videoCodecParsers = mutableMapOf() + + init { + streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { + av1DDExtId = it + } + } + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val packet = packetInfo.packetAs() + val av1DDExtId = this.av1DDExtId // So null checks work + val payloadType = streamInformationStore.rtpPayloadTypes[packet.payloadType.toByte()] ?: run { + logger.error("Unrecognized video payload type ${packet.payloadType}, cannot parse video information") + stats.numPacketsDroppedUnknownPt++ + return null + } + val videoCodecParser: VideoCodecParser? + val parsedPacket = try { + when { + payloadType is Vp8PayloadType -> { + val (vp8Packet, parser) = parseNormalPayload(packetInfo, ::Vp8Packet) { source -> + Vp8Parser(source, logger) + } + videoCodecParser = parser + vp8Packet + } + + payloadType is Vp9PayloadType -> { + val (vp9Packet, parser) = parseNormalPayload(packetInfo, ::Vp9Packet) { source -> + Vp9Parser(source, logger) + } + videoCodecParser = parser + vp9Packet + } + + av1DDExtId != null && packet.getHeaderExtension(av1DDExtId) != null -> { + videoCodecParser = checkParserType(packetInfo) { source -> + Av1DDParser(source, logger, diagnosticContext) + } + + val av1DDPacket = videoCodecParser?.createFrom(packet, av1DDExtId)?.also { + packetInfo.packet = it + packetInfo.resetPayloadVerification() + } + + av1DDPacket + } + + else -> { + val curParser = videoCodecParsers[packet.ssrc] + if (curParser != null) { + logger.cdebug { + "Removing videoCodecParser on ${payloadType.javaClass} packet, " + + "current videoCodecParser is ${curParser.javaClass}" + } + sources.findRtpSource(packet)?.let { source -> + resetSource(source) + source.rtpEncodings.forEach { + videoCodecParsers.remove(it.primarySSRC) + } + } + packetInfo.layeringChanged = true + } + return packetInfo + } + }.also { + videoCodecParser?.parse(packetInfo) + } + } catch (e: Exception) { + logger.error( + "Exception parsing video packet. Packet data is: " + + packet.buffer.toHex(packet.offset, Math.min(packet.length, 80)), + e + ) + return null + } + + /* Some codecs mark keyframes in every packet of the keyframe - only count the start of the frame, + * so the count is correct. */ + /* Alternately we could keep track of keyframes we've already seen, by timestamp, but that seems unnecessary. */ + if (parsedPacket != null && parsedPacket.isKeyframe && parsedPacket.isStartOfFrame) { + logger.cdebug { "Received a keyframe for ssrc ${packet.ssrc} at seq ${packet.sequenceNumber}" } + stats.numKeyframes++ + } + if (packetInfo.layeringChanged) { + logger.cdebug { "Layering structure changed for ssrc ${packet.ssrc} at seq ${packet.sequenceNumber}" } + stats.numLayeringChanges++ + } + + return packetInfo + } + + /** A normal payload is one where we choose the subclass of the ParsedVideoPacket and VideoCodecParser + * based on the payload type, as opposed to the header extension (like AV1). If the packet doesn't + * satisfy [ParsedVideoPacket.meetsRoutingNeeds] but it has an AV1 DD header extension, we will parse + * this packet as AV1 rather than as its normal type. + * */ + private inline fun parseNormalPayload( + packetInfo: PacketInfo, + otherTypeCreator: (ByteArray, Int, Int) -> ParsedVideoPacket, + parserConstructor: (MediaSourceDesc) -> T + ): Pair { + val parsedPacket = packetInfo.packet.toOtherType(otherTypeCreator) + if (!parsedPacket.meetsRoutingNeeds()) { + // See if we can parse this packet as AV1 + val packet = packetInfo.packetAs() + val av1DDExtId = this.av1DDExtId // So null checks work + if (av1DDExtId != null && packet.getHeaderExtension(av1DDExtId) != null) { + val parser = checkParserType(packetInfo) { source -> + Av1DDParser(source, logger, diagnosticContext) + } + + val av1DDPacket = parser?.createFrom(packet, av1DDExtId)?.also { + packetInfo.packet = it + packetInfo.resetPayloadVerification() + } + + return Pair(av1DDPacket, parser) + } + } + packetInfo.packet = parsedPacket + packetInfo.resetPayloadVerification() + + val parser = checkParserType(packetInfo, parserConstructor) + + return Pair(parsedPacket, parser) + } + + private inline fun checkParserType( + packetInfo: PacketInfo, + constructor: (MediaSourceDesc) -> T + ): T? { + val packet = packetInfo.packetAs() + val parser = videoCodecParsers[packet.ssrc] + if (parser is T) { + return parser + } + + val source = sources.findRtpSource(packet) + ?: // VideoQualityLayerLookup will drop this packet later, so no need to warn about it now + return null + logger.cdebug { + "Creating new ${T::class.java.simpleName} for source ${source.sourceName}, " + + "current videoCodecParser is ${parser?.javaClass?.simpleName}" + } + resetSource(source) + packetInfo.layeringChanged = true + val newParser = constructor(source) + source.rtpEncodings.forEach { + videoCodecParsers[it.primarySSRC] = newParser + } + + return newParser + } + + override fun handleEvent(event: Event) { + when (event) { + is SetMediaSourcesEvent -> { + sources = event.mediaSourceDescs + signaledSources = event.signaledMediaSourceDescs + val ssrcsSeen = mutableSetOf() + sources.forEach { source -> + source.rtpEncodings.forEach { + videoCodecParsers[it.primarySSRC]?.source = source + ssrcsSeen.add(it.primarySSRC) + } + } + videoCodecParsers.keys.removeIf { !ssrcsSeen.contains(it) } + } + } + super.handleEvent(event) + } + + private fun resetSource(source: MediaSourceDesc) { + val signaledSource = signaledSources.findRtpSourceByPrimary(source.primarySSRC) + if (signaledSource == null) { + logger.warn("Unable to find signaled source corresponding to ${source.primarySSRC}") + return + } + logger.cdebug { "Resetting source ${source.sourceName} to signaled source: $signaledSource" } + for (signaledEncoding in signaledSource.rtpEncodings) { + source.setEncodingLayers(signaledEncoding.layers, signaledEncoding.primarySSRC) + } + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { stats.addToNodeStatsBlock(this) } + override fun statsJson() = super.statsJson().apply { stats.addToJson(this) } + + fun getStats() = stats.snapshot() + + class Stats { + var numKeyframes = 0 + var numLayeringChanges = 0 + var numPacketsDroppedUnknownPt = 0 + + fun snapshot() = Snapshot(numKeyframes, numLayeringChanges, numPacketsDroppedUnknownPt) + + fun addToNodeStatsBlock(nodeStatsBlock: NodeStatsBlock) = nodeStatsBlock.apply { + addNumber("num_packets_dropped_unknown_pt", numPacketsDroppedUnknownPt) + addNumber("num_keyframes", numKeyframes) + addNumber("num_layering_changes", numLayeringChanges) + } + fun addToJson(o: ObjectNode) { + o.put("num_packets_dropped_unknown_pt", numPacketsDroppedUnknownPt) + o.put("num_keyframes", numKeyframes) + o.put("num_layering_changes", numLayeringChanges) + } + + data class Snapshot( + val numKeyframes: Int, + var numLayeringChanges: Int, + var numPacketsDroppedUnknownPt: Int + ) { + fun toJson() = JsonNodeFactory.instance.objectNode().apply { + put("num_packets_dropped_unknown_pt", numPacketsDroppedUnknownPt) + put("num_keyframes", numKeyframes) + put("num_layering_changes", numLayeringChanges) + } + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt new file mode 100644 index 0000000..276651b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookup.kt @@ -0,0 +1,70 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.Event +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.findRtpEncodingId +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.TransformerNode +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import java.util.concurrent.atomic.AtomicInteger + +/** + * Set video packets' quality layer info + */ +class VideoQualityLayerLookup( + parentLogger: Logger +) : TransformerNode("Video quality layer lookup") { + private val logger = createChildLogger(parentLogger) + private var sources: Array = arrayOf() + private val numPacketsDroppedNoEncoding = AtomicInteger() + + override fun transform(packetInfo: PacketInfo): PacketInfo? { + val videoPacket = packetInfo.packetAs() + val encodingId = sources.findRtpEncodingId(videoPacket) + if (encodingId == null) { + logger.warn( + "Unable to find encoding matching packet! packet=$videoPacket; " + + "sources=${sources.joinToString(separator = "\n")}" + ) + numPacketsDroppedNoEncoding.incrementAndGet() + return null + } + videoPacket.encodingId = encodingId + + return packetInfo + } + + override fun handleEvent(event: Event) { + when (event) { + is SetMediaSourcesEvent -> { + sources = event.mediaSourceDescs + } + } + super.handleEvent(event) + } + + override fun trace(f: () -> Unit) = f.invoke() + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_packets_dropped_no_encoding", numPacketsDroppedNoEncoding.get()) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt new file mode 100644 index 0000000..4f16fe4 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt @@ -0,0 +1,117 @@ +/* + * Copyright @ 2024-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import org.jitsi.nlj.Event +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.findRtpSource +import org.jitsi.nlj.rtp.RtpExtensionType.VLA +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.kbps +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.VlaExtension +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import kotlin.math.min + +/** + * A node which reads the Video Layers Allocation (VLA) RTP header extension and updates the media sources. + */ +class VlaReaderNode( + streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger = LoggerImpl(VlaReaderNode::class.simpleName) +) : ObserverNode("Video Layers Allocation reader") { + private val logger = createChildLogger(parentLogger) + private var vlaExtId: Int? = null + private var mediaSourceDescs: Array = arrayOf() + + init { + streamInformationStore.onRtpExtensionMapping(VLA) { + vlaExtId = it + logger.debug("VLA extension ID set to $it") + } + } + + override fun handleEvent(event: Event) { + when (event) { + is SetMediaSourcesEvent -> { + mediaSourceDescs = event.mediaSourceDescs.copyOf() + logger.cdebug { "Media sources changed:\n${mediaSourceDescs.joinToString()}" } + } + } + } + + override fun observe(packetInfo: PacketInfo) { + val rtpPacket = packetInfo.packetAs() + vlaExtId?.let { + rtpPacket.getHeaderExtension(it)?.let { ext -> + val vla = try { + VlaExtension.parse(ext) + } catch (e: Exception) { + logger.warn("Failed to parse VLA extension", e) + return + } + + val sourceDesc = mediaSourceDescs.findRtpSource(rtpPacket) + + logger.debug("Found VLA=$vla for sourceDesc=$sourceDesc") + + vla.forEachIndexed { streamIdx, stream -> + val rtpEncoding = sourceDesc?.rtpEncodings?.get(streamIdx) + stream.spatialLayers.forEach { spatialLayer -> + val maxTl = spatialLayer.targetBitratesKbps.size - 1 + + spatialLayer.targetBitratesKbps.forEachIndexed { tlIdx, targetBitrateKbps -> + rtpEncoding?.layers?.find { + // With VP8 simulcast all layers have sid -1 + (it.sid == spatialLayer.id || it.sid == -1) && it.tid == tlIdx + }?.let { layer -> + logger.debug { + "Setting target bitrate for rtpEncoding=$rtpEncoding layer=$layer to " + + "${targetBitrateKbps.kbps} (res=${spatialLayer.res})" + } + layer.targetBitrate = targetBitrateKbps.kbps + spatialLayer.res?.let { res -> + // Treat the lesser of width and height as the height + // in order to handle portrait-mode video correctly + val minDimension = min(res.height, res.width) + if (layer.height > 0 && layer.height != minDimension) { + logger.info { + "Updating layer height for source ${sourceDesc.sourceName} " + + "encoding ${rtpEncoding.primarySSRC} layer ${layer.indexString()} " + + "from ${layer.height} to $minDimension" + } + } + layer.height = minDimension + /* Presume 2:1 frame rate ratios for temporal layers */ + val framerateFraction = 1.0 / (1 shl (maxTl - tlIdx)) + layer.frameRate = res.maxFramerate.toDouble() * framerateFraction + } + } + } + } + } + } + } + } + + override fun trace(f: () -> Unit) {} +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/AbsSendTime.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/AbsSendTime.kt new file mode 100644 index 0000000..c6ea46f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/AbsSendTime.kt @@ -0,0 +1,55 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.RtpExtensionType.ABS_SEND_TIME +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ModifierNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.AbsSendTimeHeaderExtension + +class AbsSendTime( + val streamInformationStore: ReadOnlyStreamInformationStore +) : ModifierNode("Absolute send time") { + private var extensionId: Int? = null + + init { + streamInformationStore.onRtpExtensionMapping(ABS_SEND_TIME) { + extensionId = it + } + } + + override fun modify(packetInfo: PacketInfo): PacketInfo { + if (streamInformationStore.supportsTcc) return packetInfo + + extensionId?.let { absSendTimeExtId -> + val rtpPacket = packetInfo.packetAs() + val ext = rtpPacket.getHeaderExtension(absSendTimeExtId) + ?: rtpPacket.addHeaderExtension(absSendTimeExtId, AbsSendTimeHeaderExtension.DATA_SIZE_BYTES) + AbsSendTimeHeaderExtension.setTime(ext, System.nanoTime()) + } + + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addString("abs_send_time_ext_id", extensionId.toString()) + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtEncoder.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtEncoder.kt new file mode 100644 index 0000000..93ea1e0 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtEncoder.kt @@ -0,0 +1,56 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ModifierNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.OneByteHeaderExtensionParser +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger + +class HeaderExtEncoder( + streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : ModifierNode("Header extension encoder") { + private val logger = createChildLogger(parentLogger) + private var extmapAllowMixed = false + + init { + streamInformationStore.onExtmapAllowMixedChanged { extmapAllowMixed = it } + } + + override fun modify(packetInfo: PacketInfo): PacketInfo { + val rtpPacket = packetInfo.packetAs() + + rtpPacket.encodeHeaderExtensions() + + if (rtpPacket.hasExtensions) { + val profileType = rtpPacket.extensionsProfileType + + if (!extmapAllowMixed && profileType != OneByteHeaderExtensionParser.headerExtensionLabel) { + logger.warn( + "Sending header extension profile type ${Integer.toHexString(profileType)} " + + "when extmap-allow-mixed is not enabled" + ) + } + } + return packetInfo + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt new file mode 100644 index 0000000..979f271 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/HeaderExtStripper.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.transform.node.ModifierNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket + +/** + * Strip all hop-by-hop header extensions. By default, this leaves ssrc-audio-level and video-orientation, + * plus the AV1 dependency descriptor if the packet is an Av1DDPacket. + */ +class HeaderExtStripper( + streamInformationStore: ReadOnlyStreamInformationStore, +) : ModifierNode("Strip header extensions") { + private var retainedExts: Set = emptySet() + private var retainedExtsWithAv1DD: Set = emptySet() + private var retainedExtTypes = defaultRetainedExtTypes + + init { + retainedExtTypes.forEach { rtpExtensionType -> + streamInformationStore.onRtpExtensionMapping(rtpExtensionType) { + it?.let { + retainedExts += it + retainedExtsWithAv1DD += it + } + } + } + streamInformationStore.onRtpExtensionMapping(RtpExtensionType.AV1_DEPENDENCY_DESCRIPTOR) { + it?.let { retainedExtsWithAv1DD += it } + } + } + + fun addRtpExtensionToRetain(extensionType: RtpExtensionType) { + retainedExtTypes += extensionType + } + + override fun modify(packetInfo: PacketInfo): PacketInfo { + val rtpPacket = packetInfo.packetAs() + + val retained = if (rtpPacket is Av1DDPacket) retainedExtsWithAv1DD else retainedExts + + // TODO: we should also retain any extensions that were not signaled. + rtpPacket.removeHeaderExtensionsExcept(retained) + + return packetInfo + } + + override fun trace(f: () -> Unit) = f.invoke() + + companion object { + val defaultRetainedExtTypes: Set = setOf( + RtpExtensionType.SSRC_AUDIO_LEVEL, + RtpExtensionType.VIDEO_ORIENTATION + ) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/MidStamper.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/MidStamper.kt new file mode 100644 index 0000000..6c3888b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/MidStamper.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.RtpExtensionType.MID +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ModifierNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.SdesHeaderExtension + +/** + * Stamps the sdes:mid (media identification) RTP header extension on outgoing packets, using a per-SSRC mid supplied by + * [getMidBySsrc]. This is used under SSRC rewriting so that the receiving client can demux forwarded media by mid (which + * disables Chrome's payload-type demuxing fallback, the trigger for the audio-demux wedge). + * + * This node sits in the shared portion of the outgoing pipeline (after RTX encapsulation) so that both media packets and + * their retransmissions are stamped with the same mid. The mid is resolved from the packet's (already rewritten) SSRC, + * which is stable per slot, so primary and RTX SSRCs of a slot resolve to the same mid. + * + * Does nothing unless the mid extension has been negotiated for this endpoint and [getMidBySsrc] returns a mid for the + * packet's SSRC (i.e. the endpoint opted into mid-based demuxing and the SSRC is a rewritten slot SSRC). + */ +class MidStamper( + private val streamInformationStore: ReadOnlyStreamInformationStore, + private val getMidBySsrc: (Long) -> String? +) : ModifierNode("Mid stamper") { + private var extensionId: Int? = null + private var numStamped = 0 + + init { + streamInformationStore.onRtpExtensionMapping(MID) { + extensionId = it + } + } + + override fun modify(packetInfo: PacketInfo): PacketInfo { + val midExtId = extensionId ?: return packetInfo + val rtpPacket = packetInfo.packetAs() + val mid = getMidBySsrc(rtpPacket.ssrc) ?: return packetInfo + + // The header extensions of outgoing packets have been stripped by this point, so a mid extension is only present + // if we (or RTX, re-using the same packet) added it. Avoid adding it twice. + if (rtpPacket.getHeaderExtension(midExtId) == null) { + val ext = rtpPacket.addHeaderExtension(midExtId, mid.length) + SdesHeaderExtension.setTextValue(ext, mid) + numStamped++ + } + + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addString("mid_ext_id", extensionId.toString()) + addNumber("num_stamped", numStamped) + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt new file mode 100644 index 0000000..312b27f --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/OutgoingStatisticsTracker.kt @@ -0,0 +1,164 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.nlj.transform.node.incoming.BitrateCalculator +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import java.util.concurrent.ConcurrentHashMap + +class OutgoingStatisticsTracker( + private val diagnosticContext: DiagnosticContext, +) : ObserverNode("Outgoing statistics tracker") { + /** + * Per-SSRC statistics + */ + private val ssrcStats: MutableMap = ConcurrentHashMap() + + private var numAudioPackets = 0 + private var numVideoPackets = 0 + + private val videoBitrate = BitrateCalculator.createBitrateTracker() + + private val videoBitratesByOrigin = mutableMapOf() + + override fun observe(packetInfo: PacketInfo) { + val rtpPacket = packetInfo.packetAs() + + when (rtpPacket) { + is AudioRtpPacket -> numAudioPackets++ + + is VideoRtpPacket -> { + numVideoPackets++ + + /* These bitrate measurements are only used in the timeseries log below, so only calculate them + * if the timeseries is enabled. */ + if (timeseriesLogger.isTraceEnabled) { + videoBitrate.update(rtpPacket.length.bytes) + videoBitratesByOrigin.getOrPut(packetInfo.packetOrigin) { + BitrateCalculator.createBitrateTracker() + }.update(rtpPacket.length.bytes) + } + } + } + + val stats = ssrcStats.computeIfAbsent(rtpPacket.ssrc) { + OutgoingSsrcStats(rtpPacket.ssrc) + } + stats.packetSent(rtpPacket.length, rtpPacket.timestamp) + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + val stats = getSnapshot() + stats.ssrcStats.forEach { (ssrc, streamStats) -> + addJson(ssrc.toString(), streamStats.toJson()) + } + } + + /** + * Don't aggregate the per-SSRC stats. + */ + override fun getNodeStatsToAggregate(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_audio_packets", numAudioPackets) + addNumber("num_video_packets", numVideoPackets) + } + + override fun trace(f: () -> Unit) = f.invoke() + + fun getSnapshot(): OutgoingStatisticsSnapshot = OutgoingStatisticsSnapshot( + ssrcStats.map { (ssrc, stats) -> + Pair(ssrc, stats.getSnapshot()) + }.toMap() + ).also { + if (timeseriesLogger.isTraceEnabled) { + val point = diagnosticContext.makeTimeSeriesPoint("sent_video_stream_stats") + .addField("bitrate_bps", videoBitrate.rate.bps) + videoBitratesByOrigin.forEach { (origin, tracker) -> + point.addField("video_${origin}_bitrate", tracker.rate.bps) + } + + timeseriesLogger.trace(point) + } + } + + fun getSsrcSnapshot(ssrc: Long): OutgoingSsrcStats.Snapshot? = ssrcStats[ssrc]?.getSnapshot() + + companion object { + private val timeseriesLogger = TimeSeriesLogger.getTimeSeriesLogger(OutgoingStatisticsTracker::class.java) + } +} + +class OutgoingStatisticsSnapshot( + /** + * Per-ssrc stats. + */ + val ssrcStats: Map +) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + ssrcStats.forEach { (ssrc, snapshot) -> + set(ssrc.toString(), snapshot.toJson()) + } + } +} + +class OutgoingSsrcStats( + private val ssrc: Long +) { + private var statsLock = Any() + + // Start variables protected by statsLock + private var packetCount: Int = 0 + private var octetCount: Int = 0 + private var mostRecentRtpTimestamp: Long = 0 + // End variables protected by statsLock + + fun packetSent(packetSizeOctets: Int, rtpTimestamp: Long) { + synchronized(statsLock) { + packetCount++ + octetCount += packetSizeOctets + mostRecentRtpTimestamp = rtpTimestamp + } + } + + fun getSnapshot(): Snapshot { + synchronized(statsLock) { + return Snapshot(packetCount, octetCount, mostRecentRtpTimestamp) + } + } + + data class Snapshot( + val packetCount: Int, + val octetCount: Int, + val mostRecentRtpTimestamp: Long + ) { + fun toJson() = JsonNodeFactory.instance.objectNode().apply { + put("packet_count", packetCount) + put("octet_count", octetCount) + put("most_recent_rtp_timestamp", mostRecentRtpTimestamp) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/ProbingDataSender.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/ProbingDataSender.kt new file mode 100644 index 0000000..9c8e997 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/ProbingDataSender.kt @@ -0,0 +1,217 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.outgoing + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.Event +import org.jitsi.nlj.EventHandler +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.SetLocalSsrcEvent +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.format.VideoPayloadType +import org.jitsi.nlj.rtp.PaddingVideoPacket +import org.jitsi.nlj.util.PacketCache +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtp.RtpHeader +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import java.util.Random +import java.util.concurrent.ConcurrentHashMap + +/** + * [ProbingDataSender] currently supports probing via 2 methods: + * 1) retransmitting previous packets via RTX via [sendRedundantDataOverRtx]. + * 2) If RTX is not available, or, not enough packets to retransmit are available, we + * can send empty media packets using the bridge's ssrc + * + */ +class ProbingDataSender( + private val packetCache: PacketCache, + private val rtxDataSender: PacketHandler, + private val garbageDataSender: PacketHandler, + private val diagnosticContext: DiagnosticContext, + streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : EventHandler { + + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass) + private val logger = createChildLogger(parentLogger) + + private var rtxSupported = false + private val videoPayloadTypes: MutableSet = ConcurrentHashMap.newKeySet() + private var localVideoSsrc: Long? = null + + // Stats + private var numProbingBytesSentRtx: Long = 0 + private var numProbingBytesSentDummyData: Long = 0 + + private var lastMediaSsrcs: Collection = emptyList() + + init { + streamInformationStore.onRtpPayloadTypesChanged { currentRtpPayloadTypes -> + if (currentRtpPayloadTypes.isEmpty()) { + videoPayloadTypes.clear() + rtxSupported = false + } else { + currentRtpPayloadTypes.values.forEach { pt -> + if (!rtxSupported && pt is RtxPayloadType) { + rtxSupported = true + logger.cdebug { "RTX payload type signaled, enabling RTX probing" } + } + if (pt is VideoPayloadType) { + videoPayloadTypes.add(pt) + } + } + } + } + } + + fun sendProbing(mediaSsrcsIn: Collection?, numBytes: Int, probingInfo: Any?): Int { + var totalBytesSent = 0 + + val mediaSsrcs = if (mediaSsrcsIn != null) { + lastMediaSsrcs = mediaSsrcsIn + mediaSsrcsIn + } else { + lastMediaSsrcs + } + + if (rtxSupported) { + for (mediaSsrc in mediaSsrcs) { + if (totalBytesSent >= numBytes) { + break + } + val rtxBytesSent = sendRedundantDataOverRtx(mediaSsrc, numBytes - totalBytesSent, probingInfo) + numProbingBytesSentRtx += rtxBytesSent + totalBytesSent += rtxBytesSent + if (timeSeriesLogger.isTraceEnabled()) { + timeSeriesLogger.trace( + diagnosticContext + .makeTimeSeriesPoint("rtx_probing_bytes") + .addField("ssrc", mediaSsrc) + .addField("bytes", rtxBytesSent) + ) + } + } + } + if (totalBytesSent < numBytes) { + val dummyBytesSent = sendDummyData(numBytes - totalBytesSent, probingInfo) + numProbingBytesSentDummyData += dummyBytesSent + totalBytesSent += dummyBytesSent + if (timeSeriesLogger.isTraceEnabled()) { + timeSeriesLogger.trace( + diagnosticContext + .makeTimeSeriesPoint("dummy_probing_bytes") + .addField("bytes", dummyBytesSent) + ) + } + } + + return totalBytesSent + } + + /** + * Using the RTX stream associated with [mediaSsrc], send [numBytes] of data + * by re-transmitting previously sent packets from the outgoing packet cache. + * Returns the number of bytes transmitted + */ + private fun sendRedundantDataOverRtx(mediaSsrc: Long, numBytes: Int, probingInfo: Any?): Int { + var bytesSent = 0 + // TODO(brian): we're in a thread context mess here. we'll be sending these out from the bandwidthprobing + // context (or whoever calls this) which i don't think we want. Need look at getting all the pipeline + // work posted to one thread so we don't have to worry about concurrency nightmares + + // Get the most recent packets whose length add up to no more than numBytes. + packetCache.getMany(mediaSsrc, numBytes).forEach { + bytesSent += it.length + val packetInfo = PacketInfo(it) + packetInfo.probingInfo = probingInfo + packetInfo.packetOrigin = if (probingInfo != null) PacketOrigin.Probing else PacketOrigin.Padding + rtxDataSender.processPacket(packetInfo) + } + return bytesSent + } + + private var currDummyTimestamp = random.nextLong() and 0xFFFFFFFF + private var currDummySeqNum = random.nextInt(0xFFFF) + + private fun sendDummyData(numBytes: Int, probingInfo: Any?): Int { + var bytesSent = 0 + val pt = videoPayloadTypes.firstOrNull() ?: return bytesSent + val senderSsrc = localVideoSsrc ?: return bytesSent + + while (bytesSent < numBytes) { + val remainingBytes = numBytes - bytesSent + if (remainingBytes < RtpHeader.FIXED_HEADER_SIZE_BYTES) { + break + } + val paddingSize = (remainingBytes - RtpHeader.FIXED_HEADER_SIZE_BYTES).coerceAtMost(0xFF) + val packetLength = RtpHeader.FIXED_HEADER_SIZE_BYTES + paddingSize + + val paddingPacket = PaddingVideoPacket.create(packetLength) + paddingPacket.payloadType = pt.pt.toPositiveInt() + paddingPacket.ssrc = senderSsrc + paddingPacket.timestamp = currDummyTimestamp + paddingPacket.sequenceNumber = currDummySeqNum + val packetInfo = PacketInfo(paddingPacket) + packetInfo.probingInfo = probingInfo + packetInfo.packetOrigin = if (probingInfo != null) PacketOrigin.Probing else PacketOrigin.Padding + garbageDataSender.processPacket(packetInfo) + + currDummySeqNum++ + bytesSent += packetLength + } + currDummyTimestamp += 3000 + + return bytesSent + } + + override fun handleEvent(event: Event) { + when (event) { + is SetLocalSsrcEvent -> { + if (MediaType.VIDEO == event.mediaType) { + logger.cdebug { "Setting video ssrc to ${event.ssrc}" } + localVideoSsrc = event.ssrc + } + } + } + } + + fun debugState(mode: DebugStateMode) = JsonNodeFactory.instance.objectNode().apply { + put("num_bytes_of_probing_data_sent_as_rtx", numProbingBytesSentRtx) + put("num_bytes_of_probing_data_sent_as_dummy", numProbingBytesSentDummyData) + put("rtx_supported", rtxSupported) + if (mode == DebugStateMode.FULL) { + put("local_video_ssrc", localVideoSsrc.toString()) + put("curr_dummy_timestamp", currDummyTimestamp.toString()) + put("curr_dummy_seq_num", currDummySeqNum.toString()) + put("video_payload_types", videoPayloadTypes.toString()) + } + } + + companion object { + val random = Random() + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt new file mode 100644 index 0000000..01e25de --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSender.kt @@ -0,0 +1,128 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.rtp.RtxPacket +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ModifierNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import java.util.concurrent.ConcurrentHashMap + +class RetransmissionSender( + private val streamInformationStore: ReadOnlyStreamInformationStore, + parentLogger: Logger +) : ModifierNode("Retransmission sender") { + private val logger = createChildLogger(parentLogger) + + /** + * Maps an original payload type (Int) to its [RtxPayloadType] + */ + private val origPtToRtxPayloadType: MutableMap = ConcurrentHashMap() + + /** + * A map of rtx stream ssrc to the current sequence number for that stream + */ + private val rtxStreamSeqNums: MutableMap = mutableMapOf() + + private var numRetransmissionsRequested = 0 + private var numRetransmittedRtxPackets = 0 + private var numRetransmittedPlainPackets = 0 + + init { + streamInformationStore.onRtpPayloadTypesChanged { currentRtpPayloadTypes -> + origPtToRtxPayloadType.clear() + currentRtpPayloadTypes.values + .filterIsInstance() + .map { + origPtToRtxPayloadType[it.associatedPayloadType] = it + } + } + } + + /** + * Transform an original RTP packet into an RTX-encapsulated form of that packet. + */ + override fun modify(packetInfo: PacketInfo): PacketInfo { + val rtpPacket = packetInfo.packetAs() + numRetransmissionsRequested++ + val rtxPt = origPtToRtxPayloadType[rtpPacket.payloadType.toPositiveInt()] + ?: return retransmitPlain(packetInfo) + val rtxSsrc = streamInformationStore.getRemoteSecondarySsrc(rtpPacket.ssrc, SsrcAssociationType.RTX) + ?: return retransmitPlain(packetInfo) + + return retransmitRtx(packetInfo, rtxPt.pt.toPositiveInt(), rtxSsrc) + } + + private fun retransmitRtx(packetInfo: PacketInfo, rtxPt: Int, rtxSsrc: Long): PacketInfo { + // Get a default value of 1 to start if it isn't present in the map. If it is present + // in the map, get the value and increment it by 1 + val rtxSeqNum = rtxStreamSeqNums.merge(rtxSsrc, 1, Integer::sum)!! + val rtpPacket = packetInfo.packetAs() + logger.cdebug { + "${hashCode()} sending RTX packet with ssrc $rtxSsrc with pt $rtxPt and seqNum " + + "$rtxSeqNum with original ssrc ${rtpPacket.ssrc}, original sequence number " + + "${rtpPacket.sequenceNumber} and original payload type: ${rtpPacket.payloadType}" + } + RtxPacket.addOriginalSequenceNumber(rtpPacket) + rtpPacket.ssrc = rtxSsrc + rtpPacket.payloadType = rtxPt + rtpPacket.sequenceNumber = rtxSeqNum + + packetInfo.resetPayloadVerification() + numRetransmittedRtxPackets++ + + return packetInfo + } + + private fun retransmitPlain(packetInfo: PacketInfo): PacketInfo { + val rtpPacket = packetInfo.packetAs() + logger.cdebug { + "${hashCode()} plain retransmission packet with original ssrc " + + "${rtpPacket.ssrc}, original sequence number ${rtpPacket.sequenceNumber} and original " + + "payload type: ${rtpPacket.payloadType}" + } + + numRetransmittedPlainPackets++ + // No work needed + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_retransmissions_requested", numRetransmissionsRequested) + addNumber("num_retransmissions_rtx_sent", numRetransmittedRtxPackets) + addNumber("num_retransmissions_plain_sent", numRetransmittedPlainPackets) + addString( + "rtx_payload_types(orig -> rtx)", + this@RetransmissionSender.origPtToRtxPayloadType.toString() + ) + } + + override fun statsJson() = super.statsJson().apply { + put("num_retransmissions_requested", numRetransmissionsRequested) + put("num_retransmissions_rtx_sent", numRetransmittedRtxPackets) + put("num_retransmissions_plain_sent", numRetransmittedPlainPackets) + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/SentRtcpStats.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/SentRtcpStats.kt new file mode 100644 index 0000000..0172cc7 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/SentRtcpStats.kt @@ -0,0 +1,46 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ObserverNode +import org.jitsi.rtp.rtcp.RtcpPacket +import java.util.concurrent.ConcurrentHashMap + +class SentRtcpStats : ObserverNode("Sent RTCP stats") { + private var sentRtcpCounts: MutableMap = ConcurrentHashMap() + + override fun observe(packetInfo: PacketInfo) { + val rtcpPacket: RtcpPacket = packetInfo.packetAs() + sentRtcpCounts.merge(rtcpPacket::class.simpleName!!, 1, Int::plus) + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + sentRtcpCounts.forEach { (rtcpType, count) -> + addNumber("num_${rtcpType}_tx", count) + } + } + + override fun statsJson() = super.statsJson().apply { + sentRtcpCounts.forEach { (rtcpType, count) -> + put("num_${rtcpType}_tx", count) + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/TccSeqNumTagger.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/TccSeqNumTagger.kt new file mode 100644 index 0000000..b3322d0 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/outgoing/TccSeqNumTagger.kt @@ -0,0 +1,79 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.outgoing + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.RtpExtensionType.TRANSPORT_CC +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ModifierNode +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.TccHeaderExtension +import java.lang.ref.WeakReference + +class TccSeqNumTagger( + transportCcEngine: TransportCcEngine, + streamInformationStore: ReadOnlyStreamInformationStore +) : ModifierNode("TCC sequence number tagger") { + private var currTccSeqNum: Long = 1 + private var tccExtensionId: Int? = null + + init { + streamInformationStore.onRtpExtensionMapping(TRANSPORT_CC) { + tccExtensionId = it + } + } + + private val weakTcc = WeakReference(transportCcEngine) + + override fun modify(packetInfo: PacketInfo): PacketInfo { + tccExtensionId?.let { tccExtId -> + when (val rtpPacket = packetInfo.packetAs()) { + is VideoRtpPacket -> { + val ext = rtpPacket.getHeaderExtension(tccExtId) + ?: rtpPacket.addHeaderExtension(tccExtId, TccHeaderExtension.DATA_SIZE_BYTES) + + val curSeq = currTccSeqNum + + TccHeaderExtension.setSequenceNumber(ext, curSeq.toInt()) + + weakTcc.get()?.mediaPacketTagged(packetInfo, curSeq) + + packetInfo.onSent { pkt -> weakTcc.get()?.mediaPacketSent(pkt, curSeq) } + + currTccSeqNum++ + } + + else -> Unit + } + } + + return packetInfo + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addString("tcc_ext_id", tccExtensionId.toString()) + } + + override fun stop() { + super.stop() + weakTcc.get()?.stop() + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt new file mode 100644 index 0000000..d169614 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ArrayCache.kt @@ -0,0 +1,277 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.NodeStatsProducer +import java.lang.Integer.max +import java.lang.Long.max +import java.time.Clock +import java.util.concurrent.atomic.AtomicInteger + +/** + * Implements a fixed-sized cache based on a pre-filled array. The main use-case is the outgoing RTP packet cache. + * + * @author Boris Grozev + */ +open class ArrayCache( + val size: Int, + private val cloneItem: (T) -> T, + protected val synchronize: Boolean = true, + /** + * The function to use to clone items. The cache always saves copies of the items that are inserted. + */ + protected val clock: Clock = Clock.systemUTC() +) : NodeStatsProducer { + private val cache: Array = Array(size) { Container() } + protected val syncRoot = Any() + + /** + * The index in [cache] where the item with the highest index is stored. + */ + private var head = -1 + + var numInserts = 0 + var numOldInserts = 0 + private val _numHits = AtomicInteger() + private val _numMisses = AtomicInteger() + val numHits + get() = _numHits.get() + val numMisses + get() = _numMisses.get() + val hitRate + get() = _numHits.get() * 1.0 / max(1, _numHits.get() + _numMisses.get()) + + protected val lastIndex: Long + get() = if (head == -1) -1 else cache[head].index + + val empty: Boolean + get() = (head == -1) + + /** + * Inserts an item with a specific index in the cache. Stores a copy. + */ + fun insertItem(item: T, index: Long, timeAdded: Long): Boolean = if (synchronize) { + synchronized(syncRoot) { + doInsert(item, index, timeAdded) + } + } else { + doInsert(item, index, timeAdded) + } + + /** + * Inserts an item with a specific index in the cache, computing time + * from [clock]. Stores a copy. + */ + fun insertItem(item: T, index: Long): Boolean = insertItem(item, index, clock.millis()) + + private fun position(diff: Long): Int = ((head + diff) floorMod size.toLong()).toInt() + + private fun doInsert(item: T, index: Long, timeAdded: Long): Boolean { + val diff = if (head == -1) -1 else index - cache[head].index + val position = when { + head == -1 -> { + head = 0 + head + } + + diff <= -size -> { + // The item is too old + numOldInserts++ + return false + } + + diff < 0 -> position(diff) + + else -> { + head = position(diff) + head + } + } + + numInserts++ + cache[position].item?.let { discardItem(it) } + cache[position].item = cloneItem(item) + cache[position].index = index + cache[position].timeAdded = timeAdded + return true + } + + /** + * Called when an item in the cache is replaced/discarded. + */ + protected open fun discardItem(item: T) {} + + /** + * Gets an item from the cache with a given index. Returns 'null' if there is no item with this index in the cache. + * The item is wrapped in a [Container] to allow access to the time it was added to the cache, and we provide a + * copy. + */ + @JvmOverloads + fun getContainer(index: Long, shouldCloneItem: Boolean = true): Container? { + val result = when { + synchronize -> synchronized(syncRoot) { + doGet(index, shouldCloneItem) + } + + else -> doGet(index, shouldCloneItem) + } + + result?.let { _numHits.incrementAndGet() } ?: _numMisses.incrementAndGet() + return result + } + + private fun doGet(index: Long, shouldCloneItem: Boolean): Container? { + if (index < 0) { + return null + } + if (head == -1) { + // Not initialized (empty), or newer than head. + return null + } + + val diff = index - cache[head].index + if (diff > 0) { + // The requested index is newer than the last index we have. + return null + } + + val position = position(diff) + if (cache[position].index == index) { + return cache[position].clone(shouldCloneItem) + } + return null + } + + /** + * Checks whether the cache contains an item with a given index. + */ + fun containsIndex(index: Long): Boolean = if (synchronize) { + synchronized(syncRoot) { + doContains(index) + } + } else { + doContains(index) + } + + private fun doContains(index: Long): Boolean { + if (head == -1) { + return false + } + + val diff = index - cache[head].index + if (diff > 0) { + return false + } + + val position = position(diff) + return (cache[position].index == index) + } + + /** + * Updates the [timeAdded] value of an item with a particular index, if it is in the cache. + */ + protected fun updateTimeAdded(index: Long, timeAdded: Long) = if (synchronize) { + synchronized(syncRoot) { + doUpdateTimeAdded(index, timeAdded) + } + } else { + doUpdateTimeAdded(index, timeAdded) + } + + private fun doUpdateTimeAdded(index: Long, timeAdded: Long) { + if (head == -1 || index > cache[head].index) { + return + } + val diff = index - cache[head].index + val position = position(diff) + if (cache[position].index == index) { + cache[position].timeAdded = timeAdded + } + } + + /** + * Iterates from the last index added to the cache down at most [size] elements. For each item, if it is in the + * last 'window', calls [predicate] with the item. If [predicate] returns [false] for any item, stops the + * iteration and returns. Note that if the caller must clone the item on their own if they want to keep or modify + * it in any way. + */ + fun forEachDescending(predicate: (T) -> Boolean) = if (synchronize) { + synchronized(syncRoot) { + doForEachDescending(predicate) + } + } else { + doForEachDescending(predicate) + } + + private fun doForEachDescending(predicate: (T) -> Boolean) { + if (head == -1) return + + val indexRange = cache[head].index downTo max(cache[head].index - size, 1L) + for (i in 0 until size) { + val position = (head - i) floorMod size + if (cache[position].index in indexRange) { + // We maintain the invariant [index==-1 iff item==null] + if (!predicate(cache[position].item!!)) { + return + } + } + } + } + + /** + * Removes all items stored in the cache, calling [discardItem] for each one. + */ + fun flush() = if (synchronize) { + synchronized(syncRoot) { + doFlush() + } + } else { + doFlush() + } + + private fun doFlush() { + for (container in cache) { + with(container) { + index = -1 + item?.let { discardItem(it) } + item = null + timeAdded = -1 + } + } + head = -1 + } + + override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("ArrayCache").apply { + addNumber("size", size) + addNumber("numInserts", numInserts) + addNumber("numOldInserts", numOldInserts) + addNumber("numHits", numHits) + addNumber("numMisses", numMisses) + addNumber("numRequests", numHits + numMisses) + addRatio("hitRate", "numHits", "numRequests", 1) + } + + inner class Container( + var item: T? = null, + var index: Long = -1, + var timeAdded: Long = -1 + ) { + fun clone(shouldCloneItem: Boolean): Container = + Container(item?.let { if (shouldCloneItem) cloneItem(it) else it }, index, timeAdded) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt new file mode 100644 index 0000000..6e717d1 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Bandwidth.kt @@ -0,0 +1,178 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.utils.toDouble +import org.jitsi.utils.toRoundedMicros +import java.text.DecimalFormat +import java.time.Duration +import kotlin.math.round + +/** + * [Bandwidth] models a current bandwidth, represented as a rate of bits per second. + */ +@JvmInline +value class Bandwidth(val bps: Long) : Comparable { + + constructor(bps: Double) : this(bps.toLong()) + val bytesPerSec: Double + get() = bps.toDouble() / 8 + val kbps: Double + get() = bps.toDouble() / 1000 + val mbps: Double + get() = bps.toDouble() / (1000 * 1000) + + operator fun minus(other: Bandwidth): Bandwidth = Bandwidth(bps - other.bps) + + operator fun plus(other: Bandwidth): Bandwidth = Bandwidth(bps + other.bps) + + /** + * For multiplication, we support multiplying against + * a normal number (not another bandwidth). This allows + * applying some factor to a given bandwidth, for example + * + * currentBandwidth *= 0.95 + * + * to reduce 'currentBandwidth' by 5% + */ + operator fun times(other: Double): Bandwidth = Bandwidth(round(bps.toDouble() * other)) + + operator fun times(other: Int): Bandwidth = Bandwidth(bps * other) + + /** + * For division, we support both dividing by + * a normal number (giving a bandwidth), and dividing + * by another bandwidth, giving a number + */ + operator fun div(other: Double): Bandwidth = Bandwidth(round(bps / other)) + + operator fun div(other: Int): Bandwidth = Bandwidth(bps / other) + + operator fun div(other: Bandwidth): Double = bps.toDouble() / other.bps.toDouble() + + override fun compareTo(other: Bandwidth): Int = bps.compareTo(other.bps) + + override fun toString(): String { + // To determine which unit we'll print in, + // find the biggest one which has a value + // in the ones place + val format = DecimalFormat("0.##") + return when { + mbps >= 1 -> "${format.format(mbps)} mbps" + kbps >= 1 -> "${format.format(kbps)} kbps" + else -> "${format.format(bps)} bps" + } + } + + fun isInfinite() = (this == INFINITY || this == MINUS_INFINITY) + + fun isFinite() = !isInfinite() + + companion object { + fun fromString(str: String): Bandwidth { + val (digits, notDigits) = str.partition { it.isDigit() } + val amount = digits.toInt() + return when (val unit = notDigits.trim().lowercase()) { + "bps" -> amount.bps + "kbps" -> amount.kbps + "mbps" -> amount.mbps + else -> throw IllegalArgumentException("Unrecognized unit $unit") + } + } + + val INFINITY = Bandwidth(Long.MAX_VALUE) + val ZERO = Bandwidth(0.0) + val MINUS_INFINITY = Bandwidth(Long.MIN_VALUE) + } +} + +operator fun Double.times(other: Bandwidth): Bandwidth = Bandwidth(round(this * other.bps)) +operator fun Int.times(other: Bandwidth): Bandwidth = Bandwidth(this * other.bps) + +val Int.bps: Bandwidth + get() = Bandwidth(this.toLong()) +val Int.bytesPerSec: Bandwidth + get() = Bandwidth(this * 8L) +val Int.kbps: Bandwidth + get() = Bandwidth(this * 1000L) +val Int.mbps: Bandwidth + get() = Bandwidth(this * 1000L * 1000L) + +val Float.bps: Bandwidth + get() = Bandwidth(this.toLong()) +val Float.bytesPerSec: Bandwidth // Bytes per second + get() = Bandwidth(this.toDouble() * 8) +val Float.kbps: Bandwidth + get() = Bandwidth(this.toDouble() * 1000) +val Float.mbps: Bandwidth + get() = Bandwidth(this.toDouble() * 1000 * 1000) + +val Double.bps: Bandwidth + get() = Bandwidth(this) +val Double.bytesPerSec: Bandwidth // Bytes per second + get() = Bandwidth(this * 8) +val Double.kbps: Bandwidth + get() = Bandwidth(this * 1000) +val Double.mbps: Bandwidth + get() = Bandwidth(this * 1000 * 1000) + +val Long.bps: Bandwidth + get() = Bandwidth(this) +val Long.bytesPerSec: Bandwidth // Bytes per second + get() = Bandwidth(this * 8) +val Long.kbps: Bandwidth + get() = Bandwidth(this * 1000) +val Long.mbps: Bandwidth + get() = Bandwidth(this * 1000 * 1000) + +/** + * Create a [Bandwidth] from a [DataSize] over a given time + */ +fun DataSize.per(duration: Duration): Bandwidth = Bandwidth((this.bits * 1_000_000) / duration.toRoundedMicros()) + +/** + * Create a [Duration] from a [DataSize] per [Bandwidth] + */ +operator fun DataSize.div(bandwidth: Bandwidth): Duration = + Duration.ofNanos(round((this.bits.toDouble() / bandwidth.bps * 1e9)).toLong()) + +/** + * create a [DataSize] from a [Bandwidth] times [Duration] + */ +operator fun Bandwidth.times(duration: Duration): DataSize = round(bps * duration.toDouble()).toLong().bits + +/** + * Returns the sum of all elements in the collection. + */ +fun Iterable.sum(): Bandwidth = reduce(Bandwidth::plus) + +/** + * Returns the maximum of two [Bandwidth]s + */ +fun max(a: Bandwidth, b: Bandwidth): Bandwidth = if (a >= b) a else b + +/** + * Returns the minimum of two [Bandwidth]s + */ +fun min(a: Bandwidth, b: Bandwidth): Bandwidth = if (a <= b) a else b + +/** + * Ensures that this value is not greater than the specified [maximumValue]. + * + * @return this value if it's less than or equal to the [maximumValue] or the [maximumValue] otherwise. + */ +public fun Bandwidth.coerceAtMost(maximumValue: Bandwidth): Bandwidth = if (this > maximumValue) maximumValue else this diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt new file mode 100644 index 0000000..5f91556 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BitrateTracker.kt @@ -0,0 +1,49 @@ +/* + * Copyright @ 2020 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import org.jitsi.utils.ms +import org.jitsi.utils.stats.RateTracker +import java.time.Clock +import java.time.Duration + +open class BitrateTracker @JvmOverloads constructor( + private val windowSize: Duration, + private val bucketSize: Duration = 1.ms, + private val clock: Clock = Clock.systemUTC() +) { + // Use composition to expose functions with the data types we want ([DataSize], [Bandwidth]) and not the raw types + // that RateTracker uses. + private val tracker = RateTracker(windowSize, bucketSize, clock) + open fun getRate(nowMs: Long = clock.millis()): Bandwidth = tracker.getRate(nowMs).bps + + @JvmOverloads + open fun getRateBps(nowMs: Long = clock.millis()): Long = tracker.getRate(nowMs) + + /** + * The rate over the full window, that is without the ramp-up which [getRate] applies while the window has not yet + * filled up. When a stream has had no packets for a whole window the tracker restarts, and [getRate] then divides + * by the time since it resumed rather than by the window; for a bursty stream that turns the first frame after an + * idle period into a rate many times the one the stream actually sustains. This under-states the rate of a stream + * which has genuinely just started, which is the safer direction when deciding whether it can be forwarded. + */ + fun getRateOverFullWindow(nowMs: Long = clock.millis()): Bandwidth = getAccumulatedSize(nowMs).per(windowSize) + + val rate: Bandwidth + get() = getRate() + fun update(dataSize: DataSize, now: Long = clock.millis()) = tracker.update(dataSize.bits, now) + fun getAccumulatedSize(now: Long = clock.millis()) = tracker.getAccumulatedCount(now).bits +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BufferPool.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BufferPool.kt new file mode 100644 index 0000000..75b8411 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/BufferPool.kt @@ -0,0 +1,29 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +/** + * All ByteBuffers JMT needs in the pipeline will be acquired via [getBuffer]. + * When we're done with a buffer, we'll pass it to [returnBuffer]. These methods + * can be overridden so that a user of this library can use a pool for the buffers. + */ +class BufferPool { + companion object { + var getBuffer: (Int) -> ByteArray = { size -> ByteArray(size) } + var returnBuffer: (ByteArray) -> Unit = { } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt new file mode 100644 index 0000000..af6bfee --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/DataSize.kt @@ -0,0 +1,119 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import java.text.DecimalFormat +import kotlin.math.roundToLong + +/** + * Model an amount of data, internally represented as a number of bits. + */ +@SuppressFBWarnings( + value = ["SING_SINGLETON_HAS_NONPRIVATE_CONSTRUCTOR"], + justification = "Spotbugs thinks this is a singleton, but it isn't" +) +class DataSize( + bits: Long +) : Comparable { + + var bits: Long = bits + private set + + val bytes: Double = bits / 8.0 + val kiloBytes: Double = bytes / 1000.0 + val megaBytes: Double = kiloBytes / 1000.0 + + operator fun minus(other: DataSize): DataSize = DataSize(bits - other.bits) + + operator fun plus(other: DataSize): DataSize = DataSize(bits + other.bits) + + operator fun times(other: Int): DataSize = DataSize(bits * other) + + operator fun times(other: Double): DataSize = DataSize((bits * other).roundToLong()) + + operator fun div(other: Double): DataSize = DataSize((bits / other).roundToLong()) + + operator fun div(other: DataSize): Double = bits.toDouble() / other.bits.toDouble() + + override fun toString(): String { + // To determine which unit we'll print in, + // find the biggest one which has a value + // in the ones place + val format = DecimalFormat("0.##") + return when { + megaBytes >= 1 -> "${format.format(megaBytes)} MB" + kiloBytes >= 1 -> "${format.format(kiloBytes)} KB" + bytes >= 1 -> "${format.format(bytes)} B" + else -> "${format.format(bits)} bits" + } + } + + override fun compareTo(other: DataSize): Int = when { + bits < other.bits -> -1 + bits > other.bits -> 1 + else -> 0 + } + + override fun equals(other: Any?): Boolean { + if (other !is DataSize) { + return false + } + return compareTo(other) == 0 + } + + override fun hashCode(): Int = bits.hashCode() + + fun toWholeBytes(): DataSize = (bits / 8.0).roundToLong().bytes + + companion object { + val ZERO = DataSize(0) + val INFINITY = DataSize(Long.MAX_VALUE) + } +} + +operator fun Int.times(other: DataSize): DataSize = other * this + +operator fun Double.times(other: DataSize): DataSize = other * this + +val Int.bits: DataSize + get() = DataSize(this.toLong()) +val Int.bytes: DataSize + get() = DataSize(this.toLong() * 8) +val Int.kilobytes: DataSize + get() = DataSize(this.toLong() * 1000 * 8) +val Int.megabytes: DataSize + get() = DataSize(this.toLong() * 1000 * 1000 * 8) + +val Long.bits: DataSize + get() = DataSize(this) +val Long.bytes: DataSize + get() = DataSize(this * 8) +val Long.kilobytes: DataSize + get() = DataSize(this * 1000 * 8) +val Long.megabytes: DataSize + get() = DataSize(this * 1000 * 1000 * 8) + +/** + * Returns the maximum of two [DataSize]s + */ +fun max(a: DataSize, b: DataSize): DataSize = if (a >= b) a else b + +/** + * Returns the minimum of two [DataSize]s + */ +fun min(a: DataSize, b: DataSize): DataSize = if (a <= b) a else b diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/EvictingConcurrentQueue.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/EvictingConcurrentQueue.kt new file mode 100644 index 0000000..d3e77a2 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/EvictingConcurrentQueue.kt @@ -0,0 +1,29 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import java.util.concurrent.ConcurrentLinkedQueue + +class EvictingConcurrentQueue(private val maxSize: Int) : ConcurrentLinkedQueue() { + override fun add(element: T): Boolean { + val result = super.add(element) + if (size > maxSize) { + // We check on every add, so we can't be more than 1 over the max size + poll() + } + return result + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt new file mode 100644 index 0000000..83ec21b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ExecutorUtils.kt @@ -0,0 +1,40 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import java.time.Duration +import java.util.concurrent.ExecutorService +import java.util.concurrent.TimeUnit + +class ExecutorShutdownTimeoutException : Exception("Timed out trying to shutdown executor service") + +/** + * Shutdown [executorService] normally via [ExecutorService.shutdown]. If, after [timeout] / 2, the + * service has still not shutdown, try to stop it more forcefully via [ExecutorService.shutdownNow]. If, + * after [timeout] / 2, the service has still not shutdown then throw [ExecutorShutdownTimeoutException]. + * + */ +@Throws(ExecutorShutdownTimeoutException::class) +fun ExecutorService.safeShutdown(timeout: Duration) { + shutdown() + if (!awaitTermination(timeout.toMillis() / 2, TimeUnit.MILLISECONDS)) { + shutdownNow() + if (!awaitTermination(timeout.toMillis() / 2, TimeUnit.MILLISECONDS)) { + throw ExecutorShutdownTimeoutException() + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/IndexTracker.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/IndexTracker.kt new file mode 100644 index 0000000..61419c7 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/IndexTracker.kt @@ -0,0 +1,155 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +/* + * Index tracker inspired by the RFC3711 RTP sequence number index tracker. + */ + +/** A tracker suitable for the VP8/VP9 15-bit PictureID field. */ +class PictureIdIndexTracker : IntIndexTracker(15) + +/** A tracker suitable for RTP 16-bit sequence numbers. */ +class RtpSequenceIndexTracker : IntIndexTracker(16) + +/** A tracker suitable for RTP 32-bit timestamps. */ +class RtpTimestampIndexTracker : LongIndexTracker(32) + +open class IntIndexTracker(val bits: Int) : IndexTracker() { + init { + require(bits in 1..31) { "bits must be between 1 and 31" } + } + override fun addRollover(seqNum: Int, roc: Long) = (1 shl bits) * roc + seqNum + override fun rollsOver(a: Int, b: Int): Boolean = isOlderThan(a, b) && b < a + override fun isOlderThan(a: Int, b: Int): Boolean = delta(a, b) < 0 + override fun toLong(t: Int): Long = t.toLong() + override fun isValid(t: Int): Boolean = t >= 0 && t < (1 shl bits) + + private fun delta(a: Int, b: Int): Int { + val diff = a - b + return when { + diff < -(1 shl (bits - 1)) -> diff + (1 shl bits) + diff > (1 shl (bits - 1)) -> diff - (1 shl bits) + else -> diff + } + } +} + +open class LongIndexTracker(val bits: Int) : IndexTracker() { + init { + require(bits in 1..63) { "bits must be between 1 and 63" } + } + override fun addRollover(seqNum: Long, roc: Long) = (1L shl bits) * roc + seqNum + override fun rollsOver(a: Long, b: Long): Boolean = isOlderThan(a, b) && b < a + override fun isOlderThan(a: Long, b: Long): Boolean = delta(a, b) < 0 + override fun toLong(t: Long): Long = t + override fun isValid(t: Long): Boolean = t >= 0L && t < (1L shl bits) + + private fun delta(a: Long, b: Long): Long { + val diff = a - b + return when { + diff < -(1L shl (bits - 1)) -> diff + (1L shl bits) + diff > (1L shl (bits - 1)) -> diff - (1L shl bits) + else -> diff + } + } +} + +sealed class IndexTracker { + private var roc: Long = 0L + + private var highestSeqNumReceived: T? = null + + internal abstract fun addRollover(seqNum: T, roc: Long): Long + internal abstract fun rollsOver(a: T, b: T): Boolean + internal abstract fun isOlderThan(a: T, b: T): Boolean + internal abstract fun toLong(t: T): Long + internal abstract fun isValid(t: T): Boolean + private fun isNewerThan(a: T, b: T): Boolean = !isOlderThan(a, b) && a != b + private fun validate(t: T): T = if (!isValid(t)) { + throw IllegalArgumentException("Invalid sequence number: $t") + } else { + t + } + + /** + * return the index (as defined by RFC3711 at https://tools.ietf.org/html/rfc3711#section-3.3.1) + * for the given seqNum, updating our ROC if we roll over. + * NOTE that this method must be called for all 'received' sequence numbers so that it may keep + * its rollover counter accurate + */ + fun update(seqNum: T): Long = getIndex(validate(seqNum), true) + + /** + * Interprets an RTP sequence number in the context of the highest sequence number received. Returns the index + * which corresponds to the packet, but does not update the ROC. + */ + fun interpret(seqNum: T): Long = getIndex(validate(seqNum), false) + + /** + * return the index (as defined by RFC3711 at https://tools.ietf.org/html/rfc3711#section-3.3.1) + * for the given [seqNum]. If [updateRoc] is [true] and we've rolled over, updates our ROC. + */ + private fun getIndex(seqNum: T, updateRoc: Boolean): Long { + val highestSeqNumReceived = this.highestSeqNumReceived + if (highestSeqNumReceived == null) { + if (updateRoc) { + this.highestSeqNumReceived = seqNum + } + return toLong(seqNum) + } + + val v = when { + rollsOver(seqNum, highestSeqNumReceived) -> { + // Seq num was from the previous roc value + roc - 1 + } + + rollsOver(highestSeqNumReceived, seqNum) -> { + // We've rolled over, so update the roc in place if updateRoc + // is set, otherwise return the right value (our current roc + // + 1) + if (updateRoc) { + ++roc + } else { + roc + 1 + } + } + + else -> roc + } + + if (updateRoc && isNewerThan(seqNum, highestSeqNumReceived)) { + this.highestSeqNumReceived = seqNum + } + + return addRollover(seqNum, v) + } + + /** Force this sequence number to be interpreted as the new highest, regardless of its rollover state. */ + fun resetAt(seq: T) { + validate(seq) + val highestSeqNumReceived = this.highestSeqNumReceived + if (highestSeqNumReceived == null || isOlderThan(seq, highestSeqNumReceived)) { + roc++ + this.highestSeqNumReceived = seq + } + getIndex(seq, true) + } + + fun debugState(): String = "{roc=$roc, highestSeqNumReceived=$highestSeqNumReceived}" +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt new file mode 100644 index 0000000..8d1701e --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/InstantUtils.kt @@ -0,0 +1,21 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import java.time.Instant + +fun latest(vararg instants: Instant): Instant = instants.reduce { a, b -> maxOf(a, b) } diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt new file mode 100644 index 0000000..17e569d --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/NodeStatsBlockExtensions.kt @@ -0,0 +1,56 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.util.Util.Companion.getMbps +import java.time.Duration + +/** + * Adds a compound value which computes the bitrate in megabits per second given two values + * that represent number of bytes and duration in milliseconds. + * + * @param name the name of the new value to add. + * @param bytesKey the name of the stat which has the number of bytes. + * @param durationMsKey the name of the stat which has the duration in milliseconds. + */ +fun NodeStatsBlock.addMbps(name: String, bytesKey: String, durationMsKey: String) = addCompoundValue(name) { + getMbps( + it.getNumberOrDefault(bytesKey, 0), + Duration.ofMillis(it.getNumberOrDefault(durationMsKey, 1).toLong()) + ) +} + +/** + * Adds a compound value which computes the ratio of two values. + * + * @param name the name of the new value to add. + * @param numeratorKey the name of the stat which holds the numerator of the ratio. + * @param denominatorKey the name of the stat which holds the denominator of the ratio. + * @param defaultDenominator default value for the denominator in case the value with the given key + * doesn't exist or is 0. + */ +fun NodeStatsBlock.addRatio( + name: String, + numeratorKey: String, + denominatorKey: String, + defaultDenominator: Number = 1 +) = addCompoundValue(name) { + val numerator = it.getNumber(numeratorKey) ?: 0 + val denominator = it.getNumber(denominatorKey) ?: defaultDenominator + + numerator.toDouble() / (if (denominator.toDouble() == 0.0) defaultDenominator else denominator).toDouble() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt new file mode 100644 index 0000000..0a89e95 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketCache.kt @@ -0,0 +1,205 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.NodeStatsProducer +import org.jitsi.rtp.rtp.RtpPacket +import java.util.Collections + +/** + * Creates a packet cache for packets by SSRC + */ +class PacketCache( + /** + * A function which dictates which packets to cache. + */ + val packetPredicate: (RtpPacket) -> Boolean = { it is VideoRtpPacket }, + /** + * The max number of packets to cache per SSRC. + */ + val size: Int = 500 +) : NodeStatsProducer { + private val packetCaches = Collections.synchronizedMap( + LinkedHashMap( + // These are the default values of initialCapacity and loadFactor - we have to set them to be able to set + // accessOrder + // initialCapacity + 16, + // loadFactor + 0.75F, + // accessOrder + true + ) + ) + + private var stopped = false + + private fun getCache(ssrc: Long): RtpPacketCache = packetCaches.computeIfAbsent(ssrc) { + RtpPacketCache(size) + }.also { it.setLastAccess() } + .also { expireCaches(it.lastAccessMillis) } + + private fun expireCaches(now: Long) { + synchronized(packetCaches) { + val i = packetCaches.iterator() + while (i.hasNext()) { + val cache = i.next().value + if (now - cache.lastAccessMillis >= MAX_CACHE_LIFETIME_MILLIS) { + cache.flush() + i.remove() + } else { + break + } + } + } + } + + /** + * Stores a copy of the given packet in the cache. + */ + fun insert(packet: RtpPacket) = !stopped && packetPredicate(packet) && getCache(packet.ssrc).insert(packet) + + /** + * Gets a copy of the packet in the cache with the given SSRC and sequence number, if the cache contains it. + * The instance is wrapped in a [ArrayCache.Container]. + */ + fun get(ssrc: Long, seqNum: Int): ArrayCache.Container? = getCache(ssrc).get(seqNum) + + /** + * Gets copies of the latest packets in the cache. Returns packets which add up to no more than + * [numBytes] bytes. + */ + fun getMany(ssrc: Long, numBytes: Int): Set = getCache(ssrc).getMany(numBytes) + + /** + * Updates the timestamp of a packet in the cache (if it is in the cache). This is used when we re-transmit a + * packet in order to update the timestamp without re-adding the packet to the cache (which is expensive). + */ + fun updateTimestamp(ssrc: Long, seqNum: Int, timeAdded: Long) = getCache(ssrc).updateTimestamp(seqNum, timeAdded) + + fun stop() { + stopped = true + synchronized(packetCaches) { + packetCaches.forEach { (_, cache) -> cache.flush() } + } + } + + override fun getNodeStats(): NodeStatsBlock = NodeStatsBlock("PacketCache").apply { + synchronized(packetCaches) { + packetCaches.values.forEach { + aggregate(it.getNodeStats()) + } + } + } + + companion object { + val MAX_CACHE_LIFETIME_MILLIS = 15000 + } +} + +/** + * Implements a cache for RTP packets. + */ +class RtpPacketCache( + size: Int, + synchronize: Boolean = true +) : ArrayCache( + size = size, + cloneItem = RtpPacket::clone, + synchronize = synchronize +) { + var lastAccessMillis: Long = 0 + private set + + private val rtpSequenceIndexTracker = RtpSequenceIndexTracker() + + override fun discardItem(item: RtpPacket) { + BufferPool.returnBuffer(item.buffer) + } + + fun setLastAccess() { + if (synchronize) { + synchronized(syncRoot) { + doSetLastAccess() + } + } else { + doSetLastAccess() + } + } + + private fun doSetLastAccess() { + val t = clock.millis() + if (lastAccessMillis < t) { + lastAccessMillis = t + } + } + + /** + * Gets a packet with a given RTP sequence number from the cache (clones the packet). + */ + fun get(sequenceNumber: Int): Container? = doGet(sequenceNumber, true) + + fun doGet(sequenceNumber: Int, shouldCloneItem: Boolean): Container? { + // Note that we use [interpret] because we don't want the ROC to get out of sync because of funny requests + // (NACKs) + val index = rtpSequenceIndexTracker.interpret(sequenceNumber) + // The RFC3711 tracker may produce negative numbers (example, if it is initialized with 0, + // then 65535 is interpreted). These are invalid indexes. + return if (index < 0) null else super.getContainer(index, shouldCloneItem) + } + + /** + * Gets a packet with a given RTP sequence number from the cache (does not clone the packet). + */ + fun peek(sequenceNumber: Int): Container? = doGet(sequenceNumber, false) + + fun contains(sequenceNumber: Int): Boolean { + // Note that we use [interpret] because we don't want the ROC to get out of sync because of funny requests + // (NACKs) + val index = rtpSequenceIndexTracker.interpret(sequenceNumber) + return super.containsIndex(index) + } + + fun insert(rtpPacket: RtpPacket): Boolean { + val index = rtpSequenceIndexTracker.update(rtpPacket.sequenceNumber) + return super.insertItem(rtpPacket, index) + } + + fun updateTimestamp(seqNum: Int, timeAdded: Long) { + val index = rtpSequenceIndexTracker.interpret(seqNum) + super.updateTimeAdded(index, timeAdded) + } + + fun getMany(numBytes: Int): Set { + var bytesRemaining = numBytes + val packets = mutableSetOf() + + forEachDescending { + if (it.length <= bytesRemaining) { + packets.add(it.clone()) + bytesRemaining -= it.length + true + } else { + false + } + } + + return packets + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketInfoQueue.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketInfoQueue.kt new file mode 100644 index 0000000..6ee05f4 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/PacketInfoQueue.kt @@ -0,0 +1,35 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.nlj.PacketInfo +import org.jitsi.utils.queue.PacketQueue +import java.util.concurrent.ExecutorService + +/** + * A [PacketQueue] of [PacketInfo]s, returning buffers when released. + */ +class PacketInfoQueue( + id: String, + executor: ExecutorService, + handler: (PacketInfo) -> Boolean, + capacity: Int = 1024 +) : PacketQueue(capacity, null, id, handler, executor) { + override fun releasePacket(pkt: PacketInfo) { + BufferPool.returnBuffer(pkt.packet.buffer) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt new file mode 100644 index 0000000..ebd3819 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RateUtils.kt @@ -0,0 +1,40 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import java.time.Duration + +/** + * This method isn't technically necessary, but provides better readability, i.e.: + * + * val time = howLongToSend(1.megabytes()) atRate 1.mbps() + * + * as opposed to + * + * val time = 1.megabytes() atRate 1.mbps() + */ +fun howLongToSend(size: DataSize): DataSize = size + +infix fun DataSize.atRate(bw: Bandwidth): Duration { + val bitsPerNano = bw.bps / 1e9 + return Duration.ofNanos((this.bits / bitsPerNano).toLong()) +} + +fun howMuchCanISendAtRate(bw: Bandwidth): Bandwidth = bw + +@Suppress("ktlint:standard:function-naming") +infix fun Bandwidth.`in`(time: Duration): DataSize = DataSize((bps * (time.seconds + time.nano / 1e9)).toLong()) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ReceiveSsrcStore.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ReceiveSsrcStore.kt new file mode 100644 index 0000000..0525d89 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ReceiveSsrcStore.kt @@ -0,0 +1,97 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.utils.MediaType +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArraySet + +class ReceiveSsrcStore( + private val ssrcAssociationStore: SsrcAssociationStore +) { + // NOTE: to enable efficient lookup for various use cases, we store + // different 'views' of the receive SSRCs in multiple data structures + // (below). Updates to these data structures do not happen atomically, + // so it's possible the view across them may be inconsistent while changes + // are propagating. We don't currently have code that cares about this, + // so it's done this way to avoid having to synchronize updates and access + // across all of them. + /** + * All signaled receive SSRCs + */ + val receiveSsrcs: MutableSet = CopyOnWriteArraySet() + + /** + * All receive SSRCs indexed by their media type + */ + private val receiveSsrcsByMediaType: MutableMap> = + ConcurrentHashMap() + + /** + * 'Primary' receive media SSRCs (excludes things like RTX) + */ + val primaryMediaSsrcs: MutableSet = CopyOnWriteArraySet() + + /** + * 'Primary' *video* SSRCs + */ + val primaryVideoSsrcs: MutableSet = CopyOnWriteArraySet() + + init { + ssrcAssociationStore.onAssociation(this::onSsrcAssociation) + } + + fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) { + receiveSsrcsByMediaType.getOrPut(mediaType, { mutableSetOf() }).add(ssrc) + receiveSsrcs.add(ssrc) + if (ssrcAssociationStore.isPrimarySsrc(ssrc)) { + primaryMediaSsrcs.add(ssrc) + if (mediaType == MediaType.VIDEO) { + primaryVideoSsrcs.add(ssrc) + } + } + } + + fun removeReceiveSsrc(ssrc: Long) { + receiveSsrcs.remove(ssrc) + receiveSsrcsByMediaType.values.forEach { it.remove(ssrc) } + primaryMediaSsrcs.remove(ssrc) + primaryVideoSsrcs.remove(ssrc) + } + + /** + * Handle new [SsrcAssociation]s + */ + private fun onSsrcAssociation(ssrcAssociation: SsrcAssociation) { + // Secondary SSRCs in associations shouldn't be considered + // primary media or video SSRCs + primaryVideoSsrcs.remove(ssrcAssociation.secondarySsrc) + primaryMediaSsrcs.remove(ssrcAssociation.secondarySsrc) + } + + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + val mapper = ObjectMapper() + set( + "receive_ssrcs", + mapper.valueToTree(receiveSsrcsByMediaType.mapValues { it.value.toString() }.toMap()) + ) + set("primary_media_ssrcs", mapper.valueToTree(primaryMediaSsrcs.toSet())) + set("primary_video_ssrcs", mapper.valueToTree(primaryVideoSsrcs.toSet())) + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RtpPacketExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RtpPacketExtensions.kt new file mode 100644 index 0000000..5d2ab11 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/RtpPacketExtensions.kt @@ -0,0 +1,35 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.rtp.rtp.RtpPacket + +/** + * Shifts the payload byte 'numBytes' to the right, increasing the length of the packet by 'numBytes' + * and growing (i.e. re-allocating) the underlying buffer if necessary. + */ +fun RtpPacket.shiftPayloadRight(numBytes: Int) { + // Make sure we have enough space. + // TODO: if grow() needs to allocate a new buffer, we end up copying the payload twice, which can be avoided. + // We expect that most of the time we have enough space and grow() doesn't need to do anything, but this should + // be verified and optimized if it doesn't hold. + grow(numBytes) + + val payloadOffset = payloadOffset + System.arraycopy(buffer, payloadOffset, buffer, payloadOffset + 2, payloadLength) + length += 2 +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ScheduleExecutorServiceExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ScheduleExecutorServiceExtensions.kt new file mode 100644 index 0000000..9eb2810 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ScheduleExecutorServiceExtensions.kt @@ -0,0 +1,25 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import java.time.Duration +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import kotlin.reflect.KFunction0 + +fun ScheduledExecutorService.schedule(command: KFunction0, delay: Duration): ScheduledFuture<*> = + schedule(command, delay.toMillis(), TimeUnit.MILLISECONDS) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociation.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociation.kt new file mode 100644 index 0000000..aba51f6 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociation.kt @@ -0,0 +1,49 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.nlj.rtp.SsrcAssociationType + +abstract class SsrcAssociation( + val primarySsrc: Long, + val secondarySsrc: Long, + val type: SsrcAssociationType +) { + override fun toString(): String = "$secondarySsrc -> $primarySsrc ($type)" +} + +/** + * A [LocalSsrcAssociation] is an SSRC association which belongs to + * the endpoint itself; i.e. the processor of a [LocalSsrcAssociation] + * is also the 'owner' of the SSRCs signaled in the association. + */ +class LocalSsrcAssociation( + primarySsrc: Long, + secondarySsrc: Long, + type: SsrcAssociationType +) : SsrcAssociation(primarySsrc, secondarySsrc, type) + +/** + * A [RemoteSsrcAssociation] is an SSRC association which belongs to + * another endpoint; i.e. the processor of a [RemoteSsrcAssociation] + * does not 'own' the SSRCs signaled in the association. + */ +class RemoteSsrcAssociation( + primarySsrc: Long, + secondarySsrc: Long, + type: SsrcAssociationType +) : SsrcAssociation(primarySsrc, secondarySsrc, type) diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt new file mode 100644 index 0000000..c8da347 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/SsrcAssociationStore.kt @@ -0,0 +1,81 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.nlj.rtp.SsrcAssociationType +import java.util.concurrent.CopyOnWriteArrayList + +typealias SsrcAssociationHandler = (SsrcAssociation) -> Unit + +class SsrcAssociationStore { + private val ssrcAssociations: MutableList = CopyOnWriteArrayList() + + /** + * The SSRC associations indexed by the primary SSRC. Since an SSRC may have + * multiple secondary SSRC mappings, the primary SSRC maps to a list of its + * SSRC associations + */ + private var ssrcAssociationsByPrimarySsrc = mapOf>() + private var ssrcAssociationsBySecondarySsrc = mapOf() + + private val handlers: MutableList = CopyOnWriteArrayList() + + /** + * Each time an association is added, we want to invoke the handlers + * and each time a handler is added, want to invoke it with all existing + * associations. In order to make each of those operations a single, + * atomic operation, we use this lock to synchronize them. + */ + private val lock = Any() + + fun addAssociation(ssrcAssociation: SsrcAssociation) { + synchronized(lock) { + ssrcAssociations.add(ssrcAssociation) + rebuildMaps() + handlers.forEach { it(ssrcAssociation) } + } + } + + private fun rebuildMaps() { + ssrcAssociationsByPrimarySsrc = ssrcAssociations.groupBy(SsrcAssociation::primarySsrc) + ssrcAssociationsBySecondarySsrc = ssrcAssociations.associateBy(SsrcAssociation::secondarySsrc) + } + + fun getPrimarySsrc(secondarySsrc: Long): Long? = ssrcAssociationsBySecondarySsrc[secondarySsrc]?.primarySsrc + + fun getSecondarySsrc(primarySsrc: Long, associationType: SsrcAssociationType): Long? = + ssrcAssociationsByPrimarySsrc[primarySsrc]?.find { it.type == associationType }?.secondarySsrc + + /** + * When an SSRC has no associations at all (audio, for example), we consider it a + * 'primary' SSRC. So to perform this check we assume the given SSRC has been + * signalled and simply verify that it's *not* signaled as a secondary SSRC. + * Note that this may mean there is a slight window before the SSRC associations are + * processed during which we return true for an SSRC which will later be denoted + * as a secondary ssrc. + */ + fun isPrimarySsrc(ssrc: Long): Boolean = !ssrcAssociationsBySecondarySsrc.containsKey(ssrc) + + fun onAssociation(handler: (SsrcAssociation) -> Unit) { + synchronized(lock) { + handlers.add(handler) + ssrcAssociations.forEach(handler) + } + } + + override fun toString(): String = ssrcAssociations.toString() +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StateChangeLogger.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StateChangeLogger.kt new file mode 100644 index 0000000..e87eb03 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StateChangeLogger.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cinfo + +/** + * Class that logs when state first becomes false, or changes from false to true, + * without logging on every packet. + */ +class StateChangeLogger( + val desc: String, + val logger: Logger +) { + var state: Boolean? = null + + fun setState(newState: Boolean, instance: Any, instanceDesc: () -> String) { + if (state != newState) { + if (newState == false) { + logger.cinfo { "Packet $instance has $desc. ${instanceDesc()}" } + } else if (state != null) { + logger.cinfo { "Packet $instance source no longer has $desc. ${instanceDesc()}" } + } + state = newState + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt new file mode 100644 index 0000000..c4b7b29 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StreamInformation.kt @@ -0,0 +1,243 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.format.supportsPli +import org.jitsi.nlj.format.supportsRemb +import org.jitsi.nlj.format.supportsTcc +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.utils.MediaType +import java.util.Collections +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList + +/** + * A handler installed on a specific [RtpExtensionType] to be notified + * when that type is mapped to an id. A null id value indicates the + * extension mapping has been removed + */ +typealias RtpExtensionHandler = (Int?) -> Unit + +typealias RtpPayloadTypesChangedHandler = (Map) -> Unit + +typealias ExtmapAllowMixedChangedHandler = (Boolean) -> Unit + +/** + * Makes information about stream metadata (RTP extensions, payload types, + * etc.) available and allows interested parties to add handlers for when certain + * information is available. + */ +interface ReadOnlyStreamInformationStore { + val rtpExtensions: List + fun onRtpExtensionMapping(rtpExtensionType: RtpExtensionType, handler: RtpExtensionHandler) + + val rtpPayloadTypes: Map + fun onRtpPayloadTypesChanged(handler: RtpPayloadTypesChangedHandler) + + val extmapAllowMixed: Boolean + fun onExtmapAllowMixedChanged(handler: ExtmapAllowMixedChangedHandler) + + fun getLocalPrimarySsrc(secondarySsrc: Long): Long? + fun getRemoteSecondarySsrc(primarySsrc: Long, associationType: SsrcAssociationType): Long? + fun debugState(mode: DebugStateMode): ObjectNode + + /** + * All signaled receive SSRCs + */ + val receiveSsrcs: Set + + /** + * A list of all primary media (audio and video) SSRCs for which the + * endpoint associated with this stream information store sends video + * (does not include things like RTX) + */ + val primaryMediaSsrcs: Set + + val supportsPli: Boolean + val supportsFir: Boolean + val supportsRemb: Boolean + val supportsTcc: Boolean +} + +/** + * A writable stream information store + */ +interface StreamInformationStore : ReadOnlyStreamInformationStore { + fun addRtpExtensionMapping(rtpExtension: RtpExtension) + fun clearRtpExtensions() + + fun addRtpPayloadType(payloadType: PayloadType) + fun clearRtpPayloadTypes() + + fun setExtmapAllowMixed(allow: Boolean) + + fun addSsrcAssociation(ssrcAssociation: SsrcAssociation) + + fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) + fun removeReceiveSsrc(ssrc: Long) +} + +class StreamInformationStoreImpl : StreamInformationStore { + private val extensionsLock = Any() + private val extensionHandlers = + mutableMapOf>() + private val _rtpExtensions: MutableList = CopyOnWriteArrayList() + override val rtpExtensions: List + get() = _rtpExtensions + private val extmapAllowMixedHandlers = + mutableListOf() + + private val payloadTypesLock = Any() + private val payloadTypeHandlers = mutableListOf() + private val _rtpPayloadTypes: MutableMap = ConcurrentHashMap() + override val rtpPayloadTypes: Map = Collections.unmodifiableMap(_rtpPayloadTypes) + + private val localSsrcAssociations = SsrcAssociationStore() + private val remoteSsrcAssociations = SsrcAssociationStore() + + private val receiveSsrcStore = ReceiveSsrcStore(localSsrcAssociations) + override val receiveSsrcs: Set + get() = receiveSsrcStore.receiveSsrcs + override val primaryMediaSsrcs: Set + get() = receiveSsrcStore.primaryMediaSsrcs + + // Support for FIR, PLI, REMB and TCC is declared per-payload type, but currently our code is not payload-type + // aware. So until this changes we will just check if any of the PTs supports the relevant feedback. + // We always assume support for FIR. + override var supportsFir: Boolean = true + private set + + override var supportsPli: Boolean = false + private set + + override var supportsRemb: Boolean = false + private set + + override var supportsTcc: Boolean = false + private set + + override fun addRtpExtensionMapping(rtpExtension: RtpExtension) { + synchronized(extensionsLock) { + _rtpExtensions.add(rtpExtension) + extensionHandlers.get(rtpExtension.type)?.forEach { it(rtpExtension.id.toInt()) } + } + } + + override fun clearRtpExtensions() { + synchronized(extensionsLock) { + _rtpExtensions.clear() + extensionHandlers.values.forEach { handlers -> handlers.forEach { it(null) } } + } + } + + override fun onRtpExtensionMapping(rtpExtensionType: RtpExtensionType, handler: RtpExtensionHandler) { + synchronized(extensionsLock) { + extensionHandlers.getOrPut(rtpExtensionType, { mutableListOf() }).add(handler) + _rtpExtensions.find { it.type == rtpExtensionType }?.let { handler(it.id.toInt()) } + } + } + + override var extmapAllowMixed: Boolean = false + private set + + override fun setExtmapAllowMixed(allow: Boolean) { + synchronized(extensionsLock) { + val changed = (extmapAllowMixed != allow) + extmapAllowMixed = allow + if (changed) { + extmapAllowMixedHandlers.forEach { it(allow) } + } + } + } + + override fun onExtmapAllowMixedChanged(handler: ExtmapAllowMixedChangedHandler) { + synchronized(extensionsLock) { + extmapAllowMixedHandlers.add(handler) + handler(extmapAllowMixed) + } + } + + override fun addRtpPayloadType(payloadType: PayloadType) { + synchronized(payloadTypesLock) { + _rtpPayloadTypes[payloadType.pt] = payloadType + supportsPli = rtpPayloadTypes.values.find { it.rtcpFeedbackSet.supportsPli() } != null + supportsRemb = rtpPayloadTypes.values.find { it.rtcpFeedbackSet.supportsRemb() } != null + supportsTcc = rtpPayloadTypes.values.find { it.rtcpFeedbackSet.supportsTcc() } != null + payloadTypeHandlers.forEach { it(_rtpPayloadTypes) } + } + } + + override fun clearRtpPayloadTypes() { + synchronized(payloadTypesLock) { + _rtpPayloadTypes.clear() + supportsPli = false + supportsRemb = false + supportsTcc = false + payloadTypeHandlers.forEach { it(_rtpPayloadTypes) } + } + } + + override fun onRtpPayloadTypesChanged(handler: RtpPayloadTypesChangedHandler) { + synchronized(payloadTypesLock) { + payloadTypeHandlers.add(handler) + handler(_rtpPayloadTypes) + } + } + + // NOTE(brian): Currently, we only have a use case to do a mapping of + // secondary -> primary for local SSRCs and primary -> secondary for + // remote SSRCs + override fun getLocalPrimarySsrc(secondarySsrc: Long): Long? = localSsrcAssociations.getPrimarySsrc(secondarySsrc) + + override fun getRemoteSecondarySsrc(primarySsrc: Long, associationType: SsrcAssociationType): Long? = + remoteSsrcAssociations.getSecondarySsrc(primarySsrc, associationType) + + override fun addSsrcAssociation(ssrcAssociation: SsrcAssociation) { + when (ssrcAssociation) { + is LocalSsrcAssociation -> localSsrcAssociations.addAssociation(ssrcAssociation) + is RemoteSsrcAssociation -> remoteSsrcAssociations.addAssociation(ssrcAssociation) + } + } + + override fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) = receiveSsrcStore.addReceiveSsrc(ssrc, mediaType) + + override fun removeReceiveSsrc(ssrc: Long) = receiveSsrcStore.removeReceiveSsrc(ssrc) + + override fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("supports_pli", supportsPli) + put("supports_fir", supportsFir) + val rtpExtNode = JsonNodeFactory.instance.objectNode().apply { + rtpExtensions.forEach { put(it.id.toString(), it.type.toString()) } + } + set("rtp_extensions", rtpExtNode) + val rtpPayloadNode = JsonNodeFactory.instance.objectNode().apply { + rtpPayloadTypes.forEach { put(it.key.toString(), it.value.toString()) } + } + set("rtp_payload_types", rtpPayloadNode) + if (mode == DebugStateMode.FULL) { + put("local_ssrc_associations", localSsrcAssociations.toString()) + put("remote_ssrc_associations", remoteSsrcAssociations.toString()) + set("receive_ssrc_store", receiveSsrcStore.debugState()) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StringBufferExtensions.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StringBufferExtensions.kt new file mode 100644 index 0000000..33cca16 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/StringBufferExtensions.kt @@ -0,0 +1,21 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +fun StringBuffer.appendLnIndent(numSpaces: Int, msg: String) { + append(" ".repeat(numSpaces)).appendLine(msg) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ThreadSafeVetoable.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ThreadSafeVetoable.kt new file mode 100644 index 0000000..6a46731 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/ThreadSafeVetoable.kt @@ -0,0 +1,40 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import kotlin.properties.ObservableProperty +import kotlin.properties.ReadWriteProperty +import kotlin.reflect.KProperty + +/** + * Same as [kotlin.properties.Delegates.vetoable], but thread safe + */ +public inline fun threadSafeVetoable( + initialValue: T, + crossinline onChange: (property: KProperty<*>, oldValue: T, newValue: T) -> Boolean +): ReadWriteProperty = object : ObservableProperty(initialValue) { + private val lock = Any() + + override fun beforeChange(property: KProperty<*>, oldValue: T, newValue: T): Boolean = + onChange(property, oldValue, newValue) + + override fun setValue(thisRef: Any?, property: KProperty<*>, value: T) { + synchronized(lock) { + super.setValue(thisRef, property, value) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TimeExpiringCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TimeExpiringCache.kt new file mode 100644 index 0000000..3f61f54 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TimeExpiringCache.kt @@ -0,0 +1,105 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import java.time.Clock +import java.time.Duration +import java.util.TreeMap + +/** + * A cache which holds an arbitrary [DataType] and stores it, as well + * as the time at which it was added to the cache. On updates, it will + * check to prune itself to remove anything older than the configured + * timeout. + * NOTE: it's possible the pruning does not remove ALL elements older + * than the configured timeout as we store the elements according to + * the value of their [IndexType], not their insertion timestamp. + * This is because retrieval is much handier when done via the value + * of [IndexType]. This means that it's possible older values will + * not be pruned when expected, but if the ordering of [IndexType] + * roughly follows the insertion order, it should not be too bad. + */ +class TimeExpiringCache( + /** + * The amount of time after which an element should be pruned from the cache + */ + private val dataTimeout: Duration, + /** + * The maximum amount of elements we'll allow in the cache + */ + private val maxNumElements: Int, + private val clock: Clock = Clock.systemUTC() +) { + private val cache: TreeMap> = TreeMap() + + fun insert(index: IndexType, data: DataType) { + val container = Container(data, clock.millis()) + synchronized(cache) { + cache[index] = container + clean(clock.millis() - dataTimeout.toMillis()) + } + } + + fun get(index: IndexType): DataType? { + synchronized(cache) { + val container = cache.getOrDefault(index, null) ?: return null + return container.data + } + } + + /** + * For each element in this cache, in descending order of the value of their + * [IndexType], run the given [predicate]. If [predicate] returns false, stop + * the iteration. + */ + fun forEachDescending(predicate: (DataType) -> Boolean) { + synchronized(cache) { + val iter = cache.descendingMap().iterator() + while (iter.hasNext()) { + val (_, container) = iter.next() + if (!predicate(container.data)) { + break + } + } + } + } + + private fun clean(expirationTimestamp: Long) { + synchronized(cache) { + val iter = cache.entries.iterator() + while (iter.hasNext()) { + val (_, container) = iter.next() + if (cache.size > maxNumElements) { + iter.remove() + } else if (container.timeAdded <= expirationTimestamp) { + iter.remove() + } else { + // We'll break out of the loop once we find an insertion time that + // is not older than the expiration cutoff + break + } + } + } + } + + /** + * Holds an instance of [Type] as well as when it was + * added to the data structure so that we can time out + * old [Container]s + */ + private data class Container(var data: Type, var timeAdded: Long) +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt new file mode 100644 index 0000000..180050b --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/TreeCache.kt @@ -0,0 +1,62 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import java.util.* + +/** + * Implements a cache based on integer values, optimized for sparse values, such that you can find the most + * recent cached value before a specific value. + * + * The intended use case is AV1 Dependency Descriptor history. + */ +open class TreeCache( + private val minSize: Int +) { + private val map = TreeMap() + + private var highestIndex = -1L + + fun insert(index: Long, value: T) { + map[index] = value + + updateState(index) + } + + fun getEntryBefore(index: Long): Map.Entry? { + updateState(index) + return map.floorEntry(index) + } + + fun get(index: Long): T? = map[index] + + private fun updateState(index: Long) { + if (highestIndex < index) { + highestIndex = index + } + + /* Keep at most one entry older than highestIndex - minSize. */ + val headMap = map.headMap(highestIndex - minSize) + if (headMap.size > 1) { + val last = headMap.keys.last() + headMap.keys.removeIf { it < last } + } + } + + val size + get() = map.size +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Util.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Util.kt new file mode 100644 index 0000000..04f11e5 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/util/Util.kt @@ -0,0 +1,80 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.rtp.Packet +import java.time.Duration +import java.util.Collections +import java.util.function.Predicate + +class Util { + companion object { + fun getMbps(numBytes: Number, duration: Duration): Double { + val mbps = (numBytes.toLong() * 8.0) / (duration.toMillis() * 1000) + return if (mbps.isNaN()) -1.0 else mbps + } + } +} + +/** + * This method should only be called when the caller is confident the + * contents of the iterable contain [Expected] types. Because of this, + * throwing an exception if that isn't the case is desired. + */ +@Suppress("UNCHECKED_CAST") +inline fun Iterable<*>.forEachAs(action: (Expected) -> Unit) { + for (element in this) action(element as Expected) +} +inline fun Iterable<*>.forEachIf(action: (Expected) -> Unit) { + for (element in this) { + if (element is Expected) action(element) + } +} + +infix fun Int.floorMod(other: Int): Int = Math.floorMod(this, other) + +infix fun Long.floorMod(other: Long): Long = Math.floorMod(this, other) + +/** Set the value at position [index] in a [MutableList] to [element]. If the list has fewer + * than [index] entries, extend the intermediate entries between its current size and + * [index] with [fillerElement]. + */ +fun MutableList.setAndExtend(index: Int, element: T, fillerElement: T) { + if (index >= size) { + addAll(Collections.nCopies(index - size, fillerElement)) + add(element) + } else { + this[index] = element + } +} + +/* We inline this so getStackTrace itself doesn't show up in the stack trace. */ +@Suppress("NOTHING_TO_INLINE") +inline fun getStackTrace(): String = with(StringBuffer()) { + for (ste in Thread.currentThread().stackTrace) { + appendLine(ste.toString()) + } + toString() +} + +fun ObjectNode.appendAll(other: ObjectNode): ObjectNode = this.also { + other.fields().forEach { (key, value) -> + set(key, value) + } +} + +typealias PacketPredicate = Predicate diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/config/OveruseDetectorConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/config/OveruseDetectorConfig.kt new file mode 100644 index 0000000..119cc52 --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/remotebitrateestimator/config/OveruseDetectorConfig.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.rtp.remotebitrateestimator.config + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class OveruseDetectorConfig { + companion object { + val initialThreshold: Double by config { + "jmt.bwe.overuse-detector.initial-threshold".from(JitsiConfig.newConfig) + } + + val overusingTimeThreshold: Double by config { + "jmt.bwe.overuse-detector.overusing-time-threshold".from(JitsiConfig.newConfig) + } + } +} diff --git a/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt b/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt new file mode 100644 index 0000000..5d3a0de --- /dev/null +++ b/jitsi-media-transform/src/main/kotlin/org/jitsi_modified/impl/neomedia/rtp/sendsidebandwidthestimation/config/SendSideBandwidthEstimationConfig.kt @@ -0,0 +1,124 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi_modified.impl.neomedia.rtp.sendsidebandwidthestimation.config + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.kbps + +class SendSideBandwidthEstimationConfig { + companion object { + private val defaultLowLossThreshold: Double by + config("jmt.bwe.send-side.low-loss-threshold".from(JitsiConfig.newConfig)) + + /** + * The low-loss threshold (expressed as a proportion of lost packets) when the loss probability + * experiment is *not* active + */ + @JvmStatic + fun defaultLowLossThreshold() = defaultLowLossThreshold + + private val defaultHighLossThreshold: Double by + config("jmt.bwe.send-side.high-loss-threshold".from(JitsiConfig.newConfig)) + + /** + * The high-loss threshold (expressed as a proportion of lost packets) when the loss probability + * experiment is *not* active. + */ + @JvmStatic + fun defaultHighLossThreshold() = defaultHighLossThreshold + + private val defaultBitrateThreshold: Bandwidth by config { + "jmt.bwe.send-side.bitrate-threshold".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + + /** + * The bitrate threshold when the loss probability experiment is *not* active. + */ + @JvmStatic + fun defaultBitrateThreshold() = defaultBitrateThreshold + + @JvmStatic + fun defaultBitrateThresholdBps() = defaultBitrateThreshold.bps + + private const val LEGACY_BASE_NAME = + "org.jitsi.impl.neomedia.rtp.sendsidebandwidthestimation.SendSideBandwidthEstimation" + + private val lossExperimentProbability: Double by config { + "$LEGACY_BASE_NAME.lossExperimentProbability".from(JitsiConfig.legacyConfig) + "jmt.bwe.send-side.loss-experiment.probability".from(JitsiConfig.newConfig) + } + + /** + * The probability of enabling the loss-based experiment. + */ + @JvmStatic + fun lossExperimentProbability() = lossExperimentProbability + + private val experimentalLowLossThreshold: Double by config { + "$LEGACY_BASE_NAME.lowLossThreshold".from(JitsiConfig.legacyConfig) + "jmt.bwe.send-side.loss-experiment.low-loss-threshold".from(JitsiConfig.newConfig) + } + + /** + * The low-loss threshold (expressed as a proportion of lost packets) when the loss probability + * experiment is active. + */ + @JvmStatic + fun experimentalLowLossThreshold() = experimentalLowLossThreshold + + private val experimentalHighLossThreshold: Double by config { + "$LEGACY_BASE_NAME.highLossThreshold".from(JitsiConfig.legacyConfig) + "jmt.bwe.send-side.loss-experiment.high-loss-threshold".from(JitsiConfig.newConfig) + } + + /** + * The high-loss threshold (expressed as a proportion of lost packets). + */ + @JvmStatic + fun experimentalHighLossThreshold() = experimentalHighLossThreshold + + private val experimentalBitrateThreshold: Bandwidth by config { + "$LEGACY_BASE_NAME.bitrateThresholdKbps".from(JitsiConfig.legacyConfig).convertFrom { it.kbps } + "jmt.bwe.send-side.loss-experiment.bitrate-threshold".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + + /** + * The bitrate threshold when the loss probability experiment is active. + */ + @JvmStatic + fun experimentalBitrateThreshold() = experimentalBitrateThreshold + + @JvmStatic + fun experimentalBitrateThresholdBps() = experimentalBitrateThreshold.bps + + private val timeoutExperimentProbability: Double by config { + "$LEGACY_BASE_NAME.timeoutExperimentProbability".from(JitsiConfig.legacyConfig) + "jmt.bwe.send-side.timeout-experiment.probability".from(JitsiConfig.newConfig) + } + + /** + * The probability of enabling the timeout experiment. + */ + @JvmStatic + fun timeoutExperimentProbability() = timeoutExperimentProbability + } +} diff --git a/jitsi-media-transform/src/main/resources/reference.conf b/jitsi-media-transform/src/main/resources/reference.conf new file mode 100644 index 0000000..2578fe4 --- /dev/null +++ b/jitsi-media-transform/src/main/resources/reference.conf @@ -0,0 +1,235 @@ +jmt { + bwe { + estimator { + // Version of bandwidth estimator to use. Current supported values are + // GoogleCc, GoogleCc2. + engine = GoogleCc2 + + initial-bw = 2500 kbps + min-bw = 30 kbps + max-bw = 20 mbps + + // Defaults for some other parameters are based on the value selected for 'engine' + GoogleCc { + default-window-size = 5 seconds + default-bucket-size = 100 ms + default-initial-ignore-bwe-period = 10 seconds + } + GoogleCc2 { + default-window-size = 1 seconds + default-bucket-size = 20 ms + default-initial-ignore-bwe-period = 0 seconds + } + } + + send-side { + low-loss-threshold = 0.02 + high-loss-threshold = 0.1 + bitrate-threshold = 100 kbps + + loss-experiment { + probability = 0 + low-loss-threshold = ${jmt.bwe.send-side.low-loss-threshold} + high-loss-threshold = ${jmt.bwe.send-side.high-loss-threshold} + bitrate-threshold = ${jmt.bwe.send-side.bitrate-threshold} + } + + timeout-experiment { + probability = 0 + } + } + + overuse-detector { + initial-threshold = 25.0 + overusing-time-threshold = 100.0 + } + } + dtls { + // The maximum length of time to wait for a DTLS handshake to complete. + handshake-timeout = 30 seconds + + // The list of cipher suites to use for DTLS. The names must correspond to the constants in BouncyCastle's + // CipherSuite class. + cipher-suites = [ + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 + // Looks like none of the browsers supports AES 256 for DTLS yet: + // TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + // TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 + ] + // The hash function to use to generate certificate fingerprints + local-fingerprint-hash-function = sha-256 + // The hash functions that are accepted for remote certificate fingerprints, in decreasing strength order + accepted-fingerprint-hash-functions = [ sha-512, sha-384, sha-256, sha-1 ] + + } + keyframe { + // The minimum interval between consecutive keyframe requests from a particular receiver for a media source. + // (This interval will also be limited by the current round-trip time to the sender of that source.) + min-interval = 200 ms + // The maximum number of requests from a particular receiver for a media source per max-request-interval + max-requests = 3 + // The interval over which to compute max-requests + max-request-interval = 10 s + + // The minimum interval between consecutive keyframe requests for any receiver for a media source. This also + // applies to requests which can not be attributed to a receiver, such as those relayed from other bridges. + // A keyframe costs many times a delta frame, and a sender's encoder has to serve every receiver's requests from + // a single bitrate budget, so this is kept much lower than the per-receiver rate. A VP9 K-SVC source capped at + // 1.2 Mbps was observed to degrade badly at one keyframe per second, so the interval is longer than that; it is + // also the longest a receiver waiting for a keyframe can be delayed when the source is in demand. + source-wide-min-interval = 2 s + source-wide-max-requests = 1 + source-wide-max-request-interval = 2 s + } + srtp { + // The maximum number of packets that can be discarded early (without going through the SRTP stack for + // authentication), or -1 to authenticate all packets. + // This is useful to avoid unnecessary computation for audio silence. + max-consecutive-packets-discarded-early=-1 + + // The DTLS-SRTP protection profiles that are supported, in preference order. + protection-profiles=[ + "SRTP_AEAD_AES_128_GCM", + "SRTP_AES128_CM_HMAC_SHA1_80" + ] + + // The engine (cryptographic provider) to use for SRTP cryptography. + // Valid values are "OpenSSL", "SunJCE", "SunPKCS11", "BouncyCastle", + // or null to let the implementation pick the provider at runtime. + // "OpenSSL" is currently only supported on Linux (x86_64, arm64, and ppc64el) + // and macOS (x86_64 and arm64). + // Custom class values are also possible - see Aes.setFactoryClassName + // in jitsi-srtp for more details. + // Generally the best performance will be obtained by "OpenSSL" where supported, + // and "SunJCE" otherwise, but see the logs for the measured performance of + // each provider. + factory-class="OpenSSL" + } + audio { + red { + // The policy for handling RED. The valid values are: + // NOOP: does not strip RED from incoming streams (unless the receiver lacks RED support), and does not + // encapsulate audio streams in RED. + // STRIP: strips RED from all incoming streams (recovering lost packets) + // PROTECT_ALL: encapsulates all forwarded audio streams in RED (unless they are already RED encoded, or the + // receiver lacks RED support). + policy = NOOP + // The "distance" to use when encoding streams in RED. The valid values are: + // ONE: encode only the previous packet as redundancy (increasing bitrate by ~100%) + // TWO: encode the previous two packets as redundancy (increasing bitrate by ~200%, but further increasing + // resilience) + distance = TWO + // Whether redundancy should be added for all packets (if `false`), or just packets with the Voice Activity + // Detection bit set (if `true`). + vad-only = true + } + level { + // Number of RTP packets with 0 audio level to forward before start discarding (to avoid CNG noise in muted state + // if DTX is enabled). + forwarded-silence-packets-limit = 3 + + // Whether to discard packets marked with silence (audio level 0) in the receive pipeline. + discard-silence = true + } + } + + rtp { + // The node which calculates the bitrates of the incoming RTP streams and layers. + // Default values are set based on the value of jmt.bwe.estimator.engine + bitrate-calculator { + // By default these values will depend on the BWE algorithm chosen based on values set under + // jmt.bwe.estimator, but they can also be specified explicitly. + + // The window over which average bitrate is calculated. + # window-size = 5 seconds + // The bucket size used to calculate average bitrate (smaller bucket size provides more precision, but requires + // more memory). This must evenly divide [window-size]. + # bucket-size = 100 ms + + // The window used for sources whose bitrate is bursty (screen sharing), for which the window above -- which is + // matched to the bandwidth estimator's window -- is too short to be representative. It should exceed both the + // rate control credit a screen sharing encoder may accumulate (about 1 second) and the maximum interval it may + // leave between frames (2.75 seconds in libwebrtc). + smoothed-window-size = 5 seconds + // The bucket size used with smoothed-window-size. This must evenly divide it. + smoothed-bucket-size = 100 ms + } + } + + transceiver { + recv { + // The size of the queue that handles incoming rtp/rtcp packets. This is + // a per-participant queue and it's where we temporarily store the + // packets that the endpoint is sending, right after an srtp packet has + // been received. + // + // Assuming 300pps for high-definition, 200pps for standard-definition, + // 100pps for low-definition and 50pps for audio, this queue is fed + // 650pps, so its size in terms of millis is 1024/650*1000 ~= 1575ms. + queue-size=1024 + } + + send { + // The size of the queue that handles outgoing rtp/rtcp packets. This is + // a per-participant queue and it's where we temporarily store the + // packets right after the ingress pipeline has run. + // + // Packets from all the other endpoints in a call end up in the send + // queue. + // + // In a 100 participant call, assuming 300pps for the on-stage and + // 100pps for low-definition, last-n 20 and 2 participants talking, so + // 2*50pps for audio, this queue is fed 300+19*100+2*50 = 2300pps, so its + // size in terms of millis is 1024/2300*1000 ~= 445ms. + queue-size=1024 + } + } + + debug { + pcap { + // Whether to permit the API to dynamically enable the capture of + // unencrypted PCAP files of media traffic. + // Should only be enabled in test or debug environments. + enabled = false + // Whether to start pcaps for every endpoint automatically. Requires "enabled" to be true. + // Definitely should only be enabled in debug environments. + start-enabled = false + // The directory in which to place captured PCAP files. + directory = "/tmp" + } + packet-loss { + // Artificial loss to introduce in the receive pipeline. + incoming { + // Rate of random packet loss with uniform distribution. + uniform-rate = 0 % + // Bursty-loss: drops [burst-size] consecutive packets in every [burst-interval] packets. Applied prior to + // uniform loss. + burst-size = 0 + burst-interval = 0 + } + // Artificial loss to introduce in the send pipeline. + outgoing { + // Rate of random packet loss with uniform distribution. + uniform-rate = 0 % + // Bursty-loss: drops [burst-size] consecutive packets in every [burst-interval] packets. Applied prior to + // uniform loss + burst-size = 0 + burst-interval = 0 + } + } + packet-timeline { + // Whether to enable the packet timeline. This is an expensive option used for debugging. + enabled = false + // Log a packet timeline for every one out of [log-fraction] packets. + log-fraction = 10000 + } + payload-verification { + // Whether to enable payload verification on startup. This is a very expensive option only used for debugging. + enabled = false + } + } +} diff --git a/jitsi-media-transform/src/test/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264Test.java b/jitsi-media-transform/src/test/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264Test.java new file mode 100644 index 0000000..db76d8f --- /dev/null +++ b/jitsi-media-transform/src/test/java/org/jitsi_modified/impl/neomedia/codec/video/h264/H264Test.java @@ -0,0 +1,170 @@ +/* + * Copyright @ 2015 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi_modified.impl.neomedia.codec.video.h264; + +import org.junit.jupiter.api.*; +import static org.junit.jupiter.api.Assertions.*; + +import static org.jitsi_modified.impl.neomedia.codec.video.h264.H264.kNalUSize; +import static org.jitsi_modified.impl.neomedia.codec.video.h264.H264.kStapA; +import static org.jitsi_modified.impl.neomedia.codec.video.h264.H264.verifyStapANaluLengths; + +public class H264Test +{ + private void populateNalUnits(byte[] data, int[] naluSizes, int offset) + { + for (int naluSize : naluSizes) + { + data[offset] = (byte) ((naluSize & 0xff00) >> 8); + data[offset+1] = (byte) (naluSize & 0xff); + offset += naluSize + kNalUSize; + } + } + + private int packetLength(int[] naluSizes) + { + int accumulator = 0; + for (int nalSize : naluSizes) + { + accumulator += nalSize + kNalUSize; + } + return accumulator; + } + + private byte[] buildStapA(int[] naluSizes, int offset, int length) + { + byte[] data = new byte[offset + length]; + populateNalUnits(data, naluSizes, offset); + return data; + } + + private boolean buildAndVerifyStapA(int[] naluSizes, int offset) + { + return buildAndVerifyStapA(naluSizes, offset, packetLength(naluSizes)); + } + + private boolean buildAndVerifyStapA(int[] naluSizes, int offset, int length) + { + int realPacketLength = packetLength(naluSizes); + byte[] data = buildStapA(naluSizes, offset, realPacketLength); + return verifyStapANaluLengths(data, offset, length); + } + + private static final int kMaxNalSize = 0xffff; + + @Test + public void getUint16_BuildsUint16() + { + assertEquals(0x0, H264.getUint16(new byte[] {0x0, 0x0}, 0)); + assertEquals(0xffff, H264.getUint16(new byte[] {(byte) 0xff, (byte) 0xff}, 0)); + assertEquals(0x1234, H264.getUint16(new byte[] {0x12, 0x34}, 0)); + } + + @Test + public void getUint16_ObeysOffset() + { + assertEquals(0x1234, H264.getUint16(new byte[] {(byte) 0xff, 0x12, 0x34}, 1)); + } + + @Test + public void getUint16_MostSignificantBitSetInFirstByte() + { + assertEquals(0xff00, H264.getUint16(new byte[] {(byte) 0xff, 0}, 0)); + } + + @Test + public void getUint16_MostSignificantBitSetInSecondByte() + { + assertEquals(0xff, H264.getUint16(new byte[] { 0, (byte) 0xff}, 0)); + } + + @Test + public void verifyStapANaluLengths_EmptyData_ShouldPass() + { + assertTrue(verifyStapANaluLengths(new byte[] {kStapA}, 1, 0)); + } + + @Test + public void verifyStapANaluLengths_LengthLessThanOneNal_ShouldFail() + { + assertFalse(verifyStapANaluLengths(new byte[] {kStapA}, 1, 1)); + assertFalse(verifyStapANaluLengths(new byte[] {kStapA}, 1, 2)); + } + + @Test + public void verifyStapANaluLengths_PacketTooShort_ShouldFail() + { + assertFalse(verifyStapANaluLengths(new byte[] {kStapA, 0, 1 }, 1, 3)); + } + + @Test + public void verifyStapANaluLengths_ValidPacketWithOneNal_ShouldPass() + { + assertTrue(verifyStapANaluLengths(new byte[] {kStapA, 0, 1, (byte)0xfa }, 1, 3)); + } + + @Test + public void verifyStapANaluLengths_OneMinSizedNal_ShouldPass() + { + assertTrue(buildAndVerifyStapA(new int[]{ 0 }, 1)); + } + + @Test + public void verifyStapANaluLengts_OneNal_ShouldPass() + { + assertTrue(buildAndVerifyStapA(new int[]{ 10 }, 1)); + } + + @Test + public void verifyStapANaluLengths_InsufficientLengthForNal_ShouldFail() + { + int[] naluSizes = { 10 }; + assertFalse(buildAndVerifyStapA(naluSizes, 1, packetLength(naluSizes) - 1)); + } + + @Test + public void verifyStapANaluLengths_InsufficentLengthForLargeNal_ShouldFail() + { + assertFalse(buildAndVerifyStapA(new int[]{ kMaxNalSize }, 1, 3)); + } + + @Test + public void verifyStapANaluLengths_MostSignificantBitNalLength_ShouldProperlyHandleSignExtensionWithoutInfiniteLoop() + { + assertTrue(buildAndVerifyStapA(new int[]{ kMaxNalSize - 2 }, 1)); + } + + @Test + public void verifyStapANaluLengths_TwoNal_ShouldPass() + { + int[] naluSizes = { 10, 10 }; + assertTrue(buildAndVerifyStapA(naluSizes, 1)); + } + + @Test + public void verifyStapANaluLengths_TwoNalInsufficientNalHeaderLength_ShouldFail() + { + int[] naluSizes = { 10, 10 }; + assertFalse(buildAndVerifyStapA(naluSizes, 1, packetLength(naluSizes) - naluSizes[0] - 1)); + } + + @Test + public void verifyStapANaluLengths_TwoNalInsufficientLength_ShouldFail() + { + int[] naluSizes = { 10, 10 }; + assertFalse(buildAndVerifyStapA(naluSizes, 1, packetLength(naluSizes) - 1)); + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt new file mode 100644 index 0000000..c03b59a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/KotestProjectConfig.kt @@ -0,0 +1,28 @@ +/* + * Copyright @ 2022 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj + +import io.kotest.core.config.AbstractProjectConfig +import org.jitsi.metaconfig.MetaconfigSettings + +class KotestProjectConfig : AbstractProjectConfig() { + override suspend fun beforeProject() = super.beforeProject().also { + // The only purpose of config caching is performance. We always want caching disabled in tests (so we can + // freely modify the config without affecting other tests executing afterwards). + MetaconfigSettings.cacheEnabled = false + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt new file mode 100644 index 0000000..09c44cd --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourceDescTest.kt @@ -0,0 +1,259 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.bps +import org.jitsi.utils.secs + +class MediaSourceDescTest : ShouldSpec() { + init { + val ssrcs = arrayOf(0xdeadbeefL, 0xcafebabeL, 0x01234567L) + val source = createSource( + ssrcs, + 1, + 3, + "Fake owner", + "Fake name", + VideoType.DESKTOP + ) + + context("Source properties should be correct") { + source.owner shouldBe "Fake owner" + source.sourceName shouldBe "Fake name" + source.videoType shouldBe VideoType.DESKTOP + source.rtpEncodings.size shouldBe 3 + + source.rtpLayers.size shouldBe 9 + source.hasRtpLayers() shouldBe true + source.numRtpLayers() shouldBe 9 + + source.matches(0xdeadbeef) shouldBe true + } + + context("Encoding and layer properties should be correct") { + for (i in source.rtpEncodings.indices) { + val e = source.rtpEncodings[i] + e.primarySSRC shouldBe ssrcs[i] + e.eid shouldBe i + e.layers.size shouldBe 3 + for (j in e.layers.indices) { + val l = e.layers[j] + l should beInstanceOf() + l as VpxRtpLayerDesc + + l.eid shouldBe i + l.tid shouldBe j + l.sid shouldBe -1 + + /* Set up rate statistics testing */ + /* Set non-zero rates for (0,0), (0,1), (1,0), (1,1), and (2,2) */ + if ((i < source.rtpEncodings.size - 1 && j < e.layers.size - 1) || + (i == source.rtpEncodings.size - 1 && j == e.layers.size - 1) + ) { + /* Encode the layer ID into the rate, so it's unambiguous which layers are getting summed. */ + l.inheritStatistics(FakeBitrateTracker(1L shl (i * source.rtpEncodings.size + j))) + } + } + } + } + + context("Layer bitrates should be correct") { + val t = 0L // Doesn't actually matter for fake rate statistics + + /* Rate from layer 0 */ + source.getBitrate(t, RtpLayerDesc.getIndex(0, 0, 0)) shouldBe 1.bps + /* Rates from layer 1 and its dependency, 0 */ + source.getBitrate(t, RtpLayerDesc.getIndex(0, 0, 1)) shouldBe (2 + 1).bps + /* Layer 2's own rate is 0, so rates from its dependencies, 1 and 0 */ + source.getBitrate(t, RtpLayerDesc.getIndex(0, 0, 2)) shouldBe (2 + 1).bps + + /* Rate from layer 3 */ + source.getBitrate(t, RtpLayerDesc.getIndex(1, 0, 0)) shouldBe 8.bps + /* Rates from layer 4 and its dependency, 3 */ + source.getBitrate(t, RtpLayerDesc.getIndex(1, 0, 1)) shouldBe (16 + 8).bps + /* Layer 5's own rate is 0, so rates from its dependencies, 4 and 3 */ + source.getBitrate(t, RtpLayerDesc.getIndex(1, 0, 2)) shouldBe (16 + 8).bps + + /* If a layer returns a 0 rate, the function gets the next lower non-zero rate */ + /* Layer 6's own rate is 0, and it has no dependencies, so get the rate from the next-lower + layer, 5 */ + source.getBitrate(t, RtpLayerDesc.getIndex(2, 0, 0)) shouldBe (16 + 8).bps + /* Layer 7's own rate is also 0, so similarly get the rate from layer 5 */ + source.getBitrate(t, RtpLayerDesc.getIndex(2, 0, 1)) shouldBe (16 + 8).bps + /* Layer 8's rate. Its dependencies (7 and 6) are 0. */ + source.getBitrate(t, RtpLayerDesc.getIndex(2, 0, 2)) shouldBe 256.bps + } + } +} + +/* The following creation functions are adapted from MediaSourceFactory in jitsi-videobridge. */ + +/** + * Calculates the array position of an RTP layer description specified by its + * spatial index (SVC) and temporal index (SVC). + * + * @param spatialIdx the spatial layer index. + * @param temporalIdx the temporal layer index. + * + * @return the subjective quality index of the flow specified in the + * arguments. + */ +private fun idx(spatialIdx: Int, temporalIdx: Int, temporalLen: Int) = spatialIdx * temporalLen + temporalIdx + +/* + * Creates layers for an encoding. + * + * @param spatialLen the number of spatial encodings per simulcast stream. + * @param temporalLen the number of temporal encodings per simulcast stream. + * @param height the maximum height of the top spatial layer + * @return an array that holds the layer descriptions. + */ +private fun createRTPLayerDescs(spatialLen: Int, temporalLen: Int, encodingIdx: Int, height: Int): Array { + val rtpLayers = arrayOfNulls(spatialLen * temporalLen) + for (spatialIdx in 0 until spatialLen) { + var frameRate = 30.toDouble() / (1 shl temporalLen - 1) + for (temporalIdx in 0 until temporalLen) { + val idx: Int = idx(spatialIdx, temporalIdx, temporalLen) + val dependencies: Array = if (spatialIdx > 0 && temporalIdx > 0) { + // this layer depends on spatialIdx-1 and temporalIdx-1. + arrayOf( + rtpLayers[ + idx( + spatialIdx, + temporalIdx - 1, + temporalLen + ) + ]!!, + rtpLayers[ + idx( + spatialIdx - 1, + temporalIdx, + temporalLen + ) + ]!! + ) + } else if (spatialIdx > 0) { + // this layer depends on spatialIdx-1. + arrayOf( + rtpLayers[ + idx( + spatialIdx - 1, + temporalIdx, + temporalLen + ) + ]!! + ) + } else if (temporalIdx > 0) { + // this layer depends on temporalIdx-1. + arrayOf( + rtpLayers[ + idx( + spatialIdx, + temporalIdx - 1, + temporalLen + ) + ]!! + ) + } else { + // this is a base layer without any dependencies. + emptyArray() + } + val temporalId = if (temporalLen > 1) temporalIdx else -1 + val spatialId = if (spatialLen > 1) spatialIdx else -1 + rtpLayers[idx] = VpxRtpLayerDesc( + encodingIdx, + temporalId, + spatialId, + height, + frameRate, + dependencies + ) + frameRate *= 2.0 + } + } + return rtpLayers as Array +} + +/** + * Creates an RTP encoding. + * @param primarySsrc the primary SSRC for the encoding. + * @param spatialLen the number of spatial layers of the encoding. + * @param temporalLen the number of temporal layers of the encodings. + * @param secondarySsrcs a list of pairs, where each + * pair has the secondary ssrc as its key, and the type (rtx, etc.) as its + * value + * @param encodingIdx the index of the encoding + * @return a description of the encoding. + */ +private fun createRtpEncodingDesc( + primarySsrc: Long, + spatialLen: Int, + temporalLen: Int, + encodingIdx: Int, + height: Int +): RtpEncodingDesc { + val layers: Array = createRTPLayerDescs( + spatialLen, + temporalLen, + encodingIdx, + height + ) + val enc = RtpEncodingDesc(primarySsrc, layers) + return enc +} + +private fun createSource( + primarySsrcs: Array, + numSpatialLayersPerStream: Int, + numTemporalLayersPerStream: Int, + owner: String, + name: String, + videoType: VideoType, +): MediaSourceDesc { + var height = 180 + + val encodings = Array(primarySsrcs.size) { encodingIdx -> + val primarySsrc: Long = primarySsrcs[encodingIdx] + val ret = createRtpEncodingDesc( + primarySsrc, + numSpatialLayersPerStream, + numTemporalLayersPerStream, + encodingIdx, + height + ) + height *= 2 + ret + } + + return MediaSourceDesc(encodings, owner, name, videoType) +} + +/** A fake rate statistics object, for testing */ +private class FakeBitrateTracker( + private val fakeRateBps: Long +) : BitrateTracker(1.secs) { + override fun getRate(nowMs: Long): Bandwidth = fakeRateBps.bps + + override fun getRateBps(nowMs: Long): Long = fakeRateBps +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourcesTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourcesTest.kt new file mode 100644 index 0000000..6f80e2c --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/MediaSourcesTest.kt @@ -0,0 +1,94 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class MediaSourcesTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + init { + val mediaSources = MediaSources() + + val sourceA = createSource(1000) + val sourceA2 = createSource(1000) + val sourceB = createSource(2000, 2001) + val sourceB2 = createSource(2000, 2001) + val sourceC = createSource(3000, 3001) + + var changed = mediaSources.setMediaSources(arrayOf(sourceA, sourceB)) + context("Setting initially must signal a change.") { + changed shouldBe true + + val newSources = mediaSources.getMediaSources() + newSources[0] shouldBe sourceA + newSources[1] shouldBe sourceB + } + + context("Setting the same sources must not signal a change.") { + changed = mediaSources.setMediaSources(arrayOf(sourceA, sourceB)) + changed shouldBe false + + val newSources = mediaSources.getMediaSources() + newSources[0] shouldBe sourceA + newSources[1] shouldBe sourceB + } + + context("Setting matching sources must not signal a change, or change the saved sources") { + changed = mediaSources.setMediaSources(arrayOf(sourceA2, sourceB2)) + changed shouldBe false + + val newSources = mediaSources.getMediaSources() + newSources[0] shouldBe sourceA + newSources[1] shouldBe sourceB + } + + context("Adding a new source must signal a change, but not change the previous sources") { + changed = mediaSources.setMediaSources(arrayOf(sourceA2, sourceB2, sourceC)) + changed shouldBe true + + val newSources = mediaSources.getMediaSources() + newSources[0] shouldBe sourceA + newSources[1] shouldBe sourceB + newSources[2] shouldBe sourceC + } + + context("Removing a source must signal a change, but not change the previous source") { + changed = mediaSources.setMediaSources(arrayOf(sourceA)) + changed shouldBe true + + val newSources = mediaSources.getMediaSources() + newSources[0] shouldBe sourceA + } + + context("Adding and removing must signal a change, but not change the previous source") { + changed = mediaSources.setMediaSources(arrayOf(sourceA2, sourceC)) + changed shouldBe true + + val newSources = mediaSources.getMediaSources() + newSources[0] shouldBe sourceA + newSources[1] shouldBe sourceC + } + } + + companion object { + fun createSource(vararg ssrcs: Long): MediaSourceDesc { + val encodings = Array(ssrcs.size) { i -> RtpEncodingDesc(ssrcs[i], i) } + return MediaSourceDesc(encodings, sourceName = ssrcs.joinToString(","), owner = ssrcs.joinToString(",")) + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt new file mode 100644 index 0000000..99f4959 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PacketInfoTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class PacketInfoTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + context("EventTimeline test") { + val timeline = EventTimeline().apply { + addEvent("A") + addEvent("B") + } + val clone = timeline.clone() + timeline.size shouldBe 2 + clone.size shouldBe 2 + + timeline.addEvent("original") + timeline.size shouldBe 3 + clone.size shouldBe 2 + + clone.addEvent("clone") + clone.addEvent("clone2") + timeline.size shouldBe 3 + clone.size shouldBe 4 + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PerSsrcMapCapTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PerSsrcMapCapTest.kt new file mode 100644 index 0000000..fae03e3 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/PerSsrcMapCapTest.kt @@ -0,0 +1,152 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.format.Vp8PayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.rtcp.RetransmissionRequester +import org.jitsi.nlj.rtcp.RtcpRrGenerator +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.transform.node.AudioRedHandler +import org.jitsi.nlj.transform.node.incoming.DiscardableDiscarder +import org.jitsi.nlj.transform.node.incoming.DuplicateTermination +import org.jitsi.nlj.transform.node.incoming.IncomingStatisticsTracker +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.RtcpSrPacketBuilder +import org.jitsi.rtp.rtcp.SenderInfoBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.concurrent.FakeScheduledExecutorService +import java.time.Instant + +/** + * Verifies that the per-SSRC tracking structures are bounded, i.e. that the number of distinct SSRCs tracked for a + * single stream is limited regardless of how many distinct SSRCs are received. + */ +class PerSsrcMapCapTest : ShouldSpec() { + init { + // More than any of the caps, so the maps are driven to eviction. + val n = 2000 + + context("IncomingStatisticsTracker") { + val store = StreamInformationStoreImpl().apply { + addRtpPayloadType(Vp8PayloadType(100, emptyMap(), emptySet())) + } + val tracker = IncomingStatisticsTracker(store) + repeat(n) { i -> + tracker.processPacket(PacketInfo(rtpPacket(i.toLong())).apply { receivedTime = Instant.EPOCH }) + } + should("bound the number of tracked SSRCs") { + tracker.getSnapshot().ssrcStats.size shouldBe IncomingStatisticsTracker.MAX_SSRCS + } + } + + context("DuplicateTermination") { + val node = DuplicateTermination() + repeat(n) { i -> node.processPacket(PacketInfo(rtpPacket(i.toLong()))) } + should("bound the number of tracked replay contexts") { + node.mapField("replayContexts").size shouldBe DuplicateTermination.MAX_SSRCS + } + } + + context("DiscardableDiscarder") { + val node = DiscardableDiscarder("test", keepHistory = false) + repeat(n) { i -> node.processPacket(PacketInfo(rtpPacket(i.toLong()))) } + should("bound the number of tracked rewriters") { + node.rewriters.size shouldBe DiscardableDiscarder.MAX_SSRCS + } + } + + context("AudioRedHandler") { + val handler = AudioRedHandler(StreamInformationStoreImpl(), StdoutLogger()) + repeat(n) { i -> handler.processPacket(PacketInfo(audioPacket(i.toLong()))) } + should("bound the number of tracked RED handlers") { + handler.mapField("ssrcRedHandlers").size shouldBe AudioRedHandler.MAX_SSRCS + } + } + + context("RetransmissionRequester") { + val scheduler = FakeScheduledExecutorService() + val requester = RetransmissionRequester({}, scheduler, StdoutLogger(), scheduler.clock) + repeat(n) { i -> requester.packetReceived(i.toLong(), 1) } + should("bound the number of tracked stream requesters") { + requester.mapField("streamPacketRequesters").size shouldBe RetransmissionRequester.MAX_SSRCS + } + } + + context("RtcpRrGenerator") { + val scheduler = FakeScheduledExecutorService() + val store = StreamInformationStoreImpl() + val generator = RtcpRrGenerator( + scheduler, + {}, + IncomingStatisticsTracker(store), + scheduler.clock + ) { emptyList() } + repeat(n) { i -> + val sr = RtcpSrPacketBuilder( + RtcpHeaderBuilder(senderSsrc = i.toLong()), + SenderInfoBuilder(rtpTimestamp = 12345L) + ).build() + generator.rtcpPacketReceived(sr, Instant.EPOCH) + } + should("bound the number of tracked sender infos") { + generator.mapField("senderInfos").size shouldBe RtcpRrGenerator.MAX_SSRCS + } + } + } + + private fun rtpPacket(ssrcVal: Long, seq: Int = 1, pt: Int = 100): RtpPacket = + RtpPacket(ByteArray(50), 0, 50).apply { + version = 2 + hasPadding = false + hasExtensions = false + isMarked = false + payloadType = pt + sequenceNumber = seq + timestamp = 456L + ssrc = ssrcVal + } + + private fun audioPacket(ssrcVal: Long, seq: Int = 1, pt: Int = 111): AudioRtpPacket = + AudioRtpPacket(ByteArray(50), 0, 50).apply { + version = 2 + hasPadding = false + hasExtensions = false + isMarked = false + payloadType = pt + sequenceNumber = seq + timestamp = 456L + ssrc = ssrcVal + } + + /** Reads a private (possibly inherited) [Map] field by reflection, for asserting its size. */ + private fun Any.mapField(name: String): Map<*, *> { + var cls: Class<*>? = this::class.java + while (cls != null) { + try { + val f = cls.getDeclaredField(name) + f.isAccessible = true + return f.get(this) as Map<*, *> + } catch (e: NoSuchFieldException) { + cls = cls.superclass + } + } + throw NoSuchFieldException(name) + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt new file mode 100644 index 0000000..826fbe7 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/RtpLayerDescTest.kt @@ -0,0 +1,152 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc +import org.jitsi.nlj.util.bytes + +class RtpLayerDescTest : FunSpec({ + + test("VP8 layer ids") { + // mostly for documenting the encoding -> index mapping. + val vp8Layers = arrayOf( + VpxRtpLayerDesc(0, 0, 0, 180, 7.5), + VpxRtpLayerDesc(0, 1, 0, 180, 15.0), + VpxRtpLayerDesc(0, 2, 0, 180, 30.0), + VpxRtpLayerDesc(1, 0, 0, 360, 7.5), + VpxRtpLayerDesc(1, 1, 0, 360, 15.0), + VpxRtpLayerDesc(1, 2, 0, 360, 30.0), + VpxRtpLayerDesc(2, 0, 0, 720, 7.5), + VpxRtpLayerDesc(2, 1, 0, 720, 15.0), + VpxRtpLayerDesc(2, 2, 0, 720, 30.0) + ) + + vp8Layers[0].index shouldBe 0 + vp8Layers[1].index shouldBe 1 + vp8Layers[2].index shouldBe 2 + vp8Layers[3].index shouldBe 64 + vp8Layers[4].index shouldBe 65 + vp8Layers[5].index shouldBe 66 + vp8Layers[6].index shouldBe 128 + vp8Layers[7].index shouldBe 129 + vp8Layers[8].index shouldBe 130 + + vp8Layers[0].layerId shouldBe 0 + vp8Layers[1].layerId shouldBe 1 + vp8Layers[2].layerId shouldBe 2 + vp8Layers[3].layerId shouldBe 0 + vp8Layers[4].layerId shouldBe 1 + vp8Layers[5].layerId shouldBe 2 + vp8Layers[6].layerId shouldBe 0 + vp8Layers[7].layerId shouldBe 1 + vp8Layers[8].layerId shouldBe 2 + } + + test("VP9 layer ids") { + // same here, mostly for documenting the encoding -> index mapping. + val vp9Layers = arrayOf( + VpxRtpLayerDesc(0, 0, 0, 180, 7.5), + VpxRtpLayerDesc(0, 1, 0, 180, 15.0), + VpxRtpLayerDesc(0, 2, 0, 180, 30.0), + VpxRtpLayerDesc(0, 0, 1, 360, 7.5), + VpxRtpLayerDesc(0, 1, 1, 360, 15.0), + VpxRtpLayerDesc(0, 2, 1, 360, 30.0), + VpxRtpLayerDesc(0, 0, 2, 720, 7.5), + VpxRtpLayerDesc(0, 1, 2, 720, 15.0), + VpxRtpLayerDesc(0, 2, 2, 720, 30.0) + ) + + vp9Layers[0].index shouldBe 0 + vp9Layers[1].index shouldBe 1 + vp9Layers[2].index shouldBe 2 + vp9Layers[3].index shouldBe 8 + vp9Layers[4].index shouldBe 9 + vp9Layers[5].index shouldBe 10 + vp9Layers[6].index shouldBe 16 + vp9Layers[7].index shouldBe 17 + vp9Layers[8].index shouldBe 18 + + vp9Layers[0].layerId shouldBe 0 + vp9Layers[1].layerId shouldBe 1 + vp9Layers[2].layerId shouldBe 2 + vp9Layers[3].layerId shouldBe 8 + vp9Layers[4].layerId shouldBe 9 + vp9Layers[5].layerId shouldBe 10 + vp9Layers[6].layerId shouldBe 16 + vp9Layers[7].layerId shouldBe 17 + vp9Layers[8].layerId shouldBe 18 + } + + test("Smoothed bitrate is not tracked until enabled") { + val layer = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + layer.updateBitrate(1000.bytes, 0) + + // Fall back to the bitrate over the estimator-matched window. + layer.getSmoothedBitrate(100) shouldBe layer.getBitrate(100) + } + + test("Smoothed bitrate falls back until its tracker has a full window of history") { + val layer = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + layer.enableSmoothedBitrate(0) + layer.updateBitrate(1000.bytes, 0) + + layer.getSmoothedBitrate(100) shouldBe layer.getBitrate(100) + } + + test("Smoothed bitrate is lower than the short-window rate once its window has filled") { + val layer = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + layer.enableSmoothedBitrate(0) + val nowMs = 6000L // Past the smoothed window (5s by default). + layer.updateBitrate(1000.bytes, nowMs) + + layer.getSmoothedBitrate(nowMs + 100) shouldBeLessThan layer.getBitrate(nowMs + 100) + } + + test("Enabling the smoothed bitrate twice does not discard the data collected so far") { + val layer = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + layer.enableSmoothedBitrate(0) + val nowMs = 6000L + layer.updateBitrate(1000.bytes, nowMs) + val before = layer.getSmoothedBitrate(nowMs + 100) + + layer.enableSmoothedBitrate(nowMs + 100) + layer.getSmoothedBitrate(nowMs + 100) shouldBe before + } + + test("Disabling the smoothed bitrate reverts to the short-window rate") { + val layer = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + layer.enableSmoothedBitrate(0) + val nowMs = 6000L + layer.updateBitrate(1000.bytes, nowMs) + + layer.disableSmoothedBitrate() + layer.getSmoothedBitrate(nowMs + 100) shouldBe layer.getBitrate(nowMs + 100) + } + + test("A layer inheriting from another keeps an established smoothed tracker") { + val old = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + old.enableSmoothedBitrate(0) + val nowMs = 6000L + old.updateBitrate(1000.bytes, nowMs) + + val new = VpxRtpLayerDesc(0, 0, -1, 180, 30.0) + new.inheritFrom(old) + new.getSmoothedBitrate(nowMs + 100) shouldBe old.getSmoothedBitrate(nowMs + 100) + } +}) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/VpxUtilsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/VpxUtilsTest.kt new file mode 100644 index 0000000..f101a79 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/codec/vpx/VpxUtilsTest.kt @@ -0,0 +1,80 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.codec.vpx + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.data.forAll +import io.kotest.data.row +import io.kotest.matchers.shouldBe + +class VpxUtilsTest : ShouldSpec() { + + init { + context("getPicIdDelta") { + should("work correctly") { + forAll( + row(1, 10, -9), + row(10, 1, 9), + row(1, 32760, 9), + row(32760, 1, -9), + row(1234, 1234, 0) + ) { a, b, expected -> + VpxUtils.getExtendedPictureIdDelta(a, b) shouldBe expected + } + } + } + context("applyPicIdDelta") { + should("work correctly") { + forAll( + row(10, -9, 1), + row(1, 9, 10), + row(32760, 9, 1), + row(1, -9, 32760), + row(1234, 0, 1234) + ) { start, delta, expected -> + VpxUtils.applyExtendedPictureIdDelta(start, delta) shouldBe expected + } + } + } + context("getTl0PicIdxDelta") { + should("work correctly") { + forAll( + row(1, 10, -9), + row(10, 1, 9), + row(1, 250, 7), + row(250, 1, -7), + row(34, 34, 0) + ) { a, b, expected -> + VpxUtils.getTl0PicIdxDelta(a, b) shouldBe expected + } + } + } + context("applyTl0PicIdxDelta") { + should("work correctly") { + forAll( + row(10, -9, 1), + row(1, 9, 10), + row(250, 7, 1), + row(1, -7, 250), + row(34, 0, 34) + ) { start, delta, expected -> + VpxUtils.applyTl0PicIdxDelta(start, delta) shouldBe expected + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt new file mode 100644 index 0000000..b0b9ad9 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsConfigTest.kt @@ -0,0 +1,122 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.dtls + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import org.bouncycastle.tls.CipherSuite +import org.jitsi.config.withNewConfig +import org.jitsi.metaconfig.ConfigException + +class DtlsConfigTest : ShouldSpec() { + init { + context("Valid cipher suites") { + withNewConfig( + """ + jmt.dtls.cipher-suites = [ + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + ] + """.trimIndent() + ) { + DtlsConfig.config.cipherSuites shouldBe listOf( + CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + ) + } + } + context("Invalid cipher suites") { + context("Invalid name") { + withNewConfig( + """ + jmt.dtls.cipher-suites = [ + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + invalid + ] + """.trimIndent() + ) { + shouldThrow { DtlsConfig.config.cipherSuites } + } + } + + context("Empty") { + withNewConfig("jmt.dtls.cipher-suites = []") { + shouldThrow { DtlsConfig.config.cipherSuites } + } + } + context("Wrong type") { + withNewConfig("jmt.dtls.cipher-suites = 42") { + shouldThrow { DtlsConfig.config.cipherSuites } + } + } + } + context("Valid fingerprint hash functions") { + withNewConfig( + """ + jmt.dtls.local-fingerprint-hash-function = sha-512 + jmt.dtls.accepted-fingerprint-hash-functions = [ sha-512, sha-384, sha-256 ] + """.trimIndent() + ) { + DtlsConfig.config.localFingerprintHashFunction shouldBe "sha-512" + DtlsConfig.config.acceptedFingerprintHashFunctions shouldContainExactly + setOf("sha-512", "sha-384", "sha-256") + } + context("With inconsistent capitalization") { + withNewConfig( + """ + jmt.dtls.local-fingerprint-hash-function = SHA-512 + jmt.dtls.accepted-fingerprint-hash-functions = [ Sha-512, sHa-384, shA-256 ] + """.trimIndent() + ) { + DtlsConfig.config.localFingerprintHashFunction shouldBe "sha-512" + DtlsConfig.config.acceptedFingerprintHashFunctions shouldContainExactly + setOf("sha-512", "sha-384", "sha-256") + } + } + } + context("Invalid local fingerprint hash function") { + context("Invalid name") { + withNewConfig("jmt.dtls.local-fingerprint-hash-function = sha-257") { + shouldThrow { DtlsConfig.config.localFingerprintHashFunction } + } + } + context("Forbidden function") { + withNewConfig("jmt.dtls.local-fingerprint-hash-function = md5") { + shouldThrow { DtlsConfig.config.localFingerprintHashFunction } + } + } + } + context("Invalid accepted accepted fingerprint hash functions") { + context("Invalid entry") { + withNewConfig("jmt.dtls.accepted-fingerprint-hash-functions = [ sha-256, sha-257 ]") { + shouldThrow { DtlsConfig.config.acceptedFingerprintHashFunctions } + } + } + context("Empty") { + withNewConfig("jmt.dtls.accepted-fingerprint-hash-functions = []") { + shouldThrow { DtlsConfig.config.acceptedFingerprintHashFunctions } + } + } + context("Forbidden function") { + withNewConfig("jmt.dtls.accepted-fingerprint-hash-functions = [ sha-1, md5 ]") { + shouldThrow { DtlsConfig.config.acceptedFingerprintHashFunctions } + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsStackRoleTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsStackRoleTest.kt new file mode 100644 index 0000000..3b937cd --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsStackRoleTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright @ 2026 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.resources.logging.StdoutLogger +import java.util.logging.Level + +/** + * The role of a [DtlsStack] is set from the peer's `setup` attribute, which the peer re-signals with transport + * updates that have nothing to do with DTLS. Re-applying the role the stack already has must leave it alone: + * building a new [DtlsRole] would discard the one that negotiated the connection. + */ +class DtlsStackRoleTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + private val stack = DtlsStack(StdoutLogger(_level = Level.OFF)) + + init { + context("Setting the role for the first time") { + should("set it to server") { + stack.actAsServer() shouldBe true + (stack.role is DtlsServer) shouldBe true + } + should("set it to client") { + stack.actAsClient() shouldBe true + (stack.role is DtlsClient) shouldBe true + } + } + + context("Setting the role again") { + should("keep the server role that is already set") { + stack.actAsServer() shouldBe true + val role = stack.role + + stack.actAsServer() shouldBe false + (stack.role === role) shouldBe true + } + should("keep the client role that is already set") { + stack.actAsClient() shouldBe true + val role = stack.role + + stack.actAsClient() shouldBe false + (stack.role === role) shouldBe true + } + } + + context("Changing the role") { + should("replace a server role with a client one") { + stack.actAsServer() shouldBe true + + stack.actAsClient() shouldBe true + (stack.role is DtlsClient) shouldBe true + } + should("replace a client role with a server one") { + stack.actAsClient() shouldBe true + + stack.actAsServer() shouldBe true + (stack.role is DtlsServer) shouldBe true + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt new file mode 100644 index 0000000..537a419 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/dtls/DtlsTest.kt @@ -0,0 +1,126 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.dtls + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.transform.node.PcapWriter +import org.jitsi.rtp.UnparsedPacket +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit +import java.util.logging.Level +import kotlin.concurrent.thread + +class DtlsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + private val debugEnabled = true + private val pcapEnabled = false + private val logger = StdoutLogger(_level = Level.OFF) + + fun debug(s: String) { + if (debugEnabled) { + println(s) + } + } + + init { + val dtlsServer = DtlsStack(logger).apply { actAsServer() } + val dtlsClient = DtlsStack(logger).apply { actAsClient() } + val pcapWriter = if (pcapEnabled) PcapWriter(logger, "/tmp/dtls-test.pcap") else null + + dtlsClient.remoteFingerprints = mapOf( + dtlsServer.localFingerprintHashFunction to listOf(dtlsServer.localFingerprint) + ) + dtlsServer.remoteFingerprints = mapOf( + dtlsClient.localFingerprintHashFunction to listOf(dtlsClient.localFingerprint) + ) + + // The DTLS server's send is wired directly to the DTLS client's receive + dtlsServer.outgoingDataHandler = object : DtlsStack.OutgoingDataHandler { + override fun sendData(data: ByteArray, off: Int, len: Int) { + pcapWriter?.observe(PacketInfo(UnparsedPacket(data, off, len)), outbound = false) + dtlsClient.processIncomingProtocolData(data, off, len) + } + } + + val serverReceivedData = CompletableFuture() + val serverToClientMessage = "Goodbye, world" + dtlsServer.incomingDataHandler = object : DtlsStack.IncomingDataHandler { + override fun dataReceived(data: ByteArray, off: Int, len: Int) { + val packetData = ByteBuffer.wrap(data, off, len) + val receivedStr = StandardCharsets.UTF_8.decode(packetData).toString() + debug("Server received message: '$receivedStr'") + serverReceivedData.complete(receivedStr) + val serverToClientData = serverToClientMessage.toByteArray() + + dtlsServer.sendApplicationData(serverToClientData, 0, serverToClientData.size) + } + } + + // The DTLS client's send is wired directly to the DTLS server's receive + dtlsClient.outgoingDataHandler = object : DtlsStack.OutgoingDataHandler { + override fun sendData(data: ByteArray, off: Int, len: Int) { + pcapWriter?.observe(PacketInfo(UnparsedPacket(data, off, len)), outbound = true) + dtlsServer.processIncomingProtocolData(data, off, len) + } + } + + val clientReceivedData = CompletableFuture() + dtlsClient.incomingDataHandler = object : DtlsStack.IncomingDataHandler { + override fun dataReceived(data: ByteArray, off: Int, len: Int) { + val packetData = ByteBuffer.wrap(data, off, len) + val receivedStr = StandardCharsets.UTF_8.decode(packetData).toString() + debug("Client received message: '$receivedStr'") + clientReceivedData.complete(receivedStr) + } + } + + val serverThread = thread { + debug("Server accepting") + dtlsServer.start() + debug("Server accepted connection") + } + + debug("Client connecting") + dtlsClient.start() + // Ensure the server has fully established things on its side as well before we send the + // message by waiting for the server accept thread to finish + serverThread.join() + debug("Client connected, sending message") + val clientToServerMessage = "Hello, world" + val clientToServerData = clientToServerMessage.toByteArray() + dtlsClient.sendApplicationData(clientToServerData, 0, clientToServerData.size) + + serverReceivedData.get(5, TimeUnit.SECONDS) shouldBe clientToServerMessage + clientReceivedData.get(5, TimeUnit.SECONDS) shouldBe serverToClientMessage + + // The peer re-signals its setup attribute with transport updates long after the handshake, so the role is + // re-applied then. It must not replace the role that negotiated this connection. + val serverRole = dtlsServer.role + dtlsServer.actAsServer() shouldBe false + (dtlsServer.role === serverRole) shouldBe true + + val clientRole = dtlsClient.role + dtlsClient.actAsClient() shouldBe false + (dtlsClient.role === clientRole) shouldBe true + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/format/PayloadTypeEncodingTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/format/PayloadTypeEncodingTest.kt new file mode 100644 index 0000000..d9f8d33 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/format/PayloadTypeEncodingTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.format + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +internal class PayloadTypeEncodingTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("createFrom") { + should("be case insensitive") { + val encoding = PayloadTypeEncoding.createFrom("vp8") + val encoding2 = PayloadTypeEncoding.createFrom("vP8") + val encoding3 = PayloadTypeEncoding.createFrom("Vp8") + + encoding shouldBe PayloadTypeEncoding.VP8 + encoding2 shouldBe PayloadTypeEncoding.VP8 + encoding3 shouldBe PayloadTypeEncoding.VP8 + + encoding shouldBe encoding2 + encoding shouldBe encoding3 + encoding2 shouldBe encoding3 + } + should("return OTHER in the invalid case") { + PayloadTypeEncoding.createFrom("blah") shouldBe PayloadTypeEncoding.OTHER + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/AbstractPacketProducer.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/AbstractPacketProducer.kt new file mode 100644 index 0000000..1e52a57 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/AbstractPacketProducer.kt @@ -0,0 +1,88 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.module_tests + +import io.pkts.Pcap +import io.pkts.packet.Packet +import io.pkts.packet.UDPPacket +import io.pkts.protocol.Protocol +import org.jitsi.rtp.UnparsedPacket +import java.util.concurrent.TimeUnit + +abstract class AbstractPacketProducer : PacketProducer { + private val handlers = mutableListOf() + + override fun subscribe(handler: PacketReceiver) { + handlers.add(handler) + } + + protected fun onPacket(packet: org.jitsi.rtp.Packet) { + handlers.forEach { it(packet) } + } +} + +/** + * Read data from a PCAP file and play it out at a rate consistent with the packet arrival times. I.e. if the PCAP + * file captured data flowing at 2mbps, this producer will play it out at 2mbps + */ +class PcapPacketProducer( + pcapFilePath: String +) : AbstractPacketProducer() { + private val pcap = Pcap.openStream(pcapFilePath) + var running: Boolean = true + + companion object { + private fun translateToUnparsedPacket(pktsPacket: Packet): UnparsedPacket { + // We always allocate a buffer with capacity 1500, so the packet has room to 'grow' + val packetBuf = ByteArray(1500) + return if (pktsPacket.hasProtocol(Protocol.UDP)) { + val udpPacket = pktsPacket.getPacket(Protocol.UDP) as UDPPacket + System.arraycopy(udpPacket.payload.array, 0, packetBuf, 0, udpPacket.payload.array.size) + UnparsedPacket(packetBuf, 0, udpPacket.payload.array.size) + } else { + // When capturing on the loopback interface, the packets have a null ethernet + // frame which messes up the pkts libary's parsing, so instead use a hack to + // grab the buffer directly + System.arraycopy(pktsPacket.payload.rawArray, 32, packetBuf, 0, pktsPacket.payload.rawArray.size - 32) + UnparsedPacket(packetBuf, 0, pktsPacket.payload.rawArray.size - 32) + } + } + + private fun nowMicros(): Long = System.nanoTime() / 1000 + } + + fun run() { + var firstPacketArrivalTime = -1L + val startTime = nowMicros() + while (running) { + pcap.loop { pkt -> + if (firstPacketArrivalTime == -1L) { + firstPacketArrivalTime = pkt.arrivalTime + } + val expectedSendTime = pkt.arrivalTime - firstPacketArrivalTime + val nowClockTime = nowMicros() - startTime + if (expectedSendTime > nowClockTime) { + TimeUnit.MICROSECONDS.sleep(expectedSendTime - nowClockTime) + } + + val packet = translateToUnparsedPacket(pkt) + onPacket(packet) + true + } + running = false + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt new file mode 100644 index 0000000..c22b46d --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/EndToEndHarness.kt @@ -0,0 +1,109 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.safeShutdown +import org.jitsi.test_utils.Pcaps +import org.jitsi.utils.secs +import java.time.Clock +import java.util.concurrent.Executors + +/** + * Read packets from a PCAP file and feed them through a receiver + * and then to a sender to simulate an end-to-end run of packets. + * + * NOTE that in some places this file tries to simulate behavior + * of the bridge (for example cloning the packet before sending + * it to the sender) but it does not perfectly simulate the bridge + * (e.g. it does not do the simulcast filter/rewriting, send + * probing, or other behaviors) + */ + +fun main() { + val pcap = Pcaps.Incoming.ONE_PARTICIPANT_RTP_RTCP_SIM_RTX + + var numBuffersRequested = 0 + fun getBuffer(size: Int): ByteArray { + numBuffersRequested++ + return ByteArray(size) + } + var numBuffersReturned = 0 + fun returnBuffer(@Suppress("UNUSED_PARAMETER") buf: ByteArray) { + numBuffersReturned++ + } + BufferPool.getBuffer = ::getBuffer + BufferPool.returnBuffer = ::returnBuffer + org.jitsi.rtp.util.BufferPool.getArray = ::getBuffer + org.jitsi.rtp.util.BufferPool.returnArray = ::returnBuffer + + val producer = PcapPacketProducer(pcap.filePath) + + val backgroundExecutor = Executors.newSingleThreadScheduledExecutor() + val executor = Executors.newSingleThreadExecutor() + + val sender = SenderFactory.createSender( + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations + ) + + val receiver = ReceiverFactory.createReceiver( + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations, + { rtcpPacket -> sender.processPacket(PacketInfo(rtcpPacket)) } + ) + + producer.subscribe { pkt -> + val packetInfo = PacketInfo(pkt) + packetInfo.receivedTime = Clock.systemUTC().instant() + receiver.enqueuePacket(packetInfo) + } + + receiver.packetHandler = object : PacketHandler { + override fun processPacket(packetInfo: PacketInfo) { + sender.processPacket(packetInfo) + } + } + + sender.onOutgoingPacket(object : PacketHandler { + override fun processPacket(packetInfo: PacketInfo) { + BufferPool.returnBuffer(packetInfo.packet.getBuffer()) + } + }) + + producer.run() + + receiver.stop() + sender.stop() + executor.safeShutdown(10.secs) + backgroundExecutor.safeShutdown(10.secs) + + println(receiver.getNodeStats().prettyPrint()) + println(sender.getNodeStats().prettyPrint()) + + println("gave out $numBuffersRequested buffers, returned $numBuffersReturned") +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/PacketProducer.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/PacketProducer.kt new file mode 100644 index 0000000..ad3df63 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/PacketProducer.kt @@ -0,0 +1,24 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.module_tests + +import org.jitsi.rtp.Packet + +typealias PacketReceiver = (Packet) -> Unit + +interface PacketProducer { + fun subscribe(handler: PacketReceiver) +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/ReceiverFactory.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/ReceiverFactory.kt new file mode 100644 index 0000000..cf4a4ab --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/ReceiverFactory.kt @@ -0,0 +1,76 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.RtpReceiver +import org.jitsi.nlj.RtpReceiverImpl +import org.jitsi.nlj.TransceiverEventHandler +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.util.LocalSsrcAssociation +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.test_utils.SourceAssociation +import org.jitsi.test_utils.SrtpData +import org.jitsi.utils.logging2.Logger +import java.util.Random +import java.util.concurrent.ExecutorService +import java.util.concurrent.ScheduledExecutorService + +class ReceiverFactory { + companion object { + fun createReceiver( + executor: ExecutorService, + backgroundExecutor: ScheduledExecutorService, + srtpData: SrtpData, + payloadTypes: List, + headerExtensions: List, + ssrcAssociations: List, + rtcpSender: (RtcpPacket) -> Unit = {}, + logger: Logger = StdoutLogger() + ): RtpReceiver { + val streamInformationStore = StreamInformationStoreImpl() + val receiver = RtpReceiverImpl( + id = Random().nextLong().toString(), + rtcpSender = rtcpSender, + rtcpEventNotifier = RtcpEventNotifier(), + executor = executor, + backgroundExecutor = backgroundExecutor, + streamInformationStore = streamInformationStore, + eventHandler = object : TransceiverEventHandler {}, + parentLogger = logger + ) + receiver.setSrtpTransformers(SrtpTransformerFactory.createSrtpTransformers(srtpData)) + + payloadTypes.forEach { + streamInformationStore.addRtpPayloadType(it) + } + headerExtensions.forEach { + streamInformationStore.addRtpExtensionMapping(it) + } + ssrcAssociations.forEach { + streamInformationStore.addSsrcAssociation( + LocalSsrcAssociation(it.primarySsrc, it.secondarySsrc, it.associationType) + ) + } + + return receiver + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt new file mode 100644 index 0000000..a098679 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverHarness.kt @@ -0,0 +1,77 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpReceiver +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.util.safeShutdown +import org.jitsi.test_utils.Pcaps +import org.jitsi.utils.secs +import java.time.Clock +import java.util.concurrent.Executors +import java.util.logging.Level +import kotlin.system.measureTimeMillis + +/** + * Feed media data from a PCAP file through N receivers. This harness + * allows: + * 1) Changing the PCAP file to feed in different data + * 2) Changing the number of RTP Receivers we feed the incoming data to + * 3) Changing the number of threads used for all of the receivers + * + * This can give a sense of how many threads would be necessary to run + * N receivers given a certain input bit rate (determined by the PCAP file) + */ + +fun main() { + val pcap = Pcaps.Incoming.ONE_PARTICIPANT_RTP_RTCP_SIM_RTX + + val producer = PcapPacketProducer(pcap.filePath) + + val backgroundExecutor = Executors.newSingleThreadScheduledExecutor() + val executor = Executors.newSingleThreadExecutor() + val numReceivers = 1 + val receivers = mutableListOf() + repeat(numReceivers) { + val receiver = ReceiverFactory.createReceiver( + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations, + logger = StdoutLogger("receiver", Level.ALL) + ) + receivers.add(receiver) + } + + producer.subscribe { pkt -> + val packetInfo = PacketInfo(pkt) + packetInfo.receivedTime = Clock.systemUTC().instant() + receivers.forEach { it.enqueuePacket(packetInfo.clone()) } + } + + val time = measureTimeMillis { + producer.run() + } + println("took $time ms") + receivers.forEach(RtpReceiver::stop) + executor.safeShutdown(10.secs) + backgroundExecutor.safeShutdown(10.secs) + + receivers.forEach { println(it.getNodeStats().prettyPrint()) } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt new file mode 100644 index 0000000..15219ba --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpReceiverNoNewBufferTest.kt @@ -0,0 +1,88 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.util.safeShutdown +import org.jitsi.test_utils.Pcaps +import org.jitsi.utils.secs +import java.time.Clock +import java.util.concurrent.Executors +import java.util.concurrent.LinkedBlockingQueue + +/** + * Verify that the same buffer we feed into the receive pipeline is the one we get + * out (in other words: we don't re-allocate a new buffer). Verifying this means + * we can easily pull the original buffer from a pool and know that we can return + * the array coming out the other side. + * NOTE: we don't technically care about the ByteBuffer instance itself, but + * the backing byte array. + */ +// TODO: turn this into a real test (as well as the rest) and tag them as 'slow' +fun main() { + val pcap = Pcaps.Incoming.ONE_PARTICIPANT_RTP_RTCP_SIM_RTX + + val producer = PcapPacketProducer(pcap.filePath) + + val backgroundExecutor = Executors.newSingleThreadScheduledExecutor() + val executor = Executors.newSingleThreadExecutor() + + val receiver = ReceiverFactory.createReceiver( + executor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations + ) + + val sentArrays = LinkedBlockingQueue() + + /** + * Verify that the arrayId (acquired via calling System.identifyHashCode + * on the array) matches one that we went out. + */ + var numNewArrays = 0 + fun verifyArray(arrayId: Int) { + if (!sentArrays.any { it == arrayId }) { + println("Array $arrayId was not among the sent out arrays!") + numNewArrays++ + } + } + + receiver.packetHandler = object : PacketHandler { + override fun processPacket(packetInfo: PacketInfo) { + verifyArray(System.identityHashCode(packetInfo.packet.buffer)) + } + } + + producer.subscribe { pkt -> + sentArrays.offer(System.identityHashCode(pkt.buffer)) + val packetInfo = PacketInfo(pkt) + packetInfo.receivedTime = Clock.systemUTC().instant() + receiver.enqueuePacket(packetInfo) + } + + producer.run() + + receiver.stop() + executor.safeShutdown(10.secs) + backgroundExecutor.safeShutdown(10.secs) + + println("Num new arrays: $numNewArrays") +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt new file mode 100644 index 0000000..fd348cf --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/RtpSenderHarness.kt @@ -0,0 +1,67 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpSender +import org.jitsi.nlj.util.safeShutdown +import org.jitsi.rtp.extensions.looksLikeRtp +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.test_utils.Pcaps +import org.jitsi.utils.secs +import java.util.concurrent.Executors +import kotlin.system.measureTimeMillis + +fun main() { + val pcap = Pcaps.Outgoing.ONE_PARITICPANT_RTP_AND_RTCP_DECRYPTED + + val producer = PcapPacketProducer(pcap.filePath) + + val senderExecutor = Executors.newSingleThreadExecutor() + val backgroundExecutor = Executors.newSingleThreadScheduledExecutor() + val numSenders = 1 + val senders = mutableListOf() + repeat(numSenders) { + val sender = SenderFactory.createSender( + senderExecutor, + backgroundExecutor, + pcap.srtpData, + pcap.payloadTypes, + pcap.headerExtensions, + pcap.ssrcAssociations + ) + senders.add(sender) + } + + producer.subscribe { pkt -> + val packetInfo = when { + pkt.looksLikeRtp() -> PacketInfo(RtpPacket(pkt.buffer, pkt.offset, pkt.length)) + else -> PacketInfo(RtcpPacket.parse(pkt.buffer, pkt.offset, pkt.length)) + } + senders.forEach { it.processPacket(packetInfo.clone()) } + } + + val time = measureTimeMillis { + producer.run() + } + println("took $time ms") + senders.forEach(RtpSender::stop) + senderExecutor.safeShutdown(10.secs) + backgroundExecutor.safeShutdown(10.secs) + + senders.forEach { println(it.getNodeStats().prettyPrint()) } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SenderFactory.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SenderFactory.kt new file mode 100644 index 0000000..3f1932b --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SenderFactory.kt @@ -0,0 +1,77 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.RtpSender +import org.jitsi.nlj.RtpSenderImpl +import org.jitsi.nlj.SetLocalSsrcEvent +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.util.RemoteSsrcAssociation +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.test_utils.SourceAssociation +import org.jitsi.test_utils.SrtpData +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging2.Logger +import java.util.Random +import java.util.concurrent.ExecutorService +import java.util.concurrent.ScheduledExecutorService + +class SenderFactory { + companion object { + fun createSender( + executor: ExecutorService, + backgroundExecutor: ScheduledExecutorService, + srtpData: SrtpData, + payloadTypes: List, + headerExtensions: List, + ssrcAssociations: List, + logger: Logger = StdoutLogger() + ): RtpSender { + val streamInformationStore = StreamInformationStoreImpl() + val sender = RtpSenderImpl( + Random().nextLong().toString(), + RtcpEventNotifier(), + executor, + backgroundExecutor, + streamInformationStore, + logger + ) + sender.setSrtpTransformers(SrtpTransformerFactory.createSrtpTransformers(srtpData)) + + payloadTypes.forEach { + streamInformationStore.addRtpPayloadType(it) + } + headerExtensions.forEach { + streamInformationStore.addRtpExtensionMapping(it) + } + ssrcAssociations.forEach { + streamInformationStore.addSsrcAssociation( + RemoteSsrcAssociation(it.primarySsrc, it.secondarySsrc, it.associationType) + ) + } + + // Set some dummy sender SSRCs so RTCP can be forwarded + sender.handleEvent(SetLocalSsrcEvent(MediaType.VIDEO, 123)) + sender.handleEvent(SetLocalSsrcEvent(MediaType.AUDIO, 456)) + + return sender + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt new file mode 100644 index 0000000..718d12d --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/module_tests/SrtpTransformerFactory.kt @@ -0,0 +1,35 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.module_tests + +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.test_utils.SrtpData + +class SrtpTransformerFactory { + companion object { + fun createSrtpTransformers(srtpData: SrtpData): SrtpTransformers = SrtpUtil.initializeTransformer( + srtpData.srtpProfileInformation, + srtpData.keyingMaterial, + srtpData.tlsRole, + // TODO: add tests for the cryptex=true case + cryptex = false, + StdoutLogger() + ) + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/node/PacketLossTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/node/PacketLossTest.kt new file mode 100644 index 0000000..d1c3e3a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/node/PacketLossTest.kt @@ -0,0 +1,54 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.node + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import org.jitsi.config.withNewConfig +import org.jitsi.nlj.rtp.audioPacketBytes +import org.jitsi.nlj.transform.node.PacketLossConfig +import org.jitsi.nlj.transform.node.PacketLossNode +import org.jitsi.rtp.rtp.RtpPacket + +class PacketLossTest : ShouldSpec() { + init { + context("Burst loss") { + withNewConfig( + """ + prefix.uniform-rate=0 + prefix.burst-size=3 + prefix.burst-interval=100""" + ) { + val config = PacketLossConfig("prefix") + + config.uniformRate shouldBe 0 + config.burstSize shouldBe 3 + config.burstInterval shouldBe 100 + + val node = PacketLossNode(config) + + // The first interval starts at 100 + node.processPackets(List(99) { rtpPacket }) shouldHaveSize 99 + node.processPackets(List(3) { rtpPacket }) shouldHaveSize 0 + node.processPackets(List(97) { rtpPacket }) shouldHaveSize 97 + node.processPackets(List(3) { rtpPacket }) shouldHaveSize 0 + } + } + } +} + +private val rtpPacket = RtpPacket(audioPacketBytes, 0, audioPacketBytes.size) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/node/Utils.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/node/Utils.kt new file mode 100644 index 0000000..e3e9bdb --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/node/Utils.kt @@ -0,0 +1,38 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.Node +import org.jitsi.rtp.rtp.RtpPacket + +class CollectorNode : ConsumerNode("Collector") { + val result = mutableListOf() + override fun consume(packetInfo: PacketInfo) { + result.add(packetInfo.packetAs()) + } + + override fun trace(f: () -> Unit) {} +} + +fun Node.processPackets(packets: List): List { + val collector = CollectorNode() + attach(collector) + packets.forEach { processPacket(PacketInfo(it)) } + detachNext() + return collector.result +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt new file mode 100644 index 0000000..8c0bd09 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/logging/StdoutLogger.kt @@ -0,0 +1,143 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.resources.logging + +import org.jitsi.utils.logging2.LogContext +import org.jitsi.utils.logging2.Logger +import java.util.function.Supplier +import java.util.logging.Handler +import java.util.logging.Level + +/** + * Logger which logs all messages to stdout (but obeys a set log level). By default + * the log level is [Level.OFF], so nothing is logged. Child loggers inherit the current + * log level from the parent, but future changes to the parent logger's level don't affect + * children. + */ +class StdoutLogger( + private val name: String = "root", + private var _level: Level = Level.OFF, + context: Map = emptyMap() +) : Logger { + private val context = LogContext(context) + override fun createChildLogger(name: String): Logger = StdoutLogger(name, level) + override fun createChildLogger(name: String, context: MutableMap): Logger = + StdoutLogger(name, _level, context) + + override fun addContext(key: String, value: String) { + context.addContext(key, value) + } + + override fun addContext(addedContext: MutableMap) { + context.addContext(addedContext) + } + + override fun removeContext(key: String) { + context.removeContext(key) + } + + private fun isLoggable(level: Level): Boolean = level.intValue() >= _level.intValue() + + private fun log(level: Level, msg: Any) { + if (isLoggable(level)) { + println("$name: $context $msg") + } + } + private fun log(level: Level, msg: Any, throwable: Throwable) { + log(level, "$msg $throwable") + } + + override fun setLevelOff() { + setLevel(Level.OFF) + } + + override fun setLevelError() { + _level = Level.SEVERE + } + override fun error(msg: Any) { + log(Level.SEVERE, msg) + } + + override fun error(msgSupplier: Supplier) { + log(Level.SEVERE, msgSupplier) + } + override fun error(msg: Any, throwable: Throwable) { + log(Level.SEVERE, msg, throwable) + } + + override fun isWarnEnabled(): Boolean = isLoggable(Level.WARNING) + override fun setLevelWarn() { + setLevel(Level.WARNING) + } + override fun warn(msg: Any) { + log(Level.WARNING, msg) + } + + override fun warn(msgSupplier: Supplier) { + log(Level.WARNING, msgSupplier) + } + + override fun warn(msg: Any, throwable: Throwable) { + log(Level.WARNING, msg, throwable) + } + + override fun isInfoEnabled(): Boolean = isLoggable(Level.INFO) + override fun setLevelInfo() { + setLevel(Level.INFO) + } + override fun info(msg: Any) { + log(Level.INFO, msg) + } + override fun info(msgSupplier: Supplier) { + log(Level.INFO, msgSupplier) + } + + override fun isDebugEnabled(): Boolean = isLoggable(Level.FINE) + override fun setLevelDebug() { + setLevel(Level.FINE) + } + override fun debug(msg: Any) { + log(Level.FINE, msg) + } + override fun debug(msgSupplier: Supplier) { + log(Level.FINE, msgSupplier) + } + + override fun isTraceEnabled(): Boolean = isLoggable(Level.FINER) + override fun setLevelTrace() { + setLevel(Level.FINER) + } + override fun trace(msg: Any) { + log(Level.FINER, msg) + } + override fun trace(msgSupplier: Supplier) { + log(Level.FINER, msgSupplier) + } + + override fun setLevelAll() { + setLevel(Level.ALL) + } + override fun getLevel(): Level = _level + override fun setLevel(level: Level) { + _level = level + } + + /* These can be stubs */ + override fun setUseParentHandlers(useParentHandlers: Boolean) = Unit + override fun addHandler(handler: Handler?) = Unit + override fun removeHandler(handler: Handler?) = Unit +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/node/NodeTestExtensions.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/node/NodeTestExtensions.kt new file mode 100644 index 0000000..d1338ad --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/node/NodeTestExtensions.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.resources.node + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.Node + +internal fun Node.onOutput(func: (PacketInfo) -> Unit) { + this.attach(object : ConsumerNode("verifier") { + override fun consume(packetInfo: PacketInfo) { + func(packetInfo) + } + + override fun trace(f: () -> Unit) = f.invoke() + }) +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt new file mode 100644 index 0000000..49161fc --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/resources/srtp_samples/SrtpSample.kt @@ -0,0 +1,162 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.resources.srtp_samples + +import org.jitsi.nlj.srtp.SrtpProfileInformation +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.byteBufferOf + +/** + * Contains all the information needed for instantiating SRTP and SRTCP transformers + * and the following before and after packets: + * SRTCP -> RTCP + * RTCP -> RTCP + * SRTP -> RTP + * RTP -> RTCP + * which can be used to test encryption and decryption + */ +class SrtpSample { + companion object { + val srtpProfileInformation = SrtpProfileInformation( + cipherKeyLength = 16, + cipherSaltLength = 14, + cipherName = 1, + authFunctionName = 1, + authKeyLength = 20, + rtcpAuthTagLength = 10, + rtpAuthTagLength = 10 + ) + val keyingMaterial = byteBufferOf( + 0xB4, 0x04, 0x3B, 0x87, 0x67, 0xF6, 0xC4, 0x67, + 0xB2, 0x3E, 0xE1, 0xBE, 0x0C, 0xEB, 0x8E, 0x24, + 0xA0, 0x4F, 0xA4, 0x36, 0xC4, 0x17, 0x87, 0xF5, + 0xF5, 0x0C, 0xE4, 0x1A, 0x39, 0xFC, 0xB8, 0x21, + 0xDD, 0xC5, 0x60, 0x46, 0xCE, 0x69, 0x63, 0x55, + 0x8E, 0xF1, 0x9A, 0x35, 0x73, 0x0A, 0x4B, 0x69, + 0x17, 0x80, 0xD8, 0x96, 0x19, 0x85, 0xD0, 0xEF, + 0x32, 0x00, 0xCC, 0x27 + ) + + val tlsRole = TlsRole.CLIENT + + // Before and after packets + + // SRTP -> RTP + private val incomingEncryptedRtpData = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x90, 0xEF, 0x43, 0xD7, 0xCF, 0x6F, 0xDE, 0x8F, + 0x56, 0x29, 0x97, 0x7A, 0xBE, 0xDE, 0x00, 0x01, + 0x10, 0xFF, 0x00, 0x00, 0x40, 0xC9, 0xDC, 0x4E, + 0xDD, 0x95, 0x80, 0x41, 0x88, 0x8E, 0xFA, 0x32, + 0x1C, 0x42, 0xB4, 0x03, 0x8B, 0x2D, 0xE5, 0x79, + 0x61, 0xE2, 0x23, 0x7E, 0x17, 0x9C, 0x6E, 0xD7, + 0x6C, 0x6A, 0x11, 0x0D, 0x44, 0x91, 0x33, 0xBE, + 0xE1, 0xD7, 0x0D, 0x41, 0xE4, 0x8B + ) + val incomingEncryptedRtpPacket = RtpPacket(incomingEncryptedRtpData, 0, incomingEncryptedRtpData.size) + + val expectedDecryptedRtpPacket = RtpPacket( + org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x90, 0xEF, 0x43, 0xD7, 0xCF, 0x6F, 0xDE, 0x8F, + 0x56, 0x29, 0x97, 0x7A, 0xBE, 0xDE, 0x00, 0x01, + 0x10, 0xFF, 0x00, 0x00, 0x78, 0x0B, 0xE4, 0xC1, + 0x36, 0xEC, 0xC5, 0x8D, 0x8C, 0x49, 0x46, 0x99, + 0x04, 0xC5, 0xAA, 0xED, 0x92, 0xE7, 0x63, 0x4A, + 0x3A, 0x18, 0x98, 0xEE, 0x62, 0xCB, 0x60, 0xFF, + 0x6C, 0x1B, 0x29, 0x00 + ) + ) + + // SRTCP -> RTCP + // NOTE(brian): this ended up being a compound RTCP packet, + // so there are multiple in here + private val incomingEncryptedRtcpData = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x80, 0xC8, 0x00, 0x06, 0x75, 0x6D, 0x56, 0x40, + 0x0C, 0x24, 0x4E, 0x44, 0xF9, 0xE1, 0x4A, 0x5E, + 0x1C, 0xD1, 0xEA, 0x0B, 0xB3, 0xC5, 0x34, 0xD1, + 0xF7, 0x3D, 0x5F, 0x44, 0x2B, 0x9A, 0xD9, 0x04, + 0xB0, 0xC1, 0x48, 0xF6, 0x22, 0x5F, 0x2E, 0xFE, + 0x3A, 0xD9, 0x5B, 0xA5, 0x77, 0x52, 0xE0, 0xFD, + 0x8F, 0x46, 0x9C, 0x96, 0x29, 0xE9, 0x64, 0x1A, + 0x80, 0x00, 0x00, 0x01, 0x3C, 0xB4, 0xC8, 0xE6, + 0xB8, 0x19, 0xFB, 0xEE, 0xCE, 0xA2 + ) + val incomingEncryptedRtcpPacket = + UnparsedPacket(incomingEncryptedRtcpData, 0, incomingEncryptedRtcpData.size) + + val expectedDecryptedRtcpPacket = UnparsedPacket( + org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x80, 0xC8, 0x00, 0x06, 0x75, 0x6D, 0x56, 0x40, + 0xE0, 0x0E, 0x30, 0x93, 0x24, 0x77, 0x57, 0x4F, + 0xD3, 0x16, 0xD8, 0x0E, 0x00, 0x00, 0x00, 0x16, + 0x00, 0x00, 0x1F, 0x18, 0x81, 0xCA, 0x00, 0x06, + 0x75, 0x6D, 0x56, 0x40, 0x01, 0x10, 0x4C, 0x51, + 0x47, 0x4C, 0x32, 0x35, 0x4E, 0x76, 0x33, 0x35, + 0x30, 0x57, 0x7A, 0x38, 0x72, 0x7A, 0x00, 0x00 + ) + ) + + // RTP -> SRTP + private val outgoingUnencryptedRtpData = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x90, 0xEF, 0x36, 0xD6, 0x6C, 0x25, 0xF2, 0x81, + 0xCC, 0x8E, 0x67, 0xDB, 0xBE, 0xDE, 0x00, 0x03, + 0x10, 0xFF, 0x32, 0x88, 0xB7, 0xBD, 0x51, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x78, 0x0B, 0xE4, 0xC1, + 0x36, 0xEC, 0xC5, 0x8D, 0x8C, 0x49, 0x46, 0x99, + 0x04, 0xC5, 0xAA, 0xED, 0x92, 0xE7, 0x63, 0x4A, + 0x3A, 0x18, 0x98, 0xEE, 0x62, 0xCB, 0x60, 0xFF, + 0x6C, 0x1B, 0x29, 0x00 + ) + val outgoingUnencryptedRtpPacket = + RtpPacket(outgoingUnencryptedRtpData, 0, outgoingUnencryptedRtpData.size) + + val expectedEncryptedRtpData = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x90, 0xEF, 0x36, 0xD6, 0x6C, 0x25, 0xF2, 0x81, + 0xCC, 0x8E, 0x67, 0xDB, 0xBE, 0xDE, 0x00, 0x03, + 0x10, 0xFF, 0x32, 0x88, 0xB7, 0xBD, 0x51, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x1C, 0x6D, 0xA8, 0xEC, + 0x3D, 0x3D, 0xE5, 0x70, 0x2B, 0x2E, 0xB7, 0x94, + 0x9D, 0x4E, 0x2F, 0x4C, 0x9D, 0xF1, 0x3A, 0x68, + 0x2E, 0x3D, 0xC9, 0xA3, 0xA5, 0xCE, 0x76, 0x3F, + 0xEA, 0x82, 0xBD, 0xD0, 0xB9, 0x39, 0x02, 0x5E, + 0x32, 0x11, 0xD6, 0x70, 0x13, 0x81 + ) + + val expectedEncryptedRtpPacket = RtpPacket(expectedEncryptedRtpData, 0, expectedEncryptedRtpData.size) + + // RTCP -> SRTCP + private val outgoingUnencryptedRtcpData = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x8F, 0xCD, 0x00, 0x05, 0xE7, 0x46, 0x52, 0x23, + 0x56, 0x29, 0x97, 0x7A, 0x00, 0x01, 0x00, 0x01, + 0x95, 0xBE, 0x64, 0x00, 0xD0, 0x00, 0x00, 0x00 + ) + val outgoingUnencryptedRtcpPacket = + RtcpPacket.parse(outgoingUnencryptedRtcpData, 0, outgoingUnencryptedRtcpData.size) + + val expectedEncryptedRtcpPacket = UnparsedPacket( + org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x8F, 0xCD, 0x00, 0x05, 0xE7, 0x46, 0x52, 0x23, + 0x05, 0x67, 0x3D, 0xC0, 0x6C, 0x35, 0xF4, 0x40, + 0xE2, 0x2A, 0x7E, 0xCD, 0x99, 0x06, 0x5D, 0x34, + 0x80, 0x00, 0x00, 0x00, 0x48, 0x9E, 0x8E, 0xD1, + 0x8F, 0x2B, 0x76, 0xCC, 0xD7, 0x1F + ) + ) + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/CompoundRtcpParserTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/CompoundRtcpParserTest.kt new file mode 100644 index 0000000..8d956b4 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/CompoundRtcpParserTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.resources.node.onOutput +import org.jitsi.rtp.Packet +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtcp.RtcpByePacket +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.UnsupportedRtcpPacket + +class CompoundRtcpParserTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + private val logger = StdoutLogger() + + private val parser = CompoundRtcpParser(logger) + + private val invalidRtcpData = byteArrayOf( + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + + private val unsupportedRtcpData = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + // V=2, PT=195, length = 2 + 0x80, 0xC3, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + + private val rtcpByeNoReason = RtcpHeaderBuilder( + packetType = RtcpByePacket.PT, + reportCount = 1, + senderSsrc = 12345L, + length = 1 + ).build() + + init { + context("A compound RTCP packet with invalid data") { + context("at the beginning of the packet") { + val packet = DummyPacket(invalidRtcpData) + val packetInfo = PacketInfo(packet) + should("drop the packet entirely") { + parser.onOutput { it shouldBe null } + parser.processPacket(packetInfo) + } + } + context("after a valid packet") { + val packet = DummyPacket(rtcpByeNoReason + invalidRtcpData) + val packetInfo = PacketInfo(packet) + should("drop the packet entirely") { + parser.onOutput { it shouldBe null } + parser.processPacket(packetInfo) + } + } + } + context("A compound RTCP packet with an unsupported RTCP type") { + context("at the beginning of the packet") { + val packet = DummyPacket(unsupportedRtcpData) + val packetInfo = PacketInfo(packet) + should("parse it as UnsupportedRtcpPacket") { + parser.onOutput { + it.packet.shouldBeInstanceOf() + val compoundPacket = it.packetAs() + compoundPacket.packets shouldHaveSize 1 + compoundPacket.packets[0].shouldBeInstanceOf() + } + parser.processPacket(packetInfo) + } + } + context("in between other supported RTCP packets") { + val packet = DummyPacket(rtcpByeNoReason + unsupportedRtcpData + rtcpByeNoReason) + val packetInfo = PacketInfo(packet) + should("parse it as UnsupportedRtcpPacket") { + parser.onOutput { + it.packet.shouldBeInstanceOf() + val compoundPacket = it.packetAs() + compoundPacket.packets shouldHaveSize 3 + compoundPacket.packets[0].shouldBeInstanceOf() + compoundPacket.packets[1].shouldBeInstanceOf() + compoundPacket.packets[2].shouldBeInstanceOf() + } + parser.processPacket(packetInfo) + } + } + } + } + + private class DummyPacket( + buf: ByteArray + ) : Packet(buf, 0, buf.size) { + override fun clone(): Packet = DummyPacket(cloneBuffer(0)) + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/KeyframeRequesterTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/KeyframeRequesterTest.kt new file mode 100644 index 0000000..49068f2 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/KeyframeRequesterTest.kt @@ -0,0 +1,217 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.resources.node.onOutput +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.util.ExtmapAllowMixedChangedHandler +import org.jitsi.nlj.util.ReadOnlyStreamInformationStore +import org.jitsi.nlj.util.RtpExtensionHandler +import org.jitsi.nlj.util.RtpPayloadTypesChangedHandler +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock + +class KeyframeRequesterTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val streamInformationStore = object : ReadOnlyStreamInformationStore { + override val rtpExtensions: List = mutableListOf() + override val rtpPayloadTypes: Map = mutableMapOf() + override var supportsFir: Boolean = true + override var supportsPli: Boolean = true + override val supportsRemb: Boolean = true + override val supportsTcc: Boolean = true + override fun onRtpExtensionMapping(rtpExtensionType: RtpExtensionType, handler: RtpExtensionHandler) { + // no-op + } + override fun onRtpPayloadTypesChanged(handler: RtpPayloadTypesChangedHandler) { + // no-op + } + + override val extmapAllowMixed: Boolean = false + override fun onExtmapAllowMixedChanged(handler: ExtmapAllowMixedChangedHandler) { + // no-op + } + + override val primaryMediaSsrcs: Set = setOf(123L, 456L, 789L) + override val receiveSsrcs: Set = setOf(123L, 456L, 789L, 321L, 654L) + + override fun getLocalPrimarySsrc(secondarySsrc: Long): Long? = null + + override fun getRemoteSecondarySsrc(primarySsrc: Long, associationType: SsrcAssociationType): Long? = null + + override fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode() + } + private val logger = StdoutLogger() + private val clock: FakeClock = FakeClock() + + private val keyframeRequester = KeyframeRequester(streamInformationStore, logger, clock) + private val sentKeyframeRequests = mutableListOf() + + init { + keyframeRequester.onOutput { sentKeyframeRequests.add(it) } + + context("requesting a keyframe") { + context("without a specific SSRC") { + keyframeRequester.requestKeyframe("ep1") + should("result in a sent PLI request with the first video SSRC") { + sentKeyframeRequests shouldHaveSize 1 + val packet = sentKeyframeRequests.last().packet + packet.shouldBeInstanceOf() + packet.mediaSourceSsrc shouldBe 123L + } + } + context("when PLI is supported") { + keyframeRequester.requestKeyframe("ep1", 123L) + should("result in a sent PLI request") { + sentKeyframeRequests shouldHaveSize 1 + val packet = sentKeyframeRequests.last().packet + packet.shouldBeInstanceOf() + packet.mediaSourceSsrc shouldBe 123L + } + context("and then requesting again") { + sentKeyframeRequests.clear() + context("within the wait interval") { + clock.elapse(10.ms) + context("on the same SSRC") { + keyframeRequester.requestKeyframe("ep1", 123L) + should("not send anything") { + sentKeyframeRequests.shouldBeEmpty() + } + } + context("for a different SSRC") { + keyframeRequester.requestKeyframe("ep1", 456L) + should("result in a sent PLI request") { + sentKeyframeRequests shouldHaveSize 1 + val packet = sentKeyframeRequests.last().packet + packet.shouldBeInstanceOf() + packet.mediaSourceSsrc shouldBe 456L + } + } + } + context("after the wait and source-wide intervals have expired") { + clock.elapse(3.secs) + keyframeRequester.requestKeyframe("ep1", 123L) + should("result in a sent PLI request") { + sentKeyframeRequests shouldHaveSize 1 + val packet = sentKeyframeRequests.last().packet + packet.shouldBeInstanceOf() + packet.mediaSourceSsrc shouldBe 123L + } + } + } + } + context("when PLI isn't supported") { + streamInformationStore.supportsPli = false + keyframeRequester.requestKeyframe("ep1", 123L) + should("result in a sent FIR request") { + sentKeyframeRequests shouldHaveSize 1 + val packet = sentKeyframeRequests.last().packet + packet.shouldBeInstanceOf() + packet.mediaSenderSsrc shouldBe 123L + } + } + context("when neither PLI nor FIR is supported") { + streamInformationStore.supportsFir = false + streamInformationStore.supportsPli = false + keyframeRequester.requestKeyframe("ep1", 123L) + should("not send anything") { + sentKeyframeRequests.shouldBeEmpty() + } + } + } + + context("requesting a keyframe with no requester id") { + context("repeatedly") { + repeat(4) { keyframeRequester.requestKeyframe(null, 123L) } + should("still be limited by the source-wide limit") { + sentKeyframeRequests shouldHaveSize 1 + } + } + context("from more than one requester for the same source") { + keyframeRequester.requestKeyframe("ep1", 123L) + keyframeRequester.requestKeyframe(null, 123L) + should("share the source-wide limit with attributed requests") { + sentKeyframeRequests shouldHaveSize 1 + } + } + context("after the source-wide interval has expired") { + keyframeRequester.requestKeyframe(null, 123L) + clock.elapse(3.secs) + keyframeRequester.requestKeyframe(null, 123L) + should("be allowed again") { + sentKeyframeRequests shouldHaveSize 2 + } + } + } + + context("requests dropped by the source-wide limit") { + // ep1 opens the source-wide limit's 2s min-interval. ep2 arrives just after and, like a receiver waiting + // for a keyframe, keeps re-requesting as often as its own 200ms min-interval allows, using up its 3 + // requests per 10s long before the source-wide limit reopens. + keyframeRequester.requestKeyframe("ep1", 123L) + repeat(3) { + keyframeRequester.requestKeyframe("ep2", 123L) + clock.elapse(200.ms) + } + should("not have been sent") { + sentKeyframeRequests shouldHaveSize 1 + } + context("and the source-wide limit reopens") { + clock.elapse(2.secs) + keyframeRequester.requestKeyframe("ep2", 123L) + should("not count against the requester's per-receiver limit") { + sentKeyframeRequests shouldHaveSize 2 + } + } + } + + context("requests dropped by the per-receiver limit") { + keyframeRequester.requestKeyframe("ep1", 123L) + // Within ep1's 200ms per-receiver min-interval, so dropped there, before the source-wide limit. + clock.elapse(100.ms) + keyframeRequester.requestKeyframe("ep1", 123L) + should("not have been sent") { + sentKeyframeRequests shouldHaveSize 1 + } + context("when another receiver requests once the source-wide interval has expired") { + clock.elapse(3.secs) + keyframeRequester.requestKeyframe("ep2", 123L) + should("not have counted against the source-wide limit") { + sentKeyframeRequests shouldHaveSize 2 + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt new file mode 100644 index 0000000..9931505 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtcp/StreamPacketRequesterTest.kt @@ -0,0 +1,147 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtcp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacket +import org.jitsi.utils.concurrent.FakeScheduledExecutorService + +class StreamPacketRequesterTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val scheduler = FakeScheduledExecutorService() + + private val nackPacketsSent = mutableListOf() + private fun rtcpSender(rtcpPacket: RtcpPacket) { + nackPacketsSent.add(rtcpPacket) + } + + private val streamPacketRequester = RetransmissionRequester.StreamPacketRequester( + 123L, + scheduler, + scheduler.clock, + ::rtcpSender, + StdoutLogger() + ) + + init { + context("receiving the first packet") { + streamPacketRequester.packetReceived(1) + should("not schedule any work") { + scheduler.numPendingJobs() shouldBe 0 + } + context("and then a packet sequentially after that") { + streamPacketRequester.packetReceived(2) + should("not schedule any work") { + scheduler.numPendingJobs() shouldBe 0 + } + } + context("and then a packet with a gap") { + streamPacketRequester.packetReceived(3) + should("schedule work") { + scheduler.numPendingJobs() shouldBe 1 + } + context("the scheduled work") { + scheduler.runOne() + should("send a nack") { + nackPacketsSent shouldHaveSize 1 + val nackPacket = nackPacketsSent.first() as RtcpFbNackPacket + nackPacket.missingSeqNums shouldContainExactly listOf(2) + } + } + } + context("and then receiving it again") { + streamPacketRequester.packetReceived(1) + should("not schedule any work") { + scheduler.numPendingJobs() shouldBe 0 + } + } + } + context("a nack for a specific packet") { + streamPacketRequester.packetReceived(1) + streamPacketRequester.packetReceived(3) + should("be resent 10 times") { + repeat(10) { scheduler.runOne() } + nackPacketsSent shouldHaveSize 10 + // It shouldn't reschedule the job after the 10th time + scheduler.numPendingJobs() shouldBe 0 + } + should("not be sent after the packet has been received") { + // Have it send a few nacks + repeat(3) { scheduler.runOne() } + // Now have the missing packet come in + streamPacketRequester.packetReceived(2) + scheduler.numPendingJobs() shouldBe 0 + } + } + context("multiple missing packets") { + context("all at once") { + streamPacketRequester.packetReceived(1) + streamPacketRequester.packetReceived(10) + should("all be nacked at once") { + scheduler.runOne() + nackPacketsSent shouldHaveSize 1 + val nackPacket = nackPacketsSent.first() as RtcpFbNackPacket + nackPacket.missingSeqNums shouldContainExactly (2..9).toSortedSet() + } + should("all be nacked 10 times") { + repeat(10) { scheduler.runOne() } + nackPacketsSent shouldHaveSize 10 + nackPacketsSent.forEach { packet -> + packet as RtcpFbNackPacket + packet.missingSeqNums shouldContainExactly (2..9).toSortedSet() + } + scheduler.numPendingJobs() shouldBe 0 + } + } + context("spread out over time") { + streamPacketRequester.packetReceived(1) + streamPacketRequester.packetReceived(3) + repeat(5) { scheduler.runOne() } + streamPacketRequester.packetReceived(5) + repeat(5) { scheduler.runOne() } + streamPacketRequester.packetReceived(7) + repeat(10) { scheduler.runOne() } + + should("all be nacked 10 times each over time") { + for (missingSeqNum in listOf(2, 4, 6)) { + val numNacksForSeqNum = nackPacketsSent.map { it as RtcpFbNackPacket } + .filter { it.missingSeqNums.contains(missingSeqNum) } + .size + numNacksForSeqNum shouldBe 10 + } + scheduler.numPendingJobs() shouldBe 0 + } + } + } + context("a packet jump of larger than the max") { + streamPacketRequester.packetReceived(1) + streamPacketRequester.packetReceived(3) + should("reset and clear all pending nacks") { + scheduler.numPendingJobs() shouldBe 1 + streamPacketRequester.packetReceived(3000) + scheduler.numPendingJobs() shouldBe 0 + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt new file mode 100644 index 0000000..cbded5f --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/AudioRedHandlerTest.kt @@ -0,0 +1,272 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.Spec +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeTypeOf +import org.jitsi.config.withNewConfig +import org.jitsi.metaconfig.MetaconfigSettings +import org.jitsi.nlj.format.AudioRedPayloadType +import org.jitsi.nlj.node.processPackets +import org.jitsi.nlj.transform.node.AudioRedHandler +import org.jitsi.nlj.transform.node.RedPolicy +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.createLogger + +/** + * Tests the handling and generation of RED packets with two simple streams. One is a stream of Opus packets with + * sequence numbers 1, 2, 3, 5, 6 (packet 4 is lost). The other is a stream of RED packets with the same sequence + * numbers and redundancy with distance 2 whenever available. + */ +class AudioRedHandlerTest : ShouldSpec() { + val logger = createLogger() + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + override suspend fun beforeSpec(spec: Spec) { + super.beforeSpec(spec) + MetaconfigSettings.cacheEnabled = false + } + + private val streamInformationStore = StreamInformationStoreImpl() + + init { + context("Policy STRIP") { + withNewConfig("jmt.audio.red.policy=STRIP") { + val redHandler = AudioRedHandler(streamInformationStore, logger) + AudioRedHandler.config.policy shouldBe RedPolicy.STRIP + + context("Target supports RED") { + streamInformationStore.addRtpPayloadType(AudioRedPayloadType(112)) + redHandler.redPayloadType shouldBe 112 + + context("Receiving RED") { + val result = redHandler.processRedPackets() + + // We should strip RED from the stream and recover the lost packet + result.shouldBeTheExpectedOpusStream(packet4isPresent = true) + } + context("Receiving Opus") { + val result = redHandler.processOpusPackets() + + // We should forward the Opus packets without encapsulation, the lost packet is not recovered + result.shouldBeTheExpectedOpusStream(packet4isPresent = false) + } + } + context("Target does not support RED") { + redHandler.redPayloadType shouldBe null + + context("Receiving RED") { + val result = redHandler.processRedPackets() + + // We should strip RED from the stream and recover the lost packet + result.shouldBeTheExpectedOpusStream(packet4isPresent = true) + } + context("Receiving Opus") { + val result = redHandler.processOpusPackets() + + // We should forward the Opus packets without encapsulation, the lost packet is not recovered + result.shouldBeTheExpectedOpusStream(packet4isPresent = false) + } + } + } + } + context("Policy PROTECT_ALL") { + withNewConfig( + """ + jmt.audio.red.policy=PROTECT_ALL + # TODO: add tests for the VAD-only case too. + jmt.audio.red.vad-only=false + """.trimIndent() + ) { + val redHandler = AudioRedHandler(streamInformationStore, logger) + AudioRedHandler.config.policy shouldBe RedPolicy.PROTECT_ALL + + context("Target supports RED") { + streamInformationStore.addRtpPayloadType(AudioRedPayloadType(112)) + redHandler.redPayloadType shouldBe 112 + + context("Receiving RED") { + val result = redHandler.processRedPackets() + + // We should forward the RED without changes + result.shouldBeTheExpectedRedStream(packet4WasAvailable = true) + } + context("Receiving Opus") { + val result = redHandler.processOpusPackets() + + // We should encapsulate the Opus packets in RED. + result.shouldBeTheExpectedRedStream(packet4WasAvailable = false) + } + } + context("Target does not support RED") { + redHandler.redPayloadType shouldBe null + + context("Receiving RED") { + val result = redHandler.processRedPackets() + + // We should strip RED from the stream and recover the lost packet + result.shouldBeTheExpectedOpusStream(packet4isPresent = true) + } + context("Receiving Opus") { + val result = redHandler.processOpusPackets() + + // We should forward the Opus packets without encapsulation, the lost packet is not recovered + result.shouldBeTheExpectedOpusStream(packet4isPresent = false) + } + } + } + } + } + + /** + * Process the sample stream of RED packets. + */ + private fun AudioRedHandler.processRedPackets() = processPackets( + listOf( + redPackets[0], + redPackets[1], + redPackets[2], + redPackets[3], + // 4 is lost + redPackets[5], + redPackets[6] + ) + ) + + /** + * Process the sample stream of Opus packets. + */ + private fun AudioRedHandler.processOpusPackets() = processPackets( + listOf( + audioPackets[0], + audioPackets[1], + audioPackets[2], + audioPackets[3], + // 4 is lost + audioPackets[5], + audioPackets[6] + ) + ) + + private val redPt = 112 + + private fun createAudioPacket(seq: Int) = AudioRtpPacket(audioPacketBytes, 0, audioPacketBytes.size).apply { + sequenceNumber = seq + timestamp = seq * 960.toLong() + // Encode the SEQ in the payload + buffer[payloadOffset] = seq.toByte() + } + + /** + * Get the ID encoded in the payload of the audio packets we create (since the sequence number might be lost) + */ + private fun RtpPacket.getPacketId(): Int = buffer[payloadOffset].toInt() + + private val audioPackets = List(7) { i -> createAudioPacket(i) } + private val redPackets = List(7) { i -> + when (i) { + 0 -> RedAudioRtpPacket.builder.build(redPt, audioPackets[0].clone(), emptyList()) + + 1 -> RedAudioRtpPacket.builder.build(redPt, audioPackets[1].clone(), listOf(audioPackets[0])) + + else -> RedAudioRtpPacket.builder.build( + redPt, + audioPackets[i].clone(), + listOf(audioPackets[i - 2], audioPackets[i - 1]) + ) + } + } + + private fun List.shouldBeTheExpectedRedStream( + /** + * Whether the packet with sequence number 4 was available when the RED stream was encoded (it would have been + * available to the original encoder, but not to the SFU receiving a partial opus stream). + */ + packet4WasAvailable: Boolean + ) { + size shouldBe 6 + map { it.sequenceNumber }.toList() shouldBe listOf(0, 1, 2, 3, 5, 6) // 4 is lost + forEach { + it.payloadType shouldBe 112 + it.shouldBeTypeOf() + + val parsedRedundancy = it.removeRedAndGetRedundancyPackets() + when (it.sequenceNumber) { + 0 -> parsedRedundancy.size shouldBe 0 + + 1 -> { + parsedRedundancy.size shouldBe 1 + parsedRedundancy[0].sequenceNumber shouldBe 0 + parsedRedundancy[0].getPacketId() shouldBe 0 + } + + 2, 3 -> { + parsedRedundancy.size shouldBe 2 + parsedRedundancy[0].sequenceNumber shouldBe it.sequenceNumber - 2 + parsedRedundancy[0].getPacketId() shouldBe it.sequenceNumber - 2 + parsedRedundancy[1].sequenceNumber shouldBe it.sequenceNumber - 1 + parsedRedundancy[1].getPacketId() shouldBe it.sequenceNumber - 1 + } + + 5 -> { + if (packet4WasAvailable) { + parsedRedundancy.size shouldBe 2 + parsedRedundancy[0].sequenceNumber shouldBe 3 + parsedRedundancy[0].getPacketId() shouldBe 3 + parsedRedundancy[1].sequenceNumber shouldBe 4 + parsedRedundancy[1].getPacketId() shouldBe 4 + } else { + // When 4 was unavailable the encoder should have produced no redundancy. + parsedRedundancy.size shouldBe 0 + } + } + + 6 -> { + if (packet4WasAvailable) { + parsedRedundancy.size shouldBe 2 + parsedRedundancy[0].sequenceNumber shouldBe 4 + parsedRedundancy[0].getPacketId() shouldBe 4 + parsedRedundancy[1].sequenceNumber shouldBe 5 + parsedRedundancy[1].getPacketId() shouldBe 5 + } else { + parsedRedundancy.size shouldBe 1 + parsedRedundancy[0].sequenceNumber shouldBe 5 + parsedRedundancy[0].getPacketId() shouldBe 5 + } + } + } + } + } + + private fun List.shouldBeTheExpectedOpusStream( + /** + * Whether the packet with sequence number 4 should be present in the stream. + */ + packet4isPresent: Boolean + ) { + size shouldBe if (packet4isPresent) 7 else 6 + forEach { + it.payloadType shouldBe 111 + it.shouldBeTypeOf() + } + + val expectedSequenceNumbers = if (packet4isPresent) listOf(0, 1, 2, 3, 4, 5, 6) else listOf(0, 1, 2, 3, 5, 6) + map { it.sequenceNumber }.toList() shouldBe expectedSequenceNumbers + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacketTest.kt new file mode 100644 index 0000000..de8b8ea --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/PaddingVideoPacketTest.kt @@ -0,0 +1,57 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.util.BufferPool +import org.jitsi.rtp.rtp.RtpHeader + +class PaddingVideoPacketTest : ShouldSpec() { + + init { + context("Creating a padding packet") { + context("from a buffer that previously held other data") { + BufferPool.getBuffer = { + // A buffer with bogus CSRC count and header ext length values + org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0xF7, 0xF8, 0x04, 0x54, + 0x47, 0x35, 0x08, 0x10, + 0x25, 0xA2, 0x71, 0x9C, + 0x23, 0x9F, 0xCA, 0x3D + ) + ByteArray(1484) { 0 } + } + val paddingPacket = PaddingVideoPacket.create(267) + should("have the right header length") { + paddingPacket.headerLength shouldBe 12 + } + should("have the right payload length") { + paddingPacket.payloadLength shouldBe (267 - paddingPacket.headerLength) + } + should("have the padding bit set") { + paddingPacket.hasPadding shouldBe true + } + should("have the padding value set") { + paddingPacket.paddingSize shouldBe paddingPacket.payloadLength + } + should("have the version set") { + paddingPacket.version shouldBe RtpHeader.VERSION + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/RtpExtensionTypeTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/RtpExtensionTypeTest.kt new file mode 100644 index 0000000..4cb8e45 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/RtpExtensionTypeTest.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class RtpExtensionTypeTest : ShouldSpec() { + + init { + context("Creating an extension from a valid URI") { + should("work correctly") { + RtpExtensionType.createFromUri(RtpExtensionType.TRANSPORT_CC.uri) shouldBe + RtpExtensionType.TRANSPORT_CC + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/RtxPacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/RtxPacketTest.kt new file mode 100644 index 0000000..a0e12bb --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/RtxPacketTest.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.rtp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.test_utils.matchers.haveSameContentAs +import org.jitsi.rtp.rtp.RtpPacket +import java.nio.ByteBuffer + +class RtxPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + // OSN = 57005 + val osn = org.jitsi.rtp.extensions.bytearray.byteArrayOf(0xDE, 0xAD) + + val payload = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x01, 0x02, 0x03, 0x04, + 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C + ) + + val header = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + // Dummy header data + // V=2,P=false,X=false,CC=0,M=false,PT=111,SeqNum=5807 + 0x80, 0x6f, 0x16, 0xaf, + // Timestamp: 1710483662 + 0x65, 0xf3, 0xe8, 0xce, + // SSRC: 839852602 + 0x32, 0x0f, 0x22, 0x3a + ) + + val rtxPacket = RtpPacket(header + osn + payload) + val rtpPacket = RtpPacket(header + payload) + + init { + context("Getting the original sequence number") { + should("work correctly") { + RtxPacket.getOriginalSequenceNumber(rtxPacket) shouldBe 57005 + } + } + context("Removing the original sequence number") { + should("work correctly") { + RtxPacket.removeOriginalSequenceNumber(rtxPacket) + val newPayload = + ByteBuffer.wrap(rtxPacket.buffer, rtxPacket.payloadOffset, rtxPacket.payloadLength).slice() + newPayload should haveSameContentAs(ByteBuffer.wrap(payload)) + } + } + context("Adding an original sequence number") { + RtxPacket.addOriginalSequenceNumber(rtpPacket) + should("work correctly") { + RtxPacket.getOriginalSequenceNumber(rtpPacket) shouldBe 5807 + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt new file mode 100644 index 0000000..fb90f67 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/TransportCcEngineTest.kt @@ -0,0 +1,168 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.mockk.mockk +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.instantOfEpochMicro +import org.jitsi.utils.time.FakeClock +import java.util.logging.Level + +class TransportCcEngineTest : FunSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val bandwidthEstimator: BandwidthEstimator = mockk(relaxed = true) + private val clock: FakeClock = FakeClock() + private val logger = StdoutLogger(_level = Level.INFO) + + private val lossListener = object : LossListener { + var numReceived = 0 + var numLost = 0 + + override fun packetReceived(previouslyReportedLost: Boolean) { + numReceived++ + if (previouslyReportedLost) { + numLost-- + } + } + + override fun packetLost(numLost: Int) { + this.numLost += numLost + } + } + + private val transportCcEngine = ClassicTransportCcEngine(bandwidthEstimator, logger, clock).also { + it.addLossListener(lossListener) + } + + private fun makePacket(length: DataSize): PacketInfo { + val len = length.bytes.toInt() + val buf = BufferPool.getBuffer(len) + buf.fill(0, 0, len) + + val packet = RtpPacket(buf, 0, len).apply { + version = 2 + } + return PacketInfo(packet) + } + + init { + test("Missing packet reports") { + transportCcEngine.mediaPacketSent(makePacket(1300.bytes), 4) + + val tccPacket = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 0)) { + SetBase(1, instantOfEpochMicro(100)) + AddReceivedPacket(1, instantOfEpochMicro(100)) + AddReceivedPacket(2, instantOfEpochMicro(110)) + AddReceivedPacket(3, instantOfEpochMicro(120)) + AddReceivedPacket(4, instantOfEpochMicro(130)) + build() + } + + transportCcEngine.rtcpPacketReceived(tccPacket, clock.instant()) + + with(transportCcEngine.getStatistics()) { + numMissingPacketReports shouldBe 3 + numDuplicateReports shouldBe 0 + numPacketsReportedAfterLost shouldBe 0 + numPacketsUnreported shouldBe 0 + } + lossListener.numReceived shouldBe 1 + lossListener.numLost shouldBe 0 + } + test("Duplicate packet reports") { + transportCcEngine.mediaPacketSent(makePacket(1300.bytes), 4) + + val tccPacket = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 1)) { + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(4, instantOfEpochMicro(130)) + build() + } + transportCcEngine.rtcpPacketReceived(tccPacket, clock.instant()) + + val tccPacket2 = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 2)) { + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(4, instantOfEpochMicro(130)) + build() + } + transportCcEngine.rtcpPacketReceived(tccPacket2, clock.instant()) + + with(transportCcEngine.getStatistics()) { + numMissingPacketReports shouldBe 0 + numDuplicateReports shouldBe 1 + numPacketsReportedAfterLost shouldBe 0 + numPacketsUnreported shouldBe 0 + } + lossListener.numReceived shouldBe 1 + lossListener.numLost shouldBe 0 + } + test("Packets reported after lost") { + transportCcEngine.mediaPacketSent(makePacket(1300.bytes), 4) + transportCcEngine.mediaPacketSent(makePacket(1300.bytes), 5) + + val tccPacket = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 1)) { + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(5, instantOfEpochMicro(130)) + build() + } + transportCcEngine.rtcpPacketReceived(tccPacket, clock.instant()) + + lossListener.numReceived shouldBe 1 + lossListener.numLost shouldBe 1 + + val tccPacket2 = with(RtcpFbTccPacketBuilder(mediaSourceSsrc = 123, feedbackPacketSeqNum = 2)) { + SetBase(4, instantOfEpochMicro(130)) + AddReceivedPacket(4, instantOfEpochMicro(130)) + build() + } + + transportCcEngine.rtcpPacketReceived(tccPacket2, clock.instant()) + + with(transportCcEngine.getStatistics()) { + numMissingPacketReports shouldBe 0 + numDuplicateReports shouldBe 0 + numPacketsReportedAfterLost shouldBe 1 + numPacketsUnreported shouldBe 0 + } + lossListener.numReceived shouldBe 2 + lossListener.numLost shouldBe 0 + } + test("Packet unreported") { + transportCcEngine.mediaPacketSent(makePacket(1300.bytes), 4) + /* Force the report of sequence 4 to be evicted from the packet history */ + transportCcEngine.mediaPacketSent(makePacket(1300.bytes), 1004) + + with(transportCcEngine.getStatistics()) { + numMissingPacketReports shouldBe 0 + numDuplicateReports shouldBe 0 + numPacketsReportedAfterLost shouldBe 0 + numPacketsUnreported shouldBe 1 + } + lossListener.numReceived shouldBe 0 + lossListener.numLost shouldBe 0 + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/Utils.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/Utils.kt new file mode 100644 index 0000000..2a5de55 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/Utils.kt @@ -0,0 +1,36 @@ +package org.jitsi.nlj.rtp + +private val audioPacketBytes_ = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + // RTP Header. PT=111, seq=0x7b2b + 0x90, 0x6f, 0x7b, 0x2b, + // TS + 0x44, 0xb2, 0x83, 0x6e, + // SSRC + 0x16, 0x49, 0x3f, 0x2d, + // Extension + 0xbe, 0xde, 0x00, 0x01, + // ID=1, value=0xbd (VAD=true) + 0x10, 0xbd, 0x00, 0x00, + + // RTP Payload (length = 65) + 0x68, 0x2f, 0x3b, 0x25, + 0xab, 0x1a, 0x72, 0xfb, + 0x75, 0x2d, 0xf0, 0x9b, + 0xa2, 0xa3, 0xfc, 0x20, + 0x51, 0xf7, 0x9c, 0xe8, + 0x75, 0xe9, 0x02, 0xd6, + 0xc1, 0xe4, 0xa1, 0x51, + 0x02, 0x00, 0x59, 0x55, + 0x04, 0x56, 0x5e, 0xed, + 0x31, 0x55, 0xb5, 0x04, + 0x9d, 0xf6, 0x1c, 0x40, + 0x7b, 0xb7, 0x00, 0x0c, + 0xd9, 0x7b, 0x5d, 0x13, + 0x4c, 0xeb, 0x7d, 0xf1, + 0x74, 0xf8, 0xd5, 0xb9, + 0x07, 0xda, 0x18, 0x19, + 0x92 +) + +val audioPacketBytes + get() = audioPacketBytes_.clone() diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/.gitignore b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/.gitignore new file mode 100644 index 0000000..e26c5a5 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/.gitignore @@ -0,0 +1 @@ +bwe-output.js diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimationTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimationTest.kt new file mode 100644 index 0000000..99d2385 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/BandwidthEstimationTest.kt @@ -0,0 +1,247 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.longs.shouldBeBetween +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.atRate +import org.jitsi.nlj.util.bits +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.mbps +import org.jitsi.utils.NEVER +import org.jitsi.utils.concurrent.FakeScheduledExecutorService +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.LoggerImpl +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.ArrayDeque +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import kotlin.properties.Delegates + +/** A simulated packet, for bandwidth estimation testing. */ +data class SimulatedPacket( + val sendTime: Instant, + val packetSize: DataSize, + val ssrc: Long +) + +abstract class FixedRateSender( + private val executor: ScheduledExecutorService, + val clock: Clock, + var receiver: (SimulatedPacket) -> Unit +) { + private var nextPacket: ScheduledFuture<*>? = null + private var lastSendTime: Instant = NEVER + + var running = false + + var rate: Bandwidth by Delegates.observable(0.bps) { _, _, _ -> + nextPacket?.cancel(false) + schedulePacket(false) + } + + abstract fun nextPacketSize(): DataSize + + fun schedulePacket(justSent: Boolean) { + if (!running || rate <= 0.bps || nextPacketSize().bits == 0L) { + nextPacket = null + } else { + val packetDelayTime = when (lastSendTime) { + NEVER -> Duration.ZERO + + else -> { + var delayTime = (nextPacketSize() atRate rate) + if (!justSent) { + delayTime -= Duration.between(lastSendTime, clock.instant()) + } + delayTime + } + } + + nextPacket = executor.schedule(::doSendPacket, packetDelayTime.toNanos(), TimeUnit.NANOSECONDS) + } + } + + private fun doSendPacket() { + val now = clock.instant() + val sendNext = sendPacket(now) + + if (sendNext) { + lastSendTime = now + schedulePacket(true) + } + } + + abstract fun sendPacket(now: Instant): Boolean + + fun start() { + running = true + doSendPacket() + } + + fun stop() { + running = false + nextPacket?.cancel(false) + } +} + +class PacketGenerator( + executor: ScheduledExecutorService, + clock: Clock, + receiver: (SimulatedPacket) -> Unit, + private var packetSize: DataSize = 1250.bytes, + private val ssrc: Long = 0xcafebabe +) : FixedRateSender(executor, clock, receiver) { + override fun nextPacketSize() = packetSize + + override fun sendPacket(now: Instant): Boolean { + val packet = SimulatedPacket(now, packetSize, ssrc) + receiver(packet) + return true + } +} + +class PacketBottleneck( + executor: ScheduledExecutorService, + clock: Clock, + private val ctx: DiagnosticContext, + receiver: (SimulatedPacket) -> Unit +) : FixedRateSender(executor, clock, receiver) { + private val timeSeriesLogger: TimeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass) + + private val queue = ArrayDeque() + + fun enqueue(packet: SimulatedPacket) { + if (!running) { + return + } + assert(packet.sendTime <= clock.instant()) + val queueWasEmpty = queue.isEmpty() + queue.addFirst(packet) + if (queueWasEmpty) { + schedulePacket(false) + } + } + + override fun sendPacket(now: Instant): Boolean { + if (queue.isEmpty()) { + return false + } + + val packet = queue.removeLast() + receiver(packet) + + if (timeSeriesLogger.isTraceEnabled) { + val delay = Duration.between(packet.sendTime, now) + timeSeriesLogger.trace( + ctx.makeTimeSeriesPoint("queue", now) + .addField("depth", queue.size) + .addField("delay", delay.toNanos() / 1e6) + ) + } + + return true + } + + override fun nextPacketSize(): DataSize = queue.peek()?.packetSize ?: 0.bits +} + +class PacketDelayer( + private val executor: ScheduledExecutorService, + private val delay: Duration, + val receiver: (SimulatedPacket) -> Unit +) { + fun enqueue(packet: SimulatedPacket) { + executor.schedule({ receiver(packet) }, delay.toNanos(), TimeUnit.NANOSECONDS) + } +} + +class PacketReceiver( + private val clock: Clock, + private val estimator: BandwidthEstimator, + private val ctx: DiagnosticContext, + val rateReceiver: (Bandwidth) -> Unit +) { + private val timeSeriesLogger: TimeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(this.javaClass) + private var seq = 0 + + fun receivePacket(packet: SimulatedPacket) { + val now = clock.instant() + assert(packet.sendTime <= now) + /* All delay is send -> receive in this simulation, so one-way delay is rtt. */ + estimator.onRttUpdate(now, Duration.between(packet.sendTime, now)) + estimator.processPacketArrival(now, packet.sendTime, now, seq, packet.packetSize) + estimator.feedbackComplete(now) + seq++ + val bw = estimator.getCurrentBw(now) + if (timeSeriesLogger.isTraceEnabled) { + timeSeriesLogger.trace(ctx.makeTimeSeriesPoint("bw", now).addField("bw", bw.bps)) + } + rateReceiver(bw) + } +} + +class BandwidthEstimationTest : ShouldSpec() { + private val scheduler = FakeScheduledExecutorService() + private val clock: Clock = scheduler.clock + + private val ctx = DiagnosticContext(clock) + init { + /* Emulate the fields that jitsi-videobridge puts in its DiagnosticContexts. */ + ctx["conf_name"] = "BandwidthEstimationTest" + ctx["conf_creation_time_ms"] = clock.instant().toEpochMilli() + ctx["endpoint_id"] = "00000000" + } + + private val logger = LoggerImpl(BandwidthEstimationTest::class.qualifiedName) + private val estimator: BandwidthEstimator = GoogleCcEstimator(ctx, logger) + + private val rtt: Duration = Duration.ofMillis(200) + private val bottleneckRate = 4.mbps + + private val generator: PacketGenerator = PacketGenerator(scheduler, clock, { bottleneck.enqueue(it) }) + private val bottleneck: PacketBottleneck = PacketBottleneck(scheduler, clock, ctx) { delayer.enqueue(it) } + private val delayer: PacketDelayer = PacketDelayer(scheduler, rtt) { receiver.receivePacket(it) } + private val receiver: PacketReceiver = PacketReceiver(clock, estimator, ctx) { generator.rate = it } + + init { + context("Running bandwidth estimation test") { + should("work correctly") { + bottleneck.rate = bottleneckRate + generator.rate = estimator.getCurrentBw(clock.instant()) + + bottleneck.start() + generator.start() + + scheduler.runUntil(clock.instant().plus(120, ChronoUnit.SECONDS)) + + generator.stop() + bottleneck.stop() + + val finalBw = estimator.getCurrentBw(clock.instant()) + finalBw.bps.shouldBeBetween((bottleneckRate / 1.2).bps, (bottleneckRate * 1.2).bps) + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/chart-bwe-output.html b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/chart-bwe-output.html new file mode 100644 index 0000000..9b4eb7a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/chart-bwe-output.html @@ -0,0 +1,15 @@ + + + + + + + + + + + + +
+ + diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/chart-bwe-output.js b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/chart-bwe-output.js new file mode 100644 index 0000000..f5a5f18 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/chart-bwe-output.js @@ -0,0 +1,61 @@ +// Load the Visualization API and the corechart package. +google.charts.load('current', {'packages':['corechart', 'scatter']}); + +// Set a callback to run when the Google Visualization API is loaded. +google.charts.setOnLoadCallback(drawCharts); + +// Callback that creates and populates a data table, +// instantiates the pie chart, passes in the data and +// draws it. +function drawChart(chart) { + // Create the data table. + var data = google.visualization.arrayToDataTable(chart.data); + + var options = { + width: 800, + height: 600, + title: chart.title, + + series: {}, + vAxes: {}, + + pointSize: 0, + lineWidth: 1, + interpolateNulls: true, + } + + var series = 0; + var msAxis; + if (chart.columns.bw) { + options.series["0"] = {targetAxisIndex: 0}; + options.vAxes["0"] = {title: 'Bandwidth (bps)', minValue:0}; + msAxis = 1; + series++; + } + else { + msAxis = 0; + } + if (chart.columns.rtt || chart.columns.delay) { + options.vAxes[msAxis] = {title: 'ms', minValue:0}; + } + if (chart.columns.rtt) { + options.series[series] = {targetAxisIndex: msAxis}; + series++; + } + if (chart.columns.delay) { + options.series[series] = {targetAxisIndex: msAxis}; + series++; + } + + var div = document.createElement("div"); + document.getElementById('chart_div').appendChild(div); + var chart = new google.visualization.ScatterChart(div); + chart.draw(data, options); +} + +function drawCharts() { + var chart; + for (chart in charts) { + drawChart(charts[chart]); + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/parse-bwe-timeseries.pl b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/parse-bwe-timeseries.pl new file mode 100755 index 0000000..d2597c3 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation/parse-bwe-timeseries.pl @@ -0,0 +1,209 @@ +#!/usr/bin/perl -w + +# Parse the bandwidth estimation timeseries log output, and convert +# to JSON in the format expected by chart-bwe-output.html. +# Timeseries logging should be enabled for the relevant subclass of +# org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimator, e.g. +# timeseries.org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator.level=ALL +# +# Output file should be bwe-output.js. + +use strict; +use Data::Dump 'dump'; + +my %endpoints; + +my $check_old_bwe = 0; + +sub parse_line($) +{ + my ($line) = @_; + + my %ret; + + if ($line !~ /^{/gc) { + return undef; + } + + while (1) { + if ($line !~ /\G"([A-Za-z0-9_]*)":/gc) { + return undef; + } + my $field = $1; + if ($line =~ /\G"((?:[^\\"]|\\.)*)"/gc) { + $ret{$field} = $1; + } + elsif ($line =~ /\G([^",}]*)/gc) { + $ret{$field} = $1; + } + if ($line !~ /\G,/gc) { + last; + } + } + + if ($line !~ /\G}/gc) { + return undef; + } + + return \%ret; +} + +sub get_field($$) +{ + my ($line, $field) = @_; + if ($line =~ /"$field":"((?:[^\\"]|\\.)*)"[,}]/) { + return $1; + } + elsif ($line =~ /"$field":([^",}]*)/) { + return $1; + } + return undef; +} + +sub get_ep_key($) +{ + my ($line) = @_; + + my $conf_name = $line->{conf_name}; + my $ep_id = $line->{endpoint_id}; + my $conf_time = $line->{conf_creation_time_ms}; + + return undef if (!defined($conf_name) || !defined($ep_id) || !defined($conf_time)); + + my $ep_key = "$conf_name:$conf_time:$ep_id"; + if (!exists($endpoints{$ep_key})) { + $endpoints{$ep_key}{info} = [$conf_name, $conf_time, $ep_id]; + } + + return $ep_key; +} + +# Determine which of two values is "smaller" modulo a modulus. +# (Assume modulus is an even number.) +sub min_modulo($$$) +{ + my ($a, $b, $modulus) = @_; + return $a if !defined($b); + return $b if !defined($a); + + my $delta = ($b - $a) % $modulus; + $delta -= $modulus if ($delta > $modulus/2); + + return $delta < 0 ? $b : $a; +} + +while (<>) { + my ($line) = parse_line($_); + next if !defined($line); + + my $key = get_ep_key($line); + # Use 1e8 since the epoch (March 1973) as the threshold between fake clocks and real clocks. + my $time = $line->{time} < 1e8 ? $line->{time} / 1e3 : "new Date($line->{time})"; + my $series = $line->{series}; + next if (!defined($key) || !defined($time) || !defined($series)); + + if ($check_old_bwe) { + if ($series eq "in_pkt") { + # send_ts_ms is mod 64000, so do all math like that. + my $delta = ($line->{recv_ts_ms} - $line->{send_ts_ms}) % 64000; + + $endpoints{$key}{min_delta} = min_modulo($delta, $endpoints{$key}{min_delta}, 64000); + push(@{$endpoints{$key}{trace}}, ["delay", $time, $delta]); + $endpoints{columns}{delay} = 1; + } + elsif ($series eq "aimd_rtt") { + my $entry = ["rtt", $time, $line->{rtt}]; + push(@{$endpoints{$key}{trace}}, $entry); + $endpoints{columns}{rtt} = 1; + } + elsif ($series eq "bwe_incoming") { + push(@{$endpoints{$key}{trace}}, ["bw", $time, $line->{bitrate_bps}]); + $endpoints{columns}{bw} = 1; + } + } + else { + if ($series eq "bwe_packet_arrival") { + my $delta = ($line->{recvTime} - $line->{sendTime}); + + if (!defined($endpoints{$key}{min_delta}) || $delta < $endpoints{$key}{min_delta}) { + $endpoints{$key}{min_delta} = $delta; + } + push(@{$endpoints{$key}{trace}}, ["delay", $time, $delta]); + $endpoints{$key}{columns}{delay} = 1; + } + elsif ($series eq "bwe_rtt") { + my $entry = ["rtt", $time, $line->{rtt}]; + push(@{$endpoints{$key}{trace}}, $entry); + $endpoints{$key}{columns}{rtt} = 1; + } + elsif ($series eq "bwe_estimate") { + push(@{$endpoints{$key}{trace}}, ["bw", $time, $line->{bw}]); + $endpoints{$key}{columns}{bw} = 1; + } + } +} + +sub print_row($$$$$;$) { + my ($cols, $time, $bw, $rtt, $delay, $skip_comma) = @_; + + my $have_bw = exists($cols->{bw}); + my $have_rtt = exists($cols->{rtt}); + my $have_delay = exists($cols->{delay}); + + print "," if (!defined($skip_comma) || $skip_comma); + print ("[", $time); + print ",$bw" if ($have_bw); + print ",$rtt" if ($have_rtt); + print ",$delay" if ($have_delay); + print "]\n"; +} + +print "var charts={"; +my $subsequent=0; +foreach my $ep (sort keys %endpoints) { + my ($conf_name, $conf_time, $ep_id) = @{$endpoints{$ep}{info}}; + + my $cols = $endpoints{$ep}{columns}; + my $have_bw = exists($cols->{bw}); + my $have_rtt = exists($cols->{rtt}); + my $have_delay = exists($cols->{delay}); + + next if (!$have_delay && !$have_rtt && !$have_bw); + + print "," if ($subsequent); $subsequent = 1; + + my $conf_time_str; + # See above about 1e8 + if ($conf_time > 1e8) { + $conf_time_str = "new Date($conf_time).toISOString()"; + } + else { + $conf_time_str = "\"$conf_time\""; + } + + print "\"$ep\":{\"title\":\"Endpoint $ep_id in Conference $conf_name at \" + $conf_time_str,\n"; + + print "\"columns\":{"; + print "\"bw\":", $have_bw ? "true," : "false,"; + print "\"rtt\":", $have_rtt ? "true," : "false,"; + print "\"delay\":", $have_delay ? "true},\n" : "false},\n"; + + print "\"data\":["; + print_row($cols, "\"Time\"","\"Bandwidth\"","\"RTT\"","\"Excess One-Way Delay\"", 0); + + foreach my $row (@{$endpoints{$ep}{trace}}) { + if ($row->[0] eq "delay") { + print_row($cols, $row->[1], "null", "null", $row->[2] - $endpoints{$ep}{min_delta}); + } + elsif ($row->[0] eq "rtt") { + print_row($cols, $row->[1], "null", $row->[2], "null"); + } + elsif ($row->[0] eq "bw") { + if ($row->[2] != -1) { + print_row ($cols, $row->[1], $row->[2], "null", "null"); + } + } + } + print "]}\n"; +} +print "};\n"; diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorTest.kt new file mode 100644 index 0000000..a48f66f --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AcknowledgedBitrateEstimatorTest.kt @@ -0,0 +1,125 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.shouldBe +import io.mockk.confirmVerified +import io.mockk.every +import io.mockk.spyk +import io.mockk.verifySequence +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.kbps +import java.time.Instant + +/** + * Unit tests of AcknowledgedBitrateEstimator. + * + * Based on WebRTC modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +class AcknowledgedBitrateEstimatorTestStates { + val mockBitrateEstimator = spyk(BitrateEstimator()) + + val acknowledgedBitrateEstimator = AcknowledgedBitrateEstimator(mockBitrateEstimator) +} + +fun createFeedbackVector(): List { + val packetFeedbackVector = MutableList(2) { PacketResult() } + packetFeedbackVector[0].receiveTime = Instant.ofEpochMilli(kFirstArrivalTimeMs) + packetFeedbackVector[0].sentPacket.sendTime = Instant.ofEpochMilli(kFirstSendTimeMs) + packetFeedbackVector[0].sentPacket.sequenceNumber = kSequenceNumber + packetFeedbackVector[0].sentPacket.size = kPayloadSize.bytes + packetFeedbackVector[1].receiveTime = + Instant.ofEpochMilli(kFirstArrivalTimeMs + 10) + packetFeedbackVector[1].sentPacket.sendTime = + Instant.ofEpochMilli(kFirstSendTimeMs + 10) + packetFeedbackVector[1].sentPacket.sequenceNumber = + kSequenceNumber + packetFeedbackVector[1].sentPacket.size = + (kPayloadSize + 10).bytes + return packetFeedbackVector +} + +const val kFirstArrivalTimeMs = 10L +const val kFirstSendTimeMs = 10L +const val kSequenceNumber = 1L +const val kPayloadSize = 1L + +class AcknowledgedBitrateEstimatorTest : FreeSpec() { + init { + "UpdateBandwidth" { + val states = AcknowledgedBitrateEstimatorTestStates() + val packetFeedbackVector = createFeedbackVector() + + states.acknowledgedBitrateEstimator.incomingPacketFeedbackVector(packetFeedbackVector) + + verifySequence { + states.mockBitrateEstimator.update( + packetFeedbackVector[0].receiveTime, + packetFeedbackVector[0].sentPacket.size, + false + ) + states.mockBitrateEstimator.update( + packetFeedbackVector[1].receiveTime, + packetFeedbackVector[1].sentPacket.size, + false + ) + } + + confirmVerified(states.mockBitrateEstimator) + } + + "ExpectFastRateChangeWhenLeftAlr" { + val states = AcknowledgedBitrateEstimatorTestStates() + val packetFeedbackVector = createFeedbackVector() + + states.acknowledgedBitrateEstimator.setAlrEndedTime(Instant.ofEpochMilli(kFirstArrivalTimeMs + 1)) + states.acknowledgedBitrateEstimator.incomingPacketFeedbackVector(packetFeedbackVector) + + verifySequence { + states.mockBitrateEstimator.update( + packetFeedbackVector[0].receiveTime, + packetFeedbackVector[0].sentPacket.size, + false + ) + states.mockBitrateEstimator.expectFastRateChange() + states.mockBitrateEstimator.update( + packetFeedbackVector[1].receiveTime, + packetFeedbackVector[1].sentPacket.size, + false + ) + } + } + + "ReturnBitrate".config(enabled = false) { + val states = AcknowledgedBitrateEstimatorTestStates() + val returnValue = 42.kbps + + // Unfortunately this goes into an infinite loop in mockk. + every { states.mockBitrateEstimator.bitrate() } returns returnValue + + states.acknowledgedBitrateEstimator.bitrate() shouldBe returnValue + + verifySequence { + states.mockBitrateEstimator.bitrate() + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControlTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControlTest.kt new file mode 100644 index 0000000..5ce4c8a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AimdRateControlTest.kt @@ -0,0 +1,194 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeLessThanOrEqualTo +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.longs.shouldBeInRange +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.times +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +/** + * Unit tests of AimdRateControl + * + * Based on WebRTC modules/remote_bitrate_estimator/aimd_rate_control_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +val kInitialTime = Instant.ofEpochMilli(123_456) + +val kMinBwePeriod = 2.secs +val kDefaultPeriod = 3.secs +val kMaxBwePeriod = 50.secs + +// After an overuse, we back off to 85% to the received bitrate. +val kFractionAfterOveruse = 0.85 + +class AimdRateControlTest : FreeSpec() { + init { + "MinNearMaxIncreaseRateOnLowBandwith" { + val aimdRateControl = AimdRateControl() + aimdRateControl.setEstimate(30_000.bps, kInitialTime) + aimdRateControl.getNearMaxIncreaseRateBpsPerSecond() shouldBe 4_000 + } + + "NearMaxIncreaseRateIs5kbpsOn90kbpsAnd200msRtt" { + val aimdRateControl = AimdRateControl() + aimdRateControl.setEstimate(90_000.bps, kInitialTime) + aimdRateControl.getNearMaxIncreaseRateBpsPerSecond() shouldBe 5_000 + } + + "NearMaxIncreaseRateIs5kbpsOn60kbpsAnd100msRtt" { + val aimdRateControl = AimdRateControl() + aimdRateControl.setEstimate(60_000.bps, kInitialTime) + aimdRateControl.rtt = 100.ms + aimdRateControl.getNearMaxIncreaseRateBpsPerSecond() shouldBe 5_000 + } + + "GetIncreaseRateAndBandwidthPeriod" { + val aimdRateControl = AimdRateControl() + val kBitrate = 300_000.bps + aimdRateControl.setEstimate(kBitrate, kInitialTime) + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwOverusing, kBitrate), kInitialTime) + aimdRateControl.getNearMaxIncreaseRateBpsPerSecond().shouldBe(14_000.0 plusOrMinus 1_000.0) + aimdRateControl.getExpectedBandwidthPeriod() shouldBe kDefaultPeriod + } + + "BweLimitedByAckedBitrate" { + val aimdRateControl = AimdRateControl() + val kAckedBitrate = 10_000.bps + var now = kInitialTime + aimdRateControl.setEstimate(kAckedBitrate, now) + while (Duration.between(kInitialTime, now) < 20.secs) { + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwNormal, kAckedBitrate), now) + now += 100.ms + } + aimdRateControl.validEstimate() shouldBe true + aimdRateControl.latestEstimate() shouldBe 1.5 * kAckedBitrate + 10_000.bps + } + + "BweLimitedByDecreasingAckedBitrate" { + val aimdRateControl = AimdRateControl() + val kAckedBitrate = 10_000.bps + var now = kInitialTime + aimdRateControl.setEstimate(kAckedBitrate, now) + while (Duration.between(kInitialTime, now) < 20.secs) { + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwNormal, kAckedBitrate), now) + now += 100.ms + } + aimdRateControl.validEstimate() shouldBe true + val prevEstimate = aimdRateControl.latestEstimate() + aimdRateControl.update( + RateControlInput(BandwidthUsage.kBwNormal, kAckedBitrate / 2), + now + ) + val newEstimate = aimdRateControl.latestEstimate() + newEstimate shouldBe prevEstimate + newEstimate.bps shouldBeInRange ((1.5 * kAckedBitrate + 10_000.bps).bps plusOrMinus 2_000) + } + + "DefaultPeriodUntilFirstOveruse" { + val aimdRateControl = AimdRateControl() + aimdRateControl.setStartBitrate(300.kbps) + aimdRateControl.getExpectedBandwidthPeriod() shouldBe kDefaultPeriod + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwOverusing, 280.kbps), kInitialTime) + aimdRateControl.getExpectedBandwidthPeriod() shouldNotBe kDefaultPeriod + } + + "ExpectedPeriodAfterTypicalDrop" { + val aimdRateControl = AimdRateControl() + // The rate increase at 216 kbps should be 12 kbps. If we drop from + // 216 + 4*12 = 264 kbps, it should take 4 seconds to recover. Since we + // back off to 0.85*acked_rate-5kbps, the acked bitrate needs to be 260 + // kbps to end up at 216 kbps. + val kInitialBitrate = 264_000.bps + val kUpdatedBitrate = 216_000.bps + val kAckedBitrate = + (kUpdatedBitrate + 5_000.bps) / kFractionAfterOveruse + var now = kInitialTime + aimdRateControl.setEstimate(kInitialBitrate, now) + now += 100.ms + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwOverusing, kAckedBitrate), now) + aimdRateControl.latestEstimate() shouldBe kUpdatedBitrate + aimdRateControl.getNearMaxIncreaseRateBpsPerSecond() shouldBe 12_000 + aimdRateControl.getExpectedBandwidthPeriod() shouldBe 4.secs + } + + "BandwidthPeriodIsNotBelowMin" { + val aimdRateControl = AimdRateControl() + val kInitialBitrate = 10_000.bps + var now = kInitialTime + aimdRateControl.setEstimate(kInitialBitrate, now) + now += 100.ms + aimdRateControl.update( + RateControlInput(BandwidthUsage.kBwOverusing, kInitialBitrate - 1.bps), + now + ) + aimdRateControl.getExpectedBandwidthPeriod() shouldBe kMinBwePeriod + } + + "BandwidthPeriodIsNotAboveMaxNoSmoothingExp" { + val aimdRateControl = AimdRateControl() + val kInitialBitrate = 10_010_000.bps + var now = kInitialTime + aimdRateControl.setEstimate(kInitialBitrate, now) + now += 100.ms + // Make a large (10 Mbps) bitrate drop to 10 kbps. + val kAckedBitrate = 10_000.bps / kFractionAfterOveruse + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwOverusing, kAckedBitrate), now) + aimdRateControl.getExpectedBandwidthPeriod() shouldBe kMaxBwePeriod + } + + "SendingRateBoundedWhenThroughputNotEstimated" { + val aimdRateControl = AimdRateControl() + val kInitialBitrate = 123_000.bps + var now = kInitialTime + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwNormal, kInitialBitrate), now) + // AimdRateControl sets the initial bit rate to what it receives after + // five seconds has passed. + // TODO(bugs.webrtc.org/9379): The comment in the AimdRateControl does not + // match the constant. + val kInitializationTime = 5.secs + now += kInitializationTime + 1.ms + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwNormal, kInitialBitrate), now) + for (i in 0 until 100) { + aimdRateControl.update(RateControlInput(BandwidthUsage.kBwNormal, null), now) + now += 100.ms + } + aimdRateControl.latestEstimate() shouldBeLessThanOrEqualTo kInitialBitrate * 1.5 + 10_000.bps + } + + // Skipped: EstimateDoesNotIncreaseInAlr: uses field trial WebRTC-DontIncreaseDelayBasedBweInAlr/Enabled/ + // Skipped: SetEstimateIncreaseBweInAlr: uses field trial WebRTC-DontIncreaseDelayBasedBweInAlr/Enabled/ + // Skipped: SetEstimateUpperLimitedByNetworkEstimate: uses setNetworkStateEstimate + // Skipped: SetEstimateDefaultUpperLimitedByCurrentBitrateIfNetworkEstimateIsLow: uses setNetworkStateEstimate + // Skipped: SetEstimateNotUpperLimitedByCurrentBitrateIfNetworkEstimateIsLowIf: uses setNetworkStateEstimate + // Skipped: SetEstimateLowerLimitedByNetworkEstimate: uses setNetworkStateEstimate + // Skipped: SetEstimateIgnoredIfLowerThanNetworkEstimateAndCurrent: uses setNetworkStateEstimate + // Skipped: EstimateIncreaseWhileNotInAlr: uses field trial WebRTC-DontIncreaseDelayBasedBweInAlr/Enabled/ + // Skipped: EstimateNotLimitedByNetworkEstimateIfDisabled: uses setNetworkStateEstimate + } +} + +infix fun Long.plusOrMinus(tolerance: Long) = LongRange(this - tolerance, this + tolerance) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetectorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetectorTest.kt new file mode 100644 index 0000000..1a0288a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/AlrDetectorTest.kt @@ -0,0 +1,164 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This file uses WebRTC's naming style for constants +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 +/* + * Unit tests for AlrDetector. + * + * Based on WebRTC modules/congestion_controller/goog_cc/alr_detector_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138), skipping tests of field trials. + */ + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe + +const val kEstimatedBitrateBps = 300000 + +data class TimestampHolder( + var time: Long +) + +class SimulateOutgoingTrafficIn( + val alrDetector: AlrDetector, + var timestampMs: TimestampHolder +) { + fun forTimeMs(timeMs: Int): SimulateOutgoingTrafficIn { + intervalMs = timeMs + produceTraffic() + return this + } + + fun atPercentOfEstimatedBitrate(usagePercentage: Int): SimulateOutgoingTrafficIn { + this.usagePercentage = usagePercentage + produceTraffic() + return this + } + + private fun produceTraffic() { + val intervalMs = intervalMs + val usagePercentage = usagePercentage + if (intervalMs == null || usagePercentage == null) { + return + } + val kTimeStepMs = 10 + for (t in 0 until intervalMs step kTimeStepMs) { + timestampMs.time += kTimeStepMs + alrDetector.onBytesSent( + kEstimatedBitrateBps.toLong() * usagePercentage * kTimeStepMs / (8 * 100 * 1000), + timestampMs.time + ) + } + val remainderMs = intervalMs % kTimeStepMs + if (remainderMs > 0) { + timestampMs.time += kTimeStepMs + alrDetector.onBytesSent( + kEstimatedBitrateBps.toLong() * usagePercentage * kTimeStepMs / (8 * 100 * 1000), + timestampMs.time + ) + } + } + + private var intervalMs: Int? = null + private var usagePercentage: Int? = null +} + +class AlrDetectorTest : FreeSpec() { + init { + "AlrDetection" { + val timestampMs = TimestampHolder(1000) + val alrDetector = AlrDetector() + alrDetector.setEstimatedBitrate(kEstimatedBitrateBps) + + // Start in non-ALR state + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + + // Stay in non-ALR state when usage is close to 100%. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(1000) + .atPercentOfEstimatedBitrate(90) + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + + // Verify that we ALR starts when bitrate drops below 20%. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(1500) + .atPercentOfEstimatedBitrate(20) + alrDetector.getApplicationLimitedRegionStartTime() shouldNotBe null + + // Verify that ALR ends when usage is above 65%. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(4000) + .atPercentOfEstimatedBitrate(100) + + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + } + + "ShortSpike" { + val timestampMs = TimestampHolder(1000) + val alrDetector = AlrDetector() + alrDetector.setEstimatedBitrate(kEstimatedBitrateBps) + + // Start in non-ALR state + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + + // Verify that we ALR starts when bitrate drops below 20%. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(1000) + .atPercentOfEstimatedBitrate(20) + alrDetector.getApplicationLimitedRegionStartTime() shouldNotBe null + + // Verify that we stay in ALR region even after a short bitrate spike. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(100) + .atPercentOfEstimatedBitrate(150) + alrDetector.getApplicationLimitedRegionStartTime() shouldNotBe null + + // ALR ends when usage is above 65%. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(3000) + .atPercentOfEstimatedBitrate(100) + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + } + + "BandwidthEstimateChanges" { + val timestampMs = TimestampHolder(1000) + val alrDetector = AlrDetector() + alrDetector.setEstimatedBitrate(kEstimatedBitrateBps) + + // Start in non-ALR state + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + + // ALR starts when bitrate drops below 20%. + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(1000) + .atPercentOfEstimatedBitrate(20) + alrDetector.getApplicationLimitedRegionStartTime() shouldNotBe null + + // When bandwidth estimate drops the detector should stay in ALR mode and quit + // it shortly afterwards as the sender continues sending the same amount of + // traffic. This is necessary to ensure that ProbeController can still react + // to the BWE drop by initiating a new probe. + alrDetector.setEstimatedBitrate(kEstimatedBitrateBps / 5) + alrDetector.getApplicationLimitedRegionStartTime() shouldNotBe null + SimulateOutgoingTrafficIn(alrDetector, timestampMs) + .forTimeMs(1000) + .atPercentOfEstimatedBitrate(50) + alrDetector.getApplicationLimitedRegionStartTime() shouldBe null + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProberTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProberTest.kt new file mode 100644 index 0000000..cb6ba55 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/BitrateProberTest.kt @@ -0,0 +1,509 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.comparables.shouldBeGreaterThanOrEqualTo +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.per +import org.jitsi.nlj.util.times +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.max +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +/** Unit tests for BitrateProber, + * based on WebRTC modules/pacing/bitrate_prober_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class BitrateProberTest : FreeSpec() { + private val logger = createLogger() + + init { + "VerifyStatesAndTimeBetweenProbes" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + prober.isProbing() shouldBe false + + var now = Instant.EPOCH + val startTime = now + prober.nextProbeTime(now) shouldBe Instant.MAX + + val kTestBitrate1 = 900.kbps + val kTestBitrate2 = 1800.kbps + val kClusterSize = 5 + val kProbeSize = 1000.bytes + val kMinProbeDuration = 15.ms + + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kTestBitrate1, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kTestBitrate2, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 1 + ) + ) + prober.isProbing() shouldBe false + + prober.onIncomingPacket(kProbeSize) + prober.isProbing() shouldBe true + prober.currentCluster(now)!!.probeClusterId shouldBe 0 + + // First packet should probe as soon as possible + prober.nextProbeTime(now) shouldBe Instant.MIN + + for (i in 0 until kClusterSize) { + now = max(now, prober.nextProbeTime(now)) + now shouldBe max(now, prober.nextProbeTime(now)) + prober.currentCluster(now)!!.probeClusterId shouldBe 0 + prober.probeSent(now, kProbeSize) + } + + Duration.between(startTime, now) shouldBeGreaterThanOrEqualTo kMinProbeDuration + // Verify that the actual bitrate is withing 10% of the target. + var bitrate = (kProbeSize * (kClusterSize - 1)).per(Duration.between(startTime, now)) + bitrate shouldBeGreaterThan kTestBitrate1 * 0.9 + bitrate shouldBeLessThan kTestBitrate1 * 1.1 + + now = max(now, prober.nextProbeTime(now)) + val probe2Started = now + + for (i in 0 until kClusterSize) { + now = max(now, prober.nextProbeTime(now)) + now shouldBe prober.nextProbeTime(now) + prober.currentCluster(now)!!.probeClusterId shouldBe 1 + prober.probeSent(now, kProbeSize) + } + + val duration = Duration.between(probe2Started, now) + duration shouldBeGreaterThanOrEqualTo kMinProbeDuration + // Verify that the actual bitrate is withing 10% of the target. + bitrate = (kProbeSize * (kClusterSize - 1)).per(duration) + bitrate shouldBeGreaterThan kTestBitrate2 * 0.9 + bitrate shouldBeLessThan kTestBitrate2 * 1.1 + + prober.nextProbeTime(now) shouldBe Instant.MAX + prober.isProbing() shouldBe false + } + + "DoesntProbeWithoutRecentPackets" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + val kProbeSize = 1000.bytes + + val now = Instant.EPOCH + prober.nextProbeTime(now) shouldBe Instant.MAX + + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = 900.kbps, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.isProbing() shouldBe false + + prober.onIncomingPacket(kProbeSize) + prober.isProbing() shouldBe true + max(now, prober.nextProbeTime(now)) shouldBe now + prober.probeSent(now, kProbeSize) + } + + "DiscardsDelayedProbes" { + val kMaxProbeDelay = 3.ms + val config = BitrateProberConfig( + maxProbeDelay = 3.ms + ) + val prober = BitrateProber(logger, config) + val kProbeSize = 1000.bytes + + var now = Instant.EPOCH + + // Add two probe clusters. + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = 900.kbps, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + + prober.onIncomingPacket(kProbeSize) + prober.isProbing() shouldBe true + prober.currentCluster(now)!!.probeClusterId shouldBe 0 + // Advance to first probe time and indicate sent probe. + now = max(now, prober.nextProbeTime(now)) + prober.probeSent(now, kProbeSize) + + // Advance time 1ms past timeout for the next probe. + val nextProbeTime = prober.nextProbeTime(now) + nextProbeTime shouldBeGreaterThan now + now = nextProbeTime + kMaxProbeDelay + 1.ms + + // Still indicates the time we wanted to probe at. + prober.nextProbeTime(now) shouldBe nextProbeTime + prober.currentCluster(now) shouldBe null + } + + "LimitsNumberOfPendingProbeClusters" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + val kProbeSize = 1000.bytes + var now = Instant.EPOCH + prober.nextProbeTime(now) shouldBe Instant.MAX + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = 900.kbps, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.onIncomingPacket(kProbeSize) + prober.isProbing() shouldBe true + prober.currentCluster(now)!!.probeClusterId shouldBe 0 + + for (i in 1 until 11) { + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = 900.kbps, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = i + ) + ) + prober.onIncomingPacket(kProbeSize) + } + // Expect some clusters has been dropped. + prober.isProbing() shouldBe true + prober.currentCluster(now)!!.probeClusterId shouldBeGreaterThanOrEqualTo 5 + + val maxExpectedProbeTime = now + 1.secs + while (prober.isProbing() && now < maxExpectedProbeTime) { + now = max(now, prober.nextProbeTime(now)) + prober.probeSent(now, kProbeSize) + } + prober.isProbing() shouldBe false + } + + "DoesntInitializeProbingForSmallPackets" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + prober.setEnabled(true) + prober.isProbing() shouldBe false + + prober.createProbeCluster( + ProbeClusterConfig( + atTime = Instant.EPOCH, + targetDataRate = 1000.kbps, + targetDuration = 15.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.onIncomingPacket(100.bytes) + + prober.isProbing() shouldBe false + } + + "DoesInitializeProbingForSmallPacketsIfConfigured" { + val config = BitrateProberConfig( + minPacketSize = 0.bytes + ) + + val prober = BitrateProber(logger, config) + prober.setEnabled(true) + prober.isProbing() shouldBe false + + prober.createProbeCluster( + ProbeClusterConfig( + atTime = Instant.EPOCH, + targetDataRate = 1000.kbps, + targetDuration = 15.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.onIncomingPacket(10.bytes) + + prober.isProbing() shouldBe true + } + + "VerifyProbeSizeOnHighBitrate" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + + val kHighBitrate = 10000.kbps + + prober.createProbeCluster( + ProbeClusterConfig( + atTime = Instant.EPOCH, + targetDataRate = kHighBitrate, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + // Probe size should ensure a minimum of 1 ms interval. + prober.recommendedMinProbeSize() shouldBeGreaterThan kHighBitrate * 1.ms + } + + "ProbeSizeCanBeSetInProbeClusterConfig" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + prober.setEnabled(true) + + val kHighBitrate = 10000.kbps + + prober.createProbeCluster( + ProbeClusterConfig( + atTime = Instant.EPOCH, + targetDataRate = kHighBitrate, + targetDuration = 15.ms, + minProbeDelta = 20.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.recommendedMinProbeSize() shouldBe kHighBitrate * 20.ms + + prober.onIncomingPacket(1000.bytes) + // Next time to send probe should be "min_probe_delta" if the recommended + // number of bytes has been sent. + prober.probeSent(Instant.EPOCH, prober.recommendedMinProbeSize()) + prober.nextProbeTime(Instant.EPOCH) shouldBe Instant.EPOCH + 20.ms + } + + "MinumumNumberOfProbingPackets" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + // Even when probing at a low bitrate we expect a minimum number + // of packets to be sent. + val kBitrate = 100.kbps + val kPacketSize = 1000.bytes + + val now = Instant.EPOCH + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kBitrate, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + + prober.onIncomingPacket(kPacketSize) + for (i in 0 until 5) { + prober.isProbing() shouldBe true + prober.probeSent(now, kPacketSize) + } + + prober.isProbing() shouldBe false + } + + "ScaleBytesUsedForProbing" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + val kBitrate = 10000.kbps // 10 Mbps. + val kPacketSize = 1000.bytes + val kExpectedDataSent = kBitrate * 15.ms + + val now = Instant.EPOCH + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kBitrate, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.onIncomingPacket(kPacketSize) + var dataSent = DataSize.ZERO + while (dataSent < kExpectedDataSent) { + prober.isProbing() shouldBe true + prober.probeSent(now, kPacketSize) + dataSent += kPacketSize + } + + prober.isProbing() shouldBe false + } + + "HighBitrateProbing" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + val kBitrate = 1000000.kbps // 1 Gbps. + val kPacketSize = 1000.bytes + val kExpectedDataSent = kBitrate * 15.ms + + val now = Instant.EPOCH + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kBitrate, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.onIncomingPacket(kPacketSize) + var dataSent = DataSize.ZERO + while (dataSent < kExpectedDataSent) { + prober.isProbing() shouldBe true + prober.probeSent(now, kPacketSize) + dataSent += kPacketSize + } + + prober.isProbing() shouldBe false + } + + "ProbeClusterTimeout" { + val config = BitrateProberConfig() + val prober = BitrateProber(logger, config) + val kBitrate = 300.kbps + val kSmallPacketSize = 20.bytes + // Expecting two probe clusters of 5 packets each. + val kExpectedDataSent = kSmallPacketSize * 2 * 5 + val kTimeout = 5000.ms + + var now = Instant.EPOCH + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kBitrate, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 0 + ) + ) + prober.onIncomingPacket(kSmallPacketSize) + prober.isProbing() shouldBe false + now += kTimeout + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kBitrate / 10, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 1 + ) + ) + prober.onIncomingPacket(kSmallPacketSize) + prober.isProbing() shouldBe false + now += 1.ms + prober.createProbeCluster( + ProbeClusterConfig( + atTime = now, + targetDataRate = kBitrate / 10, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 5, + id = 2 + ) + ) + prober.onIncomingPacket(kSmallPacketSize) + prober.isProbing() shouldBe true + var dataSent = DataSize.ZERO + while (dataSent < kExpectedDataSent) { + prober.isProbing() shouldBe true + prober.probeSent(now, kSmallPacketSize) + dataSent += kSmallPacketSize + } + + prober.isProbing() shouldBe false + } + + "CanProbeImmediatelyIfConfigured" { + val config = BitrateProberConfig( + minPacketSize = 0.bytes, + ) + val prober = BitrateProber(logger, config) + prober.createProbeCluster( + ProbeClusterConfig( + atTime = Instant.EPOCH, + targetDataRate = 300.kbps, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + id = 0 + ) + ) + prober.isProbing() shouldBe true + } + + "CanProbeImmediatelyAgainAfterProbeIfConfigured" { + val config = BitrateProberConfig( + minPacketSize = 0.bytes, + ) + val prober = BitrateProber(logger, config) + val clusterConfig = ProbeClusterConfig( + atTime = Instant.EPOCH, + targetDataRate = 300.kbps, + targetDuration = 15.ms, + minProbeDelta = 2.ms, + targetProbeCount = 1, + id = 0 + ) + prober.createProbeCluster(clusterConfig) + prober.isProbing() shouldBe true + prober.probeSent( + Instant.EPOCH + 1.ms, + clusterConfig.targetDataRate * clusterConfig.targetDuration + ) + prober.isProbing() shouldBe false + + clusterConfig.id = 2 + clusterConfig.atTime = Instant.EPOCH + 100.ms + prober.createProbeCluster(clusterConfig) + prober.isProbing() shouldBe true + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackControllerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackControllerTest.kt new file mode 100644 index 0000000..abfbcd5 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/CongestionWindowPushbackControllerTest.kt @@ -0,0 +1,78 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.util.bytes +/** + * Unit tests for CongestionWindowPushbackController. + * + * Based on WebRTC modules/congestion_controller/goog_cc/congestion_window_pushback_controller_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138), skipping tests of field trials. + */ +class CongestionWindowPushbackControllerTest : FreeSpec() { + init { + "FullCongestionWindow" { + val cwndController = CongestionWindowPushbackController() + cwndController.updateOutstandingData(100000) + cwndController.setDataWindow(50000.bytes) + + var bitrateBps = 80000 + bitrateBps = cwndController.updateTargetBitrate(bitrateBps) + bitrateBps shouldBe 72000 + + cwndController.setDataWindow(50000.bytes) + bitrateBps = cwndController.updateTargetBitrate(bitrateBps) + bitrateBps shouldBe (72000 * 0.9 * 0.9).toInt() + } + + "NormalCongestionWindow" { + val cwndController = CongestionWindowPushbackController() + cwndController.updateOutstandingData(199999) + cwndController.setDataWindow(200000.bytes) + + var bitrateBps = 80000 + bitrateBps = cwndController.updateTargetBitrate(bitrateBps) + bitrateBps shouldBe 80000 + } + + "LowBitrate" { + val cwndController = CongestionWindowPushbackController() + cwndController.updateOutstandingData(100000) + cwndController.setDataWindow(50000.bytes) + + var bitrateBps = 35000 + bitrateBps = cwndController.updateTargetBitrate(bitrateBps) + bitrateBps shouldBe (35000 * 0.9).toInt() + + cwndController.setDataWindow(20000.bytes) + bitrateBps = cwndController.updateTargetBitrate(bitrateBps) + bitrateBps shouldBe 30000 + } + + "NoPushbackOnDataWindowUnset" { + val cwndController = CongestionWindowPushbackController() + cwndController.updateOutstandingData(1e8.toLong()) // Large number + + var bitrateBps = 80000 + bitrateBps = cwndController.updateTargetBitrate(bitrateBps) + bitrateBps shouldBe 80000 + } + + /* Skipping tests with field trials set */ + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBweTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBweTest.kt new file mode 100644 index 0000000..46a6a45 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBweTest.kt @@ -0,0 +1,254 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.equals.shouldNotBeEqual +import io.kotest.matchers.longs.shouldBeGreaterThan +import io.kotest.matchers.longs.shouldBeInRange +import io.kotest.matchers.longs.shouldBeLessThanOrEqual +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.util.kbps +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms + +/** + * Unit tests for Delay-Based BWE, + * based on WebRTC modules/congestion_controller/goog_cc/delay_based_bwe_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + */ +class DelayBasedBweTest : FreeSpec() { + val logger = createLogger() + val diagnosticContext = DiagnosticContext() + + init { + "ProbeDetection" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + + var nowMs = test.clock.millis() + + // First burst sent at 8 * 1000 / 10 = 800 kbps. + repeat(kNumProbesCluster0) { + test.clock.elapse(10.ms) + nowMs = test.clock.millis() + test.incomingFeedback(nowMs, nowMs, 1000, kPacingInfo0) + } + test.bitrateObserver.updated shouldBe true + + // Second burst sent at 8 * 1000 / 5 = 1600 kbps. + repeat(kNumProbesCluster1) { + test.clock.elapse(5.ms) + nowMs = test.clock.millis() + test.incomingFeedback(nowMs, nowMs, 1000, kPacingInfo1) + } + test.bitrateObserver.updated shouldBe true + test.bitrateObserver.latestBitrate shouldBeGreaterThan 1500000 + } + + "ProbeDetectionNonPacedPackets" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + var nowMs = test.clock.millis() + // First burst sent at 8 * 1000 / 10 = 800 kbps, but with every other packet + // not being paced which could mess things up. + repeat(kNumProbesCluster0) { + test.clock.elapse(5.ms) + nowMs = test.clock.millis() + test.incomingFeedback(nowMs, nowMs, 1000, kPacingInfo0) + // Non-paced packet, arriving 5 ms after. + test.clock.elapse(5.ms) + test.incomingFeedback(nowMs, nowMs, 100, PacedPacketInfo()) + } + + test.bitrateObserver.updated shouldBe true + test.bitrateObserver.latestBitrate shouldBeGreaterThan 800000 + } + + "ProbeDetectionFasterArrival" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + var nowMs = test.clock.millis() + // First burst sent at 8 * 1000 / 10 = 800 kbps. + // Arriving at 8 * 1000 / 5 = 1600 kbps. + var sendTimeMs = 0L + repeat(kNumProbesCluster0) { + test.clock.elapse(1.ms) + sendTimeMs += 10 + nowMs = test.clock.millis() + test.incomingFeedback(nowMs, sendTimeMs, 1000, kPacingInfo0) + } + + test.bitrateObserver.updated shouldBe false + } + + "ProbeDetectionSlowerArrival" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + var nowMs = test.clock.millis() + // First burst sent at 8 * 1000 / 5 = 1600 kbps. + // Arriving at 8 * 1000 / 7 = 1142 kbps. + // Since the receive rate is significantly below the send rate, we expect to + // use 95% of the estimated capacity. + var sendTimeMs = 0L + repeat(kNumProbesCluster1) { + test.clock.elapse(7.ms) + sendTimeMs += 5 + nowMs = test.clock.millis() + test.incomingFeedback(nowMs, sendTimeMs, 1000, kPacingInfo1) + } + + test.bitrateObserver.updated shouldBe true + test.bitrateObserver.latestBitrate shouldBeInRange + ((kTargetUtilizationFraction * 1140000).toLong() plusOrMinus 10000) + } + + "ProbeDetectionSlowerArrivalHighBitrate" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + var nowMs = test.clock.millis() + // Burst sent at 8 * 1000 / 1 = 8000 kbps. + // Arriving at 8 * 1000 / 2 = 4000 kbps. + // Since the receive rate is significantly below the send rate, we expect to + // use 95% of the estimated capacity. + var sendTimeMs = 0L + repeat(kNumProbesCluster1) { + test.clock.elapse(2.ms) + sendTimeMs += 1 + nowMs = test.clock.millis() + test.incomingFeedback(nowMs, sendTimeMs, 1000, kPacingInfo1) + } + + test.bitrateObserver.updated shouldBe true + test.bitrateObserver.latestBitrate shouldBeInRange + ((kTargetUtilizationFraction * 4000000).toLong() plusOrMinus 10000) + } + + "GetExpectedBwePeriodMs" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + val defaultInterval = test.bitrateEstimator.getExpectedBwePeriod() + defaultInterval.toMillis() shouldBeGreaterThan 0 + test.capacityDropTestHelper(1, true, 533, 0) + val interval = test.bitrateEstimator.getExpectedBwePeriod() + interval.toMillis() shouldBeGreaterThan 0 + interval shouldNotBeEqual defaultInterval + } + + "InitialBehavior" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.initialBehaviorTestHelper(730000) + } + + "InitializeResult" { + val result = DelayBasedBwe.Result() + result.delayDetectorState shouldBe BandwidthUsage.kBwNormal + } + + "RateIncreaseReordering" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.rateIncreaseReorderingTestHelper(730000) + } + + "RateIncreaseRtpTimestamps" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.rateIncreaseRtpTimestampsTestHelper(617) + } + + "CapacityDropOneStream" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.capacityDropTestHelper(1, false, 500, 0) + } + + "CapacityDropPosOffsetChange" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.capacityDropTestHelper(1, false, 867, 30000) + } + + "CapacityDropNegOffsetChange" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.capacityDropTestHelper(1, false, 933, -30000) + } + + "CapacityDropOneStreamWrap" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.capacityDropTestHelper(1, true, 533, 0) + } + + "TestTimestampGrouping" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + test.testTimestampGroupingTestHelper() + } + + "TestShortTimeoutAndWrap" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + // Simulate a client leaving and rejoining the call after 35 seconds. This + // will make abs send time wrap, so if streams aren't timed out properly + // the next 30 seconds of packets will be out of order. + test.testWrappingHelper(35) + } + + "TestLongTimeoutAndWrap" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + // Simulate a client leaving and rejoining the call after some multiple of + // 64 seconds later. This will cause a zero difference in abs send times due + // to the wrap, but a big difference in arrival time, if streams aren't + // properly timed out. + test.testWrappingHelper(10 * 64) + } + + "TestInitialOveruse" { + val test = OneDelayBasedBweTest(logger, diagnosticContext) + val kStartBitrate = 300.kbps + val kInitialCapacity = 200.kbps + // High FPS to ensure that we send a lot of packets in a short time. + val kFps = 90 + + test.streamGenerator.addStream(RtpStream(kFps, kStartBitrate.bps)) + test.streamGenerator.setCapacityBps(kInitialCapacity.bps) + + // Needed to initialize the AimdRateControl. + test.bitrateEstimator.setStartBitrate(kStartBitrate) + + // Produce 40 frames (in 1/3 second) and give them to the estimator. + var bitrateBps = kStartBitrate.bps + var seenOveruse = false + for (i in 0 until 40) { + val overuse = test.generateAndProcessFrame(bitrateBps) + if (overuse) { + test.bitrateObserver.updated shouldBe true + test.bitrateObserver.latestBitrate shouldBeLessThanOrEqual kInitialCapacity.bps + test.bitrateObserver.latestBitrate shouldBeGreaterThan (0.8 * kInitialCapacity.bps).toLong() + bitrateBps = test.bitrateObserver.latestBitrate + seenOveruse = true + break + } else if (test.bitrateObserver.updated) { + bitrateBps = test.bitrateObserver.latestBitrate + test.bitrateObserver.reset() + } + } + seenOveruse shouldBe true + test.bitrateObserver.latestBitrate shouldBeLessThanOrEqual kInitialCapacity.bps + test.bitrateObserver.latestBitrate shouldBeGreaterThan (0.8 * kInitialCapacity.bps).toLong() + } + } + + companion object { + const val kNumProbesCluster0 = 5 + const val kNumProbesCluster1 = 8 + val kPacingInfo0 = PacedPacketInfo(0, kNumProbesCluster0, 2000) + val kPacingInfo1 = PacedPacketInfo(1, kNumProbesCluster1, 4000) + const val kTargetUtilizationFraction = 0.95 + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBweUnittestHelper.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBweUnittestHelper.kt new file mode 100644 index 0000000..3c9e726 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/DelayBasedBweUnittestHelper.kt @@ -0,0 +1,538 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.longs.shouldBeGreaterThanOrEqual +import io.kotest.matchers.longs.shouldBeInRange +import io.kotest.matchers.longs.shouldBeLessThan +import io.kotest.matchers.longs.shouldBeLessThanOrEqual +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.bytes +import org.jitsi.utils.instantOfEpochMicro +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.ms +import org.jitsi.utils.roundedMillis +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import org.jitsi.utils.toEpochMicro +import org.jitsi.utils.toRoundedEpochMilli +import java.lang.Long.max +import java.time.Duration +import java.time.Instant +import java.util.* + +/** + * Helper classes for the unit tests for Delay-Based BWE, + * based on WebRTC modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.{h,cc} in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + */ + +class TestBitrateObserver { + var updated = false + private set + var latestBitrate = 0L + private set + + fun onReceivedBitrateChanged(bitrate: Long) { + latestBitrate = bitrate + updated = true + } + + fun reset() { + updated = false + } +} + +class RtpStream( + val fps: Int, + var bitrateBps: Long +) { + init { + require(fps > 0) + } + + var nextRtpTime: Long = 0L + private set + + // Generates a new frame for this stream. If called too soon after the + // previous frame, no frame will be generated. The frame is split into + // packets. + fun generateFrame(timeNowUs: Long, nextSequenceNumber: Ref, packets: MutableList): Long { + if (timeNowUs < nextRtpTime) { + return nextRtpTime + } + val bitsPerFrame = (bitrateBps + fps / 2) / fps + val nPackets = ((bitsPerFrame + 4 * kMtu) / (8 * kMtu)).coerceAtLeast(1).toInt() + val payloadSize = (bitsPerFrame + 4 * nPackets) / (8 * nPackets) + repeat(nPackets) { + val packet = PacketResult() + packet.sentPacket.sendTime = instantOfEpochMicro(timeNowUs + kSendSideOffsetUs) + packet.sentPacket.size = payloadSize.bytes + packet.sentPacket.sequenceNumber = nextSequenceNumber.v++ + packets.add(packet) + } + nextRtpTime = timeNowUs + (1000000 + fps / 2) / fps + return nextRtpTime + } + + companion object { + const val kSendSideOffsetUs = 1000000 + const val kMtu = 1200 + } +} + +class StreamGenerator( + private var capacity: Long, + timeNow: Long +) { + /** All streams being transmitted on this simulated channel. */ + private val streams = ArrayList() + + private var prevArrivalTimeUs = timeNow + + /** Add a new stream */ + fun addStream(stream: RtpStream) { + streams.add(stream) + } + + /** Set the link capacity */ + fun setCapacityBps(capacityBps: Long) { + require(capacityBps > 0) + capacity = capacityBps + } + + /** Divides `bitrate_bps` among all streams. The allocated bitrate per stream + is decided by the initial allocation ratios. */ + fun setBitrateBps(bitrateBps: Long) { + check(streams.isNotEmpty()) + var totalBitrateBefore = 0L + for (stream in streams) { + totalBitrateBefore += stream.bitrateBps + } + var bitrateBefore = 0L + var totalBitrateAfter = 0L + for (stream in streams) { + bitrateBefore += stream.bitrateBps + val bitrateAfter: Long = + (bitrateBefore * bitrateBps + totalBitrateBefore / 2) / + totalBitrateBefore + stream.bitrateBps = bitrateAfter - totalBitrateAfter + totalBitrateAfter += stream.bitrateBps + } + check(bitrateBefore == totalBitrateBefore) + check(totalBitrateAfter == bitrateBps) + } + + // TODO(holmer): Break out the channel simulation part from this class to make + // it possible to simulate different types of channels. + fun generateFrame(packets: MutableList, nextSequenceNumber: Ref, timeNowUs: Long): Long { + check(packets.isEmpty()) + check(capacity > 0) + var it = streams.minByOrNull { it.nextRtpTime } + it!!.generateFrame(timeNowUs, nextSequenceNumber, packets) + for (packet in packets) { + val capacityBpus = capacity / 1000 + val requiredNetworkTimeUs = + (8 * 1000 * packet.sentPacket.size.bytes.toLong() + capacityBpus / 2) / + capacityBpus + prevArrivalTimeUs = + max( + timeNowUs + requiredNetworkTimeUs, + prevArrivalTimeUs + requiredNetworkTimeUs + ) + packet.receiveTime = instantOfEpochMicro(prevArrivalTimeUs) + } + it = streams.minByOrNull { it.nextRtpTime } + return max(it!!.nextRtpTime, timeNowUs) + } +} + +class OneDelayBasedBweTest(parentLogger: Logger, diagnosticContext: DiagnosticContext) { + val clock = FakeClock().also { it.setTime(instantOfEpochMicro(100000000)) } + val bitrateObserver = TestBitrateObserver() + val acknowledgedBitrateEstimator: AcknowledgedBitrateEstimatorInterface = + AcknowledgedBitrateEstimatorInterface.create() + val probeBitrateEstimator = ProbeBitrateEstimator(parentLogger, diagnosticContext) + val bitrateEstimator = DelayBasedBwe(parentLogger, diagnosticContext) + val streamGenerator = StreamGenerator(1e6.toLong(), clock.instant().toEpochMicro()) + + var arrivalTimeOffsetMs: Long = 0L + var nextSequenceNumber: Ref = Ref(0) + var firstUpdate: Boolean = true + + fun addDefaultStream() { + streamGenerator.addStream(RtpStream(30, 3e5.toLong())) + } + + // Helpers to insert a single packet into the delay-based BWE. + fun incomingFeedback(arrivalTimeMs: Long, sendTimeMs: Long, payloadSize: Long) = + incomingFeedback(arrivalTimeMs, sendTimeMs, payloadSize, PacedPacketInfo()) + + fun incomingFeedback(arrivalTimeMs: Long, sendTimeMs: Long, payloadSize: Long, pacingInfo: PacedPacketInfo) { + check(arrivalTimeMs + arrivalTimeOffsetMs >= 0) + incomingFeedback( + Instant.ofEpochMilli(arrivalTimeMs + arrivalTimeOffsetMs), + Instant.ofEpochMilli(sendTimeMs), + payloadSize, + pacingInfo + ) + } + + fun incomingFeedback(receiveTime: Instant, sendTime: Instant, payloadSize: Long, pacingInfo: PacedPacketInfo) { + val packet = PacketResult() + packet.receiveTime = receiveTime + packet.sentPacket.sendTime = sendTime + packet.sentPacket.size = payloadSize.bytes + packet.sentPacket.pacingInfo = pacingInfo + packet.sentPacket.sequenceNumber = nextSequenceNumber.v++ + if (packet.sentPacket.pacingInfo.probeClusterId != PacedPacketInfo.kNotAProbe) { + probeBitrateEstimator.handleProbeAndEstimateBitrate(packet) + } + + val msg = TransportPacketsFeedback() + msg.feedbackTime = clock.instant() + msg.packetFeedbacks.add(packet) + acknowledgedBitrateEstimator.incomingPacketFeedbackVector(msg.sortedByReceiveTime()) + val result = bitrateEstimator.incomingPacketFeedbackVector( + msg, + acknowledgedBitrateEstimator.bitrate(), + probeBitrateEstimator.fetchAndResetLastEstimatedBitrate(), + false + ) + if (result.updated) { + bitrateObserver.onReceivedBitrateChanged(result.targetBitrate.bps) + } + } + + /** Generates a frame of packets belonging to a stream at a given bitrate ~~and + with a given ssrc~~. The stream is pushed through a very simple simulated + network, and is then given to the receive-side bandwidth estimator. + Returns true if an over-use was seen, false otherwise. + The StreamGenerator::updated() should be used to check for any changes in + target bitrate after the call to this function. */ + fun generateAndProcessFrame(bitrateBps: Long): Boolean { + streamGenerator.setBitrateBps(bitrateBps) + val packets = ArrayList() + + val nextTimeUs = streamGenerator.generateFrame(packets, nextSequenceNumber, clock.instant().toEpochMicro()) + if (packets.isEmpty()) { + return false + } + + var overuse = false + bitrateObserver.reset() + clock.elapse(Duration.between(clock.instant(), packets.last().receiveTime)) + + for (packet in packets) { + check(packet.receiveTime.toRoundedEpochMilli() + arrivalTimeOffsetMs >= 0) + packet.receiveTime += Duration.ofMillis(arrivalTimeOffsetMs) + + if (packet.sentPacket.pacingInfo.probeClusterId != PacedPacketInfo.kNotAProbe) { + probeBitrateEstimator.handleProbeAndEstimateBitrate(packet) + } + } + + acknowledgedBitrateEstimator.incomingPacketFeedbackVector(packets) + val msg = TransportPacketsFeedback() + msg.packetFeedbacks = packets + msg.feedbackTime = Instant.ofEpochMilli(clock.roundedMillis()) + + val result = + bitrateEstimator.incomingPacketFeedbackVector( + msg, + acknowledgedBitrateEstimator.bitrate(), + probeBitrateEstimator.fetchAndResetLastEstimatedBitrate(), + false + ) + if (result.updated) { + bitrateObserver.onReceivedBitrateChanged(result.targetBitrate.bps) + if (!firstUpdate && result.targetBitrate.bps < bitrateBps) { + overuse = true + } + firstUpdate = false + } + + clock.setTime(instantOfEpochMicro(nextTimeUs)) + return overuse + } + + /** Run the bandwidth estimator with a stream of `number_of_frames` frames, or + until it reaches `target_bitrate`. + Can for instance be used to run the estimator for some time to get it + into a steady state. */ + fun steadyStateRun( + maxNumberOfFrames: Int, + startBitrate: Long, + minBitrate: Long, + maxBitrate: Long, + targetBitrate: Long + ): Long { + var bitrateBps = startBitrate + var bitrateUpdateSeen = false + for (i in 0 until maxNumberOfFrames) { + // Produce `number_of_frames` frames and give them to the estimator. + val overuse = generateAndProcessFrame(bitrateBps) + if (overuse) { + bitrateObserver.latestBitrate shouldBeLessThanOrEqual maxBitrate + bitrateObserver.latestBitrate shouldBeGreaterThanOrEqual minBitrate + bitrateBps = bitrateObserver.latestBitrate + bitrateUpdateSeen = true + } else if (bitrateObserver.updated) { + bitrateBps = bitrateObserver.latestBitrate + bitrateObserver.reset() + } + if (bitrateUpdateSeen && bitrateBps >= targetBitrate) { + break + } + } + bitrateUpdateSeen shouldBe true + return bitrateBps + } + + fun initialBehaviorTestHelper(expectedConvergeBitrate: Long) { + val kFramerate = 50 // 50 fps to avoid rounding errors. + val kFrameIntervalMs = 1000L / kFramerate + val kPacingInfo = PacedPacketInfo(0, 5, 5000) + var sendTimeMs = 0L + bitrateEstimator.latestEstimate() shouldBe null + bitrateObserver.updated shouldBe false + bitrateObserver.reset() + clock.elapse(1000.ms) + // Inserting packets for 5 seconds to get a valid estimate. + for (i in 0 until 5 * kFramerate + 1 + kNumInitialPackets) { + val pacingInfo = if (i < kInitialProbingPackets) { + kPacingInfo + } else { + PacedPacketInfo() + } + if (i == kNumInitialPackets) { + bitrateEstimator.latestEstimate() shouldBe null + bitrateObserver.updated shouldBe false + bitrateObserver.reset() + } + incomingFeedback(clock.roundedMillis(), sendTimeMs, kMtu, pacingInfo) + clock.elapse((1000 / kFramerate).ms) + sendTimeMs = kFrameIntervalMs + } + val bitrate = bitrateEstimator.latestEstimate() + bitrate shouldNotBe null + bitrate!!.bps shouldBeInRange (expectedConvergeBitrate plusOrMinus kAcceptedBitrateErrorBps) + bitrateObserver.updated shouldBe true + bitrateObserver.latestBitrate shouldBe bitrate.bps + } + + fun rateIncreaseReorderingTestHelper(expectedBitrateBps: Long) { + val kFramerate = 50 // fps to avoid rounding errors. + val kFrameIntervalMs = 1000L / kFramerate + val kPacingInfo = PacedPacketInfo(0, 5, 5000) + var sendTimeMs = 0L + // Inserting packets for five seconds to get a valid estimate. + for (i in 0 until 5 * kFramerate + 1 + kNumInitialPackets) { + // NOTE!!! If the following line is moved under the if case then this test + // wont work on windows realease bots. + val pacingInfo = if (i < kInitialProbingPackets) kPacingInfo else PacedPacketInfo() + + // TODO(sprang): Remove this hack once the single stream estimator is gone, + // as it doesn't do anything in Process(). + if (i == kNumInitialPackets) { + // Process after we have enough frames to get a valid input rate estimate. + + bitrateObserver.updated shouldBe false // No valid estimate. + } + incomingFeedback(clock.roundedMillis(), sendTimeMs, kMtu, pacingInfo) + clock.elapse(kFrameIntervalMs.ms) + sendTimeMs += kFrameIntervalMs + } + bitrateObserver.updated shouldBe true + expectedBitrateBps shouldBeInRange (bitrateObserver.latestBitrate plusOrMinus kAcceptedBitrateErrorBps) + + repeat(10) { + clock.elapse((2 * kFrameIntervalMs).ms) + sendTimeMs += 2 * kFrameIntervalMs + incomingFeedback(clock.roundedMillis(), sendTimeMs, 1000) + incomingFeedback(clock.roundedMillis(), sendTimeMs - kFrameIntervalMs, 1000) + } + bitrateObserver.updated shouldBe true + expectedBitrateBps shouldBeInRange (bitrateObserver.latestBitrate plusOrMinus kAcceptedBitrateErrorBps) + } + + // Make sure we initially increase the bitrate as expected. + fun rateIncreaseRtpTimestampsTestHelper(expectedIterations: Int) { + // This threshold corresponds approximately to increasing linearly with + // bitrate(i) = 1.04 * bitrate(i-1) + 1000 + // until bitrate(i) > 500000, with bitrate(1) ~= 30000. + var bitrateBps = 30000L + var iterations = 0 + addDefaultStream() + // Feed the estimator with a stream of packets and verify that it reaches + // 500 kbps at the expected time. + while (bitrateBps < 5e5) { + val overuse = generateAndProcessFrame(bitrateBps) + if (overuse) { + bitrateObserver.latestBitrate shouldBeGreaterThan bitrateBps + bitrateBps = bitrateObserver.latestBitrate + bitrateObserver.reset() + } else if (bitrateObserver.updated) { + bitrateBps = bitrateObserver.latestBitrate + bitrateObserver.reset() + } + ++iterations + } + iterations shouldBe expectedIterations + } + + fun capacityDropTestHelper( + numberOfStreams: Int, + wrapTimeStamp: Boolean, + expectedBitrateDropDelta: Long, + receiverClockOffsetChangeMs: Long + ) { + val kFramerate = 30 + val kStartBitrate = 900e3.toLong() + val kMinExpectedBitrate = 800e3.toLong() + val kMaxExpectedBitrate = 1100e3.toLong() + val kInitialCapacityBps = 1000e3.toLong() + val kReducedCapacityBps = 500e3.toLong() + + var steadyStateTime = 0 + if (numberOfStreams <= 1) { + steadyStateTime = 10 + addDefaultStream() + } else { + steadyStateTime = 10 * numberOfStreams + var bitrateSum = 0L + val kBitrateDenom = numberOfStreams * (numberOfStreams - 1) + for (i in 0 until numberOfStreams) { + // First stream gets half available bitrate, while the rest share the + // remaining half i.e.: 1/2 = Sum[n/(N*(N-1))] for n=1..N-1 (rounded up) + var bitrate = kStartBitrate / 2 + if (i > 0) { + bitrate = (kStartBitrate * i + kBitrateDenom / 2) / kBitrateDenom + } + streamGenerator.addStream(RtpStream(kFramerate, bitrate)) + bitrateSum += bitrate + } + check(bitrateSum == kStartBitrate) + } + + // Run in steady state to make the estimator converge. + streamGenerator.setCapacityBps(kInitialCapacityBps) + var bitrateBps = steadyStateRun( + steadyStateTime * kFramerate, + kStartBitrate, + kMinExpectedBitrate, + kMaxExpectedBitrate, + kInitialCapacityBps + ) + bitrateBps shouldBeInRange (kInitialCapacityBps plusOrMinus 180000) + bitrateObserver.reset() + + // Add an offset to make sure the BWE can handle it. + arrivalTimeOffsetMs += receiverClockOffsetChangeMs + + // Reduce the capacity and verify the decrease time. + streamGenerator.setCapacityBps(kReducedCapacityBps) + val overuseStartTime = clock.roundedMillis() + var bitrateDropTime = -1L + for (i in 0 until 100 * numberOfStreams) { + generateAndProcessFrame(bitrateBps) + if (bitrateDropTime == -1L && + bitrateObserver.latestBitrate <= kReducedCapacityBps + ) { + bitrateDropTime = clock.roundedMillis() + } + if (bitrateObserver.updated) { + bitrateBps = bitrateObserver.latestBitrate + } + } + + bitrateDropTime - overuseStartTime shouldBeInRange (expectedBitrateDropDelta plusOrMinus 33) + } + + fun testTimestampGroupingTestHelper() { + val kFramerate = 50 // 50 fps to avoid rounding errors. + val kFrameIntervalMs = 1000L / kFramerate + var sendTimeMs = 0L + // Initial set of frames to increase the bitrate. 6 seconds to have enough + // time for the first estimate to be generated and for Process() to be called. + for (i in 0 until 6 * kFramerate) { + incomingFeedback(clock.roundedMillis(), sendTimeMs, 1000) + + clock.elapse(kFrameIntervalMs.ms) + sendTimeMs += kFrameIntervalMs + } + bitrateObserver.updated shouldBe true + bitrateObserver.latestBitrate shouldBeGreaterThanOrEqual 400000 + + // Insert batches of frames which were sent very close in time. Also simulate + // capacity over-use to see that we back off correctly. + val kTimestampGroupLength = 15 + repeat(100) { + repeat(kTimestampGroupLength) { + // Insert `kTimestampGroupLength` frames with just 1 timestamp ticks in + // between. Should be treated as part of the same group by the estimator. + incomingFeedback(clock.roundedMillis(), sendTimeMs, 100) + clock.elapse((kFrameIntervalMs / kTimestampGroupLength).ms) + sendTimeMs += 1 + } + // Increase time until next batch to simulate over-use. + clock.elapse(10.ms) + sendTimeMs += kFrameIntervalMs - kTimestampGroupLength + } + bitrateObserver.updated shouldBe true + // Should have reduced the estimate. + bitrateObserver.latestBitrate shouldBeLessThan 400000 + } + + fun testWrappingHelper(silenceTimeS: Int) { + val kFramerate = 100 + val kFrameIntervalMs = 1000 / kFramerate + var sendTimeMs = 0L + + repeat(3000) { + incomingFeedback(clock.roundedMillis(), sendTimeMs, 1000) + clock.elapse(kFrameIntervalMs.ms) + sendTimeMs += kFrameIntervalMs + } + val bitrateBefore = bitrateEstimator.latestEstimate() + + clock.elapse(silenceTimeS.secs) + + repeat(24) { + incomingFeedback(clock.roundedMillis(), sendTimeMs, 1000) + clock.elapse((2 * kFrameIntervalMs).ms) + sendTimeMs += kFrameIntervalMs + } + val bitrateAfter = bitrateEstimator.latestEstimate() + bitrateAfter!! shouldBeLessThan bitrateBefore!! + } + + companion object { + const val kMtu = 1200L + const val kAcceptedBitrateErrorBps = 50000L + + // Number of packets needed before we have a valid estimate. + const val kNumInitialPackets = 2 + const val kInitialProbingPackets = 5 + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkControllerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkControllerTest.kt new file mode 100644 index 0000000..5713083 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/GoogCcNetworkControllerTest.kt @@ -0,0 +1,291 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.times +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Instant + +/** Tests of GoogCcNetworkController, + * based on WebRTC modules/congestion_controller/goog_cc/goog_cc_network_control_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Only tests that don't require the full network simulation environment are included. + */ + +private const val kInitialBitrateKbps = 60 +private val kInitialBitrate = kInitialBitrateKbps.kbps +private const val kDefaultPacingRate = 2.5 + +private fun createRouteChange( + time: Instant, + startRate: Bandwidth? = null, + minRate: Bandwidth? = null, + maxRate: Bandwidth? = null +): NetworkRouteChange = NetworkRouteChange( + atTime = time, + constraints = TargetRateConstraints(minDataRate = minRate, maxDataRate = maxRate, startingRate = startRate) +) + +private fun createPacketResult( + arrivalTime: Instant, + sendTime: Instant, + payloadSize: Long, + pacingInfo: PacedPacketInfo +): PacketResult { + val packetResult = PacketResult() + packetResult.sentPacket = SentPacket() + packetResult.sentPacket.sendTime = sendTime + packetResult.sentPacket.size = payloadSize.bytes + packetResult.sentPacket.pacingInfo = pacingInfo + packetResult.receiveTime = arrivalTime + return packetResult +} + +class Ref( + var v: T +) + +// Simulate sending packets and receiving transport feedback during +// `runtime_ms`, then return the final target birate. +fun packetTransmissionAndFeedbackBlock( + controller: NetworkControllerInterface, + runtimeMs: Long, + delay: Long, + currentTimeRef: Ref +): Bandwidth? { + var currentTime = currentTimeRef.v + var update: NetworkControlUpdate + var targetBitrate: Bandwidth? = null + var delayBuildup = 0L + val startTimeMs = currentTime.toEpochMilli() + while (currentTime.toEpochMilli() - startTimeMs < runtimeMs) { + val kPayloadSize = 1000L + val packet = createPacketResult(currentTime + delayBuildup.ms, currentTime, kPayloadSize, PacedPacketInfo()) + delayBuildup += delay + update = controller.onSentPacket(packet.sentPacket) + if (update.targetRate != null) { + targetBitrate = update.targetRate!!.targetRate + } + val feedback = TransportPacketsFeedback() + feedback.feedbackTime = packet.receiveTime + feedback.packetFeedbacks.add(packet) + update = controller.onTransportPacketsFeedback(feedback) + if (update.targetRate != null) { + targetBitrate = update.targetRate!!.targetRate + } + currentTime += 50.ms + update = controller.onProcessInterval(ProcessInterval(atTime = currentTime)) + if (update.targetRate != null) { + targetBitrate = update.targetRate!!.targetRate + } + } + currentTimeRef.v = currentTime + return targetBitrate +} + +// Scenarios: + +class NetworkControllerTextFixture( + googccConfig: GoogCcFactoryConfig = GoogCcFactoryConfig() +) { + private val logger = createLogger() + private val diagnosticContext = DiagnosticContext() + + private val factory = GoogCcNetworkControllerFactory(googccConfig) + + fun createController(): NetworkControllerInterface { + val config = initialConfig() + val controller = factory.create(config) + return controller + } + + private fun initialConfig( + startingBandwidthKbps: Int = kInitialBitrateKbps, + minDataRateKbps: Int = 0, + maxDataRateKbps: Int = 5 * kInitialBitrateKbps + ): NetworkControllerConfig { + val config = NetworkControllerConfig( + logger, + diagnosticContext, + constraints = TargetRateConstraints( + atTime = Instant.ofEpochSecond(0), + minDataRate = minDataRateKbps.kbps, + maxDataRate = maxDataRateKbps.kbps, + startingRate = startingBandwidthKbps.kbps + ) + ) + return config + } +} + +class GoogCcNetworkControllerTest : FreeSpec() { + init { + "InitializeTargetRateOnFirstProcessInterval" { + val fixture = NetworkControllerTextFixture() + val controller = fixture.createController() + + var update = + controller.onNetworkAvailability( + NetworkAvailability( + atTime = Instant.ofEpochMilli(123456), + networkAvailable = true + ) + ) + update = controller.onProcessInterval(ProcessInterval(atTime = Instant.ofEpochMilli(123456))) + + update.targetRate!!.targetRate shouldBe kInitialBitrate + update.pacerConfig!!.dataRate() shouldBe kInitialBitrate * kDefaultPacingRate + update.probeClusterConfigs[0].targetDataRate shouldBe kInitialBitrate * 3 + update.probeClusterConfigs[1].targetDataRate shouldBe kInitialBitrate * 5 + } + + "ReactsToChangedNetworkConditions" { + val fixture = NetworkControllerTextFixture() + val controller = fixture.createController() + var currentTime = Instant.ofEpochMilli(123) + var update = + controller.onNetworkAvailability(NetworkAvailability(atTime = currentTime, networkAvailable = true)) + update = controller.onProcessInterval(ProcessInterval(atTime = currentTime)) + update = controller.onRemoteBitrateReport( + RemoteBitrateReport(receiveTime = currentTime, bandwidth = kInitialBitrate * 2) + ) + + currentTime += 25.ms + update = controller.onProcessInterval(ProcessInterval(atTime = currentTime)) + update.targetRate!!.targetRate shouldBe kInitialBitrate * 2 + update.pacerConfig!!.dataRate() shouldBe kInitialBitrate * 2 * kDefaultPacingRate + + update = controller.onRemoteBitrateReport( + RemoteBitrateReport(receiveTime = currentTime, bandwidth = kInitialBitrate) + ) + currentTime += 25.ms + update = controller.onProcessInterval(ProcessInterval(atTime = currentTime)) + update.targetRate!!.targetRate shouldBe kInitialBitrate + update.pacerConfig!!.dataRate() shouldBe kInitialBitrate * kDefaultPacingRate + } + + "OnNetworkRouteChanged" { + val fixture = NetworkControllerTextFixture() + val controller = fixture.createController() + val currentTime = Instant.ofEpochMilli(123) + var update = + controller.onNetworkAvailability(NetworkAvailability(atTime = currentTime, networkAvailable = true)) + val newBitrate = 200000.bps + + update = controller.onNetworkRouteChange(createRouteChange(currentTime, newBitrate)) + update.targetRate!!.targetRate shouldBe newBitrate + update.pacerConfig!!.dataRate() shouldBe newBitrate * kDefaultPacingRate + update.probeClusterConfigs.size shouldBe 2 + + // If the bitrate is reset to -1, the new starting bitrate will be + // the minimum default bitrate. + val kDefaultMinBitrate = 5.kbps + update = controller.onNetworkRouteChange(createRouteChange(currentTime)) + update.targetRate!!.targetRate shouldBe kDefaultMinBitrate + update.pacerConfig!!.dataRate().bps.toDouble() shouldBe + kDefaultMinBitrate.bps * kDefaultPacingRate plusOrMinus 10.0 + update.probeClusterConfigs.size shouldBe 2 + } + + "ProbeOnRouteChange" { + val fixture = NetworkControllerTextFixture() + val controller = fixture.createController() + var currentTime = Instant.ofEpochMilli(123) + var update = + controller.onNetworkAvailability(NetworkAvailability(atTime = currentTime, networkAvailable = true)) + currentTime += 1.secs + + update = controller.onNetworkRouteChange( + createRouteChange(currentTime, 2 * kInitialBitrate, Bandwidth.ZERO, 20 * kInitialBitrate) + ) + + update.pacerConfig shouldNotBe null + update.targetRate!!.targetRate shouldBe kInitialBitrate * 2 + update.probeClusterConfigs.size shouldBe 2 + update.probeClusterConfigs[0].targetDataRate shouldBe kInitialBitrate * 6 + update.probeClusterConfigs[1].targetDataRate shouldBe kInitialBitrate * 12 + + update = controller.onProcessInterval(ProcessInterval(atTime = currentTime)) + } + + "ProbeAfterRouteChangeWhenTransportWritable" { + val fixture = NetworkControllerTextFixture() + val controller = fixture.createController() + val currentTime = Instant.ofEpochMilli(123) + + var update = controller.onNetworkAvailability( + NetworkAvailability(atTime = currentTime, networkAvailable = false) + ) + update.probeClusterConfigs.isEmpty() shouldBe true + + update = controller.onNetworkRouteChange( + createRouteChange( + currentTime, + 2 * kInitialBitrate, + Bandwidth.ZERO, + 20 * kInitialBitrate + ) + ) + // Transport is not writable. So not point in sending a probe. + update.probeClusterConfigs.isEmpty() shouldBe true + + // Probe is sent when transport becomes writable. + update = controller.onNetworkAvailability( + NetworkAvailability(atTime = currentTime, networkAvailable = true) + ) + update.probeClusterConfigs.isNotEmpty() shouldBe true + } + + // Bandwidth estimation is updated when feedbacks are received. + // Feedbacks which show an increasing delay cause the estimation to be reduced. + "UpdatesDelayBasedEstimate" { + val fixture = NetworkControllerTextFixture() + val controller = fixture.createController() + val kRunTimeMs = 6000L + val currentTime = Ref(Instant.ofEpochMilli(123)) + var update = + controller.onNetworkAvailability(NetworkAvailability(atTime = currentTime.v, networkAvailable = true)) + + // The test must run and insert packets/feedback long enough that the + // BWE computes a valid estimate. This is first done in an environment which + // simulates no bandwidth limitation, and therefore not built-up delay. + val targetBitrateBeforeDelay = + packetTransmissionAndFeedbackBlock(controller, kRunTimeMs, 0, currentTime) + targetBitrateBeforeDelay shouldNotBe null + + // Repeat, but this time with a building delay, and make sure that the + // estimation is adjusted downwards. + val targetBitrateAfterDelay = + packetTransmissionAndFeedbackBlock(controller, kRunTimeMs, 50, currentTime) + targetBitrateAfterDelay!! shouldBeLessThan targetBitrateBeforeDelay!! + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudgetTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudgetTest.kt new file mode 100644 index 0000000..5e500c5 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/IntervalBudgetTest.kt @@ -0,0 +1,119 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// This file uses WebRTC's naming style for constants +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.shouldBe + +const val kWindowMs = 500L +const val kBitrateKbps = 100 +const val kCanBuildUpUnderuse = true +const val kCanNotBuildUpUnderuse = false +fun timeToBytes(bitrateKbps: Int, timeMs: Long): Long = bitrateKbps.toLong() * timeMs / 8 + +/** Unit tests for IntervalBudget, + * based on WebRTC modules/pacing/interval_budget_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +class IntervalBudgetTest : FreeSpec() { + init { + "InitailState" { + val intervalBudget = IntervalBudget(kBitrateKbps) + intervalBudget.budgetRatio() shouldBe 0.0 + intervalBudget.bytesRemaining() shouldBe 0 + } + + "Underuse" { + val intervalBudget = IntervalBudget(kBitrateKbps) + val deltaTimeMs = 50L + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.budgetRatio() shouldBe kWindowMs / (100 * deltaTimeMs).toDouble() + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, deltaTimeMs) + } + + "DontUnderuseMoreThanMaxWindow" { + val intervalBudget = IntervalBudget(kBitrateKbps) + val deltaTimeMs = 1000L + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.budgetRatio() shouldBe 1.0 + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, kWindowMs) + } + + "DontUnderuseMoreThanMaxWindowWhenChangeBitrate" { + val intervalBudget = IntervalBudget(kBitrateKbps) + val deltaTimeMs = kWindowMs / 2 + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.setTargetRateKbps(kBitrateKbps / 10) + intervalBudget.budgetRatio() shouldBe 1.0 + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps / 10, kWindowMs) + } + + "BalanceChangeOnBitrateChange" { + val intervalBudget = IntervalBudget(kBitrateKbps) + val deltaTimeMs = kWindowMs + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.setTargetRateKbps(kBitrateKbps * 2) + intervalBudget.budgetRatio() shouldBe 0.5 + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, kWindowMs) + } + + "Overuse" { + val intervalBudget = IntervalBudget(kBitrateKbps) + val overuseTimeMs = 50L + val usedBytes = timeToBytes(kBitrateKbps, overuseTimeMs) + intervalBudget.useBudget(usedBytes) + intervalBudget.budgetRatio() shouldBe -kWindowMs / (100 * overuseTimeMs).toDouble() + intervalBudget.bytesRemaining() shouldBe 0 + } + + "DontOveruseMoreThanMaxWindow" { + val intervalBudget = IntervalBudget(kBitrateKbps) + val overuseTimeMs = 1000L + val usedBytes = timeToBytes(kBitrateKbps, overuseTimeMs) + intervalBudget.useBudget(usedBytes) + intervalBudget.budgetRatio() shouldBe -1.0 + intervalBudget.bytesRemaining() shouldBe 0 + } + + "CanBuildUpUnderuseWhenConfigured" { + val intervalBudget = IntervalBudget(kBitrateKbps, kCanBuildUpUnderuse) + val deltaTimeMs = 50L + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.budgetRatio() shouldBe kWindowMs / (100 * deltaTimeMs).toDouble() + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, deltaTimeMs) + + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.budgetRatio() shouldBe 2 * kWindowMs / (100 * deltaTimeMs).toDouble() + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, 2 * deltaTimeMs) + } + + "CanNotBuildUpUnderuseWhenConfigured" { + val intervalBudget = IntervalBudget(kBitrateKbps, kCanNotBuildUpUnderuse) + val deltaTimeMs = 50L + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.budgetRatio() shouldBe kWindowMs / (100 * deltaTimeMs).toDouble() + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, deltaTimeMs) + + intervalBudget.increaseBudget(deltaTimeMs) + intervalBudget.budgetRatio() shouldBe kWindowMs / (100 * deltaTimeMs).toDouble() + intervalBudget.bytesRemaining() shouldBe timeToBytes(kBitrateKbps, deltaTimeMs) + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2Test.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2Test.kt new file mode 100644 index 0000000..03a5db0 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/LossBasedBweV2Test.kt @@ -0,0 +1,1609 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.comparables.shouldBeGreaterThanOrEqualTo +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.comparables.shouldBeLessThanOrEqualTo +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.div +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.times +import org.jitsi.utils.NEVER +import org.jitsi.utils.div +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.times +import java.time.Duration +import java.time.Instant + +/** + * Unit tests for Loss-Based BWE, + * based on WebRTC modules/congestion_controller/goog_cc/loss_based_bwe_v2_test.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ + +val kObservationDurationLowerBound = 250.ms +val kDelayedIncreaseWindow = 300.ms +const val kMaxIncreaseFactor = 1.5 +const val kPacketSize = 15_000 + +private var transportSequenceNumber = 0L + +private fun config(enabled: Boolean, valid: Boolean): LossBasedBweV2.Config = LossBasedBweV2.Config( + enabled = enabled, + bandwidthRampupUpperBoundFactor = if (valid) 1.2 else 0.0, + candidateFactors = doubleArrayOf(1.1, 1.0, 0.95), + higherBandwidthBiasFactor = 0.01, + inherentLossLowerBound = 0.001, + inherentLossUpperBoundBandwidthBalance = 14.kbps, + inherentLossUpperBoundOffset = 0.9, + initialInherentLossEstimate = 0.01, + newtonIterations = 2, + newtonStepSize = 0.4, + observationWindowSize = 15, + sendingRateSmoothingFactor = 0.01, + instantUpperBoundTemporalWeightFactor = 0.97, + instantUpperBoundBandwidthBalance = 90.kbps, + instantUpperBoundLossOffset = 0.1, + temporalWeightFactor = 0.98, + minNumObservations = 1, + observationDurationLowerBound = kObservationDurationLowerBound, + maxIncreaseFactor = kMaxIncreaseFactor, + delayedIncreaseWindow = kDelayedIncreaseWindow +) + +private fun shortObservationConfig(config: LossBasedBweV2.Config = LossBasedBweV2.Config()): LossBasedBweV2.Config = + config.copy( + minNumObservations = 1, + observationWindowSize = 2 + ) + +private fun createPacketResultsWithReceivedPackets(firstPacketTimestamp: Instant): List { + val enoughFeedback = List(2) { PacketResult() } + enoughFeedback[0].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[1].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[0].sentPacket.size = kPacketSize.bytes + enoughFeedback[1].sentPacket.size = kPacketSize.bytes + enoughFeedback[0].sentPacket.sendTime = firstPacketTimestamp + enoughFeedback[1].sentPacket.sendTime = firstPacketTimestamp + kObservationDurationLowerBound + enoughFeedback[0].receiveTime = firstPacketTimestamp + kObservationDurationLowerBound + enoughFeedback[1].receiveTime = firstPacketTimestamp + 2 * kObservationDurationLowerBound + + return enoughFeedback +} + +private fun createPacketResultsWith10pPacketLossRate( + firstPacketTimestamp: Instant, + lostPacketSize: DataSize = kPacketSize.bytes +): List { + val enoughFeedback = List(10) { PacketResult() } + + for (i in enoughFeedback.indices) { + enoughFeedback[i].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[i].sentPacket.size = kPacketSize.bytes + enoughFeedback[i].sentPacket.sendTime = firstPacketTimestamp + i * kObservationDurationLowerBound + enoughFeedback[i].receiveTime = firstPacketTimestamp + (i + 1) * kObservationDurationLowerBound + } + enoughFeedback[9].receiveTime = NEVER + enoughFeedback[9].sentPacket.size = lostPacketSize + return enoughFeedback +} + +private fun createPacketResultsWith50pPacketLossRate(firstPacketTimestamp: Instant): List { + val enoughFeedback = List(2) { PacketResult() } + enoughFeedback[0].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[1].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[0].sentPacket.size = kPacketSize.bytes + enoughFeedback[1].sentPacket.size = kPacketSize.bytes + enoughFeedback[0].sentPacket.sendTime = firstPacketTimestamp + enoughFeedback[1].sentPacket.sendTime = firstPacketTimestamp + kObservationDurationLowerBound + enoughFeedback[0].receiveTime = firstPacketTimestamp + kObservationDurationLowerBound + enoughFeedback[1].receiveTime = NEVER + return enoughFeedback +} + +private fun createPacketResultsWith100pLossRate( + firstPacketTimestamp: Instant, + numPackets: Int = 2 +): List { + val enoughFeedback = List(numPackets) { PacketResult() } + for (i in 0 until numPackets - 1) { + enoughFeedback[i].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[i].sentPacket.size = kPacketSize.bytes + enoughFeedback[i].sentPacket.sendTime = firstPacketTimestamp + (i * 10).ms + enoughFeedback[i].receiveTime = NEVER + } + enoughFeedback[numPackets - 1].sentPacket.sequenceNumber = transportSequenceNumber++ + enoughFeedback[numPackets - 1].sentPacket.size = kPacketSize.bytes + enoughFeedback[numPackets - 1].sentPacket.sendTime = firstPacketTimestamp + kObservationDurationLowerBound + enoughFeedback[numPackets - 1].receiveTime = NEVER + return enoughFeedback +} + +class LossBasedBweV2Test : FreeSpec() { + init { + "EnabledWhenGivenValidConfigurationValues" { + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.isEnabled() shouldBe true + } + "DisabledWhenGivenDisabledConfiguration" { + val config = config(enabled = false, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.isEnabled() shouldBe false + } + "DisabledWhenGivenNonValidConfigurationValues" { + val config = config(enabled = true, valid = false) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.isEnabled() shouldBe false + } + "DisabledWhenGivenNonPositiveCandidateFactor" { + val configNegativeCandidateFactor = + LossBasedBweV2.Config(candidateFactors = doubleArrayOf(-1.3, 1.1)) + val lossBasedBandwidthEstimator1 = LossBasedBweV2(configNegativeCandidateFactor) + lossBasedBandwidthEstimator1.isEnabled() shouldBe false + + val configZeroCandidateFactor = + LossBasedBweV2.Config(candidateFactors = doubleArrayOf(-0.0, 1.1)) + val lossBasedBandwidthEstimator2 = LossBasedBweV2(configZeroCandidateFactor) + lossBasedBandwidthEstimator2.isEnabled() shouldBe false + } + "DisabledWhenGivenConfigurationThatDoesNotAllowGeneratingCandidates" { + val config = + LossBasedBweV2.Config( + candidateFactors = doubleArrayOf(1.0), + appendAcknowledgedRateCandidate = false, + appendDelayBasedEstimateCandidate = false + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.isEnabled() shouldBe false + } + "ReturnsDelayBasedEstimateWhenDisabled" { + val config = config(enabled = false, valid = true,) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + listOf(), + 100.kbps, + false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 100.kbps + } + "ReturnsDelayBasedEstimateWhenWhenGivenNonValidConfigurationValues" { + val config = config(enabled = true, valid = false) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + listOf(), + 100.kbps, + false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 100.kbps + } + "BandwidthEstimateGivenInitializationAndThenFeedback" { + val enoughFeedback = createPacketResultsWithReceivedPackets(Instant.ofEpochMilli(0)) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator.isReady() shouldBe true + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate.isFinite() shouldBe true + } + "NoBandwidthEstimateGivenNoInitialization" { + val enoughFeedback = createPacketResultsWithReceivedPackets(Instant.EPOCH) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback, + Bandwidth.INFINITY, + false + ) + lossBasedBandwidthEstimator.isReady() shouldBe false + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe Bandwidth.INFINITY + } + "NoBandwidthEstimateGivenNotEnoughFeedback" { + val notEnoughFeedback = List(2) { PacketResult() } + notEnoughFeedback[0].sentPacket.size = 15_000.bytes + notEnoughFeedback[1].sentPacket.size = 15_000.bytes + notEnoughFeedback[0].sentPacket.sendTime = Instant.EPOCH + notEnoughFeedback[1].sentPacket.sendTime = Instant.EPOCH + kObservationDurationLowerBound / 2 + notEnoughFeedback[0].receiveTime = Instant.EPOCH + kObservationDurationLowerBound / 2 + notEnoughFeedback[1].receiveTime = Instant.EPOCH + kObservationDurationLowerBound + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + lossBasedBandwidthEstimator.isReady() shouldBe false + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe Bandwidth.INFINITY + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + notEnoughFeedback, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator.isReady() shouldBe false + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe Bandwidth.INFINITY + } + + "SetValueIsTheEstimateUntilAdditionalFeedbackHasBeenReceived" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldNotBe 600.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 600.kbps + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldNotBe 600.kbps + } + "SetAcknowledgedBitrateOnlyAffectsTheBweWhenAdditionalFeedbackIsGiven" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator1 = LossBasedBweV2(config) + val lossBasedBandwidthEstimator2 = LossBasedBweV2(config) + + lossBasedBandwidthEstimator1.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator2.setBandwidthEstimate(600.kbps) + + lossBasedBandwidthEstimator1.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator2.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator1.getLossBasedResult().bandwidthEstimate shouldBe 660.kbps + + lossBasedBandwidthEstimator1.setAcknowledgedBitrate(900.kbps) + + lossBasedBandwidthEstimator1.getLossBasedResult().bandwidthEstimate shouldBe 660.kbps + + lossBasedBandwidthEstimator1.updateBandwidthEstimate( + enoughFeedback2, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator2.updateBandwidthEstimate( + enoughFeedback2, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator1.getLossBasedResult().bandwidthEstimate shouldNotBe + lossBasedBandwidthEstimator2.getLossBasedResult().bandwidthEstimate + } + "BandwidthEstimateIsCappedToBeTcpFairGivenTooHighLossRate" { + val enoughFeedbackNoReceivedPackets = createPacketResultsWith100pLossRate(Instant.EPOCH) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedbackNoReceivedPackets, + Bandwidth.INFINITY, + false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 100.kbps + } + + // When network is normal, estimate can increase but never be higher than + // the delay based estimate. + "BandwidthEstimateCappedByDelayBasedEstimateWhenNetworkNormal" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + + // If the delay based estimate is infinity, then loss based estimate increases + // and not bounded by delay based estimate. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeGreaterThan 600.kbps + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + 500.kbps, + false + ) + // If the delay based estimate is not infinity, then loss based estimate is + // bounded by delay based estimate. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 500.kbps + } + + // When loss based bwe receives a strong signal of overusing and an increase in + // loss rate, it should acked bitrate for emegency backoff. + "UseAckedBitrateForEmegencyBackOff" { + // Create two packet results, first packet has 50% loss rate, second packet + // has 100% loss rate. + val enoughFeedback1 = createPacketResultsWith50pPacketLossRate(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWith100pLossRate( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ) + + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + val ackedBitrate = 300.kbps + + lossBasedBandwidthEstimator.setAcknowledgedBitrate(ackedBitrate) + // Update estimate when network is overusing, and 50% loss rate. + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + // Update estimate again when network is continuously overusing, and 100% + // loss rate. + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + Bandwidth.INFINITY, + false + ) + // The estimate bitrate now is backed off based on acked bitrate. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThanOrEqualTo + ackedBitrate + } + + // When receiving the same packet feedback, loss based bwe ignores the feedback + // and returns the current estimate. + "NoBweChangeIfObservationDurationUnchanged" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(300.kbps) + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + + val estimate1 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + + // Use the same feedback and check if the estimate is unchanged. + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + + val estimate2 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + estimate1 shouldBe estimate2 + } + + // When receiving feedback of packets that were sent within an observation + // duration, and network is in the normal state, loss based bwe returns the + // current estimate. + "NoBweChangeIfObservationDurationIsSmallAndNetworkNormal" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound - 1.ms + ) + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + val estimate1 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + Bandwidth.INFINITY, + false + ) + val estimate2 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + estimate2 shouldBe estimate1 + } + + // When receiving feedback of packets that were sent within an observation + // duration, and network is in the underusing state, loss based bwe returns the + // current estimate. + "NoBweIncreaseIfObservationDurationIsSmallAndNetworkUnderusing" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound - 1.ms + ) + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + Bandwidth.INFINITY, + false + ) + val estimate1 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + Bandwidth.INFINITY, + false + ) + val estimate2 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + estimate2 shouldBeLessThanOrEqualTo estimate1 + } + + "IncreaseToDelayBasedEstimateIfNoLossOrDelayIncrease" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ) + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate, + false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe delayBasedEstimate + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + delayBasedEstimate, + false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe delayBasedEstimate + } + + "IncreaseByHoldFactorAfterLossBasedBweBacksOff" { + val config = shortObservationConfig() + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + val ackedRate = 100.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(ackedRate) + + // Create some loss to create the loss limited scenario. + val enoughFeedback1 = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate, + inAlr = false + ) + val resultAtLoss = lossBasedBandwidthEstimator.getLossBasedResult() + + // Network recovers after loss. + var feedbackCount = 2 + while (lossBasedBandwidthEstimator.getLossBasedResult().state != LossBasedState.kIncreaseUsingPadding) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound * feedbackCount + ), + delayBasedEstimate = delayBasedEstimate, + inAlr = false + ) + feedbackCount++ + } + + val resultAfterRecovery = lossBasedBandwidthEstimator.getLossBasedResult() + // BwRampupUpperBoundInHoldFactor is 1.2. + resultAfterRecovery.bandwidthEstimate shouldBe resultAtLoss.bandwidthEstimate * 1.2 + } + + "LossBasedStateIsDelayBasedEstimateAfterNetworkRecovering" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + candidateFactors = doubleArrayOf(100.0, 1.0, 0.5), + observationDurationLowerBound = 200.ms, + instantUpperBoundBandwidthBalance = 10000.kbps, + maxIncreaseFactor = 100.0, + notIncreaseIfInherentLossLessThanAverageLoss = false + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 600.kbps + val ackedRate = 300.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(ackedRate) + + // Create some loss to create the loss limited scenario. + val enoughFeedback1 = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + + // Network recovers after loss. + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound + ) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + delayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDelayBasedEstimate + + // Network recovers continuing. + val enoughFeedback3 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound * 2 + ) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback3, + delayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDelayBasedEstimate + } + + "LossBasedStateIsNotDelayBasedEstimateIfDelayBasedEstimateInfinite" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + candidateFactors = doubleArrayOf(100.0, 1.0, 0.5), + instantUpperBoundBandwidthBalance = 10000.kbps, + maxIncreaseFactor = 100.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + // Create some loss to create the loss limited scenario. + val enoughFeedback1 = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + + // Network recovers after loss. + val enoughFeedback2 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound + ) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(600.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldNotBe LossBasedState.kDelayBasedEstimate + } + + // Ensure that the state can switch to kIncrease even when the bandwidth is + // bounded by acked bitrate. + "EnsureIncreaseEvenIfAckedBitrateBound" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lossThresholdOfHighBandwidthPreference = 0.99, + bandwidthRampupUpperBoundFactor = 1.2, + // Set InstantUpperBoundBwBalance high to disable InstantUpperBound cap. + instantUpperBoundBandwidthBalance = 10000.kbps + ) + ) + val enoughFeedback1 = + createPacketResultsWith100pLossRate(Instant.EPOCH) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(300.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate(enoughFeedback1, delayBasedEstimate, inAlr = false) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + var result = lossBasedBandwidthEstimator.getLossBasedResult() + val estimate1 = result.bandwidthEstimate + estimate1.kbps shouldBeLessThan 600.0 + + // Set a low acked bitrate. + lossBasedBandwidthEstimator.setAcknowledgedBitrate(estimate1 / 2) + + var feedbackCount = 1 + while (result.state != LossBasedState.kIncreaseUsingPadding) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackCount++ * kObservationDurationLowerBound + ), + delayBasedEstimate, + inAlr = false + ) + result = lossBasedBandwidthEstimator.getLossBasedResult() + } + + result.state shouldBe LossBasedState.kIncreaseUsingPadding + // The estimate increases by 1kbps. + result.bandwidthEstimate shouldBe estimate1 + 1.bps + } + + // After loss based bwe backs off, the next estimate is capped by + // a factor of acked bitrate. + "IncreaseByFactorOfAckedBitrateAfterLossBasedBweBacksOff" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lossThresholdOfHighBandwidthPreference = 0.99, + bandwidthRampupUpperBoundFactor = 1.2, + // Set InstantUpperBoundBwBalance high to disable InstantUpperBound cap. + instantUpperBoundBandwidthBalance = 10000.kbps + ) + ) + val enoughFeedback1 = createPacketResultsWith100pLossRate(Instant.EPOCH) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(300.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + var result = lossBasedBandwidthEstimator.getLossBasedResult() + val estimate1 = result.bandwidthEstimate + estimate1.kbps shouldBeLessThan 600.0 + + lossBasedBandwidthEstimator.setAcknowledgedBitrate(estimate1 * 0.9) + var feedbackCount = 1 + while (result.state != LossBasedState.kIncreaseUsingPadding) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackCount++ * kObservationDurationLowerBound + ), + delayBasedEstimate, + inAlr = false + ) + result = lossBasedBandwidthEstimator.getLossBasedResult() + } + result.state shouldBe LossBasedState.kIncreaseUsingPadding + + // The estimate is capped by acked_bitrate * BwRampupUpperBoundFactor. + result.bandwidthEstimate shouldBe estimate1 * 0.9 * 1.2 + + // But if acked bitrate decreases, BWE does not decrease when there is no + // loss. + lossBasedBandwidthEstimator.setAcknowledgedBitrate(estimate1 * 0.9) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackCount++ * kObservationDurationLowerBound + ), + delayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe result.bandwidthEstimate + } + + "EstimateBitrateIsBoundedDuringDelayedWindowAfterLossBasedBweBacksOff" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = createPacketResultsWith50pPacketLossRate( + Instant.EPOCH + kDelayedIncreaseWindow - 2.ms + ) + val enoughFeedback3 = createPacketResultsWithReceivedPackets( + Instant.EPOCH + kDelayedIncreaseWindow - 1.ms + ) + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(300.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate, + inAlr = false + ) + // Increase the acknowledged bitrate to make sure that the estimate is not + // capped too low. + lossBasedBandwidthEstimator.setAcknowledgedBitrate(5000.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + delayBasedEstimate, + inAlr = false + ) + + // The estimate is capped by current_estimate * kMaxIncreaseFactor because + // it recently backed off. + val estimate2 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback3, + delayBasedEstimate, + inAlr = false + ) + // The latest estimate is the same as the previous estimate since the sent + // packets were sent within the DelayedIncreaseWindow. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe estimate2 + } + + // The estimate is not bounded after the delayed increase window. + "KeepIncreasingEstimateAfterDelayedIncreaseWindow" { + val enoughFeedback1 = createPacketResultsWithReceivedPackets(Instant.EPOCH) + val enoughFeedback2 = + createPacketResultsWithReceivedPackets(Instant.EPOCH + kDelayedIncreaseWindow - 1.ms) + val enoughFeedback3 = + createPacketResultsWithReceivedPackets(Instant.EPOCH + kDelayedIncreaseWindow + 1.ms) + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(300.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback1, + delayBasedEstimate, + inAlr = false + ) + // Increase the acknowledged bitrate to make sure that the estimate is not + // capped too low. + lossBasedBandwidthEstimator.setAcknowledgedBitrate(5000.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback2, + delayBasedEstimate, + inAlr = false + ) + + // The estimate is capped by current_estimate * kMaxIncreaseFactor because + // it recently backed off. + val estimate2 = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback3, + delayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeGreaterThanOrEqualTo + estimate2 + } + + "NotIncreaseIfInherentLossLessThanAverageLoss" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + candidateFactors = doubleArrayOf(1.2), + notIncreaseIfInherentLossLessThanAverageLoss = true + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + val enoughFeedback10pLoss1 = createPacketResultsWith10pPacketLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss1, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + val enoughFeedback10pLoss2 = + createPacketResultsWith10pPacketLossRate(Instant.EPOCH + kObservationDurationLowerBound) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss2, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + // Do not increase the bitrate because inherent loss is less than average loss + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 600.kbps + } + + "SelectHighBandwidthCandidateIfLossRateIsLessThanThreshold" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lossThresholdOfHighBandwidthPreference = 0.20, + notIncreaseIfInherentLossLessThanAverageLoss = false + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + val enoughFeedback10pLoss1 = createPacketResultsWith10pPacketLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss1, + delayBasedEstimate, + inAlr = false + ) + + val enoughFeedback10pLoss2 = + createPacketResultsWith10pPacketLossRate(Instant.EPOCH + kObservationDurationLowerBound) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss2, + delayBasedEstimate, + inAlr = false + ) + + // Because LossThresholdOfHighBandwidthPreference is 20%, the average loss is + // 10%, bandwidth estimate should increase. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeGreaterThan 600.kbps + } + + "SelectLowBandwidthCandidateIfLossRateIsIsHigherThanThreshold" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lossThresholdOfHighBandwidthPreference = 0.05 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val delayBasedEstimate = 5000.kbps + + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + val enoughFeedback10pLoss1 = createPacketResultsWith10pPacketLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss1, + delayBasedEstimate, + inAlr = false + ) + + val enoughFeedback10pLoss2 = + createPacketResultsWith10pPacketLossRate(Instant.EPOCH + kObservationDurationLowerBound) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss2, + delayBasedEstimate, + inAlr = false + ) + + // Because LossThresholdOfHighBandwidthPreference is 5%, the average loss is + // 10%, bandwidth estimate should decrease. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan 600.kbps + } + + "EstimateIsNotHigherThanMaxBitrate" { + val config = config(enabled = true, valid = true) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000.kbps) + lossBasedBandwidthEstimator.setBandwidthEstimate(1000.kbps) + val enoughFeedback = createPacketResultsWithReceivedPackets(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThanOrEqualTo 1000.kbps + } + + "NotBackOffToAckedRateInAlr" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + instantUpperBoundBandwidthBalance = 100.kbps + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000000.kbps) + val delayBasedEstimate = 5000.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + val ackedRate = 100.kbps + lossBasedBandwidthEstimator.setAcknowledgedBitrate(ackedRate) + val enoughFeedback100pLoss1 = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback100pLoss1, + delayBasedEstimate, + inAlr = true + ) + + // Make sure that the estimate decreases but higher than acked rate. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeGreaterThan ackedRate + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan 600.kbps + } + + "BackOffToAckedRateIfNotInAlr" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + instantUpperBoundBandwidthBalance = 100.kbps + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000000.kbps) + val delayBasedEstimate = 5000.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + + val ackedRate = 100.kbps + lossBasedBandwidthEstimator.setAcknowledgedBitrate(ackedRate) + val enoughFeedback100pLoss1 = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback100pLoss1, + delayBasedEstimate, + inAlr = false + ) + + // Make sure that the estimate decreases but higher than acked rate. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe ackedRate + } + + "NotReadyToUseInStartPhase" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + useInStartPhase = true + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + // Make sure that the estimator is not ready to use in start phase because of + // lacking TWCC feedback. + lossBasedBandwidthEstimator.readyToUseInStartPhase() shouldBe false + } + + "ReadyToUseInStartPhase" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + useInStartPhase = true + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val enoughFeedback = createPacketResultsWith100pLossRate(Instant.EPOCH) + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback, + delayBasedEstimate = 600.kbps, + inAlr = false + ) + lossBasedBandwidthEstimator.readyToUseInStartPhase() shouldBe true + } + + "BoundEstimateByAckedRate" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lowerBoundByAckedRateFactor = 1.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000000.kbps) + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(500.kbps) + + val enoughFeedback100pLossRate = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback100pLossRate, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 500.kbps + } + + "NotBoundEstimateByAckedRate" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lowerBoundByAckedRateFactor = 0.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000000.kbps) + lossBasedBandwidthEstimator.setBandwidthEstimate(600.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(500.kbps) + + val enoughFeedback100pLossRate = createPacketResultsWith100pLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback100pLossRate, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan 500.kbps + } + + "HasDecreaseStateBecauseOfUpperBound" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + candidateFactors = doubleArrayOf(1.0), + instantUpperBoundBandwidthBalance = 10.kbps + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000000.kbps) + lossBasedBandwidthEstimator.setBandwidthEstimate(500.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(100.kbps) + + val enoughFeedback10pLoss1 = createPacketResultsWith10pPacketLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback10pLoss1, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + // Verify that the instant upper bound decreases the estimate, and state is + // updated to kDecreasing. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 200.kbps + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + } + + "HasIncreaseStateBecauseOfLowerBound" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + candidateFactors = doubleArrayOf(1.0), + lowerBoundByAckedRateFactor = 10.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000000.kbps) + lossBasedBandwidthEstimator.setBandwidthEstimate(100.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(1.kbps) + + // Network has a high loss to create a loss scenario. + val enoughFeedback50pLoss1 = createPacketResultsWith50pPacketLossRate(Instant.EPOCH) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback50pLoss1, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + + // Network still has a high loss, but better acked rate. + lossBasedBandwidthEstimator.setAcknowledgedBitrate(200.kbps) + val enoughFeedback50pLoss2 = + createPacketResultsWith50pPacketLossRate(Instant.EPOCH + kObservationDurationLowerBound * 2) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + enoughFeedback50pLoss2, + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + // Verify that the instant lower bound increases the estimate, and state is + // updated to kIncreaseUsingPadding. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 200.kbps * 10 + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kIncreaseUsingPadding + } + + "EstimateIncreaseSlowlyFromInstantUpperBoundInAlrIfFieldTrial" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + appendUpperBoundCandidateInAlr = true + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(1000.kbps) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(150.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + val resultAfterLoss = lossBasedBandwidthEstimator.getLossBasedResult() + resultAfterLoss.state shouldBe LossBasedState.kDecreasing + + for (feedbackCount in 1..3) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackCount * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + } + + // Expect less than 100% increase + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan + 2 * resultAfterLoss.bandwidthEstimate + } + + "HasDelayBasedStateIfLossBasedBweIsMax" { + val config = shortObservationConfig() + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setMinMaxBitrate(10.kbps, 1000.kbps) + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + packetResults = createPacketResultsWithReceivedPackets( + Instant.EPOCH + ), + delayBasedEstimate = 2000.kbps, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDelayBasedEstimate + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 1000.kbps + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + packetResults = createPacketResultsWith50pPacketLossRate( + Instant.EPOCH + kObservationDurationLowerBound + ), + delayBasedEstimate = 2000.kbps, + inAlr = false + ) + var result = lossBasedBandwidthEstimator.getLossBasedResult() + result.state shouldBe LossBasedState.kDecreasing + result.bandwidthEstimate shouldBeLessThan 1000.kbps + + // Eventually the estimator recovers to delay based state. + var feedbackCount = 2 + while (result.state != LossBasedState.kDelayBasedEstimate) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + packetResults = createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackCount++ * kObservationDurationLowerBound + ), + delayBasedEstimate = 2000.kbps, + inAlr = false + ) + result = lossBasedBandwidthEstimator.getLossBasedResult() + } + result.state shouldBe LossBasedState.kDelayBasedEstimate + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 1000.kbps + } + + "IncreaseUsingPaddingStateIfFieldTrial" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + paddingDuration = 1000.ms + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(2500.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + + var feedbackCount = 1 + while (lossBasedBandwidthEstimator.getLossBasedResult().state != LossBasedState.kIncreaseUsingPadding) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound * feedbackCount + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + feedbackCount++ + } + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kIncreaseUsingPadding + } + + "BestCandidateResetsToUpperBoundInFieldTrial" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + paddingDuration = 1000.ms, + boundBestCandidate = true + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(2500.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + val resultAfterLoss = lossBasedBandwidthEstimator.getLossBasedResult() + resultAfterLoss.state shouldBe LossBasedState.kDecreasing + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + // After a BWE decrease due to large loss, BWE is expected to ramp up slowly + // and follow the acked bitrate. + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kIncreaseUsingPadding + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate.kbps shouldBe + (resultAfterLoss.bandwidthEstimate.kbps plusOrMinus 100.0) + } + + "DecreaseToAckedCandidateIfPaddingInAlr" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + paddingDuration = 1000.ms, + // Set InstantUpperBoundBwBalance high to disable InstantUpperBound cap. + instantUpperBoundBandwidthBalance = 10000.kbps + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(1000.kbps) + var feedbackId = 0 + while (lossBasedBandwidthEstimator.getLossBasedResult().state != + LossBasedState.kDecreasing + ) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith100pLossRate( + Instant.EPOCH + feedbackId * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + feedbackId++ + } + + while (lossBasedBandwidthEstimator.getLossBasedResult().state != + LossBasedState.kIncreaseUsingPadding + ) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackId * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + feedbackId++ + } + + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeGreaterThan 900.kbps + + lossBasedBandwidthEstimator.setAcknowledgedBitrate(100.kbps) + // Padding is sent now, create some lost packets. + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith100pLossRate( + Instant.EPOCH + feedbackId * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = true + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 100.kbps + } + + "DecreaseAfterPadding" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + paddingDuration = 1000.ms, + bandwidthRampupUpperBoundFactor = 2.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(2500.kbps) + var acknowledgedBitrate = 51.kbps + lossBasedBandwidthEstimator.setAcknowledgedBitrate(acknowledgedBitrate) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe acknowledgedBitrate + + acknowledgedBitrate = 26.kbps + lossBasedBandwidthEstimator.setAcknowledgedBitrate(acknowledgedBitrate) + var feedbackId = 1 + while (lossBasedBandwidthEstimator.getLossBasedResult().state != LossBasedState.kIncreaseUsingPadding) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackId * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + feedbackId++ + } + + val estimateIncreased = Instant.EPOCH + feedbackId * kObservationDurationLowerBound + // The state is kIncreaseUsingPadding for a while without changing the + // estimate, which is limited by 2 * acked rate. + while (lossBasedBandwidthEstimator.getLossBasedResult().state == LossBasedState.kIncreaseUsingPadding) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + feedbackId * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + feedbackId++ + } + + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + val startDecreasing = Instant.EPOCH + (feedbackId - 1) * kObservationDurationLowerBound + Duration.between(estimateIncreased, startDecreasing) shouldBe 1.secs + } + + "HoldRateNotLowerThanAckedRate" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + holdDurationFactor = 10.0, + lowerBoundByAckedRateFactor = 1.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(2500.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + // During the hold duration, hold rate is not lower than the acked rate. + lossBasedBandwidthEstimator.setAcknowledgedBitrate(1000.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate( + Instant.EPOCH + kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 1000.kbps + } + + "EstimateNotLowerThanAckedRate" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + lowerBoundByAckedRateFactor = 1.0 + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(2500.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith100pLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan 1000.kbps + + lossBasedBandwidthEstimator.setAcknowledgedBitrate(1000.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate( + Instant.EPOCH + kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe 1000.kbps + + lossBasedBandwidthEstimator.setAcknowledgedBitrate(1000.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + 2 * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.setAcknowledgedBitrate(1000.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + Instant.EPOCH + 3 * kObservationDurationLowerBound + ), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + + // Verify that the estimate recovers from the acked rate. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeGreaterThan 1000.kbps + } + + "EndHoldDurationIfDelayBasedEstimateWorks" { + val config = shortObservationConfig() + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(2500.kbps) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith50pPacketLossRate(Instant.EPOCH), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + val estimate = lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets(Instant.EPOCH + kObservationDurationLowerBound), + delayBasedEstimate = estimate + 10.kbps, + inAlr = false + ) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets(Instant.EPOCH + kObservationDurationLowerBound * 2), + delayBasedEstimate = estimate + 10.kbps, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDelayBasedEstimate + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe estimate + 10.kbps + } + + "UseByteLossRate" { + val config = shortObservationConfig( + LossBasedBweV2.Config( + useByteLossRate = true + ) + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + lossBasedBandwidthEstimator.setBandwidthEstimate(500.kbps) + // Create packet feedback having 10% packet loss but more than 50% byte loss. + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith10pPacketLossRate(Instant.EPOCH, (kPacketSize * 20).bytes), + delayBasedEstimate = Bandwidth.INFINITY, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + // The estimate is bounded by the instant upper bound due to high loss. + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan 160.kbps + } + + "UseByteLossRateIgnoreLossSpike" { + val config = LossBasedBweV2.Config( + useByteLossRate = true, + observationWindowSize = 5 + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val kDelayBasedEstimate = 500.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(kDelayBasedEstimate) + + // Fill the observation window + for (i in 0 until 5) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + firstPacketTimestamp = Instant.EPOCH + i * kObservationDurationLowerBound + ), + delayBasedEstimate = kDelayBasedEstimate, + inAlr = false + ) + } + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith100pLossRate(Instant.EPOCH + 5 * kObservationDurationLowerBound), + delayBasedEstimate = kDelayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + firstPacketTimestamp = Instant.EPOCH + 6 * kObservationDurationLowerBound + ), + delayBasedEstimate = kDelayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe kDelayBasedEstimate + + // But if more loss happen in a new observation, BWE back down. + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith100pLossRate( + firstPacketTimestamp = Instant.EPOCH + 7 * kObservationDurationLowerBound + ), + delayBasedEstimate = kDelayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan kDelayBasedEstimate + } + + "UseByteLossRateDoesNotIgnoreLossSpikeOnSendBurst" { + val config = LossBasedBweV2.Config( + useByteLossRate = true, + observationWindowSize = 5 + ) + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val kDelayBasedEstimate = 500.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(kDelayBasedEstimate) + + // Fill the observation window + for (i in 0 until 5) { + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWithReceivedPackets( + firstPacketTimestamp = Instant.EPOCH + i * kObservationDurationLowerBound + ), + delayBasedEstimate = kDelayBasedEstimate, + inAlr = false + ) + } + + // If the loss happens when increasing sending rate, then + // the BWE should back down + lossBasedBandwidthEstimator.updateBandwidthEstimate( + createPacketResultsWith100pLossRate( + firstPacketTimestamp = Instant.EPOCH + 5 * kObservationDurationLowerBound, + numPackets = 5 + ), + delayBasedEstimate = kDelayBasedEstimate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().state shouldBe LossBasedState.kDecreasing + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBeLessThan kDelayBasedEstimate + } + + "EstimateDoesNotBackOffDueToPacketReorderingBetweenFeedback" { + val config = shortObservationConfig() + val lossBasedBandwidthEstimator = LossBasedBweV2(config) + val kStartBitrate = 2500.kbps + lossBasedBandwidthEstimator.setBandwidthEstimate(kStartBitrate) + + val feedback1 = List(3) { PacketResult() } + feedback1[0].sentPacket.sequenceNumber = 1 + feedback1[0].sentPacket.size = kPacketSize.bytes + feedback1[0].sentPacket.sendTime = Instant.EPOCH + feedback1[0].receiveTime = feedback1[0].sentPacket.sendTime + 10.ms + feedback1[1].sentPacket.sequenceNumber = 2 + feedback1[1].sentPacket.size = kPacketSize.bytes + feedback1[1].sentPacket.sendTime = Instant.EPOCH + // Lost or reordered + feedback1[1].receiveTime = Instant.MAX + + feedback1[2].sentPacket.sequenceNumber = 3 + feedback1[2].sentPacket.size = kPacketSize.bytes + feedback1[2].sentPacket.sendTime = Instant.EPOCH + feedback1[2].receiveTime = feedback1[2].sentPacket.sendTime + 10.ms + + val feedback2 = List(3) { PacketResult() } + feedback2[0].sentPacket.sequenceNumber = 2 + feedback2[0].sentPacket.size = kPacketSize.bytes + feedback2[0].sentPacket.sendTime = Instant.EPOCH + feedback2[0].receiveTime = feedback1[0].sentPacket.sendTime + 10.ms + feedback2[1].sentPacket.sequenceNumber = 4 + feedback2[1].sentPacket.size = kPacketSize.bytes + feedback2[1].sentPacket.sendTime = Instant.EPOCH + feedback2[1].receiveTime = Instant.MAX + feedback2[2].sentPacket.sequenceNumber = 5 + feedback2[2].sentPacket.size = kPacketSize.bytes + feedback2[2].sentPacket.sendTime = Instant.EPOCH + feedback2[2].receiveTime = feedback1[2].sentPacket.sendTime + 10.ms + + lossBasedBandwidthEstimator.updateBandwidthEstimate( + feedback1, + delayBasedEstimate = kStartBitrate, + inAlr = false + ) + lossBasedBandwidthEstimator.updateBandwidthEstimate( + feedback2, + delayBasedEstimate = kStartBitrate, + inAlr = false + ) + lossBasedBandwidthEstimator.getLossBasedResult().bandwidthEstimate shouldBe kStartBitrate + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimatorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimatorTest.kt new file mode 100644 index 0000000..566c8b2 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeBitrateEstimatorTest.kt @@ -0,0 +1,261 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.longs.shouldBeInRange +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bytes +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import java.time.Instant + +/** + * Unit tests of ProbeBitrateEstimator + * + * Based on WebRTC modules/congestion_controller/goog_cc/probe_bitrate_estimator_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class TestProbeBitrateEstimator { + var measuredDataRate: Bandwidth? = null + val probeBitrateEstimator = ProbeBitrateEstimator(logger, diagnosticContext) + + // TODO(philipel): Use PacedPacketInfo when ProbeBitrateEstimator is rewritten + // to use that information. + fun addPacketFeedback( + probeClusterId: Int, + sizeBytes: Long, + sendTimeMs: Long, + arrivalTimeMs: Long, + minProbes: Int = kDefaultMinProbes, + minBytes: Int = kDefaultMinBytes + ) { + val kReferenceTime = Instant.ofEpochSecond(1000) + val feedback = PacketResult() + feedback.sentPacket.sendTime = kReferenceTime + sendTimeMs.ms + feedback.sentPacket.size = sizeBytes.bytes + feedback.sentPacket.pacingInfo = PacedPacketInfo(probeClusterId, minProbes, minBytes) + feedback.receiveTime = kReferenceTime + arrivalTimeMs.ms + measuredDataRate = probeBitrateEstimator.handleProbeAndEstimateBitrate(feedback) + } + + companion object { + val logger = createLogger() + val diagnosticContext = DiagnosticContext() + } +} + +class ProbeBitrateEstimatorTest : FreeSpec() { + init { + "OneCluster" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + test.addPacketFeedback(0, 1000, 30, 40) + + test.measuredDataRate!!.bps shouldBeInRange (800000L plusOrMinus 10L) + } + + "OneClusterTooFewProbes" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + + test.measuredDataRate shouldBe null + } + + "OneClusterTooFewBytes" { + val test = TestProbeBitrateEstimator() + val kMinBytes = 6000 + test.addPacketFeedback(0, 1000, 0, 10, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 1000, 10, 20, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 1000, 20, 30, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 1000, 30, 40, kDefaultMinProbes, kMinBytes) + + test.measuredDataRate shouldBe null + } + + "SmallCluster" { + val test = TestProbeBitrateEstimator() + val kMinBytes = 1000 + test.addPacketFeedback(0, 150, 0, 10, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 150, 10, 20, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 150, 20, 30, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 150, 30, 40, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 150, 40, 50, kDefaultMinProbes, kMinBytes) + test.addPacketFeedback(0, 150, 50, 60, kDefaultMinProbes, kMinBytes) + + test.measuredDataRate!!.bps shouldBeInRange (120000L plusOrMinus 10L) + } + + "LargeCluster" { + val test = TestProbeBitrateEstimator() + val kMinProbes = 30 + val kMinBytes = 312500 + + var sendTime = 0L + var receiveTime = 5L + repeat(25) { + test.addPacketFeedback(0, 12500, sendTime, receiveTime, kMinProbes, kMinBytes) + ++sendTime + ++receiveTime + } + + test.measuredDataRate!!.bps shouldBeInRange (100000000L plusOrMinus 10L) + } + + "FastReceive" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 15) + test.addPacketFeedback(0, 1000, 10, 30) + test.addPacketFeedback(0, 1000, 20, 35) + test.addPacketFeedback(0, 1000, 30, 40) + + test.measuredDataRate!!.bps shouldBeInRange (800000L plusOrMinus 10L) + } + + "TooFastReceive" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 19) + test.addPacketFeedback(0, 1000, 10, 22) + test.addPacketFeedback(0, 1000, 20, 25) + test.addPacketFeedback(0, 1000, 30, 27) + + test.measuredDataRate shouldBe null + } + + "SlowReceive" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 40) + test.addPacketFeedback(0, 1000, 20, 70) + test.addPacketFeedback(0, 1000, 30, 85) + // Expected send rate = 800 kbps, expected receive rate = 320 kbps. + + test.measuredDataRate!!.bps shouldBeInRange + ((kTargetUtilizationFraction * 320000.0).toLong() plusOrMinus 10) + } + + "BurstReceive" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 50) + test.addPacketFeedback(0, 1000, 10, 50) + test.addPacketFeedback(0, 1000, 20, 50) + test.addPacketFeedback(0, 1000, 30, 50) + + test.measuredDataRate shouldBe null + } + + "MultipleClusters" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + test.addPacketFeedback(0, 1000, 40, 60) + // Expected send rate = 600 kbps, expected receive rate = 480 kbps. + test.measuredDataRate!!.bps shouldBeInRange + ((kTargetUtilizationFraction * 480000.0).toLong() plusOrMinus 10) + + test.addPacketFeedback(0, 1000, 50, 60) + // Expected send rate = 640 kbps, expected receive rate = 640 kbps. + test.measuredDataRate!!.bps shouldBeInRange (640000L plusOrMinus 10L) + + test.addPacketFeedback(1, 1000, 60, 70) + test.addPacketFeedback(1, 1000, 65, 77) + test.addPacketFeedback(1, 1000, 70, 84) + test.addPacketFeedback(1, 1000, 75, 90) + // Expected send rate = 1600 kbps, expected receive rate = 1200 kbps. + + test.measuredDataRate!!.bps shouldBeInRange + ((kTargetUtilizationFraction * 1200000.0).toLong() plusOrMinus 10) + } + + "IgnoreOldClusters" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + + test.addPacketFeedback(1, 1000, 60, 70) + test.addPacketFeedback(1, 1000, 65, 77) + test.addPacketFeedback(1, 1000, 70, 84) + test.addPacketFeedback(1, 1000, 75, 90) + // Expected send rate = 1600 kbps, expected receive rate = 1200 kbps. + + test.measuredDataRate!!.bps shouldBeInRange + ((kTargetUtilizationFraction * 1200000.0).toLong() plusOrMinus 10) + + // Coming in 6s later + test.addPacketFeedback(0, 1000, 40 + 6000, 60 + 6000) + test.measuredDataRate shouldBe null + } + + "IgnoreSizeLastSendPacket" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + test.addPacketFeedback(0, 1000, 30, 40) + test.addPacketFeedback(0, 1500, 40, 50) + // Expected send rate = 800 kbps, expected receive rate = 900 kbps. + + test.measuredDataRate!!.bps shouldBeInRange (800000L plusOrMinus 10L) + } + + "IgnoreSizeFirstReceivePacket" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1500, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + test.addPacketFeedback(0, 1000, 30, 40) + // Expected send rate = 933 kbps, expected receive rate = 800 kbps. + + test.measuredDataRate!!.bps shouldBeInRange + ((kTargetUtilizationFraction * 800000.0).toLong() plusOrMinus 10) + } + + "NoLastEstimatedBitrateBps" { + val test = TestProbeBitrateEstimator() + test.probeBitrateEstimator.fetchAndResetLastEstimatedBitrate() shouldBe null + } + + "FetchLastEstimatedBitrateBps" { + val test = TestProbeBitrateEstimator() + test.addPacketFeedback(0, 1000, 0, 10) + test.addPacketFeedback(0, 1000, 10, 20) + test.addPacketFeedback(0, 1000, 20, 30) + test.addPacketFeedback(0, 1000, 30, 40) + + val estimatedBitrate = test.probeBitrateEstimator.fetchAndResetLastEstimatedBitrate() + + estimatedBitrate shouldNotBe null + estimatedBitrate!!.bps shouldBeInRange (800000L plusOrMinus 10L) + test.probeBitrateEstimator.fetchAndResetLastEstimatedBitrate() shouldBe null + } + } +} + +const val kDefaultMinProbes = 5 +const val kDefaultMinBytes = 5000 +const val kTargetUtilizationFraction = 0.95 diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeControllerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeControllerTest.kt new file mode 100644 index 0000000..b70b4c2 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/ProbeControllerTest.kt @@ -0,0 +1,1007 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.ints.shouldBeGreaterThanOrEqual +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.times +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import java.time.Duration +import java.time.Instant + +val kMinBitrate = 100.bps +val kStartBitrate = 300.bps +val kMaxBitrate = 10000.bps +val kMbpsMultiplier = 1000.kbps + +val kExponentialProbingTimeout = 5.secs + +val kAlrProbeInterval = 5.secs +private val kAlrEndedTimeout = 3.secs +private val kBitrateDropTimeout = 5.secs + +/** Unit tests for ProbeController, + * based on WebRTC modules/congestion_controller/goog_cc/probe_controller_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Field trial settings have been generally removed, set to their default settings. + */ +class ProbeControllerTest : FreeSpec() { + class ProbeControllerFixture( + val config: ProbeControllerConfig = ProbeControllerConfig() + ) { + private val clock = FakeClock().apply { setTime(Instant.ofEpochMilli(100000L)) } + + fun createController(): ProbeController = ProbeController(logger, diagnosticContext, config) + + fun currentTime() = clock.instant() + fun advanceTime(delta: Duration) = clock.elapse(delta) + } + + init { + "InitiatesProbingAfterSetBitrates" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + + val probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThanOrEqual 2 + } + + "InitiatesProbingWhenNetworkAvailable" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe true + probes = probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)) + probes.size shouldBeGreaterThanOrEqual 2 + } + + "SetsDefaultTargetDurationAndTargetProbeCount" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + val probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThanOrEqual 2 + + probes[0].targetDuration shouldBe 15.ms + probes[0].targetProbeCount shouldBe 5 + } + + "FieldTrialsOverrideDefaultTargetDurationAndTargetProbeCount" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + minProbePacketsSent = 2, + minProbeDuration = 123.ms + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + val probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThanOrEqual 2 + + probes[0].targetDuration shouldBe 123.ms + probes[0].targetProbeCount shouldBe 2 + } + + "ProbeOnlyWhenNetworkIsUp" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + var probes = probeController.onNetworkAvailability( + NetworkAvailability(atTime = fixture.currentTime(), networkAvailable = false) + ) + probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe true + probes = probeController.onNetworkAvailability( + NetworkAvailability(atTime = fixture.currentTime(), networkAvailable = true) + ) + probes.size shouldBeGreaterThanOrEqual 2 + } + + "CanConfigureInitialProbeRateFactor" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + firstExponentialProbeScale = 2.0, + secondExponentialProbeScale = 3.0 + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + val probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThanOrEqual 2 + probes[0].targetDataRate shouldBe kStartBitrate * 2 + probes[1].targetDataRate shouldBe kStartBitrate * 3 + } + + "DisableSecondInitialProbeIfRateFactorZero" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + firstExponentialProbeScale = 2.0, + secondExponentialProbeScale = 0.0 + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + val probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe kStartBitrate * 2 + } + + "InitiatesProbingOnMaxBitrateIncrease" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + // Long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.setEstimatedBitrate( + kStartBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setBitrates( + kMinBitrate, + kStartBitrate, + kMaxBitrate + 100.bps, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe kMaxBitrate.bps + 100 + } + + "ProbesOnMaxAllocatedBitrateIncreaseOnlyWhenInAlr" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + kMaxBitrate - 1.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + // Wait long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + + // Probe when in alr. + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + probes = probeController.onMaxTotalAllocatedBitrate( + kMaxBitrate + 1.bps, + fixture.currentTime() + ) + probes.size shouldBe 2 + probes[0].targetDataRate shouldBe kMaxBitrate + + // Do not probe when not in alr. + probeController.setAlrStartTimeMs(null) + probes = probeController.onMaxTotalAllocatedBitrate( + kMaxBitrate + 2.bps, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + } + + "ProbesOnMaxAllocatedBitrateLimitedByCurrentBwe" { + val fixture = ProbeControllerFixture() + kMaxBitrate shouldBeGreaterThan 1.5 * kStartBitrate + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + kStartBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + // Wait long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + + // Probe when in alr. + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + probes = probeController.onMaxTotalAllocatedBitrate(kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe 2.0 * kStartBitrate + + // Continue probing if probe succeeds. + probes = probeController.setEstimatedBitrate( + 1.5 * kStartBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBeGreaterThan 2.0 * kStartBitrate + } + + "CanDisableProbingOnMaxTotalAllocatedBitrateIncrease" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + probeOnMaxAllocatedBitrateChange = false + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + kMaxBitrate - 1.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + + // Do no probe, since probe_max_allocation:false. + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + probes = probeController.onMaxTotalAllocatedBitrate(kMaxBitrate + 1.bps, fixture.currentTime()) + probes.isEmpty() shouldBe true + } + + "InitiatesProbingOnMaxBitrateIncreaseAtMaxBitrate" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + // Long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.setEstimatedBitrate( + kStartBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + kMaxBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes = probeController.setBitrates( + kMinBitrate, + kStartBitrate, + kMaxBitrate + 100.bps, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe kMaxBitrate + 100.bps + } + + "TestExponentialProbing" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + + // Repeated probe should only be sent when estimated bitrate climbs above + // 0.7 * 6 * kStartBitrate = 1260. + probes = probeController.setEstimatedBitrate( + 1000.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + + probes = probeController.setEstimatedBitrate( + 1800.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe 2 * 1800 + } + + "ExponentialProbingStopIfMaxBitrateLow" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + abortFurtherProbeIfMaxLowerThanCurrent = true + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThan 0 + + // Repeated probe normally is sent when estimated bitrate climbs above + // 0.7 * 6 * kStartBitrate = 1260. But since max bitrate is low, expect + // exponential probing to stop. + probes = probeController.setBitrates( + kMinBitrate, + kStartBitrate, + maxBitrate = kStartBitrate, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + + probes = probeController.setEstimatedBitrate( + 1800.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + } + + "ExponentialProbingStopIfMaxAllocatedBitrateLow" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + abortFurtherProbeIfMaxLowerThanCurrent = true + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThan 0 + + // Repeated probe normally is sent when estimated bitrate climbs above + // 0.7 * 6 * kStartBitrate = 1260. But since allocated bitrate i slow, expect + // exponential probing to stop. + probes = probeController.onMaxTotalAllocatedBitrate(kStartBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe true + + probes = probeController.setEstimatedBitrate( + 1800.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + } + + "InitialProbingToLowMaxAllocatedbitrate" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThan 0 + + // Repeated probe is sent when estimated bitrate climbs above + // 0.7 * 6 * kStartBitrate = 1260. + probes = probeController.onMaxTotalAllocatedBitrate(kStartBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe true + + // If the inital probe result is received, a new probe is sent at 2x the + // needed max bitrate. + probes = probeController.setEstimatedBitrate( + 1800.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe 2 * kStartBitrate.bps + } + + "InitialProbingTimeout" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThan 0 + // Advance far enough to cause a time out in waiting for probing result. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + probes = probeController.setEstimatedBitrate( + 1800.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + } + + "RepeatedInitialProbingSendsNewProbeAfterTimeout" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.enableRepeatedInitialProbing(true) + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThan 0 + val startTime = fixture.currentTime() + var lastProbeTime = fixture.currentTime() + while (fixture.currentTime() < startTime + 5.secs) { + fixture.advanceTime(100.ms) + probes = probeController.process(fixture.currentTime()) + if (probes.isNotEmpty()) { + // Expect a probe every second. + Duration.between(lastProbeTime, fixture.currentTime()) shouldBe 1100.ms + probes[0].minProbeDelta shouldBe 20.ms + probes[0].targetDuration shouldBe 100.ms + lastProbeTime = fixture.currentTime() + } else { + Duration.between(lastProbeTime, fixture.currentTime()) shouldBeLessThan 1100.ms + } + } + fixture.advanceTime( + 1.secs + ) + // After 5s, repeated initial probing stops. + probeController.process(fixture.currentTime()).isEmpty() shouldBe true + } + + "RepeatedInitialProbingStopIfMaxAllocatedBitrateSet" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.enableRepeatedInitialProbing(true) + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThan 0 + + fixture.advanceTime(1100.ms) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes = probeController.onMaxTotalAllocatedBitrate(kMinBitrate, fixture.currentTime()) + fixture.advanceTime(1100.ms) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + } + + "RequestProbeInAlr" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBeGreaterThanOrEqual 2 + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(kAlrEndedTimeout + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + 250.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes = probeController.requestProbe(fixture.currentTime()) + + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe (0.85 * 500).toLong() + } + + "RequestProbeWhenAlrEndedRecently" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 2 + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + probeController.setAlrStartTimeMs(null) + fixture.advanceTime(kAlrProbeInterval + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + 250.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probeController.setAlrEndedTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(kAlrEndedTimeout - 1.ms) + probes = probeController.requestProbe(fixture.currentTime()) + + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe (0.85 * 500).toLong() + } + + "RequestProbeWhenAlrNotEndedRecently" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 2 + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + probeController.setAlrStartTimeMs(null) + fixture.advanceTime(kAlrProbeInterval + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + 250.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probeController.setAlrEndedTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(kAlrEndedTimeout + 1.ms) + probes = probeController.requestProbe(fixture.currentTime()) + + probes.isEmpty() shouldBe true + } + + "RequestProbeWhenBweDropNotRecent" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 2 + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(kAlrProbeInterval + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + 250.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + fixture.advanceTime(kBitrateDropTimeout + 1.ms) + probes = probeController.requestProbe(fixture.currentTime()) + probes.isEmpty() shouldBe true + } + + "PeriodicProbing" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + probeController.enablePeriodicAlrProbing(true) + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 2 + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + val startTime = fixture.currentTime() + + // Expect the controller to send a new probe after 5s has passed. + probeController.setAlrStartTimeMs(startTime.toEpochMilli()) + fixture.advanceTime(5.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe 1000 + + // The following probe should be sent at 10s into ALR. + probeController.setAlrStartTimeMs(startTime.toEpochMilli()) + fixture.advanceTime(4.secs) + probes = probeController.process(fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + + probeController.setAlrStartTimeMs(startTime.toEpochMilli()) + fixture.advanceTime(1.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + } + + "PeriodicProbingAfterReset" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + val alrStartTime = fixture.currentTime() + + probeController.setAlrStartTimeMs(alrStartTime.toEpochMilli()) + probeController.enablePeriodicAlrProbing(true) + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probeController.reset(fixture.currentTime()) + + fixture.advanceTime(10.secs) + probes = probeController.process(fixture.currentTime()) + // Since bitrates are not yet set, no probe is sent event though we are in ALR + // mode. + probes.isEmpty() shouldBe true + + probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.size shouldBe 2 + + // Make sure we use `kStartBitrateBps` as the estimated bitrate + // until SetEstimatedBitrate is called with an updated estimate. + fixture.advanceTime(10.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe kStartBitrate * 2 + } + + "NoProbesWhenTransportIsNotWritable" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.enablePeriodicAlrProbing(true) + + val probes = probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = false)) + probes.isEmpty() shouldBe true + probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + .isEmpty() shouldBe true + fixture.advanceTime(10.secs) + probeController.process(fixture.currentTime()).isEmpty() shouldBe true + + // Controller is reset after a network route change. + // But, a probe should not be sent since the transport is not writable. + // Transport is not writable until after DTLS negotiation completes. + // However, the bitrate constraints may change. + probeController.reset(fixture.currentTime()) + probeController.setBitrates( + 2 * kMinBitrate, + 2 * kStartBitrate, + 2 * kMaxBitrate, + fixture.currentTime() + ).isEmpty() shouldBe true + fixture.advanceTime(10.secs) + probeController.process(fixture.currentTime()).isEmpty() shouldBe true + } + + "TestExponentialProbingOverflow" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates( + kMinBitrate, + 10 * kMbpsMultiplier, + 100 * kMbpsMultiplier, + fixture.currentTime() + ) + // Verify that probe bitrate is capped at the specified max bitrate. + probes = probeController.setEstimatedBitrate( + 60 * kMbpsMultiplier, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe 100 * kMbpsMultiplier + // Verify that repeated probes aren't sent. + probes = probeController.setEstimatedBitrate( + 100 * kMbpsMultiplier, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + } + + "TestAllocatedBitrateCap" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + + var probes = probeController.setBitrates( + kMinBitrate, + 10 * kMbpsMultiplier, + 100 * kMbpsMultiplier, + fixture.currentTime() + ) + + // Configure ALR for periodic probing. + probeController.enablePeriodicAlrProbing(true) + val alrStartTime = fixture.currentTime() + probeController.setAlrStartTimeMs(alrStartTime.toEpochMilli()) + + val estimatedBitrate = 10 * kMbpsMultiplier + probes = probeController.setEstimatedBitrate( + estimatedBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + // Set a max allocated bitrate below the current estimate. + val maxAllocated = estimatedBitrate - 1 * kMbpsMultiplier + probes = probeController.onMaxTotalAllocatedBitrate(maxAllocated, fixture.currentTime()) + probes.isEmpty() shouldBe true + + // Probes such as ALR capped at 2x the max allocation limit. + fixture.advanceTime(5.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe 2 * maxAllocated + + // Remove allocation limit. + probeController.onMaxTotalAllocatedBitrate(Bandwidth.ZERO, fixture.currentTime()) + .isEmpty() shouldBe true + fixture.advanceTime(5.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe estimatedBitrate * 2 + } + + "ConfigurableProbingFieldTrial" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + firstExponentialProbeScale = 2.0, + secondExponentialProbeScale = 5.0, + furtherExponentialProbeScale = 3.0, + furtherProbeThreshold = 0.8, + firstAllocationProbeScale = 2.0, + allocationProbeLimitByCurrentScale = 1000.0, + secondAllocationProbeScale = null, + minProbePacketsSent = 2 + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, 5000.kbps, fixture.currentTime()) + probes.size shouldBe 2 + probes[0].targetDataRate.bps shouldBe 600 + probes[0].targetProbeCount shouldBe 2 + probes[1].targetDataRate.bps shouldBe 1500 + probes[1].targetProbeCount shouldBe 2 + + // Repeated probe should only be sent when estimated bitrate climbs above + // 0.8 * 5 * kStartBitrateBps = 1200. + probes = probeController.setEstimatedBitrate( + 1100.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.size shouldBe 0 + + probes = probeController.setEstimatedBitrate( + 1250.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe 3 * 1250 + + fixture.advanceTime(5.secs) + probes = probeController.process(fixture.currentTime()) + + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + probes = probeController.onMaxTotalAllocatedBitrate(200.kbps, fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate.bps shouldBe 400_000 + } + + "LimitAlrProbeWhenLossBasedBweLimited" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig() + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + probeController.enablePeriodicAlrProbing(true) + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes = probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + // Expect the controller to send a new probe after 5s has passed. + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(5.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + + probeController.setEstimatedBitrate( + 500.bps, + BandwidthLimitedCause.kLossLimitedBweIncreasing, + fixture.currentTime() + ) + fixture.advanceTime(6.secs) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe 1.5 * 500.bps + + probes = probeController.setEstimatedBitrate( + 1.5 * 500.bps, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + fixture.advanceTime(6.secs) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe false + probes[0].targetDataRate shouldBeGreaterThan 1.5 * 1.5 * 500.bps + } + + /* Skipping tests involving network state: + * "PeriodicProbeAtUpperNetworkStateEstimate", + * "LimitProbeAtUpperNetworkStateEstimateIfLossBasedLimited", + * "AlrProbesLimitedByNetworkStateEstimate", + * "CanSetLongerProbeDurationAfterNetworkStateEstimate" + */ + + "ProbeInAlrIfLossBasedIncreasing" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig() + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probeController.enablePeriodicAlrProbing(true) + probes = probeController.setEstimatedBitrate( + kStartBitrate, + BandwidthLimitedCause.kLossLimitedBweIncreasing, + fixture.currentTime() + ) + + // Wait long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + + // Probe when in alr. + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(kAlrProbeInterval + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes.size shouldBe 1 + probes[0].targetDataRate shouldBe 1.5 * kStartBitrate + } + + "NotProbeWhenInAlrIfLossBasedDecreases" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig() + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probeController.enablePeriodicAlrProbing(true) + probes = probeController.setEstimatedBitrate( + kStartBitrate, + BandwidthLimitedCause.kLossLimitedBwe, + fixture.currentTime() + ) + + // Wait long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + + // Not probe in alr when loss based estimate decreases. + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + fixture.advanceTime(kAlrProbeInterval + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + } + + "NotProbeIfLossBasedIncreasingOutsideAlr" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig() + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probeController.enablePeriodicAlrProbing(true) + probes = probeController.setEstimatedBitrate( + kStartBitrate, + BandwidthLimitedCause.kLossLimitedBwe, + fixture.currentTime() + ) + + // Wait long enough to time out exponential probing. + fixture.advanceTime(kExponentialProbingTimeout) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + + probeController.setAlrStartTimeMs(null) + fixture.advanceTime(kAlrProbeInterval + 1.ms) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + } + + /* Skipping tests with network_state: + * "ProbeFurtherWhenLossBasedIsSameAsDelayBasedEstimate", + * "ProbeIfEstimateLowerThanNetworkStateEstimate", + * "DontProbeFurtherWhenLossLimited", + * "ProbeFurtherWhenDelayBasedLimited", + * "ProbeAfterTimeoutIfNetworkStateEstimateIncreaseAfterProbeSent", + * "SkipProbeFurtherIfAlreadyProbedToMaxRate" + */ + + "MaxAllocatedBitrateNotReset" { + val fixture = ProbeControllerFixture() + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe false + + probes = probeController.onMaxTotalAllocatedBitrate(kStartBitrate / 4, fixture.currentTime()) + probeController.reset(fixture.currentTime()) + + probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe false + probes[0].targetDataRate shouldBe kStartBitrate / 4 * 2 + } + + "SkipAlrProbeIfEstimateLargerThanMaxProbe" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + skipIfEstimateLargerThanFractionOfMax = 0.9 + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + probeController.enablePeriodicAlrProbing(true) + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe false + + probes = probeController.setEstimatedBitrate( + kMaxBitrate, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + probes.isEmpty() shouldBe true + + // But if the max rate increase, A new probe is sent. + probes = probeController.setBitrates(kMinBitrate, kStartBitrate, 2 * kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe false + } + + "SkipAlrProbeIfEstimateLargerThanFractionOfMaxAllocated" { + val fixture = ProbeControllerFixture( + config = ProbeControllerConfig( + skipIfEstimateLargerThanFractionOfMax = 1.0 + ) + ) + val probeController = fixture.createController() + probeController.onNetworkAvailability(NetworkAvailability(networkAvailable = true)).isEmpty() shouldBe true + probeController.enablePeriodicAlrProbing(true) + var probes = probeController.setBitrates(kMinBitrate, kStartBitrate, kMaxBitrate, fixture.currentTime()) + probes.isEmpty() shouldBe false + probes = probeController.setEstimatedBitrate( + kMaxBitrate / 2, + BandwidthLimitedCause.kDelayBasedLimited, + fixture.currentTime() + ) + + fixture.advanceTime(10.secs) + probeController.setAlrStartTimeMs(fixture.currentTime().toEpochMilli()) + probes = probeController.onMaxTotalAllocatedBitrate(kMaxBitrate / 2, fixture.currentTime()) + // No probes since total allocated is not higher than the current estimate. + probes.isEmpty() shouldBe true + fixture.advanceTime(2.secs) + probes = probeController.process(fixture.currentTime()) + probes.isEmpty() shouldBe true + + // But if the max allocated increase, A new probe is sent. + probes = probeController.onMaxTotalAllocatedBitrate( + kMaxBitrate / 2 + 1.bps, + fixture.currentTime() + ) + probes.isEmpty() shouldBe false + } + + /* Skipping tests with network_state: + * "SkipNetworkStateProbeIfEstimateLargerThanMaxProbe", + * "SendsProbeIfNetworkStateEstimateLowerThanMaxProbe", + * "ProbeNotLimitedByNetworkStateEsimateIfLowerThantCurrent", + * "DontProbeIfDelayIncreased", + * "DontProbeIfHighRtt" */ + } + + companion object { + val logger = createLogger() + val diagnosticContext = DiagnosticContext() + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimatorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimatorTest.kt new file mode 100644 index 0000000..fa3ab86 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/RobustThroughputEstimatorTest.kt @@ -0,0 +1,524 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeGreaterThanOrEqualTo +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.comparables.shouldBeLessThanOrEqualTo +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.nlj.util.bytesPerSec +import org.jitsi.nlj.util.div +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import java.time.Duration +import java.time.Instant + +class FeedbackGenerator { + fun createFeedbackVector( + numberOfPackets: Int, + packetSize: DataSize, + sendRate: Bandwidth, + recvRate: Bandwidth + ): List { + val packetFeedbackVector = MutableList(numberOfPackets) { PacketResult() } + for (i in 0 until numberOfPackets) { + packetFeedbackVector[i].sentPacket.sendTime = sendClock + packetFeedbackVector[i].sentPacket.sequenceNumber = sequenceNumber + packetFeedbackVector[i].sentPacket.size = packetSize + sendClock += packetSize / sendRate + recvClock += packetSize / recvRate + sequenceNumber += 1 + packetFeedbackVector[i].receiveTime = recvClock + } + return packetFeedbackVector + } + + fun currentReceiveClock() = recvClock + fun advanceReceiveClock(delta: Duration) { + recvClock += delta + } + fun advanceSendClock(delta: Duration) { + sendClock += delta + } + + private var sendClock = Instant.ofEpochMilli(100000) + private var recvClock = Instant.ofEpochMilli(10000) + private var sequenceNumber = 100L +} + +/** + * Unit tests of AcknowledgedBitrateEstimator. + * + * Based on WebRTC modules/congestion_controller/goog_cc/robust_throughput_estimator_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138) + */ +@SuppressFBWarnings("IM_BAD_CHECK_FOR_ODD") +class RobustThroughputEstimatorTest : FreeSpec() { + init { + "DefaultEnabled" { + val settings = RobustThroughputEstimatorSettings() + settings.enabled shouldBe true + } + + "CanDisable" { + val settings = RobustThroughputEstimatorSettings(enabled = false) + settings.enabled shouldBe false + } + + "InitialEstimate" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + val sendRate = 100000.bytesPerSec + val recvRate = 100000.bytesPerSec + + // No estimate until the estimator has enough data. + var packetFeedback = feedbackGenerator.createFeedbackVector( + 9, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughputEstimator.bitrate() shouldBe null + + // Estimate once `required_packets` packets have been received. + packetFeedback = feedbackGenerator.createFeedbackVector( + 1, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + var throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + + // Estimate remains stable when send and receive rates are stable. + packetFeedback = feedbackGenerator.createFeedbackVector( + 15, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + + "EstimateAdapts" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + + // 1 second, 800kbps, estimate is stable. + var sendRate = 100000.bytesPerSec + var recvRate = 100000.bytesPerSec + + repeat(10) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + + // 1 second, 1600kbps, estimate increases + sendRate = 200000.bytesPerSec + recvRate = 200000.bytesPerSec + repeat(20) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!! shouldBeGreaterThanOrEqualTo 100000.bytesPerSec + throughput shouldBeLessThanOrEqualTo sendRate + } + + // 1 second, 1600kbps, estimate is stable + repeat(20) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + + // 1 second, 400kbps, estimate decreases + sendRate = 50000.bytesPerSec + recvRate = 50000.bytesPerSec + repeat(5) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!! shouldBeLessThanOrEqualTo 200000.bytesPerSec + throughput shouldBeGreaterThanOrEqualTo sendRate + } + + // 1 second, 400kbps, estimate is stable + sendRate = 50000.bytesPerSec + recvRate = 50000.bytesPerSec + repeat(5) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + } + + "CappedByReceiveRate" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + val sendRate = 100000.bytesPerSec + val recvRate = 25000.bytesPerSec + + val packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!!.bytesPerSec shouldBe recvRate.bytesPerSec plusOrMinus + 0.05 * recvRate.bytesPerSec // Allow 5% error + } + + "CappedBySendRate" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + val sendRate = 50000.bytesPerSec + val recvRate = 100000.bytesPerSec + + val packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!!.bytesPerSec shouldBe sendRate.bytesPerSec plusOrMinus + 0.05 * sendRate.bytesPerSec // Allow 5% error + } + + "DelaySpike" { + val feedbackGenerator = FeedbackGenerator() + // This test uses a 500ms window to amplify the effect + // of a delay spike. + val throughputEstimator = RobustThroughputEstimator( + RobustThroughputEstimatorSettings(enabled = true, minWindowDuration = 500.ms) + ) + val sendRate = 100000.bytesPerSec + var recvRate = 100000.bytesPerSec + + var packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + var throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + + // Delay spike. 25 packets sent, but none received. + feedbackGenerator.advanceReceiveClock(250.ms) + + // Deliver all of the packets during the next 50 ms. (During this time, + // we'll have sent an additional 5 packets, so we need to receive 30 + // packets at 1000 bytes each in 50 ms, i.e. 600000 bytes per second). + recvRate = 600000.bytesPerSec + // Estimate should not drop + repeat(30) { + packetFeedback = feedbackGenerator.createFeedbackVector( + 1, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!!.bytesPerSec shouldBe sendRate.bytesPerSec plusOrMinus + 0.05 * sendRate.bytesPerSec // Allow 5% error + } + + // Delivery at normal rate. When the packets received before the gap + // has left the estimator's window, the receive rate will be high, but the + // estimate should be capped by the send rate. + recvRate = 100000.bytesPerSec + repeat(20) { + packetFeedback = feedbackGenerator.createFeedbackVector( + 5, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!!.bytesPerSec shouldBe sendRate.bytesPerSec plusOrMinus + 0.05 * sendRate.bytesPerSec // Allow 5% error + } + } + + "HighLoss" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + val sendRate = 100000.bytesPerSec + val recvRate = 100000.bytesPerSec + + var packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + + // 50% loss + for (i in packetFeedback.indices) { + if (i % 2 == 1) { + packetFeedback[i].receiveTime = Instant.MAX + } + } + + packetFeedback = packetFeedback.sortedBy { it.receiveTime } + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!!.bytesPerSec shouldBe sendRate.bytesPerSec / 2 plusOrMinus + 0.05 * sendRate.bytesPerSec / 2 // Allow 5% error + } + + "ReorderedFeedback" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + val sendRate = 100000.bytesPerSec + val recvRate = 100000.bytesPerSec + + var packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + var throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + + val delayedFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + + // Since we're missing some feedback, it's expected that the + // estimate will drop. + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!! shouldBeLessThan sendRate + + // But it should completely recover as soon as we get the feedback. + throughputEstimator.incomingPacketFeedbackVector(delayedFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + + // It should then remain stable (as if the feedbacks weren't reordered.) + repeat(10) { + packetFeedback = feedbackGenerator.createFeedbackVector( + 15, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + } + + "DeepReordering" { + val feedbackGenerator = FeedbackGenerator() + // This test uses a 500ms window to amplify the + // effect of reordering. + val throughputEstimator = RobustThroughputEstimator( + RobustThroughputEstimatorSettings(enabled = true, minWindowDuration = 500.ms) + ) + val sendRate = 100000.bytesPerSec + val recvRate = 100000.bytesPerSec + + val delayedPackets = feedbackGenerator.createFeedbackVector( + 1, + 1000.bytes, + sendRate, + recvRate + ) + + repeat(10) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + + // Delayed packet arrives ~1 second after it should have. + // Since the window is 500 ms, the delayed packet was sent ~500 + // ms before the second oldest packet. However, the send rate + // should not drop. + run { + delayedPackets.first().receiveTime = feedbackGenerator.currentReceiveClock() + throughputEstimator.incomingPacketFeedbackVector(delayedPackets) + val throughput = throughputEstimator.bitrate() + throughput shouldNotBe null + throughput!!.bytesPerSec shouldBe sendRate.bytesPerSec plusOrMinus + 0.05 * sendRate.bytesPerSec // Allow 5% error + } + + // Thoughput should stay stable. + repeat(10) { + val packetFeedback = feedbackGenerator.createFeedbackVector( + 10, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + val throughput = throughputEstimator.bitrate() + throughput!!.bytesPerSec shouldBe sendRate.bytesPerSec plusOrMinus + 0.05 * sendRate.bytesPerSec // Allow 5% error + } + } + + "ResetsIfClockChangeBackwards" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + var sendRate = 100000.bytesPerSec + var recvRate = 100000.bytesPerSec + + var packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughputEstimator.bitrate() shouldBe sendRate + + feedbackGenerator.advanceReceiveClock((-2).secs) + sendRate = 200000.bytesPerSec + recvRate = 200000.bytesPerSec + packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughputEstimator.bitrate() shouldBe sendRate + } + + "StreamPausedAndResumed" { + val feedbackGenerator = FeedbackGenerator() + val throughputEstimator = RobustThroughputEstimator(RobustThroughputEstimatorSettings(enabled = true)) + val sendRate = 100000.bytesPerSec + val recvRate = 100000.bytesPerSec + + var packetFeedback = feedbackGenerator.createFeedbackVector( + 20, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + var throughput = throughputEstimator.bitrate() + val expectedBytesPerSec = 100 * 1000.0 + throughput!!.bytesPerSec shouldBe expectedBytesPerSec plusOrMinus + 0.05 * expectedBytesPerSec // Allow 5% error + + // No packets sent or feedback received for 60s. + feedbackGenerator.advanceSendClock(60.secs) + feedbackGenerator.advanceReceiveClock(60.secs) + + // Resume sending packets at the same rate as before. The estimate + // will initially be invalid, due to lack of recent data. + packetFeedback = feedbackGenerator.createFeedbackVector( + 5, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughputEstimator.bitrate() shouldBe null + + // But be back to the normal level once we have enough data. + repeat(4) { + packetFeedback = feedbackGenerator.createFeedbackVector( + 5, + 1000.bytes, + sendRate, + recvRate + ) + throughputEstimator.incomingPacketFeedbackVector(packetFeedback) + throughput = throughputEstimator.bitrate() + throughput shouldBe sendRate + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimationTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimationTest.kt new file mode 100644 index 0000000..a310893 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/SendSideBandwidthEstimationTest.kt @@ -0,0 +1,194 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.comparables.shouldBeGreaterThanOrEqualTo +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.comparables.shouldBeLessThanOrEqualTo +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.util.bps +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import java.time.Instant + +/** + * Unit tests for Loss-Based BWE, + * based on WebRTC modules/congestion_controller/goog_cc/send_side_bandwidth_estimation_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + * + * Tests of WebRTC's RtcEventLog and tests of field trial parameters are skipped. + */ + +class SendSideBandwidthEstimationTest : FreeSpec() { + val logger = createLogger() + val diagnosticContext = DiagnosticContext() + + fun testProbing(useDelayBased: Boolean) { + val bwe = SendSideBandwidthEstimation(logger, diagnosticContext) + var nowMs = 0L + bwe.setMinMaxBitrate(100000.bps, 1500000.bps) + bwe.setSendBitrate(200000.bps, Instant.ofEpochMilli(nowMs)) + + val kRembBps = 1000000 + val kSecondRembBps = kRembBps + 500000 + + bwe.updatePacketsLost(packetsLost = 0, numberOfPackets = 1, Instant.ofEpochMilli(nowMs)) + bwe.updateRtt(50.ms, Instant.ofEpochMilli(nowMs)) + + // Initial REMB applies immediately. + if (useDelayBased) { + bwe.updateDelayBasedEstimate(Instant.ofEpochMilli(nowMs), kRembBps.bps) + } else { + bwe.updateReceiverEstimate(Instant.ofEpochMilli(nowMs), kRembBps.bps) + } + bwe.updateEstimate(Instant.ofEpochMilli(nowMs)) + bwe.targetRate().bps shouldBe kRembBps + + // Second REMB doesn't apply immediately. + nowMs += 2001 + if (useDelayBased) { + bwe.updateDelayBasedEstimate(Instant.ofEpochMilli(nowMs), kSecondRembBps.bps) + } else { + bwe.updateReceiverEstimate(Instant.ofEpochMilli(nowMs), kSecondRembBps.bps) + } + bwe.updateEstimate(Instant.ofEpochMilli(nowMs)) + bwe.targetRate().bps shouldBe kRembBps + } + + init { + "InitialRembWithProbing" { + testProbing(false) + } + + "InitialDelayBasedBweWithProbing" { + testProbing(true) + } + + "DoesntReapplyBitrateDecreaseWithoutFollowingRemb" { + val bwe = SendSideBandwidthEstimation(logger, diagnosticContext) + val kMinBitrateBps = 100000L + val kInitialBitrateBps = 1000000L + var nowMs = 1000L + bwe.setMinMaxBitrate(kMinBitrateBps.bps, 1500000.bps) + bwe.setSendBitrate(kInitialBitrateBps.bps, Instant.ofEpochMilli(nowMs)) + + val kFractionLoss: UByte = 128u + val kRttMs = 50L + nowMs += 10000 + + bwe.targetRate().bps shouldBe kInitialBitrateBps + bwe.fractionLoss() shouldBe 0u + bwe.roundTripTime().toMillis() shouldBe 0 + + // Signal heavy loss to go down in bitrate. + bwe.updatePacketsLost(packetsLost = 50, numberOfPackets = 100, Instant.ofEpochMilli(nowMs)) + bwe.updateRtt(kRttMs.ms, Instant.ofEpochMilli(nowMs)) + + // Trigger an update 2 seconds later to not be rate limited. + nowMs += 1000 + bwe.updateEstimate(Instant.ofEpochMilli(nowMs)) + bwe.targetRate().bps shouldBeLessThan kInitialBitrateBps + // Verify that the obtained bitrate isn't hitting the min bitrate, or this + // test doesn't make sense. If this ever happens, update the thresholds or + // loss rates so that it doesn't hit min bitrate after one bitrate update. + bwe.targetRate().bps shouldBeGreaterThan kMinBitrateBps + bwe.fractionLoss() shouldBe kFractionLoss + bwe.roundTripTime().toMillis() shouldBe kRttMs + + // Triggering an update shouldn't apply further downgrade nor upgrade since + // there's no intermediate receiver block received indicating whether this is + // currently good or not. + val lastBitrateBps = bwe.targetRate().bps + // Trigger an update 2 seconds later to not be rate limited (but it still + // shouldn't update). + nowMs += 1000 + bwe.updateEstimate(Instant.ofEpochMilli(nowMs)) + + bwe.targetRate().bps shouldBe lastBitrateBps + // The old loss rate should still be applied though. + bwe.fractionLoss() shouldBe kFractionLoss + bwe.roundTripTime().toMillis() shouldBe kRttMs + } + + "SettingSendBitrateOverridesDelayBasedEstimate" { + val bwe = SendSideBandwidthEstimation(logger, diagnosticContext) + val kMinBitrateBps = 10000L + val kMaxBitrateBps = 10000000L + val kInitialBitrateBps = 300000L + val kDelayBasedBitrateBps = 350000L + val kForcedHighBitrate = 2500000L + + var nowMs = 0L + + bwe.setMinMaxBitrate(kMinBitrateBps.bps, kMaxBitrateBps.bps) + bwe.setSendBitrate(kInitialBitrateBps.bps, Instant.ofEpochMilli(nowMs)) + + bwe.updateDelayBasedEstimate(Instant.ofEpochMilli(nowMs), kDelayBasedBitrateBps.bps) + bwe.updateEstimate(Instant.ofEpochMilli(nowMs)) + bwe.targetRate().bps shouldBeGreaterThanOrEqualTo kInitialBitrateBps + bwe.targetRate().bps shouldBeLessThanOrEqualTo kDelayBasedBitrateBps + + bwe.setSendBitrate(kForcedHighBitrate.bps, Instant.ofEpochMilli(nowMs)) + bwe.targetRate().bps shouldBe kForcedHighBitrate + } + + "FractionLossIsNotOverflowed" { + val bwe = SendSideBandwidthEstimation(logger, diagnosticContext) + val kMinBitrateBps = 100000L + val kInitialBitrateBps = 1000000L + var nowMs = 1000L + + bwe.setMinMaxBitrate(kMinBitrateBps.bps, 1500000.bps) + bwe.setSendBitrate(kInitialBitrateBps.bps, Instant.ofEpochMilli(nowMs)) + + nowMs += 10000 + + bwe.targetRate().bps shouldBe kInitialBitrateBps + bwe.fractionLoss() shouldBe 0u + + // Signal negative loss. + bwe.updatePacketsLost(packetsLost = -1, numberOfPackets = 100, Instant.ofEpochMilli(nowMs)) + bwe.fractionLoss() shouldBe 0u + } + + "RttIsAboveLimitIfRttGreaterThanLimit" { + val bwe = SendSideBandwidthEstimation(logger, diagnosticContext) + val kMinBitrateBps = 10000L + val kMaxBitrateBps = 10000000L + val kInitialBitrateBps = 300000L + var nowMs = 0L + bwe.setMinMaxBitrate(kMinBitrateBps.bps, kMaxBitrateBps.bps) + bwe.setSendBitrate(kInitialBitrateBps.bps, Instant.ofEpochMilli(nowMs)) + bwe.updatePropagationRtt(Instant.ofEpochMilli(nowMs), 5000.ms) + bwe.isRttAboveLimit() shouldBe true + } + + "RttIsBelowLimitIfRttLessThanLimit" { + val bwe = SendSideBandwidthEstimation(logger, diagnosticContext) + val kMinBitrateBps = 10000L + val kMaxBitrateBps = 10000000L + val kInitialBitrateBps = 300000L + var nowMs = 0L + bwe.setMinMaxBitrate(kMinBitrateBps.bps, kMaxBitrateBps.bps) + bwe.setSendBitrate(kInitialBitrateBps.bps, Instant.ofEpochMilli(nowMs)) + bwe.updatePropagationRtt(Instant.ofEpochMilli(nowMs), 1000.ms) + bwe.isRttAboveLimit() shouldBe false + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TransportFeedbackAdapterTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TransportFeedbackAdapterTest.kt new file mode 100644 index 0000000..e18e2e0 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TransportFeedbackAdapterTest.kt @@ -0,0 +1,692 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.assertions.withClue +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.bytes +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.EcnMarking +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.ReceivedPacketInfo +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.RtcpFbCcfbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.RtcpFbCcfbPacketBuilder +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.UnreceivedPacketInfo +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.NEVER +import org.jitsi.utils.TimeUtils +import org.jitsi.utils.isFinite +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import org.jitsi.utils.roundTo +import org.jitsi.utils.time.FakeClock +import org.jitsi.utils.times +import org.jitsi.utils.toDoubleMillis +import java.time.Duration +import java.time.Instant + +/** Unit tests for TransportFeedbackAdapter, + * based loosely on WebRTC modules/congestion_controller/rtp/transport_feedback_adapter_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138) + * modified to use Jitsi types for objects outside the congestion controller, + * and not using Network Routes. + */ +private val kPacingInfo0 = PacedPacketInfo(0, 5, 2000) + +data class PacketTemplate( + val ssrc: Long = 1, + val transportSequenceNumber: Long = 0, + val rtpSequenceNumber: Int = 2, + // TODO do we need mediaType? + val packetSize: DataSize = 100.bytes, + + val ecn: EcnMarking = EcnMarking.kNotEct, + val sendTimestamp: Instant = Instant.EPOCH, + val pacingInfo: PacedPacketInfo = PacedPacketInfo(), + var receiveTimestamp: Instant = NEVER, + + val isAudio: Boolean = false +) + +fun createPacketTemplates( + numberOfSsrcs: Int, + packetsPerSsrc: Int, + firstTransportSequenceNumber: Long = 99 +): List { + var transportSequenceNumber: Long = firstTransportSequenceNumber + var sendTime = Instant.EPOCH + 200.ms + var receiveTime = Instant.EPOCH + 100.ms + val packets = mutableListOf() + + for (s in 0 until numberOfSsrcs) { + val ssrc = (s + 3).toLong() + for (r in ssrc * 10 until ssrc * 10 + packetsPerSsrc) { + val rtpSequenceNumber = r.toInt() + packets.add( + PacketTemplate( + ssrc = ssrc, + transportSequenceNumber = transportSequenceNumber++, + rtpSequenceNumber = rtpSequenceNumber, + sendTimestamp = sendTime, + pacingInfo = kPacingInfo0, + receiveTimestamp = receiveTime, + ) + ) + sendTime += 10.ms + receiveTime += 13.ms + } + } + return packets +} + +private fun comparePacketFeedbackVectors(truth: List, input: List) { + input.size shouldBe truth.size + // truth contains the input data for the test, and input is what will be + // sent to the bandwidth estimator. truth.arrival_tims_ms is used to + // populate the transport feedback messages. As these times may be changed + // (because of resolution limits in the packets, and because of the time + // base adjustment performed by the TransportFeedbackAdapter at the first + // packet, the truth[x].arrival_time and input[x].arrival_time may not be + // equal. However, the difference must be the same for all x. + val arrivalTimeDelta = Duration.between(input[0].receiveTime, truth[0].receiveTimestamp).toDoubleMillis() + for (i in truth.indices) { + withClue("feedback vector packet $i") { + input[i].isReceived() shouldBe truth[i].receiveTimestamp.isFinite() + if (input[i].isReceived()) { + // Jitsi change from WebRTC: since we encode the feedback packets to their wire format, we + // need to account for the rounding of arrival times in CCFB packets. + Duration.between(input[i].receiveTime, truth[i].receiveTimestamp).toDoubleMillis() shouldBe + (arrivalTimeDelta plusOrMinus 1000.0 / 1024.0) + } + input[i].sentPacket.sendTime shouldBe truth[i].sendTimestamp + input[i].sentPacket.sequenceNumber shouldBe truth[i].transportSequenceNumber + input[i].sentPacket.size shouldBe truth[i].packetSize + input[i].sentPacket.pacingInfo shouldBe truth[i].pacingInfo + input[i].sentPacket.audio shouldBe truth[i].isAudio + input[i].rtpPacketInfo?.rtpSequenceNumber shouldBe truth[i].rtpSequenceNumber + input[i].rtpPacketInfo?.ssrc shouldBe truth[i].ssrc + // TODO do we need to compare isRetransmission? + } + } +} + +private fun createPacketToSend(packet: PacketTemplate): org.jitsi.nlj.PacketInfo { + val length = packet.packetSize.bytes.toInt() + val buf = BufferPool.getBuffer(length) + buf.fill(0, 0, length) + + val sendPacket = RtpPacket(buffer = buf, offset = 0, length = length).apply { + version = 2 + ssrc = packet.ssrc + sequenceNumber = packet.rtpSequenceNumber + } + val packetInfo = org.jitsi.nlj.PacketInfo(sendPacket).apply { + probingInfo = packet.pacingInfo + } + + return packetInfo +} + +private fun buildRtcpTransportFeedbackPacket(packets: List): RtcpFbTccPacket { + val feedbackBuilder = RtcpFbTccPacketBuilder() + feedbackBuilder.SetBase( + (packets[0].transportSequenceNumber and 0xFFFF).toInt(), + packets[0].receiveTimestamp + ) + packets.forEach { packet -> + if (packet.receiveTimestamp.isFinite()) { + feedbackBuilder.AddReceivedPacket( + (packet.transportSequenceNumber and 0xFFFF).toInt(), + packet.receiveTimestamp + ) + } + } + return feedbackBuilder.build() +} + +private fun buildRtcpCongestionControlFeedbackPacket(packets: List): RtcpFbCcfbPacket { + // Assume the feedback was sent when the last packet was received. + // JITSI modification: interpret "last" by time rather than by sequence number + val feedbackSentTime = + packets.filter { it.receiveTimestamp.isFinite() }.maxOfOrNull { it.receiveTimestamp } ?: NEVER + + val compactNtp = TimeUtils.toNtpShortFormat(TimeUtils.toNtpTime(feedbackSentTime.toEpochMilli())) + + val packetInfos = mutableListOf() + packets.forEach { packet -> + val packetInfo = if (packet.receiveTimestamp.isFinite()) { + ReceivedPacketInfo( + ssrc = packet.ssrc, + sequenceNumber = packet.rtpSequenceNumber, + arrivalTimeOffset = Duration.between(packet.receiveTimestamp, feedbackSentTime), + ecn = packet.ecn + ) + } else { + UnreceivedPacketInfo(ssrc = packet.ssrc, sequenceNumber = packet.rtpSequenceNumber) + } + packetInfos.add(packetInfo) + } + val feedbackBuilder = RtcpFbCcfbPacketBuilder(reportTimestampCompactNtp = compactNtp, packets = packetInfos) + + return feedbackBuilder.build() +} + +enum class FeedbackType { + Ccfb, + Tcc +} + +private fun timeNow() = Instant.EPOCH + 1234.ms + +class OneTransportFeedbackAdapterTest(val feedbackType: FeedbackType) { + private val logger = createLogger() + + fun createAndProcessFeedback(packets: List): TransportPacketsFeedback? { + when (feedbackType) { + FeedbackType.Tcc -> { + val rtcpFeedback = buildRtcpTransportFeedbackPacket(packets) + return adapter.processTransportFeedback(rtcpFeedback, timeNow()) + } + + FeedbackType.Ccfb -> { + val rtcpFeedback = buildRtcpCongestionControlFeedbackPacket(packets) + return adapter.processCongestionControlFeedback(rtcpFeedback, timeNow()) + } + } + } + + val clock = FakeClock() + val adapter = TransportFeedbackAdapter(logger) +} + +class TransportFeedbackAdapterTest : FreeSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + BufferPool.getBuffer = { size -> ByteArray(size + 10) } + } + + init { + "AdaptsFeedbackAndPopulatesSendTimes" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val packets = createPacketTemplates(numberOfSsrcs = 2, packetsPerSsrc = 3) + + packets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + val adaptedFeedback = test.createAndProcessFeedback(packets) + comparePacketFeedbackVectors(packets, adaptedFeedback!!.packetFeedbacks) + } + } + } + + "FeedbackVectorReportsUnreceived" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val sentPackets = createPacketTemplates(numberOfSsrcs = 2, packetsPerSsrc = 3) + + sentPackets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + + // Note: Important to include the last packet per SSRC, as only unreceived + // packets in between received packets can be inferred. + sentPackets[1].receiveTimestamp = NEVER + sentPackets[4].receiveTimestamp = NEVER + val adaptedFeedback = test.createAndProcessFeedback(sentPackets) + comparePacketFeedbackVectors(sentPackets, adaptedFeedback!!.packetFeedbacks) + } + } + } + + "HandlesDroppedPackets" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val packets = + createPacketTemplates(numberOfSsrcs = 2, packetsPerSsrc = 3, firstTransportSequenceNumber = 0) + + val kSendSideDropBefore = 1 + val kReceiveSideDropAfter = 3 + + val sentPackets = mutableListOf() + + packets.forEach { packet -> + if (packet.transportSequenceNumber >= kSendSideDropBefore) { + sentPackets.add(packet) + } + } + + sentPackets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + + val receivedPackets = mutableListOf() + packets.forEach { packet -> + if (packet.transportSequenceNumber <= kReceiveSideDropAfter) { + receivedPackets.add(packet) + } + } + + val adaptedFeedback = test.createAndProcessFeedback(receivedPackets) + + val expectedPackets = packets.subList(kSendSideDropBefore, kReceiveSideDropAfter + 1) + + comparePacketFeedbackVectors(expectedPackets, adaptedFeedback!!.packetFeedbacks) + } + } + } + + // Skipped: "FeedbackReportsIfPacketIsAudio" + + "ReceiveTimeWrapsBothWays" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val kHighArrivalTime = RtcpFbTccPacket.Companion.kDeltaScaleFactor * (1 shl 8) * ((1 shl 23) - 1) + val packets = listOf( + PacketTemplate( + transportSequenceNumber = 0, + rtpSequenceNumber = 102, + receiveTimestamp = Instant.EPOCH + kHighArrivalTime + 64.ms + ), + PacketTemplate( + transportSequenceNumber = 1, + rtpSequenceNumber = 103, + receiveTimestamp = Instant.EPOCH + kHighArrivalTime - 64.ms + ), + PacketTemplate( + transportSequenceNumber = 2, + rtpSequenceNumber = 104, + receiveTimestamp = Instant.EPOCH + kHighArrivalTime + ) + ) + + packets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + + packets.forEach { packet -> + val receivedPackets = listOf(packet) + val result: TransportPacketsFeedback? + when (test.feedbackType) { + FeedbackType.Ccfb -> { + val feedback = buildRtcpCongestionControlFeedbackPacket(receivedPackets) + val rawBuffer = feedback.buffer + val offset = feedback.offset + val length = feedback.length + + val parsedFeedback = RtcpFbCcfbPacket(rawBuffer, offset, length) + result = test.adapter.processCongestionControlFeedback(parsedFeedback, timeNow()) + } + + FeedbackType.Tcc -> { + val feedback = buildRtcpTransportFeedbackPacket(receivedPackets) + val rawBuffer = feedback.buffer + val offset = feedback.offset + val length = feedback.length + + val parsedFeedback = RtcpFbTccPacket(rawBuffer, offset, length) + result = test.adapter.processTransportFeedback(parsedFeedback, timeNow()) + } + } + result shouldNotBe null + comparePacketFeedbackVectors(receivedPackets, result!!.packetFeedbacks) + } + } + } + } + + "HandlesArrivalReordering" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val packets = listOf( + PacketTemplate( + transportSequenceNumber = 0, + rtpSequenceNumber = 101, + sendTimestamp = Instant.EPOCH + 200.ms, + receiveTimestamp = Instant.EPOCH + 120.ms + ), + PacketTemplate( + transportSequenceNumber = 1, + rtpSequenceNumber = 102, + sendTimestamp = Instant.EPOCH + 210.ms, + receiveTimestamp = Instant.EPOCH + 110.ms + ), + PacketTemplate( + transportSequenceNumber = 2, + rtpSequenceNumber = 103, + sendTimestamp = Instant.EPOCH + 220.ms, + receiveTimestamp = Instant.EPOCH + 100.ms + ), + ) + + packets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + + // Adapter keeps the packets ordered by sequence number (which is itself + // assigned by the order of transmission). Reordering by some other criteria, + // eg. arrival time, is up to the observers. + val adaptedFeedback = test.createAndProcessFeedback(packets) + comparePacketFeedbackVectors(packets, adaptedFeedback!!.packetFeedbacks) + } + } + } + + "IgnoreDuplicatePacketSentCalls" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val packet = PacketTemplate() + // Add a packet and then mark it as sent. + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + val sentPacket = test.adapter.processSentPacket( + SentPacketInfo(packet.transportSequenceNumber, packet.sendTimestamp, PacketInfo()) + ) + sentPacket shouldNotBe null + + // Call ProcessSentPacket() again with the same sequence number. This packet + // has already been marked as sent and the call should be ignored. + val duplicatePacket = test.adapter.processSentPacket( + SentPacketInfo(packet.transportSequenceNumber, packet.sendTimestamp, PacketInfo()) + ) + duplicatePacket shouldBe null + } + } + } + + "SendReceiveTimeDiffTimeContinuouseBetweenFeedback" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + val packets = listOf( + PacketTemplate( + transportSequenceNumber = 1, + rtpSequenceNumber = 101, + sendTimestamp = Instant.EPOCH + 100.ms, + pacingInfo = kPacingInfo0, + receiveTimestamp = Instant.EPOCH + 200.ms + ), + PacketTemplate( + transportSequenceNumber = 2, + rtpSequenceNumber = 102, + sendTimestamp = Instant.EPOCH + 110.ms, + pacingInfo = kPacingInfo0, + receiveTimestamp = Instant.EPOCH + 210.ms + ) + ) + + packets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + + val adaptedFeedback1 = test.createAndProcessFeedback(listOf(packets[0])) + val adaptedFeedback2 = test.createAndProcessFeedback(listOf(packets[1])) + + adaptedFeedback1!!.packetFeedbacks.size shouldBe adaptedFeedback2!!.packetFeedbacks.size + adaptedFeedback1.packetFeedbacks.size shouldBe 1 + Duration.between( + adaptedFeedback1.packetFeedbacks[0].sentPacket.sendTime, + adaptedFeedback1.packetFeedbacks[0].receiveTime + ).roundTo(1.ms) shouldBe + Duration.between( + adaptedFeedback2.packetFeedbacks[0].sentPacket.sendTime, + adaptedFeedback2.packetFeedbacks[0].receiveTime + ).roundTo(1.ms) + } + } + } + + "ProcessSentPacketIncreaseOutstandingData" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + + val packet1 = PacketTemplate(transportSequenceNumber = 1, packetSize = 200.bytes) + val packet2 = PacketTemplate(transportSequenceNumber = 2, packetSize = 300.bytes) + test.adapter.addPacket( + createPacketToSend(packet1), + packet1.transportSequenceNumber, + 0.bytes, + timeNow() + ) + val sentPacket1 = test.adapter.processSentPacket( + SentPacketInfo(packet1.transportSequenceNumber, packet1.sendTimestamp) + ) + + sentPacket1 shouldNotBe null + sentPacket1!!.sequenceNumber shouldBe packet1.transportSequenceNumber + // Only one packet in flight + sentPacket1.dataInFlight shouldBe packet1.packetSize + test.adapter.getOutstandingData() shouldBe packet1.packetSize + + test.adapter.addPacket( + createPacketToSend(packet2), + packet2.transportSequenceNumber, + 0.bytes, + timeNow() + ) + val sentPacket2 = test.adapter.processSentPacket( + SentPacketInfo(packet2.transportSequenceNumber, packet2.sendTimestamp) + ) + + sentPacket2 shouldNotBe null + // Two packets in flight. + sentPacket2!!.dataInFlight shouldBe packet1.packetSize + packet2.packetSize + + test.adapter.getOutstandingData() shouldBe packet1.packetSize + packet2.packetSize + } + } + } + + "TransportPacketFeedbackHasDataInFlight" { + FeedbackType.entries.forEach { + withClue(it) { + val test = OneTransportFeedbackAdapterTest(it) + + val packets = listOf( + PacketTemplate( + transportSequenceNumber = 1, + rtpSequenceNumber = 101, + packetSize = 200.bytes, + sendTimestamp = Instant.EPOCH + 100.ms, + pacingInfo = kPacingInfo0, + receiveTimestamp = Instant.EPOCH + 200.ms, + ), + PacketTemplate( + transportSequenceNumber = 2, + rtpSequenceNumber = 102, + packetSize = 300.bytes, + sendTimestamp = Instant.EPOCH + 110.ms, + pacingInfo = kPacingInfo0, + receiveTimestamp = Instant.EPOCH + 210.ms, + ) + ) + + packets.forEach { packet -> + test.adapter.addPacket( + createPacketToSend(packet), + packet.transportSequenceNumber, + 0.bytes, + timeNow() + ) + test.adapter.processSentPacket( + SentPacketInfo( + packet.transportSequenceNumber, + packet.sendTimestamp + ) + ) + } + + val adaptedFeedback1 = test.createAndProcessFeedback(listOf(packets[0])) + val adaptedFeedback2 = test.createAndProcessFeedback(listOf(packets[1])) + adaptedFeedback1!!.dataInFlight shouldBe packets[1].packetSize + adaptedFeedback2!!.dataInFlight shouldBe DataSize.ZERO + } + } + } + + "CongestionControlFeedbackResultHasEcn" { + val test = OneTransportFeedbackAdapterTest(feedbackType = FeedbackType.Ccfb) + + val packets = listOf( + PacketTemplate( + transportSequenceNumber = 1, + rtpSequenceNumber = 101, + ecn = EcnMarking.kCe, + sendTimestamp = Instant.EPOCH + 100.ms, + receiveTimestamp = Instant.EPOCH + 200.ms + ), + PacketTemplate( + transportSequenceNumber = 2, + rtpSequenceNumber = 102, + ecn = EcnMarking.kEct1, + sendTimestamp = Instant.EPOCH + 110.ms, + receiveTimestamp = Instant.EPOCH + 210.ms + ), + ) + + packets.forEach { packet -> + test.adapter.addPacket(createPacketToSend(packet), packet.transportSequenceNumber, 0.bytes, timeNow()) + test.adapter.processSentPacket(SentPacketInfo(packet.transportSequenceNumber, packet.sendTimestamp)) + } + + val rtcpFeedback = buildRtcpCongestionControlFeedbackPacket(packets) + val adaptedFeedback = test.adapter.processCongestionControlFeedback(rtcpFeedback, timeNow()) + + adaptedFeedback!!.packetFeedbacks.size shouldBe 2 + adaptedFeedback.packetFeedbacks[0].ecn shouldBe EcnMarking.kCe + adaptedFeedback.packetFeedbacks[1].ecn shouldBe EcnMarking.kEct1 + adaptedFeedback.transportSupportsEcn shouldBe true + } + + "ReportTransportDoesNotSupportEcnIfFeedbackContainNotEctPacket" { + val test = OneTransportFeedbackAdapterTest(feedbackType = FeedbackType.Ccfb) + + val packets = listOf( + PacketTemplate( + transportSequenceNumber = 1, + rtpSequenceNumber = 101, + ecn = EcnMarking.kCe, + sendTimestamp = Instant.EPOCH + 100.ms, + receiveTimestamp = Instant.EPOCH + 200.ms + ), + PacketTemplate( + transportSequenceNumber = 2, + rtpSequenceNumber = 102, + ecn = EcnMarking.kNotEct, + sendTimestamp = Instant.EPOCH + 110.ms, + receiveTimestamp = Instant.EPOCH + 210.ms + ), + ) + + packets.forEach { packet -> + test.adapter.addPacket(createPacketToSend(packet), packet.transportSequenceNumber, 0.bytes, timeNow()) + test.adapter.processSentPacket(SentPacketInfo(packet.transportSequenceNumber, packet.sendTimestamp)) + } + + val rtcpFeedback = buildRtcpCongestionControlFeedbackPacket(packets) + val adaptedFeedback = test.adapter.processCongestionControlFeedback(rtcpFeedback, timeNow()) + + adaptedFeedback!!.transportSupportsEcn shouldBe false + adaptedFeedback.packetFeedbacks.size shouldBe 2 + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TrendlineEstimatorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TrendlineEstimatorTest.kt new file mode 100644 index 0000000..0c41a3f --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/bandwidthestimation2/TrendlineEstimatorTest.kt @@ -0,0 +1,138 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming") + +package org.jitsi.nlj.rtp.bandwidthestimation2 + +import io.kotest.core.spec.style.FreeSpec +import io.kotest.matchers.shouldBe +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger + +/** + * Unit tests of TrendlineEstimator + * + * Based on WebRTC modules/congestion_controller/goog_cc/trendline_estimator_unittest.cc in + * WebRTC tag branch-heads/7204 (Chromium 138). + */ +class OneTrendlineEstimatorTest( + sendTimeGenerator: (Int) -> Long, + recvTimeGenerator: (Int) -> Long, + packetSize: Long = kPacketSizeBytes +) { + val sendTimes = LongArray(kPacketCount, sendTimeGenerator) + val recvTimes = LongArray(kPacketCount, recvTimeGenerator) + val packetSizes = LongArray(kPacketCount) { packetSize } + + var count = 1 + + val estimator = TrendlineEstimator(createLogger(), DiagnosticContext()) + + fun runTestUntilStateChange() { + check(count >= 1) + check(count < kPacketCount) + + val initialState = estimator.state() + while (count < kPacketCount) { + val recvDelta = recvTimes[count] - recvTimes[count - 1] + val sendDelta = sendTimes[count] - sendTimes[count - 1] + estimator.update( + recvDelta.toDouble(), + sendDelta.toDouble(), + sendTimes[count], + recvTimes[count], + packetSizes[count], + true + ) + if (estimator.state() != initialState) { + return + } + count++ + } + } + + companion object { + const val kPacketCount = 25 + const val kPacketSizeBytes = 1200L + } +} + +class TrendlineEstimatorTest : FreeSpec() { + init { + "Normal usage" { + val sendTimeGenerator = PacketTimeGenerator(initialClock = 123456789, timeBetweenPackets = 20.0) + /* Delivered at the same pace */ + val recvTimeGenerator = PacketTimeGenerator(initialClock = 987654321, timeBetweenPackets = 20.0) + + val test = OneTrendlineEstimatorTest(sendTimeGenerator::get, recvTimeGenerator::get) + test.estimator.state() shouldBe BandwidthUsage.kBwNormal + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwNormal + test.count shouldBe OneTrendlineEstimatorTest.kPacketCount // All packets processed + } + + "Overusing" { + val sendTimeGenerator = PacketTimeGenerator(initialClock = 123456789, timeBetweenPackets = 20.0) + /* 10% slower delivery */ + val recvTimeGenerator = PacketTimeGenerator(initialClock = 987654321, timeBetweenPackets = 1.1 * 20.0) + + val test = OneTrendlineEstimatorTest(sendTimeGenerator::get, recvTimeGenerator::get) + test.estimator.state() shouldBe BandwidthUsage.kBwNormal + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwOverusing + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwOverusing + test.count shouldBe OneTrendlineEstimatorTest.kPacketCount // All packets processed + } + + "Underusing" { + val sendTimeGenerator = PacketTimeGenerator(initialClock = 123456789, timeBetweenPackets = 20.0) + /* 15% faster delivery */ + val recvTimeGenerator = PacketTimeGenerator(initialClock = 987654321, timeBetweenPackets = 0.85 * 20.0) + + val test = OneTrendlineEstimatorTest(sendTimeGenerator::get, recvTimeGenerator::get) + test.estimator.state() shouldBe BandwidthUsage.kBwNormal + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwUnderusing + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwUnderusing + test.count shouldBe OneTrendlineEstimatorTest.kPacketCount // All packets processed + } + + "IncludesSmallPacketsByDefault" { + val sendTimeGenerator = PacketTimeGenerator(initialClock = 123456789, timeBetweenPackets = 20.0) + /* 10% slower delivery */ + val recvTimeGenerator = PacketTimeGenerator(initialClock = 987654321, timeBetweenPackets = 1.1 * 20.0) + + val test = OneTrendlineEstimatorTest(sendTimeGenerator::get, recvTimeGenerator::get, 100) + test.estimator.state() shouldBe BandwidthUsage.kBwNormal + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwOverusing + test.runTestUntilStateChange() + test.estimator.state() shouldBe BandwidthUsage.kBwOverusing + test.count shouldBe OneTrendlineEstimatorTest.kPacketCount // All packets processed + } + } +} + +class PacketTimeGenerator( + private val initialClock: Long, + private val timeBetweenPackets: Double +) { + private var packets = 0 + + fun get(idx: Int): Long = (initialClock + timeBetweenPackets * idx).toLong() +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt new file mode 100644 index 0000000..16a0b6b --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDPacketTest.kt @@ -0,0 +1,242 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.codec.av1 + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.LoggerImpl +import javax.xml.bind.DatatypeConverter + +class Av1DDPacketTest : ShouldSpec() { + val logger = LoggerImpl(javaClass.name) + + private data class SampleAv1DDPacket( + val description: String, + val data: ByteArray, + val structureSource: SampleAv1DDPacket?, + // ... + val scalabilityStructure: RtpEncodingDesc? = null + + ) { + constructor( + description: String, + hexData: String, + structureSource: SampleAv1DDPacket?, + // ... + scalabilityStructure: RtpEncodingDesc? = null + ) : this( + description = description, + data = DatatypeConverter.parseHexBinary(hexData), + structureSource = structureSource, + // ... + scalabilityStructure = scalabilityStructure + ) + } + + /* Packets captured from AV1 test calls */ + + private val nonScalableKeyframe = + SampleAv1DDPacket( + "non-scalable keyframe", + // RTP header + "902939b91ff6a695114ed316" + + // Header extension header + "bede0006" + + // Other header extensions + "3248c482" + + "510002" + + // AV1 DD + "bc80000180003a410180ef808680" + + // Padding + "000000" + + // AV1 Media payload, truncated. + "680b0800000004477e1a00d004301061", + null, + RtpEncodingDesc( + 0x114ed316L, + arrayOf( + Av1DDRtpLayerDesc(0, 0, 0, 0, 270, 30.0) + ) + ) + ) + + private val scalableKeyframe = + SampleAv1DDPacket( + "scalable keyframe", + // RTP header + "906519aed780a8f32ab2873c" + + // Header extension header (2-byte) + "1000001a" + + // Other header extensions + "030302228e" + + "050219f9" + + // AV1 DD + "0b5a8003ca80081485214eaaaaa8000600004000100002aa80a80006" + + "00004000100002a000a800060000400016d549241b5524906d549231" + + "57e001974ca864330e222396eca8655304224390eca87753013f00b3" + + "027f016704ff02cf" + + // Padding + "000000" + + // AV1 Media payload, truncated. + "aabbdc101a58014000b4028001680500", + null, + RtpEncodingDesc( + 0x2ab2873cL, + arrayOf( + Av1DDRtpLayerDesc(0, 0, 0, 0, 180, 7.5), + Av1DDRtpLayerDesc(0, 1, 1, 0, 180, 15.0), + Av1DDRtpLayerDesc(0, 2, 2, 0, 180, 30.0), + Av1DDRtpLayerDesc(0, 3, 1, 0, 360, 7.5), + Av1DDRtpLayerDesc(0, 4, 1, 1, 360, 15.0), + Av1DDRtpLayerDesc(0, 5, 1, 2, 360, 30.0), + Av1DDRtpLayerDesc(0, 6, 2, 0, 720, 7.5), + Av1DDRtpLayerDesc(0, 7, 2, 1, 720, 15.0), + Av1DDRtpLayerDesc(0, 8, 2, 2, 720, 30.0) + ) + ) + ) + + private val testPackets = arrayOf( + nonScalableKeyframe, + SampleAv1DDPacket( + "non-scalable following packet", + // RTP header + "90a939ba1ff6a695114ed316" + + // Header extension header + "bede0003" + + // Other header extensions + "3248d02a" + + "510003" + + // AV1 DD + "b2400001" + + // Padding + "00" + + // AV1 Media payload, truncated. + "9057f7c3f51ba803b0c397e938589750", + nonScalableKeyframe + ), + scalableKeyframe, + SampleAv1DDPacket( + "scalable following packet changing available DTs", + // RTP header + "90e519c2d780b1bd2ab2873c" + + // Header extension header + "bede0004" + + // Other header extensions + "3202f824" + + "511a19" + + // AV1 DD + "b4c303cd401c" + + // Padding + "000000" + + // AV1 Media payload, truncated. + "edbbdd501a87000000027e016704ff02", + scalableKeyframe, + RtpEncodingDesc( + 0x2ab2873cL, + arrayOf( + Av1DDRtpLayerDesc(0, 0, 0, 0, 180, 7.5), + Av1DDRtpLayerDesc(0, 1, 0, 1, 180, 15.0), + Av1DDRtpLayerDesc(0, 2, 0, 2, 180, 30.0), + ) + ) + ), + SampleAv1DDPacket( + "temporally-scalable portrait-mode keyframe", + // RTP header + "906487db2fb5ae06481d4dca" + + // Two-byte header extension header + "1000000f" + + // Other header extensions + "030387b1a4" + + "0502827e" + + // AV1 DD + "0b178023c58002044eaaaf2860414d34538a094040b3c13fc0" + + // VLA + "0c17a1009601f403dc0b00b3013f060167027f0602cf04ff06" + + // padding + "00" + + // Media payload, truncated + "109096029d012ad0020005396b00", + null, + RtpEncodingDesc( + 0x481d4dcaL, + arrayOf( + // Make sure the correct "height" value is picked from portrait mode (720x1280) + // TODO: I think the frame rate calculation gets this DD wrong, but I'm not quite sure + // what structure the encoder is using + Av1DDRtpLayerDesc(0, 0, 0, 0, 720, 10.0), + Av1DDRtpLayerDesc(0, 1, 1, 0, 720, 20.0), + Av1DDRtpLayerDesc(0, 2, 2, 0, 720, 30.0), + ) + ) + + ) + ) + + init { + context("AV1 packets") { + should("be parsed correctly") { + for (t in testPackets) { + withClue(t.description) { + val structure = if (t.structureSource != null) { + val sourceR = + RtpPacket(t.structureSource.data, 0, t.structureSource.data.size) + val sourceP = Av1DDPacket(sourceR, AV1_DD_HEADER_EXTENSION_ID, null, logger) + sourceP.descriptor?.structure + } else { + null + } + + val r = RtpPacket(t.data, 0, t.data.size) + val p = Av1DDPacket(r, AV1_DD_HEADER_EXTENSION_ID, structure, logger) + + if (t.scalabilityStructure != null) { + val tss = t.scalabilityStructure + val ss = p.getScalabilityStructure() + ss shouldNotBe null + ss!!.primarySSRC shouldBe tss.primarySSRC + ss.layers.size shouldBe tss.layers.size + for ((index, layer) in ss.layers.withIndex()) { + val tLayer = tss.layers[index] + layer.layerId shouldBe tLayer.layerId + layer.index shouldBe tLayer.index + layer should beInstanceOf() + layer as Av1DDRtpLayerDesc + tLayer as Av1DDRtpLayerDesc + layer.dt shouldBe tLayer.dt + layer.height shouldBe tLayer.height + layer.frameRate shouldBe tLayer.frameRate + } + } else { + p.getScalabilityStructure() shouldBe null + } + } + } + } + } + } + + companion object { + const val AV1_DD_HEADER_EXTENSION_ID = 11 + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt new file mode 100644 index 0000000..9ea2523 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/rtp/codec/vp9/Vp9PacketTest.kt @@ -0,0 +1,862 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.rtp.codec.vp9 + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.rtp.codec.vpx.VpxRtpLayerDesc +import org.jitsi_modified.impl.neomedia.codec.video.vp9.DePacketizer +import javax.xml.bind.DatatypeConverter + +class Vp9PacketTest : ShouldSpec() { + private data class SampleVp9Packet( + val description: String, + val data: ByteArray, + val isStartOfFrame: Boolean, + val isEndOfFrame: Boolean, + val isEndOfPicture: Boolean, + val isKeyframe: Boolean, + val isInterPicturePredicted: Boolean, + val pictureId: Int?, + val hasExtendedPictureId: Boolean, + val isUpperLevelReference: Boolean, + val tid: Int?, + val sid: Int?, + val isSwitchingUpPoint: Boolean, + val usesInterLayerDependency: Boolean, + val tL0PICIDX: Int?, + val descriptorSize: Int, + val scalabilityStructure: RtpEncodingDesc? = null + ) { + constructor( + description: String, + hexData: String, + isStartOfFrame: Boolean, + isEndOfFrame: Boolean, + isEndOfPicture: Boolean, + isKeyframe: Boolean, + isInterPicturePredicted: Boolean, + pictureId: Int?, + hasExtendedPictureId: Boolean, + isUpperLevelReference: Boolean, + tid: Int?, + sid: Int?, + isSwitchingUpPoint: Boolean, + usesInterLayerDependency: Boolean, + tL0PICIDX: Int?, + descriptorSize: Int, + scalabilityStructure: RtpEncodingDesc? = null + ) : this( + description = description, + data = DatatypeConverter.parseHexBinary(hexData), + isStartOfFrame = isStartOfFrame, + isEndOfFrame = isEndOfFrame, + isEndOfPicture = isEndOfPicture, + isKeyframe = isKeyframe, + isInterPicturePredicted = isInterPicturePredicted, + pictureId = pictureId, + hasExtendedPictureId = hasExtendedPictureId, + isUpperLevelReference = isUpperLevelReference, + tid = tid, + sid = sid, + isSwitchingUpPoint = isSwitchingUpPoint, + usesInterLayerDependency = usesInterLayerDependency, + tL0PICIDX = tL0PICIDX, + descriptorSize = descriptorSize, + scalabilityStructure = scalabilityStructure + ) + } + + /* Packets captured from Chrome VP9 call */ + private val testPackets = arrayOf( + /* Live video - Chrome 81 */ + SampleVp9Packet( + "Chrome: Start of keyframe (with SS) in K-SVC stream", + // RTP + "906536b69f3077686098017b" + + // RTP header extension + "bede0002" + "3202168751210700" + + // I=1,P=0,L=1,F=0,B=1,E=0,V=1,Z=0 + "aa" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0xfd=253 + "fd" + + // Begin SS: N_S=2,Y=1,G=1 + "58" + + // WIDTH=320 + "0140" + + // HEIGHT=180 + "00b4" + + // WIDTH=640 + "0280" + + // HEIGHT=360 + "0168" + + // WIDTH=1280 + "0500" + + // HEIGHT=720 + "02d0" + + // N_G=4 + "04" + + // TID=0,U=0,R=1 + "04" + + // P_DIFF=4 + "04" + + // TID=2,U=1,R=1 + "54" + + // P_DIFF=1 + "01" + + // TID=1,U=1,R=1 + "34" + + // P_DIFF=2 + "02" + + // TID=2,U=1,R=1 + "54" + + // P_DIFF=1 + "01" + + // VP9 media. Truncated. + "834983420013f00b3827f8167858e0907063a8000f", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 0, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = 253, + descriptorSize = 27, + scalabilityStructure = RtpEncodingDesc( + 0x6098017bL, + arrayOf( + VpxRtpLayerDesc(0, 0, 0, 180, 7.5), + /* TODO: dependencies */ + VpxRtpLayerDesc(0, 1, 0, 180, 15.0), + VpxRtpLayerDesc(0, 2, 0, 180, 30.0), + VpxRtpLayerDesc(0, 0, 1, 360, 7.5), + VpxRtpLayerDesc(0, 1, 1, 360, 15.0), + VpxRtpLayerDesc(0, 2, 1, 360, 30.0), + VpxRtpLayerDesc(0, 0, 2, 720, 7.5), + VpxRtpLayerDesc(0, 1, 2, 720, 15.0), + VpxRtpLayerDesc(0, 2, 2, 720, 30.0) + ) + ) + ), + SampleVp9Packet( + "Middle of keyframe (SID = 0)", + "906536b79f3077686098017b" + + "bede0002" + "3202168751210800" + + // I=1,P=0,L=1,F=0,B=0,E=0,V=0,Z=0 + "a0" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "1989b602dbf8c9bf071d00b3e3ccf1135bd14a", + isStartOfFrame = false, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 0, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "End of keyframe (SID = 0)", + "906536bb9f3077686098017b" + + "bede0002" + "32021ba651210c00" + + // I=1,P=0,L=1,F=0,B=0,E=1,V=0,Z=0 + "a4" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "65019dea53fb1ec89768361a113523c6da88c9", + isStartOfFrame = false, + isEndOfFrame = true, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 0, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "Beginning of SID=1 in keyframe picture", + "906536bc9f3077686098017b" + + "bede0002" + "32021ba651210d00" + + // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=0 + "a8" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=1,D=1 + "03" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "8702020004fe02cf04ff02cf9121c120e0c81080", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 1, + isSwitchingUpPoint = false, + usesInterLayerDependency = true, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "Middle of SID=1 in keyframe picture", + "906536c09f3077686098017b" + + "bede0002" + "320221cb51211100" + + // I=1,P=0,L=1,F=0,B=0,E=0,V=0,Z=0 + "a0" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=1,D=1 + "03" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "08d6c732b4351ba6a440acb18ae6c444d02f89e2", + isStartOfFrame = false, + isEndOfFrame = false, + isEndOfPicture = false, + isInterPicturePredicted = false, + isKeyframe = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 1, + isSwitchingUpPoint = false, + usesInterLayerDependency = true, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "End of SID=1 in keyframe picture", + "906536c39f3077686098017b" + + "bede0002" + "320227f051211400" + + // I=1,P=0,L=1,F=0,B=0,E=1,V=0,Z=0 + "a4" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=1,D=1 + "03" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "2993d7ab77552b9ff92cb2f64cfd3dc9f81c8db7", + isStartOfFrame = false, + isEndOfFrame = true, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 1, + isSwitchingUpPoint = false, + usesInterLayerDependency = true, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "Beginning of SID=2 in keyframe picture", + "906536c49f3077686098017b" + + "bede0002" + "320227f051211500" + + // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=1 + "a9" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=2,D=1 + "05" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "8704242009fe059ec48704838320830089007fbc", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = 0, + sid = 2, + isSwitchingUpPoint = false, + usesInterLayerDependency = true, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "End of SID=2 in keyframe picture -- end of picture", + "90e536dd9f3077686098017b" + + "bede0002" + "32025a1d51212e00" + + // I=1,P=0,L=1,F=0,B=0,E=1,V=0,Z=1 + "a5" + + // M=1,PID=0x1e65=7781 + "9e65" + + // TID=0,U=0,SID=2,D=1 + "05" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "646f37516a6889fea39038cbc29dcde9702e8495", + isStartOfFrame = false, + isEndOfFrame = true, + isEndOfPicture = true, + isKeyframe = false, + isInterPicturePredicted = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = 0, + sid = 2, + isSwitchingUpPoint = false, + usesInterLayerDependency = true, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "Complete SID=0,TID=1 K-SVC frame", + "906536de9f3081f46098017b" + + "bede0002" + "32025a1d51212f00" + + // I=1,P=1,L=1,F=0,B=1,E=1,V=0,Z=1 + "ed" + + // M=1,PID=0x1e66=7782 + "9e66" + + // TID=2,U=1,SID=0,D=0 + "50" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "87080060027e016704ff02cfc487048383208000", + isStartOfFrame = true, + isEndOfFrame = true, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = true, + pictureId = 7782, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = 2, + sid = 0, + isSwitchingUpPoint = true, + usesInterLayerDependency = false, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "Complete SID=1,TID=1 K-SVC frame", + "906536df9f3081f46098017b" + + "bede0002" + "32025a1d51213000" + + // I=1,P=1,L=1,F=0,B=1,E=1,V=0,Z=1 + "ed" + + // M=1,PID=0x1e66=7782 + "9e66" + + // TID=2,U=1,SID=1,D=0 + "52" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "8710228004fe02cf04ff02cf9211c120e0cab080", + isStartOfFrame = true, + isEndOfFrame = true, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = true, + pictureId = 7782, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = 2, + sid = 1, + isSwitchingUpPoint = true, + usesInterLayerDependency = false, + tL0PICIDX = 253, + descriptorSize = 5 + ), + SampleVp9Packet( + "Complete SID=2,TID=1 K-SVC frame", + "90e536e09f3081f46098017b" + + "bede0002" + "32025a1d51213100" + + // I=1,P=1,L=1,F=0,B=1,E=1,V=0,Z=1 + "ed" + + // M=1,PID=0x1e66=7782 + "9e66" + + // TID=2,U=1,SID=2,D=0 + "54" + + // TL0PICIDX=0xfd=253 + "fd" + + // VP9 media. Truncated. + "8700444009fe059ec8c70483832b83001c007e20", + isStartOfFrame = true, + isEndOfFrame = true, + isEndOfPicture = true, + isKeyframe = false, + isInterPicturePredicted = true, + pictureId = 7782, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = 2, + sid = 2, + isSwitchingUpPoint = true, + usesInterLayerDependency = false, + tL0PICIDX = 253, + descriptorSize = 5 + ), + + /* Window capture - Chrome 81 */ + SampleVp9Packet( + "Beginning of window capture keyframe - contains SS, 1 layer", + "90656dc9440dac37184b0cc4" + + "bede0002" + "326bcdd351000100" + + // I=1,P=0,L=0,F=0,B=1,E=0,V=1,Z=1 + "8b" + + // M=1,PID=0x2558=9560 + "a558" + + // Begin SS: N_S=0,Y=1,G=1 + "18" + + // WIDTH=1576 + "0628" + + // HEIGHT=1158 + "0486" + + // N_G=1 + "01" + + // TID=0,U=0,R=1 + "04" + + // P_DIFF=1 + "01" + + // VP9 media. Truncated. + "8249834200627048547638241c19a01803105f85", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 9560, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = null, + sid = null, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 11, + scalabilityStructure = RtpEncodingDesc( + 0x184b0cc4L, + arrayOf( + VpxRtpLayerDesc(0, 0, 0, 1158, 30.0) + ) + ) + ), + SampleVp9Packet( + "Middle of window capture keyframe", + "90656dca440dac37184b0cc4" + + "bede0002" + "326bd1ec51000200" + + // I=1,P=0,L=0,F=0,B=0,E=0,V=0,Z=1 + "81" + + // M=1,PID=0x2558=9560 + "a558" + + // VP9 media. Truncated. + "9089bebb979590638f183ac76bc650cde64144de", + isStartOfFrame = false, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 9560, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = null, + sid = null, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 3 + ), + SampleVp9Packet( + "End of window capture keyframe", + "90e56de4440dac37184b0cc4" + + "bede0002" + "326cf8d551001c00" + + // I=1,P=0,L=0,F=0,B=0,E=1,V=0,Z=1 + "85" + + // M=1,PID=0x2558=9560 + "a558" + + // VP9 media. Truncated. + "7da09c7d64214a918874bb6b9cde16ef67545921", + isStartOfFrame = false, + isEndOfFrame = true, + isEndOfPicture = true, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 9560, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = null, + sid = null, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 3 + ), + SampleVp9Packet( + "Complete window capture non-keyframe", + "90656e01440eb32f184b0cc4" + + "bede0002" + "326e77cf51003900" + + // I=1,P=1,L=0,F=0,B=1,E=1,V=0,Z=1 + "c9" + + // M=1,PID=0x255A=9562 + "a55a" + + // VP9 media. Truncated. + "8600409218fc5a0330101c060301f8c9e03049a2", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = true, + pictureId = 9562, + hasExtendedPictureId = true, + isUpperLevelReference = false, + tid = null, + sid = null, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 3 + ), + + /* Live video - Firefox 75 */ + SampleVp9Packet( + "Beginning of Firefox 75 keyframe - no scalability", + "9065385f33e8e7666538459e" + + "bede0001" + "32a4e45a" + + // I=1,P=0,L=0,F=0,B=1,E=0,V=1,Z=0 + "8a" + + // M=1,PID=0x5bd8=23512 + "dbd8" + + // Begin SS: N_S=0,Y=1,G=1 + "18" + + // WIDTH=1280 + "0500" + + // HEIGHT=720 + "02d0" + + // N_G=1 + "01" + + // TID=0,U=0,R=1 + "04" + + // P_DIFF=1 + "01" + + // VP9 media. Truncated. + "83498342004ff02cf050e0907063486011805fcf", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 23512, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = null, + sid = null, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 11, + scalabilityStructure = RtpEncodingDesc( + 0x6538459eL, + arrayOf( + VpxRtpLayerDesc(0, 0, 0, 720, 30.0) + ) + ) + ), + + /* Live video - Firefox 77 Nightly */ + SampleVp9Packet( + "Beginning of Firefox 77 keyframe - only temporal scalability", + "90656563e256bc64a4d04528" + + "bede0001" + "329c676d" + + // I=1,P=0,L=1,F=0,B=1,E=0,V=1,Z=0 + "aa" + + // M=1,PID=0x653e=25918 + "e53e" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0x5b=91 + "5b" + + // Begin SS: N_S=0,Y=1,G=1 + "18" + + // WIDTH=1280 + "0500" + + // HEIGHT=720 + "02d0" + + // N_G=4 + "04" + + // TID=0,U=0,R=1 + "04" + + // P_DIFF=4 + "04" + + // TID=2,U=1,R=1 + "54" + + // P_DIFF=1 + "01" + + // TID=1,U=1,R=1 + "34" + + // P_DIFF=2 + "02" + + // TID=2,U=0,R=2 + "48" + + // P_DIFF=1 + "01" + + // P_DIFF=2 + "02" + + // VP9 media. Truncated. + "83498342004ff02cf098e0907064a0600f005fcb", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 25918, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 0, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = 91, + descriptorSize = 20, + scalabilityStructure = RtpEncodingDesc( + 0xa4d04528L, + arrayOf( + VpxRtpLayerDesc(0, 0, 0, 720, 7.5), + VpxRtpLayerDesc(0, 1, 0, 720, 15.0), + VpxRtpLayerDesc(0, 2, 0, 720, 30.0) + ) + ) + ), + + /* Live video - Chrome Version 91.0.4455.2 (Official Build) dev (x86_64) tab capture */ + SampleVp9Packet( + "Beginning of Chrome 91 tab capture non-key frame flexible mode with 1 reference", + "90654720f16dcb43c426ccf7" + + "bede0002324aac08510b2d00" + + // I=1,P=1,L=1,F=1,B=1,E=0,V=0,Z=0 + "f8" + + // M=1,PID=0x6c57=27735 + "ec57" + + // TID=0,U=0,SID=0,D=0 + "00" + + // P_DIFF=1,N=0 + "02" + + // VP9 media. Truncated. + "870100093f1c120e0cd0198080e030180f824f01", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = false, + isInterPicturePredicted = true, + pictureId = 27735, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 0, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 5, + scalabilityStructure = null + ), + + /* Mobile chrome, portrait-mode */ + SampleVp9Packet( + "Temporally-scalable portrait mode", + "90e553b379aa95ae6012c1c8" + + "1000001d" + + "030324e813" + + "05021b57" + + "0b5cc007e1c0081485214eaaaaa8000600004000100002aa80a8000600004000100002a000a800060000400016d549241b" + + "5524906d54923157e001974ca864330e222396eca8655304224390eca8775300b3013f0167027f02cf04ff0380" + + "0c0a01800f141c00b3013f0b" + + "00" + + // I=1,P=0,L=1,F=1,B=1,E=1,V=1,Z=0 + "be" + + // M=1, PID=0x61d6=20950 + "d1d6" + + // TID=0,U=1,SID=0,D=0 + "10" + + // Begin SS: N_S=0,Y=1,G=0 + "10" + + // WIDTH=180 + "00b4" + + // HEIGHT=320 + "0140" + + // VP9 media. Truncated. + "83498342000b3013f8167827", + isStartOfFrame = true, + isEndOfFrame = true, + isEndOfPicture = true, + isKeyframe = true, + isInterPicturePredicted = false, + pictureId = 20950, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 0, + isSwitchingUpPoint = true, + usesInterLayerDependency = false, + tL0PICIDX = null, + descriptorSize = 9, + scalabilityStructure = RtpEncodingDesc( + 0x6012c1c8, + arrayOf( + VpxRtpLayerDesc(0, 0, 0, 180, -1.0) + ) + ) + ) + ) + + /* + Template: + , + SampleVp9Packet("Description", + "", + isStartOfFrame = true, + isEndOfFrame = false, + isEndOfPicture = false, + isKeyframe = false, + pictureId = 7781, + hasExtendedPictureId = true, + isUpperLevelReference = true, + tid = 0, + sid = 1, + isSwitchingUpPoint = false, + usesInterLayerDependency = true, + tL0PICIDX = 253, + descriptorSize = 5) + */ + + init { + context("VP9 packets") { + should("be parsed correctly") { + for (t in testPackets) { + withClue(t.description) { + val p = Vp9Packet(t.data, 0, t.data.size) + p.isStartOfFrame shouldBe t.isStartOfFrame + p.isEndOfFrame shouldBe t.isEndOfFrame + p.isEndOfPicture shouldBe t.isEndOfPicture + p.isKeyframe shouldBe t.isKeyframe + p.isInterPicturePredicted shouldBe t.isInterPicturePredicted + if (t.pictureId != null) { + p.hasPictureId shouldBe true + p.pictureId shouldBe t.pictureId + } else { + p.hasPictureId shouldBe false + } + p.hasExtendedPictureId shouldBe t.hasExtendedPictureId + p.isUpperLevelReference shouldBe t.isUpperLevelReference + if (t.tid != null) { + p.hasLayerIndices shouldBe true + p.temporalLayerIndex shouldBe t.tid + } else { + p.hasLayerIndices shouldBe false + } + if (t.sid != null) { + p.hasLayerIndices shouldBe true + p.spatialLayerIndex shouldBe t.sid + } else { + p.hasLayerIndices shouldBe false + } + if (t.tL0PICIDX != null) { + p.hasTL0PICIDX shouldBe true + p.TL0PICIDX shouldBe t.tL0PICIDX + } else { + p.hasTL0PICIDX shouldBe false + } + p.isSwitchingUpPoint shouldBe t.isSwitchingUpPoint + p.usesInterLayerDependency shouldBe t.usesInterLayerDependency + if (t.scalabilityStructure != null) { + val tss = t.scalabilityStructure + p.hasScalabilityStructure shouldBe true + val ss = p.getScalabilityStructure() + ss shouldNotBe null + ss!!.primarySSRC shouldBe tss.primarySSRC + ss.layers.size shouldBe tss.layers.size + for ((index, layer) in ss.layers.withIndex()) { + val tLayer = tss.layers[index] + layer.layerId shouldBe tLayer.layerId + layer.index shouldBe tLayer.index + layer should beInstanceOf() + layer as VpxRtpLayerDesc + tLayer as VpxRtpLayerDesc + layer.sid shouldBe tLayer.sid + layer.tid shouldBe tLayer.tid + layer.height shouldBe tLayer.height + layer.frameRate shouldBe tLayer.frameRate + /* TODO: dependency layers */ + } + } else { + p.hasScalabilityStructure shouldBe false + p.getScalabilityStructure() shouldBe null + } + + val descSz = + DePacketizer.VP9PayloadDescriptor.getSize(p.buffer, p.payloadOffset, p.payloadLength) + descSz shouldBe t.descriptorSize + } + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/DelayStatsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/DelayStatsTest.kt new file mode 100644 index 0000000..23a2194 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/DelayStatsTest.kt @@ -0,0 +1,89 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.stats + +import com.fasterxml.jackson.databind.node.ObjectNode +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf +import java.lang.IllegalArgumentException + +class DelayStatsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("adding stats") { + val delayStats = DelayStats(listOf(0, 2, 5, 200, 999, Long.MAX_VALUE)) + + should("calculate the average correctly") { + repeat(100) { delayStats.addDelay(1) } + repeat(100) { delayStats.addDelay(5) } + delayStats.snapshot.average shouldBe 3.0 + } + + should("calculate the max correctly") { + repeat(100) { delayStats.addDelay(1) } + repeat(100) { delayStats.addDelay(5) } + delayStats.addDelay(100) + delayStats.snapshot.maxValue shouldBe 100 + } + + should("export the buckets correctly to json") { + repeat(100) { delayStats.addDelay(1) } + repeat(50) { delayStats.addDelay(2) } + repeat(50) { delayStats.addDelay(4) } + delayStats.addDelay(150) + delayStats.addDelay(1500) + val bucketsJson = delayStats.toJson()["buckets"] + bucketsJson.shouldBeInstanceOf() + + bucketsJson["0_to_2_ms"]?.longValue() shouldBe 100 + bucketsJson["2_to_5_ms"]?.longValue() shouldBe 100 + bucketsJson["5_to_200_ms"]?.longValue() shouldBe 1 + bucketsJson["200_to_999_ms"]?.longValue() shouldBe 0 + bucketsJson["999_to_max_ms"]?.longValue() shouldBe 1 + } + + should("calculate p99 and p999 correctly") { + delayStats.snapshot.buckets.p99bound shouldBe -1 + delayStats.snapshot.buckets.p999bound shouldBe -1 + + repeat(200) { delayStats.addDelay(1) } + delayStats.snapshot.buckets.p99bound shouldBe 2 + delayStats.snapshot.buckets.p999bound shouldBe -1 + + repeat(800) { delayStats.addDelay(1) } + delayStats.snapshot.buckets.p999bound shouldBe 2 + + delayStats.addDelay(4) + delayStats.addDelay(4) + delayStats.snapshot.buckets.p99bound shouldBe 2 + delayStats.snapshot.buckets.p999bound shouldBe 5 + + repeat(2000) { delayStats.addDelay(1) } + delayStats.snapshot.buckets.p999bound shouldBe 2 + } + } + + context("initializing with invalid thresholds should throw") { + shouldThrow { + DelayStats(listOf(2, 10, 5)) + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt new file mode 100644 index 0000000..cf498c2 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/EndpointConnectionStatsTest.kt @@ -0,0 +1,158 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.test_utils.timeline +import org.jitsi.rtp.rtcp.RtcpReportBlock +import org.jitsi.rtp.rtcp.RtcpRrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.utils.ms +import org.jitsi.utils.time.FakeClock +import java.time.Duration + +class EndpointConnectionStatsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val clock: FakeClock = FakeClock() + + private var mostRecentPublishedRtt: Double = -1.0 + private var numRttUpdates: Int = 0 + private val stats = EndpointConnectionStats(StdoutLogger(), clock).also { + it.addListener(object : EndpointConnectionStats.EndpointConnectionStatsListener { + override fun onRttUpdate(newRttMs: Double) { + mostRecentPublishedRtt = newRttMs + numRttUpdates++ + } + }) + } + + init { + context("after an SR is sent") { + context("with a non-zero timestamp") { + val srPacket = createSrPacket(senderInfoTs = 100) + stats.rtcpPacketSent(srPacket) + + context("and an RR is received which refers to it") { + clock.elapse(60.ms) + + val rrPacket = createRrPacket(lastSrTimestamp = 100, delaySinceLastSr = 50.ms) + + stats.rtcpPacketReceived(rrPacket, clock.instant()) + context("the rtt") { + should("be updated correctly") { + mostRecentPublishedRtt shouldBe (10.0 plusOrMinus .1) + } + } + } + } + context("with a timestamp of 0") { + val srPacket = createSrPacket(senderInfoTs = 0) + stats.rtcpPacketSent(srPacket) + + context("and an RR is received which refers to it") { + context("with a non-zero delaySinceLastSr") { + val rrPacket = createRrPacket(lastSrTimestamp = 0, delaySinceLastSr = 50.ms) + clock.elapse(60.ms) + + stats.rtcpPacketReceived(rrPacket, clock.instant()) + context("the rtt") { + should("be updated correctly") { + mostRecentPublishedRtt shouldBe (10.0.plusOrMinus(.1)) + } + } + } + context("with a delaySinceLastSr value of 0") { + val rrPacket = createRrPacket(lastSrTimestamp = 0, delaySinceLastSr = 0.ms) + + clock.elapse(60.ms) + + stats.rtcpPacketReceived(rrPacket, clock.instant()) + + // This case is indistinguishable from no SR being received + context("the rtt") { + should("not have been updated") { + numRttUpdates shouldBe 0 + } + } + } + } + } + } + context("when a report block is received for which we can't find an SR") { + val rrPacket = createRrPacket(lastSrTimestamp = 100, delaySinceLastSr = 50.ms) + + stats.rtcpPacketReceived(rrPacket, clock.instant()) + + context("the rtt") { + should("not have been updated") { + numRttUpdates shouldBe 0 + } + } + } + context("when a report block is received with a lastSrTimestamp of 0 but a non-zero delaySinceLastSr") { + timeline(clock) { + run { stats.rtcpPacketSent(createSrPacket(senderInfoTs = 0)) } + elapse(60.ms) + run { + stats.rtcpPacketReceived( + createRrPacket(lastSrTimestamp = 0, delaySinceLastSr = 50.ms), + clock.instant() + ) + } + }.run() + + context("the rtt") { + should("be updated correctly") { + mostRecentPublishedRtt shouldBe (10.0.plusOrMinus(.1)) + } + } + } + } + + // Takes a Duration and converts it to 1/65536ths of a second + private fun Duration.toDelaySinceLastSrFraction(): Long = (toNanos() * .000065536).toLong() + + private fun createSrPacket(senderInfoTs: Long): RtcpSrPacket { + val mockSenderInfo = mockk { + every { compactedNtpTimestamp } returns senderInfoTs + } + + return mockk { + every { senderSsrc } returns 1234L + every { senderInfo } returns mockSenderInfo + } + } + + private fun createRrPacket(lastSrTimestamp: Long, delaySinceLastSr: Duration): RtcpRrPacket { + val reportBlock = mockk { + every { ssrc } returns 1234L + every { this@mockk.lastSrTimestamp } returns lastSrTimestamp + every { this@mockk.delaySinceLastSr } returns delaySinceLastSr.toDelaySinceLastSrFraction() + } + + return mockk { + every { reportBlocks } returns listOf(reportBlock) + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/JitterStatsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/JitterStatsTest.kt new file mode 100644 index 0000000..e9ffe19 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/JitterStatsTest.kt @@ -0,0 +1,94 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import io.kotest.assertions.withClue +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.rtp.rtp.RtpPacket +import java.time.Instant + +private data class StatPacketInfo( + val packetInfo: PacketInfo, + val sentTime: Instant +) + +private data class JitterPacketInfo( + val statPacketInfo: StatPacketInfo, + val expectedJitter: Double +) + +private fun createStatPacketInfo(seqNum: Int, sentTime: Long, receivedTime: Long): StatPacketInfo { + val packetInfo = PacketInfo(RtpPacket(ByteArray(50), 0, 50)) + packetInfo.packetAs().sequenceNumber = seqNum + packetInfo.receivedTime = Instant.ofEpochMilli(receivedTime) + return StatPacketInfo(packetInfo, Instant.ofEpochMilli(sentTime)) +} + +private fun createJitterPacketInfo( + seqNum: Int, + sentTime: Long, + receivedTime: Long, + expectedJitter: Double +): JitterPacketInfo { + val statPacketInfo = createStatPacketInfo(seqNum, sentTime, receivedTime) + return JitterPacketInfo(statPacketInfo, expectedJitter) +} + +// Values taken from this example: http://toncar.cz/Tutorials/VoIP/VoIP_Basics_Jitter.html +private val JitterPacketInfos = listOf( + createJitterPacketInfo(0, 0, 10, 0.0), + createJitterPacketInfo(1, 20, 30, 0.0), + createJitterPacketInfo(2, 40, 49, .0625), + createJitterPacketInfo(3, 60, 74, .3711), + createJitterPacketInfo(4, 80, 90, .5979), + createJitterPacketInfo(5, 100, 111, .6230), + createJitterPacketInfo(6, 120, 139, 1.0841), + createJitterPacketInfo(7, 140, 150, 1.5788), + createJitterPacketInfo(8, 160, 170, 1.4802), + createJitterPacketInfo(9, 180, 191, 1.4501), + createJitterPacketInfo(10, 200, 210, 1.4220), + createJitterPacketInfo(11, 220, 229, 1.3956), + createJitterPacketInfo(12, 240, 250, 1.3709), + createJitterPacketInfo(13, 260, 271, 1.3477) +) + +class JitterStatsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("Jitter") { + val jitterStats = JitterStats() + should("update correctly") { + // Handled separately since we want to verify the jitter at each + // step along the way + JitterPacketInfos.forEachIndexed { index, jitterPacketInfo -> + jitterStats.addPacket( + jitterPacketInfo.statPacketInfo.sentTime, + jitterPacketInfo.statPacketInfo.packetInfo.receivedTime!! + ) + withClue("After adding packet $index") { + jitterStats.jitter shouldBe (jitterPacketInfo.expectedJitter plusOrMinus .001) + } + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/PacketIOActivityTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/PacketIOActivityTest.kt new file mode 100644 index 0000000..c191ae9 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/stats/PacketIOActivityTest.kt @@ -0,0 +1,81 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.stats + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import java.time.Duration + +class PacketIOActivityTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val packetIoActivity = PacketIOActivity() + private val clock: FakeClock = FakeClock() + + init { + context("Last packet time values") { + clock.elapse(Duration.ofMinutes(1)) + val oldTime = clock.instant() + clock.elapse(Duration.ofMinutes(1)) + val newTime = clock.instant() + packetIoActivity.lastRtpPacketSentInstant = newTime + packetIoActivity.lastRtpPacketReceivedInstant = newTime + packetIoActivity.lastIceActivityInstant = newTime + context("when setting an older time") { + packetIoActivity.lastRtpPacketSentInstant = oldTime + packetIoActivity.lastRtpPacketReceivedInstant = oldTime + packetIoActivity.lastIceActivityInstant = oldTime + should("not allow going backwards") { + packetIoActivity.lastRtpPacketSentInstant shouldBe newTime + packetIoActivity.lastRtpPacketReceivedInstant shouldBe newTime + packetIoActivity.lastIceActivityInstant shouldBe newTime + } + } + } + context("lastOverallRtpActivity") { + should("only reflect RTP packet time values") { + clock.elapse(30.secs) + val rtpSentTime = clock.instant() + clock.elapse(5.secs) + val rtpReceivedTime = clock.instant() + clock.elapse(10.secs) + val iceTime = clock.instant() + packetIoActivity.lastRtpPacketSentInstant = rtpSentTime + packetIoActivity.lastRtpPacketReceivedInstant = rtpReceivedTime + packetIoActivity.lastIceActivityInstant = iceTime + packetIoActivity.lastRtpActivityInstant shouldBe rtpReceivedTime + } + } + context("lastOverallActivity") { + should("only reflect all packet time values") { + clock.elapse(30.secs) + val rtpSentTime = clock.instant() + clock.elapse(10.secs) + val iceTime = clock.instant() + clock.elapse(5.secs) + val rtpReceivedTime = clock.instant() + packetIoActivity.lastRtpPacketSentInstant = rtpSentTime + packetIoActivity.lastRtpPacketReceivedInstant = rtpReceivedTime + packetIoActivity.lastIceActivityInstant = iceTime + packetIoActivity.lastRtpActivityInstant shouldBe rtpReceivedTime + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/RtpPacketGenerator.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/RtpPacketGenerator.kt new file mode 100644 index 0000000..9707157 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/RtpPacketGenerator.kt @@ -0,0 +1,132 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.test_utils + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.DataSize +import org.jitsi.nlj.util.atRate +import org.jitsi.nlj.util.howMuchCanISendAtRate +import org.jitsi.nlj.util.`in` +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.div +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import java.time.Duration +import java.time.Instant + +class RtpPacketGenerator internal constructor( + /** + * Length of each packet. + */ + val length: DataSize, + /** + * The number of packets to generate. + */ + val count: Int, + /** + * The interval to advance the clock after each packet. + */ + val interval: Duration, + val clock: FakeClock = FakeClock() +) { + + internal constructor( + /** + * The bitrate to generate packets at. + */ + targetBitrate: Bandwidth, + /** + * Length of each packet. + */ + length: DataSize, + /** + * The total duration over which packets will be generated. + */ + duration: Duration = 10.secs, + clock: FakeClock = FakeClock() + ) : this( + length = length, + count = (duration / length.atRate(targetBitrate)).toInt(), + interval = length.atRate(targetBitrate), + clock = clock + ) + + internal constructor( + /** + * The bitrate to generate packets at. + */ + targetBitrate: Bandwidth, + /** + * The interval to advance the clock after each packet. + */ + interval: Duration = Duration.ofMillis(10), + /** + * The total duration over which packets will be generated. + */ + duration: Duration = 10.secs, + clock: FakeClock = FakeClock() + ) : this( + length = howMuchCanISendAtRate(targetBitrate).`in`(interval), + count = (duration / interval).toInt(), + interval = interval, + clock = clock + ) + + private fun createPacket(seq: Int, len: Int, packetSsrc: Long, pt: Int, receivedTime: Instant): PacketInfo { + val dummyPacket = RtpPacket(ByteArray(len), 0, len).apply { + version = 2 + hasPadding = false + hasExtensions = false + isMarked = false + payloadType = pt + sequenceNumber = seq + timestamp = 456L + ssrc = packetSsrc + } + return PacketInfo(dummyPacket).apply { this.receivedTime = receivedTime } + } + + /** + * @param startSeq the sequence number of the first packet. Subsequent packets' sequence numbers increment. + * @param ssrc the SSRC. + * @param payloadType the payload type number. + * @param processPacket the function to invoke for each packet. + */ + fun generatePackets( + startSeq: Int = 0, + ssrc: Long = 0, + payloadType: Int = 100, + processPacket: (PacketInfo) -> Unit + ) { + timeline(clock) { + repeat(count) { + run { + processPacket( + createPacket( + seq = startSeq + it, + len = length.bytes.toInt(), + packetSsrc = ssrc, + pt = payloadType, + receivedTime = clock.instant() + ) + ) + } + elapse(interval) + } + }.run() + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt new file mode 100644 index 0000000..4f7a07e --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/TimelineTest.kt @@ -0,0 +1,43 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.test_utils + +import org.jitsi.utils.time.FakeClock +import java.time.Duration +import java.time.Instant + +internal class TimelineTest(private val clock: FakeClock) { + private val tasks = mutableListOf<() -> Unit>() + + fun time(time: Int) { + tasks.add { clock.setTime(Instant.ofEpochMilli(time.toLong())) } + } + + fun run(block: () -> Unit) { + tasks.add(block) + } + + fun elapse(duration: Duration) { + tasks.add { clock.elapse(duration) } + } + + fun run() { + tasks.forEach { it.invoke() } + } +} + +internal fun timeline(clock: FakeClock, block: TimelineTest.() -> Unit): TimelineTest = TimelineTest(clock).apply(block) diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteArray.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteArray.kt new file mode 100644 index 0000000..10b13e4 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteArray.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.test_utils.matchers + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.extensions.bytearray.toHex + +fun haveSameContentAs(expected: ByteArray) = object : Matcher { + override fun test(value: ByteArray): MatcherResult { + var matches = true + if (expected.size != value.size) { + matches = false + } else { + for (i in (0 until expected.size)) { + if (value.get(i) != expected.get(i)) { + matches = false + break + } + } + } + return MatcherResult( + matches, + { "\n${value.toHex()}\nwas supposed to be:\n${expected.toHex()}" }, + { "\n${value.toHex()}\nshould not have equaled \n${expected.toHex()}" } + ) + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteArrayBuffer/ByteArrayBuffer.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteArrayBuffer/ByteArrayBuffer.kt new file mode 100644 index 0000000..45b7da9 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteArrayBuffer/ByteArrayBuffer.kt @@ -0,0 +1,43 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.test_utils.matchers.ByteArrayBuffer + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.extensions.toHex +import org.jitsi.utils.ByteArrayBuffer + +fun haveSameContentAs(expected: ByteArrayBuffer) = object : Matcher { + override fun test(value: ByteArrayBuffer): MatcherResult { + var matches = true + if (expected.length != value.length) { + matches = false + } else { + for (i in (0 until expected.length)) { + if (value.buffer[value.offset + i] != expected.buffer[expected.offset + i]) { + matches = false + break + } + } + } + return MatcherResult( + matches, + { "\n${value.toHex()}\nwas supposed to be:\n${expected.toHex()}" }, + { "\n${value.toHex()}\nshould not have equaled \n${expected.toHex()}" } + ) + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteBuffer.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteBuffer.kt new file mode 100644 index 0000000..e7fa47d --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/test_utils/matchers/ByteBuffer.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.test_utils.matchers + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.extensions.compareToFromBeginning +import org.jitsi.rtp.extensions.toHex +import java.nio.ByteBuffer + +fun haveSameContentAs(expected: ByteBuffer) = object : Matcher { + override fun test(value: ByteBuffer): MatcherResult = MatcherResult( + value.compareToFromBeginning(expected) == 0, + { "Buffer\n${value.toHex()}\nwas supposed to be:\n${expected.toHex()}" }, + { "Buffer\n${value.toHex()}\nshould not have equaled buffer\n${expected.toHex()}" } + ) +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/NodeTeardownVisitorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/NodeTeardownVisitorTest.kt new file mode 100644 index 0000000..145f058 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/NodeTeardownVisitorTest.kt @@ -0,0 +1,80 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.util.PacketPredicate + +internal class NodeTeardownVisitorTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val testIncomingPipeline = pipeline { + simpleNode("Node 1") { pkts -> pkts } + demux("Node 2") { + packetPath { + name = "Node 2 path 1" + predicate = PacketPredicate { true } + path = pipeline { + simpleNode("Node 2 path 1 Node 1") { pkts -> pkts } + simpleNode("Node 2 path 1 Node 2") { pkts -> pkts } + } + } + packetPath { + name = "Node 2 path 2" + predicate = PacketPredicate { true } + path = pipeline { + simpleNode("Node 2 path 2 Node 1") { pkts -> pkts } + simpleNode("Node 2 path 2 Node 2") { pkts -> pkts } + } + } + } + } + + // 'Outgoing' style is harder to define: we actually need multiple separate pipeline that + // terminate at the same node + private val testOutgoingPipelineTermination = object : ConsumerNode("Output termination") { + override fun consume(packetInfo: PacketInfo) {} + + override fun trace(f: () -> Unit) = f.invoke() + } + + private val testOutgoingPipeline1 = pipeline { + simpleNode("OutgoingPipeline 1 Node 1") { pkts -> pkts } + simpleNode("OutgoingPipeline 1 Node 2") { pkts -> pkts } + simpleNode("OutgoingPipeline 1 Node 3") { pkts -> pkts } + node(testOutgoingPipelineTermination) + } + + private val testOutgoingPipeline2 = pipeline { + simpleNode("OutgoingPipeline 2 Node 1") { pkts -> pkts } + simpleNode("OutgoingPipeline 2 Node 2") { pkts -> pkts } + simpleNode("OutgoingPipeline 2 Node 3") { pkts -> pkts } + node(testOutgoingPipelineTermination) + } + + init { + context("tearing down an 'incoming-style' pipeline") { + NodeTeardownVisitor().visit(testIncomingPipeline) + } + context("tearing down an 'outgoing-style' pipeline") { + NodeTeardownVisitor().reverseVisit(testOutgoingPipelineTermination) + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/NodeVisitorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/NodeVisitorTest.kt new file mode 100644 index 0000000..c43331a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/NodeVisitorTest.kt @@ -0,0 +1,115 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.Node +import org.jitsi.nlj.util.PacketPredicate + +internal class NodeVisitorTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val testIncomingPipeline = pipeline { + simpleNode("Node 1") { pkts -> pkts } + demux("Node 2") { + packetPath { + name = "Node 2 path 1" + predicate = PacketPredicate { true } + path = pipeline { + simpleNode("Node 2 path 1 Node 1") { pkts -> pkts } + simpleNode("Node 2 path 1 Node 2") { pkts -> pkts } + } + } + packetPath { + name = "Node 2 path 2" + predicate = PacketPredicate { true } + path = pipeline { + simpleNode("Node 2 path 2 Node 1") { pkts -> pkts } + simpleNode("Node 2 path 2 Node 2") { pkts -> pkts } + } + } + } + } + + // 'Outgoing' style is harder to define: we actually need multiple separate pipeline that + // terminate at the same node + private val testOutgoingPipelineTermination = object : ConsumerNode("Output termination") { + override fun consume(packetInfo: PacketInfo) {} + + override fun trace(f: () -> Unit) = f.invoke() + } + + private val testOutgoingPipeline1 = pipeline { + simpleNode("OutgoingPipeline 1 Node 1") { pkts -> pkts } + simpleNode("OutgoingPipeline 1 Node 2") { pkts -> pkts } + simpleNode("OutgoingPipeline 1 Node 3") { pkts -> pkts } + node(testOutgoingPipelineTermination) + } + + private val testOutgoingPipeline2 = pipeline { + simpleNode("OutgoingPipeline 2 Node 1") { pkts -> pkts } + simpleNode("OutgoingPipeline 2 Node 2") { pkts -> pkts } + simpleNode("OutgoingPipeline 2 Node 3") { pkts -> pkts } + node(testOutgoingPipelineTermination) + } + + private class TestVisitor : NodeVisitor() { + val nodeNames = mutableListOf() + override fun doWork(node: Node) { + nodeNames += node.name + } + } + + init { + context("visiting an 'incoming-style' pipeline") { + val testVisitor = TestVisitor() + testVisitor.visit(testIncomingPipeline) + should("visit every node") { + testVisitor.nodeNames shouldHaveSize 6 + } + should("visit the nodes in the proper order") { + testVisitor.nodeNames[0] shouldBe "Node 1" + testVisitor.nodeNames[1] shouldBe "Node 2 demuxer" + testVisitor.nodeNames[2] shouldBe "Node 2 path 1 Node 1" + testVisitor.nodeNames[3] shouldBe "Node 2 path 1 Node 2" + testVisitor.nodeNames[4] shouldBe "Node 2 path 2 Node 1" + testVisitor.nodeNames[5] shouldBe "Node 2 path 2 Node 2" + } + } + context("visiting an 'outgoing-style' pipeline") { + val testVisitor = TestVisitor() + testVisitor.reverseVisit(testOutgoingPipelineTermination) + should("visit every node") { + testVisitor.nodeNames shouldHaveSize 7 + } + should("visit the nodes in the proper order") { + testVisitor.nodeNames[0] shouldBe "OutgoingPipeline 1 Node 1" + testVisitor.nodeNames[1] shouldBe "OutgoingPipeline 1 Node 2" + testVisitor.nodeNames[2] shouldBe "OutgoingPipeline 1 Node 3" + testVisitor.nodeNames[3] shouldBe "OutgoingPipeline 2 Node 1" + testVisitor.nodeNames[4] shouldBe "OutgoingPipeline 2 Node 2" + testVisitor.nodeNames[5] shouldBe "OutgoingPipeline 2 Node 3" + testVisitor.nodeNames[6] shouldBe "Output termination" + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/ExclusivePathDemuxerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/ExclusivePathDemuxerTest.kt new file mode 100644 index 0000000..3ca38a5 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/ExclusivePathDemuxerTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.Spec +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.util.PacketPredicate +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtp.RtpPacket + +internal class ExclusivePathDemuxerTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private class DummyHandler(name: String) : ConsumerNode(name) { + var numReceived = 0 + override fun consume(packetInfo: PacketInfo) { + numReceived++ + } + + override fun trace(f: () -> Unit) = f.invoke() + } + + private class DummyRtcpPacket : RtcpPacket(ByteArray(50), 0, 50) { + override fun clone(): DummyRtcpPacket = DummyRtcpPacket() + } + + private val rtpPath = ConditionalPacketPath() + private val rtpHandler = DummyHandler("RTP") + private val rtcpPath = ConditionalPacketPath() + private val rtcpHandler = DummyHandler("RTCP") + + private val demuxer = ExclusivePathDemuxer("test") + + private val rtpPacket = PacketInfo(RtpPacket(ByteArray(50), 0, 50)) + private val rtcpPacket = PacketInfo(DummyRtcpPacket()) + + override suspend fun beforeSpec(spec: Spec) { + super.beforeSpec(spec) + rtpPath.name = "RTP" + rtpPath.predicate = PacketPredicate { it is RtpPacket } + rtpPath.path = rtpHandler + + rtcpPath.name = "RTCP" + rtcpPath.predicate = PacketPredicate { it is RtcpPacket } + rtcpPath.path = rtcpHandler + + demuxer.addPacketPath(rtpPath) + demuxer.addPacketPath(rtcpPath) + } + + init { + context("a packet which matches only one predicate") { + demuxer.processPacket(rtcpPacket) + should("only be demuxed to one path") { + rtpHandler.numReceived shouldBe 0 + rtcpHandler.numReceived shouldBe 1 + } + } + context("a packet which matches more than one predicate") { + val rtpPath2 = ConditionalPacketPath() + val handler = DummyHandler("RTP 2") + rtpPath2.name = "RTP 2" + rtpPath2.predicate = PacketPredicate { it is RtpPacket } + rtpPath2.path = handler + demuxer.addPacketPath(rtpPath2) + demuxer.processPacket(rtpPacket) + should("only be demuxed to one path") { + rtpHandler.numReceived shouldBe 1 + rtcpHandler.numReceived shouldBe 0 + handler.numReceived shouldBe 0 + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingSsrcStatsTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingSsrcStatsTest.kt new file mode 100644 index 0000000..4f32a62 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/IncomingSsrcStatsTest.kt @@ -0,0 +1,166 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.incoming + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.MediaType +import java.time.Instant + +private data class StatPacketInfo( + val packetInfo: PacketInfo, + val sentTime: Instant +) + +private data class JitterPacketInfo( + val statPacketInfo: StatPacketInfo, + val expectedJitter: Double +) + +private fun createStatPacketInfo(seqNum: Int, sentTime: Long, receivedTime: Long): StatPacketInfo { + val packetInfo = PacketInfo(RtpPacket(ByteArray(50), 0, 50)) + packetInfo.packetAs().sequenceNumber = seqNum + packetInfo.receivedTime = Instant.ofEpochMilli(receivedTime) + return StatPacketInfo(packetInfo, Instant.ofEpochMilli(sentTime)) +} + +private fun createJitterPacketInfo( + seqNum: Int, + sentTime: Long, + receivedTime: Long, + expectedJitter: Double +): JitterPacketInfo { + val statPacketInfo = createStatPacketInfo(seqNum, sentTime, receivedTime) + return JitterPacketInfo(statPacketInfo, expectedJitter) +} + +// Values taken from this example: http://toncar.cz/Tutorials/VoIP/VoIP_Basics_Jitter.html +private val initialJitterPacketInfo = createJitterPacketInfo(0, 0, 10, 0.0) +private val JitterPacketInfos = listOf( + createJitterPacketInfo(1, 20, 30, 0.0), + createJitterPacketInfo(2, 40, 49, .0625), + createJitterPacketInfo(3, 60, 74, .3711), + createJitterPacketInfo(4, 80, 90, .5979), + createJitterPacketInfo(5, 100, 111, .6230), + createJitterPacketInfo(6, 120, 139, 1.0841), + createJitterPacketInfo(7, 140, 150, 1.5788), + createJitterPacketInfo(8, 160, 170, 1.4802), + createJitterPacketInfo(9, 180, 191, 1.4501), + createJitterPacketInfo(10, 200, 210, 1.4220), + createJitterPacketInfo(11, 220, 229, 1.3956), + createJitterPacketInfo(12, 240, 250, 1.3709), + createJitterPacketInfo(13, 260, 271, 1.3477) +) + +internal class IncomingSsrcStatsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("Expected packet count") { + should("Handle cases with no rollover") { + IncomingSsrcStats.calculateExpectedPacketCount( + 0, + 1, + 0, + 10 + ) shouldBe 10 + } + should("handle cases with 1 rollover") { + IncomingSsrcStats.calculateExpectedPacketCount( + 0, + 65530, + 1, + 10 + ) shouldBe 17 + } + should("handle cases with > 1 rollover") { + // Same as the scenario above but with another rollover + IncomingSsrcStats.calculateExpectedPacketCount( + 0, + 65530, + 2, + 10 + ) shouldBe 17 + 65536L + } + } + context("When receiving a series of packets with loss") { + // 17 packets between base and max sequence number, 6 packets lost + val packetSequence = listOf( + createStatPacketInfo(0, 0, 0), + createStatPacketInfo(1, 0, 0), + createStatPacketInfo(3, 0, 0), + createStatPacketInfo(5, 0, 0), + createStatPacketInfo(6, 0, 0), + createStatPacketInfo(7, 0, 0), + createStatPacketInfo(8, 0, 0), + createStatPacketInfo(9, 0, 0), + createStatPacketInfo(11, 0, 0), + createStatPacketInfo(13, 0, 0), + createStatPacketInfo(16, 0, 0) + ) + val streamStatistics = IncomingSsrcStats( + 123L, + packetSequence.first().packetInfo.packetAs().sequenceNumber, + mediaType = MediaType.VIDEO + ) + packetSequence.forEach { + streamStatistics.packetReceived( + it.packetInfo.packetAs(), + it.sentTime, + it.packetInfo.receivedTime!! + ) + } + val statSnapshot = streamStatistics.getSnapshotIfActive() + context("expected packets") { + should("be calculated correctly") { + statSnapshot?.numExpectedPackets shouldBe 17 + } + } + context("cumulative lost") { + should("be calculated correctly") { + statSnapshot?.cumulativePacketsLost shouldBe 6 + } + } + context("querying a second time with no update") { + should("be null") { + streamStatistics.getSnapshotIfActive() shouldBe null + } + } + + val packetSequence2 = listOf( + createStatPacketInfo(17, 0, 0) + ) + packetSequence2.forEach { + streamStatistics.packetReceived( + it.packetInfo.packetAs(), + it.sentTime, + it.packetInfo.receivedTime!! + ) + } + + context("querying after a new update") { + should("not be null") { + streamStatistics.getSnapshotIfActive() shouldNotBe null + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt new file mode 100644 index 0000000..dea4124 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RemoteBandwidthEstimatorTest.kt @@ -0,0 +1,105 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.transform.node.incoming + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.format.Vp8PayloadType +import org.jitsi.nlj.format.Vp9PayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.test_utils.RtpPacketGenerator +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.nlj.util.mbps +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.AbsSendTimeHeaderExtension +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import java.time.Duration + +class RemoteBandwidthEstimatorTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val clock: FakeClock = FakeClock() + private val astExtensionId = 3 + + // REMB is enabled by having at least one payload type which has "goog-remb" signaled as a rtcp-fb, and TCC is + // disabled. + private val vp8PayloadType = Vp8PayloadType(100, emptyMap(), setOf("goog-remb")) + private val vp9PayloadTypeWithTcc = Vp9PayloadType(101, emptyMap(), setOf("transport-cc")) + private val ssrc = 1234L + private val streamInformationStore = StreamInformationStoreImpl().apply { + addRtpExtensionMapping(RtpExtension(astExtensionId.toByte(), RtpExtensionType.ABS_SEND_TIME)) + addRtpPayloadType(vp8PayloadType) + } + + private val remoteBandwidthEstimator = + RemoteBandwidthEstimator(streamInformationStore, StdoutLogger(), clock = clock) + + init { + context("when REMB is not signaled") { + streamInformationStore.clearRtpPayloadTypes() + streamInformationStore.addRtpPayloadType(vp9PayloadTypeWithTcc) + sendPackets() + context("no feedback should be produced") { + remoteBandwidthEstimator.createRemb() shouldBe null + } + } + context("when both REMB and TCC are signaled") { + streamInformationStore.addRtpPayloadType(vp9PayloadTypeWithTcc) + sendPackets() + context("no feedback should be produced") { + remoteBandwidthEstimator.createRemb() shouldBe null + } + } + context("when REMB is signaled") { + sendPackets(0.5.mbps) + val rembPacket = remoteBandwidthEstimator.createRemb() + context("a feedback packet should be produced") { + rembPacket shouldNotBe null + context("with valid bitrate") { + rembPacket!!.bitrate shouldBeGreaterThan 0L + } + context("with the correct SSRCs") { + rembPacket!!.ssrcs shouldBe listOf(ssrc) + } + } + + // We generate packets with no jitter, so we expect the estimate to he higher than the receive bitrate. + val targetBitrate = 1.5.mbps + sendPackets(targetBitrate) + context("when receiving a higher bitrate, the estimate should grow") { + val rembPacket2 = remoteBandwidthEstimator.createRemb() + rembPacket2!!.bitrate shouldBeGreaterThan targetBitrate.bps + } + } + } + + private fun sendPackets(targetBitrate: Bandwidth = 1.mbps, duration: Duration = 15.secs) { + val rtpPacketGenerator = RtpPacketGenerator(targetBitrate, duration = duration, clock = clock) + rtpPacketGenerator.generatePackets(ssrc = ssrc) { + val ext = + it.packetAs().addHeaderExtension(astExtensionId, AbsSendTimeHeaderExtension.DATA_SIZE_BYTES) + AbsSendTimeHeaderExtension.setTime(ext, it.receivedTime!!.toEpochMilli() * 1_000_000) + remoteBandwidthEstimator.processPacket(it) + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTerminationTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTerminationTest.kt new file mode 100644 index 0000000..4754219 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/RtcpTerminationTest.kt @@ -0,0 +1,144 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.incoming + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.resources.node.onOutput +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.util.BufferPool +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtcp.RtcpHeader +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpReportBlock +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacketBuilder +import org.jitsi.rtp.rtcp.SenderInfoBuilder +import org.jitsi.rtp.rtcp.SenderInfoParser +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacketBuilder + +class RtcpTerminationTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val rtcpEventNotifier: RtcpEventNotifier = mockk() + private val rtcpTermination = RtcpTermination(rtcpEventNotifier, StdoutLogger()) + + val nackPacket = RtcpFbNackPacketBuilder().build() + + private var numRetrievedBuffers = 0 + private val returnedBuffers = mutableListOf() + private val outputPackets = mutableListOf() + + init { + BufferPool.getBuffer = { size -> + numRetrievedBuffers++ + ByteArray(size) + } + BufferPool.returnBuffer = { array -> + returnedBuffers.add(array) + } + rtcpTermination.onOutput { packetInfo -> + outputPackets.add(packetInfo) + } + // Force an access of RtcpSrPacket's report blocks, since it's lazy, to better + // simulate real conditions + every { rtcpEventNotifier.notifyRtcpReceived(any(), any()) } answers { + (args[0] as? RtcpSrPacket)?.reportBlocks + } + + context("Receiving an SR packet") { + val senderInfo = SenderInfoBuilder( + ntpTimestampMsw = 0xDEADBEEF, + ntpTimestampLsw = 0xBEEFDEAD, + rtpTimestamp = 12345L, + sendersPacketCount = 42, + sendersOctetCount = 4242 + ) + context("without any receiver report blocks") { + val srPacket = RtcpSrPacketBuilder( + RtcpHeaderBuilder( + senderSsrc = 12345L + ), + senderInfo + ).build() + val compoundRtcpPacket = createCompoundRtcp(srPacket) + + rtcpTermination.processPacket(PacketInfo(compoundRtcpPacket)) + + should("be forwarded intact") { + outputPackets shouldHaveSize 1 + with(outputPackets[0].packet as RtcpSrPacket) { + length shouldBe (RtcpHeader.SIZE_BYTES + SenderInfoParser.SIZE_BYTES) + reportCount shouldBe 0 + reportBlocks.size shouldBe 0 + returnedBuffers shouldHaveSize 0 + } + } + } + context("with a receiver report block") { + val reportBlock = RtcpReportBlock( + ssrc = 12345, + fractionLost = 42, + cumulativePacketsLost = 4242, + seqNumCycles = 1, + seqNum = 42, + interarrivalJitter = 4242, + lastSrTimestamp = 23456, + delaySinceLastSr = 34567 + ) + + val srPacket = RtcpSrPacketBuilder( + RtcpHeaderBuilder( + senderSsrc = 12345L + ), + senderInfo, + mutableListOf(reportBlock) + ).build() + + val compoundRtcpPacket = createCompoundRtcp(srPacket) + + rtcpTermination.processPacket(PacketInfo(compoundRtcpPacket)) + should("remove all of the report blocks") { + outputPackets shouldHaveSize 1 + with(outputPackets[0].packet as RtcpSrPacket) { + length shouldBe (RtcpHeader.SIZE_BYTES + SenderInfoParser.SIZE_BYTES) + reportCount shouldBe 0 + reportBlocks.size shouldBe 0 + } + } + should("return the original buffer to the pool") { + returnedBuffers shouldHaveSize 1 + returnedBuffers[0] shouldBe compoundRtcpPacket.buffer + } + } + } + } + + // Create a compound RTCP packet with another arbitrary RTCP packet alongside the SR + private fun createCompoundRtcp(srPacket: RtcpSrPacket): CompoundRtcpPacket = CompoundRtcpPacket( + nackPacket.buffer + srPacket.buffer, + srPacket.offset, + srPacket.length + nackPacket.length + ) +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt new file mode 100644 index 0000000..f93f780 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/SrtpDecryptTest.kt @@ -0,0 +1,120 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.incoming + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.resources.srtp_samples.SrtpSample +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.nlj.test_utils.matchers.ByteArrayBuffer.haveSameContentAs +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.srtp.SrtpErrorStatus + +internal class SrtpDecryptTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + val srtpTransformers = SrtpUtil.initializeTransformer( + SrtpSample.srtpProfileInformation, + SrtpSample.keyingMaterial.array(), + SrtpSample.tlsRole, + // TODO: add tests for cryptex case + cryptex = false, + StdoutLogger() + ) + + context("decrypting an RTCP packet") { + val packetInfo = PacketInfo(SrtpSample.incomingEncryptedRtcpPacket.clone()) + srtpTransformers.srtcpDecryptTransformer.transform(packetInfo) shouldBe SrtpErrorStatus.OK + val decryptedPacket = packetInfo.packet + + should("decrypt the data correctly") { + decryptedPacket shouldNotBe null + decryptedPacket should haveSameContentAs(SrtpSample.expectedDecryptedRtcpPacket) + } + } + + context("decrypting an RTP packet") { + val packetInfo = PacketInfo(SrtpSample.incomingEncryptedRtpPacket.clone()) + srtpTransformers.srtpDecryptTransformer.transform(packetInfo) shouldBe SrtpErrorStatus.OK + + val decryptedPacket = packetInfo.packet + should("decrypt the data correctly") { + decryptedPacket shouldNotBe null + decryptedPacket should haveSameContentAs(SrtpSample.expectedDecryptedRtpPacket) + } + } + + // Note: we don't test that the context map is bounded by its LRU cap (AbstractSrtpTransformer.MAX_SSRCS), + // because contexts are only cached once they authenticate a packet, so driving the map to eviction would + // require generating authenticating packets for that many distinct SSRCs, which the single-key sample can't do. + context("receiving RTP packets that fail to authenticate") { + val decryptTransformer = srtpTransformers.srtpDecryptTransformer + + should("not cache a context for an SSRC that never authenticates") { + repeat(5) { + val packetInfo = PacketInfo(corruptAuthTag(SrtpSample.incomingEncryptedRtpPacket.clone())) + decryptTransformer.transform(packetInfo) shouldNotBe SrtpErrorStatus.OK + } + decryptTransformer.numCachedContexts shouldBe 0 + } + + should("cache a context once a packet authenticates") { + // A failed packet leaves no state behind. + val failed = PacketInfo(corruptAuthTag(SrtpSample.incomingEncryptedRtpPacket.clone())) + decryptTransformer.transform(failed) shouldNotBe SrtpErrorStatus.OK + decryptTransformer.numCachedContexts shouldBe 0 + + // The first successful decryption for the SSRC caches the context. + val succeeded = PacketInfo(SrtpSample.incomingEncryptedRtpPacket.clone()) + decryptTransformer.transform(succeeded) shouldBe SrtpErrorStatus.OK + decryptTransformer.numCachedContexts shouldBe 1 + } + } + + context("receiving SRTCP packets that fail to authenticate") { + val decryptTransformer = srtpTransformers.srtcpDecryptTransformer + + should("not cache a context for an SSRC that never authenticates") { + repeat(5) { + val packet = SrtpSample.incomingEncryptedRtcpPacket.clone() + packet.buffer[packet.offset + packet.length - 1]++ + decryptTransformer.transform(PacketInfo(packet)) shouldNotBe SrtpErrorStatus.OK + } + decryptTransformer.numCachedContexts shouldBe 0 + } + + should("cache a context once a packet authenticates") { + decryptTransformer.transform(PacketInfo(SrtpSample.incomingEncryptedRtcpPacket.clone())) shouldBe + SrtpErrorStatus.OK + decryptTransformer.numCachedContexts shouldBe 1 + } + } + } + + /** + * Corrupts the authentication tag (the last byte) of an encrypted RTP packet so that it will fail to authenticate. + */ + private fun corruptAuthTag(packet: RtpPacket): RtpPacket = packet.apply { + buffer[offset + length - 1]++ + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt new file mode 100644 index 0000000..e2829d0 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/TccGeneratorNodeTest.kt @@ -0,0 +1,346 @@ +package org.jitsi.nlj.transform.node.incoming + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.Spec +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.ints.shouldBeLessThanOrEqual +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.Vp8PayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.rtp.LossListener +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.ReceivedPacketReport +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.UnreceivedPacketReport +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.TccHeaderExtension +import org.jitsi.utils.ms +import org.jitsi.utils.time.FakeClock +import java.time.Duration +import java.util.Random + +class TccGeneratorNodeTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val clock: FakeClock = FakeClock() + private val tccPackets = mutableListOf() + private val onTccReady = { tccPacket: RtcpPacket -> + tccPackets.add(tccPacket) + Unit + } + private val streamInformationStore = StreamInformationStoreImpl() + private val tccExtensionId = 5 + + // TCC is enabled by having at least one payload type which has "transport-cc" signaled as a rtcp-fb. + private val vp8PayloadType = Vp8PayloadType(100, emptyMap(), setOf("transport-cc")) + + private lateinit var tccGenerator: TccGeneratorNode + + private var lossListener = object : LossListener { + var numReceived = 0 + var numLost = 0 + + override fun packetReceived(previouslyReportedLost: Boolean) { + numReceived++ + if (previouslyReportedLost) { + numLost-- + } + } + + override fun packetLost(numLost: Int) { + this.numLost += numLost + } + } + + override suspend fun beforeSpec(spec: Spec) { + super.beforeSpec(spec) + streamInformationStore.addRtpExtensionMapping( + RtpExtension(tccExtensionId.toByte(), RtpExtensionType.TRANSPORT_CC) + ) + streamInformationStore.addRtpPayloadType(vp8PayloadType) + tccGenerator = TccGeneratorNode(onTccReady, streamInformationStore, StdoutLogger(), clock) + tccGenerator.addLossListener(lossListener) + } + + init { + context("when TCC is not signaled") { + streamInformationStore.clearRtpPayloadTypes() + with(clock) { + repeat(100) { tccSeqNum -> + tccGenerator.processPacket( + PacketInfo(createPacket(tccSeqNum)).apply { + receivedTime = clock.instant() + } + ) + elapse(10.ms) + } + } + context("no TCC packets should be sent") { + tccPackets.size shouldBe 0 + } + } + context("when a series of packets (without marking) is received") { + with(clock) { + repeat(11) { tccSeqNum -> + tccGenerator.processPacket( + PacketInfo(createPacket(tccSeqNum)).apply { + receivedTime = clock.instant() + } + ) + elapse(10.ms) + } + } + context("one TCC packet") { + should("be sent after 100ms") { + tccPackets.size shouldBe 1 + } + should("contain the right number of reports") { + tccPackets[0] should beInstanceOf() + with(tccPackets[0] as RtcpFbTccPacket) { + iterator().asSequence().count() shouldBe 11 + } + } + } + context("loss statistics") { + should("be correct") { + lossListener.numReceived shouldBe 11 + lossListener.numLost shouldBe 0 + } + } + } + context("when a series of packets (where one is marked) is received") { + with(clock) { + tccGenerator.processPacket( + PacketInfo(createPacket(1)).apply { + receivedTime = clock.instant() + } + ) + elapse(10.ms) + tccGenerator.processPacket( + PacketInfo(createPacket(2)).apply { + receivedTime = clock.instant() + } + ) + elapse(10.ms) + tccGenerator.processPacket( + PacketInfo( + createPacket(3).apply { + isMarked = true + } + ).apply { + receivedTime = clock.instant() + } + ) + elapse(100.ms) + tccGenerator.processPacket( + PacketInfo(createPacket(4)).apply { + receivedTime = clock.instant() + } + ) + } + context("two TCC packets") { + should("be sent") { + tccPackets.size shouldBe 2 + } + } + context("loss statistics") { + should("be correct") { + lossListener.numReceived shouldBe 4 + lossListener.numLost shouldBe 0 + } + } + } + context("when random packets are added") { + val random = Random(1234) + for (i in 1..10000) { + tccGenerator.processPacket( + PacketInfo(createPacket(random.nextInt(0xffff))).apply { + receivedTime = clock.instant() + } + ) + clock.elapse(10.ms) + + tccPackets.lastOrNull()?.let { + it.length shouldBeLessThan 1500 + } + } + } + context("when a few packets covering the seq num space are added") { + for (i in listOf(0, 10000, 20000, 30000, 40000, 50000, 60000)) { + tccGenerator.processPacket( + PacketInfo(createPacket(i % 0xffff)).apply { + receivedTime = clock.instant() + } + ) + } + context("loss statistics") { + should("be correct") { + lossListener.numReceived shouldBe 7 + lossListener.numLost shouldBe 6 * 9999 + } + } + for (i in 2..5000) { + tccGenerator.processPacket( + PacketInfo(createPacket(i % 0xffff)).apply { + receivedTime = clock.instant() + } + ) + clock.elapse(10.ms) + + tccPackets.lastOrNull()?.let { + it.length shouldBeLessThan 1500 + } + } + } + context("when sequence numbers cycle") { + var prevSize = tccPackets.size + repeat(100000) { tccSeqNum -> + val pi = PacketInfo(createPacket(tccSeqNum % 0xffff)).apply { + receivedTime = clock.instant() + } + tccGenerator.processPacket(pi) + clock.elapse(10.ms) + + tccPackets.size shouldBeLessThanOrEqual prevSize + 1 + prevSize = tccPackets.size + } + context("an appropriate number of TCC packets") { + should("be sent") { + tccPackets.size shouldBe 9999 + } + } + } + context("when sequence numbers cycle with losses") { + var prevSize = tccPackets.size + repeat(50000) { tccSeqNum -> + val pi = PacketInfo(createPacket((tccSeqNum * 2) % 0xffff)).apply { + receivedTime = clock.instant() + } + tccGenerator.processPacket(pi) + clock.elapse(20.ms) + + tccPackets.size shouldBeLessThanOrEqual prevSize + 1 + prevSize = tccPackets.size + } + context("an appropriate number of TCC packets") { + should("be sent") { + tccPackets.size shouldBe 9999 + } + } + } + context("when there is a loss after a TCC packet is sent") { + with(clock) { + repeat(11) { tccSeqNum -> + tccGenerator.processPacket( + PacketInfo(createPacket(tccSeqNum)).apply { + receivedTime = clock.instant() + } + ) + elapse(10.ms) + } + elapse(100.ms) + tccGenerator.processPacket( + PacketInfo(createPacket(20)).apply { + receivedTime = clock.instant() + } + ) + } + context("two TCC packets") { + should("be sent") { + tccPackets.size shouldBe 2 + } + } + context("last TCC packet") { + should("have a base of the lost packet") { + val lastTcc = tccPackets[1] as RtcpFbTccPacket + val firstReport = lastTcc.iterator().next() + firstReport should beInstanceOf() + firstReport.seqNum shouldBe 11 + } + } + } + context("when there is a packet reordering after a TCC packet is sent") { + with(clock) { + repeat(9) { tccSeqNum -> + tccGenerator.processPacket( + PacketInfo(createPacket(tccSeqNum)).apply { + receivedTime = clock.instant() + } + ) + elapse(10.ms) + } + elapse(10.ms) + tccGenerator.processPacket( + PacketInfo(createPacket(10)).apply { + receivedTime = clock.instant() + } + ) + + context("loss statistics") { + should("be correct before reordered packet") { + lossListener.numReceived shouldBe 10 + lossListener.numLost shouldBe 1 + } + } + + tccGenerator.processPacket( + PacketInfo(createPacket(9)).apply { + receivedTime = clock.instant() - Duration.ofMillis(10) + } + ) + + context("loss statistics") { + should("be correct after reordered packet") { + lossListener.numReceived shouldBe 11 + lossListener.numLost shouldBe 0 + } + } + + elapse(100.ms) + tccGenerator.processPacket( + PacketInfo(createPacket(20)).apply { + receivedTime = clock.instant() + } + ) + } + context("two TCC packets") { + should("be sent") { + tccPackets.size shouldBe 2 + } + } + context("last TCC packet") { + should("have a base of the reordered packet") { + val lastTcc = tccPackets[1] as RtcpFbTccPacket + val firstReport = lastTcc.iterator().next() + firstReport should beInstanceOf() + firstReport.seqNum shouldBe 9 + } + } + } + } + + private fun createPacket(tccSeqNum: Int): RtpPacket { + val dummyPacket = RtpPacket(ByteArray(1500), 0, 1500).apply { + version = 2 + hasPadding = false + hasExtensions = false + isMarked = false + payloadType = 100 + sequenceNumber = 123 + timestamp = 456L + ssrc = 1234L + } + val ext = + dummyPacket.addHeaderExtension(tccExtensionId, TccHeaderExtension.DATA_SIZE_BYTES) + TccHeaderExtension.setSequenceNumber(ext, tccSeqNum) + + return dummyPacket + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookupTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookupTest.kt new file mode 100644 index 0000000..6573bdc --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/incoming/VideoQualityLayerLookupTest.kt @@ -0,0 +1,70 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.incoming + +import io.kotest.assertions.fail +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.mockk.every +import io.mockk.mockk +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.node.onOutput +import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.createLogger + +class VideoQualityLayerLookupTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val parser = VideoQualityLayerLookup(createLogger()) + + private val vp8PacketBuf = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + // V=2,P=false,X=true,CC=0,M=false,PT=100,SeqNum=16535 + 0x90, 0x64, 0x40, 0x97, + // Timestamp: 3899068446 + 0xe8, 0x67, 0x10, 0x1e, + // SSRC: 2828806853 + 0xa8, 0x9c, 0x2a, 0xc5, + // 1 extension + 0xbe, 0xde, 0x00, 0x01, + 0x51, 0x00, 0x02, 0x00 + ) + org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + + private val vp8Packet = Vp8Packet(vp8PacketBuf, 0, vp8PacketBuf.size) + private val packetInfo = mockk(relaxed = true) { + every { packetAs() } returns vp8Packet + every { packet } returns vp8Packet + } + + init { + context("When parsing a VP8 packet") { + context("with no encoding signaled") { + parser.onOutput { _ -> + fail("Should not forward the packet") + } + parser.processPacket(packetInfo) + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/MidStamperTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/MidStamperTest.kt new file mode 100644 index 0000000..f07168b --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/MidStamperTest.kt @@ -0,0 +1,92 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.outgoing + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.node.onOutput +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.SdesHeaderExtension + +class MidStamperTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + private val midExtId = 10 + private val ssrcWithMid = 1234L + private val streamInformationStore = StreamInformationStoreImpl() + + private fun midStamper(mid: String) = MidStamper(streamInformationStore) { ssrc -> + if (ssrc == ssrcWithMid) mid else null + } + + private fun packetInfo(ssrc: Long) = PacketInfo( + RtpPacket(ByteArray(1500), 0, 1500).apply { + version = 2 + hasPadding = false + hasExtensions = false + isMarked = false + payloadType = 100 + sequenceNumber = 123 + timestamp = 456L + this.ssrc = ssrc + } + ) + + init { + context("when the mid extension is negotiated") { + streamInformationStore.addRtpExtensionMapping(RtpExtension(midExtId.toByte(), RtpExtensionType.MID)) + + // Mids of assorted lengths: single character, even and odd lengths (odd exercises the extension + // padding path), and the 16-byte maximum of a one-byte header extension. + listOf("0", "v3", "audio-mid-7", "0123456789abcdef").forEach { mid -> + should("stamp the mid \"$mid\" for an SSRC that has a mapping") { + val midStamper = midStamper(mid) + midStamper.onOutput { + val ext = it.packetAs().getHeaderExtension(midExtId) + ext.shouldNotBeNull() + SdesHeaderExtension.getTextValue(ext) shouldBe mid + } + midStamper.processPacket(packetInfo(ssrcWithMid)) + } + } + + should("not stamp an SSRC that has no mapping") { + val midStamper = midStamper("v3") + midStamper.onOutput { + it.packetAs().getHeaderExtension(midExtId) shouldBe null + } + midStamper.processPacket(packetInfo(9999L)) + } + } + + context("when the mid extension is not negotiated") { + should("not stamp anything") { + val midStamper = midStamper("v3") + midStamper.onOutput { + it.packetAs().getHeaderExtension(midExtId) shouldBe null + } + midStamper.processPacket(packetInfo(ssrcWithMid)) + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt new file mode 100644 index 0000000..79e08e3 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/RetransmissionSenderTest.kt @@ -0,0 +1,99 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.outgoing + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.core.test.TestCase +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.instanceOf +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.resources.node.onOutput +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.util.RemoteSsrcAssociation +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtp.RtpPacket + +class RetransmissionSenderTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val originalPayloadType = 100 + private val rtxPayloadType = 96 + private val originalSsrc = 1234L + private val rtxSsrc = 5678L + private val streamInformationStore = StreamInformationStoreImpl() + + // NOTE(brian): unfortunately i ran into issues trying to use mock frameworks to mock + // a packet, notably i ran into issues when trying to mock the byte[] property in + // the parent java class, mocking frameworks seem to struggle with this + private val dummyPacket = RtpPacket(ByteArray(1500), 0, 1500).apply { + version = 2 + hasPadding = false + hasExtensions = false + isMarked = false + payloadType = originalPayloadType + sequenceNumber = 123 + timestamp = 456L + ssrc = originalSsrc + } + private val dummyPacketInfo = PacketInfo(dummyPacket) + private val retransmissionSender = RetransmissionSender(streamInformationStore, StdoutLogger()) + + override suspend fun beforeTest(testCase: TestCase) { + super.beforeTest(testCase) + + // Setup: add the rtx payload type and the rtx ssrc association + streamInformationStore.addRtpPayloadType( + RtxPayloadType(rtxPayloadType.toByte(), mapOf("apt" to originalPayloadType.toString())) + ) + streamInformationStore.addSsrcAssociation( + RemoteSsrcAssociation(originalSsrc, rtxSsrc, SsrcAssociationType.RTX) + ) + } + + init { + context("retransmitting a packet") { + context("which has an associated rtx stream") { + should("rewrite the payload type and ssrc correctly") { + retransmissionSender.onOutput { + it.packet shouldBe instanceOf(RtpPacket::class) + it.packetAs().let { rtpPacket -> + rtpPacket.ssrc shouldBe rtxSsrc + rtpPacket.payloadType shouldBe rtxPayloadType + } + } + retransmissionSender.processPacket(dummyPacketInfo) + } + } + context("which does not have an associated rtx stream") { + val noRtxPacket = dummyPacket.clone().apply { + payloadType = 99 + ssrc = 9876L + } + retransmissionSender.onOutput { + it.packetAs().let { rtpPacket -> + rtpPacket.ssrc shouldBe 9876L + rtpPacket.payloadType shouldBe 99 + } + } + retransmissionSender.processPacket(PacketInfo(noRtxPacket)) + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt new file mode 100644 index 0000000..3496090 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/transform/node/outgoing/SrtpEncryptTest.kt @@ -0,0 +1,90 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.transform.node.outgoing + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.resources.logging.StdoutLogger +import org.jitsi.nlj.resources.srtp_samples.SrtpSample +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.nlj.test_utils.matchers.ByteArrayBuffer.haveSameContentAs +import org.jitsi.nlj.test_utils.matchers.haveSameContentAs +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.rtp.rtcp.RtcpHeader +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacketBuilder +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.TransportLayerRtcpFbPacket +import org.jitsi.srtp.SrtpErrorStatus + +internal class SrtpEncryptTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + val srtpTransformers = SrtpUtil.initializeTransformer( + SrtpSample.srtpProfileInformation, + SrtpSample.keyingMaterial.array(), + SrtpSample.tlsRole, + // TODO: add tests for cryptex case + cryptex = false, + StdoutLogger() + ) + + context("encrypting an RTCP packet") { + context("created from a buffer") { + val packetInfo = PacketInfo(SrtpSample.outgoingUnencryptedRtcpPacket.clone()) + srtpTransformers.srtcpEncryptTransformer.transform(packetInfo) shouldBe SrtpErrorStatus.OK + + val encryptedPacket = packetInfo.packet + should("encrypt the data correctly") { + encryptedPacket shouldNotBe null + encryptedPacket should haveSameContentAs(SrtpSample.expectedEncryptedRtcpPacket) + } + } + context("created from values") { + val originalPacket = RtcpFbNackPacketBuilder( + mediaSourceSsrc = 123, + missingSeqNums = (10..20 step 2).toSortedSet() + ).build() + val packetInfo = PacketInfo(originalPacket.clone()) + srtpTransformers.srtcpEncryptTransformer.transform(packetInfo) shouldBe SrtpErrorStatus.OK + + val encryptedPacket = packetInfo.packet + should("result in all header fields being correct") { + println("original packet:\n${originalPacket.buffer.toHex()}") + println("packet after:\n${encryptedPacket.buffer.toHex()}") + RtcpHeader.getPacketType( + encryptedPacket.buffer, + encryptedPacket.offset + ) shouldBe TransportLayerRtcpFbPacket.PT + } + } + } + + context("encrypting an RTP packet") { + val packetInfo = PacketInfo(SrtpSample.outgoingUnencryptedRtpPacket.clone()) + srtpTransformers.srtpEncryptTransformer.transform(packetInfo) shouldBe SrtpErrorStatus.OK + + val encryptedPacket = packetInfo.packet + should("encrypt the data correctly") { + encryptedPacket should haveSameContentAs(SrtpSample.expectedEncryptedRtpPacket) + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ArrayCacheTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ArrayCacheTest.kt new file mode 100644 index 0000000..8e44d56 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ArrayCacheTest.kt @@ -0,0 +1,159 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.ints.shouldBeGreaterThanOrEqual +import io.kotest.matchers.shouldBe +import org.jitsi.utils.ms +import org.jitsi.utils.time.FakeClock + +internal class ArrayCacheTest : ShouldSpec() { + + data class Dummy(val index: Long) + + private val fakeClock = FakeClock() + + private val arrayCache = object : ArrayCache(10, { Dummy(it.index) }, clock = fakeClock) { + var discarded = 0 + override fun discardItem(item: Dummy) { + discarded++ + } + + /** Expose protected [updateTimeAdded] to tests */ + fun publicUpdateTimeAdded(index: Long, timeAdded: Long) { + updateTimeAdded(index, timeAdded) + } + } + + init { + val data1 = Dummy(100) + val dataOlder = Dummy(98) + val dataNewer = Dummy(101) + val dataTooOld = Dummy(77) + + var numHits = 0 + var numMisses = 0 + var numInserts = 0 + var numOldInserts = 0 + + context("adding and retrieving items ") { + arrayCache.insertItem(data1, data1.index) shouldBe true + arrayCache.getContainer(data1.index)!!.item shouldBe data1 + numInserts++ + numHits++ + } + + context("adding and retrieving older items ") { + arrayCache.insertItem(dataOlder, dataOlder.index) shouldBe true + arrayCache.getContainer(dataOlder.index)!!.item shouldBe dataOlder + numInserts++ + numHits++ + } + + context("adding an item with an index which is too old, and retrieving existing data ") { + arrayCache.insertItem(dataTooOld, dataTooOld.index) shouldBe false + arrayCache.getContainer(data1.index)!!.item shouldBe data1 + arrayCache.getContainer(dataOlder.index)!!.item shouldBe dataOlder + numOldInserts++ + numHits += 2 + } + + context("replacing the data at the latest index") { + val otherData = Dummy(11111) + arrayCache.insertItem(otherData, 100) shouldBe true + arrayCache.getContainer(100)!!.item shouldBe otherData + numInserts++ + numHits++ + } + + context("replacing the data at an older index") { + val otherData = Dummy(22222) + arrayCache.insertItem(otherData, 98) shouldBe true + arrayCache.getContainer(98)!!.item shouldBe otherData + numInserts++ + numHits++ + } + + context("adding and retrieving more data") { + arrayCache.insertItem(dataNewer, dataNewer.index) shouldBe true + arrayCache.getContainer(dataNewer.index)!!.item shouldBe dataNewer + numInserts++ + numHits++ + + for (i in 150L..200L) { + arrayCache.insertItem(Dummy(i), i) shouldBe true + numInserts++ + } + arrayCache.getContainer(199)!!.item shouldBe Dummy(199) + numHits++ + } + + context("iterate forEachDescending") { + // This should iterate for 200..195. It should not touch the miss/hit count or + var nextExpected = 200 + val lastExpected = 195 + arrayCache.forEachDescending { + it.index shouldBe nextExpected + nextExpected shouldBeGreaterThanOrEqual lastExpected + nextExpected-- + nextExpected >= lastExpected + } + } + context("retrieving rewritten data") { + arrayCache.getContainer(data1.index) shouldBe null + numMisses++ + } + context("retrieving data with a newer index") { + arrayCache.getContainer(1000) shouldBe null + numMisses++ + } + context("keeping track of statistics ") { + arrayCache.numInserts shouldBe numInserts + arrayCache.numOldInserts shouldBe numOldInserts + arrayCache.numHits shouldBe numHits + arrayCache.numMisses shouldBe numMisses + } + context("flush should discard all items") { + arrayCache.discarded = 0 + arrayCache.flush() + arrayCache.discarded shouldBe arrayCache.size + arrayCache.getContainer(200) shouldBe null + } + context("time added should be correct") { + val time1 = fakeClock.millis() + arrayCache.insertItem(data1, data1.index) shouldBe true + + fakeClock.elapse(100.ms) + val time2 = fakeClock.millis() + arrayCache.getContainer(data1.index)!!.timeAdded shouldBe time1 + arrayCache.insertItem(dataOlder, dataOlder.index) + + fakeClock.elapse(100.ms) + arrayCache.getContainer(dataOlder.index)!!.timeAdded shouldBe time2 + arrayCache.getContainer(data1.index)!!.timeAdded shouldBe time1 + + fakeClock.elapse(100.ms) + val time3 = fakeClock.millis() + arrayCache.publicUpdateTimeAdded(dataOlder.index, time3) + + fakeClock.elapse(100.ms) + arrayCache.getContainer(dataOlder.index)!!.timeAdded shouldBe time3 + arrayCache.getContainer(data1.index)!!.timeAdded shouldBe time1 + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/BandwidthTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/BandwidthTest.kt new file mode 100644 index 0000000..1eda421 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/BandwidthTest.kt @@ -0,0 +1,119 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.shouldBe +import org.jitsi.utils.ms +import org.jitsi.utils.secs + +class BandwidthTest : ShouldSpec() { + + init { + context("equivalent bandwidths") { + should("match even if created differently") { + 2.5.mbps shouldBe 2_500_000.bps + 2500.kbps shouldBe 2.5.mbps + } + } + context("arithmetic operations on bandwidths") { + should("work correctly") { + 1.kbps * 10 shouldBe 10.kbps + 1.mbps + 500.kbps shouldBe 1.5.mbps + 1.mbps * .95 shouldBe 950.kbps + 1.mbps / 4 shouldBe 250.kbps + 1.mbps / 4.mbps shouldBe 0.25 + } + } + context("operation-assign operations on bandwidths") { + should("work correctly") { + var b = 1.mbps + b += 0.5.mbps + b shouldBe 1.5.mbps + b.kbps shouldBe 1500.0 + b.bps shouldBe 1_500_000 + + b = 1.mbps + b -= 0.5.mbps + b shouldBe 0.5.mbps + b.kbps shouldBe 500.0 + b.bps shouldBe 500_000 + + b = 1.mbps + b *= 2.0 + b shouldBe 2.0.mbps + b.kbps shouldBe 2_000.0 + b.bps shouldBe 2_000_000 + + b = 1.mbps + b /= 2.0 + b shouldBe 0.5.mbps + b.kbps shouldBe 500.0 + b.bps shouldBe 500_000 + } + } + context("printing bandwidths") { + should("print as the most appropriate unit") { + 2_500_000.bps.toString() shouldBe "2.5 mbps" + 500_000.bps.toString() shouldBe "500 kbps" + .001.mbps.toString() shouldBe "1 kbps" + } + } + context("comparing bandwidths") { + context("with integral bps") { + should("work correctly") { + 2.mbps shouldBeGreaterThan 1.mbps + 1000.bps shouldBeGreaterThan 999.bps + } + } + } + context("creation from 'per'") { + should("work correctly") { + 1.megabytes.per(1.secs) shouldBe 8.mbps + 2.bits.per(5.secs) shouldBe 0.bps // bps is an integer + 1.kilobytes.per(1.ms) shouldBe 8.mbps + 6.bits.per(1500.ms) shouldBe 4.bps + } + } + context("creating intervals from rates") { + should("work correctly") { + 1.kilobytes / 4.kbps shouldBe 2.secs + 20.megabytes / 500.kbps shouldBe 320.secs + } + } + context("calculating datasizes from rates and intervals") { + should("work correctly") { + 100.kbps * 100.ms shouldBe 10000.bits + 40.bps * 10.secs shouldBe 400.bits + } + } + context("creation from a string") { + should("work correctly") { + Bandwidth.fromString("10mbps") shouldBe 10.mbps + Bandwidth.fromString("10bps") shouldBe 10.bps + Bandwidth.fromString("10kbps") shouldBe 10.kbps + Bandwidth.fromString("10 kbps") shouldBe 10.kbps + Bandwidth.fromString("10 KbPs") shouldBe 10.kbps + } + should("throw on an invalid unit") { + shouldThrow { Bandwidth.fromString("10foos") } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/BitrateTrackerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/BitrateTrackerTest.kt new file mode 100644 index 0000000..10c1247 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/BitrateTrackerTest.kt @@ -0,0 +1,57 @@ +/* + * Copyright @ 2026 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock + +class BitrateTrackerTest : ShouldSpec() { + private val clock = FakeClock() + private val tracker = BitrateTracker(5.secs, 100.ms, clock) + + init { + context("A single burst") { + // 500 kbit, which is 100 kbps once spread over the 5 second window. + tracker.update(500_000.bits, clock.millis()) + clock.elapse(100.ms) + + should("read as a high rate while the window has not filled up") { + // getRate divides by the time since the stream started, not by the window. + tracker.getRate(clock.millis()) shouldBe 5.mbps + } + should("read as its average over the window with getRateOverFullWindow") { + tracker.getRateOverFullWindow(clock.millis()) shouldBe 100.kbps + } + context("Once the window has filled up") { + clock.elapse(4900.ms) + should("agree with getRate") { + tracker.getRate(clock.millis()) shouldBe 100.kbps + tracker.getRateOverFullWindow(clock.millis()) shouldBe 100.kbps + } + } + context("Once the burst has aged out of the window") { + clock.elapse(10.secs) + should("read as zero") { + tracker.getRateOverFullWindow(clock.millis()) shouldBe 0.bps + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/DataSizeTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/DataSizeTest.kt new file mode 100644 index 0000000..a7a4ceb --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/DataSizeTest.kt @@ -0,0 +1,57 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.shouldBe + +class DataSizeTest : ShouldSpec() { + + init { + context("equivalent data sizes") { + should("match even if created differently") { + 1000.bytes shouldBe 1.kilobytes + 1.megabytes shouldBe 8_000_000.bits + } + } + context("arithmetic operations") { + should("work correctly") { + 1.kilobytes + 1.kilobytes shouldBe 2.kilobytes + 1.megabytes - 500.kilobytes shouldBe 500.kilobytes + 1.bytes * 10 shouldBe 10.bytes + } + } + context("printing data sizes") { + should("print as the most appropriate unit") { + 8_000_000.bits.toString() shouldBe "1 MB" + 1000.bytes.toString() shouldBe "1 KB" + 1551.bytes.toString() shouldBe "1.55 KB" + 1551.kilobytes.toString() shouldBe "1.55 MB" + 32.bits.toString() shouldBe "4 B" + 36.bits.toString() shouldBe "4.5 B" + 4.bits.toString() shouldBe "4 bits" + } + } + context("comparing data sizes") { + should("work correctly") { + 2.megabytes shouldBeGreaterThan 1.megabytes + 1000.bits shouldBeGreaterThan 999.bits + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ExecutorUtilsKtTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ExecutorUtilsKtTest.kt new file mode 100644 index 0000000..43ec9b8 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ExecutorUtilsKtTest.kt @@ -0,0 +1,64 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import org.jitsi.utils.secs +import java.util.concurrent.Executors +import java.util.concurrent.LinkedBlockingQueue + +internal class ExecutorUtilsKtTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val executor = Executors.newSingleThreadExecutor() + + init { + context("shutting down an executor") { + context("with a blocked task which can be interrupted") { + val queue = LinkedBlockingQueue() + executor.submit { + queue.take() + } + + // Give it some time to make sure the executor has started the task + Thread.sleep(1000) + + // This should not throw + executor.safeShutdown(1.secs) + } + context("with an uninterruptable task") { + val queue = LinkedBlockingQueue() + executor.submit { + while (true) { + try { + queue.take() + } catch (e: Exception) {} + } + } + + // Give it some time to make sure the executor has started the task + Thread.sleep(1000) + + shouldThrow { + executor.safeShutdown(1.secs) + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/IndexTrackerTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/IndexTrackerTest.kt new file mode 100644 index 0000000..efde878 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/IndexTrackerTest.kt @@ -0,0 +1,293 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.assertions.withClue +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.shouldBe + +internal class IndexTrackerTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + context("RtpSequenceIndexTracker") { + val indexTracker = RtpSequenceIndexTracker() + + context("Not accept input outside the range") { + listOf(-1, 0x1_0000, 0x1000_0000).forEach { invalid -> + withClue("Testing $invalid") { + shouldThrow { + indexTracker.update(invalid) + } + shouldThrow { + indexTracker.interpret(invalid) + } + shouldThrow { + indexTracker.resetAt(invalid) + } + } + } + } + + context("feeding in the first sequence number") { + val firstIndex = indexTracker.update(65000) + should("return itself as the index") { + firstIndex shouldBe 65000 + } + context("and then another without rolling over") { + val secondIndex = indexTracker.update(65001) + should("return itself as the index") { + secondIndex shouldBe 65001 + } + context("and then another which does roll over") { + val rollOverIndex = indexTracker.update(2) + should("return the proper index") { + rollOverIndex shouldBe 1 * 0x1_0000 + 2L + } + context("and then a sequence number from the previous rollover") { + val prevRollOverIndex = indexTracker.update(65002) + should("return the proper index") { + prevRollOverIndex shouldBe 65002 + } + } + } + context("and then an older sequence number") { + val oldIndex = 64000 + should("return the proper index") { + indexTracker.update(oldIndex) shouldBe oldIndex + } + } + } + } + context("a series of sequence numbers") { + should("never return a negative index") { + var seqNum = 22134 + repeat(35537) { + indexTracker.update(seqNum++) shouldBeGreaterThan 0 + } + } + } + context("resetting the index tracker") { + indexTracker.update(65000) + indexTracker.update(65001) + indexTracker.update(2) + indexTracker.resetAt(65002) + should("return the proper index") { + indexTracker.update(65002) shouldBe 2 * 0x1_0000 + 65002 + } + } + } + context("LongIndexTracker") { + // This should behave identically to the RtpSequenceIndexTracker, but we can't reuse the same test because + // the API is different + val indexTracker = LongIndexTracker(16) + + context("Not accept input outside the range") { + listOf(-1L, 0x1_0000L, 0x1000_0000L).forEach { invalid -> + withClue("Testing $invalid") { + shouldThrow { + indexTracker.update(invalid) + } + shouldThrow { + indexTracker.interpret(invalid) + } + shouldThrow { + indexTracker.resetAt(invalid) + } + } + } + } + + context("feeding in the first sequence number") { + val firstIndex = indexTracker.update(65000) + should("return itself as the index") { + firstIndex shouldBe 65000 + } + context("and then another without rolling over") { + val secondIndex = indexTracker.update(65001) + should("return itself as the index") { + secondIndex shouldBe 65001 + } + context("and then another which does roll over") { + val rollOverIndex = indexTracker.update(2) + should("return the proper index") { + rollOverIndex shouldBe 1 * 0x1_0000 + 2L + } + context("and then a sequence number from the previous rollover") { + val prevRollOverIndex = indexTracker.update(65002) + should("return the proper index") { + prevRollOverIndex shouldBe 65002 + } + } + } + context("and then an older sequence number") { + val oldIndex = 64000L + should("return the proper index") { + indexTracker.update(oldIndex) shouldBe oldIndex + } + } + } + } + context("a series of sequence numbers") { + should("never return a negative index") { + var seqNum = 22134L + repeat(35537) { + indexTracker.update(seqNum++) shouldBeGreaterThan 0 + } + } + } + context("resetting the index tracker") { + indexTracker.update(65000) + indexTracker.update(65001) + indexTracker.update(2) + indexTracker.resetAt(65002) + should("return the proper index") { + indexTracker.update(65002) shouldBe 2 * 0x1_0000 + 65002 + } + } + } + context("PictureIdIndexTracker") { + val indexTracker = PictureIdIndexTracker() + + context("Not accept input outside the range") { + listOf(-1, 0x8000, 0x1000_0000).forEach { invalid -> + withClue("Testing $invalid") { + shouldThrow { + indexTracker.update(invalid) + } + shouldThrow { + indexTracker.interpret(invalid) + } + shouldThrow { + indexTracker.resetAt(invalid) + } + } + } + } + + context("feeding in the first sequence number") { + val firstIndex = indexTracker.update(32000) + should("return itself as the index") { + firstIndex shouldBe 32000 + } + context("and then another without rolling over") { + val secondIndex = indexTracker.update(32001) + should("return itself as the index") { + secondIndex shouldBe 32001 + } + context("and then another which does roll over") { + val rollOverIndex = indexTracker.update(2) + should("return the proper index") { + rollOverIndex shouldBe 1L * 0x8000 + 2 + } + context("and then a sequence number from the previous rollover") { + val prevRollOverIndex = indexTracker.update(32002) + should("return the proper index") { + prevRollOverIndex shouldBe 32002 + } + } + } + context("and then an older sequence number") { + val oldIndex = 31000 + should("return the proper index") { + indexTracker.update(oldIndex) shouldBe oldIndex + } + } + } + } + context("a series of sequence numbers") { + should("never return a negative index") { + var seqNum = 22134 + repeat(3553) { + indexTracker.update(seqNum++) shouldBeGreaterThan 0 + } + } + } + } + context("RtpTimestampIndexTracker") { + val indexTracker = RtpTimestampIndexTracker() + val base = 4000000000L // higher than 2^31 + + context("Not accept input outside the range") { + listOf(-1L, 0x1_0000_0000, 0x100_0000_0000).forEach { invalid -> + withClue("Testing $invalid") { + shouldThrow { + indexTracker.update(invalid) + } + shouldThrow { + indexTracker.interpret(invalid) + } + shouldThrow { + indexTracker.resetAt(invalid) + } + } + } + } + + context("feeding in the first timestamp") { + val firstIndex = indexTracker.update(base) + should("return itself as the index") { + firstIndex shouldBe base + } + context("and then another without rolling over") { + val secondIndex = indexTracker.update(base + 1000) + should("return itself as the index") { + secondIndex shouldBe (base + 1000) + } + context("and then another which does roll over") { + val rollOverIndex = indexTracker.update(2000L) + should("return the proper index") { + rollOverIndex shouldBe 0x1_0000_0000 + 2000L + } + context("and then a timestamp from the previous rollover") { + val prevRollOverIndex = indexTracker.update(base + 2000) + should("return the proper index") { + prevRollOverIndex shouldBe base + 2000 + } + } + } + context("and then an older timestamp") { + val oldIndex = base - 1000000 + should("return the proper index") { + indexTracker.update(oldIndex) shouldBe oldIndex + } + } + } + } + context("a series of timestamps") { + should("never return a negative index") { + var timestamp = 3000000000L + repeat(35537) { + indexTracker.update(timestamp++) shouldBeGreaterThan 0 + } + } + } + context("resetting the timestamp tracker") { + indexTracker.update(base) + indexTracker.update(base + 1000) + indexTracker.update(2000L) + indexTracker.resetAt(base + 2000) + should("return the proper index") { + indexTracker.update(base + 2000) shouldBe 2L * 0x1_0000_0000 + base + 2000 + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/RateUtilsKtTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/RateUtilsKtTest.kt new file mode 100644 index 0000000..dd39d76 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/RateUtilsKtTest.kt @@ -0,0 +1,43 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.utils.ms +import org.jitsi.utils.secs + +class RateUtilsKtTest : ShouldSpec() { + + init { + context("atRate") { + should("work correctly") { + 1.megabytes atRate 1.mbps shouldBe 8.secs + } + } + context("in") { + should("work correctly") { + val size = howMuchCanISendAtRate(1.mbps).`in`(8.secs) + size shouldBe 1.megabytes + } + should("work correctly for fractional durations") { + val size = howMuchCanISendAtRate(1.mbps).`in`(800.ms) + size shouldBe 100.kilobytes + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ResumableStreamRewriterTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ResumableStreamRewriterTest.kt new file mode 100644 index 0000000..593ac4a --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/ResumableStreamRewriterTest.kt @@ -0,0 +1,277 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.rtp.ResumableStreamRewriter + +internal class ResumableStreamRewriterTest : ShouldSpec() { + init { + context("Without history") { + var ret: Int + val snr = ResumableStreamRewriter(false) + + context("Initial state") { + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe -1 + } + + context("Accept first packet.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0xffff - 2L + ret shouldBe 0xffff - 2L + } + + context("Retransmission.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0xffff - 2L + ret shouldBe 0xffff - 2L + } + + context("Retransmission & accept toggle.") { + snr.rewriteSequenceNumber(false, 0xffff - 2) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0xffff - 2L + } + + context("Retransmission & accept toggle again.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0xffff - 2L + ret shouldBe 0xffff - 2L + } + + context("Drop ordered packet.") { + snr.rewriteSequenceNumber(false, 0xffff - 1) + snr.seqnumDelta shouldBe 1 + snr.highestSequenceNumberSent shouldBe 0xffff - 2L + } + + context("Drop re-ordered packet.") { + snr.rewriteSequenceNumber(false, 0xffff - 3) + snr.seqnumDelta shouldBe 1 + snr.highestSequenceNumberSent shouldBe 0xffff - 2L + } + + context("Accept after re-ordered drop.") { + ret = snr.rewriteSequenceNumber(true, 0xffff) + snr.seqnumDelta shouldBe 1 + snr.highestSequenceNumberSent shouldBe 0xffff - 1L + ret shouldBe 0xffff - 1L + } + + context("Drop ordered packet again.") { + snr.rewriteSequenceNumber(false, 0) + snr.seqnumDelta shouldBe 2 + snr.highestSequenceNumberSent shouldBe 0xffff - 1L + } + + context("Accept ordered packet.") { + ret = snr.rewriteSequenceNumber(true, 1) + snr.seqnumDelta shouldBe 2 + snr.highestSequenceNumberSent shouldBe 0xffff + ret shouldBe 0xffff + } + + context("Drop ordered packets.") { + for (i in 2 until 0xffff) { + withClue("index = $i") { + snr.rewriteSequenceNumber(false, i) + snr.seqnumDelta shouldBe i + 1 + snr.highestSequenceNumberSent shouldBe 0xffff + } + } + } + + context("Drop ordered packet again(2)") { + snr.rewriteSequenceNumber(false, 0xffff) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0xffff + } + + context("Accept ordered packet again") { + ret = snr.rewriteSequenceNumber(true, 0) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0 + ret shouldBe 0 + } + + context("Retransmission + accept toggle") { + ret = snr.rewriteSequenceNumber(true, 0xffff) + snr.seqnumDelta shouldBe 0 + snr.highestSequenceNumberSent shouldBe 0 + ret shouldBe 0xffff + } + } + + context("With history") { + /* The same as the trace above, other than checking internal state. */ + context("In-order trace") { + var ret: Int + val snr = ResumableStreamRewriter(true) + + context("Accept first packet.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + ret shouldBe 0xffff - 2L + } + + context("Retransmission.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + ret shouldBe 0xffff - 2L + } + + context("Retransmission & accept toggle.") { + snr.rewriteSequenceNumber(false, 0xffff - 2) + } + + context("Retransmission & accept toggle again.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + ret shouldBe 0xffff - 2L + } + + context("Drop ordered packet.") { + snr.rewriteSequenceNumber(false, 0xffff - 1) + } + + context("Drop re-ordered packet.") { + snr.rewriteSequenceNumber(false, 0xffff - 3) + } + + context("Accept after re-ordered drop.") { + ret = snr.rewriteSequenceNumber(true, 0xffff) + ret shouldBe 0xffff - 1L + } + + context("Drop ordered packet again.") { + snr.rewriteSequenceNumber(false, 0) + } + + context("Accept ordered packet.") { + ret = snr.rewriteSequenceNumber(true, 1) + ret shouldBe 0xffff + } + + context("Drop ordered packets.") { + for (i in 2 until 0xffff) { + withClue("index = $i") { + snr.rewriteSequenceNumber(false, i) + } + } + } + + context("Drop ordered packet again(2)") { + snr.rewriteSequenceNumber(false, 0xffff) + } + + context("Accept ordered packet again") { + ret = snr.rewriteSequenceNumber(true, 0) + ret shouldBe 0 + } + + context("Retransmission + accept toggle") { + ret = snr.rewriteSequenceNumber(true, 0xffff) + ret shouldBe 0xffff + } + } + + context("Reordering trace") { + var ret: Int + val snr = ResumableStreamRewriter(true) + + context("Accept first packet.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 2) + ret shouldBe 0xffff - 2L + } + + context("Drop packet one after.") { + snr.rewriteSequenceNumber(false, 0xffff - 1) + } + + context("Accept next packet.") { + ret = snr.rewriteSequenceNumber(true, 0xffff) + ret shouldBe 0xffff - 1L + } + + context("Drop packet before first.") { + snr.rewriteSequenceNumber(false, 0xffff - 3) + } + + context("Accept packet before that.") { + ret = snr.rewriteSequenceNumber(true, 0xffff - 4) + ret shouldBe 0xffff - 3L + } + + context("Drop packet after a gap") { + snr.rewriteSequenceNumber(false, 1) + } + + context("And another") { + snr.rewriteSequenceNumber(false, 2) + } + + context("Accept packet after gap") { + ret = snr.rewriteSequenceNumber(true, 3) + ret shouldBe 0 + } + + context("Accept packet filling gap") { + ret = snr.rewriteSequenceNumber(true, 0) + ret shouldBe 0xffff + } + + context("Accept packet after another gap") { + ret = snr.rewriteSequenceNumber(true, 6) + ret shouldBe 3 + } + + context("Check Gaps Left statistics") { + snr.gapsLeft shouldBe 0 + snr.rewriteSequenceNumber(false, 4) + + snr.gapsLeft shouldBe 1 + } + + context("Check Gaps Left statistics on replay of drop") { + snr.rewriteSequenceNumber(false, 4) + + snr.gapsLeft shouldBe 1 + } + + context("Verify that replay of accept works") { + ret = snr.rewriteSequenceNumber(true, 0) + ret shouldBe 0xffff + } + + context("Roll forward by the full gap") { + for (seq in 5..1005) { + ret = snr.rewriteSequenceNumber(true, seq) + ret shouldBe seq - 3 + } + } + + context("Accept packet older than wraparound.") { + ret = snr.rewriteSequenceNumber(true, 3) + ret shouldBe 0 + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/RtpPacketExtensionsKtTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/RtpPacketExtensionsKtTest.kt new file mode 100644 index 0000000..7168e3d --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/RtpPacketExtensionsKtTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.test_utils.matchers.haveSameContentAs +import org.jitsi.rtp.extensions.plus +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.byteBufferOf +import java.nio.ByteBuffer + +class RtpPacketExtensionsKtTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + val payload = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + 0x01, 0x02, 0x03, 0x04, + 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C + ) + + val buf = org.jitsi.rtp.extensions.bytearray.byteArrayOf( + // Dummy header data + // V=2,P=false,X=false,CC=0,M=false,PT=111,SeqNum=5807 + 0x80, 0x6f, 0x16, 0xaf, + // Timestamp: 1710483662 + 0x65, 0xf3, 0xe8, 0xce, + // SSRC: 839852602 + 0x32, 0x0f, 0x22, 0x3a + ) + payload + val rtpPacket = RtpPacket(buf) + + init { + context("shiftPayloadRight") { + context("when the buffer doesn't have any room") { + rtpPacket.shiftPayloadRight(2) + should("shift things correctly") { + // Header should be the same + rtpPacket.version shouldBe 2 + rtpPacket.hasPadding shouldBe false + rtpPacket.hasExtensions shouldBe false + rtpPacket.csrcCount shouldBe 0 + rtpPacket.isMarked shouldBe false + rtpPacket.payloadType shouldBe 111 + rtpPacket.sequenceNumber shouldBe 5807 + rtpPacket.timestamp shouldBe 1710483662L + + val newPayload = ByteBuffer.wrap( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength + ).slice() + newPayload should haveSameContentAs( + byteBufferOf(0x01, 0x02) + ByteBuffer.wrap(payload) + ) + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TimeExpiringCacheTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TimeExpiringCacheTest.kt new file mode 100644 index 0000000..53419e2 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TimeExpiringCacheTest.kt @@ -0,0 +1,141 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.nlj.util + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.shouldBeSameInstanceAs +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import java.time.Instant + +internal class TimeExpiringCacheTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + data class Dummy(val num: Int) + + private val fakeClock = FakeClock() + + private val timeExpiringCache = TimeExpiringCache( + 1.secs, + 20, + fakeClock + ) + + init { + context("adding data") { + val data1 = Dummy(1) + timeExpiringCache.insert(data1.num, data1) + context("and then retrieving it before the timeout has elapsed") { + val retrievedData1 = timeExpiringCache.get(data1.num) + should("return the data") { + retrievedData1 shouldNotBe null + retrievedData1 shouldBeSameInstanceAs data1 + } + } + context("and then retrieving it after the timeout has elapsed") { + fakeClock.elapse(5.secs) + val retrievedData1 = timeExpiringCache.get(data1.num) + // We don't prune on 'get', only on 'insert' + should("return the data") { + retrievedData1 shouldNotBe null + retrievedData1 shouldBeSameInstanceAs data1 + } + } + context("and then adding more data > timeout period later") { + val data2 = Dummy(2) + fakeClock.elapse(5.secs) + timeExpiringCache.insert(data2.num, data2) + context("and then trying to retrieve the old data") { + val retrievedData1 = timeExpiringCache.get(data1.num) + should("return null") { + retrievedData1 shouldBe null + } + } + } + } + context("adding lots of data over time") { + for (i in 1..10) { + timeExpiringCache.insert(i, Dummy(i)) + fakeClock.elapse(1.secs) + } + should("expire things properly") { + // Only the last value should still be present + for (i in 1..9) { + timeExpiringCache.get(i) shouldBe null + } + timeExpiringCache.get(10) shouldNotBe null + } + } + context("adding data at the same time") { + for (i in 1..10) { + timeExpiringCache.insert(i, Dummy(i)) + } + context("and then another > timeout period later") { + fakeClock.elapse(10.secs) + timeExpiringCache.insert(11, Dummy(11)) + should("expire all the old data") { + for (i in 1..10) { + timeExpiringCache.get(i) shouldBe null + } + } + } + } + context("adding data out of order") { + // Here we simulate data being inserted out of order (with relation + // to their index value) + timeExpiringCache.insert(1, Dummy(1)) + fakeClock.setTime(Instant.ofEpochMilli(400)) + timeExpiringCache.insert(3, Dummy(3)) + fakeClock.setTime(Instant.ofEpochMilli(900)) + timeExpiringCache.insert(2, Dummy(2)) + + fakeClock.setTime(Instant.ofEpochMilli(1100)) + timeExpiringCache.insert(4, Dummy(4)) + should("only expire up to the first old index") { + timeExpiringCache.get(1) shouldBe null + timeExpiringCache.get(2) shouldNotBe null + // Even though element 3 should be expired according to its insertion + // time, element 2 was new enough that it shouldn't be expired, so + // the cache will stop pruning when it hits element 2 (which is before + // it hits element 3) + timeExpiringCache.get(3) shouldNotBe null + timeExpiringCache.get(4) shouldNotBe null + } + } + context("adding more than the maximum amount of elements") { + for (i in 1..20) { + timeExpiringCache.insert(i, Dummy(i)) + } + should("remove the oldest indices") { + for (i in 21..30) { + timeExpiringCache.insert(i, Dummy(i)) + } + // 1 through 10 should not be gone... + for (i in 1..10) { + timeExpiringCache.get(i) shouldBe null + } + // ...but 11 through 30 should still be there + for (i in 11..30) { + timeExpiringCache.get(i) shouldNotBe null + } + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt new file mode 100644 index 0000000..95e3113 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/nlj/util/TreeCacheTest.kt @@ -0,0 +1,100 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.nlj.util + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import java.util.AbstractMap.SimpleImmutableEntry + +class TreeCacheTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + data class Dummy(val data: String) + + private val treeCache = TreeCache(16) + + /** Shorthand for a Map.Entry mapping [key] to a Dummy containing [dummyVal] */ + private fun ed(key: Int, dummyVal: String) = SimpleImmutableEntry(key, Dummy(dummyVal)) + + init { + context("Reading from an empty TreeCache") { + should("return null") { + treeCache.getEntryBefore(10) shouldBe null + } + should("have size 0") { + treeCache.size shouldBe 0 + } + } + context("An entry in a TreeCache") { + treeCache.insert(5, Dummy("A")) + should("be found looking up values after it") { + treeCache.getEntryBefore(10) shouldBe ed(5, "A") + } + should("be found looking up the same value") { + treeCache.getEntryBefore(5) shouldBe ed(5, "A") + } + should("not be found looking up values before it") { + treeCache.getEntryBefore(3) shouldBe null + } + should("not be expired even if values long after it are looked up") { + treeCache.getEntryBefore(10000) shouldBe ed(5, "A") + } + should("cause the tree to have size 1") { + treeCache.size shouldBe 1 + } + } + context("Multiple values in a TreeCache") { + treeCache.insert(5, Dummy("A")) + treeCache.insert(10, Dummy("B")) + should("Be looked up properly") { + treeCache.getEntryBefore(13) shouldBe ed(10, "B") + treeCache.size shouldBe 2 + } + should("Persist within the cache window") { + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.size shouldBe 2 + } + should("Not expire an older one if it is the only value outside the cache window") { + treeCache.getEntryBefore(25) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.size shouldBe 2 + } + should("Expire older ones when newer ones are outside the cache window") { + treeCache.getEntryBefore(30) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe null + treeCache.size shouldBe 1 + } + should("Expire only older ones when later values are inserted") { + treeCache.insert(40, Dummy("C")) + treeCache.getEntryBefore(13) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe null + treeCache.size shouldBe 2 + } + should("Persist values within the window while expiring values outside it") { + treeCache.insert(15, Dummy("C")) + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.getEntryBefore(25) shouldBe ed(15, "C") + treeCache.getEntryBefore(13) shouldBe ed(10, "B") + treeCache.getEntryBefore(8) shouldBe ed(5, "A") + treeCache.size shouldBe 3 + treeCache.insert(30, Dummy("D")) + treeCache.getEntryBefore(8) shouldBe null + treeCache.size shouldBe 3 + } + } + } +} diff --git a/jitsi-media-transform/src/test/kotlin/org/jitsi/test_utils/Pcap.kt b/jitsi-media-transform/src/test/kotlin/org/jitsi/test_utils/Pcap.kt new file mode 100644 index 0000000..e9c6f79 --- /dev/null +++ b/jitsi-media-transform/src/test/kotlin/org/jitsi/test_utils/Pcap.kt @@ -0,0 +1,188 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.test_utils + +import org.jitsi.nlj.format.OpusPayloadType +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.format.Vp8PayloadType +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.srtp.SrtpProfileInformation +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.rtp.util.byteBufferOf + +data class SrtpData( + var srtpProfileInformation: SrtpProfileInformation, + var keyingMaterial: ByteArray, + var tlsRole: TlsRole +) + +data class SourceAssociation( + var primarySsrc: Long, + var secondarySsrc: Long, + var associationType: SsrcAssociationType +) + +data class PcapInformation( + var filePath: String, + var srtpData: SrtpData, + var payloadTypes: List, + var headerExtensions: List, + var ssrcAssociations: List +) + +private val DEFAULT_HEADER_EXTENSIONS = listOf( + RtpExtension(1, RtpExtensionType.SSRC_AUDIO_LEVEL), + RtpExtension(3, RtpExtensionType.ABS_SEND_TIME), + RtpExtension(4, RtpExtensionType.RTP_STREAM_ID), + RtpExtension(5, RtpExtensionType.TRANSPORT_CC) +) + +object Pcaps { + object Incoming { + val ONE_PARTICIPANT_RTP_RTCP_SIM_RTX = PcapInformation( + filePath = javaClass.classLoader.getResource( + "pcaps/1_incoming_participant_rtp_rtcp_sim_rtx.pcap" + ).path, + srtpData = SrtpData( + srtpProfileInformation = SrtpProfileInformation( + cipherKeyLength = 16, + cipherSaltLength = 14, + cipherName = 1, + authFunctionName = 1, + authKeyLength = 20, + rtcpAuthTagLength = 10, + rtpAuthTagLength = 10 + ), + keyingMaterial = byteBufferOf( + 0x70, 0xD5, 0x56, 0xB1, + 0xC5, 0xB3, 0xC7, 0x7E, + 0xE6, 0x31, 0xC3, 0xA2, + 0xC2, 0x93, 0x4E, 0xAD, + 0xBE, 0x53, 0xDD, 0x22, + 0xB5, 0x6E, 0xF2, 0xD9, + 0xB6, 0x67, 0xE9, 0xEF, + 0xD5, 0xB2, 0x88, 0x6F, + 0x6C, 0x6F, 0x16, 0xAC, + 0xA2, 0x13, 0xDF, 0x1D, + 0x63, 0x60, 0x39, 0x1F, + 0x23, 0x74, 0xBA, 0x0C, + 0x5B, 0x0B, 0x14, 0x3E, + 0x2E, 0x3E, 0x6D, 0x30, + 0xFF, 0x6F, 0x54, 0x5B + ).array(), + tlsRole = TlsRole.CLIENT + ), + payloadTypes = listOf( + RtxPayloadType(96, parameters = mapOf("apt" to "100")), + Vp8PayloadType(100, rtcpFeedbackSet = setOf("nack pli", "ccm fir")), + OpusPayloadType(111) + ), + headerExtensions = DEFAULT_HEADER_EXTENSIONS, + ssrcAssociations = listOf( + SourceAssociation(1632300152, 1929115835, SsrcAssociationType.RTX), + SourceAssociation(3232245189, 3407277242, SsrcAssociationType.RTX), + SourceAssociation(1465075899, 3483093671, SsrcAssociationType.RTX) + ) + ) + val ONE_PARTICIPANT_RTP_RTCP = PcapInformation( + filePath = javaClass.classLoader.getResource( + "pcaps/capture_1_incoming_participant_1_rtp_and_rtcp.pcap" + ).path, + srtpData = SrtpData( + srtpProfileInformation = SrtpProfileInformation( + cipherKeyLength = 16, + cipherSaltLength = 14, + cipherName = 1, + authFunctionName = 1, + authKeyLength = 20, + rtcpAuthTagLength = 10, + rtpAuthTagLength = 10 + ), + keyingMaterial = byteBufferOf( + 0x2D, 0x6C, 0x37, 0xC7, + 0xA3, 0x49, 0x25, 0x82, + 0x1F, 0x3B, 0x62, 0x0D, + 0x05, 0x8A, 0x29, 0x64, + 0x6F, 0x49, 0xD6, 0x04, + 0xE6, 0xD6, 0x48, 0xE0, + 0x67, 0x43, 0xF3, 0x1F, + 0x6D, 0x2F, 0x4B, 0x33, + 0x6A, 0x61, 0xD8, 0x84, + 0x00, 0x32, 0x1A, 0x84, + 0x00, 0x8C, 0xC5, 0xC3, + 0xCB, 0x18, 0xCE, 0x8D, + 0x34, 0x3C, 0x2C, 0x70, + 0x62, 0x26, 0x39, 0x05, + 0x7D, 0x5A, 0xF9, 0xC7 + ).array(), + tlsRole = TlsRole.CLIENT + ), + payloadTypes = listOf( + Vp8PayloadType(100), + OpusPayloadType(111) + ), + headerExtensions = DEFAULT_HEADER_EXTENSIONS, + ssrcAssociations = listOf() + ) + } + object Outgoing { + val ONE_PARITICPANT_RTP_AND_RTCP_DECRYPTED = PcapInformation( + filePath = javaClass.classLoader.getResource( + "pcaps/capture_1_incoming_participant_1_rtp_and_rtcp_decrypted_2.pcap" + ).path, + srtpData = SrtpData( + srtpProfileInformation = SrtpProfileInformation( + cipherKeyLength = 16, + cipherSaltLength = 14, + cipherName = 1, + authFunctionName = 1, + authKeyLength = 20, + rtcpAuthTagLength = 10, + rtpAuthTagLength = 10 + ), + keyingMaterial = byteBufferOf( + 0x2D, 0x6C, 0x37, 0xC7, + 0xA3, 0x49, 0x25, 0x82, + 0x1F, 0x3B, 0x62, 0x0D, + 0x05, 0x8A, 0x29, 0x64, + 0x6F, 0x49, 0xD6, 0x04, + 0xE6, 0xD6, 0x48, 0xE0, + 0x67, 0x43, 0xF3, 0x1F, + 0x6D, 0x2F, 0x4B, 0x33, + 0x6A, 0x61, 0xD8, 0x84, + 0x00, 0x32, 0x1A, 0x84, + 0x00, 0x8C, 0xC5, 0xC3, + 0xCB, 0x18, 0xCE, 0x8D, + 0x34, 0x3C, 0x2C, 0x70, + 0x62, 0x26, 0x39, 0x05, + 0x7D, 0x5A, 0xF9, 0xC7 + ).array(), + tlsRole = TlsRole.CLIENT + ), + payloadTypes = listOf( + Vp8PayloadType(100), + OpusPayloadType(111), + RtxPayloadType(96, parameters = mapOf("apt" to "100")) + ), + headerExtensions = DEFAULT_HEADER_EXTENSIONS, + ssrcAssociations = listOf() + ) + } +} diff --git a/jitsi-media-transform/src/test/resources/pcaps/1_incoming_participant_rtp_rtcp_sim_rtx.pcap b/jitsi-media-transform/src/test/resources/pcaps/1_incoming_participant_rtp_rtcp_sim_rtx.pcap new file mode 100644 index 0000000..b0e7bc7 Binary files /dev/null and b/jitsi-media-transform/src/test/resources/pcaps/1_incoming_participant_rtp_rtcp_sim_rtx.pcap differ diff --git a/jitsi-media-transform/src/test/resources/pcaps/capture_1_incoming_participant_1_rtp_and_rtcp.pcap b/jitsi-media-transform/src/test/resources/pcaps/capture_1_incoming_participant_1_rtp_and_rtcp.pcap new file mode 100644 index 0000000..0c8ad6f Binary files /dev/null and b/jitsi-media-transform/src/test/resources/pcaps/capture_1_incoming_participant_1_rtp_and_rtcp.pcap differ diff --git a/jitsi-media-transform/src/test/resources/pcaps/capture_1_incoming_participant_1_rtp_and_rtcp_decrypted_2.pcap b/jitsi-media-transform/src/test/resources/pcaps/capture_1_incoming_participant_1_rtp_and_rtcp_decrypted_2.pcap new file mode 100644 index 0000000..10c8c4b Binary files /dev/null and b/jitsi-media-transform/src/test/resources/pcaps/capture_1_incoming_participant_1_rtp_and_rtcp_decrypted_2.pcap differ diff --git a/jvb/checkstyle.xml b/jvb/checkstyle.xml new file mode 100644 index 0000000..9690b13 --- /dev/null +++ b/jvb/checkstyle.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/jvb/lib/logging.properties b/jvb/lib/logging.properties new file mode 100644 index 0000000..2ab8de7 --- /dev/null +++ b/jvb/lib/logging.properties @@ -0,0 +1,25 @@ +handlers= java.util.logging.ConsoleHandler +#handlers= java.util.logging.ConsoleHandler, io.sentry.jul.SentryHandler + +java.util.logging.ConsoleHandler.level = ALL +java.util.logging.ConsoleHandler.formatter = org.jitsi.utils.logging2.JitsiLogFormatter + +org.jitsi.utils.logging2.JitsiLogFormatter.programname=JVB +.level=INFO + +# Sentry (uncomment handler to use) +io.sentry.jul.SentryHandler.level=WARNING + +# time series logging +java.util.logging.SimpleFormatter.format= %5$s%n +java.util.logging.FileHandler.level = ALL +java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter +java.util.logging.FileHandler.pattern = /tmp/jvb-series.log +java.util.logging.FileHandler.limit = 200000000 +java.util.logging.FileHandler.count = 1 +java.util.logging.FileHandler.append = false + +timeseries.level=OFF +timeseries.useParentHandlers = false +# time series logging is disabled by default. Uncomment the line below to enable it. +#timeseries.handlers = java.util.logging.FileHandler diff --git a/jvb/lib/videobridge.rc b/jvb/lib/videobridge.rc new file mode 100644 index 0000000..a8a0b75 --- /dev/null +++ b/jvb/lib/videobridge.rc @@ -0,0 +1,8 @@ +# The amount of memory that the java process can consume. +# Use the format that is in use for java's -Xmx switch +# The default is 3072m +# Only effective on linux, 64 bit +# VIDEOBRIDGE_MAX_MEMORY=3072m + +# Uncomment the next line to enable the remote debugging of the video bridge +# VIDEOBRIDGE_DEBUG_OPTIONS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8000" \ No newline at end of file diff --git a/jvb/pom.xml b/jvb/pom.xml new file mode 100644 index 0000000..907f259 --- /dev/null +++ b/jvb/pom.xml @@ -0,0 +1,611 @@ + + + 4.0.0 + + + org.jitsi + jvb-parent + 2.3-SNAPSHOT + + + + + jitsi-videobridge + jar + jitsi-videobridge + https://github.com/jitsi/jitsi-videobridge + + false + org.jitsi.videobridge.MainKt + 2.0.16 + 3.6.3 + 1.8.0 + + + + + + buildFatJar + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + build-fat-jar + package + + + + + + + + + + + com.google.guava + guava + 33.4.0-jre + + + org.eclipse.jetty + jetty-rewrite + ${jetty.version} + + + org.eclipse.jetty + jetty-security + ${jetty.version} + + + org.eclipse.jetty.ee10 + jetty-ee10-servlets + ${jetty.version} + + + org.eclipse.jetty.ee10.websocket + jetty-ee10-websocket-jetty-server + ${jetty.version} + + + org.eclipse.jetty.websocket + jetty-websocket-jetty-client + ${jetty.version} + + + org.jetbrains + annotations + 24.1.0 + + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs.version} + + + + ${project.groupId} + jitsi-media-transform + ${project.version} + + + ${project.groupId} + rtp + ${project.version} + + + ${project.groupId} + jitsi-dcsctp + 1.0-7-gb548df2 + + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-jdk14 + ${slf4j.version} + + + ${project.groupId} + ice4j + + + ${project.groupId} + jicoco-config + + + ${project.groupId} + jicoco-health-checker + + + ${project.groupId} + jicoco-jetty + + + ${project.groupId} + jicoco-mediajson + + + ${project.groupId} + jicoco-metrics + + + ${project.groupId} + jicoco-mucclient + + + ${project.groupId} + jicoco-tracing + + + ${project.groupId} + jitsi-xmpp-extensions + + + ${project.groupId} + jitsi-utils + + + ${project.groupId} + jitsi-metaconfig + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + + + org.glassfish.jersey.containers + jersey-container-jetty-http + ${jersey.version} + + + org.glassfish.jersey.containers + jersey-container-servlet + ${jersey.version} + + + org.glassfish.jersey.inject + jersey-hk2 + ${jersey.version} + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey.version} + + + com.fasterxml.jackson.module + jackson-module-kotlin + ${jackson.version} + + + + org.jetbrains.kotlin + kotlin-reflect + ${kotlin.version} + + + org.reflections + reflections + 0.9.11 + + + + io.sentry + sentry + 7.20.0 + runtime + + + + io.mockk + mockk-jvm + ${mockk.version} + test + + + org.glassfish.jersey.test-framework + jersey-test-framework-core + ${jersey.version} + test + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-jetty + ${jersey.version} + test + + + junit + junit + + + + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + ${jersey.version} + test + + + junit + junit + + + + + + org.junit.vintage + junit-vintage-engine + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + org.junit.platform + junit-platform-launcher + ${junit.platform.version} + test + + + io.kotest + kotest-runner-junit5-jvm + ${kotest.version} + test + + + io.kotest + kotest-assertions-core-jvm + ${kotest.version} + test + + + ${project.groupId} + jicoco-test-kotlin + test + + + javax.xml.bind + jaxb-api + 2.3.1 + test + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.7.1 + + + assemble-archive + package + + single + + + true + + src/assembly/archive.xml + + ${assembly.skipAssembly} + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + build-fat-jar + + shade + + + none + + ${project.name}-${project.version}-jar-with-dependencies + + + reference.conf + + + + + *:* + + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + exec-exec + + exec + + + java + + -classpath + + -Djava.util.logging.config.file=${project.basedir}/lib/logging.properties + ${exec.mainClass} + + + + + exec-java + + java + + + ${exec.mainClass} + + + + ktlint-check + verify + + exec + + + java + true + + -classpath + + com.pinterest.ktlint:ktlint-cli + + com.pinterest.ktlint.Main + --relative + src/main/kotlin/**/*.kt + src/test/kotlin/**/*.kt + + + + + ktlint-format + + exec + + + java + true + + -classpath + + com.pinterest.ktlint:ktlint-cli + + com.pinterest.ktlint.Main + --format + --relative + src/main/kotlin/**/*.kt + src/test/kotlin/**/*.kt + + + + + + + com.pinterest.ktlint + ktlint-cli + ${ktlint.version} + all + + + + + org.eclipse.jetty.ee10 + jetty-ee10-maven-plugin + ${jetty.version} + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + compile + + compile + + + + ${project.basedir}/src/main/kotlin + ${project.basedir}/src/main/java + + + + + test-compile + test-compile + + test-compile + + + + ${project.basedir}/src/test/kotlin + ${project.basedir}/src/test/java + + + + + + 17 + + -Xallow-result-return-type + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + + default-compile + none + + + + default-testCompile + none + + + java-compile + compile + + compile + + + + java-test-compile + test-compile + + testCompile + + + + + 17 + 17 + + -Xlint:all,-serial + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + ${project.basedir}/lib/logging.properties + true + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${project.basedir}/spotbugs-exclude.xml + true + + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + true + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.6.0 + + checkstyle.xml + + + + com.puppycrawl.tools + checkstyle + 10.17.0 + + + + + + check + + + + + + + + + jitsi-maven-repository-releases + default + Jitsi Maven Repository (Releases) + + true + + + false + + https://github.com/jitsi/jitsi-maven-repository/raw/master/releases/ + + + jitsi-maven-repository-snapshots + default + Jitsi Maven Repository (Snapshots) + + false + + + true + + https://github.com/jitsi/jitsi-maven-repository/raw/master/snapshots/ + + + diff --git a/jvb/resources/jvb.bat b/jvb/resources/jvb.bat new file mode 100644 index 0000000..d1bd259 --- /dev/null +++ b/jvb/resources/jvb.bat @@ -0,0 +1,14 @@ +@echo off + +:begin + +:: needed to overcome weird loop behavior in conjunction with variable expansion +SETLOCAL enabledelayedexpansion + +set mainClass=org.jitsi.videobridge.MainKt +set cp=jitsi-videobridge.jar +FOR %%F IN (lib/*.jar) DO ( + SET cp=!cp!;lib/%%F% +) + +java -Djava.util.logging.config.file=lib/logging.properties -Djdk.tls.ephemeralDHKeySize=2048 -cp %cp% %mainClass% %* diff --git a/jvb/resources/jvb.sh b/jvb/resources/jvb.sh new file mode 100755 index 0000000..fb460a6 --- /dev/null +++ b/jvb/resources/jvb.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" + +mainClass="org.jitsi.videobridge.MainKt" +cp=$SCRIPT_DIR/jitsi-videobridge.jar:$SCRIPT_DIR/lib/* +logging_config="$SCRIPT_DIR/lib/logging.properties" +videobridge_rc="$SCRIPT_DIR/lib/videobridge.rc" + +# if there is a logging config file in lib folder use it (running from source) +if [ -f $logging_config ]; then + LOGGING_CONFIG_PARAM="-Djava.util.logging.config.file=$logging_config" +fi + +if [ -f $videobridge_rc ]; then + source $videobridge_rc +fi + +if [ -z "$VIDEOBRIDGE_MAX_MEMORY" ]; then VIDEOBRIDGE_MAX_MEMORY=3072m; fi +if [ -z "$VIDEOBRIDGE_GC_TYPE" ]; then VIDEOBRIDGE_GC_TYPE=G1GC; fi + +exec java -Xmx$VIDEOBRIDGE_MAX_MEMORY $VIDEOBRIDGE_DEBUG_OPTIONS -XX:+Use$VIDEOBRIDGE_GC_TYPE -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djdk.tls.ephemeralDHKeySize=2048 $LOGGING_CONFIG_PARAM $JAVA_SYS_PROPS -cp $cp $mainClass $@ diff --git a/jvb/spotbugs-exclude.xml b/jvb/spotbugs-exclude.xml new file mode 100644 index 0000000..8f26569 --- /dev/null +++ b/jvb/spotbugs-exclude.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/jvb/src/assembly/archive.xml b/jvb/src/assembly/archive.xml new file mode 100644 index 0000000..3403af6 --- /dev/null +++ b/jvb/src/assembly/archive.xml @@ -0,0 +1,56 @@ + + archive + + zip + + true + ${project.artifactId}-${project.version} + + + lib + runtime + false + false + + + + + ${project.basedir}/target/${project.artifactId}-${project.version}.jar + ${file.separator} + jitsi-videobridge.jar + + + + + ${project.build.directory} + ${file.separator} + + *.jar + + + *-with-dependencies.jar + ${project.artifactId}-${project.version}.jar + + + + ${project.basedir}/lib + lib + + **/*.jar + README + + + + ${project.basedir}/resources/ + ${file.separator} + 755 + + jvb.sh + jvb.bat + + + + diff --git a/jvb/src/main/java/org/jitsi/videobridge/Conference.java b/jvb/src/main/java/org/jitsi/videobridge/Conference.java new file mode 100644 index 0000000..0f7e515 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/Conference.java @@ -0,0 +1,1860 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge; + +import io.opentelemetry.api.trace.*; +import io.opentelemetry.context.*; +import kotlin.*; +import org.jetbrains.annotations.*; +import org.jitsi.mediajson.*; +import org.jitsi.nlj.*; +import org.jitsi.nlj.format.*; +import org.jitsi.nlj.rtp.*; +import org.jitsi.rtp.Packet; +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket; +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.*; +import org.jitsi.rtp.rtp.*; +import org.jitsi.tracing.*; +import org.jitsi.utils.LRUCache; +import org.jitsi.utils.dsi.*; +import org.jitsi.utils.logging.*; +import org.jitsi.utils.logging2.Logger; +import org.jitsi.utils.logging2.LoggerImpl; +import org.jitsi.utils.logging2.*; +import org.jitsi.utils.queue.*; +import org.jitsi.videobridge.colibri2.*; +import org.jitsi.videobridge.export.*; +import org.jitsi.videobridge.message.*; +import org.jitsi.videobridge.metrics.*; +import org.jitsi.videobridge.relay.*; +import org.jitsi.videobridge.util.*; +import org.jitsi.videobridge.xmpp.*; +import org.jitsi.xmpp.extensions.TraceParent; +import org.jitsi.xmpp.extensions.colibri2.*; +import org.jitsi.xmpp.util.*; +import org.jivesoftware.smack.packet.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jxmpp.jid.*; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.regex.*; +import java.util.stream.*; + +import static org.jitsi.xmpp.util.ErrorUtilKt.createError; + +/** + * Represents a conference in the terms of Jitsi Videobridge. + * + * @author Lyubomir Marinov + * @author Boris Grozev + * @author Hristo Terezov + * @author George Politis + */ +public class Conference + implements AbstractEndpointMessageTransport.EndpointMessageTransportEventHandler +{ + /** + * The endpoints participating in this {@link Conference}. Although it's a + * {@link ConcurrentHashMap}, writing to it must be protected by + * synchronizing on the map itself, because it must be kept in sync with + * {@link #endpointsCache}. + */ + private final ConcurrentHashMap endpointsById = new ConcurrentHashMap<>(); + + /** + * A boolean that indicates whether or not to include RTCStats for this call. + */ + private final boolean isRtcStatsEnabled; + + public boolean isRtcStatsEnabled() + { + return isRtcStatsEnabled; + } + + /** + * A read-only cache of the endpoints in this conference. Note that it + * contains only the {@link Endpoint} instances (and not {@link RelayedEndpoint}s). + * This is because the cache was introduced for performance reasons only + * (we iterate over it for each RTP packet) and the relayed endpoints are not always needed. + */ + private List endpointsCache = Collections.emptyList(); + + private final Object endpointsCacheLock = new Object(); + + /** + * A map of the endpoints in this conference, by their ssrcs. + */ + private final ConcurrentHashMap endpointsBySsrc = new ConcurrentHashMap<>(); + + /** + * The relays participating in this conference. + */ + private final ConcurrentHashMap relaysById = new ConcurrentHashMap<>(); + + /** + * The indicator which determines whether {@link #expire()} has been called + * on this Conference. + */ + private final AtomicBoolean expired = new AtomicBoolean(false); + + public long getLocalAudioSsrc() + { + return localAudioSsrc; + } + + public long getLocalVideoSsrc() + { + return localVideoSsrc; + } + + private final long localAudioSsrc = Videobridge.RANDOM.nextLong() & 0xffff_ffffL; + private final long localVideoSsrc = Videobridge.RANDOM.nextLong() & 0xffff_ffffL; + + /** + * The locally unique identifier of this conference (i.e. unique across the + * conferences on this bridge). It is locally generated and exposed via + * colibri, and used by colibri clients to reference the conference. + */ + private final String id; + + /** + * The world readable name of this instance if any. + */ + private final @Nullable EntityBareJid conferenceName; + + /** + * The speech activity (representation) of the Endpoints of this + * Conference. + */ + private final ConferenceSpeechActivity speechActivity; + + /** + * The Videobridge which has initialized this Conference. + */ + private final Videobridge videobridge; + + /** + * The {@link Logger} to be used by this instance to print debug + * information. + */ + private final Logger logger; + + private final Tracer tracer = TracingGlobal.Companion.getSdk().getTracer("org.jitsi.videobridge"); + + /** + * Extracts a remote {@link Span} from the {@code traceparent} extension of an IQ, if present. + * Inlined from jicoco-tracing's {@code TracingUtil}, which was removed because it pulled in + * a Smack dependency that isn't available on Maven Central. + */ + private static @Nullable Span remoteSpanFromIq(IQ iq) + { + TraceParent extension = iq.getExtension(TraceParent.class); + if (extension == null) + { + return null; + } + return Span.wrap(SpanContext.createFromRemoteParent( + extension.getTraceId(), + extension.getParentId(), + TraceFlags.fromHex(extension.getTraceFlags(), 0), + TraceState.getDefault())); + } + + /** + * Extracts a remote {@link Context} from the {@code traceparent} extension of an IQ, if present, + * or the root context otherwise. + */ + private static Context remoteContextFromIq(IQ iq) + { + Context root = Context.root(); + Span span = remoteSpanFromIq(iq); + if (span == null) + { + return root; + } + return root.with(span); + } + + /** + * The time when this {@link Conference} was created. + */ + private final long creationTime = System.currentTimeMillis(); + + /** + * The shim which handles Colibri v2 related logic for this conference (translates colibri v2 signaling into the + * native videobridge APIs). + */ + @NotNull private final Colibri2ConferenceHandler colibri2Handler; + + /** + * The queue handling colibri (both v1 and v2) messages for this conference. + */ + @NotNull private final PacketQueue colibriQueue; + + @NotNull private final EncodingsManager encodingsManager = new EncodingsManager(); + + /** + * Cache here because it's accessed on every packet. + */ + private final boolean routeLoudestOnly = LoudestConfig.getRouteLoudestOnly(); + + /** + * Audio subscription manager for handling endpoint subscriptions. + */ + @NotNull + private final AudioSubscriptionManager audioSubscriptionManager = new AudioSubscriptionManager(); + + /** + * The task of updating the ordered list of endpoints in the conference. It runs periodically in order to adapt to + * endpoints stopping or starting to their video streams (which affects the order). + */ + private ScheduledFuture updateLastNEndpointsFuture; + + @NotNull + private final EndpointConnectionStatusMonitor epConnectionStatusMonitor; + + /** + * A unique meeting ID optionally set by the signaling server ({@code null} if not explicitly set). It is exposed + * via ({@link #getDebugState(DebugStateMode, String)} ()} for outside use. + */ + @Nullable + private final String meetingId; + + @NotNull + private final ExporterWrapper exporter; + + /** + * Source names for which an injected (translated) synthetic source has already been created, so we log its + * creation only once. Cleared when the source is removed, so a re-created source logs again. + */ + private final Set loggedInjectedSources = ConcurrentHashMap.newKeySet(); + + /** + * Tags of received translated-media events that didn't match any synthetic source, so we warn about an + * unrecognized tag only once rather than on every packet. Bounded (an unknown tag is never removed via + * {@link #removeAudioSources}, since it corresponds to no known source): the worst case of eviction is warning + * again for a tag we saw long ago. + */ + private final Set loggedUnknownMediaTags = + Collections.synchronizedSet(LRUCache.lruSet(SYNTHETIC_SOURCE_LOG_CACHE_SIZE, true)); + + /** + * Names of synthetic sources that have been removed (unsubscribed) and not since re-added. Media for these can + * still arrive briefly -- the translator may have sent it before it saw the unsubscribe -- so it's an expected + * transient rather than an unrecognized tag, and is dropped without a warning. Bounded (an entry is removed only + * when a source with the same name is re-added, so uniquely-named sources would otherwise accumulate for the + * conference's lifetime): the worst case of eviction is warning once for late media of a long-removed source. + */ + private final Set removedSyntheticSources = + Collections.synchronizedSet(LRUCache.lruSet(SYNTHETIC_SOURCE_LOG_CACHE_SIZE, true)); + + /** + * The conference's negotiated Opus payload type, resolved lazily by {@link #getOpusPayloadType()} and cached + * (payload types are never rewritten, so it is consistent across all endpoints for the conference's lifetime). + */ + private volatile PayloadType opusPayloadType = null; + + /** + * Whether we have warned about translated media arriving before any Opus payload type is negotiated, so the + * warning is logged once rather than per media event. + */ + private final AtomicBoolean loggedNoOpusPayloadType = new AtomicBoolean(false); + + /** + * A regex pattern to trim UUIDs to just their first 8 hex characters. + */ + private final static Pattern uuidTrimmer = Pattern.compile("(\\p{XDigit}{8})[\\p{XDigit}-]*"); + + /** + * The maximum size of the bounded sets that suppress repeated synthetic-source log messages + * ({@link #loggedUnknownMediaTags}, {@link #removedSyntheticSources}). + */ + private static final int SYNTHETIC_SOURCE_LOG_CACHE_SIZE = 1000; + + /** + * Initializes a new Conference instance which is to represent a + * conference in the terms of Jitsi Videobridge which has a specific + * (unique) ID. + * + * @param videobridge the Videobridge on which the new + * Conference instance is to be initialized + * @param id the (unique) ID of the new instance to be initialized + * @param conferenceName world readable name of this conference + */ + public Conference(Videobridge videobridge, + String id, + @Nullable EntityBareJid conferenceName, + @Nullable String meetingId, + boolean isRtcStatsEnabled) + { + this.meetingId = meetingId; + this.videobridge = Objects.requireNonNull(videobridge, "videobridge"); + this.isRtcStatsEnabled = isRtcStatsEnabled; + Map context = new HashMap<>(Map.of("confId", id)); + if (conferenceName != null) + { + context.put("conf_name", conferenceName.toString()); + } + if (meetingId != null) + { + /* We usually generate meeting IDs as a UUID - include just their first octet. */ + context.put("meeting_id", uuidTrimmer.matcher(meetingId).replaceAll("$1")); + } + + logger = new LoggerImpl(Conference.class.getName(), new LogContext(context)); + exporter = new ExporterWrapper(logger, new ExporterEventHandlerImpl()); + this.id = Objects.requireNonNull(id, "id"); + this.conferenceName = conferenceName; + this.colibri2Handler = new Colibri2ConferenceHandler(this, logger); + colibriQueue = new ColibriQueue( + request -> + { + Span span = tracer.spanBuilder("colibri.request") + .setAttribute("conference.id", request.getRequest().getMeetingId()) + .setAttribute("create", request.getRequest().getCreate()) + .setAttribute("expire", request.getRequest().getExpire()) + .setParent(remoteContextFromIq(request.getRequest())) + .startSpan(); + try (Scope s = span.makeCurrent()) + { + logger.info( () -> { + String reqStr = request.getRequest().toXML().toString(); + if (VideobridgeConfig.getRedactColibriHttpHeaders()) + { + reqStr = RedactColibri.Companion.redactHttpHeaderValues(reqStr); + } + if (VideobridgeConfig.getRedactRemoteAddresses()) + { + reqStr = RedactColibri.Companion.redactIp(reqStr); + } + return "RECV colibri2 request: " + reqStr; + }); + long start = System.currentTimeMillis(); + Pair p = colibri2Handler.handleConferenceModifyIQ(request.getRequest()); + IQ response = p.getFirst(); + boolean expire = p.getSecond(); + long end = System.currentTimeMillis(); + long processingDelay = end - start; + long totalDelay = end - request.getReceiveTime(); + request.getProcessingDelayStats().addDelay(processingDelay); + request.getTotalDelayStats().addDelay(totalDelay); + if (processingDelay > 100) + { + String reqStr = request.getRequest().toXML().toString(); + if (VideobridgeConfig.getRedactColibriHttpHeaders()) + { + reqStr = RedactColibri.Companion.redactHttpHeaderValues(reqStr); + } + if (VideobridgeConfig.getRedactRemoteAddresses()) + { + reqStr = RedactColibri.Companion.redactIp(reqStr); + } + logger.warn("Took " + processingDelay + " ms to process an IQ (total delay " + + totalDelay + " ms): " + reqStr); + } + logger.info("SENT colibri2 response: " + response.toXML()); + request.getCallback().invoke(response); + if (expire) videobridge.expireConference(this); + } + catch (Throwable e) + { + logger.warn("Failed to handle colibri request: ", e); + span.setStatus(StatusCode.ERROR, e.getMessage()); + request.getCallback().invoke( + createError( + request.getRequest(), + StanzaError.Condition.internal_server_error, + e.getMessage())); + } + finally + { + span.end(); + } + return true; + } + ) + { + }; + + speechActivity = new ConferenceSpeechActivity(new SpeechActivityListener()); + updateLastNEndpointsFuture = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(() -> { + try + { + if (speechActivity.updateLastNEndpoints()) + { + lastNEndpointsChangedAsync(); + } + } + catch (Exception e) + { + logger.warn("Failed to update lastN endpoints:", e); + } + + }, 3, 3, TimeUnit.SECONDS); + + VideobridgeMetrics.conferencesCreated.inc(); + epConnectionStatusMonitor = new EndpointConnectionStatusMonitor(this, TaskPools.SCHEDULED_POOL, logger); + epConnectionStatusMonitor.start(); + } + + public void enqueueColibriRequest(XmppConnection.ColibriRequest request) + { + colibriQueue.add(request); + } + + /** + * Creates a new diagnostic context instance that includes the conference + * name and the conference creation time. + * + * @return the new {@link DiagnosticContext} instance. + */ + public DiagnosticContext newDiagnosticContext() + { + if (conferenceName != null) + { + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("conf_name", conferenceName.toString()); + diagnosticContext.put("conf_creation_time_ms", creationTime); + return diagnosticContext; + } + else + { + return new NoOpDiagnosticContext(); + } + } + + /** + * Sends a message to a subset of endpoints in the call, primary use + * case being a message that has originated from an endpoint (as opposed to + * a message originating from the bridge and being sent to all endpoints in + * the call, for that see {@link #broadcastMessage(BridgeChannelMessage)}. + * + * @param msg the message to be sent + * @param endpoints the list of Endpoints to which the message will + * be sent. + */ + public void sendMessage(BridgeChannelMessage msg, List endpoints, boolean sendToRelays) + { + for (Endpoint endpoint : endpoints) + { + endpoint.sendMessage(msg); + } + + if (sendToRelays) + { + for (Relay relay: relaysById.values()) + { + relay.sendMessage(msg); + } + } + } + + /** + * Sends a message that originated from a relay, forwarding it to local endpoints + * (if {@param sendToEndpoints} is true) and to those relays with different mesh-id values than the source relay + * + * @param msg the message to be sent + * @param meshId the ID of the mesh from which the message was received. + */ + public void sendMessageFromRelay(BridgeChannelMessage msg, boolean sendToEndpoints, @Nullable String meshId) + { + if (sendToEndpoints) + { + for (Endpoint endpoint : getLocalEndpoints()) + { + endpoint.sendMessage(msg); + } + } + + for (Relay relay: relaysById.values()) + { + if (!Objects.equals(meshId, relay.getMeshId())) + { + relay.sendMessage(msg); + } + } + } + + /** + * Broadcasts a string message to all endpoints of the conference. + * + * @param msg the message to be broadcast. + */ + public void broadcastMessage(BridgeChannelMessage msg, boolean sendToRelays) + { + sendMessage(msg, getLocalEndpoints(), sendToRelays); + } + + /** + * Broadcasts a string message to all endpoints of the conference. + * + * @param msg the message to be broadcast. + */ + public void broadcastMessage(BridgeChannelMessage msg) + { + broadcastMessage(msg, false); + } + + /** + * Requests a keyframe from the endpoint with the specified id, if the + * endpoint is found in the conference. + * + * @param requesterID the id of the endpoint requesting a keyframe + * @param endpointID the id of the endpoint to request a keyframe from. + * @param mediaSsrc the primary SSRC of the source for which to request a keyframe + */ + public void requestKeyframe(String requesterID, String endpointID, long mediaSsrc) + { + AbstractEndpoint remoteEndpoint = getEndpoint(endpointID); + + if (remoteEndpoint != null) + { + remoteEndpoint.requestKeyframe(requesterID, mediaSsrc); + } + else if (logger.isDebugEnabled()) + { + logger.debug("Cannot request keyframe because the endpoint was not found."); + } + } + + /** + * Handles a colibri2 {@link ConferenceModifyIQ} for this conference. Exposed for the cases when it needs to be + * done synchronously (as opposed as by {@link #colibriQueue}). + * @return the response as an IQ, either an {@link ErrorIQ} or a {@link ConferenceModifiedIQ}. + */ + IQ handleConferenceModifyIQ(ConferenceModifyIQ conferenceModifyIQ) + { + Pair p = colibri2Handler.handleConferenceModifyIQ(conferenceModifyIQ); + if (p.getSecond()) + { + videobridge.expireConference(this); + } + return p.getFirst(); + } + + /** + * Runs {@link #lastNEndpointsChanged()} in an IO pool thread. + */ + private void lastNEndpointsChangedAsync() + { + TaskPools.IO_POOL.execute(() -> + { + try + { + lastNEndpointsChanged(); + } + catch (Exception e) + { + logger.warn("Failed to handle change in last N endpoints: ", e); + } + }); + } + + /** + * Updates all endpoints with a new list of ordered endpoints in the conference. + */ + private void lastNEndpointsChanged() + { + endpointsCache.forEach(Endpoint::lastNEndpointsChanged); + } + + /** + * Notifies this instance that {@link #speechActivity} has identified a speaker switch event and there is now a new + * dominant speaker. + * @param recentSpeakers the list of recent speakers (including the dominant speaker at index 0). + */ + private void recentSpeakersChanged( + List recentSpeakers, + boolean dominantSpeakerChanged, + boolean silence) + { + if (!recentSpeakers.isEmpty()) + { + List recentSpeakersIds + = recentSpeakers.stream().map(AbstractEndpoint::getId).collect(Collectors.toList()); + logger.info("Recent speakers changed: " + recentSpeakersIds + ", dominant speaker changed: " + + dominantSpeakerChanged + " silence:" + silence); + broadcastMessage(new DominantSpeakerMessage(recentSpeakersIds, silence)); + + if (dominantSpeakerChanged && !silence) + { + VideobridgeMetrics.dominantSpeakerChanges.inc(); + if (getEndpointCount() > 2) + { + maybeSendKeyframeRequest(recentSpeakers.get(0)); + } + } + } + } + + /** + * Schedules sending a pre-emptive keyframe request (if necessary) when a new dominant speaker is elected. + * @param dominantSpeaker the new dominant speaker. + */ + private void maybeSendKeyframeRequest(AbstractEndpoint dominantSpeaker) + { + if (dominantSpeaker == null) + { + return; + } + + boolean anyEndpointInStageView = false; + Set allOnStageSourceNames = new HashSet<>(); + for (Endpoint otherEndpoint : getLocalEndpoints()) + { + if (otherEndpoint != dominantSpeaker) + { + allOnStageSourceNames.addAll(otherEndpoint.getOnStageSources()); + } + } + + for (String onStageSourceName : allOnStageSourceNames) + { + AbstractEndpoint owner = findSourceOwner(onStageSourceName); + if (owner != null) + { + // Do not anticipate a switch if all on-stage sources are DESKTOP + MediaSourceDesc onStageSource = owner.findMediaSourceDesc(onStageSourceName); + if (onStageSource != null && onStageSource.getVideoType() == VideoType.CAMERA) + { + anyEndpointInStageView = true; + break; + } + } + } + + if (!anyEndpointInStageView) + { + // If all other endpoints are in tile view, there is no switch to anticipate. Don't trigger an unnecessary + // keyframe. + VideobridgeMetrics.preemptiveKeyframeRequestsSuppressed.inc(); + return; + } + VideobridgeMetrics.preemptiveKeyframeRequestsSent.inc(); + + double senderRtt = getRtt(dominantSpeaker); + double maxReceiveRtt = getMaxReceiverRtt(dominantSpeaker.getId()); + // We add an additional 10ms delay to reduce the risk of the keyframe arriving too early + double keyframeDelay = maxReceiveRtt - senderRtt + 10; + if (logger.isDebugEnabled()) + { + logger.debug("Scheduling keyframe request from " + dominantSpeaker.getId() + " after a delay" + + " of " + keyframeDelay + "ms"); + } + TaskPools.SCHEDULED_POOL.schedule( + (Runnable)dominantSpeaker::requestKeyframe, + (long)keyframeDelay, + TimeUnit.MILLISECONDS + ); + } + + private double getRtt(AbstractEndpoint endpoint) + { + if (endpoint instanceof Endpoint) + { + Endpoint localDominantSpeaker = (Endpoint)endpoint; + return localDominantSpeaker.getRtt(); + } + else + { + // This is a RelayedEndpoint + // TODO(brian): we don't currently have a way to get the RTT from this bridge to a remote endpoint, so we + // hard-code a value here. We could relatively easily get the RTT to the remote relay. Should we take that + // into account? + return 100; + } + } + + private double getMaxReceiverRtt(String excludedEndpointId) + { + return endpointsCache.stream() + .filter(ep -> !ep.getId().equalsIgnoreCase(excludedEndpointId)) + .map(Endpoint::getRtt) + .mapToDouble(Double::valueOf) + .max() + .orElse(0); + } + + /** + * Expires this Conference, its Contents and their + * respective Channels. Releases the resources acquired by this + * instance throughout its life time and prepares it to be garbage + * collected. + *

+ * NOTE: this should _only_ be called by the Conference "manager" (in this + * case, Videobridge). If you need to expire a Conference from elsewhere, use + * {@link Videobridge#expireConference(Conference)} + */ + void expire() + { + if (!expired.compareAndSet(false, true)) + { + return; + } + + logger.info("Expiring."); + + colibriQueue.close(); + + epConnectionStatusMonitor.stop(); + + if (updateLastNEndpointsFuture != null) + { + updateLastNEndpointsFuture.cancel(true); + updateLastNEndpointsFuture = null; + } + + logger.debug(() -> "Expiring endpoints."); + getEndpoints().forEach(AbstractEndpoint::expire); + getRelays().forEach(Relay::expire); + exporter.stop(); + speechActivity.expire(); + + updateStatisticsOnExpire(); + } + + /** + * Updates the statistics for this conference when it is about to expire. + */ + private void updateStatisticsOnExpire() + { + long durationSeconds = Math.round((System.currentTimeMillis() - creationTime) / 1000d); + + VideobridgeMetrics.conferencesCompleted.inc(); + VideobridgeMetrics.totalConferenceSeconds.add(durationSeconds); + + logger.info("expire_conf,duration=" + durationSeconds); + } + + /** + * Finds an Endpoint of this Conference which sends an RTP + * stream with a specific SSRC. + * + * @param receiveSSRC the SSRC of an RTP stream received by this + * Conference whose sending Endpoint is to be found + * @return Endpoint of this Conference which sends an RTP + * stream with the specified ssrc; otherwise, null + */ + AbstractEndpoint findEndpointByReceiveSSRC(long receiveSSRC) + { + return getEndpoints().stream() + .filter(ep -> ep.receivesSsrc(receiveSSRC)) + .findFirst() + .orElse(null); + } + + /** + * Gets an Endpoint participating in this Conference which + * has a specific identifier/ID. + * + * @param id the identifier/ID of the Endpoint which is to be + * returned + * @return an Endpoint participating in this Conference + * which has the specified id or null + */ + @Nullable + public AbstractEndpoint getEndpoint(@NotNull String id) + { + return endpointsById.get(Objects.requireNonNull(id, "id must be non null")); + } + + @Nullable + public Relay getRelay(@NotNull String id) + { + return relaysById.get(id); + } + + @Nullable + public AbstractEndpoint findSourceOwner(@NotNull String sourceName) + { + for (AbstractEndpoint e : endpointsById.values()) + { + if (e.findMediaSourceDesc(sourceName) != null) + { + return e; + } + } + + return null; + } + + /** + * Initializes a new Endpoint instance with the specified + * id and adds it to the list of Endpoints participating + * in this Conference. + * @param id the identifier/ID of the Endpoint which will be + * created + * @param iceControlling {@code true} if the ICE agent of this endpoint's + * transport will be initialized to serve as a controlling ICE agent; + * otherwise, {@code false} + * @param doSsrcRewriting whether this endpoint signaled SSRC rewriting support. + * @param diarize whether diarization is requested for this endpoint's audio (colibri2 {@code diarize} attribute). + * @return an Endpoint participating in this Conference + */ + @NotNull + public Endpoint createLocalEndpoint( + String id, + boolean iceControlling, + boolean doSsrcRewriting, + boolean doMidDemux, + boolean visitor, + boolean privateAddresses, + boolean diarize) + { + final AbstractEndpoint existingEndpoint = getEndpoint(id); + if (existingEndpoint != null) + { + throw new IllegalArgumentException("Local endpoint with ID = " + id + "already created"); + } + + final Endpoint endpoint = new Endpoint( + id, this, logger, iceControlling, doSsrcRewriting, doMidDemux, visitor, privateAddresses, diarize); + videobridge.localEndpointCreated(visitor); + + endpoint.addEventHandler(() -> endpointSourcesChanged(endpoint)); + + addEndpoints(Collections.singleton(endpoint)); + + return endpoint; + } + + @NotNull + public Relay createRelay(String id, @Nullable String meshId, boolean iceControlling, boolean useUniquePort) + { + final Relay existingRelay = getRelay(id); + if (existingRelay != null) + { + throw new IllegalArgumentException("Relay with ID = " + id + "already created"); + } + + final Relay relay = new Relay(id, this, logger, meshId, iceControlling, useUniquePort); + + relaysById.put(id, relay); + + return relay; + } + + /** + * One or more endpoints was added or removed. + * @param includesNonVisitors Whether any of the endpoints changed was not a visitor. + */ + private void endpointsChanged(boolean includesNonVisitors) + { + if (includesNonVisitors) + { + speechActivity.endpointsChanged(getNonVisitorEndpoints()); + } + } + + /** + * The media sources of one of the endpoints in this conference changed. + * + * @param endpoint the endpoint + */ + private void endpointSourcesChanged(@NotNull Endpoint endpoint) + { + // Force an update to be propagated to each endpoint's bitrate controller. + // We run this async because this method is called in the Colibri critical path. + lastNEndpointsChangedAsync(); + } + + /** + * Updates {@link #endpointsCache} with the current contents of + * {@link #endpointsById}. + */ + private void updateEndpointsCache() + { + synchronized (endpointsCacheLock) + { + ArrayList endpointsList = new ArrayList<>(endpointsById.size()); + endpointsById.values().forEach(e -> + { + if (e instanceof Endpoint) + { + endpointsList.add((Endpoint) e); + } + }); + + endpointsCache = Collections.unmodifiableList(endpointsList); + } + } + + /** + * Returns the number of local AND remote endpoints in this {@link Conference}. + * + * @return the number of local AND remote endpoints in this {@link Conference}. + */ + public int getEndpointCount() + { + return endpointsById.size(); + } + + /** + * @return the number of relays. + */ + public int getRelayCount() + { + return relaysById.size(); + } + + /** + * Returns the number of local endpoints in this {@link Conference}. + * + * @return the number of local endpoints in this {@link Conference}. + */ + public int getLocalEndpointCount() + { + return getLocalEndpoints().size(); + } + + /** + * Gets the Endpoints participating in/contributing to this + * Conference. + * + * @return the Endpoints participating in/contributing to this + * Conference + */ + public List getEndpoints() + { + return new ArrayList<>(this.endpointsById.values()); + } + + /** + * Gets the Endpoints participating in this conference that are not visitors. + */ + public List getNonVisitorEndpoints() + { + return this.endpointsById.values().stream().filter(ep -> !ep.getVisitor()).collect(Collectors.toList()); + } + + List getOrderedEndpoints() + { + return speechActivity.getOrderedEndpoints(); + } + + /** + * Gets the list of endpoints which are local to this bridge (as opposed to {@link RelayedEndpoint}s connected to + * one of our {@link Relay}s). + */ + public List getLocalEndpoints() + { + return endpointsCache; + } + + /** + * Gets the list of the relays this conference is using. + */ + public List getRelays() + { + return new ArrayList<>(this.relaysById.values()); + } + + /** + * Gets the (unique) identifier/ID of this instance. + * + * @return the (unique) identifier/ID of this instance + */ + public final String getID() + { + return id; + } + + /** + * Gets the signaling-server-defined meetingId of this conference, if set. + * + * @return The signaling-server-defined meetingId of this conference, or null. + */ + public final @Nullable String getMeetingId() + { + return meetingId; + } + + /** + * Gets an Endpoint participating in this Conference which + * has a specific identifier/ID. + * + * @param id the identifier/ID of the Endpoint which is to be + * returned + * @return an Endpoint participating in this Conference + * or {@code null} if endpoint with specified ID does not exist in a + * conference + */ + @Nullable + public Endpoint getLocalEndpoint(String id) + { + AbstractEndpoint endpoint = getEndpoint(id); + if (endpoint instanceof Endpoint) + { + return (Endpoint) endpoint; + } + + return null; + } + + /** + * Gets the speech activity (representation) of the Endpoints of + * this Conference. + * + * @return the speech activity (representation) of the Endpoints of + * this Conference + */ + public ConferenceSpeechActivity getSpeechActivity() + { + return speechActivity; + } + + /** + * Gets the Videobridge which has initialized this + * Conference. + * + * @return the Videobridge which has initialized this + * Conference + */ + public final Videobridge getVideobridge() + { + return videobridge; + } + + /** + * Gets the indicator which determines whether this Conference has + * expired. + * + * @return true if this Conference has expired; otherwise, + * false + */ + public boolean isExpired() + { + return expired.get(); + } + + /** + * Notifies this conference that one of its endpoints has expired. + * + * @param endpoint the Endpoint which expired. + */ + public void endpointExpired(AbstractEndpoint endpoint) + { + final AbstractEndpoint removedEndpoint; + String id = endpoint.getId(); + removedEndpoint = endpointsById.remove(id); + if (removedEndpoint == null) + { + logger.warn("No endpoint found, id=" + id); + return; + } + + if (removedEndpoint instanceof Endpoint) + { + // The removed endpoint was a local Endpoint as opposed to a RelayedEndpoint. + updateEndpointsCache(); + endpointsById.values().forEach(e -> e.otherEndpointExpired(removedEndpoint)); + videobridge.localEndpointExpired(removedEndpoint.getVisitor()); + } + + relaysById.forEach((i, relay) -> relay.endpointExpired(id)); + endpoint.getSsrcs().forEach(ssrc -> { + endpointsBySsrc.remove(ssrc, endpoint); + }); + audioSubscriptionManager.removeEndpoint(endpoint.getId()); + audioSubscriptionManager.removeSources(new HashSet<>(endpoint.getAudioSources())); + endpointsChanged(removedEndpoint.getVisitor()); + } + + /** + * Notifies this conference that one of its relays has expired. + * + * @param relay the Relay which expired. + */ + public void relayExpired(Relay relay) + { + String id = relay.getId(); + relaysById.remove(id); + + getLocalEndpoints().forEach(senderEndpoint -> senderEndpoint.removeReceiver(id)); + } + + /** + * Adds a set of {@link AbstractEndpoint} instances to the list of endpoints in this conference. + */ + public void addEndpoints(Set endpoints) + { + endpoints.forEach(endpoint -> { + if (endpoint.getConference() != this) + { + throw new IllegalArgumentException("Endpoint belong to other " + + "conference = " + endpoint.getConference()); + } + AbstractEndpoint replacedEndpoint = endpointsById.put(endpoint.getId(), endpoint); + if (replacedEndpoint != null) + { + logger.info("Endpoint with id " + endpoint.getId() + ": " + + replacedEndpoint + " has been replaced by new " + + "endpoint with same id: " + endpoint); + } + }); + + updateEndpointsCache(); + + boolean hasNonVisitor = endpoints.stream().anyMatch(endpoint -> !endpoint.getVisitor()); + + endpointsChanged(hasNonVisitor); + } + + /** + * Notifies this {@link Conference} that one of its endpoints' + * transport channel has become available. + * + * @param abstractEndpoint the endpoint whose transport channel has become + * available. + */ + @Override + public void endpointMessageTransportConnected(@NotNull AbstractEndpoint abstractEndpoint) + { + /* TODO: this is only called for actual Endpoints, but the API needs cleaning up. */ + Endpoint endpoint = (Endpoint)abstractEndpoint; + + epConnectionStatusMonitor.endpointConnected(endpoint.getId()); + + if (!isExpired()) + { + List recentSpeakers = speechActivity.getRecentSpeakers(); + + if (!recentSpeakers.isEmpty()) + { + endpoint.sendMessage(new DominantSpeakerMessage(recentSpeakers, speechActivity.isInSilence())); + } + } + } + + public AbstractEndpoint getEndpointBySsrc(long ssrc) + { + return endpointsBySsrc.get(ssrc); + } + + /** + * Whether the packet belongs to a synthetic source (bridge-generated audio, e.g. translated audio). Such packets + * are exempt from echo-suppression in {@link #sendOut}: a synthetic source is delivered purely by audio + * subscription, including to its owner endpoint, on every bridge. + */ + private boolean isSyntheticSource(PacketInfo packetInfo) + { + Packet packet = packetInfo.getPacket(); + return packet instanceof RtpPacket + && audioSubscriptionManager.isSynthetic(((RtpPacket) packet).getSsrc()); + } + + public void addEndpointSsrc(@NotNull AbstractEndpoint endpoint, long ssrc) + { + AbstractEndpoint oldEndpoint = endpointsBySsrc.put(ssrc, endpoint); + if (oldEndpoint != null && oldEndpoint != endpoint) + { + logger.warn("SSRC " + ssrc + " moved from ep " + oldEndpoint.getId() + " to ep " + endpoint.getId()); + } + } + + /** + * Gets the conference name. + * + * @return the conference name + */ + public @Nullable EntityBareJid getName() + { + return conferenceName; + } + + /** + * @return the {@link Logger} used by this instance. + */ + public Logger getLogger() + { + return logger; + } + + public void addAudioSources(Set audioSources) + { + audioSubscriptionManager.onSourcesAdded(audioSources); + audioSources.forEach(source -> removedSyntheticSources.remove(source.getSourceName())); + } + + public void removeAudioSources(Set audioSources) + { + audioSubscriptionManager.removeSources(audioSources); + audioSources.forEach(source -> + { + loggedInjectedSources.remove(source.getSourceName()); + loggedUnknownMediaTags.remove(source.getSourceName()); + if (source.getSynthetic()) + { + removedSyntheticSources.add(source.getSourceName()); + } + }); + } + + /** + * Sets the audio subscription for a given endpoint. + * @param endpointId the ID of the endpoint + * @param subscription the audio subscription message + */ + public void setEndpointAudioSubscription(String endpointId, ReceiverAudioSubscriptionMessage subscription) + { + audioSubscriptionManager.setEndpointAudioSubscription(endpointId, subscription); + } + + /** + * Checks if audio from a given SSRC is wanted by a specific endpoint. + * @param endpointId the ID of the endpoint + * @param ssrc the SSRC to check + * @return true if the audio is wanted, false otherwise + */ + public boolean isEndpointAudioWanted(String endpointId, long ssrc) + { + return audioSubscriptionManager.isEndpointAudioWanted(endpointId, ssrc); + } + + /** + * @return {@code true} if this {@link Conference} is in more than one relay mesh. + */ + private boolean inMultipleMeshes() + { + return relaysById.values().stream().map(Relay::getMeshId).collect(Collectors.toSet()).size() > 1; + } + + /** + * @return {@code true} if this {@link Conference} is ready to be expired. + */ + public boolean shouldExpire() + { + // Allow a conference to have no endpoints in the first 20 seconds. + return getEndpointCount() == 0 + && !inMultipleMeshes() + && (System.currentTimeMillis() - creationTime > 20000); + } + + /** + * Determine whether a given endpointId is currently a ranked speaker, if + * speaker ranking is currently enabled. + */ + public boolean isRankedSpeaker(AbstractEndpoint ep) + { + if (!LoudestConfig.Companion.getRouteLoudestOnly()) + { + // When "route loudest only" is disabled all speakers should be considered "ranked" (we forward all audio + // and stats). + return true; + } + return speechActivity.isAmongLoudest(ep.getId()); + } + + /** + * Broadcasts the packet to all endpoints and tentacles that want it. + * + * @param packetInfo the packet + */ + private void sendOut(PacketInfo packetInfo) + { + String sourceEndpointId = packetInfo.getEndpointId(); + // We want to avoid calling 'clone' for the last receiver of this packet + // since it's unnecessary. To do so, we'll wait before we clone and send + // to an interested handler until after we've determined another handler + // is also interested in the packet. We'll give the last handler the + // original packet (without cloning). + PotentialPacketHandler prevHandler = null; + + boolean syntheticSource = isSyntheticSource(packetInfo); + for (Endpoint endpoint : endpointsCache) + { + // Skip echoing a packet back to its source endpoint -- except for synthetic sources, which are + // bridge-generated (not the endpoint's own live audio) and whose delivery, including to their owner, is + // governed entirely by audio subscriptions. This holds whether the synthetic audio was injected locally + // or received over a relay (where the relay attributes the packet to the owner endpoint). + if (!syntheticSource && endpoint.getId().equals(sourceEndpointId)) + { + continue; + } + + if (endpoint.wants(packetInfo)) + { + if (prevHandler != null) + { + prevHandler.send(packetInfo.clone()); + } + prevHandler = endpoint; + } + } + for (Relay relay: relaysById.values()) + { + if (relay.wants(packetInfo)) + { + if (prevHandler != null) + { + prevHandler.send(packetInfo.clone()); + } + prevHandler = relay; + } + } + for (PotentialPacketHandler exporterHandler : exporter.getPacketHandlers()) + { + // Never export synthetic (bridge-generated) audio, e.g. don't feed translated audio back to the + // translator that produced it, or to a transcriber/recorder that exports all audio (empty exports list). + if (!syntheticSource && exporterHandler.wants(packetInfo)) + { + if (prevHandler != null) + { + prevHandler.send(packetInfo.clone()); + } + prevHandler = exporterHandler; + } + } + + if (prevHandler != null) + { + prevHandler.send(packetInfo); + } + else + { + // No one wanted the packet, so the buffer is now free! + ByteBufferPool.returnBuffer(packetInfo.getPacket().getBuffer()); + } + } + + public void applyConnects(List connects) + { + exporter.applyConnects(connects); + } + + public boolean hasRelays() + { + return !relaysById.isEmpty(); + } + + /** + * Handles an RTP/RTCP packet coming from a specific endpoint. + */ + public void handleIncomingPacket(PacketInfo packetInfo) + { + Packet packet = packetInfo.getPacket(); + if (packet instanceof RtpPacket) + { + // This is identical to the default 'else' below, but it defined + // because the vast majority of packet will follow this path. + sendOut(packetInfo); + } + else if (packet instanceof RtcpFbPliPacket || packet instanceof RtcpFbFirPacket) + { + AbstractEndpoint targetEndpoint = null; + boolean rewriter = false; + + long mediaSsrc = (packet instanceof RtcpFbPliPacket) + ? ((RtcpFbPliPacket) packet).getMediaSourceSsrc() + : ((RtcpFbFirPacket) packet).getMediaSenderSsrc(); + + /* If we are rewriting SSRCs to this endpoint, we must ask + it to convert back the SSRC to the media sender's SSRC. */ + String endpointId = packetInfo.getEndpointId(); + if (endpointId != null) + { + AbstractEndpoint ep = getEndpoint(endpointId); + if (ep instanceof Endpoint && ((Endpoint) ep).doesSsrcRewriting()) + { + rewriter = true; + String owner = ((Endpoint) ep).unmapRtcpFbSsrc((RtcpFbPacket) packet); + if (owner != null) + targetEndpoint = getEndpoint(owner); + } + } + + if (!rewriter) + { + // XXX we could make this faster with a map + targetEndpoint = findEndpointByReceiveSSRC(mediaSsrc); + } + + PotentialPacketHandler pph = null; + if (targetEndpoint instanceof Endpoint) + { + pph = (Endpoint) targetEndpoint; + } + else if (targetEndpoint instanceof RelayedEndpoint) + { + pph = ((RelayedEndpoint)targetEndpoint).getRelay(); + } + + // This is not a redundant check. With Octo and 3 or more bridges, + // some PLI or FIR will come from Octo but the target endpoint will + // also be Octo. We need to filter these out. + // TODO: does this still apply with Relays? + if (pph == null) + { + if (logger.isDebugEnabled()) + { + logger.debug("Dropping FIR/PLI for media ssrc " + mediaSsrc); + } + } + else if (pph.wants(packetInfo)) + { + pph.send(packetInfo); + } + } + else + { + sendOut(packetInfo); + } + } + + /** + * Process a new audio level received from an endpoint. + * + * @param endpoint the endpoint for which a new audio level was received + * @param ssrc the SSRC of the source for which the audio level was received + * @param level the new audio level which was received + * @return Whether the packet providing this audio level should be dropped, according + * to the current audio filtering configuration. + */ + public boolean levelChanged(@NotNull AbstractEndpoint endpoint, long ssrc, long level) + { + // Find the specific source this level belongs to. An endpoint may have more than one audio source (e.g. a + // real source plus a synthetic translated one), so we can't assume a single source per endpoint. The lookup + // is O(1) (indexed by SSRC) since this runs for every audio packet. + AudioSourceDesc source = endpoint.getAudioSource(ssrc); + if (source == null) + { + // The per-SSRC index is refreshed just after the endpoint's audioSources list, so a level packet arriving + // during a source update can miss the index while the list already has it. Fall back to a scan so we + // still classify the source (its synthetic flag and name gate the decisions below); missing it would + // wrongly subject a synthetic or explicitly-subscribed source to loudest-only filtering. + for (AudioSourceDesc s : endpoint.getAudioSources()) + { + if (s.getSsrc() == ssrc) + { + source = s; + break; + } + } + } + + // Synthetic sources (e.g. bridge-generated translated audio) don't participate in speech activity / + // loudest-speaker selection, and must not be dropped by loudest-only filtering -- they're still forwarded + // to relays and to endpoints that explicitly subscribe to them. + if (source != null && source.getSynthetic()) + { + return false; + } + + SpeakerRanking ranking = speechActivity.levelChanged(endpoint, level); + if (ranking == null || !routeLoudestOnly) + return false; + if (ranking.isDominant && LoudestConfig.Companion.getAlwaysRouteDominant()) + return false; + if (ranking.energyRanking < LoudestConfig.Companion.getNumLoudest()) + return false; + // return false if the source is subscribed with an "Include" subscription by any other endpoint. + if (source != null && audioSubscriptionManager.isExplicitlySubscribed(source.getSourceName())) + { + return false; + } + VideobridgeMetrics.tossedPacketsEnergy.getHistogram().observe(ranking.energyScore); + return true; + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + * + * @param mode determines how much detail to include in the debug state. + * @param endpointId the ID of the endpoint to include. If set to {@code null}, all endpoints will be included. + */ + public ObjectNode getDebugState(@NotNull DebugStateMode mode, @Nullable String endpointId) + { + if (mode == DebugStateMode.STATS && !isRtcStatsEnabled) + { + return JsonNodeFactory.instance.objectNode(); + } + + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + debugState.put("id", id); + + // Keep this camelCase for compatibility with jvb-rtcstats-push + debugState.put("rtcstatsEnabled", isRtcStatsEnabled); + + if (conferenceName != null) + { + debugState.put("name", conferenceName.toString()); + } + if (meetingId != null) + { + debugState.put("meeting_id", meetingId); + } + + if (mode == DebugStateMode.FULL) + { + debugState.put("expired", expired.get()); + debugState.put("creation_time", creationTime); + } + if (mode == DebugStateMode.FULL || mode == DebugStateMode.STATS) + { + debugState.set("speech_activity", speechActivity.getDebugState(mode)); + } + + ObjectNode endpoints = JsonNodeFactory.instance.objectNode(); + debugState.set("endpoints", endpoints); + for (Endpoint e : endpointsCache) + { + if (endpointId == null || endpointId.equals(e.getId())) + { + if (mode == DebugStateMode.SHORT) + { + endpoints.put(e.getId(), e.getStatsId()); + } + else + { + endpoints.set(e.getId(), e.debugState(mode)); + } + } + } + + ObjectNode relays = JsonNodeFactory.instance.objectNode(); + debugState.set("relays", relays); + for (Relay r : relaysById.values()) + { + if (mode == DebugStateMode.SHORT) + { + relays.put(r.getId(), r.getId()); + } + else + { + relays.set(r.getId(), r.debugState(mode)); + } + } + + if (mode != DebugStateMode.SHORT) + { + debugState.set("exporter", exporter.debugState()); + debugState.set("audio_subscriptions", audioSubscriptionManager.debugState()); + } + + return debugState; + } + + /** + * Whether this looks like a conference in which the two endpoints are + * using a peer-to-peer connection (i.e. none of them are sending audio + * or video). + * This has false positives when e.g. an endpoint doesn't support p2p + * (firefox) and both are audio/video muted. + */ + public boolean isP2p() + { + return isInactive() && getEndpointCount() == 2; + } + + /** + * Whether the conference is inactive, in the sense that none of its + * endpoints are sending audio or video. + */ + public boolean isInactive() + { + return getEndpoints().stream().noneMatch(e -> e.isSendingAudio() || e.isSendingVideo()); + } + + @NotNull public EncodingsManager getEncodingsManager() + { + return encodingsManager; + } + + /** + * The conference's implementation of the {@link ExporterEventHandler} callbacks the {@link ExporterWrapper} + * invokes: sinks for the events a peer sends back over an export websocket, plus the SSRC lookups used to + * filter and annotate exported audio. A non-static inner class, so the callbacks reach the conference's state + * directly. + */ + private class ExporterEventHandlerImpl implements ExporterEventHandler + { + /** + * Handles a transcription result received from the websocket. + * + * @param event the transcription result + */ + @Override + public void handleTranscriptionResult(TranscriptionResultEvent event) + { + try + { + EndpointMessage endpointMessage = new EndpointMessage(""); + endpointMessage.put("msgPayload", event); + endpointMessage.put("from", "transcriber"); + + broadcastMessage(endpointMessage, true); + } + catch (Exception e) + { + logger.warn("Failed to broadcast transcription message", e); + } + } + + /** + * Handles a translated-audio "media" event received from a translator over the export websocket. The event's + * tag names the synthetic source the translated audio belongs to; we build an Opus RTP packet on that + * source's SSRC and inject it into the conference, where the synthetic-source routing delivers it (only to + * endpoints that explicitly subscribed to that source, and on to relays). + */ + @Override + public void handleMediaEvent(MediaEvent mediaEvent) + { + org.jitsi.mediajson.Media media = mediaEvent.getMedia(); + String sourceName = media.getTag(); + + AudioSourceDesc source = findSyntheticAudioSource(sourceName); + if (source == null) + { + if (removedSyntheticSources.contains(sourceName)) + { + // Expected transient: the translator sent this before it saw our unsubscribe of the source. + logger.debug(() -> "Dropping translated media for removed synthetic source: " + sourceName); + } + else if (loggedUnknownMediaTags.add(sourceName)) + { + logger.warn("Received translated media for unknown synthetic source: " + sourceName); + } + return; + } + + PayloadType opusPayloadType = getOpusPayloadType(); + if (opusPayloadType == null) + { + if (loggedNoOpusPayloadType.compareAndSet(false, true)) + { + logger.warn("Received translated media but no Opus payload type is negotiated; dropping."); + } + return; + } + + if (loggedInjectedSources.add(sourceName)) + { + logger.info("Creating injected synthetic source " + sourceName + " (ssrc " + source.getSsrc() + ")"); + } + + try + { + byte[] payload = java.util.Base64.getDecoder().decode(media.getPayload()); + int headerSize = RtpHeader.FIXED_HEADER_SIZE_BYTES; + int offset = RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET; + // Use a pooled buffer with the standard head and tail room, like other packet-construction sites, so + // downstream stages (header extensions, the SRTP auth tag) don't need to reallocate and the buffer + // round-trips through the pool. + byte[] buffer = ByteBufferPool.getBuffer( + offset + headerSize + payload.length + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET); + // The pooled buffer isn't zeroed, and the setters below don't cover every header field (e.g. the + // padding/extension/CSRC-count and marker bits), so clear the header region first. + Arrays.fill(buffer, offset, offset + headerSize, (byte) 0); + System.arraycopy(payload, 0, buffer, offset + headerSize, payload.length); + + AudioRtpPacket rtpPacket = new AudioRtpPacket(buffer, offset, headerSize + payload.length); + rtpPacket.setVersion(2); + rtpPacket.setPayloadType(opusPayloadType.getPt()); + rtpPacket.setSequenceNumber(media.getChunk()); + // The translator sends a monotonic, unwrapped timestamp, but the RTP timestamp is a 32-bit wrapping + // field. setTimestamp() truncates to 32 bits on the wire, but caches the raw Long, so mask here to + // keep the packet's cached timestamp consistent with its bytes (and the sending-change notification). + rtpPacket.setTimestamp(media.getTimestamp() & 0xFFFFFFFFL); + rtpPacket.setSsrc(source.getSsrc()); + + PacketInfo packetInfo = new PacketInfo(rtpPacket); + packetInfo.setPayloadType(opusPayloadType); + // Deliberately leave the endpoint ID unset: this is bridge-generated audio, not an endpoint's own + // live audio, so there is no self-echo to suppress. Delivery (including to the source's owner) is + // governed entirely by the audio subscriptions, so an endpoint can subscribe to its own synthetic + // source. + + handleIncomingPacket(packetInfo); + } + catch (Exception e) + { + logger.warn("Failed to handle translated media for source " + sourceName, e); + } + } + + /** + * Handles a synthetic source's sending-state change, derived from the {@code start}/{@code stop} mediajson + * events a translator sends to bracket a "talk" of translated audio. Resolves the named synthetic source + * (dropping the change if it isn't a known synthetic source, like {@link #handleMediaEvent}) and broadcasts + * a {@link SyntheticSourceSendingChangeEvent} to the conference's clients (and relays). + * + * @param sourceName the synthetic source whose sending state changed + * @param sending true if the source started sending, false if it stopped + * @param timestamp the RTP timestamp (media clock) at which the change takes effect, on the same timeline as + * the source's media; the low 32 bits are used (RTP timestamps are 32-bit and wrap) + */ + @Override + public void handleSendingChange(String sourceName, boolean sending, long timestamp) + { + AudioSourceDesc source = findSyntheticAudioSource(sourceName); + if (source == null) + { + if (removedSyntheticSources.contains(sourceName)) + { + // Expected transient: the translator sent this before it saw our unsubscribe of the source. + logger.debug(() -> "Dropping sending-change for removed synthetic source: " + sourceName); + } + else if (loggedUnknownMediaTags.add(sourceName)) + { + logger.warn("Received sending-change for unknown synthetic source: " + sourceName); + } + return; + } + + try + { + // The translator sends a monotonic, unwrapped timestamp; the RTP timestamp is 32-bit and wraps, and + // the client (lib-jitsi-meet) validates 0..0xFFFFFFFF, so send the low 32 bits. This also matches the + // wrapped timestamp the client sees on this source's injected media. + long rtpTimestamp = timestamp & 0xFFFFFFFFL; + broadcastMessage(new SyntheticSourceSendingChangeEvent(sourceName, sending, rtpTimestamp), true); + } + catch (Exception e) + { + logger.warn("Failed to broadcast synthetic source sending change for " + sourceName, e); + } + } + + /** + * Resolves an audio SSRC to the name of the source it belongs to, or {@code null} if it is not known. Used to + * filter outbound audio by a connect's exported source names. + */ + @Override + @Nullable + public String getAudioSourceName(long ssrc) + { + AbstractEndpoint endpoint = getEndpointBySsrc(ssrc); + if (endpoint == null) + { + return null; + } + AudioSourceDesc source = endpoint.getAudioSource(ssrc); + return source != null ? source.getSourceName() : null; + } + + /** + * Resolves an audio SSRC to whether diarization is requested for its owning endpoint (the colibri2 + * {@code diarize} attribute), defaulting to {@code false} if the SSRC's endpoint is not known. Used to set + * the {@code diarize} flag on the transcriber start event per source. + */ + @Override + public boolean getDiarize(long ssrc) + { + AbstractEndpoint endpoint = getEndpointBySsrc(ssrc); + return endpoint != null && endpoint.getDiarize(); + } + } + + /** + * Finds the synthetic audio source with the given name, or {@code null} if there is none. + */ + @Nullable + private AudioSourceDesc findSyntheticAudioSource(String sourceName) + { + // Resolve via the subscription manager, which owns the authoritative set of sources under the same lock that + // guards the synthetic-SSRC set used at routing time. Scanning each endpoint's source field here instead + // would read state that is updated out of step with the manager (e.g. a local endpoint updates the manager + // before its field, a relayed endpoint after), so a just-added synthetic source could be missed. + return audioSubscriptionManager.findSyntheticSource(sourceName); + } + + /** + * Returns the Opus payload type negotiated in this conference, or {@code null} if none is found. Payload types + * are never rewritten, so the Opus PT is consistent across all endpoints; the first one found is cached, since + * this runs per injected media event. + */ + @Nullable + private PayloadType getOpusPayloadType() + { + PayloadType cached = opusPayloadType; + if (cached != null) + { + return cached; + } + for (Endpoint endpoint : getLocalEndpoints()) + { + PayloadType payloadType = endpoint.getOpusPayloadType(); + if (payloadType != null) + { + opusPayloadType = payloadType; + return payloadType; + } + } + return null; + } + + /** + * This is a no-op diagnostic context (one that will record nothing) meant + * to disable logging of time-series for health checks. + */ + static class NoOpDiagnosticContext + extends DiagnosticContext + { + @Override + public TimeSeriesPoint makeTimeSeriesPoint(String timeSeriesName, long tsMs) + { + return new NoOpTimeSeriesPoint(); + } + + @Override + public Object put(@NotNull String key, @NotNull Object value) + { + return null; + } + } + + static class NoOpTimeSeriesPoint + extends DiagnosticContext.TimeSeriesPoint + { + public NoOpTimeSeriesPoint() + { + this(Collections.emptyMap()); + } + + public NoOpTimeSeriesPoint(Map m) + { + super(m); + } + + @Override + public Object put(String key, Object value) + { + return null; + } + } + + private class SpeechActivityListener implements ConferenceSpeechActivity.Listener + { + @Override + public void recentSpeakersChanged( + List recentSpeakers, + boolean dominantSpeakerChanged, + boolean silence) + { + Conference.this.recentSpeakersChanged(recentSpeakers, dominantSpeakerChanged, silence); + } + + @Override + public void lastNEndpointsChanged() + { + Conference.this.lastNEndpointsChanged(); + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/ConferenceSpeechActivity.java b/jvb/src/main/java/org/jitsi/videobridge/ConferenceSpeechActivity.java new file mode 100644 index 0000000..c7e96da --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/ConferenceSpeechActivity.java @@ -0,0 +1,448 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; +import org.jitsi.utils.dsi.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.util.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.*; +import java.util.stream.*; + +/** + * Represents the speech activity of the Endpoints in a + * Conference. Identifies the dominant speaker Endpoint in the + * Conference and maintains an ordered list of the Endpoints + * in the Conference sorted by recentness of speaker domination and/or + * speech activity. + * + * @author Lyubomir Marinov + */ +public class ConferenceSpeechActivity +{ + /** + * The Logger used by the ConferenceSpeechActivity class + * and its instances to print debug information. + */ + private final Logger logger; + + /** + * The ActiveSpeakerChangedListener which listens to + * {@link #dominantSpeakerIdentification} about changes in the + * active/dominant speaker in this multipoint conference. + */ + private final ActiveSpeakerChangedListener activeSpeakerChangedListener + = ConferenceSpeechActivity.this::activeSpeakerChanged; + + /** + * The DominantSpeakerIdentification instance which detects/identifies the active/dominant speaker in a + * conference. + */ + private DominantSpeakerIdentification dominantSpeakerIdentification; + + /** + * The listener to be notified when the dominant speaker or endpoint order changes. + */ + private Listener listener; + + /** + * The list of endpoints ordered by speech activity alone with the dominant speaker at the beginning of the list + * i.e. the dominant speaker history. This contains all endpoints in the conference, including those that have + * never been the dominant speaker. Such endpoints are listed after endpoints which have been the dominant speaker, + * but the order among themselves is not specified. + * This is used internally when the list of all endpoints is required (e.g. for bandwidth allocation decisions). + */ + private final List endpointsBySpeechActivity = new ArrayList<>(); + + /** + * The list of endpoints in "LastN" order. That is, endpoints currently sending video are at the top of the list, + * ordered by speech activity, followed by the rest of the endpoints (again in speech activity order). + */ + private @NotNull List endpointsInLastNOrder = new ArrayList<>(); + + /** + * The list of endpoints which have been "dominant speaker" ordered by speech activity (with the current dominant + * speaker at the beginning). This differs from {@link #endpointsBySpeechActivity} in that it does not necessarily + * contain all endpoints in the conference. + * + * This is used when signaling "recent speakers" to endpoints. + * + * We add 1 to the configured number of recent speakers to account for the dominant speaker. + */ + @NotNull + private final RecentSpeakersList recentSpeakers + = new RecentSpeakersList<>(ConferenceSpeechActivityConfig.config.getRecentSpeakersCount() + 1); + + /** + * Whether we're currently in a period of silence. With silence detection enabled we initialize to `true` because + * (the {@link #dominantSpeakerIdentification} will fire an initial "silence" event and we don't want to act on it. + */ + private boolean inSilence = ConferenceSpeechActivityConfig.config.getEnableSilenceDetection(); + + /** + * The Object used to synchronize the access to the state of this + * instance. + */ + private final Object syncRoot = new Object(); + + public ConferenceSpeechActivity(@NotNull Listener listener) + { + this(listener, null); + } + + /** + * Initializes a new ConferenceSpeechActivity instance. + * + * @param listener the listener to be notified when the dominant speaker or endpoint order change. + */ + public ConferenceSpeechActivity(@NotNull Listener listener, Logger parentLogger) + { + this.listener = Objects.requireNonNull(listener, "listener"); + logger = + parentLogger == null ? + new LoggerImpl(ConferenceSpeechActivity.class.getName()) : + parentLogger.createChildLogger(ConferenceSpeechActivity.class.getName()); + + long silenceTimeoutMs = -1; + if (ConferenceSpeechActivityConfig.config.getEnableSilenceDetection()) + { + silenceTimeoutMs = ConferenceSpeechActivityConfig.config.getSilenceDetectionTimeout().toMillis(); + + } + dominantSpeakerIdentification = new DominantSpeakerIdentification<>(silenceTimeoutMs); + + dominantSpeakerIdentification.addActiveSpeakerChangedListener(activeSpeakerChangedListener); + int numLoudestToTrack = LoudestConfig.Companion.getRouteLoudestOnly() ? + LoudestConfig.Companion.getNumLoudest() : 0; + dominantSpeakerIdentification.setLoudestConfig(numLoudestToTrack, + (int)(LoudestConfig.Companion.getEnergyExpireTime().toMillis()), + LoudestConfig.Companion.getEnergyAlphaPct()); + } + + boolean isInSilence() + { + return inSilence; + } + + /** + * Notifies this instance that the underlying {@code dominant speaker identification} has elected a new + * active/dominant speaker. + * + * @param id the ID of the new active/dominant speaker or null if a period of silence began. + */ + protected void activeSpeakerChanged(@Nullable String id) + { + final Listener listener = this.listener; + if (listener == null) + { + return; + } + + logger.trace(() -> "The dominant speaker is now " + id + "."); + + boolean endpointListChanged; + boolean dominantSpeakerChanged; + synchronized (syncRoot) + { + if (id == null) + { + endpointListChanged = false; + dominantSpeakerChanged = false; + if (!inSilence) + { + inSilence = true; + } + } + else + { + dominantSpeakerChanged = true; + if (inSilence) + { + inSilence = false; + } + + AbstractEndpoint endpoint + = endpointsBySpeechActivity.stream() + .filter(e -> id.equals(e.getId())) + .findFirst().orElse(null); + // Move this endpoint to the top of our sorted list + if (!endpointsBySpeechActivity.remove(endpoint)) + { + logger.warn("Got active speaker notification for an unknown endpoint: " + id + ", ignoring"); + return; + } + endpointsBySpeechActivity.add(0, endpoint); + + recentSpeakers.promote(endpoint); + + endpointListChanged = updateLastNEndpoints(); + } + } + + TaskPools.IO_POOL.execute(() -> { + listener.recentSpeakersChanged(recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence); + if (endpointListChanged) + { + listener.lastNEndpointsChanged(); + } + }); + } + + /** + * Re-calculates the list of endpoints in LastN order ({@link #endpointsInLastNOrder}) based on the speech activity + * and video availability. + * + * @return {@code true} if the call resulted in a change of the ordered list of endpoints, and {@code false} + * otherwise. + */ + boolean updateLastNEndpoints() + { + synchronized (syncRoot) + { + Map> byVideoAvailable + = endpointsBySpeechActivity.stream() + .collect(Collectors.groupingBy(AbstractEndpoint::hasVideoAvailable)); + + List newEndpointsInLastNOrder = new ArrayList<>(endpointsBySpeechActivity.size()); + newEndpointsInLastNOrder.addAll(byVideoAvailable.getOrDefault(true, Collections.emptyList())); + newEndpointsInLastNOrder.addAll(byVideoAvailable.getOrDefault(false, Collections.emptyList())); + + if (!newEndpointsInLastNOrder.equals(endpointsInLastNOrder)) + { + endpointsInLastNOrder = Collections.unmodifiableList(newEndpointsInLastNOrder); + return true; + } + + return false; + } + } + + void expire() + { + synchronized (syncRoot) + { + if (dominantSpeakerIdentification != null) + { + dominantSpeakerIdentification.removeActiveSpeakerChangedListener(activeSpeakerChangedListener); + } + this.listener = null; + this.dominantSpeakerIdentification = null; + endpointsBySpeechActivity.clear(); + endpointsInLastNOrder = Collections.emptyList(); + } + } + + /** + * Gets the Endpoint which is the dominant speaker in the + * multipoint conference represented by this instance. + * + * @return the Endpoint which is the dominant speaker in the + * multipoint conference represented by this instance or null + */ + public AbstractEndpoint getDominantEndpoint() + { + synchronized (syncRoot) + { + return recentSpeakers.getRecentSpeakers().stream().findFirst().orElse(null); + } + } + + /** + * Gets a read-only list of endpoints in "LastN" order. + * + * @return a read-only list of endpoints in "LastN" order. + */ + public List getOrderedEndpoints() + { + return endpointsInLastNOrder; + } + + /** + * Get a list of recent speakers (including the current dominant one at the top of the list). + */ + @NotNull + public List getRecentSpeakers() + { + return recentSpeakers.getRecentSpeakers().stream() + .map(AbstractEndpoint::getId).collect(Collectors.toList()); + } + + /** + * Query whether an endpoint is a recent speaker. + */ + public boolean isRecentSpeaker(AbstractEndpoint endpoint) + { + return recentSpeakers.isRecentSpeaker(endpoint); + } + + public boolean isAmongLoudest(String endpointId) + { + return dominantSpeakerIdentification.isAmongLoudest(endpointId); + } + + /** + * Notifies this instance that a new audio level was received or measured by an Endpoint. + * + * @param endpoint the endpoint for which a new audio level was received or measured + * @param level the new audio level which was received or measured + * @return The current energy rank and related data. + */ + public SpeakerRanking levelChanged(@NotNull AbstractEndpoint endpoint, long level) + { + DominantSpeakerIdentification dsi = this.dominantSpeakerIdentification; + return dsi != null ? dsi.levelChanged(endpoint.getId(), (int) level) : null; + } + + /** + * Notifies this instance that the list of endpoints changed. + */ + public void endpointsChanged(List conferenceEndpoints) + { + boolean endpointsListChanged; + boolean dominantSpeakerChanged; + boolean recentSpeakersChanged; + // The list of endpoints may have changed, sync our list to make sure it matches. + synchronized (syncRoot) + { + // Remove any endpoints we have that are no longer in the conference + AbstractEndpoint previousDominantSpeaker + = endpointsBySpeechActivity.isEmpty() ? null : endpointsBySpeechActivity.get(0); + endpointsListChanged = endpointsBySpeechActivity.removeIf(ep -> !conferenceEndpoints.contains(ep)); + recentSpeakersChanged = recentSpeakers.removeAllExcept(conferenceEndpoints); + // Add any endpoints from the conf we don't have to the end of our list + for (AbstractEndpoint conferenceEndpoint : conferenceEndpoints) + { + if (!endpointsBySpeechActivity.contains(conferenceEndpoint)) + { + endpointsBySpeechActivity.add(conferenceEndpoint); + endpointsListChanged = true; + } + } + AbstractEndpoint newDominantSpeaker + = endpointsBySpeechActivity.isEmpty() ? null : endpointsBySpeechActivity.get(0); + dominantSpeakerChanged = !Objects.equals(previousDominantSpeaker, newDominantSpeaker); + recentSpeakersChanged |= dominantSpeakerChanged; + + if (endpointsListChanged) + { + endpointsListChanged = updateLastNEndpoints(); + } + } + + if (recentSpeakersChanged || endpointsListChanged) + { + final boolean finalRecentSpeakersChanged = recentSpeakersChanged; + final boolean finalEndpointsChanged = endpointsListChanged; + final Listener listener = this.listener; + if (listener == null) + { + return; + } + TaskPools.IO_POOL.execute(() -> { + if (finalRecentSpeakersChanged) + { + listener.recentSpeakersChanged( + recentSpeakers.getRecentSpeakers(), dominantSpeakerChanged, inSilence); + } + if (finalEndpointsChanged) + { + listener.lastNEndpointsChanged(); + } + }); + } + } + + public void endpointVideoAvailabilityChanged() + { + boolean endpointsListChanged; + synchronized (syncRoot) + { + endpointsListChanged = updateLastNEndpoints(); + } + if (endpointsListChanged) + { + TaskPools.IO_POOL.execute(() -> { + try + { + listener.lastNEndpointsChanged(); + } + catch (Throwable e) + { + logger.warn("Failed to fire event", e); + } + }); + } + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + public ObjectNode getDebugState(@NotNull DebugStateMode mode) + { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + + AbstractEndpoint dominantEndpoint = getDominantEndpoint(); + debugState.put("dominantEndpoint", dominantEndpoint == null ? "null" : dominantEndpoint.getId()); + if (mode != DebugStateMode.FULL) + { + return debugState; + } + + DominantSpeakerIdentification dsi = this.dominantSpeakerIdentification; + if (dsi != null) + { + debugState.set("dominantSpeakerIdentification", dsi.doGetJSON()); + } + else + { + debugState.putNull("dominantSpeakerIdentification"); + } + synchronized (syncRoot) + { + debugState.set("endpointsBySpeechActivity", + mapper.valueToTree(endpointsBySpeechActivity.stream().map(AbstractEndpoint::getId) + .collect(Collectors.toList()))); + debugState.set("endpointsInLastNOrder", + mapper.valueToTree(endpointsInLastNOrder.stream().map(AbstractEndpoint::getId) + .collect(Collectors.toList()))); + } + + return debugState; + } + + interface Listener + { + /** + * The list of recent speakers changed (either because a new dominant speaker was promoted, or because an + * endpoint was removed). + * @param recentSpeakers the new list of recent speakers (including the dominant speaker at index 0). + * @param dominantSpeakerChanged whether the dominant speaker changed. + * @param silence whether we're in a period of silence + */ + void recentSpeakersChanged( + List recentSpeakers, + boolean dominantSpeakerChanged, + boolean silence); + void lastNEndpointsChanged(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/EncodingsManager.java b/jvb/src/main/java/org/jitsi/videobridge/EncodingsManager.java new file mode 100644 index 0000000..294c2ea --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/EncodingsManager.java @@ -0,0 +1,159 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge; + +import org.jitsi.nlj.rtp.*; + +import java.util.*; +import java.util.concurrent.*; + +/** + * Process information signaled about encodings (payload types, SSRCs, SSRC + * associations, etc.) and gather it in a single place where it can be published + * out to all interested parties. + * + * The idea is to alleviate the following problem: + * When a new endpoint joins, it can use the notification of new local encoding + * information to trigger updating all other endpoints with that new information, + * but how does this new endpoint learn of this information from all existing + * endpoints? Existing endpoints don't currently have a good trigger to notify + * new endpoints about their encoding information, so this was devised as a + * single location to handle dispersing this information to all interested + * parties. + * + * TODO: Provide a way to remove associations that are no longer used. + * + * @author Brian Baldino + */ +public class EncodingsManager +{ + /** + * Maps an endpoint ID to the list of its SSRC associations. + */ + private Map> ssrcAssociations + = new ConcurrentHashMap<>(); + + /** + * Set of listeners to be notified of associations. + */ + private Set listeners + = ConcurrentHashMap.newKeySet(); + + /** + * Adds an SSRC association for a specific endpoint. + * + * @param endpointId the ID of the endpoint. + * @param primarySsrc the primary SSRC in the SSRC association. + * @param secondarySsrc the secondary SSRC in the SSRC association. + * @param type the association type. + */ + public void addSsrcAssociation( + String endpointId, + long primarySsrc, + long secondarySsrc, + SsrcAssociationType type) + { + List epSsrcAssociations + = ssrcAssociations.computeIfAbsent( + endpointId, k -> new ArrayList<>()); + epSsrcAssociations.add( + new SsrcAssociation(primarySsrc, secondarySsrc, type)); + + listeners.forEach( + listener + -> listener.onNewSsrcAssociation( + endpointId, primarySsrc, secondarySsrc, type)); + } + + /** + * Subscribe to future updates and be notified of any existing SSRC + * associations. + * @param listener + */ + public void subscribe(EncodingsUpdateListener listener) + { + listeners.add(listener); + + ssrcAssociations.forEach( + (endpointId, ssrcAssociations) + -> ssrcAssociations.forEach(ssrcAssociation -> + { + listener.onNewSsrcAssociation( + endpointId, + ssrcAssociation.primarySsrc, + ssrcAssociation.secondarySsrc, + ssrcAssociation.type); + })); + } + + /** + * Unsubscribe from SSRC updates. + */ + public void unsubscribe(EncodingsUpdateListener listener) + { + listeners.remove(listener); + } + + /** + * An interface for listening to new associations. + */ + public interface EncodingsUpdateListener + { + /** + * Notify this listener of a new association. + */ + void onNewSsrcAssociation( + String endpointId, + long primarySsrc, + long secondarySsrc, + SsrcAssociationType type); + } + + /** + * Represents an SSRC association. + */ + private static class SsrcAssociation + { + /** + * The primary SSRC. + */ + private long primarySsrc; + + /** + * The secondary SSRC. + */ + private long secondarySsrc; + + /** + * The association type. + */ + private SsrcAssociationType type; + + /** + * Initializes a new {@link SsrcAssociation} instance. + */ + private SsrcAssociation( + long primarySsrc, + long secondarySsrc, + SsrcAssociationType type) + { + this.primarySsrc = primarySsrc; + this.secondarySsrc = secondarySsrc; + this.type = type; + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java new file mode 100644 index 0000000..4cd9dab --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/EndpointMessageTransport.java @@ -0,0 +1,601 @@ +/* + * Copyright @ 2017 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge; + +import org.eclipse.jetty.websocket.api.Callback; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.core.CloseStatus; +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.datachannel.*; +import org.jitsi.videobridge.datachannel.protocol.*; +import org.jitsi.videobridge.message.*; +import org.jitsi.videobridge.metrics.*; +import org.jitsi.videobridge.relay.*; +import org.jitsi.videobridge.websocket.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.lang.ref.*; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.*; + +import static org.jitsi.videobridge.VersionConfig.config; +import static org.jitsi.videobridge.util.MultiStreamCompatibilityKt.endpointIdToSourceName; + +/** + * Handles the functionality related to sending and receiving COLIBRI messages + * for an {@link Endpoint}. Supports two underlying transport mechanisms -- + * WebRTC data channels and {@code WebSocket}s. + * + * @author Boris Grozev + */ +public class EndpointMessageTransport + extends AbstractEndpointMessageTransport + implements DataChannelStack.DataChannelMessageListener, + ColibriWebSocket.EventHandler +{ + /** + * The last accepted web-socket by this instance, if any. + */ + private ColibriWebSocket webSocket; + + /** + * User to synchronize access to {@link #webSocket} + */ + private final Object webSocketSyncRoot = new Object(); + + /** + * Whether the last active transport channel (i.e. the last to receive a + * message from the remote endpoint) was the web socket (if {@code true}), + * or the WebRTC data channel (if {@code false}). + */ + private boolean webSocketLastActive = false; + + private WeakReference dataChannel = new WeakReference<>(null); + + private final EndpointMessageTransportEventHandler eventHandler; + + private final AtomicInteger numOutgoingMessagesDropped = new AtomicInteger(0); + + /** + * The number of sent message by type. + */ + private final Map sentMessagesCounts = new ConcurrentHashMap<>(); + + @NotNull + private final Endpoint endpoint; + + /** + * Initializes a new {@link EndpointMessageTransport} instance. + * @param endpoint the associated {@link Endpoint}. + */ + EndpointMessageTransport( + @NotNull Endpoint endpoint, + EndpointMessageTransportEventHandler eventHandler, + Logger parentLogger) + { + super(parentLogger); + this.endpoint = endpoint; + this.eventHandler = eventHandler; + } + + /** + * {@inheritDoc} + */ + @Override + protected void notifyTransportChannelConnected() + { + endpoint.endpointMessageTransportConnected(); + eventHandler.endpointMessageTransportConnected(endpoint); + } + + /** + * {@inheritDoc} + */ + @Override + public BridgeChannelMessage clientHello(ClientHelloMessage message) + { + // ClientHello was introduced for functional testing purposes. It + // triggers a ServerHello response from Videobridge. The exchange + // reveals (to the client) that the transport channel between the + // remote endpoint and the Videobridge is operational. + // We take care to send the reply using the same transport channel on + // which we received the request.. + return createServerHello(); + } + + @Override + public BridgeChannelMessage videoType(VideoTypeMessage videoTypeMessage) + { + return sourceVideoType( + new SourceVideoTypeMessage( + videoTypeMessage.getVideoType(), + endpointIdToSourceName(endpoint.getId()), + videoTypeMessage.getEndpointId()) + ); + } + + @Override + public BridgeChannelMessage sourceVideoType(SourceVideoTypeMessage sourceVideoTypeMessage) + { + String sourceName = sourceVideoTypeMessage.getSourceName(); + + if (getLogger().isDebugEnabled()) + { + getLogger().debug("Received video type of " + sourceName +": " + sourceVideoTypeMessage.getVideoType()); + } + + endpoint.setVideoType(sourceName, sourceVideoTypeMessage.getVideoType()); + + Conference conference = endpoint.getConference(); + + if (conference.isExpired()) + { + getLogger().warn("Unable to forward SourceVideoTypeMessage, conference is expired"); + return null; + } + + sourceVideoTypeMessage.setEndpointId(endpoint.getId()); + + /* Forward videoType messages to Relays. */ + conference.sendMessage(sourceVideoTypeMessage, Collections.emptyList(), true); + + return null; + } + + @Override + public BridgeChannelMessage receiverAudioSubscription( + @NotNull ReceiverAudioSubscriptionMessage receiverAudioSubscriptionMessage + ) + { + if (getLogger().isDebugEnabled()) + { + getLogger().debug("Received audio subscription: " + receiverAudioSubscriptionMessage); + } + + Conference conference = endpoint.getConference(); + conference.setEndpointAudioSubscription(endpoint.getId(), receiverAudioSubscriptionMessage); + + return null; + } + + @Override + public void unhandledMessage(@NotNull BridgeChannelMessage message) + { + getLogger().warn("Received a message with an unexpected type: " + message.getClass().getSimpleName()); + } + + /** + * Sends a string via a particular transport channel. + * @param dst the transport channel. + * @param message the message to send. + */ + protected void sendMessage(Object dst, BridgeChannelMessage message) + { + super.sendMessage(dst, message); // Log message + + if (dst instanceof ColibriWebSocket) + { + sendMessage((ColibriWebSocket) dst, message); + } + else if (dst instanceof DataChannel) + { + sendMessage((DataChannel)dst, message); + } + else + { + throw new IllegalArgumentException("unknown transport:" + dst); + } + } + + /** + * Sends a string via a particular {@link DataChannel}. + * @param dst the data channel to send through. + * @param message the message to send. + */ + private void sendMessage(DataChannel dst, BridgeChannelMessage message) + { + dst.sendString(message.toJson()); + VideobridgeMetrics.dataChannelMessagesSent.inc(); + } + + /** + * Sends a string via a particular {@link ColibriWebSocket} instance. + * @param dst the {@link ColibriWebSocket} through which to send the message. + * @param message the message to send. + */ + private void sendMessage(ColibriWebSocket dst, BridgeChannelMessage message) + { + dst.sendString(message.toJson()); + VideobridgeMetrics.colibriWebSocketMessagesSent.inc(); + } + + @Override + public void onDataChannelMessage(DataChannelMessage dataChannelMessage) + { + webSocketLastActive = false; + VideobridgeMetrics.dataChannelMessagesReceived.inc(); + + if (dataChannelMessage instanceof DataChannelStringMessage) + { + DataChannelStringMessage dataChannelStringMessage = (DataChannelStringMessage)dataChannelMessage; + onMessage(dataChannel.get(), dataChannelStringMessage.data); + } + } + + /** + * {@inheritDoc} + */ + @Override + protected void sendMessage(@NotNull BridgeChannelMessage msg) + { + Object dst = getActiveTransportChannel(); + if (dst == null) + { + getLogger().debug("No available transport channel, can't send a message"); + numOutgoingMessagesDropped.incrementAndGet(); + } + else + { + sentMessagesCounts.computeIfAbsent( + msg.getClass().getSimpleName(), + (k) -> new AtomicLong()).incrementAndGet(); + sendMessage(dst, msg); + } + } + + /** + * @return the active transport channel for this + * {@link EndpointMessageTransport} (either the {@link #webSocket}, or + * the WebRTC data channel represented by a {@link DataChannel}). + *

+ * The "active" channel is determined based on what channels are available, + * and which one was the last to receive data. That is, if only one channel + * is available, it will be returned. If two channels are available, the + * last one to have received data will be returned. Otherwise, {@code null} + * will be returned. + */ + //TODO(brian): seems like it'd be nice to have the websocket and datachannel + // share a common parent class (or, at least, have a class that is returned + // here and provides a common API but can wrap either a websocket or + // datachannel) + private Object getActiveTransportChannel() + { + DataChannel dataChannel = this.dataChannel.get(); + ColibriWebSocket webSocket = this.webSocket; + + Object dst = null; + if (webSocketLastActive) + { + dst = webSocket; + } + + // Either the socket was not the last active channel, + // or it has been closed. + if (dst == null) + { + if (dataChannel != null && dataChannel.isReady()) + { + dst = dataChannel; + } + } + + // Maybe the WebRTC data channel is the last active, but it is not + // currently available. If so, and a web-socket is available -- use it. + if (dst == null && webSocket != null) + { + dst = webSocket; + } + + return dst; + } + + @Override + public boolean isConnected() + { + return getActiveTransportChannel() != null; + } + + + /** + * {@inheritDoc} + */ + @Override + public void webSocketConnected(ColibriWebSocket ws) + { + synchronized (webSocketSyncRoot) + { + // If we already have a web-socket, discard it and use the new one. + if (webSocket != null) + { + Session session = webSocket.getSession(); + if (session != null) + { + session.close(CloseStatus.NORMAL, "replaced", Callback.NOOP); + } + } + + webSocket = ws; + webSocketLastActive = true; + sendMessage(ws, createServerHello()); + } + + try + { + notifyTransportChannelConnected(); + } + catch (Exception e) + { + getLogger().warn("Caught an exception in notifyTransportConnected", e); + } + } + + private ServerHelloMessage createServerHello() + { + if (config.announceVersion()) + { + return new ServerHelloMessage(endpoint.getConference().getVideobridge().getVersion().toString()); + } + else + { + return new ServerHelloMessage(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void webSocketClosed(ColibriWebSocket ws, int statusCode, String reason) + { + synchronized (webSocketSyncRoot) + { + if (ws != null && ws.equals(webSocket)) + { + webSocket = null; + webSocketLastActive = false; + getLogger().info(() -> "Websocket closed, statusCode " + statusCode + " ( " + reason + ")."); + // 1000 is normal, 1001 is e.g. a tab closing. 1005 is "No Status Rcvd" and we see the majority of + // sockets close this way. + if (statusCode == 1000 || statusCode == 1001 || statusCode == 1005) + { + VideobridgeMetrics.colibriWebSocketCloseNormal.inc(); + } + else + { + VideobridgeMetrics.colibriWebSocketCloseAbnormal.inc(); + } + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void webSocketError(ColibriWebSocket ws, Throwable cause) + { + getLogger().error("Colibri websocket error: " + cause.getMessage()); + VideobridgeMetrics.colibriWebSocketErrors.inc(); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() + { + synchronized (webSocketSyncRoot) + { + if (webSocket != null) + { + // 1001 indicates that an endpoint is "going away", such as a server + // going down or a browser having navigated away from a page. + Session session = webSocket.getSession(); + if (session != null) + { + session.close(CloseStatus.SHUTDOWN, "endpoint closed", Callback.NOOP); + } + webSocket = null; + getLogger().debug(() -> "Endpoint expired, closed colibri web-socket."); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void webSocketTextReceived(ColibriWebSocket ws, String message) + { + if (ws == null || !ws.equals(webSocket)) + { + getLogger().warn("Received text from an unknown web socket."); + return; + } + + VideobridgeMetrics.colibriWebSocketMessagesReceived.inc(); + + webSocketLastActive = true; + onMessage(ws, message); + } + + /** + * Sets the data channel for this endpoint. + * @param dataChannel the {@link DataChannel} to use for this transport + */ + void setDataChannel(DataChannel dataChannel) + { + DataChannel prevDataChannel = this.dataChannel.get(); + if (prevDataChannel == null) + { + this.dataChannel = new WeakReference<>(dataChannel); + // We install the handler first, otherwise the 'ready' might fire after we check it but before we + // install the handler + dataChannel.onDataChannelEvents(this::notifyTransportChannelConnected); + if (dataChannel.isReady()) + { + notifyTransportChannelConnected(); + } + dataChannel.onDataChannelMessage(this); + } + else if (prevDataChannel == dataChannel) + { + //TODO: i think we should be able to ensure this doesn't happen, + // so throwing for now. if there's a good + // reason for this, we can make this a no-op + throw new Error("Re-setting the same data channel"); + } + else + { + throw new Error("Overwriting a previous data channel!"); + } + } + + @Override + public ObjectNode getDebugState() + { + ObjectNode debugState = super.getDebugState(); + debugState.put("numOutgoingMessagesDropped", numOutgoingMessagesDropped.get()); + + ObjectNode sentCounts = JsonNodeFactory.instance.objectNode(); + sentMessagesCounts.forEach((k, v) -> sentCounts.put(k, v.get())); + debugState.set("sent_counts", sentCounts); + + return debugState; + } + + @Nullable + @Override + public BridgeChannelMessage receiverVideoConstraints(@NotNull ReceiverVideoConstraintsMessage message) + { + endpoint.setBandwidthAllocationSettings(message); + return null; + } + + /** + * Notifies this {@code Endpoint} that a {@link LastNMessage} has been + * received. + * + * @param message the message that was received. + */ + @Override + public BridgeChannelMessage lastN(LastNMessage message) + { + endpoint.setLastN(message.getLastN()); + + return null; + } + + /** + * Handles an opaque message from this {@code Endpoint} that should be forwarded to either: a) another client in + * this conference (1:1 message) or b) all other clients in this conference (broadcast message). + * + * @param message the message that was received from the endpoint. + */ + @Override + public BridgeChannelMessage endpointMessage(EndpointMessage message) + { + if (endpoint.getVisitor()) + { + getLogger().warn("Not forwarding endpoint message from visitor endpoint"); + return null; + } + + // First insert/overwrite the "from" to prevent spoofing. + String from = endpoint.getId(); + message.setFrom(from); + + Conference conference = endpoint.getConference(); + + if (conference == null || conference.isExpired()) + { + getLogger().warn("Unable to send EndpointMessage, conference is null or expired"); + return null; + } + + if (message.isBroadcast()) + { + // Broadcast message to all local endpoints and relays. + List targets = new LinkedList<>(conference.getLocalEndpoints()); + targets.remove(endpoint); + conference.sendMessage(message, targets, /* sendToRelays */ true); + } + else + { + // 1:1 message + String to = message.getTo(); + + AbstractEndpoint targetEndpoint = conference.getEndpoint(to); + if (targetEndpoint instanceof Endpoint) + { + ((Endpoint)targetEndpoint).sendMessage(message); + } + else if (targetEndpoint instanceof RelayedEndpoint) + { + ((RelayedEndpoint)targetEndpoint).getRelay().sendMessage(message); + } + else if (targetEndpoint != null) + { + conference.sendMessage(message, Collections.emptyList(), /* sendToRelays */ true); + } + else + { + getLogger().warn("Unable to find endpoint to send EndpointMessage to: " + to); + } + } + + return null; + } + + /** + * Handles an endpoint statistics message from this {@code Endpoint} that should be forwarded to + * other endpoints as appropriate, and also to relays. + * + * @param message the message that was received from the endpoint. + */ + @Override + public BridgeChannelMessage endpointStats(@NotNull EndpointStats message) + { + if (endpoint.getVisitor()) + { + getLogger().warn("Not forwarding endpoint stats from visitor endpoint"); + return null; + } + + // First insert/overwrite the "from" to prevent spoofing. + String from = endpoint.getId(); + message.setFrom(from); + + Conference conference = endpoint.getConference(); + + if (conference.isExpired()) + { + getLogger().warn("Unable to send EndpointStats, conference is null or expired"); + return null; + } + + List targets = conference.getLocalEndpoints().stream() + .filter((ep) -> ep != endpoint && ep.wantsStatsFrom(endpoint)) + .collect(Collectors.toList()); + + conference.sendMessage(message, targets, true); + return null; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/PotentialPacketHandler.java b/jvb/src/main/java/org/jitsi/videobridge/PotentialPacketHandler.java new file mode 100644 index 0000000..5161a72 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/PotentialPacketHandler.java @@ -0,0 +1,37 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; + +public interface PotentialPacketHandler +{ + /** + * Checks whether or not this PotentialPacketHandler is interested + * in the RTP/RTCP packet 'packet' from 'source' + * @param packet the RTP/RTCP packet + * @return true if this handler wants the given packet, false otherwise + */ + boolean wants(@NotNull PacketInfo packet); + + /** + * Send the given RTP/RTCP 'packet' (which came from 'source') + * @param packet the RTP/RTCP packet + */ + void send(@NotNull PacketInfo packet); +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java new file mode 100644 index 0000000..4e972d3 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/Videobridge.java @@ -0,0 +1,591 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.*; +import org.jitsi.health.Result; +import org.jitsi.nlj.*; +import org.jitsi.shutdown.*; +import org.jitsi.utils.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.utils.version.*; +import org.jitsi.videobridge.health.*; +import org.jitsi.videobridge.load_management.*; +import org.jitsi.videobridge.metrics.*; +import org.jitsi.videobridge.shutdown.*; +import org.jitsi.videobridge.stats.*; +import org.jitsi.videobridge.xmpp.*; +import org.jitsi.xmpp.extensions.colibri2.*; +import org.jitsi.xmpp.extensions.health.*; +import org.jivesoftware.smack.packet.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jxmpp.jid.*; +import org.jxmpp.jid.impl.*; +import org.jxmpp.stringprep.*; + +import java.time.*; +import java.util.*; + +import static org.jitsi.videobridge.colibri2.Colibri2UtilKt.*; +import static org.jitsi.xmpp.util.ErrorUtilKt.createError; + +/** + * Represents the Jitsi Videobridge which creates, lists and destroys + * {@link Conference} instances. + * + * @author Lyubomir Marinov + * @author Hristo Terezov + * @author Boris Grozev + * @author Brian Baldino + */ +@SuppressWarnings("JavadocReference") +public class Videobridge +{ + /** + * The Logger used by the Videobridge class and its + * instances to print debug information. + */ + private static final Logger logger = new LoggerImpl(Videobridge.class.getName()); + + /** + * The pseudo-random generator which is to be used when generating + * {@link Conference} IDs in order to minimize busy + * waiting for the value of {@link System#currentTimeMillis()} to change. + */ + public static final Random RANDOM = new Random(); + + /** + * The Conferences of this Videobridge mapped by their local IDs. + *

+ * TODO: The only remaining uses of this ID are for the HTTP debug interface and the colibri WebSocket conference + * identifier. This should be replaced with meetingId (while making sure jvb-rtcstats-push doesn't break). + */ + private final Map conferencesById = new HashMap<>(); + + /** + * The Conferences of this Videobridge mapped by their + * meeting IDs. + */ + private final Map conferencesByMeetingId = new HashMap<>(); + + private final JvbHealthChecker jvbHealthChecker = new JvbHealthChecker(); + + /** + * The clock to use, pluggable for testing purposes. + *

+ * Note that currently most code uses the system clock directly. + */ + @NotNull + private final Clock clock; + + /** + * The {@link JvbLoadManager} instance used for this bridge. + */ + @NotNull + private final JvbLoadManager jvbLoadManager; + + @NotNull private final Version version; + + @NotNull private final ShutdownManager shutdownManager; + + /** + * Initializes a new Videobridge instance. + */ + public Videobridge( + @Nullable XmppConnection xmppConnection, + @NotNull ShutdownServiceImpl shutdownService, + @NotNull Version version, + @NotNull Clock clock) + { + this.clock = clock; + jvbLoadManager = JvbLoadManager.create(this); + if (xmppConnection != null) + { + xmppConnection.setEventHandler(new XmppConnectionEventHandler()); + } + this.version = version; + this.shutdownManager = new ShutdownManager(shutdownService, logger); + jvbHealthChecker.start(); + } + + @NotNull + public JvbHealthChecker getJvbHealthChecker() + { + return jvbHealthChecker; + } + + /** + * Generate conference IDs until one is found that isn't in use and create a new {@link Conference} + * object using that ID + */ + private @NotNull Conference doCreateConference( + @Nullable EntityBareJid name, + String meetingId, + boolean isRtcStatsEnabled) + { + Conference conference = null; + do + { + String id = generateConferenceID(); + + synchronized (conferencesById) + { + if (meetingId != null && conferencesByMeetingId.containsKey(meetingId)) + { + throw new IllegalStateException("Already have a meeting with meetingId " + meetingId); + } + + if (!conferencesById.containsKey(id)) + { + conference = new Conference(this, id, name, meetingId, isRtcStatsEnabled); + conferencesById.put(id, conference); + VideobridgeMetrics.currentConferences.inc(); + + if (meetingId != null) + { + conferencesByMeetingId.put(meetingId, conference); + } + } + } + } + while (conference == null); + + return conference; + } + + void localEndpointCreated(boolean visitor) + { + VideobridgeMetrics.currentLocalEndpoints.inc(); + if (visitor) + { + VideobridgeMetrics.currentVisitors.inc(); + } + } + + void localEndpointExpired(boolean visitor) + { + long remainingEndpoints = VideobridgeMetrics.currentLocalEndpoints.decAndGet(); + if (visitor) + { + VideobridgeMetrics.currentVisitors.dec(); + } + + if (remainingEndpoints < 0) + { + logger.warn("Invalid endpoint count " + remainingEndpoints + ". Disabling endpoint-count based shutdown!"); + return; + } + shutdownManager.maybeShutdown(remainingEndpoints); + } + + /** + * Initializes a new {@link Conference} instance with an ID unique to the + * Conference instances listed by this Videobridge and + * adds the new instance to the list of existing Conference + * instances. + * + * @param gid the "global" id of the conference (or + * {@link Conference#GID_NOT_SET} if it is not specified. + * @param name world readable name of the conference to create. + * @return a new Conference instance with an ID unique to the + * Conference instances listed by this Videobridge + */ + private @NotNull Conference createConference( + @Nullable EntityBareJid name, + String meetingId, + boolean isRtcStatsEnabled) + { + final Conference conference = doCreateConference(name, meetingId, isRtcStatsEnabled); + + logger.info(() -> "create_conf, id=" + conference.getID() + " meeting_id=" + meetingId); + + return conference; + } + + /** + * Expires a specific Conference of this Videobridge (i.e. + * if the specified Conference is not in the list of + * Conferences of this Videobridge, does nothing). + * + * @param conference the Conference to be expired by this + * Videobridge + */ + public void expireConference(Conference conference) + { + String id = conference.getID(); + String meetingId = conference.getMeetingId(); + + synchronized (conferencesById) + { + if (conference.equals(conferencesById.get(id))) + { + conferencesById.remove(id); + VideobridgeMetrics.currentConferences.dec(); + + if (meetingId != null) + { + if (conference.equals(conferencesByMeetingId.get(meetingId))) + { + conferencesByMeetingId.remove(meetingId); + } + } + conference.expire(); + } + } + } + + /** + * Generates a new Conference ID which is not guaranteed to be + * unique. + * + * @return a new Conference ID which is not guaranteed to be unique + */ + private String generateConferenceID() + { + return Long.toHexString(System.currentTimeMillis() + RANDOM.nextLong()); + } + + /** + * Gets an existing {@link Conference} with a specific ID. + * + * @param id the ID of the existing Conference to get + * @return an existing Conference with the specified ID. + */ + public Conference getConference(String id) + { + synchronized (conferencesById) + { + return conferencesById.get(id); + } + } + + /** + * Gets an existing {@link Conference} with a specific meeting ID. + */ + public Conference getConferenceByMeetingId(@NotNull String meetingId) + { + /* Note that conferenceByMeetingId is synchronized on conferencesById. */ + synchronized (conferencesById) + { + return conferencesByMeetingId.get(meetingId); + } + } + + /** + * Gets the Conferences of this Videobridge. + * + * @return the Conferences of this Videobridge + */ + public Collection getConferences() + { + synchronized (conferencesById) + { + return new HashSet<>(conferencesById.values()); + } + } + + /** + * Handles a COLIBRI2 request synchronously. + * @param conferenceModifyIQ The COLIBRI request. + * @return The response in the form of an {@link IQ}. It is either an error or a {@link ConferenceModifiedIQ}. + */ + public IQ handleConferenceModifyIq(ConferenceModifyIQ conferenceModifyIQ) + { + Conference conference; + try + { + conference = getOrCreateConference(conferenceModifyIQ); + } + catch (ConferenceNotFoundException e) + { + return createConferenceNotFoundError(conferenceModifyIQ, conferenceModifyIQ.getMeetingId()); + } + catch (ConferenceAlreadyExistsException e) + { + return createConferenceAlreadyExistsError(conferenceModifyIQ, conferenceModifyIQ.getMeetingId()); + } + catch (InGracefulShutdownException e) + { + return createGracefulShutdownErrorResponse(conferenceModifyIQ); + } + catch (XmppStringprepException e) + { + return createError( + conferenceModifyIQ, + StanzaError.Condition.bad_request, + "Invalid conference name (not a JID)"); + } + + return conference.handleConferenceModifyIQ(conferenceModifyIQ); + } + + /** + * Handles a COLIBRI request asynchronously. + */ + private void handleColibriRequest(XmppConnection.ColibriRequest request) + { + ConferenceModifyIQ iq = request.getRequest(); + String id = request.getRequest().getMeetingId(); + Conference conference; + + try + { + conference = getOrCreateConference(request.getRequest()); + } + catch (ConferenceNotFoundException e) + { + request.getCallback().invoke(createConferenceNotFoundError(iq, id)); + return; + } + catch (ConferenceAlreadyExistsException e) + { + request.getCallback().invoke(createConferenceAlreadyExistsError(iq, id)); + return; + } + catch (InGracefulShutdownException e) + { + request.getCallback().invoke(createGracefulShutdownErrorResponse(iq)); + return; + } + catch (XmppStringprepException e) + { + request.getCallback().invoke( + createError(iq, StanzaError.Condition.bad_request, "Invalid conference name (not a JID)")); + return; + } + + // It is now the responsibility of Conference to send a response. + conference.enqueueColibriRequest(request); + } + + private @NotNull Conference getOrCreateConference(ConferenceModifyIQ conferenceModifyIQ) + throws InGracefulShutdownException, XmppStringprepException, + ConferenceAlreadyExistsException, ConferenceNotFoundException + { + String meetingId = conferenceModifyIQ.getMeetingId(); + + synchronized(conferencesById) + { + Conference conference = getConferenceByMeetingId(meetingId); + + if (conferenceModifyIQ.getCreate()) + { + if (conference != null) + { + logger.warn("Will not create conference, conference already exists for meetingId=" + meetingId); + throw new ConferenceAlreadyExistsException(); + } + if (isInGracefulShutdown()) + { + logger.warn("Will not create conference in shutdown mode."); + throw new InGracefulShutdownException(); + } + + String conferenceName = conferenceModifyIQ.getConferenceName(); + return createConference( + conferenceName == null ? null : JidCreate.entityBareFrom(conferenceName), + meetingId, + conferenceModifyIQ.isRtcstatsEnabled() + ); + } + else + { + if (conference == null) + { + logger.warn("Conference with meeting_id=" + meetingId + " not found."); + throw new ConferenceNotFoundException(); + } + return conference; + } + } + } + + /** + * Handles a shutdown request. + */ + public void shutdown(boolean graceful) + { + shutdownManager.initiateShutdown(graceful); + shutdownManager.maybeShutdown(VideobridgeMetrics.currentLocalEndpoints.get()); + } + + /** + * Whether the bridge is in "drain" mode. The bridge will not be assigned to new + * conferences when in this mode. + */ + private boolean drainMode = VideobridgeConfig.Companion.getInitialDrainMode(); + + /** + * Handles a drain request. + */ + public void setDrainMode(boolean enable) + { + logger.info("Received drain request. enable=" + enable); + VideobridgeMetrics.INSTANCE.getDrainMode().set(enable); + drainMode = enable; + } + + /** + * Query the drain state. + */ + public boolean getDrainMode() + { + return drainMode; + } + + /** + * Returns {@code true} if this instance has entered graceful shutdown mode. + * + * @return {@code true} if this instance has entered graceful shutdown mode; + * otherwise, {@code false} + */ + public boolean isInGracefulShutdown() + { + return shutdownManager.getState() == ShutdownState.GRACEFUL_SHUTDOWN; + } + + public ShutdownState getShutdownState() + { + return shutdownManager.getState(); + } + + /** + * Stops this {@link Videobridge}. + */ + void stop() + { + jvbLoadManager.stop(); + } + + /** + * Creates a JSON for debug purposes. If a specific {@code conferenceId} + * is requested, the result will include all of the state of the conference + * considered useful for debugging (note that this is a LOT of data). + * Otherwise (if {@code conferenceId} is {@code null}), the result includes + * a shallow list of the active conferences and their endpoints. + * + * @param conferenceId the ID of a specific conference to include. If not + * specified, a shallow list of all conferences will be returned. + * @param endpointId the ID of a specific endpoint in {@code conferenceId} + * to include. If not specified, all of the conference's endpoints will be + * included. + */ + public ObjectNode getDebugState(String conferenceId, String endpointId, DebugStateMode mode) + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + + if (mode == DebugStateMode.FULL || mode == DebugStateMode.SHORT) + { + debugState.put("shutdown_state", shutdownManager.getState().toString()); + debugState.put("drain", drainMode); + debugState.put("time", System.currentTimeMillis()); + + debugState.set("load_management", jvbLoadManager.getStats()); + + Double jitter = PacketTransitStats.getBridgeJitter(); + if (jitter != null) + { + debugState.put("overall_bridge_jitter", jitter); + } + } + + ObjectNode conferences = JsonNodeFactory.instance.objectNode(); + debugState.set("conferences", conferences); + if (StringUtils.isBlank(conferenceId)) + { + getConferences().stream() + .filter(c -> mode != DebugStateMode.STATS || c.isRtcStatsEnabled()) + .forEach(conference -> + conferences.set( + conference.getID(), + conference.getDebugState(mode, null))); + } + else + { + // Using getConference will 'touch' it and prevent it from expiring + Conference conference; + synchronized (conferences) + { + conference = this.conferencesById.get(conferenceId); + } + + if (conference == null) + { + conferences.put(conferenceId, "null"); + } + else + { + conferences.set(conferenceId, conference.getDebugState(mode, endpointId)); + } + } + + return debugState; + } + + @NotNull + public Version getVersion() + { + return version; + } + + private class XmppConnectionEventHandler implements XmppConnection.EventHandler + { + @Override + public void colibriRequestReceived(@NotNull XmppConnection.ColibriRequest request) + { + handleColibriRequest(request); + } + + @NotNull + @Override + public IQ versionIqReceived(@NotNull org.jivesoftware.smackx.iqversion.packet.Version iq) + { + org.jivesoftware.smackx.iqversion.packet.Version versionResult = + new org.jivesoftware.smackx.iqversion.packet.Version( + version.getApplicationName(), + version.toString(), + System.getProperty("os.name") + ); + + versionResult.setType(IQ.Type.result); + + return versionResult; + } + + @NotNull + @Override + public IQ healthCheckIqReceived(@NotNull HealthCheckIQ iq) + { + Result result = jvbHealthChecker.getResult(); + if (result.getSuccess()) + { + return IQ.createResultIQ(iq); + } + else + { + return IQ.createErrorResponse( + iq, + StanzaError.from(StanzaError.Condition.internal_server_error, result.getMessage()).build()); + } + } + } + + private static class ConferenceNotFoundException extends Exception {} + private static class ConferenceAlreadyExistsException extends Exception {} + private static class InGracefulShutdownException extends Exception {} +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/VideobridgeExpireThread.java b/jvb/src/main/java/org/jitsi/videobridge/VideobridgeExpireThread.java new file mode 100644 index 0000000..2dd8e4b --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/VideobridgeExpireThread.java @@ -0,0 +1,164 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge; + +import java.time.*; +import java.util.*; +import java.util.concurrent.*; + +import org.jitsi.utils.concurrent.*; +import org.jitsi.utils.logging2.*; + +import static org.jitsi.videobridge.VideobridgeExpireThreadConfig.config; + +/** + * Implements a Thread which expires the {@link AbstractEndpoint}s and + * {@link Conference}s of a specific Videobridge. + * + * @author Lyubomir Marinov + */ +public class VideobridgeExpireThread +{ + /** + * The Logger used by the VideobridgeExpireThread class + * and its instances to print debug information. + */ + private static final Logger logger = + new LoggerImpl(VideobridgeExpireThread.class.getName()); + + /** + * The executor which periodically calls {@link #expire(Videobridge)} (if + * this {@link VideobridgeExpireThread} has been started). + */ + private static final RecurringRunnableExecutor EXECUTOR + = new RecurringRunnableExecutor( + VideobridgeExpireThread.class.getSimpleName()); + + /** + * The executor used to expire the individual {@link AbstractEndpoint}s + * or {@link Conference}s. + */ + private static final Executor EXPIRE_EXECUTOR + = ExecutorUtils.newCachedThreadPool( + true, VideobridgeExpireThread.class.getSimpleName() + "-channel"); + + /** + * The {@link PeriodicRunnable} registered with {@link #EXECUTOR} which is + * to run the expire task for this {@link VideobridgeExpireThread} instance. + */ + private PeriodicRunnable expireRunnable; + + /** + * The {@link Videobridge} which has its {@link Conference}s expired by this + * instance. + */ + private Videobridge videobridge; + + /** + * Initializes a new {@link VideobridgeExpireThread} instance which is to + * expire the {@link Conference}s of a specific {@link Videobridge}. + * + * @param videobridge the {@link Videobridge} which is to have its + * {@link Conference}s expired by the new instance. + */ + public VideobridgeExpireThread(Videobridge videobridge) + { + this.videobridge = Objects.requireNonNull(videobridge); + } + + /** + * Starts this {@link VideobridgeExpireThread} + */ + void start() + { + Duration expireCheckSleepDuration = config.getInterval(); + logger.info( + "Starting with " + expireCheckSleepDuration.getSeconds() + " second interval."); + + expireRunnable = new PeriodicRunnable(expireCheckSleepDuration.toMillis()) + { + @Override + public void run() + { + super.run(); + + Videobridge videobridge + = VideobridgeExpireThread.this.videobridge; + if (videobridge != null) + { + expire(videobridge); + } + + // The current implementation of the executor fails with a + // concurrent modification exception if we de-register from + // the thread running run(). So we can not de-register here + // if videobridge==null, and we will keep running until we get + // explicitly stop()ed, which is fine. + } + }; + EXECUTOR.registerRecurringRunnable(expireRunnable); + } + + /** + * Stops this {@link VideobridgeExpireThread}. + */ + void stop() + { + logger.info("Stopping."); + if (expireRunnable != null) + { + EXECUTOR.deRegisterRecurringRunnable(expireRunnable); + } + expireRunnable = null; + videobridge = null; + } + + /** + * Expires the {@link Conference}s and/or endpoints of a specific Videobridge if they + * have been inactive for more than their advertised expire number + * of seconds. + * + * @param videobridge the Videobridge which is to have its + * Channels expired if they have been inactive for more than their + * advertised expire number of seconds + */ + private void expire(Videobridge videobridge) + { + logger.debug("Running expire()"); + for (Conference conference : videobridge.getConferences()) + { + // The Conferences will live an iteration more than the Contents. + if (conference.shouldExpire()) + { + logger.info("Conference " + + conference.getID() + " should expire, expiring it"); + EXPIRE_EXECUTOR.execute( + () -> videobridge.expireConference(conference)); + } + else + { + for (AbstractEndpoint endpoint : conference.getEndpoints()) + { + if (endpoint.shouldExpire()) + { + logger.info("Expiring endpoint " + endpoint.getId()); + EXPIRE_EXECUTOR.execute(endpoint::expire); + } + } + } + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java new file mode 100644 index 0000000..2c2c075 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjection.java @@ -0,0 +1,407 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; +import org.jitsi.nlj.rtp.*; +import org.jitsi.nlj.rtp.codec.av1.*; +import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.nlj.rtp.codec.vp9.*; +import org.jitsi.rtp.rtcp.*; +import org.jitsi.utils.logging.*; +import org.jitsi.utils.logging2.Logger; +import org.jitsi.videobridge.cc.av1.*; +import org.jitsi.videobridge.cc.vp8.*; +import org.jitsi.videobridge.cc.vp9.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.lang.*; +import java.util.*; + +/** + * Filters the packets coming from a specific {@link MediaSourceDesc} + * based on the currently forwarded subjective quality index. It's also taking + * care of upscaling and downscaling. It is also responsible for rewriting the + * forwarded packets so that the the quality switches are transparent from the + * receiver. See svc.md in the doc folder for more details. + * + * XXX A projection is a "function" of taking an element of one space and + * somehow putting it on another space. This is what we're doing here, we're + * taking a source of the ingress and we're projecting it to the egress, + * depending on the layers/encodings of the input source and how these encodings are + * implemented (which depends on the codec that is used). + * + * Instances of this class are thread-safe. + * + * @author George Politis + */ +public class AdaptiveSourceProjection +{ + /** + * The Logger used by the AdaptiveSourceProjection class + * and its instances for logging output. + */ + private final Logger logger; + + /** + * The main SSRC of the source (if simulcast is used, this is the SSRC + * of the low-quality layer). We use it as the SSRC of the source projection + * and also request keyframes from this SSRC. + */ + private final long targetSsrc; + + private final DiagnosticContext diagnosticContext; + + /** + * The payload specific source projection context that's responsible for + * rewriting the packets of a projected source. + * + * XXX The more general scheme and more correct approach would be to have + * the adaptive source projection manage a context per payload type. The + * intention was to implement the more general approach but, at the time of + * this righting, we have no use case for that right now nor an easy way to + * test this dynamic payload type changes. So practically what happens is + * that we create the context once, and it remains the same throughout the + * life of the this instance. It's not really possible to simplify and make + * the context final (=> create it during construction) because we need a + * payload type (could be VP9, could be H264, could be VP8) so it has to be + * created on packet arrival. + */ + private AdaptiveSourceProjectionContext context; + + /** + * The target quality index for this source projection. + */ + private int targetIndex = RtpLayerDesc.SUSPENDED_INDEX; + + /** + * The map for persistent states. + * Currently, we only expect it for AV1. + */ + private final HashMap, Object> persistentStates = new HashMap<>(); + + /** + * Ctor. + * + * @param source the {@link MediaSourceDesc} that owns the packets + * that this instance filters. + */ + public AdaptiveSourceProjection( + @NotNull DiagnosticContext diagnosticContext, + @NotNull MediaSourceDesc source, + Runnable keyframeRequester, + Logger parentLogger + ) + { + targetSsrc = source.getPrimarySSRC(); + this.diagnosticContext = diagnosticContext; + this.logger = parentLogger.createChildLogger(AdaptiveSourceProjection.class.getName(), + Map.of("targetSsrc", Long.toString(targetSsrc), + "srcEpId", Objects.toString(source.getOwner(), ""), + "srcName", source.getSourceName() + )); + this.keyframeRequester = keyframeRequester; + } + + /** + * Sets the target index value for this source projection. + */ + public void setTargetIndex(int value) + { + targetIndex = value; + } + + /** + * The callback we'll invoke when we want to request a keyframe for a stream. + */ + private final Runnable keyframeRequester; + + /** + * Determines whether an RTP packet needs to be accepted or not. + * + * @param packetInfo packet info for the video RTP packet to determine + * whether to accept or not. + * @return true if the packet is accepted, false otherwise. + */ + public boolean accept(@NotNull PacketInfo packetInfo) + { + VideoRtpPacket videoRtpPacket = packetInfo.packetAs(); + AdaptiveSourceProjectionContext contextCopy = getContext(videoRtpPacket); + if (contextCopy == null) + { + return false; + } + + // XXX We want to let the context know that the stream has been + // suspended so that it can raise the needsKeyframe flag and also allow + // it to compute a sequence number delta when the target becomes > -1. + + if (videoRtpPacket.getEncodingId() == RtpLayerDesc.SUSPENDED_ENCODING_ID) + { + logger.warn( + "Dropping an RTP packet, because egress was unable to find " + + "an associated layer/encoding. rtpPacket=" + videoRtpPacket); + return false; + } + + int targetIndexCopy = targetIndex; + boolean accept = contextCopy.accept(packetInfo, targetIndexCopy); + + // We check if the context needs a keyframe regardless of whether or not + // the packet was accepted. + // + // XXX Upon reading this for the first time, one may think it's + // sufficient to only check for needing a key frame if the packet wasn't + // accepted. But this wouldn't be enough, as we may be accepting packets + // of low-quality, while we wish to switch to high-quality. + if (contextCopy.needsKeyframe() + && targetIndexCopy > RtpLayerDesc.SUSPENDED_INDEX) + { + keyframeRequester.run(); + } + + return accept; + } + + /** + * Gets or creates the adaptive source projection context that corresponds to + * the parsed class of the RTP packet that is specified as a parameter. If + * the payload type is different from the appropriate one for the context, then a + * new adaptive source projection context is created that is appropriate for + * the new payload type. + * + * We make no attempt for thread safety, assuming no concurrent access. + * + * @param rtpPacket the RTP packet of the adaptive source projection context + * to get or create. + * @return the adaptive source projection context that corresponds to + * the payload type of the RTP packet that is specified as a parameter. + */ + private AdaptiveSourceProjectionContext getContext(@NotNull VideoRtpPacket rtpPacket) + { + int payloadType = rtpPacket.getPayloadType(); + + if (rtpPacket instanceof Vp8Packet) + { + // Context switch between VP8 simulcast and VP8 non-simulcast (sort + // of pretend that they're different codecs). + // + // HACK: When simulcast is activated, we also get temporal + // scalability, conversely if temporal scalability is disabled + // then simulcast is disabled. + + /* Check whether this stream is projectable by the VP8AdaptiveSourceProjectionContext. */ + boolean projectable = ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex() && + ((Vp8Packet)rtpPacket).getHasPictureId(); + + if (projectable + && !(context instanceof VP8AdaptiveSourceProjectionContext)) + { + // context switch + RtpState rtpState = getRtpState(); + logger.debug(() -> "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " VP8 context for source packet ssrc " + rtpPacket.getSsrc()); + context = new VP8AdaptiveSourceProjectionContext( + diagnosticContext, rtpState, logger); + } + else if (!projectable + && (!(context instanceof GenericAdaptiveSourceProjectionContext) || + ((GenericAdaptiveSourceProjectionContext)context).getPayloadType() != payloadType)) + { + RtpState rtpState = getRtpState(); + // context switch + logger.debug(() -> { + boolean hasTemporalLayer = ((Vp8Packet)rtpPacket).getHasTemporalLayerIndex(); + boolean hasPictureId = ((Vp8Packet)rtpPacket).getHasPictureId(); + return "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " generic context for non-scalable VP8 payload " + + " (packet is " + rtpPacket.getClass().getSimpleName() + + ", ssrc " + rtpPacket.getSsrc() + + ", hasTL=" + hasTemporalLayer + ", hasPID=" + hasPictureId + ")"; + }); + context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, logger); + } + + // no context switch + return context; + } + else if (rtpPacket instanceof Vp9Packet) + { + if (!(context instanceof Vp9AdaptiveSourceProjectionContext)) + { + // context switch + RtpState rtpState = getRtpState(); + logger.debug(() -> "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " VP9 context for source packet ssrc " + rtpPacket.getSsrc()); + context = new Vp9AdaptiveSourceProjectionContext( + diagnosticContext, rtpState, logger); + } + + return context; + } + else if (rtpPacket instanceof Av1DDPacket) + { + if (!(context instanceof Av1DDAdaptiveSourceProjectionContext)) + { + // context switch + RtpState rtpState = getRtpState(); + logger.debug(() -> "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " AV1 DD context for source packet ssrc " + rtpPacket.getSsrc()); + context = new Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, rtpState, + persistentStates.get(Av1DDAdaptiveSourceProjectionContext.class), + logger); + } + + return context; + } + else + { + if (!(context instanceof GenericAdaptiveSourceProjectionContext) || + ((GenericAdaptiveSourceProjectionContext)context).getPayloadType() != payloadType) + { + RtpState rtpState = getRtpState(); + logger.debug(() -> "adaptive source projection " + + (context == null ? "creating new" : "changing to") + + " generic context for payload type " + rtpPacket.getPayloadType()); + context = new GenericAdaptiveSourceProjectionContext(payloadType, rtpState, logger); + } + return context; + } + } + + /** + * Gets the {@link RtpState}. + */ + private @NotNull RtpState getRtpState() + { + if (context == null) + { + // TODO If '1' are the starting seq number and timestamp, should + // we use random values? + return new RtpState( + targetSsrc, + 1 /* maxSequenceNumber */, + 1 /* maxTimestamp */) ; + } + else + { + savePersistentState(); + return context.getRtpState(); + } + } + + private void savePersistentState() + { + if (context == null) + { + return; + } + Object state = context.getPersistentState(); + if (state != null) + { + if (context.getClass() != Av1DDAdaptiveSourceProjectionContext.class) + { + logger.warn("Got unexpected context persistent state from class " + + context.getClass().getSimpleName()); + } + persistentStates.put(context.getClass(), state); + } + else if (persistentStates.get(context.getClass()) != null) + { + persistentStates.remove(context.getClass()); + } + } + + /** + * Rewrites an RTP packet for projection. + * + * @param packetInfo the RTP packet to rewrite. + */ + public void rewriteRtp(@NotNull PacketInfo packetInfo) + throws RewriteException + { + AdaptiveSourceProjectionContext contextCopy = context; + if (contextCopy != null) + { + contextCopy.rewriteRtp(packetInfo); + } + } + + /** + * Rewrites an RTCP packet. + * + * @param rtcpSrPacket the RTCP SR packet to rewrite. + * @return true to let the RTCP packet through, false to drop. + */ + public boolean rewriteRtcp(@NotNull RtcpSrPacket rtcpSrPacket) + { + AdaptiveSourceProjectionContext contextCopy = context; + if (contextCopy == null) + { + return true; + } + + return contextCopy.rewriteRtcp(rtcpSrPacket); + } + + /** + * @return the SSRC of the source projection. + */ + public long getTargetSsrc() + { + return targetSsrc; + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + public ObjectNode getDebugState(@NotNull DebugStateMode mode) + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + + debugState.put("targetSsrc", targetSsrc); + AdaptiveSourceProjectionContext contextCopy = context; + if (mode == DebugStateMode.FULL) + { + if (contextCopy == null) + { + debugState.putNull("context"); + } + else + { + debugState.set("context", contextCopy.getDebugState()); + } + } + else if (mode == DebugStateMode.STATS) + { + debugState.put( + "context", + contextCopy == null ? "null" : contextCopy.getClass().getSimpleName()); + } + debugState.put("needsKeyframe", contextCopy == null ? "N/A" : Boolean.toString(contextCopy.needsKeyframe())); + debugState.put("targetIndex", targetIndex); + + return debugState; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java new file mode 100644 index 0000000..b97d3d9 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/AdaptiveSourceProjectionContext.java @@ -0,0 +1,99 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc; + +import edu.umd.cs.findbugs.annotations.Nullable; +import org.jitsi.nlj.*; +import org.jitsi.rtp.rtcp.*; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * Implementations of this interface are responsible for projecting a specific + * video source of a specific payload type. + * + * One can imagine signaling a specific encoding layout (i.e. 3 temporal layers) + * and multiple codec support (e.g. VP8 SVC and VP9 SVC). In the bridge such + * signaling would translate into an AdaptiveSourceProjection instance that would + * remain active throughout the life of the source video source and would receive + * updates from the bitrate controller (i.e. ideal index and target index). The + * specific way of projecting VP9 SVC or VP8 SVC is implemented in "context" + * classes that know how to deal with codec specificities. + * + * @author George Politis + */ +public interface AdaptiveSourceProjectionContext +{ + /** + * Determines whether an RTP packet should be accepted or not. + * + * @param packetInfo the RTP packet to determine whether to accept or not. + * @param targetIndex the target quality index + * @return true if the packet should be accepted, false otherwise. + */ + boolean accept(PacketInfo packetInfo, int targetIndex); + + /** + * @return true if this stream context needs a keyframe in order to either + * start rendering again or there's a pending simulcast switch (depending + * on the implementation). + */ + boolean needsKeyframe(); + + /** + * Rewrites the timestamp, sequence number, ssrc and other codec dependent + * fields of the RTP packet that is specified as an argument. Projecting a + * video source needs to be invisible to the receiving endpoint so goal here + * is to make the resulting rtp stream continuous. + * + * @param packetInfo the RTP packet info to rewrite. + * @throws RewriteException the underlying code has failed to rewrite the + * RTP packet that is specified as an argument. + */ + void rewriteRtp(PacketInfo packetInfo) + throws RewriteException; + + /** + * Rewrites the RTCP packet that is specified as an argument. + * + * @param rtcpSrPacket the RTCP packet to transform. + * @return true if the RTCP packet is accepted, false otherwise, in which + * case it needs to be dropped. + */ + boolean rewriteRtcp(RtcpSrPacket rtcpSrPacket); + + /** + * @return the RTP state that describes the max sequence number, max + * timestamp and other RTP-level details. + */ + RtpState getRtpState(); + + /** + * @return Persistent state, if any, that should be associated with this + * particular projection context type, so that when it comes back it can + * resume data, if necessary. + */ + @Nullable + default Object getPersistentState() + { + return null; + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + ObjectNode getDebugState(); +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java new file mode 100644 index 0000000..5637492 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/GenericAdaptiveSourceProjectionContext.java @@ -0,0 +1,349 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; +import org.jitsi.nlj.rtp.*; +import org.jitsi.rtp.rtcp.*; +import org.jitsi.rtp.util.*; +import org.jitsi.utils.logging2.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A generic implementation of an adaptive source projection context that can be + * used with non-SVC codecs or when simulcast is not enabled/used or when + * support for these advanced features is not implemented in the bridge. In this + * restricted case the source projection can have only two states (or qualities), + * either off or on (or -1, 0). + * + * Instances of this class suspend a source when the target quality is set to -1. + * When the target quality is set back to 0, the request key frame flag is + * raised and the source is re-activated after a key frame is received. + * (consequently support for key frame detection for the specific media format + * of the source that is being adapted is necessary). + * + * In order to make the suspend/resume operation transparent to the receiver (at + * least in the RTP level), instances of this class rewrite the RTP sequence + * numbers of the source to hide the gaps caused by the suspend/resume + * operation. + * + * This may not be sufficient for fluid playback at the receiver as the decoder + * may be unable to handle codec specific discontinuities (such as discontinuous + * picture IDs in VP8). In this case a codec specific adaptive source projection + * context implementation will have to be used instead. + * + * Instances of this class are thread-safe. + * + * @author George Politis + */ +class GenericAdaptiveSourceProjectionContext + implements AdaptiveSourceProjectionContext +{ + /** + * The Logger used by the + * GenericAdaptiveSourceProjectionContext class and its instances to + * log debug information. + */ + private final Logger logger; + + private final long ssrc; + + /** + * Raised when a source has been resumed (after being suspended). + */ + private boolean needsKeyframe = true; + + /** + * Useful to determine when we need to change generic projection contexts. + */ + private final int payloadType; + + /** + * The maximum sequence number that we have sent. + */ + private int maxDestinationSequenceNumber; + + /** + * The delta to apply to the timestamps of the RTP packets of the incoming + * source. + */ + private long timestampDelta; + + /** + * A boolean that indicates whether or not the timestamp delta has been + * initialized. This is only necessary upon adaptive source projection + * context switches. + */ + private boolean timestampDeltaInitialized = false; + + /** + * The maximum timestamp that we have sent. + */ + private long maxDestinationTimestamp; + + /** + * The delta to apply to the sequence numbers of the RTP packets of the + * incoming source. + */ + private int sequenceNumberDelta; + + /** + * Ctor. + * + * @param payloadType the media format to expect + * @param rtpState the RTP state (i.e. seqnum, timestamp to start with, etc). + */ + GenericAdaptiveSourceProjectionContext( + int payloadType, + @NotNull RtpState rtpState, + @NotNull Logger parentLogger) + { + this.payloadType = payloadType; + this.ssrc = rtpState.ssrc; + this.maxDestinationSequenceNumber = rtpState.maxSequenceNumber; + this.maxDestinationTimestamp = rtpState.maxTimestamp; + this.logger = parentLogger.createChildLogger( + GenericAdaptiveSourceProjectionContext.class.getName()); + } + + /** + * Determines whether an RTP packet from the incoming source should be accepted + * or not. If the source is currently suspended, a key frame is necessary to + * start accepting packets again. + *

+ * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread (the translator + * thread) accessing this method at a time. + * + * @param packetInfo the RTP packet to determine whether to accept or not. + * @param targetIndex the target quality index + * @return true if the packet should be accepted, false otherwise. + */ + @Override + public synchronized boolean + accept(@NotNull PacketInfo packetInfo, int targetIndex) + { + VideoRtpPacket rtpPacket = packetInfo.packetAs(); + if (targetIndex == RtpLayerDesc.SUSPENDED_INDEX) + { + // suspend the stream. + needsKeyframe = true; + return false; + } + + int sourceSequenceNumber = rtpPacket.getSequenceNumber(); + + boolean accept; + if (needsKeyframe) + { + if (!(rtpPacket instanceof ParsedVideoPacket)) + { + /* We don't know how to parse this packet's codec type, so + * isKeyframe will never be true. The best we can do is + * to start forwarding it immediately. Hopefully the receiver + * will send a PLI/FIR if it needs a keyframe. + */ + needsKeyframe = false; + accept = true; + } + else if (((ParsedVideoPacket)rtpPacket).isKeyframe()) + { + needsKeyframe = false; + // resume after being suspended, we compute the new seqnum delta + // delta = destination - source <=> destination = source + delta + // In other words, we compute the using this formula + // "delta = destination - source" and in order to find the + // destination sequence number we use the equivalent formula + // "destination = source + delta". + int destinationSequenceNumber + = maxDestinationSequenceNumber + 1; + sequenceNumberDelta + = RtpUtils.getSequenceNumberDelta( + destinationSequenceNumber, + sourceSequenceNumber); + + logger.trace(() -> "delta ssrc=" + rtpPacket.getSsrc() + + ",src_sequence=" + sourceSequenceNumber + + ",dst_sequence=" + destinationSequenceNumber + + ",max_sequence=" + maxDestinationSequenceNumber + + ",delta=" + sequenceNumberDelta); + + accept = true; + } + else + { + accept = false; + } + } + else + { + accept = true; + } + + if (accept) + { + maybeInitializeTimestampDelta(rtpPacket.getTimestamp()); + + int destinationSequenceNumber + = RtpUtils.applySequenceNumberDelta( + sourceSequenceNumber, sequenceNumberDelta); + + long destinationTimestamp + = RtpUtils.applyTimestampDelta( + rtpPacket.getTimestamp(), timestampDelta); + + if (RtpUtils.isOlderSequenceNumberThan( + maxDestinationSequenceNumber, destinationSequenceNumber)) + { + maxDestinationSequenceNumber = destinationSequenceNumber; + } + + if (RtpUtils.isNewerTimestampThan( + destinationTimestamp, maxDestinationTimestamp)) + { + maxDestinationTimestamp = destinationTimestamp; + } + + logger.trace(() -> "accept ssrc=" + rtpPacket.getSsrc() + + ",src_sequence=" + sourceSequenceNumber + + ",dst_sequence=" + destinationSequenceNumber + + ",max_sequence=" + maxDestinationSequenceNumber); + } + else + { + logger.trace(() -> "reject ssrc=" + rtpPacket.getSsrc() + + ",src_sequence=" + sourceSequenceNumber); + } + + return accept; + } + + /** + * Initializes {@link #timestampDelta} if it hasn't been initialized + * already. + * @param sourceTimestamp + */ + private synchronized void maybeInitializeTimestampDelta(long sourceTimestamp) + { + if (timestampDeltaInitialized) + { + return; + } + + if (RtpUtils.isNewerTimestampThan( + maxDestinationTimestamp, sourceTimestamp)) + { + long destinationTimestamp = + RtpUtils.applyTimestampDelta(maxDestinationTimestamp, 3000); + + timestampDelta + = RtpUtils.getTimestampDiff( + destinationTimestamp, sourceTimestamp); + } + + timestampDeltaInitialized = true; + } + + /** + * @return true when a source has been resumed (after being suspended). + */ + @Override + public boolean needsKeyframe() + { + return needsKeyframe; + } + + /** + * Applies a delta to the sequence number of the RTP packet that is + * specified as an argument in order to make suspending/resuming of the + * incoming source transparent at the RTP level. + * + * @param packetInfo the RTP packet info to rewrite. + */ + @Override + public void rewriteRtp( + @NotNull PacketInfo packetInfo) + { + VideoRtpPacket rtpPacket = packetInfo.packetAs(); + int sourceSequenceNumber = rtpPacket.getSequenceNumber(); + int destinationSequenceNumber + = RtpUtils.applySequenceNumberDelta( + sourceSequenceNumber, sequenceNumberDelta); + + if (sourceSequenceNumber != destinationSequenceNumber) + { + rtpPacket.setSequenceNumber(destinationSequenceNumber); + } + + long sourceTimestamp = rtpPacket.getTimestamp(); + long destinationTimestamp + = RtpUtils.applyTimestampDelta(sourceTimestamp, timestampDelta); + + if (sourceTimestamp != destinationTimestamp) + { + rtpPacket.setTimestamp(destinationTimestamp); + } + + logger.debug(() -> "rewrite ssrc=" + rtpPacket.getSsrc() + + ",src_sequence=" + sourceSequenceNumber + + ",dst_sequence=" + destinationSequenceNumber + + ",max_sequence=" + maxDestinationSequenceNumber); + } + + /** + * If the first RTCP packet of the compound RTCP packet that is specified as + * a parameter is an SR, then this method updates the transmitted bytes and + * transmitted packets of that first SR. + * + * @param rtcpSrPacket the compound RTCP packet to rewrite. + * @return true. + */ + @Override + public boolean rewriteRtcp(@NotNull RtcpSrPacket rtcpSrPacket) + { + return true; + } + + @Override + public RtpState getRtpState() + { + return new RtpState( + ssrc, maxDestinationSequenceNumber, maxDestinationTimestamp); + } + + public int getPayloadType() + { + return payloadType; + } + + /** + * {@inheritDoc} + * @return + */ + @Override + public ObjectNode getDebugState() + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + debugState.put( + "class", + GenericAdaptiveSourceProjectionContext.class.getSimpleName()); + debugState.put("TODO", "export more state (or refactor)"); + + return debugState; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/RewriteException.java b/jvb/src/main/java/org/jitsi/videobridge/cc/RewriteException.java new file mode 100644 index 0000000..cf2a57c --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/RewriteException.java @@ -0,0 +1,35 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc; + +/** + * Thrown if the underlying code failed to rewrite an RTP packet. + * + * @author George Politis + */ +public class RewriteException + extends Exception +{ + public RewriteException() + { + super(); + } + + public RewriteException(String message) + { + super(message); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/RtpState.java b/jvb/src/main/java/org/jitsi/videobridge/cc/RtpState.java new file mode 100644 index 0000000..ee7d3be --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/RtpState.java @@ -0,0 +1,52 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc; + +/** + * This class represents the RTP state and is used in adaptive source context + * switches (see {@link AdaptiveSourceProjection}). + * + * @author George Politis + */ +public class RtpState +{ + /** + * The SSRC of the RTP stream that this information pertains to. + */ + public final long ssrc; + + /** + * The highest sent RTP timestamp. + */ + public final long maxTimestamp; + + /** + * The highest sent sequence number. + */ + public final int maxSequenceNumber; + + /** + * @param ssrc the SSRC of the RTP stream that this information pertains to. + * @param maxSequenceNumber the highest sent sequence number. + * @param maxTimestamp the highest sent RTP timestamp. + */ + public RtpState(long ssrc, int maxSequenceNumber, long maxTimestamp) + { + this.ssrc = ssrc; + this.maxSequenceNumber = maxSequenceNumber; + this.maxTimestamp = maxTimestamp; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java new file mode 100644 index 0000000..e46cc68 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionContext.java @@ -0,0 +1,731 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp8; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; +import org.jitsi.nlj.codec.vpx.*; +import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.rtp.rtcp.*; +import org.jitsi.rtp.util.*; +import org.jitsi.utils.logging.*; +import org.jitsi.utils.logging2.Logger; +import org.jitsi.videobridge.cc.*; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.time.*; +import java.util.*; + +/** + * This class represents a projection of a VP8 RTP stream in the RFC 7667 sense + * and it is the main entry point for VP8 simulcast/svc RTP/RTCP rewriting. Read + * svc.md for implementation details. Instances of this class are thread-safe. + * + * @author George Politis + */ +public class VP8AdaptiveSourceProjectionContext + implements AdaptiveSourceProjectionContext +{ + private final Logger logger; + + /** + * A map that stores the per-encoding VP8 frame maps. + */ + private final Map + vp8FrameMaps = new HashMap<>(); + + /** + * The {@link VP8QualityFilter} instance that does quality filtering on the + * incoming frames. + */ + private final VP8QualityFilter vp8QualityFilter; + + /** + * The diagnostic context of this instance. + */ + private final DiagnosticContext diagnosticContext; + + /** + * The "last" {@link VP8FrameProjection} that this instance has accepted. + * In this context, last here means with the highest sequence number + * and not, for example, the last one received by the bridge. + */ + private VP8FrameProjection lastVP8FrameProjection; + + /** + * Ctor. + * + * @param rtpState the RTP state to begin with. + */ + public VP8AdaptiveSourceProjectionContext( + @NotNull DiagnosticContext diagnosticContext, + @NotNull RtpState rtpState, + @NotNull Logger parentLogger) + { + this.diagnosticContext = diagnosticContext; + this.logger = parentLogger.createChildLogger( + VP8AdaptiveSourceProjectionContext.class.getName()); + this.vp8QualityFilter = new VP8QualityFilter(parentLogger); + + lastVP8FrameProjection = new VP8FrameProjection(diagnosticContext, + rtpState.ssrc, rtpState.maxSequenceNumber, rtpState.maxTimestamp); + } + + /** Lookup a Vp8Frame for a packet. */ + private VP8Frame lookupVP8Frame(@NotNull Vp8Packet vp8Packet) + { + VP8FrameMap frameMap = vp8FrameMaps.get(vp8Packet.getSsrc()); + if (frameMap == null) + return null; + + return frameMap.findFrame(vp8Packet); + } + + /** + * Insert a packet in the appropriate Vp8FrameMap. + */ + private VP8FrameMap.FrameInsertionResult insertPacketInMap( + @NotNull Vp8Packet vp8Packet) + { + VP8FrameMap frameMap = vp8FrameMaps.computeIfAbsent(vp8Packet.getSsrc(), + ssrc -> new VP8FrameMap(logger)); + /* TODO: add more context (ssrc?) to frame map's logger? */ + + return frameMap.insertPacket(vp8Packet); + } + + /** + * Find the previous frame before the given one. + */ + @Nullable + private synchronized VP8Frame prevFrame(@NotNull VP8Frame frame) + { + VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc()); + if (frameMap == null) + { + return null; + } + + return frameMap.prevFrame(frame); + } + + /** + * Find the next frame after the given one. + */ + @Nullable + private synchronized VP8Frame nextFrame(@NotNull VP8Frame frame) + { + VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc()); + if (frameMap == null) + { + return null; + } + + return frameMap.nextFrame(frame); + } + + /** + * Find the previous accepted frame before the given one. + */ + @Nullable + private VP8Frame findPrevAcceptedFrame(@NotNull VP8Frame frame) + { + VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc()); + if (frameMap == null) + { + return null; + } + + return frameMap.findPrevAcceptedFrame(frame); + } + + /** + * Find the next accepted frame after the given one. + */ + @Nullable + private VP8Frame findNextAcceptedFrame(@NotNull VP8Frame frame) + { + VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc()); + if (frameMap == null) + { + return null; + } + + return frameMap.findNextAcceptedFrame(frame); + } + + /** + * Find a subsequent Tid==0 frame after the given frame + * @param frame The frame to query + * @return A subsequent TL0 frame, or null + */ + @Nullable + private VP8Frame findNextTl0(@NotNull VP8Frame frame) + { + VP8FrameMap frameMap = vp8FrameMaps.get(frame.getSsrc()); + if (frameMap == null) + { + return null; + } + + return frameMap.findNextTl0(frame); + } + + /** + * Calculate the projected sequence number gap between two frames (of the same encoding), + * allowing collapsing for unaccepted frames. + */ + private int seqGap(@NotNull VP8Frame frame1, @NotNull VP8Frame frame2) + { + int seqGap = RtpUtils.getSequenceNumberDelta( + frame2.getEarliestKnownSequenceNumber(), + frame1.getLatestKnownSequenceNumber() + ); + + if (!frame1.isAccepted() && !frame2.isAccepted() && + frame2.isImmediatelyAfter(frame1)) + { + /* If neither frame is being projected, and they have consecutive + picture IDs, we don't need to leave any gap. */ + seqGap = 0; + } + else + { + /* If the earlier frame wasn't projected, and we haven't seen its + * final packet, we know it has to consume at least one more sequence number. */ + if (!frame1.isAccepted() && !frame1.hasSeenEndOfFrame() && seqGap > 1) + { + seqGap--; + } + /* Similarly, if the later frame wasn't projected and we haven't seen + * its first packet. */ + if (!frame2.isAccepted() && !frame2.hasSeenStartOfFrame() && seqGap > 1) + { + seqGap--; + } + if (!frame1.isAccepted() && seqGap > 0) + { + seqGap--; + } + } + + return seqGap; + } + + /** + * Calculate the projected picture ID gap between two frames (of the same encoding), + * allowing collapsing for unaccepted frames. + */ + private int picGap(@NotNull VP8Frame frame1, @NotNull VP8Frame frame2) + { + int picGap = VpxUtils.getExtendedPictureIdDelta(frame2.getPictureId(), frame1.getPictureId()); + + if (!frame1.isAccepted() && picGap > 0) + { + picGap--; + } + + return picGap; + } + + + private boolean frameIsNewSsrc(VP8Frame frame) + { + return lastVP8FrameProjection.getVP8Frame() == null || + !frame.matchesSSRC(lastVP8FrameProjection.getVP8Frame()); + } + + /** + * Determines whether a packet should be accepted or not. + * + * @param packetInfo the RTP packet to determine whether to project or not. + * @param targetIndex the target quality index we want to achieve + * @return true if the packet should be accepted, false otherwise. + */ + @Override + public synchronized boolean accept( + @NotNull PacketInfo packetInfo, int targetIndex) + { + if (!(packetInfo.getPacket() instanceof Vp8Packet)) + { + logger.warn("Packet is not VP8 packet"); + return false; + } + Vp8Packet vp8Packet = packetInfo.packetAs(); + int incomingEncoding = vp8Packet.getEncodingId(); + + VP8FrameMap.FrameInsertionResult result = insertPacketInMap(vp8Packet); + + if (result == null) + { + /* Very old frame, more than Vp8FrameMap.FRAME_MAP_SIZE old, + or something wrong with the stream. */ + return false; + } + + VP8Frame frame = result.getFrame(); + + if (result.isNewFrame()) + { + if (vp8Packet.isKeyframe() && frameIsNewSsrc(frame)) + { + /* If we're not currently projecting this SSRC, check if we've + already decided to drop a subsequent TL0 frame of this SSRC. + If we have, we can't turn on the encoding starting from this + packet, so treat this frame as though it weren't a keyframe. + */ + VP8Frame f = findNextTl0(frame); + if (f != null && !f.isAccepted()) + { + frame.setKeyframe(false); + } + } + + Instant receivedTime = packetInfo.getReceivedTime(); + boolean accepted = vp8QualityFilter + .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime); + + if (accepted) + { + accepted = checkDecodability(frame); + } + + frame.setAccepted(accepted); + + if (accepted) + { + VP8FrameProjection projection; + try + { + projection = createProjection(frame, vp8Packet, result.isReset(), + receivedTime); + } + catch (Exception e) + { + logger.warn("Failed to create frame projection", e); + /* Make sure we don't have an accepted frame without a projection in the map. */ + frame.setAccepted(false); + return false; + } + frame.setProjection(projection); + + if (RtpUtils.isNewerSequenceNumberThan(projection.getEarliestProjectedSequence(), + lastVP8FrameProjection.getLatestProjectedSequence())) + { + lastVP8FrameProjection = projection; + } + } + } + + return frame.isAccepted() && frame.getProjection().accept(vp8Packet); + } + + /** + * For a frame that's been accepted by the quality filter, verify that + * it's decodable given the projection decisions about previous frames + * (in case the targetIndex has changed). + */ + private boolean checkDecodability(@NotNull VP8Frame frame) + { + if (frame.isKeyframe() || frame.getTemporalLayer() <= 0) + { + /* We'll always project all TL0 or unknown-TL frames, and TL0PICIDX lets the + * decoder know if it's missed something, so no need to check. + */ + return true; + } + + VP8Frame f = frame, prev; + + while ((prev = prevFrame(f)) != null) + { + if (!f.isImmediatelyAfter(prev)) + { + /* If we have a gap in the projection history, we don't know + * what will be sent. Default to assuming it'll be decodable. + */ + return true; + } + if (prev.isKeyframe() || prev.getTemporalLayer() <= frame.getTemporalLayer()) + { + /* Assume temporal nesting - if the previous frame of a lower + * or equal layer was accepted, this frame is decodable, otherwise + * it probably isn't. + */ + return prev.isAccepted(); + } + + f = prev; + } + /* If we ran off the beginning of our history, we don't know what was + * sent before; assume it'll be decodable. */ + return true; + } + + /** + * Create a projection for this frame. + */ + @NotNull + private VP8FrameProjection createProjection( + @NotNull VP8Frame frame, + @NotNull Vp8Packet initialPacket, + boolean isReset, + @Nullable Instant receivedTime) + { + if (frameIsNewSsrc(frame)) + { + return createLayerSwitchProjection(frame, initialPacket, receivedTime); + } + + else if (isReset) + { + return createResetProjection(frame, initialPacket, receivedTime); + } + + return createInLayerProjection(frame, initialPacket, receivedTime); + } + + /** + * Create a projection for this frame. It is the first frame sent for a layer. + */ + @NotNull + private VP8FrameProjection createLayerSwitchProjection( + @NotNull VP8Frame frame, + @NotNull Vp8Packet initialPacket, + @Nullable Instant receivedTime) + { + assert(frame.isKeyframe()); + assert(initialPacket.isStartOfFrame()); + + int projectedSeqGap = 1; + + if (lastVP8FrameProjection.getVP8Frame() != null && + !lastVP8FrameProjection.getVP8Frame().hasSeenEndOfFrame()) + { + /* Leave a gap to signal to the decoder that the previously routed + frame was incomplete. */ + projectedSeqGap++; + + /* Make sure subsequent packets of the previous projection won't + overlap the new one. (This means the gap, above, will never be + filled in.) + */ + lastVP8FrameProjection.close(); + } + + int projectedSeq = + RtpUtils.applySequenceNumberDelta(lastVP8FrameProjection.getLatestProjectedSequence(), projectedSeqGap); + + // this is a simulcast switch. The typical incremental value = + // 90kHz / 30 = 90,000Hz / 30 = 3000 per frame or per 33ms + long tsDelta; + if (receivedTime != null && lastVP8FrameProjection.getCreated() != null) + { + tsDelta = 3000 * Math.max(1, + Duration.between(lastVP8FrameProjection.getCreated(), receivedTime).dividedBy(33).toMillis()); + } + else + { + tsDelta = 3000; + } + long projectedTs = RtpUtils.applyTimestampDelta(lastVP8FrameProjection.getTimestamp(), tsDelta); + + int picId; + int tl0PicIdx; + if (lastVP8FrameProjection.getVP8Frame() != null) + { + picId = VpxUtils.applyExtendedPictureIdDelta(lastVP8FrameProjection.getPictureId(), + 1); + tl0PicIdx = VpxUtils.applyTl0PicIdxDelta(lastVP8FrameProjection.getTl0PICIDX(), + 1); + } + else + { + picId = frame.getPictureId(); + tl0PicIdx = frame.getTl0PICIDX(); + } + + VP8FrameProjection projection = + new VP8FrameProjection(diagnosticContext, + frame, lastVP8FrameProjection.getSSRC(), projectedTs, + RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.getSequenceNumber()), + picId, tl0PicIdx, receivedTime + ); + + return projection; + } + + /** + * Create a projection for this frame. It follows a large gap in the stream's projected frames. + */ + @NotNull + private VP8FrameProjection createResetProjection(@NotNull VP8Frame frame, + @NotNull Vp8Packet initialPacket, @Nullable Instant receivedTime) + { + VP8Frame lastFrame = lastVP8FrameProjection.getVP8Frame(); + + /* Apply the latest projected frame's projections out, linearly. */ + int seqDelta = RtpUtils.getSequenceNumberDelta(lastVP8FrameProjection.getLatestProjectedSequence(), + lastFrame.getLatestKnownSequenceNumber()); + long tsDelta = RtpUtils.getTimestampDiff(lastVP8FrameProjection.getTimestamp(), + lastFrame.getTimestamp()); + int picIdDelta = VpxUtils.getExtendedPictureIdDelta(lastVP8FrameProjection.getPictureId(), + lastFrame.getPictureId()); + int tl0PicIdxDelta = VpxUtils.getTl0PicIdxDelta(lastVP8FrameProjection.getTl0PICIDX(), + lastFrame.getTl0PICIDX()); + + long projectedTs = RtpUtils.applyTimestampDelta(frame.getTimestamp(), tsDelta); + int projectedPicId = VpxUtils.applyExtendedPictureIdDelta(frame.getPictureId(), picIdDelta); + int projectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(frame.getTl0PICIDX(), tl0PicIdxDelta); + + VP8FrameProjection projection = + new VP8FrameProjection(diagnosticContext, + frame, lastVP8FrameProjection.getSSRC(), projectedTs, + seqDelta, + projectedPicId, projectedTl0PicIdx, receivedTime + ); + + return projection; + } + + /** + * Create a projection for this frame. It is being sent subsequent to other projected frames + * of this layer. + */ + @NotNull + private VP8FrameProjection createInLayerProjection(@NotNull VP8Frame frame, + @NotNull VP8Frame refFrame, @NotNull Vp8Packet initialPacket, + @Nullable Instant receivedTime) + { + long tsGap = RtpUtils.getTimestampDiff(frame.getTimestamp(), refFrame.getTimestamp()); + int tl0Gap = VpxUtils.getTl0PicIdxDelta(frame.getTl0PICIDX(), refFrame.getTl0PICIDX()); + int seqGap = 0; + int picGap = 0; + + VP8Frame f1 = refFrame, f2; + int refSeq; + int picIdDelta = VpxUtils.getExtendedPictureIdDelta(refFrame.getPictureId(), frame.getPictureId()); + if (picIdDelta < 0) + { + do + { + f2 = nextFrame(f1); + if (f2 == null) + { + throw new IllegalStateException("No next frame found after frame with picId " + f1.getPictureId() + + ", even though refFrame " + refFrame.getPictureId() + " is before frame " + + frame.getPictureId() + "!"); + } + seqGap += seqGap(f1, f2); + picGap += picGap(f1, f2); + f1 = f2; + } + while (f2 != frame); + refSeq = refFrame.getProjection().getLatestProjectedSequence(); + } + else + { + do + { + f2 = prevFrame(f1); + if (f2 == null) + { + throw new IllegalStateException("No previous frame found before frame with picId " + + f1.getPictureId() + ", even though refFrame " + refFrame.getPictureId() + " is after frame " + + frame.getPictureId() + "!"); + } + seqGap += -seqGap(f2, f1); + picGap += -picGap(f2, f1); + f1 = f2; + } + while (f2 != frame); + refSeq = refFrame.getProjection().getEarliestProjectedSequence(); + } + + int projectedSeq = RtpUtils.applySequenceNumberDelta(refSeq, seqGap); + long projectedTs = RtpUtils.applyTimestampDelta(refFrame.getProjection().getTimestamp(), tsGap); + int projectedPicId = VpxUtils.applyExtendedPictureIdDelta(refFrame.getProjection().getPictureId(), picGap); + int projectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(refFrame.getProjection().getTl0PICIDX(), tl0Gap); + + VP8FrameProjection projection = + new VP8FrameProjection(diagnosticContext, + frame, lastVP8FrameProjection.getSSRC(), projectedTs, + RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.getSequenceNumber()), + projectedPicId, projectedTl0PicIdx, receivedTime + ); + + return projection; + } + + /** + * Create a projection for this frame. It is being sent subsequent to other projected frames + * of this layer. + */ + @NotNull + private VP8FrameProjection createInLayerProjection( + @NotNull VP8Frame frame, + @NotNull Vp8Packet initialPacket, + @Nullable Instant receivedTime) + { + VP8Frame prevFrame = findPrevAcceptedFrame(frame); + if (prevFrame != null) + { + return createInLayerProjection(frame, prevFrame, initialPacket, receivedTime); + } + /* prev frame has rolled off beginning of frame map, try next frame */ + VP8Frame nextFrame = findNextAcceptedFrame(frame); + if (nextFrame != null) + { + return createInLayerProjection(frame, nextFrame, initialPacket, receivedTime); + } + + /* Neither previous or next is found. Very big frame? Use previous projected. + (This must be valid because we don't execute this function unless + frameIsNewSsrc has returned false.) + */ + return createInLayerProjection(frame, lastVP8FrameProjection.getVP8Frame(), + initialPacket, receivedTime); + } + + @Override + public boolean needsKeyframe() + { + if (vp8QualityFilter.needsKeyframe()) + { + return true; + } + + if (lastVP8FrameProjection.getVP8Frame() == null) + { + /* Never sent anything */ + return true; + } + return false; + } + + /** + * Rewrites the RTCP packet that is specified as an argument. + * + * @param rtcpSrPacket the RTCP packet to transform. + * @return true if the RTCP packet is accepted, false otherwise, in which + * case it needs to be dropped. + */ + @Override + public boolean rewriteRtcp(@NotNull RtcpSrPacket rtcpSrPacket) + { + VP8FrameProjection lastVP8FrameProjectionCopy = lastVP8FrameProjection; + if (lastVP8FrameProjectionCopy.getVP8Frame() == null + || rtcpSrPacket.getSenderSsrc() != lastVP8FrameProjectionCopy.getVP8Frame().getSsrc()) + { + return false; + } + + rtcpSrPacket.setSenderSsrc(lastVP8FrameProjectionCopy.getSSRC()); + + long srcTs = rtcpSrPacket.getSenderInfo().getRtpTimestamp(); + long delta = RtpUtils.getTimestampDiff( + lastVP8FrameProjectionCopy.getTimestamp(), + lastVP8FrameProjectionCopy.getVP8Frame().getTimestamp()); + + long dstTs = RtpUtils.applyTimestampDelta(srcTs, delta); + + if (srcTs != dstTs) + { + rtcpSrPacket.getSenderInfo().setRtpTimestamp(dstTs); + } + + return true; + } + + @Override + public RtpState getRtpState() + { + return new RtpState( + lastVP8FrameProjection.getSSRC(), + lastVP8FrameProjection.getLatestProjectedSequence(), + lastVP8FrameProjection.getTimestamp()); + } + + /** + * Rewrites the RTP packet that is specified as an argument. + * + * @param packetInfo the packet info for the RTP packet to rewrite. + * @throws RewriteException if a VP8 frame projection is not found + * for the RTP packet that is specified as a parameter. + */ + @Override + public void rewriteRtp( + @NotNull PacketInfo packetInfo) + throws RewriteException + { + if (!(packetInfo.getPacket() instanceof Vp8Packet)) + { + logger.info("Got a non-VP8 packet."); + throw new RewriteException("Non-VP8 packet in VP8 source projection"); + } + Vp8Packet vp8Packet = packetInfo.packetAs(); + + if (vp8Packet.getPictureId() == -1) + { + /* Should have been rejected in accept(). */ + logger.info("VP8 packet does not have picture ID, cannot track in frame map."); + throw new RewriteException("VP8 packet without picture ID in VP8 source projection"); + } + + VP8Frame vp8Frame = lookupVP8Frame(vp8Packet); + if (vp8Frame == null) + { + // This packet does not belong to an accepted frame. + // Possibly it aged off the frame map since accept was called? + throw new RewriteException("Frame not in tracker (aged off?)"); + } + + if (vp8Frame.getProjection() == null) + { + /* Shouldn't happen for an accepted packet whose frame is still known? */ + throw new RewriteException("Frame does not have projection?"); + } + + vp8Frame.getProjection().rewriteRtp(vp8Packet); + } + + /** + * {@inheritDoc} + */ + @Override + public synchronized ObjectNode getDebugState() + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + debugState.put( + "class", + VP8AdaptiveSourceProjectionContext.class.getSimpleName()); + + ArrayNode mapSizes = JsonNodeFactory.instance.arrayNode(); + for (Map.Entry entry: vp8FrameMaps.entrySet()) + { + ObjectNode sizeInfo = JsonNodeFactory.instance.objectNode(); + sizeInfo.put("ssrc", entry.getKey()); + sizeInfo.put("size", entry.getValue().size()); + mapSizes.add(sizeInfo); + } + debugState.set("vp8FrameMaps", mapSizes); + debugState.set("vp8QualityFilter", vp8QualityFilter.getDebugState()); + + return debugState; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8Frame.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8Frame.java new file mode 100644 index 0000000..026d44e --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8Frame.java @@ -0,0 +1,412 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp8; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.codec.vpx.*; +import org.jitsi.nlj.rtp.*; +import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.rtp.util.*; + +/** + * Groups together some RTP/VP8 fields that refer to a specific incoming VP8 + * frame. Most of those fields are final and cannot be changed, with the + * exception of the ending and max sequence number that may be unknown at the + * time of the creation of this instance. + * + * Instances of this class are *NOT* thread safe. While most internal state of + * this class instances is final, the sequence number ranges, haveStart/haveEnd, + * and isKeyframe are not. + * + * @author George Politis + * @author Jonathan Lennox + */ +class VP8Frame +{ + /** + * The RTP SSRC of the incoming frame that this instance refers to + * (RFC3550). + */ + private final long ssrc; + + /** + * The RTP timestamp of the incoming frame that this instance refers to + * (RFC3550). + */ + private final long timestamp; + + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + private int earliestKnownSequenceNumber; + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + private int latestKnownSequenceNumber; + + /** + * A boolean that indicates whether or not we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + private boolean seenStartOfFrame; + + /** + * A boolean that indicates whether or not we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + private boolean seenEndOfFrame; + + /** + * The temporal layer of this frame. + */ + private final int temporalLayer; + + /** + * The VP8 PictureID of the incoming VP8 frame that this instance refers to + * (RFC7741). + */ + private final int pictureId; + + /** + * The VP8 TL0PICIDX of the incoming VP8 frame that this instance refers to + * (RFC7741). + */ + private final int tl0PICIDX; + + /** + * A boolean that indicates whether the incoming VP8 frame that this + * instance refers to is a keyframe (RFC7741). + */ + private boolean isKeyframe; + + /** + * A record of how this frame was projected, or null if not. + */ + private VP8FrameProjection projection; + + /** + * A boolean that records whether this frame was accepted. + */ + private boolean accepted; + + /** + * Ctor. + * + * @param packet A packet from the frame to be constructed. + */ + VP8Frame(@NotNull Vp8Packet packet) + { + this.ssrc = packet.getSsrc(); + this.timestamp = packet.getTimestamp(); + this.earliestKnownSequenceNumber = packet.getSequenceNumber(); + this.latestKnownSequenceNumber = packet.getSequenceNumber(); + this.seenStartOfFrame = packet.isStartOfFrame(); + this.seenEndOfFrame = packet.isEndOfFrame(); + + this.tl0PICIDX = packet.getTL0PICIDX(); + this.isKeyframe = packet.isKeyframe(); + this.pictureId = packet.getPictureId(); + this.temporalLayer = packet.getTemporalLayerIndex(); + } + + /** + * Remember another packet of this frame. + * Note: this assumes every packet is received only once, i.e. a filter + * like {@link org.jitsi.nlj.transform.node.incoming.PaddingTermination} is in use. + * @param packet The packet to remember. This should be a packet which + * has tested true with {@link #matchesFrame(Vp8Packet)}. + */ + void addPacket(@NotNull Vp8Packet packet) + { + if (!matchesFrame(packet)) + { + throw new IllegalArgumentException("Non-matching packet added to frame"); + } + int seq = packet.getSequenceNumber(); + if (RtpUtils.isOlderSequenceNumberThan(seq, earliestKnownSequenceNumber)) + { + earliestKnownSequenceNumber = seq; + } + if (RtpUtils.isNewerSequenceNumberThan(seq, latestKnownSequenceNumber)) + { + latestKnownSequenceNumber = seq; + } + if (packet.isStartOfFrame()) + { + seenStartOfFrame = true; + } + if (packet.isEndOfFrame()) + { + seenEndOfFrame = true; + } + } + + /** + * @return true if the incoming VP8 frame that this instance refers to is a + * keyframe (RFC7741), false otherwise. + */ + boolean isKeyframe() + { + return isKeyframe; + } + + /** + * Update a frame's notion of whether it is a keyframe. + */ + void setKeyframe(boolean k) + { + isKeyframe = k; + } + + /** + * @return the temporal layer of this frame. + */ + int getTemporalLayer() + { + return temporalLayer; + } + + /** + * @return true if this is a base temporal layer frame, false otherwise + * @note We treat unknown temporal layer frames as TL0. + */ + boolean isTL0() + { + return temporalLayer <= 0; + } + + /** + * Gets the SSRC. + */ + long getSsrc() + { + return ssrc; + } + + /** + * Gets the timestamp. + */ + long getTimestamp() + { + return timestamp; + } + + /** + * Gets the earliest sequence number seen for this frame. + */ + public int getEarliestKnownSequenceNumber() + { + return earliestKnownSequenceNumber; + } + + /** + * Gets the latest sequence number seen for this frame. + */ + public int getLatestKnownSequenceNumber() + { + return latestKnownSequenceNumber; + } + + /** + * Whether the start of this frame has been seen. + */ + public boolean hasSeenStartOfFrame() + { + return seenStartOfFrame; + } + + /** + * Whether the end of this frame has been seen. + */ + public boolean hasSeenEndOfFrame() + { + return seenEndOfFrame; + } + + /** + * Get the projection of this frame, or null. + */ + public VP8FrameProjection getProjection() + { + return projection; + } + + /** + * Get whether this frame has been accepted. + */ + public boolean isAccepted() + { + return accepted; + } + + /** + * Set whether this frame has been accepted. + */ + public void setAccepted(boolean a) + { + accepted = a; + } + + /** + * Set the projection record of this frame. + */ + public void setProjection(VP8FrameProjection projection) + { + this.projection = projection; + } + + /** + * Get the picture ID of this frame. + */ + public int getPictureId() + { + return pictureId; + } + + /** + * Get the tl0picidx of this frame. + */ + public int getTl0PICIDX() + { + return tl0PICIDX; + } + + /** + * Small utility method that checks whether the {@link VP8Frame} that is + * specified as a parameter belongs to the same RTP stream as the frame that + * this instance refers to. + * + * @param vp8Frame the {@link VP8Frame} to check whether it belongs to the + * same RTP stream as the frame that this instance refers to. + * @return true if the {@link VP8Frame} that is specified as a parameter + * belongs to the same RTP stream as the frame that this instance refers to, + * false otherwise. + */ + boolean matchesSSRC(@NotNull VP8Frame vp8Frame) + { + return ssrc == vp8Frame.ssrc; + } + + /** + * Determines whether the {@link VideoRtpPacket} that is specified as an + * argument is part of the VP8 picture that is represented by this + * {@link VP8Frame} instance. + * + * @param pkt the {@link VideoRtpPacket} instance to check whether it's part + * of the VP8 picture that is represented by this {@link VP8Frame} + * instance. + * @return true if the {@link VideoRtpPacket} that is specified as an + * argument is part of the VP8 picture that is represented by this + * {@link VP8Frame} instance, false otherwise. + */ + private boolean matchesSSRC(@NotNull VideoRtpPacket pkt) + { + return ssrc == pkt.getSsrc(); + } + + /** + * Checks whether the specified RTP packet is part of this frame. + * + * @param pkt the RTP packet to check whether it's part of this frame. + * @return true if the specified RTP packet is part of this frame, false + * otherwise. + */ + boolean matchesFrame(@NotNull Vp8Packet pkt) + { + return matchesSSRC(pkt) && timestamp == pkt.getTimestamp(); + } + + /** + * Validates that the specified RTP packet consistently matches all the + * parameters of this frame. + * + * This can be useful for diagnosing invalid streams if this fails when + * {@link #matchesFrame(Vp8Packet)} is true. + * + * @param pkt the RTP packet to check whether its parameters match this frame. + * @throws RuntimeException if the specified RTP packet is inconsistent with this frame + */ + void validateConsistency(@NotNull Vp8Packet pkt) + { + if (temporalLayer == pkt.getTemporalLayerIndex() && + tl0PICIDX == pkt.getTL0PICIDX() && + pictureId == pkt.getPictureId()) + /* TODO: also check start, end, seq nums? */ + { + return; + } + + StringBuilder s = new StringBuilder().append("Packet") + .append(" ssrc ").append(pkt.getSsrc()) + .append(", seq ").append(pkt.getSequenceNumber()) + .append(", picture id ").append(pkt.getPictureId()) + .append(", timestamp ").append(pkt.getTimestamp()) + .append(" is not consistent with frame") + .append(" ssrc ").append(ssrc) + .append(", seq ").append(earliestKnownSequenceNumber).append("-").append(latestKnownSequenceNumber) + .append(", picture id ").append(pictureId) + .append(", timestamp ").append(timestamp) + .append(": "); + + boolean complained = false; + + if (temporalLayer != pkt.getTemporalLayerIndex()) + { + s.append("packet temporal layer ") + .append(pkt.getTemporalLayerIndex()) + .append(" != frame temporal layer ") + .append(temporalLayer); + complained = true; + } + if (tl0PICIDX != pkt.getTL0PICIDX()) + { + if (complained) + { + s.append("; "); + } + s.append("packet TL0PICIDX ") + .append(pkt.getTL0PICIDX()) + .append(" != frame TL0PICIDX ") + .append(tl0PICIDX); + complained = true; + } + if (pictureId != pkt.getPictureId()) + { + if (complained) + { + s.append("; "); + } + s.append("packet PictureID ") + .append(pkt.getPictureId()) + .append(" != frame PictureID ") + .append(pictureId); + } + throw new RuntimeException(s.toString()); + } + + /** + * Check whether this frame is immediately after another one, according + * to their extended picture IDs. + */ + boolean isImmediatelyAfter(@NotNull VP8Frame otherFrame) + { + return pictureId == + VpxUtils.applyExtendedPictureIdDelta(otherFrame.getPictureId(), 1); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameMap.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameMap.java new file mode 100644 index 0000000..bf70b2a --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameMap.java @@ -0,0 +1,400 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp8; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.codec.vpx.*; +import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.nlj.util.*; +import org.jitsi.rtp.util.*; +import org.jitsi.utils.logging2.*; + +import java.time.*; +import java.util.function.*; + +import static java.lang.Long.max; +import static java.lang.Long.min; + +/** + * A history of recent frames on a VP8 stream. + */ +public class VP8FrameMap +{ + static final int FRAME_MAP_SIZE = 500; /* Matches PacketCache default size. */ + + /** Cache mapping picture IDs to frames. */ + private final FrameHistory frameHistory = new FrameHistory(FRAME_MAP_SIZE); + + private final Logger logger; + + /** + * Ctor. + * + */ + public VP8FrameMap( + @NotNull Logger parentLogger) + { + this.logger = parentLogger.createChildLogger(VP8FrameMap.class.getName()); + } + + /** Find a frame in the frame map, based on a packet. */ + public synchronized VP8Frame findFrame(@NotNull Vp8Packet packet) + { + return frameHistory.get(packet.getPictureId()); + } + + /** Get the current size of the map. */ + public int size() + { + return frameHistory.numCached; + } + + /** Helper function to insert a packet into an existing frame. */ + @NotNull + @Contract("_, _ -> new") + private FrameInsertionResult doFrameInsert(@NotNull VP8Frame frame, Vp8Packet packet) + { + try + { + frame.validateConsistency(packet); + } + catch (Exception e) + { + logger.warn(e); + } + frame.addPacket(packet); + return new FrameInsertionResult(frame, false, false); + } + + /** Check whether this is a large jump from previous state, so the map should be reset. */ + private boolean isLargeJump(@NotNull Vp8Packet packet) + { + VP8Frame latestFrame = frameHistory.getLatestFrame(); + if (latestFrame == null) + { + return false; + } + + int picDelta = VpxUtils.getExtendedPictureIdDelta(packet.getPictureId(), latestFrame.getPictureId()); + if (picDelta > FRAME_MAP_SIZE) + { + return true; + } + + long tsDelta = RtpUtils.getTimestampDiff(packet.getTimestamp(), latestFrame.getTimestamp()); + + if (picDelta < 0) + { + /* if picDelta is negative but timestamp or sequence delta is positive, we've cycled. */ + if (tsDelta > 0) + { + return true; + } + if (RtpUtils.getSequenceNumberDelta( + packet.getSequenceNumber(), latestFrame.getLatestKnownSequenceNumber()) > 0) + { + return true; + } + } + + /* If tsDelta is more than twice the frame map size at 1 fps, we've cycled. */ + if (tsDelta > FRAME_MAP_SIZE * 90000 * 2) + { + return true; + } + + return false; + } + + /** Insert a packet into the frame map. Return a FrameInsertionResult + * describing what happened. + * @param packet The packet to insert. + * @return What happened. null if insertion failed. + */ + public synchronized FrameInsertionResult insertPacket(@NotNull Vp8Packet packet) + { + int pictureId = packet.getPictureId(); + + if (pictureId == -1) + { + /* Frame map indexes by picture ID. All supported browsers should currently be setting it. */ + /* Log message will have been logged by Vp8Parser in jmt. */ + return null; + } + + if (isLargeJump(packet)) + { + frameHistory.indexTracker.resetAt(pictureId); + + VP8Frame frame = new VP8Frame(packet); + + if (!frameHistory.insert(pictureId, frame)) + { + return null; + } + + return new FrameInsertionResult(frame, true, true); + } + + VP8Frame frame = frameHistory.get(pictureId); + if (frame != null) + { + if (!frame.matchesFrame(packet)) + { + if (frame.getPictureId() != pictureId) + { + throw new IllegalStateException("Frame map returned frame with picture ID " + + frame.getPictureId() + + " when asked for frame with picture ID " + pictureId); + } + logger.warn("Cannot insert packet in frame map: " + + "frame with ssrc " + frame.getSsrc() + + ", timestamp " + frame.getTimestamp() + + ", and sequence number range " + frame.getEarliestKnownSequenceNumber() + + "-" + frame.getLatestKnownSequenceNumber() + + ", and packet " + packet.getSequenceNumber() + + " with ssrc " + packet.getSsrc() + + ", timestamp " + packet.getTimestamp() + + ", and sequence number " + packet.getSequenceNumber() + + " both have picture ID " + pictureId); + return null; + } + return doFrameInsert(frame, packet); + } + + frame = new VP8Frame(packet); + + if (!frameHistory.insert(pictureId, frame)) + { + return null; + } + + return new FrameInsertionResult(frame, true, false); + } + + @Nullable + public synchronized VP8Frame nextFrame(@NotNull VP8Frame frame) + { + return frameHistory.findAfter(frame, (VP8Frame f) -> true ); + } + + @Nullable + public synchronized VP8Frame nextFrameWith(@NotNull VP8Frame frame, Predicate pred) + { + return frameHistory.findAfter(frame, pred); + } + + @Nullable + public synchronized VP8Frame findNextTl0(@NotNull VP8Frame frame) + { + return nextFrameWith(frame, VP8Frame::isTL0); + } + + @Nullable + public synchronized VP8Frame findNextAcceptedFrame(@NotNull VP8Frame frame) + { + return nextFrameWith(frame, VP8Frame::isAccepted); + } + + @Nullable + public synchronized VP8Frame prevFrame(@NotNull VP8Frame frame) + { + return frameHistory.findBefore(frame, (VP8Frame f) -> true ); + } + + @Nullable + public synchronized VP8Frame prevFrameWith(@NotNull VP8Frame frame, Predicate pred) + { + return frameHistory.findBefore(frame, pred); + } + + @Nullable + public synchronized VP8Frame findPrevAcceptedFrame(@NotNull VP8Frame frame) + { + return prevFrameWith(frame, VP8Frame::isAccepted); + } + + /** + * The result of calling {@link #insertPacket(Vp8Packet).} + */ + public static class FrameInsertionResult + { + /** The frame corresponding to the packet that was inserted. */ + private VP8Frame frame; + + /** Whether inserting the frame created a new frame. */ + private boolean newFrame; + + /** Whether inserting the frame caused a reset */ + private boolean reset; + + /** Construct a FrameInsertionResult. */ + private FrameInsertionResult(VP8Frame frame, boolean newFrame, boolean reset) + { + this.frame = frame; + this.newFrame = newFrame; + this.reset = reset; + } + + /** Get the frame corresponding to the packet that was inserted. */ + public VP8Frame getFrame() + { + return frame; + } + + /** Get whether inserting the frame created a new frame. */ + public boolean isNewFrame() + { + return newFrame; + } + + /** Get whether inserting the frame caused a reset. */ + public boolean isReset() + { + return reset; + } + } + + private static class FrameHistory extends ArrayCache + { + FrameHistory(int size) + { + super(size, (k) -> k, false, Clock.systemUTC()); + } + + int numCached = 0; + long firstIndex = -1; + + PictureIdIndexTracker indexTracker = new PictureIdIndexTracker(); + + /** + * Gets a frame with a given VP8 picture ID from the cache. + */ + public VP8Frame get(int pictureId) + { + long index = indexTracker.interpret(pictureId); + return getIndex(index); + } + + /** + * Gets a frame with a given VP8 picture ID index from the cache. + */ + private VP8Frame getIndex(long index) + { + if (index <= getLastIndex() - getSize()) + { + /* We don't want to remember old frames even if they're still + tracked; their neighboring frames may have been evicted, + so findBefore / findAfter will return bogus data. */ + return null; + } + ArrayCache.Container c = getContainer(index); + if (c == null) + { + return null; + } + return c.getItem(); + } + + /** Get the latest frame in the tracker. */ + private VP8Frame getLatestFrame() + { + return getIndex(getLastIndex()); + } + + public boolean insert(int pictureId, VP8Frame frame) + { + long index = indexTracker.update(pictureId); + boolean ret = super.insertItem(frame, index); + if (ret) + { + numCached++; + if (firstIndex == -1 || index < firstIndex) + { + firstIndex = index; + } + } + return ret; + } + + /** + * Called when an item in the cache is replaced/discarded. + */ + @Override + protected void discardItem(VP8Frame frame) + { + numCached--; + } + + @Nullable + public VP8Frame findBefore(VP8Frame frame, Predicate pred) + { + long lastIndex = getLastIndex(); + if (lastIndex == -1) + { + return null; + } + + long index = indexTracker.interpret(frame.getPictureId()); + + long searchStartIndex = min(index - 1, lastIndex); + long searchEndIndex = max(lastIndex - getSize(), firstIndex - 1); + + return doFind(pred, searchStartIndex, searchEndIndex, -1); + } + + @Nullable + public VP8Frame findAfter(VP8Frame frame, Predicate pred) + { + long lastIndex = getLastIndex(); + if (lastIndex == -1) + { + return null; + } + + long index = indexTracker.interpret(frame.getPictureId()); + + if (index >= lastIndex) + { + return null; + } + + long searchStartIndex = max(index + 1, max(lastIndex - getSize() + 1, firstIndex)); + + return doFind(pred, searchStartIndex, lastIndex + 1, 1); + } + + @Nullable + private VP8Frame doFind(Predicate pred, long startIndex, long endIndex, int increment) + { + if (!((increment > 0 && startIndex <= endIndex) || (increment < 0 && startIndex >= endIndex))) + { + throw new IllegalArgumentException("Values of startIndex=" + startIndex + ", endIndex=" + endIndex + + " , and increment=" + increment + " could lead to infinite loop"); + } + + for (long index = startIndex; index != endIndex; index += increment) + { + VP8Frame frame = getIndex(index); + if (frame != null && pred.test(frame)) + { + return frame; + } + } + return null; + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameProjection.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameProjection.java new file mode 100644 index 0000000..efa7fdd --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8FrameProjection.java @@ -0,0 +1,297 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp8; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.rtp.util.*; +import org.jitsi.utils.logging.*; + +import java.time.*; + +/** + * Represents a VP8 frame projection. It puts together all the necessary bits + * and pieces that are useful when projecting an accepted VP8 frame. A + * projection is responsible for rewriting a VP8 packet. Instances of this class + * are thread-safe. + * + * @author George Politis + */ +public class VP8FrameProjection +{ + /** + * The time series logger for this instance. + */ + private static final TimeSeriesLogger timeSeriesLogger + = TimeSeriesLogger.getTimeSeriesLogger(VP8FrameProjection.class); + + /** + * The diagnostic context for this instance. + */ + private final DiagnosticContext diagnosticContext; + + /** + * A timestamp of when this instance was created. It's used to calculate + * RTP timestamps when we switch encodings. + */ + private final @Nullable Instant created; + + /** + * The projected {@link VP8Frame}. + */ + private final VP8Frame vp8Frame; + + /** + * The RTP SSRC of the projection (RFC7667, RFC3550). + */ + private final long ssrc; + + /** + * The RTP timestamp of the projection (RFC7667, RFC3550). + */ + private final long timestamp; + + /** + * The sequence number delta for packets of this frame. + */ + private final int sequenceNumberDelta; + + /** + * The VP8 picture ID of the projection (RFC7667, RFC7741). + */ + private final int extendedPictureId; + + /** + * The VP8 TL0PICIDX of the projection (RFC7741). + */ + private final int tl0PICIDX; + + /** + * -1 if this projection is still "open" for new, later packets. + * Projections can be closed when we switch away from their encodings. + */ + private int closedSeq = -1; + + /** + * Ctor. + * + * @param ssrc the SSRC of the destination VP8 picture. + * @param timestamp The RTP timestamp of the projected frame that this + * instance refers to (RFC3550). + * @param sequenceNumberDelta The starting RTP sequence number of the + * projected frame that this instance refers to (RFC3550). + */ + VP8FrameProjection( + @NotNull DiagnosticContext diagnosticContext, + long ssrc, int sequenceNumberDelta, long timestamp) + { + this(diagnosticContext, null /* vp8Frame */, ssrc, timestamp, + sequenceNumberDelta, 0 /* extendedPictureId */, + 0 /* tl0PICIDX */, null /* created */); + } + + /** + * Ctor. + * + * @param vp8Frame The {@link VP8Frame} that's projected. + * @param ssrc The RTP SSRC of the projected frame that this instance refers + * to (RFC3550). + * @param timestamp The RTP timestamp of the projected frame that this + * instance refers to (RFC3550). + * @param sequenceNumberDelta The starting RTP sequence number of the + * projected frame that this instance refers to (RFC3550). + * @param extendedPictureId The VP8 extended picture ID of the projected VP8 + * frame that this instance refers to (RFC7741). + * @param tl0PICIDX The VP8 TL0PICIDX of the projected VP8 frame that this + * instance refers to (RFC7741). + */ + VP8FrameProjection( + @NotNull DiagnosticContext diagnosticContext, + VP8Frame vp8Frame, + long ssrc, long timestamp, int sequenceNumberDelta, + int extendedPictureId, int tl0PICIDX, @Nullable Instant created) + { + this.diagnosticContext = diagnosticContext; + this.ssrc = ssrc; + this.timestamp = timestamp; + this.sequenceNumberDelta = sequenceNumberDelta; + this.extendedPictureId = extendedPictureId; + this.tl0PICIDX = tl0PICIDX; + this.vp8Frame = vp8Frame; + this.created = created; + } + + public int rewriteSeqNo(int seq) + { + return RtpUtils.applySequenceNumberDelta(seq, sequenceNumberDelta); + } + + /** + * Rewrites an RTP packet. + * + * @param pkt the RTP packet to rewrite. + */ + void rewriteRtp(@NotNull Vp8Packet pkt) + { + int sequenceNumber = rewriteSeqNo(pkt.getSequenceNumber()); + + if (timeSeriesLogger.isTraceEnabled()) + { + timeSeriesLogger.trace(diagnosticContext + .makeTimeSeriesPoint("rtp_vp8_rewrite") + .addField("orig.rtp.ssrc", pkt.getSsrc()) + .addField("orig.rtp.timestamp", pkt.getTimestamp()) + .addField("orig.rtp.seq", pkt.getSequenceNumber()) + .addField("orig.vp8.pictureid", pkt.getPictureId()) + .addField("orig.vp8.tl0picidx", pkt.getTL0PICIDX()) + .addField("proj.rtp.ssrc", ssrc) + .addField("proj.rtp.timestamp", timestamp) + .addField("proj.rtp.seq", sequenceNumber) + .addField("proj.vp8.pictureid", extendedPictureId) + .addField("proj.vp8.tl0picidx", tl0PICIDX)); + } + + // update ssrc, sequence number, timestamp, pictureId and tl0picidx + pkt.setSsrc(ssrc); + pkt.setTimestamp(timestamp); + + pkt.setSequenceNumber(sequenceNumber); + + if (pkt.getTL0PICIDX() != -1) + { + pkt.setTL0PICIDX(tl0PICIDX); + } + pkt.setPictureId(extendedPictureId); + + } + + /** + * Determines whether a packet can be forwarded as part of this + * {@link VP8FrameProjection} instance. The check is based on the sequence + * of the incoming packet and whether or not the {@link VP8FrameProjection} + * has been "closed" or not. + * + * @param rtpPacket the {@link Vp8Packet} that will be examined. + * @return true if the packet can be forwarded as part of this + * {@link VP8FrameProjection}, false otherwise. + */ + public boolean accept(@NotNull Vp8Packet rtpPacket) + { + if (vp8Frame == null || !vp8Frame.matchesFrame(rtpPacket)) + { + // The packet does not belong to this VP8 picture. + return false; + } + + synchronized (vp8Frame) + { + if (closedSeq < 0) + { + return true; + } + + return RtpUtils + .isOlderSequenceNumberThan(rtpPacket.getSequenceNumber(), + closedSeq); + } + } + + /** + * @return The projected {@link VP8Frame}. + */ + VP8Frame getVP8Frame() + { + return vp8Frame; + } + + /** + * @return The RTP SSRC of this projection. + */ + public long getSSRC() + { + return ssrc; + } + + /** + * @return The RTP timestamp of this projection. + */ + public long getTimestamp() + { + return timestamp; + } + + /** + * @return The picture ID of this projection. + */ + public int getPictureId() + { + return extendedPictureId; + } + + /** + * @return The TL0PICIDX of this projection. + */ + public int getTl0PICIDX() + { + return tl0PICIDX; + } + + /** + * @return The system time (in ms) this projection was created. + */ + public @Nullable Instant getCreated() + { + return created; + } + + public int getEarliestProjectedSequence() + { + if (vp8Frame == null) + { + return sequenceNumberDelta; + } + synchronized (vp8Frame) + { + return rewriteSeqNo(vp8Frame.getEarliestKnownSequenceNumber()); + } + } + + public int getLatestProjectedSequence() + { + if (vp8Frame == null) + { + return sequenceNumberDelta; + } + synchronized (vp8Frame) + { + return rewriteSeqNo(vp8Frame.getLatestKnownSequenceNumber()); + } + } + + /** + * Prevents the max sequence number of this frame to grow any further. + */ + public void close() + { + if (vp8Frame != null) + { + synchronized (vp8Frame) + { + closedSeq = vp8Frame.getLatestKnownSequenceNumber(); + } + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java new file mode 100644 index 0000000..2149386 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/cc/vp8/VP8QualityFilter.java @@ -0,0 +1,387 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp8; + +import edu.umd.cs.findbugs.annotations.*; +import org.jetbrains.annotations.*; +import org.jetbrains.annotations.Nullable; +import org.jitsi.nlj.*; +import org.jitsi.utils.logging2.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.lang.SuppressWarnings; +import java.time.*; + +/** + * This class is responsible for dropping VP8 simulcast/svc packets based on + * their quality, i.e. packets that correspond to qualities that are above a + * given quality target. Instances of this class are thread-safe. + * + * @author George Politis + */ +class VP8QualityFilter +{ + /** + * The {@link Logger} to be used by this instance to print debug + * information. + */ + private final Logger logger; + + /** + * The default maximum frequency at which the media engine + * generates key frame. + */ + private static final Duration MIN_KEY_FRAME_WAIT = Duration.ofMillis(300); + + /** + * The HD, SD, LD and suspended spatial/quality layer IDs. + */ + private static final int SUSPENDED_ENCODING_ID = -1; + + /** + * Holds the arrival time of the most recent keyframe group. + * Reading/writing of this field is synchronized on this instance. + */ + private @Nullable Instant mostRecentKeyframeGroupArrivalTime = null; + + /** + * A boolean flag that indicates whether a simulcast switch is pending. This + * condition is equivalent to: + * + * internalSpatialLayerIdTarget != externalSpatialLayerIdTarget. + * + * Reading/writing of this field is synchronized on this instance. + */ + private boolean needsKeyframe = false; + + /** + * The encoding id that this instance tries to achieve. Upon + * receipt of a packet, we check whether externalSpatialLayerIdTarget + * (that's specified as an argument to the + * {@link #acceptFrame(VP8Frame, int, int, Instant)} method) is set to something + * different, in which case we set {@link #needsKeyframe} equal to true and + * update. + */ + private int internalEncodingIdTarget = SUSPENDED_ENCODING_ID; + + /** + * The encoding layer ID that we're currently forwarding. -1 + * indicates that we're not forwarding anything. Reading/writing of this + * field is synchronized on this instance. + */ + private int currentEncodingId = SUSPENDED_ENCODING_ID; + + public VP8QualityFilter(Logger parentLogger) + { + this.logger = parentLogger.createChildLogger(VP8QualityFilter.class.getName()); + } + + /** + * @return true if a the target encoding id has changed and a + * keyframe hasn't been received yet, false otherwise. + */ + boolean needsKeyframe() + { + return needsKeyframe; + } + + /** + * Determines whether to accept or drop a VP8 frame. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param frame the VP8 frame. + * @param incomingEncoding the encoding index of the incoming RTP packet + * @param externalTargetIndex the target quality index that the user of this + * instance wants to achieve. + * @param receivedTime the current time + * @return true to accept the VP8 frame, otherwise false. + */ + synchronized boolean acceptFrame( + @NotNull VP8Frame frame, + int incomingEncoding, + int externalTargetIndex, Instant receivedTime) + { + // We make local copies of the externalTemporalLayerIdTarget and the + // externalEncodingTarget (as they may be updated by some other + // thread). + int externalTemporalLayerIdTarget + = RtpLayerDesc.getTidFromIndex(externalTargetIndex); + int externalEncodingIdTarget + = RtpLayerDesc.getEidFromIndex(externalTargetIndex); + + if (externalEncodingIdTarget != internalEncodingIdTarget) + { + // The externalEncodingIdTarget has changed since accept last + // run; perhaps we should request a keyframe. + internalEncodingIdTarget = externalEncodingIdTarget; + if (externalEncodingIdTarget > SUSPENDED_ENCODING_ID) + { + needsKeyframe = true; + } + } + + if (externalEncodingIdTarget < 0 + || externalTemporalLayerIdTarget < 0) + { + // We stop forwarding immediately. We will need a keyframe in order + // to resume. + currentEncodingId = SUSPENDED_ENCODING_ID; + return false; + } + + int temporalLayerIdOfFrame = frame.getTemporalLayer(); + + if (temporalLayerIdOfFrame < 0) + { + // temporal scalability is not enabled. Pretend that + // this is the base temporal layer. + temporalLayerIdOfFrame = 0; + } + + if (frame.isKeyframe()) + { + logger.debug(() -> "Quality filter got keyframe for stream " + + frame.getSsrc()); + return acceptKeyframe(incomingEncoding, receivedTime); + } + else if (currentEncodingId > SUSPENDED_ENCODING_ID) + { + if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding)) + { + // XXX(george) i've noticed some "rogue" base layer keyframes + // that trigger this. what happens is the client sends a base + // layer key frame, the bridge switches to that layer because + // for all it knows it may be the only keyframe sent by the + // client engine. then the bridge notices that packets from the + // higher quality streams are flowing and execution ends-up + // here. it is a mystery why the engine is "leaking" base layer + // key frames + needsKeyframe = true; + } + + if (incomingEncoding != currentEncodingId) + { + // for non-keyframes, we can't route anything but the current encoding + return false; + } + + // This branch reads the {@link #currentEncodingId} and it + // filters packets based on their temporal layer. + + if (currentEncodingId > externalEncodingIdTarget) + { + // pending downscale, decrease the frame rate until we + // downscale. + return temporalLayerIdOfFrame < 1; + } + else if (currentEncodingId < externalEncodingIdTarget) + { + // pending upscale, increase the frame rate until we upscale. + return true; + } + else + { + // The currentSpatialLayerId matches exactly the target + // currentSpatialLayerId. + return temporalLayerIdOfFrame <= externalTemporalLayerIdTarget; + } + } + else + { + // In this branch we're not processing a keyframe and the + // currentEncodingId is in suspended state, which means we need + // a keyframe to start streaming again. + + // We should have already requested a keyframe, either above or when the + // internal target encoding was first moved off SUSPENDED_ENCODING. + + return false; + } + } + + /** + * Returns a boolean that indicates whether we are in layer switching phase + * or not. + * + * @param receivedTime the time the latest frame was received + * @return false if we're in layer switching phase, true otherwise. + */ + private synchronized boolean isOutOfSwitchingPhase(@Nullable Instant receivedTime) + { + if (receivedTime == null) + { + return false; + } + if (mostRecentKeyframeGroupArrivalTime == null) + { + return true; + } + + Duration delta = Duration.between(mostRecentKeyframeGroupArrivalTime, receivedTime); + return delta.compareTo(MIN_KEY_FRAME_WAIT) > 0; + } + + /** + * @return true if it looks like we can re-scale (see implementation of + * method for specific details). + */ + private synchronized boolean isPossibleToSwitch(int encodingId) + { + if (encodingId == -1) + { + // We failed to resolve the spatial/quality layer of the packet. + return false; + } + + if (encodingId > currentEncodingId + && currentEncodingId < internalEncodingIdTarget) + { + // It looks like upscaling is possible. + return true; + } + else if (encodingId < currentEncodingId + && currentEncodingId > internalEncodingIdTarget) + { + // It looks like downscaling is possible. + return true; + } + else + { + return false; + } + } + + /** + * Determines whether to accept or drop a VP8 keyframe. This method updates + * the spatial layer id. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param receivedTime the time the frame was received + * @return true to accept the VP8 keyframe, otherwise false. + */ + private synchronized boolean acceptKeyframe( + int encodingIdOfKeyframe, @Nullable Instant receivedTime) + { + // This branch writes the {@link #currentSpatialLayerId} and it + // determines whether or not we should switch to another simulcast + // stream. + if (encodingIdOfKeyframe < 0) + { + // something went terribly wrong, normally we should be able to + // extract the layer id from a keyframe. + logger.error("unable to get layer id from keyframe"); + return false; + } + + logger.debug(() -> "Received a keyframe of encoding: " + + encodingIdOfKeyframe); + + + // The keyframe request has been fulfilled at this point, regardless of + // whether we'll be able to achieve the internalEncodingIdTarget. + needsKeyframe = false; + + if (isOutOfSwitchingPhase(receivedTime)) + { + // During the switching phase we always project the first + // keyframe because it may very well be the only one that we + // receive (i.e. the endpoint is sending low quality only). Then + // we try to approach the target. + + mostRecentKeyframeGroupArrivalTime = receivedTime; + + logger.debug(() -> "First keyframe in this kf group " + + "currentEncodingId: " + encodingIdOfKeyframe + + ". Target is " + internalEncodingIdTarget); + + if (encodingIdOfKeyframe <= internalEncodingIdTarget) + { + // If the target is 180p and the first keyframe of a group of + // keyframes is a 720p keyframe we don't project it. If we + // receive a 720p keyframe, we know that there MUST be a 180p + // keyframe shortly after. + currentEncodingId = encodingIdOfKeyframe; + return true; + } + else + { + return false; + } + } + else + { + // We're within the 300ms window since the reception of the + // first key frame of a key frame group, let's check whether an + // upscale/downscale is possible. + + if (currentEncodingId <= encodingIdOfKeyframe + && encodingIdOfKeyframe <= internalEncodingIdTarget) + { + // upscale or current quality case + currentEncodingId = encodingIdOfKeyframe; + logger.debug(() -> "Upscaling to encoding " + + encodingIdOfKeyframe + + ". The target is " + internalEncodingIdTarget); + return true; + } + else if (encodingIdOfKeyframe <= internalEncodingIdTarget + && internalEncodingIdTarget < currentEncodingId) + { + // downscale case + currentEncodingId = encodingIdOfKeyframe; + logger.debug(() -> " Downscaling to encoding " + + encodingIdOfKeyframe + ". The target is + " + + internalEncodingIdTarget); + return true; + } + else + { + return false; + } + } + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + @SuppressWarnings("unchecked") + @SuppressFBWarnings( + value = "IS2_INCONSISTENT_SYNC", + justification = "We intentionally avoid synchronizing while reading" + + " fields only used in debug output.") + public ObjectNode getDebugState() + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + debugState.put( + "mostRecentKeyframeGroupArrivalTimeMs", + mostRecentKeyframeGroupArrivalTime != null ? mostRecentKeyframeGroupArrivalTime.toEpochMilli() : -1L); + debugState.put("needsKeyframe", needsKeyframe); + debugState.put( + "internalEncodingIdTarget", + internalEncodingIdTarget); + debugState.put("currentEncodingId", currentEncodingId); + + return debugState; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannel.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannel.java new file mode 100644 index 0000000..297c20f --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannel.java @@ -0,0 +1,155 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel; + +import org.jitsi.rtp.extensions.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.datachannel.protocol.*; + +import java.nio.*; + +/** + * Models a WebRTC Data Channel + * + * @author Brian Baldino + */ +public class DataChannel +{ + private final DataChannelStack.DataChannelDataSender dataChannelDataSender; + protected final int channelType; + protected final int priority; + protected final long reliability; + protected final int sid; + protected final String label; + + protected boolean ready = false; + + protected final Logger logger; + + private DataChannelStack.DataChannelEventListener eventListener; + private DataChannelStack.DataChannelMessageListener messageListener; + + /** + * Initializes a new {@link DataChannel} instance. + */ + public DataChannel( + DataChannelStack.DataChannelDataSender dataChannelDataSender, + Logger parentLogger, + int channelType, int priority, long reliability, int sid, String label + ) + { + this.dataChannelDataSender = dataChannelDataSender; + this.logger = parentLogger.createChildLogger(this.getClass().getName()); + this.channelType = channelType; + this.priority = priority; + this.reliability = reliability; + this.sid = sid; + this.label = label; + } + + public void open() + { + OpenChannelMessage openMessage = new OpenChannelMessage( + channelType, + priority, + reliability, + label, + DataChannelProtocolConstants.PROTOCOL_STRING); + + ByteBuffer msg = openMessage.getBuffer(); + if (dataChannelDataSender.send(msg, sid, DataChannelProtocolConstants.WEBRTC_DCEP_PPID) < 0) + { + logger.error("Error sending data channel open message"); + } + } + + /** + * Checks if this data channel is ready. + */ + public boolean isReady() + { + return ready; + } + + /** + * Sets the listener for data channel events. + * @param listener + */ + public void onDataChannelEvents( + DataChannelStack.DataChannelEventListener listener) + { + this.eventListener = listener; + } + + /** + * Sets the message listener. + */ + public void onDataChannelMessage( + DataChannelStack.DataChannelMessageListener + dataChannelMessageListener) + { + this.messageListener = dataChannelMessageListener; + } + + /** + * Handles an incoming message. + * @param message + */ + public void onIncomingMsg(DataChannelMessage message) + { + if (message instanceof OpenChannelAckMessage) + { + ready = true; + eventListener.onDataChannelOpened(); + } + else if (message instanceof DataChannelStringMessage) + { + DataChannelStringMessage dataChannelStringMessage = (DataChannelStringMessage)message; + logger.debug(() -> "Received data channel string message: " + dataChannelStringMessage.data); + } + else if (message instanceof DataChannelBinaryMessage) + { + DataChannelBinaryMessage dataChannelBinaryMessage = (DataChannelBinaryMessage)message; + logger.debug(() -> "Received data channel binary message: " + + ByteBufferKt.toHex(ByteBuffer.wrap(dataChannelBinaryMessage.data))); + } + + messageListener.onDataChannelMessage(message); + } + + /** + * Sends data through the sender. + */ + protected int sendData(ByteBuffer data, int sid, int ppid) + { + return dataChannelDataSender.send(data, sid, ppid); + } + + /** + * Sends a string through this data channel. + * @param message the string to send. + */ + public void sendString(String message) + { + if (logger.isDebugEnabled()) + { + logger.debug("Sending string data channel message: '" + message + "'"); + } + DataChannelStringMessage stringMessage = new DataChannelStringMessage(message); + dataChannelDataSender.send(stringMessage.getBuffer(), sid, DataChannelProtocolConstants.WEBRTC_PPID_STRING); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannelStack.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannelStack.java new file mode 100644 index 0000000..75c2bb9 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/DataChannelStack.java @@ -0,0 +1,181 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel; + +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.datachannel.protocol.*; +import org.jitsi.videobridge.metrics.*; +import org.jitsi.videobridge.sctp.*; + +import java.nio.*; +import java.util.*; + +/** + * We need the stack to look at all incoming messages so that it can listen for an 'open channel' + * message from the remote side + * + * Handles DataChannel negotiation and the routing of all Data Channel messages + * to specific {@link DataChannel} instances. + * + * @author Brian Baldino + */ +//TODO: revisit thread safety in here +//TODO: add an arbitrary ID +public class DataChannelStack +{ + private final Map dataChannels = new HashMap<>(); + private final DataChannelDataSender dataChannelDataSender; + private final Logger logger; + private DataChannelStackEventListener listener; + + /** + * Initializes a new {@link DataChannelStack} with a specific sender. + * @param dataChannelDataSender the sender. + */ + public DataChannelStack(DataChannelDataSender dataChannelDataSender, Logger parentLogger) + { + this.dataChannelDataSender = dataChannelDataSender; + logger = parentLogger.createChildLogger(DataChannelStack.class.getName()); + } + + /** + * Handles a received packet. + */ + public void onIncomingDataChannelPacket(ByteBuffer data, int sid, int ppid) + { + logger.debug(() -> "Data channel stack received SCTP message"); + DataChannelMessage message = DataChannelProtocolMessageParser.parse(data.array(), ppid); + if (message instanceof OpenChannelMessage) + { + logger.debug("Received data channel open message"); + if (dataChannels.size() >= SctpConfig.config.getMaxChannels()) + { + logger.warn("Data channel limit exceeded."); + VideobridgeMetrics.rejectedDataChannels.inc(); + return; + } + + OpenChannelMessage openChannelMessage = (OpenChannelMessage)message; + // Remote side wants to open a channel + DataChannel dataChannel = new RemotelyOpenedDataChannel( + dataChannelDataSender, + logger, + openChannelMessage.channelType, + openChannelMessage.priority, + openChannelMessage.reliability, + sid, + openChannelMessage.label); + dataChannels.put(sid, dataChannel); + listener.onDataChannelOpenedRemotely(dataChannel); + } + else + { + DataChannel dataChannel= dataChannels.get(sid); + if (dataChannel == null) + { + logger.error("Could not find data channel for sid " + sid); + return; + } + dataChannel.onIncomingMsg(message); + } + } + + public void onDataChannelStackEvents(DataChannelStackEventListener listener) + { + this.listener = listener; + } + + /** + * Opens new WebRTC data channel using specified parameters. + * @param channelType channel type as defined in control protocol description. + * Use 0 for "reliable". + * @param priority channel priority. The higher the number, the lower + * the priority. + * @param reliability Reliability Parameter
+ * + * This field is ignored if a reliable channel is used. + * If a partial reliable channel with limited number of + * retransmissions is used, this field specifies the number of + * retransmissions. If a partial reliable channel with limited + * lifetime is used, this field specifies the maximum lifetime in + * milliseconds. The following table summarizes this:

+ + +------------------------------------------------+------------------+ + | Channel Type | Reliability | + | | Parameter | + +------------------------------------------------+------------------+ + | DATA_CHANNEL_RELIABLE | Ignored | + | DATA_CHANNEL_RELIABLE_UNORDERED | Ignored | + | DATA_CHANNEL_PARTIAL_RELIABLE_REXMIT | Number of RTX | + | DATA_CHANNEL_PARTIAL_RELIABLE_REXMIT_UNORDERED | Number of RTX | + | DATA_CHANNEL_PARTIAL_RELIABLE_TIMED | Lifetime in ms | + | DATA_CHANNEL_PARTIAL_RELIABLE_TIMED_UNORDERED | Lifetime in ms | + +------------------------------------------------+------------------+ + * @param sid SCTP stream id that will be used by new channel + * (it must not be already used). + * @param label text label for the channel. + * @return new instance of WebRtcDataStream that represents opened + * WebRTC data channel. + */ + public DataChannel createDataChannel(int channelType, int priority, long reliability, int sid, String label) + { + synchronized (dataChannels) + { + DataChannel dataChannel = new DataChannel( + dataChannelDataSender, logger, channelType, priority, reliability, sid, label); + dataChannels.put(sid, dataChannel); + return dataChannel; + } + } + + /** + * TODO: these 2 feel a bit awkward since they are so similar, but we use + * a different one for a remote channel (fired by the stack) and a + * locally-created channel (fired by the data channel itself). + */ + public interface DataChannelStackEventListener + { + /** + * The data channel was opened by the remote side. + */ + void onDataChannelOpenedRemotely(DataChannel dataChannel); + } + + public interface DataChannelEventListener + { + /** + * The data channel was opened. + */ + void onDataChannelOpened(); + } + + public interface DataChannelMessageListener + { + /** + * A message received. + */ + void onDataChannelMessage(DataChannelMessage dataChannelMessage); + } + + public interface DataChannelDataSender + { + /** + * Sends a message. + */ + int send(ByteBuffer data, int sid, int ppid); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/RemotelyOpenedDataChannel.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/RemotelyOpenedDataChannel.java new file mode 100644 index 0000000..819a7fb --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/RemotelyOpenedDataChannel.java @@ -0,0 +1,56 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel; + +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.datachannel.protocol.*; + +import java.nio.*; + +/** + * Same as {@link DataChannel} but automatically sends the open channel ack message upon creation + * + * @author Brian Baldino + */ +public class RemotelyOpenedDataChannel extends DataChannel +{ + /** + * Initializes a new {@link RemotelyOpenedDataChannel} instance. + */ + public RemotelyOpenedDataChannel( + DataChannelStack.DataChannelDataSender dataChannelDataSender, + Logger parentLogger, + int channelType, int priority, long reliability, int sid, + String label) + { + super(dataChannelDataSender, parentLogger, channelType, priority, + reliability, sid, label); + ready = true; + sendOpenChannelAck(); + } + + /** + * Sends an {@link OpenChannelAckMessage} message. + */ + protected void sendOpenChannelAck() + { + OpenChannelAckMessage openChannelAckMessage = new OpenChannelAckMessage(); + + ByteBuffer msg = openChannelAckMessage.getBuffer(); + sendData(msg, sid, DataChannelProtocolConstants.WEBRTC_DCEP_PPID); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/Transport.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/Transport.java new file mode 100644 index 0000000..cd40b24 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/Transport.java @@ -0,0 +1,65 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel; + +import java.nio.*; + +/** + * TODO: This needs documentation + * + * @author Brian Baldino + */ +public interface Transport +{ + interface DataChannelDataCallback + { + /** + * A packet was received. + */ + void dataChannelPacketReceived(ByteBuffer data, int sid, int ppid); + } + + interface DataChannelTransportEventHandler + { + /** + * The transport connected. + */ + void transportConnected(); + + /** + * The transport disconnected. + */ + void transportDisconnected(); + } + + /** + * Sets the handler for data. + * @param dataChannelDataCallback + */ + void onData(DataChannelDataCallback dataChannelDataCallback); + + /** + * Sends a {@link ByteBuffer}. + */ + int send(ByteBuffer data, boolean ordered, int sid, int ppid); + + /** + * Sets the handler for events. + */ + void onEvent( + DataChannelTransportEventHandler dataChannelTransportEventHandler); +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelBinaryMessage.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelBinaryMessage.java new file mode 100644 index 0000000..2482eea --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelBinaryMessage.java @@ -0,0 +1,45 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import java.nio.*; + +/** + * @author Brian Baldino + */ +public class DataChannelBinaryMessage extends DataChannelMessage +{ + public final byte[] data; + + /** + * Initializes a new {@link DataChannelBinaryMessage} from a byte array. + * @param data the byte array. + */ + public DataChannelBinaryMessage(byte[] data) + { + this.data = data; + } + + /** + * {@inheritDoc} + */ + @Override + public ByteBuffer getBuffer() + { + return ByteBuffer.wrap(data); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelMessage.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelMessage.java new file mode 100644 index 0000000..f141bde --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelMessage.java @@ -0,0 +1,30 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import java.nio.*; + +/** + * @author Brian Baldino + */ +public abstract class DataChannelMessage +{ + /** + * Gets the buffer which holds the message. + */ + public abstract ByteBuffer getBuffer(); +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java new file mode 100644 index 0000000..10f51aa --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelPacket.java @@ -0,0 +1,59 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import org.jetbrains.annotations.*; +import org.jitsi.dcsctp4j.DcSctpMessage; +import org.jitsi.rtp.*; + +/** + * @author Brian Baldino + */ +public class DataChannelPacket extends Packet +{ + public final int sid; + public final int ppid; + + /** + * Initializes a new {@link DataChannelPacket}. + */ + public DataChannelPacket( + byte[] data, int offset, int length, int sid, int ppid) + { + super(data, offset, length); + this.sid = sid; + this.ppid = ppid; + } + + public DataChannelPacket(DcSctpMessage message) + { + super(message.getPayload(), 0, message.getPayload().length); + this.sid = message.getStreamID(); + this.ppid = message.getPpid(); + } + + /** + * {@inheritDoc} + */ + @NotNull + @Override + public Packet clone() + { + return new DataChannelPacket( + getBuffer().clone(), getOffset(), getLength(), sid, ppid); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolConstants.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolConstants.java new file mode 100644 index 0000000..b0a4ee3 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolConstants.java @@ -0,0 +1,61 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +public class DataChannelProtocolConstants +{ + /** + * The String value of the Protocol field of the + * DATA_CHANNEL_OPEN message. + */ + public static final String PROTOCOL_STRING = "http://jitsi.org/protocols/colibri"; + + /** + * DataChannel message types + //TODO: enum? + */ + public static final int MSG_TYPE_CHANNEL_OPEN = 0x3; + public static final int MSG_TYPE_CHANNEL_ACK = 0x2; + + /** + * Channel types + * TODO: enum? + */ + public static final int RELIABLE = 0x00; + public static final int RELIABLE_UNORDERED = 0x80; + public static final int PARTIAL_RELIABLE_REXMIT = 0x01; + public static final int PARTIAL_RELIABLE_REXMIT_UNORDERED = 0x81; + public static final int PARTIAL_RELIABLE_TIMED = 0x02; + public static final int PARTIAL_RELIABLE_TIMED_UNORDERED = 0x82; + + /** + * This is the SCTP PPID used for datachannel establishment protocol + * https://tools.ietf.org/html/draft-ietf-rtcweb-data-protocol-08#section-8.1 + */ + public static final int WEBRTC_DCEP_PPID = 50; + + /** + * Payload protocol id that identifies text data UTF8 encoded in WebRTC data + * channels. + */ + public static final int WEBRTC_PPID_STRING = 51; + + /** + * Payload protocol id that identifies binary data in WebRTC data channel. + */ + public static final int WEBRTC_PPID_BIN = 53; +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessage.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessage.java new file mode 100644 index 0000000..cd424cf --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessage.java @@ -0,0 +1,68 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import java.nio.*; + +/** + * https://tools.ietf.org/html/draft-ietf-rtcweb-data-protocol-08#section-5 + * Base class for Data Channel 'protocol' messages; that is, messages + * used in Data Channel setup. + * + * @author Brian Baldino + */ +public class DataChannelProtocolMessage extends DataChannelMessage +{ + /** + * The only common field is the message type, so we'll call that the 'header' + */ + private static final int HEADER_SIZE_BYTES = 1; + static final int MAX_LABEL_LENGTH = 0xFFFF; + static final int MAX_PROTOCOL_LENGTH = 0xFFFF; + + private final int messageType; + + /** + * Initializes a new {@link DataChannelProtocolMessage} instance. + * @param messageType the message type. + */ + public DataChannelProtocolMessage(int messageType) + { + this.messageType = messageType; + } + + protected int getSizeBytes() + { + return HEADER_SIZE_BYTES; + } + + /** + * {@inheritDoc} + */ + @Override + public ByteBuffer getBuffer() + { + ByteBuffer buf = ByteBuffer.allocate(getSizeBytes()); + serialize(buf); + return buf; + } + + public void serialize(ByteBuffer destination) + { + destination.put((byte)messageType); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessageParser.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessageParser.java new file mode 100644 index 0000000..1f629b0 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelProtocolMessageParser.java @@ -0,0 +1,78 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import org.jitsi.utils.logging2.*; + +import java.nio.*; + +/** + * Parses Data Channel protocol messages + * + * @author Brian Baldino + */ +public class DataChannelProtocolMessageParser +{ + private static Logger logger = new LoggerImpl(DataChannelProtocolMessageParser.class.getName()); + /** + * Reads the message type from a byte array. + * @param data + * @return + */ + private static int getMessageType(byte[] data) + { + return ByteBuffer.wrap(data).get(0) & 0xFF; + } + + /** + * TODO(brian): change data to be a ByteBuffer + */ + public static DataChannelMessage parse(byte[] data, long ppid) + { + if (ppid == DataChannelProtocolConstants.WEBRTC_DCEP_PPID) + { + int messageType = getMessageType(data); + switch (messageType) + { + case DataChannelProtocolConstants.MSG_TYPE_CHANNEL_ACK: { + return new OpenChannelAckMessage(); + } + case DataChannelProtocolConstants.MSG_TYPE_CHANNEL_OPEN: { + return OpenChannelMessage.parse(data); + } + default: + { + logger.warn("Unrecognized datachannel control message type: " + messageType); + return null; + } + } + } + else if (ppid == DataChannelProtocolConstants.WEBRTC_PPID_STRING) + { + return DataChannelStringMessage.parse(data); + } + else if (ppid == DataChannelProtocolConstants.WEBRTC_PPID_BIN) + { + return new DataChannelBinaryMessage(data); + } + else + { + logger.warn("Unrecognized data channel ppid: " + ppid); + } + return null; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelStringMessage.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelStringMessage.java new file mode 100644 index 0000000..bb6cc01 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/DataChannelStringMessage.java @@ -0,0 +1,57 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import java.nio.*; +import java.nio.charset.*; + +/** + * @author Brian Baldino + */ +public class DataChannelStringMessage extends DataChannelMessage +{ + public final String data; + + /** + * Initializes a new {@link DataChannelStringMessage} instance. + * @param data + */ + public DataChannelStringMessage(String data) + { + this.data = data; + } + + /** + * Parses a {@link DataChannelStringMessage} from a byte array. + * @param data + * @return + */ + public static DataChannelStringMessage parse(byte[] data) + { + String stringData = new String(data, StandardCharsets.UTF_8); + return new DataChannelStringMessage(stringData); + } + + /** + * {@inheritDoc} + */ + @Override + public ByteBuffer getBuffer() + { + return ByteBuffer.wrap(data.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelAckMessage.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelAckMessage.java new file mode 100644 index 0000000..9573c83 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelAckMessage.java @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +/** + * @author Brian Baldino + */ +public class OpenChannelAckMessage extends DataChannelProtocolMessage +{ + /** + * Initializes a new {@link OpenChannelAckMessage} instance. + */ + public OpenChannelAckMessage() + { + super(DataChannelProtocolConstants.MSG_TYPE_CHANNEL_ACK); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelMessage.java b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelMessage.java new file mode 100644 index 0000000..703c1b4 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/datachannel/protocol/OpenChannelMessage.java @@ -0,0 +1,115 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.datachannel.protocol; + +import java.nio.*; +import java.nio.charset.*; + +/** + * TODO: this needs documentation + * @author Brian Baldino + */ +public class OpenChannelMessage extends DataChannelProtocolMessage +{ + public final int channelType; + public final int priority; + public final long reliability; + public final String label; + public final String protocol; + + /** + * Initializes a new {@link OpenChannelMessage} instance. + */ + public OpenChannelMessage( + int channelType, int priority, + long reliability, String label, String protocol) + { + super(DataChannelProtocolConstants.MSG_TYPE_CHANNEL_OPEN); + this.channelType = channelType; + this.priority = priority; + this.reliability = reliability; + this.label = label; + this.protocol = protocol; + } + + public static OpenChannelMessage parse(byte[] data) + { + ByteBuffer buf = ByteBuffer.wrap(data); + int channelType = buf.get(1); + int priority = buf.getShort(2); + long reliability = buf.getInt(4); + int labelLength = buf.getShort(8); + int protocolLength = buf.getShort(10); + String label = null; + if (labelLength > 0) + { + // The cast is needed when the code is compiled with JDK9+ and run on JRE8. + // https://stackoverflow.com/questions/48693695/java-nio-buffer-not-loading-clear-method-on-runtime + ((Buffer) buf).position(12); + byte[] labelBytes = new byte[labelLength]; + buf.get(labelBytes, 0, labelLength); + label = new String(labelBytes); + } + String protocol = null; + if (protocolLength > 0) + { + // The cast is needed when the code is compiled with JDK9+ and run on JRE8. + // https://stackoverflow.com/questions/48693695/java-nio-buffer-not-loading-clear-method-on-runtime + ((Buffer) buf).position(12 + labelLength); + byte[] protocolBytes = new byte[protocolLength]; + buf.get(protocolBytes, 0, protocolLength); + protocol = new String(protocolBytes); + } + return new OpenChannelMessage( + channelType, priority, reliability, label, protocol); + } + + /** + * {@inheritDoc} + */ + @Override + protected int getSizeBytes() + { + byte[] labelBytes = label.getBytes(StandardCharsets.UTF_8); + int labelLength = Math.min(labelBytes.length, MAX_LABEL_LENGTH); + byte[] protocolBytes = protocol.getBytes(StandardCharsets.UTF_8); + int protocolLength = Math.min(protocolBytes.length, MAX_PROTOCOL_LENGTH); + + return super.getSizeBytes() + 11 + labelLength + protocolLength; + } + + /** + * {@inheritDoc} + */ + @Override + public void serialize(ByteBuffer destination) + { + super.serialize(destination); + destination.put((byte)channelType); + destination.putShort((short)priority); + destination.putInt((int)reliability); + + byte[] labelBytes = label.getBytes(StandardCharsets.UTF_8); + int labelLength = Math.min(labelBytes.length, MAX_LABEL_LENGTH); + byte[] protocolBytes = protocol.getBytes(StandardCharsets.UTF_8); + int protocolLength = Math.min(protocolBytes.length, MAX_PROTOCOL_LENGTH); + destination.putShort((short)labelLength); + destination.putShort((short)protocolLength); + destination.put(labelBytes, 0, labelLength); + destination.put(protocolBytes, 0, protocolLength); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/ice/TransportUtils.java b/jvb/src/main/java/org/jitsi/videobridge/ice/TransportUtils.java new file mode 100644 index 0000000..eb3b739 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/ice/TransportUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.ice; + +import org.ice4j.ice.*; + +public class TransportUtils +{ + /** + * Determines whether at least one LocalCandidate of a specific ICE + * Component can reach (in the terms of the ice4j library) a + * specific RemoteCandidate + * + * @param component the ICE Component which contains the + * LocalCandidates to check whether at least one of them can reach + * the specified remoteCandidate + * @param remoteCandidate the RemoteCandidate to check whether at + * least one of the LocalCandidates of the specified + * component can reach it + * @return true if at least one LocalCandidate of the + * specified component can reach the specified + * remoteCandidate + */ + public static boolean canReach( + Component component, + RemoteCandidate remoteCandidate) + { + return component.getLocalCandidates().stream(). + anyMatch( + localCandidate -> localCandidate.canReach(remoteCandidate)); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/relay/RelayedPacketInfo.java b/jvb/src/main/java/org/jitsi/videobridge/relay/RelayedPacketInfo.java new file mode 100644 index 0000000..114dc3f --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/relay/RelayedPacketInfo.java @@ -0,0 +1,39 @@ +/* + * Copyright @ 2019-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.relay; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; +import org.jitsi.rtp.*; + +/** + * A packet received from a {@link Relay}. + */ +public class RelayedPacketInfo extends PacketInfo +{ + @Nullable private final String meshId; + + public RelayedPacketInfo(@NotNull Packet packet, @Nullable String meshId) + { + super(packet); + this.meshId = meshId; + } + + public String getMeshId() + { + return meshId; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/annotations/EnabledByConfig.java b/jvb/src/main/java/org/jitsi/videobridge/rest/annotations/EnabledByConfig.java new file mode 100644 index 0000000..00c0503 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/annotations/EnabledByConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.annotations; + +import org.jitsi.videobridge.rest.*; + +import java.lang.annotation.*; + +/** + * An annotation which allows a REST resource to be enabled/disabled by a property value in {@link RestConfig}. + * + * See {@link org.jitsi.videobridge.rest.filters.ConfigFilter} + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +@Inherited +public @interface EnabledByConfig +{ + RestApis value(); +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/BadRequestExceptionWithMessage.java b/jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/BadRequestExceptionWithMessage.java new file mode 100644 index 0000000..005271d --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/BadRequestExceptionWithMessage.java @@ -0,0 +1,35 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.exceptions; + +import jakarta.servlet.http.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; + +/** + * The ctor for {@link BadRequestException} which takes in a String doesn't + * actually include that String in the response. A much longer syntax (seen + * in the constructor below) is necessary. This class exists in order to expose + * that behavior in a more concise way + */ +public class BadRequestExceptionWithMessage extends BadRequestException +{ + public BadRequestExceptionWithMessage(String message) + { + super(Response.status(HttpServletResponse.SC_BAD_REQUEST, message).build()); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/InternalServerErrorExceptionWithMessage.java b/jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/InternalServerErrorExceptionWithMessage.java new file mode 100644 index 0000000..e6001a8 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/exceptions/InternalServerErrorExceptionWithMessage.java @@ -0,0 +1,35 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.exceptions; + +import jakarta.servlet.http.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; + +/** + * The ctor for {@link InternalServerErrorException} which takes in a String doesn't + * actually include that String in the response. A much longer syntax (seen + * in the constructor below) is necessary. This class exists in order to expose + * that behavior in a more concise way + */ +public class InternalServerErrorExceptionWithMessage extends InternalServerErrorException +{ + public InternalServerErrorExceptionWithMessage(String message) + { + super(Response.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, message).build()); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/filters/ConfigFilter.java b/jvb/src/main/java/org/jitsi/videobridge/rest/filters/ConfigFilter.java new file mode 100644 index 0000000..fe16e93 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/filters/ConfigFilter.java @@ -0,0 +1,49 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.filters; + +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; + +import jakarta.ws.rs.*; +import jakarta.ws.rs.container.*; +import jakarta.ws.rs.core.*; + +/** + * A filter which returns 404 not found for any path which: + * 1) Is marked with the {@link EnabledByConfig} annotation and + * 2) The resulting config value in {@link RestConfig} is `false`. + */ +public class ConfigFilter implements ContainerRequestFilter +{ + @Context + protected ResourceInfo resourceInfo; + + @Override + public void filter(ContainerRequestContext containerRequestContext) + { + Class clazz = resourceInfo.getResourceClass(); + if (clazz.isAnnotationPresent(EnabledByConfig.class)) + { + EnabledByConfig anno = clazz.getAnnotation(EnabledByConfig.class); + if (!RestConfig.config.isEnabled(anno.value())) + { + throw new NotFoundException(); + } + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java b/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java new file mode 100644 index 0000000..454b06f --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/prometheus/Prometheus.java @@ -0,0 +1,69 @@ +/* + * Copyright @ 2022 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.prometheus; + +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; +import kotlin.*; +import org.jetbrains.annotations.NotNull; +import org.jitsi.metrics.*; + +import java.util.*; +import java.util.stream.*; + +/** + * A REST endpoint exposing JVB stats for Prometheus. + * Any scraper supporting Prometheus' text-based formats ({@code text/plain; version=0.0.4} or OpenMetrics) + * is compatible with this {@code /metrics} endpoint.
+ * JSON is provided when the client performs a request with the 'Accept' header set to {@code application/json}.
+ * The response defaults to {@code text/plain; version=0.0.4} formatted output. + * + * @see + * Prometheus' exposition formats + */ +@Path("/metrics") +public class Prometheus +{ + static private final Comparator comparator = Comparator.comparing(Prometheus::getQValue).reversed(); + + private static double getQValue(MediaType m) + { + return m.getParameters().get("q") == null ? 1.0 : Double.parseDouble(m.getParameters().get("q")); + } + + @NotNull + private final MetricsContainer metricsContainer; + + public Prometheus(@NotNull MetricsContainer metricsContainer) + { + this.metricsContainer = metricsContainer; + } + + @GET + public Response x(@HeaderParam("Accept") String accept) + { + List acceptMediaTypes + = Arrays.stream(accept.split(",")) + .map(MediaType::valueOf) + .sorted(comparator) + .map(m -> m.getType() + "/" + m.getSubtype()) + .collect(Collectors.toList()); + Pair m = metricsContainer.getMetrics(acceptMediaTypes); + + return Response.ok(m.getFirst(), m.getSecond()).build(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java new file mode 100644 index 0000000..8e85f63 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/Application.java @@ -0,0 +1,67 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root; + +import org.glassfish.jersey.server.*; +import org.jetbrains.annotations.*; +import org.jitsi.utils.version.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.health.*; +import org.jitsi.videobridge.metrics.*; +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.binders.*; +import org.jitsi.videobridge.rest.filters.*; +import org.jitsi.videobridge.rest.prometheus.*; +import org.jitsi.videobridge.xmpp.*; + +import static org.jitsi.videobridge.rest.RestConfig.config; + +public class Application extends ResourceConfig +{ + public Application( + Videobridge videobridge, + XmppConnection xmppConnection, + @NotNull Version version, + @NotNull JvbHealthChecker healthChecker) + + { + register( + new ServiceBinder( + videobridge, + xmppConnection, + healthChecker + ) + ); + // Filters + register(ConfigFilter.class); + // Register all resources in the package + packages("org.jitsi.videobridge.rest.root"); + + if (config.isEnabled(RestApis.HEALTH)) + { + register(new org.jitsi.rest.Health(healthChecker)); + } + if (config.isEnabled(RestApis.VERSION)) + { + register(new org.jitsi.rest.Version(version)); + } + if (config.isEnabled(RestApis.PROMETHEUS)) + { + register(new Prometheus(VideobridgeMetricsContainer.getInstance())); + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/debug/Debug.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/debug/Debug.java new file mode 100644 index 0000000..d783577 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/debug/Debug.java @@ -0,0 +1,57 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.debug; + +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; + +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; +import jakarta.ws.rs.core.Response; +import java.net.*; + +/** + * We moved /colibri/debug to /debug. + */ +@Path("/colibri/debug") +@EnabledByConfig(RestApis.COLIBRI) +public class Debug +{ + @POST + @Path("{path:.+}") + public Response post(@PathParam("path") String path, @Context UriInfo uriInfo) + { + return create301(path, uriInfo); + } + + @GET + @Path("{path:.+}") + public Response get(@PathParam("path") String path, @Context UriInfo uriInfo) + { + return create301(path, uriInfo); + } + + private Response create301(String path, UriInfo uriInfo) + { + String target = uriInfo.getBaseUri() + "debug/" + path; + if (uriInfo.getQueryParameters().get("full") != null) + { + target += "?full=" + uriInfo.getQueryParameters().get("full"); + } + return Response.status(301).location(URI.create(target)).build(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/drain/Drain.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/drain/Drain.java new file mode 100644 index 0000000..bdf27b0 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/drain/Drain.java @@ -0,0 +1,75 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.drain; + +import org.eclipse.jetty.http.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import jakarta.inject.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; + +/** + * A resource for putting the videobridge in drain via REST. + */ +@Path("/colibri/drain") +@EnabledByConfig(RestApis.DRAIN) +public class Drain +{ + @Inject + protected Videobridge videobridge; + + private Response setDrainMode(boolean enable) + { + try + { + videobridge.setDrainMode(enable); + return Response.ok().build(); + } + catch (Throwable t) + { + return Response.status(HttpStatus.INTERNAL_SERVER_ERROR_500).build(); + } + } + + @POST + @Path("/enable") + public Response drainEnable() + { + return setDrainMode(true); + } + + @POST + @Path("/disable") + public Response drainDisable() + { + return setDrainMode(false); + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + public String getDrainState() + { + ObjectNode obj = JsonNodeFactory.instance.objectNode(); + obj.put("drain", videobridge.getDrainMode()); + return obj.toString(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClient.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClient.java new file mode 100644 index 0000000..c809144 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClient.java @@ -0,0 +1,100 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.mucclient; + +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; +import org.jitsi.videobridge.xmpp.*; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import jakarta.inject.*; +import jakarta.servlet.http.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; + +/** + * Add or remove XMPP environments to which the bridge will connect + */ +@Path("/colibri/muc-client") +@EnabledByConfig(RestApis.COLIBRI) +public class MucClient +{ + @Inject + protected XmppConnection xmppConnection; + + @Path("/add") + @POST + @Consumes(MediaType.APPLICATION_JSON) + public Response addMucClient(String requestBody) + { + //NOTE: unfortunately MucClientConfiguration is not a compliant bean (it doesn't have + // a no-arg ctor) so we can't parse the json directly into a MucClientConfiguration + // instance and just take that as an argument here, we have to read the json + // ourselves. + try + { + JsonNode node = new ObjectMapper().readTree(requestBody); + if (!(node instanceof ObjectNode)) + { + return Response.status(HttpServletResponse.SC_BAD_REQUEST).build(); + } + if (xmppConnection.addMucClient((ObjectNode)node)) + { + return Response.ok().build(); + } + return Response.status(HttpServletResponse.SC_BAD_REQUEST).build(); + } + catch (Exception e) + { + return Response.status(HttpServletResponse.SC_BAD_REQUEST).build(); + } + } + + @Path("/remove") + @POST + @Consumes(MediaType.APPLICATION_JSON) + public Response removeMucClient(String requestBody) + { + try + { + JsonNode node = new ObjectMapper().readTree(requestBody); + if (!(node instanceof ObjectNode)) + { + return Response.status(HttpServletResponse.SC_BAD_REQUEST).build(); + } + if (xmppConnection.removeMucClient((ObjectNode)node)) + { + return Response.ok().build(); + } + return Response.status(HttpServletResponse.SC_BAD_REQUEST).build(); + } + catch (Exception e) + { + return Response.status(HttpServletResponse.SC_BAD_REQUEST).build(); + } + } + + @Path("/list") + @GET + @Produces(MediaType.APPLICATION_JSON) + public String listMucClientIDs() + { + return xmppConnection.getMucClientIds(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/shutdown/Shutdown.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/shutdown/Shutdown.java new file mode 100644 index 0000000..71106c6 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/shutdown/Shutdown.java @@ -0,0 +1,79 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.shutdown; + +import com.fasterxml.jackson.annotation.*; +import org.eclipse.jetty.http.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; + +import jakarta.inject.*; +import jakarta.servlet.http.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; + +/** + * A resource for shutting down the videobridge via REST. + */ +@Path("/colibri/shutdown") +@EnabledByConfig(RestApis.SHUTDOWN) +public class Shutdown +{ + @Inject + protected Videobridge videobridge; + + @POST + @Consumes(MediaType.APPLICATION_JSON) + public Response shutdown(ShutdownJson shutdown, @Context HttpServletRequest request) + { + try + { + videobridge.shutdown(shutdown.isGraceful); + return Response.ok().build(); + } + catch (Throwable t) + { + return Response.status(HttpStatus.INTERNAL_SERVER_ERROR_500).build(); + } + } + + /** + * A class binding for the shutdown JSON passed to the shutdown request + * Currently, it only has a single field: + * { + * graceful-shutdown: Boolean [required] + * } + */ + public static class ShutdownJson + { + // Unfortunately, using @JsonProperty here alone is not enough to throw an error + // if the property is missing from the JSON, but we can't leave it out entirely + // as it's needed for serialization (since the constructor param JsonProperty + // is only for the parameter, not the member, so it isn't used when + // serializing). + @JsonProperty(value = "graceful-shutdown", required = true) + private Boolean isGraceful; + + @JsonCreator + public ShutdownJson(@JsonProperty(value = "graceful-shutdown", required = true) Boolean isGraceful) + { + this.isGraceful = isGraceful; + } + } +} + diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java new file mode 100644 index 0000000..c737158 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/stats/Stats.java @@ -0,0 +1,36 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.stats; + +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; +import org.jitsi.videobridge.stats.*; + +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; + +@Path("/colibri/stats") +@EnabledByConfig(RestApis.COLIBRI) +public class Stats +{ + @GET + @Produces(MediaType.APPLICATION_JSON) + public String getStats() + { + return VideobridgeStatisticsShim.INSTANCE.getStatsJson().toString(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/v2/conferences/Conferences.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/v2/conferences/Conferences.java new file mode 100644 index 0000000..484e619 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/colibri/v2/conferences/Conferences.java @@ -0,0 +1,184 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.v2.conferences; + +import org.jitsi.nlj.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; +import org.jitsi.videobridge.rest.exceptions.*; +import org.jitsi.xmpp.extensions.colibri2.*; +import org.jitsi.xmpp.extensions.colibri2.json.*; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jivesoftware.smack.packet.*; + +import jakarta.inject.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; +import java.util.*; + +@Path("/colibri/v2/conferences") +@EnabledByConfig(RestApis.COLIBRI) +public class Conferences +{ + private static final ObjectMapper jsonMapper = new ObjectMapper(); + + @Inject + private Videobridge videobridge; + + @GET + @Produces(MediaType.APPLICATION_JSON) + public String getConferences() + { + /* Not clear what this should do. If you have a use case please contact us with requirements. */ + throw new ServerErrorException(Response.Status.NOT_IMPLEMENTED); + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Path("/{meetingId}") + public String getConference(@PathParam("meetingId") String meetingId) + { + Conference conference = videobridge.getConferenceByMeetingId(meetingId); + + if (conference == null) + { + throw new NotFoundException(); + } + + /* Not clear what this should do. If you have a use case please contact us with requirements. */ + throw new ServerErrorException(Response.Status.NOT_IMPLEMENTED); + } + + @GET + @Produces(MediaType.APPLICATION_JSON) + @Path("/{meetingId}/dominant-speaker-identification") + public String getDominantSpeakerIdentification(@PathParam("meetingId") String meetingId) + { + Conference conference = videobridge.getConferenceByMeetingId(meetingId); + + if (conference == null) + { + throw new NotFoundException(); + } + + ConferenceSpeechActivity conferenceSpeechActivity = conference.getSpeechActivity(); + + return conferenceSpeechActivity.getDebugState(DebugStateMode.FULL).toString(); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public String createConference(String requestBody) + { + JsonNode requestJson; + try + { + requestJson = jsonMapper.readTree(requestBody); + if (!(requestJson instanceof ObjectNode)) + { + throw new BadRequestException(); + } + } + catch (JsonProcessingException pe) + { + throw new BadRequestExceptionWithMessage( + "Failed to create conference, could not parse JSON: " + pe.getMessage()); + } + + ConferenceModifyIQ conferenceModifyIQ; + + try + { + conferenceModifyIQ + = Colibri2JSONDeserializer.deserializeConferenceModify((ObjectNode) requestJson).build(); + } + catch (Exception e) + { + throw new BadRequestExceptionWithMessage("Failed to parse ConferenceModify JSON:" + e.getMessage()); + } + + return getVideobridgeIqResponseAsJson(conferenceModifyIQ); + } + + @PATCH + @Path("/{meetingId}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public String patchConference(@PathParam("meetingId") String meetingId, String requestBody) + { + Conference conference = videobridge.getConferenceByMeetingId(meetingId); + if (conference == null) + { + throw new NotFoundException(); + } + JsonNode requestJson; + try + { + requestJson = jsonMapper.readTree(requestBody); + if (!(requestJson instanceof ObjectNode)) + { + throw new BadRequestException(); + } + } + catch (JsonProcessingException e) + { + throw new BadRequestException(); + } + + ConferenceModifyIQ.Builder conferenceModifyIQBuilder; + try + { + conferenceModifyIQBuilder + = Colibri2JSONDeserializer.deserializeConferenceModify((ObjectNode) requestJson); + } + catch (Exception e) + { + throw new BadRequestExceptionWithMessage("Failed to parse ConferenceModify JSON:" + e.getMessage()); + } + + conferenceModifyIQBuilder.setMeetingId(meetingId); + + ConferenceModifyIQ conferenceModifyIQ = conferenceModifyIQBuilder.build(); + + return getVideobridgeIqResponseAsJson(conferenceModifyIQ); + } + + private String getVideobridgeIqResponseAsJson(ConferenceModifyIQ request) + { + IQ responseIq = videobridge.handleConferenceModifyIq(request); + + if (responseIq.getError() != null) + { + throw new BadRequestExceptionWithMessage( + "Failed to create conference: " + responseIq.getError().getDescriptiveText()); + } + + if (!(responseIq instanceof ConferenceModifiedIQ)) + { + throw new InternalServerErrorExceptionWithMessage("Non-error, non-colibri IQ result"); + } + + ObjectNode responseJson = Colibri2JSONSerializer.serializeConferenceModified((ConferenceModifiedIQ)responseIq); + + return responseJson.toString(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java new file mode 100644 index 0000000..04cbcc2 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/Debug.java @@ -0,0 +1,431 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.debug; + +import org.jitsi.health.*; +import org.jitsi.nlj.*; +import org.jitsi.nlj.transform.node.*; +import org.jitsi.nlj.transform.node.debug.*; +import org.jitsi.utils.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.utils.logging2.Logger; +import org.jitsi.utils.queue.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.metrics.*; +import org.jitsi.videobridge.relay.*; +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; +import org.jitsi.videobridge.stats.*; +import org.jitsi.videobridge.transport.ice.*; +import org.jitsi.videobridge.util.*; +import org.jitsi.videobridge.xmpp.*; + +import jakarta.inject.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.*; +import jakarta.ws.rs.core.MediaType; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.net.*; + +/** + * A REST interface for retrieving debug information about the bridge. + * + * Note that using this interface MAY disrupt running conferences or even + * cause a deadlock. It is really meant only for debugging, which is why it is + * disabled by default. Use at your own risk. + * + * @author bbaldino + */ +@Path("/debug") +@EnabledByConfig(RestApis.DEBUG) +public class Debug +{ + @Inject + @SuppressWarnings("unused") + private Videobridge videobridge; + + @Inject + private HealthCheckService healthCheckService; + + private final Logger logger = new LoggerImpl(Debug.class.getName()); + + // Functions to enable or disable features + + /** + * Set the state of a given JVB feature + * @param feature the feature to enable or disable + * @param enabled whether the feature should be enabled + * @return HTTP response + */ + @POST + @Path("/features/jvb/{feature}/{enabled}") + public Response setJvbFeatureState( + @PathParam("feature") DebugFeatures feature, + @PathParam("enabled") Boolean enabled) + { + logger.info((enabled ? "Enabling" : "Disabling") + " feature " + feature.getValue()); + setFeature(feature, enabled); + return Response.ok().build(); + } + + @GET + @Path("/features/jvb/{feature}/{enabled}") + public Response setJvbFeatureState2( + @PathParam("feature") DebugFeatures feature, + @PathParam("enabled") Boolean enabled) + { + logger.info((enabled ? "Enabling" : "Disabling") + " feature " + feature.getValue()); + setFeature(feature, enabled); + return Response.ok().build(); + } + + /** + * Find out whether the given JVB feature is currently enabled or disabled + * @param feature the feature to check + * @return true if the feature is enabled, false otherwise + */ + @GET + @Path("/features/jvb/{feature}") + public Boolean getJvbFeatureState(@PathParam("feature") DebugFeatures feature) + { + switch (feature) + { + case PAYLOAD_VERIFICATION: { + return Node.Companion.isPayloadVerificationEnabled(); + } + case NODE_STATS: { + return StatsKeepingNode.Companion.getEnableStatistics(); + } + case POOL_STATS: { + return ByteBufferPool.statisticsEnabled(); + } + case POOL_BOOKKEEPING: { + return ByteBufferPool.bookkeepingEnabled(); + } + case QUEUE_STATS: { + return QueueStatistics.DEBUG; + } + case QUEUE_TIMING_STATS: { + return QueueStatistics.TRACK_TIMES; + } + case NODE_TRACING: { + return Node.Companion.isNodeTracingEnabled(); + } + case TRANSIT_STATS: { + // Always enabled (worth modeling as a 'feature' then?) + return true; + } + case TASK_POOL_STATS: { + // Always enabled (worth modeling as a 'feature' then?) + return true; + } + case TOSSED_PACKET_STATS: { + return true; + } + default: { + throw new NotFoundException(); + } + } + } + + @POST + @Path("/features/endpoint/{confId}/{epId}/{feature}/{enabled}") + public Response setEndpointFeatureState( + @PathParam("confId") String confId, + @PathParam("epId") String epId, + @PathParam("feature") EndpointDebugFeatures feature, + @PathParam("enabled") Boolean enabled) + { + Conference conference = videobridge.getConference(confId); + if (conference == null) + { + throw new NotFoundException("No conference was found with the specified id."); + } + + Endpoint endpoint = conference.getLocalEndpoint(epId); + if (endpoint == null) + { + throw new NotFoundException("No endpoint was found with the specified id."); + } + + logger.info("Setting feature state: feature=" + feature.getValue() + ", enabled? " + enabled); + try + { + endpoint.setFeature(feature, enabled); + } + catch (IllegalStateException e) + { + return Response.status(403, e.getMessage()).build(); + } + + return Response.ok().build(); + } + + @GET + @Path("/features/endpoint/{confId}/{epId}/{feature}/") + public Boolean getEndpointFeatureState( + @PathParam("confId") String confId, + @PathParam("epId") String epId, + @PathParam("feature") EndpointDebugFeatures feature) + { + Conference conference = videobridge.getConference(confId); + if (conference == null) + { + throw new NotFoundException("No conference was found with the specified id."); + } + + Endpoint endpoint = conference.getLocalEndpoint(epId); + if (endpoint == null) + { + throw new NotFoundException("No endpoint was found with the specified id."); + } + + try + { + return endpoint.isFeatureEnabled(feature); + } + catch (Exception e) + { + throw new ServerErrorException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + } + } + + @POST + @Path("/features/relay/{confId}/{rId}/{feature}/{enabled}") + public Response setRelayFeatureState( + @PathParam("confId") String confId, + @PathParam("rId") String rId, + @PathParam("feature") EndpointDebugFeatures feature, + @PathParam("enabled") Boolean enabled) + { + Conference conference = videobridge.getConference(confId); + if (conference == null) + { + throw new NotFoundException("No conference was found with the specified id."); + } + + Relay relay = conference.getRelay(rId); + if (relay == null) + { + throw new NotFoundException("No relay was found with the specified id."); + } + + logger.info("Setting feature state: feature=" + feature.getValue() + ", enabled? " + enabled); + try + { + relay.setFeature(feature, enabled); + } + catch (IllegalStateException e) + { + return Response.status(403, e.getMessage()).build(); + } + + return Response.ok().build(); + } + + @GET + @Path("/features/relay/{confId}/{rId}/{feature}/") + public Boolean getRelayFeatureState( + @PathParam("confId") String confId, + @PathParam("rId") String rId, + @PathParam("feature") EndpointDebugFeatures feature) + { + Conference conference = videobridge.getConference(confId); + if (conference == null) + { + throw new NotFoundException("No conference was found with the specified id."); + } + + Relay relay = conference.getRelay(rId); + if (relay == null) + { + throw new NotFoundException("No relay was found with the specified id."); + } + + try + { + return relay.isFeatureEnabled(feature); + } + catch (Exception e) + { + throw new ServerErrorException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); + } + } + + + private void setFeature(DebugFeatures feature, boolean enabled) + { + switch (feature) + { + case PAYLOAD_VERIFICATION: { + Node.Companion.enablePayloadVerification(enabled); + break; + } + case NODE_STATS: { + StatsKeepingNode.Companion.setEnableStatistics(enabled); + break; + } + case POOL_STATS: { + ByteBufferPool.enableStatistics(enabled); + break; + } + case POOL_BOOKKEEPING: { + ByteBufferPool.enableBookkeeping(enabled); + break; + } + case QUEUE_STATS: { + QueueStatistics.DEBUG = enabled; + break; + } + case QUEUE_TIMING_STATS: { + QueueStatistics.TRACK_TIMES = enabled; + break; + } + case NODE_TRACING: { + Node.Companion.enableNodeTracing(enabled); + break; + } + case TRANSIT_STATS: { + //TODO + break; + } + case TASK_POOL_STATS: { + //TODO + break; + } + default: { + throw new NotFoundException(); + } + } + } + + // Functions to actually get statistics + + @GET + @Produces(MediaType.APPLICATION_JSON) + public String bridgeDebug(@DefaultValue("false") @QueryParam("full") boolean full) + { + ObjectNode debugState = videobridge.getDebugState( + null, + null, + full ? DebugStateMode.FULL : DebugStateMode.SHORT); + + // Append the health status. + Result result = healthCheckService.getResult(); + ObjectNode health = JsonNodeFactory.instance.objectNode(); + health.put("success", result.getSuccess()); + health.put("hardFailure", result.getHardFailure()); + health.put("responseCode", result.getResponseCode()); + health.put("message", result.getMessage()); + debugState.set("health", health); + + return debugState.toString(); + } + + @GET + @Path("/{confId}") + @Produces(MediaType.APPLICATION_JSON) + public String confDebug( + @PathParam("confId") String confId, + @DefaultValue("true") @QueryParam("full") boolean full) + { + ObjectNode confJson = videobridge.getDebugState( + confId, + null, + full ? DebugStateMode.FULL : DebugStateMode.SHORT); + return confJson.toString(); + } + + @GET + @Path("/{confId}/{epId}") + @Produces(MediaType.APPLICATION_JSON) + public String epDebug( + @PathParam("confId") String confId, + @PathParam("epId") String epId, + @DefaultValue("true") @QueryParam("full") boolean full) + { + ObjectNode confJson = videobridge.getDebugState( + confId, + epId, + full ? DebugStateMode.FULL : DebugStateMode.SHORT); + return confJson.toString(); + } + + @GET + @Path("/stats/jvb/{feature}") + @Produces(MediaType.APPLICATION_JSON) + public String getJvbFeatureStats(@PathParam("feature") DebugFeatures feature) + { + switch (feature) + { + case NODE_STATS: { + return StatsKeepingNode.Companion.getStatsJson().toString(); + } + case POOL_STATS: { + return ByteBufferPool.getStatsJson().toString(); + } + case QUEUE_STATS: { + return QueueStats.getQueueStats().toString(); + } + case TRANSIT_STATS: { + return PacketTransitStats.getStatsJson().toString(); + } + case TASK_POOL_STATS: { + return TaskPools.getStatsJson().toString(); + } + case XMPP_DELAY_STATS: { + return XmppConnection.getStatsJson().toString(); + } + case PAYLOAD_VERIFICATION: { + return PayloadVerificationPlugin.getStatsJson().toString(); + } + case ICE_STATS: { + return IceStatistics.Companion.getStats().toJson().toString(); + } + case CONFERENCE_PACKET_STATS: { + return ConferencePacketStats.stats.toJson().toString(); + } + case TOSSED_PACKET_STATS: { + return VideobridgeMetrics.tossedPacketsEnergy.get().toString(); + } + default: { + throw new NotFoundException(); + } + } + } + + // Old deprecated paths + + /** + * Depreacted, use {@link Debug#getJvbFeatureStats(DebugFeatures)} + * @param featureName the feature name + * @param uriInfo the URI info of the request + * @return HTTP response + */ + @Deprecated + @GET + @Path("/stats/{feature_name:.+}") + @Produces(MediaType.APPLICATION_JSON) + public Response getStats(@PathParam("feature_name") String featureName, @Context UriInfo uriInfo) + { + // Redirect to the new location + String newTarget = uriInfo.getBaseUri() + "debug/stats/jvb/" + featureName; + return Response.status(302).location(URI.create(newTarget)).build(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java new file mode 100644 index 0000000..dc1ca6b --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/DebugFeatures.java @@ -0,0 +1,67 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.debug; + +/** + * Enumerates different debug features which can be enabled and disabled via + * the debug REST API. + */ +public enum DebugFeatures +{ + PAYLOAD_VERIFICATION("payload-verification"), + NODE_STATS("node-stats"), + POOL_STATS("pool-stats"), + POOL_BOOKKEEPING("pool-bookkeeping"), + QUEUE_STATS("queue-stats"), + QUEUE_TIMING_STATS("queue-timing-stats"), + TRANSIT_STATS("transit-stats"), + TASK_POOL_STATS("task-pool-stats"), + NODE_TRACING("node-tracing"), + ICE_STATS("ice-stats"), + XMPP_DELAY_STATS("xmpp-delay-stats"), + TOSSED_PACKET_STATS("tossed-packet-stats"), + CONFERENCE_PACKET_STATS("conference-packet-stats"); + + private final String value; + + DebugFeatures(String value) + { + this.value = value; + } + + public String getValue() + { + return this.value; + } + + /** + * A custom 'fromString' implementation which allows creating an instance of + * this enum from its value. This assumes that the String values are derived using + * the following transformation of the 'keys': + * 1) lower case + * 2) underscores are replaced with hyphens + * + * @param value the String value of the enum + * @return an instance of the enum, if one can be derived by reversing the transformation + * detailed above + */ + public static DebugFeatures fromString(String value) + { + String normalized = value.toUpperCase().replace("-", "_"); + return DebugFeatures.valueOf(normalized); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java new file mode 100644 index 0000000..917a984 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/debug/EndpointDebugFeatures.java @@ -0,0 +1,52 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.debug; + +public enum EndpointDebugFeatures +{ + PCAP_DUMP("pcap-dump"), + SCTP_PCAP_DUMP("sctp-pcap-dump"); + + private final String value; + + EndpointDebugFeatures(String value) + { + this.value = value; + } + + public String getValue() + { + return this.value; + } + + /** + * A custom 'fromString' implementation which allows creating an instance of + * this enum from its value. This assumes that the String values are derived using + * the following transformation of the 'keys': + * 1) lower case + * 2) underscores are replaced with hyphens + * + * @param value the String value of the enum + * @return an instance of the enum, if one can be derived by reversing the transformation + * detailed above + */ + public static EndpointDebugFeatures fromString(String value) + { + String normalized = value.toUpperCase().replace("-", "_"); + return EndpointDebugFeatures.valueOf(normalized); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/rest/root/stats/Stats.java b/jvb/src/main/java/org/jitsi/videobridge/rest/root/stats/Stats.java new file mode 100644 index 0000000..a13aec9 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/rest/root/stats/Stats.java @@ -0,0 +1,45 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.rest.root.stats; + +import jakarta.inject.*; +import jakarta.ws.rs.*; +import jakarta.ws.rs.core.MediaType; +import org.jitsi.nlj.*; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.rest.*; +import org.jitsi.videobridge.rest.annotations.*; + +@Path("/stats") +@EnabledByConfig(RestApis.RTCSTATS) +public class Stats +{ + @Inject + private Videobridge videobridge; + + @GET + @Produces(MediaType.APPLICATION_JSON) + public String rtcstats() + { + ObjectNode debugState = videobridge.getDebugState( + null, + null, + DebugStateMode.STATS); + + return debugState.toString(); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/util/ByteBufferPool.java b/jvb/src/main/java/org/jitsi/videobridge/util/ByteBufferPool.java new file mode 100644 index 0000000..c3c00cc --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/util/ByteBufferPool.java @@ -0,0 +1,327 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.util; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jetbrains.annotations.*; +import org.jitsi.nlj.util.*; +import org.jitsi.utils.logging2.*; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; +import java.util.stream.*; + +/** + * A pool of reusable byte[] + * + * @author Brian Baldino + * @author Boris Grozev + */ +public class ByteBufferPool +{ + /** + * The pool of buffers is segmented by size to make the overall memory + * footprint smaller. + * These thresholds were chosen to minimize the used memory for a trace of + * requests from a test conference. Using finer segmentation (4 thresholds) + * results in only a marginal improvement. + */ + private static int T1 = 220; + private static int T2 = 775; + private static int T3 = 1500; + + /** + * The pool of buffers with size <= T1 + */ + private static final PartitionedByteBufferPool pool1 + = new PartitionedByteBufferPool(T1); + /** + * The pool of buffers with size in (T1, T2] + */ + private static final PartitionedByteBufferPool pool2 + = new PartitionedByteBufferPool(T2); + /** + * The pool of buffers with size in (T2, T3] + */ + private static final PartitionedByteBufferPool pool3 + = new PartitionedByteBufferPool(T3); + + /** + * The {@link Logger} + */ + private static final Logger logger = new LoggerImpl(ByteBufferPool.class.getName()); + + private static final Set outstandingBuffers = + Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>())); + + private static final Set returnedBuffers = + Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<>())); + + /** + * A debug data structure which tracks all events (requests and returns) for a given + * buffer + */ + private static final Map> bufferEvents = + Collections.synchronizedMap(new IdentityHashMap<>()); + + /** + * Whether to enable keeping track of statistics. + */ + private static boolean enableStatistics = false; + + /** + * Whether to enable or disable book keeping. + */ + private static Boolean bookkeepingEnabled = false; + + /** + * Total number of buffers requested. + */ + private static final LongAdder numRequests = new LongAdder(); + + /** + * The number of requests which were larger than our threshold and were + * allocated from java instead. + */ + private static final LongAdder numLargeRequests = new LongAdder(); + + /** + * Total number of buffers returned. + */ + private static final LongAdder numReturns = new LongAdder(); + + /** + * Gets the current thread ID. + */ + private static long threadId() + { + return Thread.currentThread().getId(); + } + + /** + * Gets a stack trace as a multi-line string. + */ + private static String stackTraceAsString(StackTraceElement[] stack) + { + StringBuilder sb = new StringBuilder(); + for (StackTraceElement ste : stack) + { + sb.append(ste.toString()).append("\n"); + } + return sb.toString(); + } + + /** + * Returns a buffer from the pool. + * + * @param size the minimum size. + */ + public static byte[] getBuffer(int size) + { + if (enableStatistics) + { + numRequests.increment(); + } + + byte[] buf; + if (size <= T1) + { + buf = pool1.getBuffer(size); + } + else if (size <= T2) + { + buf = pool2.getBuffer(size); + } + else if (size <= T3) + { + buf = pool3.getBuffer(size); + } + else + { + buf = new byte[size]; + numLargeRequests.increment(); + } + + if (bookkeepingEnabled) + { + Exception stackTrace = new Exception(); + outstandingBuffers.add(buf); + returnedBuffers.remove(buf); + bufferEvents.computeIfAbsent(buf, k -> new LinkedBlockingQueue<>()) + .add(new BufferEvent(BufferEvent.ALLOCATION, System.currentTimeMillis(), stackTrace)); + } + return buf; + } + + /** + * Returns a buffer to the pool. + * @param buf + */ + public static void returnBuffer(@NotNull byte[] buf) + { + if (enableStatistics) + { + numReturns.increment(); + } + + int len = buf.length; + + if (bookkeepingEnabled) + { + int arrayId = System.identityHashCode(buf); + Exception s; + Exception stackTrace = new Exception(); + bufferEvents.computeIfAbsent(buf, k -> new LinkedBlockingQueue<>()) + .add(new BufferEvent(BufferEvent.RETURN, System.currentTimeMillis(), stackTrace)); + if (outstandingBuffers.remove(buf)) + { + returnedBuffers.add(buf); + } + else if (returnedBuffers.contains(buf)) + { + String bufferTimeline = bufferEvents.get(buf).stream() + .map(BufferEvent::toString) + .collect(Collectors.joining()); + logger.error("Thread " + threadId() + + " returned a previously-returned " + len + "-byte buffer " + arrayId + + " its timeline:\n" + bufferTimeline); + } + else + { + logger.error("Thread " + threadId() + + " returned a " + len + "-byte buffer we didn't give out from\n" + + stackTraceAsString(stackTrace.getStackTrace())); + } + } + + if (len <= T1) + { + pool1.returnBuffer(buf); + } + else if (len <= T2) + { + pool2.returnBuffer(buf); + } + else if (len <= T3) + { + pool3.returnBuffer(buf); + } + else + { + logger.warn( + "Received a suspiciously large buffer (size = " + len + ")\n +" + + UtilKt.getStackTrace()); + } + } + + /** + * Gets a JSON representation of the statistics about the pool. + */ + public static ObjectNode getStatsJson() + { + ObjectNode stats = JsonNodeFactory.instance.objectNode(); + long numLargeRequestsSum = numLargeRequests.sum(); + + stats.put("num_large_requests", numLargeRequestsSum); + + if (enableStatistics) + { + long numRequestsSum = numRequests.sum(); + long allAllocations = numLargeRequestsSum + pool1.getNumAllocations() + + pool2.getNumAllocations() + pool3.getNumAllocations(); + long storedBytes = pool1.getStoredBytes() + pool2.getStoredBytes() + + pool3.getStoredBytes(); + stats.put("num_requests", numRequestsSum); + stats.put("num_returns", numReturns.sum()); + stats.put("num_allocations", allAllocations); + stats.put( + "allocation_percent", + (100.0 * allAllocations) / numRequestsSum); + stats.put("stored_bytes", storedBytes); + + stats.set("pool1", pool1.getStats()); + stats.set("pool2", pool2.getStats()); + stats.set("pool3", pool3.getStats()); + } + + if (bookkeepingEnabled) + { + stats.put("outstanding_buffers", outstandingBuffers.size()); + } + + return stats; + } + + /** + * Enables of disables tracking of statistics for the pool. + * @param enable whether to enable it or disable it. + */ + public static void enableStatistics(boolean enable) + { + enableStatistics = enable; + pool1.enableStatistics(enable); + pool2.enableStatistics(enable); + pool3.enableStatistics(enable); + } + + public static boolean statisticsEnabled() + { + return enableStatistics; + } + + public static void enableBookkeeping(boolean enable) + { + bookkeepingEnabled = enable; + if (!enable) + { + outstandingBuffers.clear(); + returnedBuffers.clear(); + bufferEvents.clear(); + } + } + + public static boolean bookkeepingEnabled() + { + return bookkeepingEnabled; + } + + private static class BufferEvent + { + final String context; + final Long timestamp; + final Exception trace; + + public BufferEvent(String context, Long timestamp, Exception trace) + { + this.context = context; + this.timestamp = timestamp; + this.trace = trace; + } + + @Override + public String toString() + { + return context + " BufferEvent: timestamp=" + timestamp + + "\n" + stackTraceAsString(trace.getStackTrace()); + } + + public static final String ALLOCATION = "allocation"; + public static final String RETURN = "return"; + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/util/PartitionedByteBufferPool.java b/jvb/src/main/java/org/jitsi/videobridge/util/PartitionedByteBufferPool.java new file mode 100644 index 0000000..4c814c2 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/util/PartitionedByteBufferPool.java @@ -0,0 +1,420 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.util; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.utils.stats.*; + +import java.time.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.*; + +/** + * Implements a byte array pool based on a number of independent partitions. + * Buffers are requested and returned to a random partition, which helps with + * contention. + * + * @author Brian Baldino + * @author Boris Grozev + */ +class PartitionedByteBufferPool +{ + /** + * The number of partitions. + */ + private static final int NUM_PARTITIONS = 8; + + /** + * How many buffers to pre-allocate in each partition. + */ + private static final int INITIAL_SIZE = 10; + + /** + * Whether to accept small buffers (<1500) that are returned. + */ + private static final boolean ACCEPT_SMALL_BUFFERS = false; + + /** + * The {@link Logger} + */ + private static final Logger logger = new LoggerImpl(PartitionedByteBufferPool.class.getName()); + + /** + * The partitions. + */ + private final Partition[] partitions = new Partition[NUM_PARTITIONS]; + + /** + * Whether to keep track of request/return rates and other basic statistics. + * As opposed to {@link ByteBufferPool#bookkeepingEnabled()} this has a + * relatively low overhead and can be kept on in production if necessary. + */ + private boolean enableStatistics = false; + + private final int defaultBufferSize; + + /** + * Initializes a new {@link PartitionedByteBufferPool} instance with + * a given initial size for each partition. + */ + PartitionedByteBufferPool(int defaultBufferSize) + { + this.defaultBufferSize = defaultBufferSize; + for (int i = 0; i < NUM_PARTITIONS; ++i) + { + partitions[i] = new Partition(i, INITIAL_SIZE); + } + logger.info("Initialized a new " + getClass().getSimpleName() + + " with " + NUM_PARTITIONS + " partitions."); + } + + /** + * Enables or disables tracking of statistics. + * @param enable whether to enable or disable. + */ + void enableStatistics(boolean enable) + { + enableStatistics = enable; + } + + /** + * Returns a random partition. + */ + private Partition getPartition() + { + return partitions[ThreadLocalRandom.current().nextInt(NUM_PARTITIONS)]; + } + + /** + * {@inheritDoc} + */ + byte[] getBuffer(int size) + { + return getPartition().getBuffer(size); + } + + /** + * {@inheritDoc} + */ + void returnBuffer(byte[] buf) + { + getPartition().returnBuffer(buf); + } + + /** + * Adds statistics for this pool to the given JSON object. + */ + ObjectNode getStats() + { + ObjectNode stats = JsonNodeFactory.instance.objectNode(); + + stats.put("default_size", defaultBufferSize); + + long requests = 0; + long returns = 0; + long allocations = 0; + long storedBytes = 0; + ArrayNode partitionStats = JsonNodeFactory.instance.arrayNode(); + + for (Partition p : partitions) + { + requests += p.numRequests.sum(); + returns += p.numReturns.sum(); + allocations += p.numAllocations.sum(); + storedBytes += p.storedBytes.get(); + + partitionStats.add(p.getStatsJson()); + } + stats.put("num_requests", requests); + stats.put("num_returns", returns); + stats.put("num_allocations", allocations); + stats.put("stored_bytes", storedBytes); + stats.put( + "allocation_percent", + 100D * allocations / Math.max(1, requests)); + + stats.set("partitions", partitionStats); + return stats; + } + + /** + * Gets the total number of times a new byte[] was allocated. + */ + long getNumAllocations() + { + long allocations = 0; + for (int i = 0; i < NUM_PARTITIONS; i++) + { + allocations += partitions[i].numAllocations.sum(); + } + + return allocations; + } + + /** + * Gets the total number of bytes currently stored in the pool + */ + long getStoredBytes() + { + long storedBytes = 0; + for (int i = 0; i < NUM_PARTITIONS; i++) + { + storedBytes += partitions[i].storedBytes.get(); + } + + return storedBytes; + } + + /** + * A byte array pool with a single {@link LinkedBlockingQueue}. + */ + private class Partition + { + /** + * The queue in which we store available packets. + */ + private final LinkedBlockingQueue pool + = new LinkedBlockingQueue<>(); + + /** + * The ID of the partition (used for debugging). + */ + private final int id; + + /** + * The number of times a request was satisfied with a buffer from the + * pool. + */ + private final LongAdder numNoAllocationNeeded = new LongAdder(); + + /** + * The number of times a new {@code byte[]} had to be allocated. + */ + private final LongAdder numAllocations = new LongAdder(); + + /** + * The number of times a new {@code byte[]} had to be allocated because + * the pool was empty. + */ + private final LongAdder numEmptyPoolAllocations = new LongAdder(); + + /** + * The number of times a new {@code byte[]} had to be allocated because + * the pool did not have a buffer with the required size (but it was + * not empty). + */ + private final LongAdder numWrongSizeAllocations = new LongAdder(); + + /** + * Total number of requests. + */ + private final LongAdder numRequests = new LongAdder(); + + /** + * Total number of returned buffers. + */ + private final LongAdder numReturns = new LongAdder(); + + /** + * The number of times a small buffer (<1500 bytes) was returned. + */ + private final LongAdder numSmallReturns = new LongAdder(); + + /** + * The number of times a buffer from the pool was discarded because it + * was too small to satisfy a request. + */ + private final LongAdder numSmallBuffersDiscarded = new LongAdder(); + + /** + * The number of times a large buffer (>1500 bytes) was requested. + */ + private final LongAdder numLargeRequests = new LongAdder(); + + /** + * The total size of buffers stored in the pool. + */ + private final AtomicLong storedBytes = new AtomicLong(); + + /** + * Request rate in requests per second over the last 10 seconds. + */ + private final RateTracker requestRate = new RateTracker(Duration.ofSeconds(10), Duration.ofMillis(100)); + + /** + * Return rate in requests per second over the last 10 seconds. + */ + private final RateTracker returnRate = new RateTracker(Duration.ofSeconds(10), Duration.ofMillis(100)); + + /** + * Initializes a new partition. + * @param id + * @param initialSize + */ + Partition(int id, int initialSize) + { + this.id = id; + for (int i = 0; i < initialSize; ++i) + { + pool.add(new byte[defaultBufferSize]); + } + } + + /** + * Returns a buffer from this pool (allocates a new one if necessary). + * @param requiredSize the minimum size. + * + * @return a {@code byte[]} with size at least {@code requiredSize}. + */ + private byte[] getBuffer(int requiredSize) + { + if (enableStatistics) + { + numRequests.increment(); + requestRate.update(1); + if (requiredSize > defaultBufferSize) + { + numLargeRequests.increment(); + } + } + + byte[] buf = pool.poll(); + if (buf == null) + { + buf = new byte[Math.max(defaultBufferSize, requiredSize)]; + if (enableStatistics) + { + numAllocations.increment(); + numEmptyPoolAllocations.increment(); + } + } + else if (buf.length < requiredSize) + { + if (ByteBufferPool.bookkeepingEnabled()) + { + logger.info("Needed buffer of size " + requiredSize + + ", got size " + buf.length + " retrying"); + } + + // This is unusual and shouldn't happen often, so we will just + // allocate a new buffer. + + // If the size is smaller than the default AND we just witnessed + // the buffer to be too small in practice, we'll throw it away. + // This makes sure that if someone returns a small buffer (in + // the practical sense) it will not get stuck in the pool. + if (buf.length >= defaultBufferSize) + { + pool.offer(buf); + } + else if (enableStatistics) + { + storedBytes.getAndAdd(-buf.length); + numSmallBuffersDiscarded.increment(); + } + + buf = new byte[Math.max(defaultBufferSize, requiredSize)]; + if (enableStatistics) + { + numAllocations.increment(); + numWrongSizeAllocations.increment(); + } + } + else if (enableStatistics) + { + storedBytes.getAndAdd(-buf.length); + numNoAllocationNeeded.increment(); + } + + return buf; + } + + /** + * Returns a buffer to this pool. + * @param buf + */ + private void returnBuffer(@NotNull byte[] buf) + { + if (enableStatistics) + { + numReturns.increment(); + returnRate.update(1); + } + + if (buf.length < defaultBufferSize) + { + if (enableStatistics) + { + numSmallReturns.increment(); + } + if (ACCEPT_SMALL_BUFFERS) + { + pool.offer(buf); + if (enableStatistics) + { + storedBytes.getAndAdd(buf.length); + } + } + } + else + { + pool.offer(buf); + if (enableStatistics) + { + storedBytes.getAndAdd(buf.length); + } + } + } + + /** + * Gets a snapshot of the statistics of this partition in JSON format. + */ + private ObjectNode getStatsJson() + { + long now = System.currentTimeMillis(); + ObjectNode stats = JsonNodeFactory.instance.objectNode(); + stats.put("id", id); + + long numRequestsSum = numRequests.sum(); + long numAllocationsSum = numAllocations.sum(); + + stats.put("num_requests", numRequestsSum); + stats.put("num_returns", numReturns.sum()); + stats.put("requests_rate_rps", requestRate.getRate(now)); + stats.put("returns_rate_rps", returnRate.getRate(now)); + stats.put("current_size", pool.size()); + stats.put("stored_bytes", storedBytes.get()); + + stats.put("num_allocations", numAllocationsSum); + stats.put("num_allocations_empty_pool", numEmptyPoolAllocations.sum()); + stats.put("num_allocations_wrong_size", numWrongSizeAllocations.sum()); + stats.put("num_no_allocation_needed", numNoAllocationNeeded.sum()); + stats.put( + "allocation_percent", + 100D * numAllocationsSum / Math.max(1, numRequestsSum)); + stats.put("num_small_returns", numSmallReturns.sum()); + stats.put("num_large_requests", numLargeRequests.sum()); + stats.put("num_small_discarded", numSmallBuffersDiscarded.sum()); + + return stats; + } + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/util/TaskPools.java b/jvb/src/main/java/org/jitsi/videobridge/util/TaskPools.java new file mode 100644 index 0000000..83b9c60 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/util/TaskPools.java @@ -0,0 +1,90 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.util; + +import org.jitsi.utils.concurrent.*; +import org.jitsi.utils.logging2.*; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.concurrent.*; + +public class TaskPools +{ + private static final Logger classLogger = new LoggerImpl(TaskPools.class.getName()); + /** + * A global executor service which can be used for non-CPU-intensive tasks. + */ + public static final ExecutorService IO_POOL = + Executors.newCachedThreadPool(new CustomizableThreadFactory("Global IO pool", false)); + + /** + * An executor to be used for CPU-intensive tasks. NOTE that tasks which block should + * NOT use this pool! + */ + public static final ExecutorService CPU_POOL = + Executors.newFixedThreadPool( + Runtime.getRuntime().availableProcessors(), + new CustomizableThreadFactory("Global CPU pool", false) + ); + private final static ScheduledExecutorService DEFAULT_SCHEDULED_POOL = + Executors.newSingleThreadScheduledExecutor(new CustomizableThreadFactory("Global scheduled pool", false)); + public static ScheduledExecutorService SCHEDULED_POOL = DEFAULT_SCHEDULED_POOL; + + public static void resetScheduledPool() + { + SCHEDULED_POOL = DEFAULT_SCHEDULED_POOL; + } + + + public static ObjectNode getStatsJson(ExecutorService es) + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + debugState.put("executor_class", es.getClass().getSimpleName()); + + if (es instanceof ThreadPoolExecutor) + { + ThreadPoolExecutor ex = (ThreadPoolExecutor)es; + debugState.put("pool_size", ex.getPoolSize()); + debugState.put("active_task_count", ex.getActiveCount()); + debugState.put("completed_task_count", ex.getCompletedTaskCount()); + debugState.put("core_pool_size", ex.getCorePoolSize()); + debugState.put("maximum_pool_size", ex.getMaximumPoolSize()); + debugState.put("largest_pool_size", ex.getLargestPoolSize()); + debugState.put("queue_class", ex.getQueue().getClass().getSimpleName()); + debugState.put("pending_task_count", ex.getQueue().size()); + } + + return debugState; + } + + public static ObjectNode getStatsJson() + { + ObjectNode debugState = JsonNodeFactory.instance.objectNode(); + + debugState.set("IO_POOL", getStatsJson(IO_POOL)); + debugState.set("CPU_POOL", getStatsJson(CPU_POOL)); + + return debugState; + } + + static { + classLogger.info("TaskPools detected " + Runtime.getRuntime().availableProcessors() + + " processors, creating the CPU pool with that many threads"); + + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/util/UlimitCheck.java b/jvb/src/main/java/org/jitsi/videobridge/util/UlimitCheck.java new file mode 100644 index 0000000..4df8e73 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/util/UlimitCheck.java @@ -0,0 +1,113 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.util; + +import org.jitsi.utils.logging2.*; + +import java.io.*; + +/** + * Checks the `ulimit` values for the process and prints a warning message if + * they are too low. + * + * @author Boris Grozev + */ +public class UlimitCheck +{ + /** + * The {@link Logger} to be used by the {@link UlimitCheck} class + * and its instances to print debug information. + */ + private static final Logger logger = new LoggerImpl(UlimitCheck.class.getName()); + + /** + * Executes a command in {@code bash} and returns the output ({@code stdin} + * and {@code stderr} combined), or {@code null} on failure. + * @param command the command to execute. + * @return the output of the command or {@code null}. + */ + private static String getOutputFromCommand(String command) + { + ProcessBuilder pb = new ProcessBuilder("bash", "-c", command); + pb.redirectErrorStream(true); + + try + { + Process p = pb.start(); + + BufferedReader br + = new BufferedReader( + new InputStreamReader(p.getInputStream())); + String output = br.lines().reduce(String::concat).orElse("null?"); + br.close(); + return output; + } + catch (IOException e) + { + return null; + } + } + + private static int parseInt(String s) + { + try + { + return Integer.parseInt(s); + } + catch (NumberFormatException n) + { + return Integer.MAX_VALUE; + } + } + + /** + * Extracts the current limits for number of open files and user processes + * (threads) by running {@code bash}'s {@code ulimit} builtin, and logs + * the values. + */ + public static void printUlimits() + { + + String fileLimit = getOutputFromCommand("ulimit -n"); + String fileLimitHard = getOutputFromCommand("ulimit -Hn"); + String threadLimit = getOutputFromCommand("ulimit -u"); + String threadLimitHard = getOutputFromCommand("ulimit -Hu"); + + StringBuilder sb + = new StringBuilder("Running with open files limit ") + .append(fileLimit) + .append(" (hard ").append(fileLimitHard).append(')') + .append(", thread limit ").append(threadLimit) + .append(" (hard ").append(threadLimitHard).append(")."); + + // These are not precise limits, they are based on numbers observed on a moderately loaded 8-core machine in + // production (Nov 2022). + boolean warn = parseInt(fileLimit) <= 4096 || parseInt(threadLimit) <= 2048; + + if (warn) + { + sb.append(" These values are too low and they will limit the ") + .append("number of participants that the bridge can serve ") + .append("simultaneously."); + logger.warn(sb); + } + else + { + logger.info(sb); + } + + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java new file mode 100644 index 0000000..38e7b45 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocket.java @@ -0,0 +1,204 @@ +/* + * Copyright @ 2017 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.websocket; + +import org.eclipse.jetty.websocket.api.Callback; +import org.eclipse.jetty.websocket.api.Session; +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.util.*; +import org.jitsi.videobridge.websocket.config.*; + +import java.nio.*; +import java.time.*; +import java.util.*; +import java.util.concurrent.*; + +/** + * @author Boris Grozev + */ +public class ColibriWebSocket implements Session.Listener.AutoDemanding +{ + /** + * The logger instance used by this {@link ColibriWebSocket}. + */ + private final Logger logger; + + /** + * The {@link EventHandler}, if any, associated with this web socket. + */ + private final EventHandler eventHandler; + + /** + * The clock used to compute lastSendTime. + */ + private final Clock clock = Clock.systemUTC(); + + /** The last time something was sent on this web socket */ + private Instant lastSendTime = Instant.MIN; + + /** The recurring task to send pings on the connection, if needed. */ + private ScheduledFuture pinger = null; + + /** The {@link Session} associated with this socket, set once the upgrade completes. */ + private volatile Session session; + + /** + * Initializes a new {@link ColibriWebSocket} instance. + */ + public ColibriWebSocket( + String id, + EventHandler eventHandler + ) + { + this.logger = new LoggerImpl(getClass().getName(), new LogContext(Map.of("id", id))); + this.eventHandler = Objects.requireNonNull(eventHandler, "eventHandler"); + } + + public Session getSession() + { + return session; + } + + /** + * Handles an a text message received on this web socket. + * @param message the message. + */ + @Override + public void onWebSocketText(String message) + { + logger.debug(() -> "Received text: " + message); + eventHandler.webSocketTextReceived(this, message); + } + + /** + * Handles the event of this web socket being connected. Finds the + * destination COLIBRI {@link Endpoint} and authenticates the request + * based on the password. + */ + @Override + public void onWebSocketOpen(Session sess) + { + this.session = sess; + + if (WebsocketServiceConfig.config.getSendKeepalivePings()) + { + pinger = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(this::maybeSendPing, + WebsocketServiceConfig.config.getKeepalivePingInterval().toMillis(), + WebsocketServiceConfig.config.getKeepalivePingInterval().toMillis(), + TimeUnit.MILLISECONDS); + } + + eventHandler.webSocketConnected(this); + } + + /** + * {@inheritDoc} + */ + @Override + public void onWebSocketClose(int statusCode, String reason) + { + session = null; + eventHandler.webSocketClosed(this, statusCode, reason); + if (pinger != null) + { + pinger.cancel(true); + } + } + + public void sendString(String message) + { + Session s = session; + if (s != null) + { + // We'll use the async version of sendText since this may be called + // from multiple threads. It's just fire-and-forget though, so we + // don't wait on the result. + s.sendText(message, Callback.NOOP); + synchronized (this) + { + lastSendTime = clock.instant(); + } + } + } + + private void maybeSendPing() + { + try + { + Instant now = clock.instant(); + synchronized (this) + { + if (Duration.between(lastSendTime, now). + compareTo(WebsocketServiceConfig.config.getKeepalivePingInterval()) < 0) + { + Session s = session; + if (s != null) + { + s.sendPing(ByteBuffer.allocate(0), Callback.NOOP); + lastSendTime = clock.instant(); + } + } + } + } + catch (Exception e) + { + logger.error("Error sending websocket ping", e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void onWebSocketError(Throwable cause) + { + eventHandler.webSocketError(this, cause); + if (pinger != null) + { + pinger.cancel(true); + } + } + + public interface EventHandler + { + /** + * Notifies that a specific {@link ColibriWebSocket} + * instance associated with it has been closed. + * @param ws the {@link ColibriWebSocket} which has been closed. + */ + void webSocketClosed(ColibriWebSocket ws, int statusCode, String reason); + /** + * Notifies that a specific {@link ColibriWebSocket} + * instance associated with it has connected. + * @param ws the {@link ColibriWebSocket} which has connected. + */ + void webSocketConnected(ColibriWebSocket ws); + /** + * Notifies that a message has been received from a + * specific {@link ColibriWebSocket} instance associated with it. + * @param ws the {@link ColibriWebSocket} from which a message was received. + */ + void webSocketTextReceived(ColibriWebSocket ws, String message); + + /** + * An error occurred for a specific websocket. + * @param ws the websocket. + * @param cause the cause of the error. + */ + void webSocketError(ColibriWebSocket ws, Throwable cause); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java new file mode 100644 index 0000000..5bd19f3 --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/websocket/ColibriWebSocketServlet.java @@ -0,0 +1,234 @@ +/* + * Copyright @ 2017 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.websocket; + +import org.eclipse.jetty.ee10.websocket.server.*; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.websocket.api.*; +import org.jetbrains.annotations.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.*; +import org.jitsi.videobridge.relay.*; +import org.jitsi.videobridge.websocket.config.*; + +import static org.jitsi.videobridge.websocket.config.WebsocketServiceConfig.config; + +import java.io.*; +import java.time.*; +import java.util.*; +import java.util.stream.*; + +/** + * @author Boris Grozev + */ +class ColibriWebSocketServlet + extends JettyWebSocketServlet +{ + /** + * The logger instance used by this {@link ColibriWebSocketServlet}. + */ + private static final Logger logger = new LoggerImpl(ColibriWebSocketServlet.class.getName()); + + /** + * The server ID of this websocket servlet. Incoming requests must have a matching + * server ID to be handled. + */ + private final String serverId; + + @NotNull + private final Videobridge videobridge; + + /** + * Initializes a new {@link ColibriWebSocketServlet} instance. + */ + ColibriWebSocketServlet( + @NotNull String serverId, + @NotNull Videobridge videobridge) + { + this.serverId = serverId; + this.videobridge = videobridge; + } + + /** + * {@inheritDoc} + */ + @Override + public void configure(JettyWebSocketServletFactory webSocketServletFactory) + { + // set a timeout of 1min + webSocketServletFactory.setIdleTimeout(WebsocketServiceConfig.config.getIdleTimeout()); + + webSocketServletFactory.setCreator((request, response) -> + { + try + { + return + ColibriWebSocketServlet + .this.createWebSocket(request, response); + } + catch (IOException ioe) + { + response.setStatusCode(HttpStatus.INTERNAL_SERVER_ERROR_500); + return null; + } + }); + } + + /** + * Handles a request for a web-socket. Validates the request URI and either + * accepts the request and creates a {@link ColibriWebSocket} instance, or + * rejects the request and sends an error. + */ + private ColibriWebSocket createWebSocket( + JettyServerUpgradeRequest request, + JettyServerUpgradeResponse response) + throws IOException + { + // A valid request URI looks like this: + // /colibri-ws/server-id/conf-id/endpoint-id?pwd=password + // or /colibri-ws/relay/server-id/conf-id/relay-id?pwd=password + // The "path" does not include "?pwd=password", which is in the "query" + String path = request.getRequestURI().getPath(); + logger.debug(() -> "Got a create websocket request at path " + path); + boolean isRelay; + + if (path == null) + { + logger.debug(() -> "Received request for an invalid path: " + path); + response.sendError(404, "invalid path"); + return null; + } + else if (path.startsWith(ColibriWebSocketService.COLIBRI_WS_PATH)) + { + isRelay = false; + } + else if (path.startsWith(ColibriWebSocketService.COLIBRI_RELAY_WS_PATH)) + { + isRelay = true; + } + else + { + logger.debug(() -> "Received request for an invalid path: " + path); + response.sendError(404, "invalid path"); + return null; + } + + String wsPath + = path.substring((isRelay ? + ColibriWebSocketService.COLIBRI_RELAY_WS_PATH : + ColibriWebSocketService.COLIBRI_WS_PATH).length()); + String[] ids = wsPath.split("/"); + if (ids.length < 3) + { + logger.debug(() -> "Received request for an invalid path: " + wsPath); + response.sendError(404, "invalid path"); + return null; + } + + if (!serverId.equals(ids[0])) + { + logger.warn("Received request with a mismatching server ID " + + "(expected '" + serverId + + "', received '" + ids[0] + "')."); + response.sendError(404, "server ID mismatch"); + return null; + } + + // Note: intentionally fail with the exact same message (as to not leak + // info) + String authFailed = "authentication failed"; + Conference conference = videobridge.getConference(ids[1]); + if (conference == null) + { + logger.warn("Received request for an nonexistent conference: " + + ids[1]); + response.sendError(403, authFailed); + return null; + } + + ColibriWebSocket.EventHandler eventHandler; + + if (isRelay) + { + Relay relay = conference.getRelay(ids[2]); + if (relay == null) + { + logger.warn("Received request for a nonexistent relay: " + + ids[2] + " (conference " + conference.getID() + ")"); + response.sendError(403, authFailed); + return null; + } + + String pwd = getPwd(request.getRequestURI().getQuery()); + if (!relay.acceptWebSocket(pwd)) + { + response.sendError(403, authFailed); + return null; + } + + eventHandler = relay.getMessageTransport(); + } + else + { + Endpoint endpoint = conference.getLocalEndpoint(ids[2]); + if (endpoint == null) + { + logger.warn("Received request for a nonexistent endpoint: " + + ids[2] + " (conference " + conference.getID() + ")"); + response.sendError(403, authFailed); + return null; + } + + String pwd = getPwd(request.getRequestURI().getQuery()); + if (!endpoint.acceptWebSocket(pwd)) + { + response.sendError(403, authFailed); + return null; + } + + eventHandler = endpoint.getMessageTransport(); + } + + if (!config.shouldEnableCompression()) + { + List extensions = response.getExtensions().stream(). + filter((ext) -> !ext.getName().equals("permessage-deflate")). + collect(Collectors.toList()); + response.setExtensions(extensions); + } + + return new ColibriWebSocket(ids[2], eventHandler); + } + + /** + * Extracts the password from the "query" part of a URI (e.g. "pwd=12345"). + * @param query the "query" part of a URI (e.g. "pwd=12345"). + * @return the password (e.g. "12345"), or null if parsing failed. + */ + private String getPwd(String query) + { + // TODO: this only deals with the simplest case. + if (query == null) + { + return null; + } + if (!query.startsWith("pwd=")) + { + return null; + } + return query.substring("pwd=".length()); + } +} diff --git a/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java new file mode 100644 index 0000000..190446e --- /dev/null +++ b/jvb/src/main/java/org/jitsi/videobridge/xmpp/MediaSourceFactory.java @@ -0,0 +1,840 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.xmpp; + +import org.jetbrains.annotations.*; +import org.jitsi.nlj.*; +import org.jitsi.nlj.rtp.*; +import org.jitsi.nlj.rtp.codec.vpx.*; +import org.jitsi.utils.logging2.*; +import org.jitsi.xmpp.extensions.colibri.*; +import org.jitsi.xmpp.extensions.jingle.*; +import org.jitsi.xmpp.extensions.jitsimeet.*; +import org.jivesoftware.smack.packet.*; +import org.jxmpp.jid.parts.*; +import org.jxmpp.util.*; + +import java.util.*; +import java.util.Objects; +import java.util.stream.*; + +/** + * A factory of {@link MediaSourceDesc}s from jingle signaling. + * + * @author George Politis + */ +public class MediaSourceFactory +{ + /** + * The {@link Logger} used by the {@link MediaSourceDesc} class and its + * instances for logging output. + */ + private static final Logger logger = new LoggerImpl(MediaSourceFactory.class.getName()); + + /** + * The resolution of the base stream when activating simulcast for VP8. + * + * FIXME: hardcoded ugh.. this should be either signaled or somehow included + * in the RTP stream. + */ + private static final int VP8_SIMULCAST_BASE_LAYER_HEIGHT = 180; + + /** + * libjitsi isn't aware of the group semantics names defined in + * {@link SourceGroupPacketExtension}, which is how we distinguish secondary + * ssrcs, so we'll translate them into constants defined in libjitsi + */ + private static Map secondarySsrcTypeMap = null; + + private static synchronized + Map getSecondarySsrcTypeMap() + { + if (secondarySsrcTypeMap == null) + { + secondarySsrcTypeMap = new HashMap<>(); + secondarySsrcTypeMap.put( + SourceGroupPacketExtension.SEMANTICS_FID, SsrcAssociationType.RTX); + } + + return secondarySsrcTypeMap; + } + + private static final VpxRtpLayerDesc[] noDependencies = new VpxRtpLayerDesc[0]; + + /* + * Creates layers for an encoding. + * + * @param spatialLen the number of spatial encodings per simulcast stream. + * @param temporalLen the number of temporal encodings per simulcast stream. + * @param height the maximum height of the top spatial layer + * @return an array that holds the layer descriptions. + */ + private static RtpLayerDesc[] createRTPLayerDescs( + int spatialLen, int temporalLen, int encodingIdx, int height) + { + VpxRtpLayerDesc[] rtpLayers + = new VpxRtpLayerDesc[spatialLen * temporalLen]; + + for (int spatialIdx = 0; spatialIdx < spatialLen; spatialIdx++) + { + double frameRate = (double) 30 / (1 << (temporalLen - 1)); + for (int temporalIdx = 0; + temporalIdx < temporalLen; temporalIdx++) + { + int idx = idx(spatialIdx, temporalIdx, + temporalLen); + + VpxRtpLayerDesc[] dependencies; + if (spatialIdx > 0 && temporalIdx > 0) + { + // this layer depends on spatialIdx-1 and temporalIdx-1. + dependencies = new VpxRtpLayerDesc[]{ + rtpLayers[ + idx(spatialIdx, temporalIdx - 1, + temporalLen)], + rtpLayers[ + idx(spatialIdx - 1, temporalIdx, + temporalLen)] + }; + } + else if (spatialIdx > 0) + { + // this layer depends on spatialIdx-1. + dependencies = new VpxRtpLayerDesc[] + {rtpLayers[ + idx(spatialIdx - 1, temporalIdx, + temporalLen)]}; + } + else if (temporalIdx > 0) + { + // this layer depends on temporalIdx-1. + dependencies = new VpxRtpLayerDesc[] + {rtpLayers[ + idx(spatialIdx, temporalIdx - 1, + temporalLen)]}; + } + else + { + // this is a base layer without any dependencies. + dependencies = noDependencies; + } + + int temporalId = temporalLen > 1 ? temporalIdx : -1; + int spatialId = spatialLen > 1 ? spatialIdx : -1; + + rtpLayers[idx] + = new VpxRtpLayerDesc(encodingIdx, + temporalId, spatialId, height, frameRate, dependencies); + + frameRate *= 2; + } + } + return rtpLayers; + } + + + /** + * Creates an RTP encoding. + * @param primarySsrc the primary SSRC for the encoding. + * @param spatialLen the number of spatial layers of the encoding. + * @param temporalLen the number of temporal layers of the encodings. + * @param secondarySsrcs a list of pairs, where each + * pair has the secondary ssrc as its key, and the type (rtx, etc.) as its + * value + * @param encodingIdx the index of the encoding + * @return a description of the encoding. + */ + private static RtpEncodingDesc createRtpEncodingDesc(Long primarySsrc, + int spatialLen, int temporalLen, SecondarySsrcs secondarySsrcs, + int encodingIdx, int height) + { + RtpLayerDesc[] layers = createRTPLayerDescs(spatialLen, temporalLen, + encodingIdx, height); + + RtpEncodingDesc enc = new RtpEncodingDesc(primarySsrc, layers, encodingIdx); + + if (secondarySsrcs != null) + { + secondarySsrcs.forEach(ssrcSecondarySsrc -> { + SsrcAssociationType type + = getSecondarySsrcTypeMap() + .get(ssrcSecondarySsrc.type); + if (type == null) + { + logger.error("Unable to find a mapping for" + + " secondary ssrc type " + + ssrcSecondarySsrc.type + + " will NOT included this secondary ssrc as" + + " an encoding"); + } + else + { + enc.addSecondarySsrc( + ssrcSecondarySsrc.ssrc, type); + } + }); + } + + return enc; + } + + + /** + * Get the 'secondary' ssrcs for the given primary ssrc. 'Secondary' here + * is defined as things like rtx or fec ssrcs. + * @param ssrc the primary ssrc for which to get the secondary ssrcs + * @param sourceGroups the source groups + * @return a map of secondary ssrc -> type (rtx, fec, etc.) + */ + private static List getSecondarySsrcs( + long ssrc, Collection sourceGroups) + { + List secondarySsrcs = new ArrayList<>(); + for (SourceGroupPacketExtension sourceGroup : sourceGroups) + { + if (sourceGroup.getSemantics().equalsIgnoreCase( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST)) + { + // Simulcast does not fall under the definition of 'secondary' + // we want here. + continue; + } + long groupPrimarySsrc = sourceGroup.getSources().get(0).getSSRC(); + long groupSecondarySsrc = sourceGroup.getSources().get(1).getSSRC(); + if (groupPrimarySsrc == ssrc) + { + secondarySsrcs.add( + new SecondarySsrc( + groupSecondarySsrc, sourceGroup.getSemantics())); + } + } + return secondarySsrcs; + } + + /** + * Build a map of source ssrc -> a list of secondary ssrc, secondary ssrc + * type + * @param ssrcs the ssrcs to get secondary ssrcs for + * @param sourceGroups the signaled source groups + * @return map of source ssrc -> a list of secondary ssrc, secondary ssrc + * type + */ + private static Map getAllSecondarySsrcs( + SourceSsrcs ssrcs, Collection sourceGroups) + { + Map allSecondarySsrcs = new HashMap<>(); + + for (long ssrc : ssrcs) + { + List secondarySsrcs + = getSecondarySsrcs(ssrc, sourceGroups); + allSecondarySsrcs.put(ssrc, new SecondarySsrcs(secondarySsrcs)); + } + return allSecondarySsrcs; + } + + /** + * Get all groups which have the given semantics + * @param semantics + * @param groups + * @return + */ + private static List getGroups( + final String semantics, final List groups) + { + return groups.stream() + .filter(sg -> sg.getSemantics().equalsIgnoreCase(semantics)) + .collect(Collectors.toList()); + } + + /** + * Removes all sources which correspond to an ssrc in sourceSsrcs, or any + * groups with a primary ssrc in sourceSsrcs. NOTE: modifies + * sources and sourceGroups in place + * @param sourceSsrcs the primary ssrcs for which all corresponding ssrcs + * will be removed from the given sources and groups + * @param sources the set of sources + * @param sourceGroups the set of groups + */ + private static void removeReferences( + SourceSsrcs sourceSsrcs, + List sources, + List sourceGroups) + { + // Remove any groups to which any of the ssrcs of this source belong + List groupsToRemove + = sourceGroups.stream() + .filter( + group -> group.getSources().stream().anyMatch( + source -> sourceSsrcs.contains(source.getSSRC()))) + .collect(Collectors.toList()); + + sourceGroups.removeAll(groupsToRemove); + + /* + * Remove not only the ssrcs in the source itself, but any ssrcs that + * were in groups along with ssrcs from the source. E.g. if we have:` + * SIM 1 2 3 + * RTX 1 10 + * RTX 2 20 + * RTX 3 30 + * then we need to make sure ssrcs 10, 20 and 30 don't create sources of + * their own and are removed along with the processing of the source + * with ssrcs 1, 2 and 3. + */ + Set ssrcsToRemove = extractSsrcs(groupsToRemove); + sources.removeIf( + source -> + sourceSsrcs.contains(source.getSSRC()) + || ssrcsToRemove.contains(source.getSSRC())); + } + + /** + * Extracts all SSRCs from all sources of a list of source groups. + * @param groups the list of groups. + * @return the set of SSRCs contained in one of the groups. + */ + private static Set extractSsrcs( + List groups) + { + Set ssrcs = new HashSet<>(); + groups.forEach( + group -> group.getSources().forEach( + source -> ssrcs.add(source.getSSRC()))); + return ssrcs; + }; + + /** + * Given the sources and groups, return a list of the ssrcs for each + * unique source + * @param sources the set of sources + * @param sourceGroups the set of source groups + * @return a list of {@link SourceSsrcs}. Each SourceSsrc object represents + * a set of primary video ssrcs belonging to a single source (video source) + */ + private static List getSourceSsrcs( + Collection sources, + Collection sourceGroups + ) + { + //FIXME: determining the individual sources should be done via msid, + // but somewhere along the line we seem to lose the msid information + // in the packet extensions + List sourceSsrcsList = new ArrayList<>(); + // We'll need to keep track of which sources and groups have been + // processed, so make copies of the lists we've been given so we can + // modify them. + List sourcesCopy = new ArrayList<>(sources); + List sourceGroupsCopy + = new ArrayList<>(sourceGroups); + + // Note that the order we process these groups is important + Arrays.asList( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST, + SourceGroupPacketExtension.SEMANTICS_FID, + SourceGroupPacketExtension.SEMANTICS_FEC + ).forEach(groupSem -> { + List groups + = getGroups(groupSem, sourceGroupsCopy); + groups.forEach(group -> { + // An empty group is the signal that we want to clear all + // the groups. + // https://github.com/jitsi/jitsi/blob/7eabaab0fca37711813965d66a0720d1545f6c48/src/net/java/sip/communicator/impl/protocol/jabber/extensions/colibri/ColibriBuilder.java#L188 + if (group.getSources() == null || group.getSources().isEmpty()) + { + if (groups.size() > 1) + { + logger.warn("Received empty group, which is " + + "a signal to clear all groups, but there were " + + "other groups present, which shouldn't happen"); + } + return; + } + List ssrcs; + // For a simulcast group, all the ssrcs are considered primary + // ssrcs, but for others, only the main ssrc of the group is + if (groupSem.equalsIgnoreCase( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST)) + { + ssrcs + = group.getSources().stream() + .map(SourcePacketExtension::getSSRC) + .collect(Collectors.toList()); + } + else + { + ssrcs = Arrays.asList(group.getSources().get(0).getSSRC()); + } + + SourceSsrcs sourceSsrcs = new SourceSsrcs(ssrcs); + // Now we need to remove any groups with these ssrcs as their + // primary, or sources that correspond to one of these ssrcs + removeReferences(sourceSsrcs, sourcesCopy, sourceGroupsCopy); + + sourceSsrcsList.add(sourceSsrcs); + }); + }); + + if (!sourceGroupsCopy.isEmpty()) + { + logger.warn( + "Unprocessed source groups: " + + sourceGroupsCopy.stream() + .map(Element::toXML) + .map(Object::toString) + .reduce(String::concat)); + } + + // The remaining sources are not part of any group. Add them as sources + // with their own primary SSRC. + // NOTE: we need to ignore sources with an ssrc of -1, because the + // ColibriBuilder will use that as a signal to remove sources + // https://github.com/jitsi/jitsi/blob/7eabaab0fca37711813965d66a0720d1545f6c48/src/net/java/sip/communicator/impl/protocol/jabber/extensions/colibri/ColibriBuilder.java#L162 + sourcesCopy.forEach(source -> { + if (source.getSSRC() != -1) + { + sourceSsrcsList.add(new SourceSsrcs(source.getSSRC())); + } + else + { + if (sourcesCopy.size() > 1) + { + logger.warn("Received an empty source, which is " + + "a signal to clear all sources, but there were " + + "other sources present, which shouldn't happen"); + } + } + }); + + setOwnersAndNames(sources, sourceSsrcsList); + + return sourceSsrcsList; + } + + /** + * Updates the given list of {@link SourceSsrcs}, setting the {@code owner} + * field according to the {@code owner} attribute in {@code ssrc-info} + * extensions contained in {@code sources}, if not specified externally + * @param sources the list of {@link SourcePacketExtension} which may contain + * the {@code owner} as an attribute of a {@code ssrc-info} child. The + * list or the objects in the list will not be modified. + * @param sourceSsrcsList the list of {@link SourceSsrcs} to update. + */ + private static void setOwnersAndNames( + Collection sources, + Collection sourceSsrcsList + ) + { + for (SourceSsrcs sourceSsrcs : sourceSsrcsList) + { + // Look for the "owner" tag in the sources. We assume that all + // sources contain the "owner" tag so we just check for the + // first SSRC of the source's SSRCs. + long primarySsrc = sourceSsrcs.get(0); + SourcePacketExtension sourceSource + = sources.stream() + .filter(source -> source.getSSRC() == primarySsrc) + .findAny().orElse(null); + + sourceSsrcs.owner = getOwner(sourceSource); + sourceSsrcs.name = sourceSource != null ? sourceSource.getName() : null; + } + } + + /** + * Extracts the owner/endpoint ID as a {@link String} from a + * {@link SourcePacketExtension}. + * Jicofo includes the full occupant JID of the endpoint as the owner of + * a {@link SSRCInfoPacketExtension}, but in jitsi-videobridge we only + * care about the resource part, which coincides with the ID of the Colibri + * endpoint associated with the owner. + * + * @param source the {@link SourcePacketExtension} from which to extract + * the owner. + * @return the owner/endpoint ID as a {@link String}. + */ + public static String getOwner(SourcePacketExtension source) + { + SSRCInfoPacketExtension ssrcInfoPacketExtension + = source == null + ? null : source.getFirstChildOfType( + SSRCInfoPacketExtension.class); + + if (ssrcInfoPacketExtension != null) + { + /* + * N.B.: this is semantically the same as + * return + * ssrcInfoPacketExtension.getOwner() + * .getResourceOrEmpty().toString(); + * + * but avoids expensive construction of a lot of JID parts we just throw away. + */ + String ownerAttr = ssrcInfoPacketExtension.getAttributeAsString(SSRCInfoPacketExtension.OWNER_ATTR_NAME); + if (ownerAttr != null) + { + return getResourceFromJid(ownerAttr); + } + } + return null; + } + + /** + * Given a string containing a JID, return its StringPrep'd Resource part, + * or {@link Resourcepart#EMPTY}, as a string. + * + * Helper for {@link #getOwner} + */ + private static String getResourceFromJid(String jid) + { + String unpreppedResource = XmppStringUtils.parseResource(jid); + if (unpreppedResource.isEmpty()) + { + return Resourcepart.EMPTY.toString(); + } + return Resourcepart.fromOrThrowUnchecked(unpreppedResource).toString(); + } + + /** + * Creates {@link MediaSourceDesc}s from signaling params + * + * will receive the created {@link MediaSourceDesc}s. + * @param sources The {@link List} of {@link SourcePacketExtension} that + * describes the list of jingle sources. + * @param sourceGroups The {@link List} of + * {@link SourceGroupPacketExtension} that describes the list of jingle + * source groups. + * @return an array of {@link MediaSourceDesc} that are described in the + * jingle sources and source groups. + * @deprecated Use createMediaSource. + */ + @Deprecated + public static MediaSourceDesc[] createMediaSources( + Collection sources, + Collection sourceGroups) + { + final Collection finalSourceGroups + = sourceGroups == null ? new ArrayList<>() : sourceGroups; + final Collection finalSources + = sources == null ? new ArrayList<>() : sources; + + List sourceSsrcsList = getSourceSsrcs(finalSources, finalSourceGroups); + List mediaSources = new ArrayList<>(); + + sourceSsrcsList.forEach(sourceSsrcs -> { + // As of now, we only ever have 1 spatial layer per stream + int numSpatialLayersPerStream = 1; + // Previously, we assumed that when simulcast was enabled, 3 temporal layers would + // be present in each stream and if simulcast wasn't enabled, there would be only + // 1 temporal layer per stream, but we found that screenshare started sending 2 + // temporal layers, and we'd therefore start dropping layer 1 packets since we didn't + // recognize them. The truth is we have no way of _knowing_ how many temporal layers + // there are going to be (as it's not signaled) unless we dynamically read the layers + // as the packets come in (which we may want to do in the future)--but for now we'll + // over-estimate how many layers there are per stream to avoid dropping packets for a + // layer we don't recognize. Since there are already situations where chrome sends fewer + // temporal layers than we assume, this should be safe. + final int numTemporalLayersPerStream = 3; + Map secondarySsrcs + = getAllSecondarySsrcs(sourceSsrcs, finalSourceGroups); + MediaSourceDesc mediaSource + = createSource( + sourceSsrcs, + numSpatialLayersPerStream, + numTemporalLayersPerStream, + secondarySsrcs, + sourceSsrcs.owner, + sourceSsrcs.name, + getVideoType(finalSources), + false + ); + mediaSources.add(mediaSource); + }); + + return mediaSources.toArray(new MediaSourceDesc[0]); + } + + // This method is to replace createMediaSources when Colibri V1 is removed + public static MediaSourceDesc createMediaSource( + Collection sources, + Collection sourceGroups, + String owner, + String name) + { + return createMediaSource(sources, sourceGroups, owner, name, false); + } + + public static MediaSourceDesc createMediaSource( + Collection sources, + Collection sourceGroups, + String owner, + String name, + boolean synthetic) + { + Objects.requireNonNull(owner, "owner is required"); + Objects.requireNonNull(name, "name is required"); + + final Collection finalSourceGroups + = sourceGroups == null ? new ArrayList<>() : sourceGroups; + if (sources == null) + { + return null; + } + + List sourceSsrcsList = getSourceSsrcs(sources, finalSourceGroups); + + if (sourceSsrcsList.size() != 1) + { + logger.warn("sourceSsrcsList.size() != 1 for: " + sources + " groups: " + finalSourceGroups); + } + + if (sourceSsrcsList.size() > 0) + { + SourceSsrcs sourceSsrcs = sourceSsrcsList.get(0); + + // As of now, we only ever have 1 spatial layer per stream + int numSpatialLayersPerStream = 1; + // Previously, we assumed that when simulcast was enabled, 3 temporal layers would + // be present in each stream and if simulcast wasn't enabled, there would be only + // 1 temporal layer per stream, but we found that screenshare started sending 2 + // temporal layers, and we'd therefore start dropping layer 1 packets since we didn't + // recognize them. The truth is we have no way of _knowing_ how many temporal layers + // there are going to be (as it's not signaled) unless we dynamically read the layers + // as the packets come in (which we may want to do in the future)--but for now we'll + // over-estimate how many layers there are per stream to avoid dropping packets for a + // layer we don't recognize. Since there are already situations where chrome sends fewer + // temporal layers than we assume, this should be safe. + final int numTemporalLayersPerStream = 3; + Map secondarySsrcs = getAllSecondarySsrcs(sourceSsrcs, finalSourceGroups); + + return createSource( + sourceSsrcs, + numSpatialLayersPerStream, + numTemporalLayersPerStream, + secondarySsrcs, + owner, + name, + getVideoType(sources), + synthetic + ); + } + else + { + return null; + } + } + + + /** + * Calculates the array position of an RTP layer description specified by its + * spatial index (SVC) and temporal index (SVC). + * + * @param spatialIdx the spatial layer index. + * @param temporalIdx the temporal layer index. + * + * @return the subjective quality index of the flow specified in the + * arguments. + */ + private static int idx(int spatialIdx, int temporalIdx, + int temporalLen) + { + return spatialIdx * temporalLen + temporalIdx; + } + + /** + * Describes a single secondary ssrc and its group semantics to + * its primary ssrc (e.g. FID or FEC-FR) + */ + private static class SecondarySsrc + { + public long ssrc; + public String type; + + /** + * Initializes a new {@link SecondarySsrc} instance. + */ + public SecondarySsrc(long ssrc, String type) + { + this.ssrc = ssrc; + this.type = type; + } + } + + /** + * Groups a set of secondary ssrcs for a single primary ssrc (e.g. its + * rtx and fec ssrcs) + */ + private static class SecondarySsrcs + implements Iterable + { + public List secondarySsrcs; + + /** + * Initializes a new {@link SecondarySsrcs} instance. + */ + public SecondarySsrcs(List secondarySsrcs) + { + this.secondarySsrcs = secondarySsrcs; + } + + /** + * {@inheritDoc} + */ + @Override + public Iterator iterator() + { + return secondarySsrcs.iterator(); + } + } + + /** + * Describes one or more primary video ssrcs for a single + * {@link MediaSourceDesc} + */ + private static class SourceSsrcs + implements Iterable + { + private String name; + + private List sourceSsrcs; + + private String owner; + + /** + * Initializes a new {@link SourceSsrcs} instance for a single SSRC. + */ + private SourceSsrcs(Long ssrc) + { + this(Collections.singletonList(ssrc)); + } + + /** + * Initializes a new {@link SourceSsrcs} instance for a list of SSRCs. + */ + public SourceSsrcs(List sourceSsrcs) + { + this.sourceSsrcs = sourceSsrcs; + } + + /** + * Checks whether this instance contains a specific SSRC. + */ + public boolean contains(Long ssrc) + { + return sourceSsrcs.contains(ssrc); + } + + /** + * @return the number of SSRCs in this instance. + */ + public int size() + { + return sourceSsrcs.size(); + } + + /** + * Gets the SSRC at a specific index. + */ + public Long get(int index) + { + return sourceSsrcs.get(index); + } + + /** + * {@inheritDoc} + */ + @Override + public Iterator iterator() + { + return sourceSsrcs.iterator(); + } + } + + /** + * Creates a single MediaSourceDesc with the given information + * @param primarySsrcs the set of primary video ssrcs belonging to this source + * @param numSpatialLayersPerStream the number of spatial layers per stream + * for this source + * @param numTemporalLayersPerStream the number of temporal layers per stream + * for this source + * @param allSecondarySsrcs a map of primary ssrc -> SecondarySsrcs, which lists + * the ssrc and type of all the secondary ssrcs for a given primary (e.g. + * its corresponding rtx and fec ssrcs) + * @param owner An externally specified owner, if available. + * @param name An externally specified name, if available. + * @return the created MediaSourceDesc + */ + private static MediaSourceDesc createSource( + SourceSsrcs primarySsrcs, + int numSpatialLayersPerStream, + int numTemporalLayersPerStream, + Map allSecondarySsrcs, + String owner, + String name, + VideoType videoType, + boolean synthetic + ) + { + RtpEncodingDesc[] encodings = + new RtpEncodingDesc[primarySsrcs.size()]; + + // this loop builds a subjective quality index array that looks like + // this: + // + // [s0t0, s0t1, s0t2, s1t0, s1t1, s1t2, s2t0, s2t1, s2t2] + // + // The spatial layer is offered either by simulcast (VP8) or spatial + // scalability (VP9). Exotic cases might do simulcast + spatial + // scalability. + + //TODO(brian): this is only correct if the highest res is 720p + int height = VP8_SIMULCAST_BASE_LAYER_HEIGHT; + + for (int encodingIdx = 0; encodingIdx < primarySsrcs.size(); encodingIdx++) { + Long primarySsrc = primarySsrcs.get(encodingIdx); + SecondarySsrcs ssrcSecondarySsrcs = allSecondarySsrcs.get(primarySsrc); + + encodings[encodingIdx] = createRtpEncodingDesc(primarySsrc, + numSpatialLayersPerStream, numTemporalLayersPerStream, + ssrcSecondarySsrcs, encodingIdx, height); + + height *= 2; + } + + if (name == null) + { + throw new IllegalArgumentException("The 'name' is missing in the source description"); + } + if (owner == null) + { + throw new IllegalArgumentException("The 'owner' is missing in the source description"); + } + + return new MediaSourceDesc(encodings, owner, name, videoType, synthetic); + } + + private static VideoType getVideoType(@NotNull Collection sources) + { + for (SourcePacketExtension source : sources) + { + if (source.getVideoType() != null) + { + return VideoType.valueOf(source.getVideoType().toUpperCase()); + } + } + return VideoType.CAMERA; + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt new file mode 100644 index 0000000..b029b4e --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpoint.kt @@ -0,0 +1,407 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.VideoType +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.utils.NEVER +import org.jitsi.utils.event.EventEmitter +import org.jitsi.utils.event.SyncEventEmitter +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.cc.allocation.MediaSourceContainer +import org.jitsi.videobridge.cc.allocation.ReceiverConstraintsMap +import org.jitsi.videobridge.cc.allocation.VideoConstraints +import org.jitsi.videobridge.relay.AudioSourceDesc +import org.jitsi.videobridge.util.TaskPools +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap + +/** + * Represents an endpoint in a conference (i.e. the entity associated with + * a participant in the conference, which connects the participant's audio + * and video channel). This might be an endpoint connected to this instance of + * jitsi-videobridge, or a relayed endpoint connected to another bridge in the + * same conference. + * + * @author Boris Grozev + * @author Brian Baldino + */ +abstract class AbstractEndpoint protected constructor( + /** + * The [Conference] which this endpoint is to be part of. + */ + val conference: Conference, + /** + * The ID of the endpoint. + */ + final override val id: String, + parentLogger: Logger +) : MediaSourceContainer { + + /** + * The [Logger] used by the [AbstractEndpoint] class to print debug + * information. + */ + protected val logger: Logger = parentLogger.createChildLogger(this.javaClass.name, mapOf("epId" to id)) + + /** + * The map of source name -> ReceiverConstraintsMap. + */ + private val receiverVideoConstraints = ConcurrentHashMap() + + private data class SourceAndHeight( + val sourceName: String, + val maxHeight: Int + ) + + private val receiverVideoConstraintsQueue = PacketQueue( + 128, + null, + "receiver-constraints-$id", + { (sourceName, newMaxHeight) -> + receiverVideoConstraintsChanged(sourceName, newMaxHeight) + true + }, + TaskPools.IO_POOL + ) + + /** + * The statistics id of this Endpoint. + */ + var statsId: String? = null + set(value) { + field = value + if (value != null) { + logger.addContext("stats_id", value) + } + } + + /** + * The indicator which determines whether [.expire] has been called + * on this Endpoint. + */ + var isExpired = false + private set + + /** + * A map of source name -> VideoConstraints. + * + * Stores the maximum set of constraints applied by all receivers for each media source sent by this endpoint. + * The client needs to send _at least_ this for a media source to satisfy all its receivers. + */ + protected var maxReceiverVideoConstraints = mutableMapOf() + + protected val eventEmitter: EventEmitter = SyncEventEmitter() + + private val videoTypeCache = mutableMapOf() + + fun hasVideoAvailable(): Boolean { + for (source in mediaSources) { + if (source.videoType !== VideoType.NONE) { + return true + } + } + return videoTypeCache.values.any { it !== VideoType.NONE } + } + + fun setVideoType(sourceName: String, videoType: VideoType) { + val mediaSourceDesc = findMediaSourceDesc(sourceName) + if (mediaSourceDesc != null) { + if (mediaSourceDesc.videoType !== videoType) { + if (mediaSourceDesc.videoType.isEnabled() && videoType.isEnabled()) { + logger.warn( + "Changing video type from ${mediaSourceDesc.videoType} to $videoType for $sourceName. " + + "This will not trigger re-signaling the mapping." + ) + } + mediaSourceDesc.videoType = videoType + conference.speechActivity.endpointVideoAvailabilityChanged() + } + } + videoTypeCache[sourceName] = videoType + } + + protected fun applyVideoTypeCache(mediaSourceDescs: Array) { + // Video types are signaled over JVB data channel while MediaStreamDesc over Colibri. The two channels need + // to be synchronized. + for (mediaSourceDesc in mediaSourceDescs) { + val videoType = videoTypeCache[mediaSourceDesc.sourceName] + if (videoType != null) { + mediaSourceDesc.videoType = videoType + videoTypeCache.remove(mediaSourceDesc.sourceName) + } + } + } + + /** + * Checks whether a specific SSRC belongs to this endpoint. + * @param ssrc + * @return + */ + abstract fun receivesSsrc(ssrc: Long): Boolean + + /** + * The set of SSRCs received from this endpoint. + * @return + */ + abstract val ssrcs: Set + + /** + * The [AbstractEndpointMessageTransport] associated with + * this endpoint. + */ + open val messageTransport: AbstractEndpointMessageTransport? + get() = null + + /** Whether this endpoint represents a visitor. */ + abstract val visitor: Boolean + + /** + * Whether diarization is requested for this endpoint's audio, as signaled via the colibri2 `diarize` attribute. + * Propagated to the transcriber via the mediajson start event. Defaults to false for endpoint types that don't + * signal it (e.g. relayed endpoints). + */ + open val diarize: Boolean + get() = false + + /** + * Gets the description of the video [MediaSourceDesc] that this endpoint has advertised, or `null` if + * it hasn't advertised any video sources. + */ + protected val mediaSource: MediaSourceDesc? + get() = mediaSources.firstOrNull() + + fun findMediaSourceDesc(sourceName: String): MediaSourceDesc? = mediaSources.firstOrNull { + sourceName == it.sourceName + } + + fun addEventHandler(eventHandler: EventHandler) { + eventEmitter.addHandler(eventHandler) + } + + fun removeEventHandler(eventHandler: EventHandler) { + eventEmitter.removeHandler(eventHandler) + } + + override fun toString() = "${javaClass.name} $id" + + /** + * Expires this [AbstractEndpoint]. + */ + open fun expire() { + logger.debug("Expiring.") + isExpired = true + conference.endpointExpired(this) + } + + /** + * Return true if this endpoint should expire (based on whatever logic is + * appropriate for that endpoint implementation. + * + * @return true if this endpoint should expire, false otherwise + */ + abstract fun shouldExpire(): Boolean + + /** + * Get the last 'incoming activity' (packets received) this endpoint has seen + * @return the timestamp, in milliseconds, of the last activity of this endpoint + */ + open val lastIncomingActivity: Instant? + get() = NEVER + + /** + * Requests a keyframe from this endpoint for the specified media SSRC. + * + * @param mediaSsrc the media SSRC to request a keyframe from. + */ + abstract fun requestKeyframe(requesterID: String, mediaSsrc: Long) + + /** + * Requests a keyframe from this endpoint on the first video SSRC + * it finds. Being able to request a keyframe without passing a specific + * SSRC is useful for things like requesting a pre-emptive keyframes when a new + * active speaker is detected (where it isn't convenient to try and look up + * a particular SSRC). + */ + abstract fun requestKeyframe() + + /** A JSON representation of the parts of this object's state that are deemed useful for debugging. */ + open fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + val receiverVideoConstraints = JsonNodeFactory.instance.objectNode() + this@AbstractEndpoint.receiverVideoConstraints.forEach { (sourceName, receiverConstraints) -> + receiverVideoConstraints.set(sourceName, receiverConstraints.getDebugState()) + } + set("receiver_video_constraints", receiverVideoConstraints) + put("max_receiver_video_constraints", HashMap(maxReceiverVideoConstraints).toString()) + put("expired", isExpired) + put("stats_id", statsId) + } + + /** + * Enqueues a call to send a receiver video constrains message. + * We use a queue to prevent races in sending these constraint messages. + */ + private fun sendReceiverVideoConstraintsChanged(sourceName: String, newMaxHeight: Int) { + receiverVideoConstraintsQueue.add(SourceAndHeight(sourceName, newMaxHeight)) + } + + /** + * Computes and sets the [.maxReceiverVideoConstraints] from the specified video constraints of the media + * source identified by the given source name. + * + * @param sourceName the name of the media source for which the constraints have changed. + * @param newMaxHeight the maximum height resulting from the current set of constraints. + * (Currently we only support constraining the height, and not frame rate.) + */ + private fun receiverVideoConstraintsChanged(sourceName: String, newMaxHeight: Int) { + val oldReceiverMaxVideoConstraints = maxReceiverVideoConstraints[sourceName] + val newReceiverMaxVideoConstraints = VideoConstraints(newMaxHeight, -1.0) + if (newReceiverMaxVideoConstraints != oldReceiverMaxVideoConstraints) { + maxReceiverVideoConstraints[sourceName] = newReceiverMaxVideoConstraints + sendVideoConstraints(sourceName, newReceiverMaxVideoConstraints) + } + } + + /** + * Whether the remote endpoint is currently sending (non-silence) audio. + */ + abstract val isSendingAudio: Boolean + + /** + * Whether the remote endpoint is currently sending video. + */ + abstract val isSendingVideo: Boolean + + /** + * The set of [AudioSourceDesc]s that this endpoint has advertised. + */ + abstract var audioSources: List + + /** + * Index of this endpoint's audio sources by SSRC, for O(1) lookup on the per-packet audio path (e.g. + * [Conference.levelChanged]). Rebuilt only when [audioSources] changes; subclasses must call + * [audioSourcesChanged] from their [audioSources] setter to keep it current. + */ + @Volatile + private var audioSourcesBySsrc: Map = emptyMap() + + /** Returns the advertised audio source with the given [ssrc], or null if there is none. */ + fun getAudioSource(ssrc: Long): AudioSourceDesc? = audioSourcesBySsrc[ssrc] + + /** Keeps the [getAudioSource] index in sync; call from the [audioSources] setter whenever it changes. */ + protected fun audioSourcesChanged(sources: List) { + audioSourcesBySsrc = sources.associateBy { it.ssrc } + } + + /** + * Adds a payload type to this endpoint. + */ + abstract fun addPayloadType(payloadType: PayloadType) + + /** + * Adds an RTP extension to this endpoint + */ + abstract fun addRtpExtension(rtpExtension: RtpExtension) + + /** + * Sets extmap-allow-mixed for this endpoint + */ + abstract fun setExtmapAllowMixed(allow: Boolean) + + /** + * Notifies this instance that the max video constraints that the bridge needs to receive from a source of this + * endpoint has changed. Each implementation handles this notification differently. + * + * @param sourceName the name of the media source + * @param maxVideoConstraints the max video constraints that the bridge needs to receive from the source + */ + protected abstract fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints) + + /** + * Notifies this instance that a specified received wants to receive the specified video constraints from the media + * source with the given source name. + * + * The receiver can be either another endpoint, or a remote bridge. + * + * @param receiverId the id that specifies the receiver endpoint. + * @param sourceName the name of the media source for which the constraints are to be applied. + * @param newVideoConstraints the video constraints that the receiver wishes to receive. + */ + fun addReceiver(receiverId: String, sourceName: String, newVideoConstraints: VideoConstraints) { + val sourceConstraints = receiverVideoConstraints.computeIfAbsent(sourceName) { ReceiverConstraintsMap() } + synchronized(sourceConstraints) { + val oldVideoConstraints = sourceConstraints.put(receiverId, newVideoConstraints) + if (oldVideoConstraints == null || oldVideoConstraints != newVideoConstraints) { + logger.debug { + "Changed receiver constraints: $receiverId->$sourceName: ${newVideoConstraints.maxHeight}" + } + sendReceiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight) + } + } + } + + /** Notify this endpoint that another endpoint expired */ + open fun otherEndpointExpired(expired: AbstractEndpoint) { + removeReceiver(expired.id) + } + + /** + * Notifies this instance that the specified receiver no longer wants or + * needs to receive anything from the endpoint attached to this + * instance (the sender). + * + * @param receiverId the id that specifies the receiver endpoint + */ + fun removeReceiver(receiverId: String) { + for ((sourceName, sourceConstraints) in receiverVideoConstraints) { + synchronized(sourceConstraints) { + if (sourceConstraints.remove(receiverId) != null) { + logger.debug { "Removed receiver $receiverId for $sourceName" } + sendReceiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight) + } + } + } + } + + /** + * Notifies this instance that the specified receiver no longer wants or needs to receive anything from the media + * source attached to this instance (the sender). + * + * @param receiverId the id that specifies the receiver endpoint + * @param sourceName the media source name + */ + fun removeSourceReceiver(receiverId: String, sourceName: String) { + val sourceConstraints = receiverVideoConstraints[sourceName] + if (sourceConstraints != null) { + synchronized(sourceConstraints) { + if (sourceConstraints.remove(receiverId) != null) { + logger.debug { "Removed receiver $receiverId for $sourceName" } + sendReceiverVideoConstraintsChanged(sourceName, sourceConstraints.maxHeight) + } + } + } + } + + interface EventHandler { + fun sourcesChanged() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt new file mode 100644 index 0000000..f44b9ff --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AbstractEndpointMessageTransport.kt @@ -0,0 +1,125 @@ +/* + * Copyright @ 2017 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.message.BridgeChannelMessage +import org.jitsi.videobridge.message.BridgeChannelMessage.Companion.parse +import org.jitsi.videobridge.message.MessageHandler +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.util.TaskPools +import java.io.IOException +import java.time.Clock + +abstract class AbstractEndpointMessageTransport(parentLogger: Logger) : MessageHandler() { + + protected val logger: Logger = parentLogger.createChildLogger(javaClass.name) + + abstract val isConnected: Boolean + + private val incomingMessageQueue: PacketQueue = PacketQueue( + 50, + true, + INCOMING_MESSAGE_QUEUE_ID, + { messageAndSource -> + try { + handleMessage(messageAndSource.message)?.let { response -> + sendMessage(messageAndSource.source, response) + } + } catch (e: Exception) { + logger.warn("Failed to handle message: ", e) + } + true + }, + TaskPools.IO_POOL, + Clock.systemUTC() + ).apply { setErrorHandler(queueErrorCounter) } + + /** + * Fires the message transport ready event for the associated endpoint. + */ + protected open fun notifyTransportChannelConnected() {} + + /** + * Notifies this [EndpointMessageTransport] that a specific message + * has been received on a specific transport channel. + * @param src the transport channel on which the message has been received. + * @param msg the message that was received. + */ + fun onMessage(src: Any?, msg: String) { + val message = try { + parse(msg) + } catch (ioe: IOException) { + logger.warn("Invalid message received (${ioe.message}: $msg") + return + } + + logger.debug { "RECV: $msg" } + incomingMessageQueue.add(MessageAndSource(message, src)) + } + + /** + * Sends [msg] over the active transport channel of this [EndpointMessageTransport]. + */ + protected open fun sendMessage(msg: BridgeChannelMessage) {} + protected open fun sendMessage(dst: Any?, message: BridgeChannelMessage) = + logger.debug { "SENT: " + message.toJson() } + + open fun close() {} + + open val debugState: ObjectNode + get() = JsonNodeFactory.instance.objectNode().apply { + val counts = JsonNodeFactory.instance.objectNode() + getReceivedCounts().forEach { (k, v) -> counts.put(k, v) } + set("received_counts", counts) + } + + /** + * Events generated by [AbstractEndpointMessageTransport] types which are of interest to other entities. + */ + interface EndpointMessageTransportEventHandler { + fun endpointMessageTransportConnected(endpoint: AbstractEndpoint) + } + + private data class MessageAndSource(val message: BridgeChannelMessage, val source: Any?) + + companion object { + const val INCOMING_MESSAGE_QUEUE_ID = "bridge-channel-message-incoming-queue" + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "endpoint_receive_message_queue_dropped_packets", + "Number of packets dropped out of the Endpoint receive message queue." + ) + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "endpoint_receive_message_queue_exceptions", + "Number of exceptions from the Endpoint receive message queue." + ) + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscription.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscription.kt new file mode 100644 index 0000000..aaf9802 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscription.kt @@ -0,0 +1,81 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.videobridge.message.ReceiverAudioSubscriptionMessage +import org.jitsi.videobridge.relay.AudioSourceDesc + +class AudioSubscription { + /** Whether to receive sources not named in [include]/[exclude] (synthetic sources excepted -- see below). */ + private var all: Boolean = false + + /** The source names being explicitly included / excluded, and the SSRCs they currently resolve to. */ + private var include: Set = emptySet() + private var exclude: Set = emptySet() + private var includedSsrcs: Set = emptySet() + private var excludedSsrcs: Set = emptySet() + + private val lock: Any = Any() + + fun updateSubscription(subscription: ReceiverAudioSubscriptionMessage, sources: List) = + synchronized(lock) { + all = subscription.all + include = subscription.include.toSet() + exclude = subscription.exclude.toSet() + includedSsrcs = ssrcsForNames(include, sources) + excludedSsrcs = ssrcsForNames(exclude, sources) + } + + private fun ssrcsForNames(names: Set, sources: Collection): Set = + sources.filter { it.sourceName != null && it.sourceName in names }.map(AudioSourceDesc::ssrc).toSet() + + fun isSsrcWanted(ssrc: Long): Boolean = when { + excludedSsrcs.contains(ssrc) -> false + includedSsrcs.contains(ssrc) -> true + else -> all + } + + /** + * Whether the given SSRC is explicitly wanted, i.e. named in the [include] list (and not excluded). Unlike + * [isSsrcWanted] this is not satisfied by [all], so it gates synthetic sources, which are forwarded only on an + * explicit subscription. + */ + fun isExplicitlyWanted(ssrc: Long): Boolean = includedSsrcs.contains(ssrc) && !excludedSsrcs.contains(ssrc) + + fun onConferenceSourceAdded(descs: Set) = synchronized(lock) { + includedSsrcs = includedSsrcs.union(ssrcsForNames(include, descs)) + excludedSsrcs = excludedSsrcs.union(ssrcsForNames(exclude, descs)) + } + + fun onConferenceSourceRemoved(descs: Set) = synchronized(lock) { + val removed = descs.map(AudioSourceDesc::ssrc).toSet() + includedSsrcs = includedSsrcs.subtract(removed) + excludedSsrcs = excludedSsrcs.subtract(removed) + } + + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + synchronized(lock) { + put("all", all) + putArray("include").apply { include.forEach { add(it) } } + putArray("exclude").apply { exclude.forEach { add(it) } } + putArray("included_ssrcs").apply { includedSsrcs.forEach { add(it) } } + putArray("excluded_ssrcs").apply { excludedSsrcs.forEach { add(it) } } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscriptionManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscriptionManager.kt new file mode 100644 index 0000000..7d9c8a3 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/AudioSubscriptionManager.kt @@ -0,0 +1,208 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.videobridge.message.ReceiverAudioSubscriptionMessage +import org.jitsi.videobridge.relay.AudioSourceDesc +import java.util.concurrent.ConcurrentHashMap + +class AudioSubscriptionManager() { + /** + * A map of endpoint IDs to their audio subscriptions. + */ + private val audioSubscriptions = ConcurrentHashMap() + + /** + * A map of local audio source names to a set of endpoint IDs that subscribe to the source with an "Include" type. + */ + private val subscribedLocalAudioSources = ConcurrentHashMap>() + + /** + * The audio sources currently known to the conference. This is the authoritative set against which subscriptions + * resolve their source names to SSRCs. It is maintained solely via [onSourcesAdded]/[removeSources] under [lock], + * which is the same lock that guards subscription updates. Resolving against this (rather than a source list + * captured by the caller outside [lock]) closes the race where a subscription arriving concurrently with the + * colibri signaling of a source would resolve against a stale snapshot and permanently drop the source. + */ + private val knownSources = mutableSetOf() + + /** + * An immutable snapshot classifying each currently-known audio SSRC as synthetic (true) or regular (false). An + * SSRC absent from the map is not a currently-known source and is never routed (see [isEndpointAudioWanted]). + * + * This is the single value read on the per-packet hot path: reading one volatile reference to an immutable map + * yields a consistent "is it known? is it synthetic?" answer with no tearing between two separate sets. It is + * rebuilt from [knownSources] under [lock] whenever sources change. + * + * Gating routing on membership here is what closes the synthetic-source leak in both directions: a synthetic + * SSRC that is not yet known (media arriving before colibri signals the source) or no longer known (media still + * arriving just after the source was removed) is absent from this map and so is dropped, rather than falling + * through to the permissive default and being forwarded as ordinary audio to non-subscribers. + */ + @Volatile + private var ssrcSynthetic: Map = emptyMap() + + private val lock: Any = Any() + + /** + * Sets the audio subscription for a given endpoint. + * @param endpointId the ID of the endpoint + * @param subscription the audio subscription message + */ + fun setEndpointAudioSubscription(endpointId: String, subscription: ReceiverAudioSubscriptionMessage) = + synchronized(lock) { + val audioSubscription = audioSubscriptions.getOrPut(endpointId) { + AudioSubscription() + } + + // Update subscribed local sources before updating subscription + updateSubscribedLocalAudioSourcesForEndpoint(endpointId, subscription) + // Resolve against the authoritative set of known sources, not a snapshot supplied by the caller: the caller + // gathers sources outside [lock], so a source signaled concurrently could otherwise be missed permanently. + audioSubscription.updateSubscription(subscription, knownSources.toList()) + } + + /* + * Checks if audio from a given SSRC is wanted by a specific endpoint. + * @param endpointId the ID of the endpoint + * @param ssrc the SSRC to check + * @return true if the audio is wanted, false otherwise + */ + + /** Whether the given SSRC belongs to a (currently-known) synthetic source. */ + fun isSynthetic(ssrc: Long): Boolean = ssrcSynthetic[ssrc] == true + + /** + * Returns the synthetic source with the given name, or null if there is none. Resolved against [knownSources] + * under [lock] so it is consistent with [isSynthetic]/[ssrcSynthetic]: callers (e.g. injecting translated media) + * see a synthetic source as resolvable exactly while it is treated as synthetic at routing time, rather than + * relying on a separately-maintained, lock-skewed view of the conference's sources. + */ + fun findSyntheticSource(sourceName: String): AudioSourceDesc? = synchronized(lock) { + knownSources.firstOrNull { it.synthetic && it.sourceName == sourceName } + } + + fun isEndpointAudioWanted(endpointId: String, ssrc: Long): Boolean { + // Single consistent read: null => not a currently-known source. + val synthetic = ssrcSynthetic[ssrc] + // Don't route audio that doesn't correspond to a currently-known source. Real audio always has a source + // signaled via colibri; an unrecognized SSRC is either a synthetic source whose desc hasn't arrived yet + // or was just removed (which must not leak as ordinary audio), or a stray SSRC -- none should be routed. + ?: return false + if (synthetic) { + // Synthetic sources are never routed automatically; only to an endpoint that explicitly (Include) + // subscribed to them by name. + return audioSubscriptions[endpointId]?.isExplicitlyWanted(ssrc) ?: false + } + val subscription = audioSubscriptions[endpointId] + return subscription?.isSsrcWanted(ssrc) ?: true + } + + /** + * Checks if a specific local audio source is explicitly subscribed to by any endpoint. + * @param sourceName the name of the audio source + * @return true if the source is explicitly subscribed, false otherwise + */ + fun isExplicitlySubscribed(sourceName: String?): Boolean = subscribedLocalAudioSources.containsKey(sourceName) + + /** + * Updates the subscribed local audio sources for a specific endpoint based on their subscription. + * @param endpointId the ID of the endpoint + * @param subscription the audio subscription message + */ + private fun updateSubscribedLocalAudioSourcesForEndpoint( + endpointId: String, + subscription: ReceiverAudioSubscriptionMessage + ) { + subscribedLocalAudioSources.values.forEach { endpointSet -> + endpointSet.remove(endpointId) + } + subscribedLocalAudioSources.entries.removeIf { it.value.isEmpty() } + // Only the explicit `include` list counts as an explicit subscription. + subscription.include.forEach { sourceName -> + subscribedLocalAudioSources.getOrPut(sourceName) { mutableSetOf() }.add(endpointId) + } + } + + /** + * Called when new audio sources are added to the conference. + * @param sources the new audio source descriptions + */ + fun onSourcesAdded(sources: Set) = synchronized(lock) { + knownSources.addAll(sources) + rebuildSsrcClassification() + audioSubscriptions.values.forEach { subscription -> + subscription.onConferenceSourceAdded(sources) + } + } + + /** Rebuilds the [ssrcSynthetic] snapshot from [knownSources]. Must be called under [lock]. */ + private fun rebuildSsrcClassification() { + ssrcSynthetic = knownSources.associate { it.ssrc to it.synthetic } + } + + /** + * Called when an endpoint is removed from the conference. + * @param id the endpoint ID that was removed + */ + fun removeEndpoint(id: String) = synchronized(lock) { + // Remove the endpoint from all sets + // This is necessary to precisely maintain the number of subscriptions to a source + subscribedLocalAudioSources.values.forEach { endpointSet -> + endpointSet.remove(id) + } + subscribedLocalAudioSources.entries.removeIf { it.value.isEmpty() } + audioSubscriptions.remove(id) + } + + fun removeSources(sources: Set) = synchronized(lock) { + knownSources.removeAll(sources) + rebuildSsrcClassification() + subscribedLocalAudioSources.keys.removeAll(sources.mapNotNull { it.sourceName }) + audioSubscriptions.values.forEach { subscription -> + subscription.onConferenceSourceRemoved(sources) + } + } + + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + synchronized(lock) { + // The conference's known audio sources, including which are synthetic. This is the authoritative set that + // routing (isEndpointAudioWanted) and subscription name resolution are gated on. + val sources = putArray("known_sources") + knownSources.forEach { src -> + sources.addObject().apply { + put("ssrc", src.ssrc) + put("name", src.sourceName) + put("owner", src.owner) + put("synthetic", src.synthetic) + } + } + // Local source names explicitly subscribed (Include) to by at least one endpoint -> the subscribers. + val subscribed = putObject("subscribed_local_sources") + subscribedLocalAudioSources.forEach { (name, endpointIds) -> + subscribed.putArray(name).apply { endpointIds.forEach { add(it) } } + } + // Per-endpoint subscription state (mode + resolved SSRCs). + val subscriptions = putObject("subscriptions") + audioSubscriptions.forEach { (endpointId, subscription) -> + subscriptions.set(endpointId, subscription.debugState()) + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt new file mode 100644 index 0000000..a491566 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ColibriQueue.kt @@ -0,0 +1,68 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.xmpp.XmppConnection +import java.time.Clock +import kotlin.Int.Companion.MAX_VALUE + +abstract class ColibriQueue(packetHandler: PacketHandler) : + PacketQueue( + MAX_VALUE, + true, + QUEUE_NAME, + packetHandler, + TaskPools.IO_POOL, + // TODO: using the Videobridge clock breaks tests somehow + Clock.systemUTC(), + // Allow running tasks to complete (so we can close the queue from within the task). + false, + ) { + init { + setErrorHandler(queueErrorCounter) + } + + companion object { + val QUEUE_NAME = "colibri-queue" + + val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "colibri_queue_dropped_packets", + "Number of packets dropped out of the Colibri queue." + ) + + val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "colibri_queue_exceptions", + "Number of exceptions from the Colibri queue." + ) + + /** Count the number of dropped packets and exceptions. */ + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ConferenceSpeechActivityConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ConferenceSpeechActivityConfig.kt new file mode 100644 index 0000000..21b844e --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ConferenceSpeechActivityConfig.kt @@ -0,0 +1,39 @@ +/* + * Copyright @ 2021 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import java.time.Duration + +class ConferenceSpeechActivityConfig { + val recentSpeakersCount: Int by config { + "videobridge.speech-activity.recent-speakers-count".from(JitsiConfig.newConfig) + } + + val enableSilenceDetection: Boolean by config { + "videobridge.speech-activity.enable-silence-detection".from(JitsiConfig.newConfig) + } + + val silenceDetectionTimeout: Duration by config { + "videobridge.speech-activity.silence-detection-timeout".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = ConferenceSpeechActivityConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ConnectionStatsConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ConnectionStatsConfig.kt new file mode 100644 index 0000000..5d484b4 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ConnectionStatsConfig.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import java.time.Duration + +class ConnectionStatsConfig private constructor() { + companion object { + val enabled: Boolean by config( + "videobridge.connection-stats.enabled".from(JitsiConfig.newConfig) + ) + val interval: Duration by config( + "videobridge.connection-stats.interval".from(JitsiConfig.newConfig) + ) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/CryptexConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/CryptexConfig.kt new file mode 100644 index 0000000..58577fb --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/CryptexConfig.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from + +class CryptexConfig private constructor() { + companion object { + val endpoint: Boolean by config( + "videobridge.cryptex.endpoint".from(JitsiConfig.newConfig) + ) + val relay: Boolean by config( + "videobridge.cryptex.relay".from(JitsiConfig.newConfig) + ) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt new file mode 100644 index 0000000..77aa8c7 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Endpoint.kt @@ -0,0 +1,1296 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.ice4j.util.Buffer +import org.jitsi.config.JitsiConfig +import org.jitsi.dcsctp4j.DcSctpMessage +import org.jitsi.dcsctp4j.ErrorKind +import org.jitsi.dcsctp4j.SendPacketStatus +import org.jitsi.dcsctp4j.SendStatus +import org.jitsi.metaconfig.config +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.Features +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.Transceiver +import org.jitsi.nlj.TransceiverEventHandler +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.format.PayloadTypeEncoding +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.ParsedVideoPacket +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.nlj.stats.EndpointConnectionStats +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.pipeline +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.LocalSsrcAssociation +import org.jitsi.nlj.util.PacketInfoQueue +import org.jitsi.nlj.util.RemoteSsrcAssociation +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket +import org.jitsi.utils.MediaType +import org.jitsi.utils.NEVER +import org.jitsi.utils.concurrent.PeriodicRunnable +import org.jitsi.utils.concurrent.RecurringRunnableExecutor +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.mins +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.sumOf +import org.jitsi.videobridge.cc.BandwidthProbing +import org.jitsi.videobridge.cc.allocation.BandwidthAllocation +import org.jitsi.videobridge.cc.allocation.BitrateController +import org.jitsi.videobridge.cc.allocation.EffectiveConstraintsMap +import org.jitsi.videobridge.cc.allocation.VideoConstraints +import org.jitsi.videobridge.datachannel.DataChannelStack +import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket +import org.jitsi.videobridge.dcsctp.DcSctpBaseCallbacks +import org.jitsi.videobridge.dcsctp.DcSctpHandler +import org.jitsi.videobridge.dcsctp.DcSctpTransport +import org.jitsi.videobridge.message.BridgeChannelMessage +import org.jitsi.videobridge.message.ConnectionStats +import org.jitsi.videobridge.message.ForwardedSourcesMessage +import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage +import org.jitsi.videobridge.message.SenderSourceConstraintsMessage +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.relay.AudioSourceDesc +import org.jitsi.videobridge.relay.RelayedEndpoint +import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures +import org.jitsi.videobridge.stats.PacketTransitStats +import org.jitsi.videobridge.transport.dtls.DtlsTransport +import org.jitsi.videobridge.transport.ice.IceRestartResult +import org.jitsi.videobridge.transport.ice.IceTransport +import org.jitsi.videobridge.util.ByteBufferPool +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.util.looksLikeDtls +import org.jitsi.videobridge.websocket.colibriWebSocketServiceSupplier +import org.jitsi.videobridge.websocket.generateColibriWebSocketPassword +import org.jitsi.xmpp.extensions.colibri.WebSocketPacketExtension +import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import java.nio.ByteBuffer +import java.security.SecureRandom +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import kotlin.collections.ArrayList +import kotlin.collections.HashSet +import org.jitsi.videobridge.sctp.SctpConfig.Companion.config as sctpConfig + +/** + * Models a local endpoint (participant) in a [Conference] + */ +class Endpoint @JvmOverloads constructor( + id: String, + conference: Conference, + parentLogger: Logger, + /** + * True if the ICE agent for this Endpoint will be initialized to serve + * as a controlling ICE agent, false otherwise + */ + iceControlling: Boolean, + private val doSsrcRewriting: Boolean, + /** + * Whether the bridge should stamp the sdes:mid header extension on packets forwarded to this endpoint (and signal + * the per-slot mids in the source maps), so the client can demux by mid. Only meaningful when [doSsrcRewriting]. + */ + private val doMidDemux: Boolean, + /** + * Whether this endpoint is in "visitor" mode, i.e. should be invisible to other endpoints. + */ + override val visitor: Boolean, + supportsPrivateAddresses: Boolean, + /** + * Whether diarization is requested for this endpoint's audio, as signaled via the colibri2 `diarize` attribute. + */ + override val diarize: Boolean, + private val clock: Clock = Clock.systemUTC() +) : AbstractEndpoint(conference, id, parentLogger), + PotentialPacketHandler, + EncodingsManager.EncodingsUpdateListener, + SsrcRewriter { + + /** The time at which this endpoint was created */ + val creationTime = clock.instant() + + private val sctpHandler = if (sctpConfig.enabled) DcSctpHandler() else null + + /** The [DcSctpTransport] instance we'll use to manage the SCTP connection */ + private var sctpTransport: DcSctpTransport? = null + + private var dataChannelStack = DataChannelStack( + { data, sid, ppid -> + val message = DcSctpMessage(sid.toShort(), ppid, data.array()) + val status = sctpTransport?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) + return@DataChannelStack if (status == SendStatus.kSuccess) { + 0 + } else { + logger.error("Error sending to SCTP: $status") + -1 + } + }, + logger + ) + + private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-sctp") + private val sctpRecvPcap = toggleablePcapWriter.newObserverNode(outbound = false, suffix = "rx_sctp") + private val sctpSendPcap = toggleablePcapWriter.newObserverNode(outbound = true, suffix = "tx_sctp") + + private val sctpPipeline = pipeline { + node(sctpRecvPcap) + sctpHandler?.let { + node(it) + } + } + + /** + * The password which authenticates the colibri WebSocket of this endpoint. It is independent of ICE and + * stays the same for the lifetime of the endpoint, because the client re-dials the URL which contains it + * each time the WebSocket reconnects. See [generateColibriWebSocketPassword]. + */ + private val webSocketPassword = generateColibriWebSocketPassword() + + /* TODO: do we ever want to support useUniquePort for an Endpoint? */ + private val iceTransport = IceTransport(id, iceControlling, false, supportsPrivateAddresses, logger) + private val dtlsTransport = DtlsTransport(logger, id).also { it.cryptex = CryptexConfig.endpoint } + + private var cryptex: Boolean = CryptexConfig.endpoint + + private val diagnosticContext = conference.newDiagnosticContext().apply { + put("endpoint_id", id) + } + + private val timelineLogger = logger.createChildLogger("timeline.${this.javaClass.name}") + + /** + * Whether this endpoint should accept audio packets. We set this according + * to whether the endpoint has an audio Colibri channel whose direction + * allows sending. + */ + var acceptAudio = false + + /** + * Whether this endpoint should accept video packets. We set this according + * to whether the endpoint has a video Colibri channel whose direction + * allows sending. + */ + var acceptVideo = false + + /** + * Whether a [ReceiverVideoConstraintsMessage] has been received from the endpoint. Setting initial-last-n only + * has an effect prior to constraints being received on the message transport. + */ + var initialReceiverConstraintsReceived = false + + /** + * The queue we put outgoing SRTP packets onto so they can be sent + * out via the [IceTransport] on an IO thread. + */ + private val outgoingSrtpPacketQueue = PacketInfoQueue( + "${javaClass.simpleName}-outgoing-packet-queue", + TaskPools.IO_POOL, + this::doSendSrtp, + TransportConfig.queueSize + ).apply { + setErrorHandler(queueErrorCounter) + } + + /** + * The queue which enforces sequential processing of incoming data channel messages + * to maintain processing order. + */ + private val incomingDataChannelMessagesQueue = PacketInfoQueue( + "${javaClass.simpleName}-incoming-data-channel-queue", + TaskPools.IO_POOL, + { packetInfo -> + packetInfo.packetAs().let { packet -> + dataChannelStack.onIncomingDataChannelPacket( + ByteBuffer.wrap(packet.buffer), + packet.sid, + packet.ppid + ) + } + true + }, + TransportConfig.queueSize + ) + + private val bitrateController = BitrateController( + object : BitrateController.EventHandler { + override fun allocationChanged(allocation: BandwidthAllocation) { + // Intentional no-op + } + + override fun forwardedSourcesChanged(forwardedSources: Set) { + sendForwardedSourcesMessage(forwardedSources) + } + + override fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap, + ) = this@Endpoint.effectiveVideoConstraintsChanged(oldEffectiveConstraints, newEffectiveConstraints) + + override fun keyframeNeeded(endpointId: String?, ssrc: Long) = + conference.requestKeyframe(id, endpointId, ssrc) + }, + { getOrderedEndpoints() }, + diagnosticContext, + logger + ) + + /** Whether any sources are suspended from being sent to this endpoint because of BWE. */ + fun hasSuspendedSources() = bitrateController.hasSuspendedSources() + + /** + * The instance which manages the Colibri messaging (over a data channel + * or web sockets). + */ + override val messageTransport = EndpointMessageTransport( + this, + conference, + logger + ) + + /** + * Gets the endpoints in the conference in LastN order, with this {@link Endpoint} removed. + */ + fun getOrderedEndpoints(): List = conference.orderedEndpoints.filterNot { it == this } + + /** + * Listen for RTT updates from [transceiver] and update the ICE stats the first time an RTT is available. Note that + * the RTT is measured via RTCP, since we don't expose response time for STUN requests. + */ + private val rttListener: EndpointConnectionStats.EndpointConnectionStatsListener = + object : EndpointConnectionStats.EndpointConnectionStatsListener { + override fun onRttUpdate(newRttMs: Double) { + if (newRttMs > 0) { + transceiver.removeEndpointConnectionStatsListener(this) + iceTransport.updateStatsOnInitialRtt(newRttMs) + } + } + } + + val transceiver = Transceiver( + id, + TaskPools.CPU_POOL, + TaskPools.CPU_POOL, + TaskPools.SCHEDULED_POOL, + diagnosticContext, + logger, + TransceiverEventHandlerImpl(), + clock, + getMidBySsrc = this::getMidBySsrc + ).apply { + setIncomingPacketHandler(object : ConsumerNode("receiver chain handler") { + override fun consume(packetInfo: PacketInfo) { + this@Endpoint.handleIncomingPacket(packetInfo) + } + + override fun trace(f: () -> Unit) = f.invoke() + }) + addEndpointConnectionStatsListener(rttListener) + setLocalSsrc(MediaType.AUDIO, conference.localAudioSsrc) + setLocalSsrc(MediaType.VIDEO, conference.localVideoSsrc) + rtpSender.preProcesor = { packetInfo -> preProcess(packetInfo) } + } + + /** + * Perform processing of the packet before it goes through the rest of the [transceiver] send pipeline: + * 1. Update the bitrate controller state and apply the source projection logic + * 2. Perform SSRC re-writing if [doSsrcRewriting] is set. + */ + private fun preProcess(packetInfo: PacketInfo): PacketInfo? { + when (val packet = packetInfo.packet) { + is VideoRtpPacket -> { + if (!bitrateController.transformRtp(packetInfo)) { + logger.warn("Dropping a packet which was supposed to be accepted:$packet") + return null + } + // The original packet was transformed in place. + if (doSsrcRewriting) { + val start = packet !is ParsedVideoPacket || (packet.isKeyframe && packet.isStartOfFrame) + if (!videoSsrcs.rewriteRtp(packet, start)) { + return null + } + } + } + + is AudioRtpPacket -> if (doSsrcRewriting) audioSsrcs.rewriteRtp(packet) + + is RtcpSrPacket -> { + // Allow the BC to update the timestamp (in place). + bitrateController.transformRtcp(packet) + if (doSsrcRewriting) { + // Just check both tables instead of looking up the type first. + if (!videoSsrcs.rewriteRtcp(packet) && !audioSsrcs.rewriteRtcp(packet)) { + return null + } + } + } + } + return packetInfo + } + + private val bandwidthProbing = BandwidthProbing( + object : BandwidthProbing.ProbingDataSender { + override fun sendProbing(mediaSsrcs: Collection, numBytes: Int): Int = + transceiver.sendProbing(mediaSsrcs, numBytes) + }, + { bitrateController.getStatusSnapshot() } + ).apply { + diagnosticsContext = this@Endpoint.diagnosticContext + enabled = true + }.also { + recurringRunnableExecutor.registerRecurringRunnable(it) + } + + /** + * Manages remapping of video SSRCs when enabled. + */ + private val videoSsrcs = VideoSsrcCache(SsrcLimitConfig.config.maxVideoSsrcs, this, doMidDemux, logger) + + /** + * Manages remapping of audio SSRCs when enabled. + */ + private val audioSsrcs = AudioSsrcCache(SsrcLimitConfig.config.maxAudioSsrcs, this, doMidDemux, logger) + + /** + * Last advertised forwarded-sources in remapping mode. + */ + private var activeSources: Set = emptySet() + + /** + * Track allocated send SSRCs to avoid collisions. + */ + private val sendSsrcs = mutableSetOf() + + /** + * Next allocated send SSRC, when allocating serially (off by default). + */ + private var nextSendSsrc = 777000001L + + /** + * Latest bandwidth received from bandwidth estimator. + */ + private var latestBandwidth: Bandwidth? = null + + /** + * Recurring event to send connection stats messages. + */ + private val connectionStatsSender = + if (ConnectionStatsConfig.enabled) { + object : PeriodicRunnable(ConnectionStatsConfig.interval.toMillis()) { + override fun run() { + super.run() + latestBandwidth?.let { sendMessage(ConnectionStats(it.bps)) } + } + }.also { + recurringRunnableExecutor.registerRecurringRunnable(it) + } + } else { + null + } + + init { + conference.encodingsManager.subscribe(this) + setupIceTransport() + setupDtlsTransport() + + VideobridgeMetrics.totalEndpoints.inc() + if (visitor) { + VideobridgeMetrics.totalVisitors.inc() + } + + logger.info("Created new endpoint, iceControlling=$iceControlling") + } + + override var mediaSources: Array + get() = transceiver.getMediaSources() + set(value) { + applyVideoTypeCache(value) + val wasEmpty = transceiver.getMediaSources().isEmpty() + if (transceiver.setMediaSources(value)) { + eventEmitter.fireEvent { sourcesChanged() } + } + if (wasEmpty) { + sendAllVideoConstraints() + } + } + + /** + * Keep track of this endpoint's audio sources. + */ + override var audioSources: List = ArrayList() + set(newValue) { + val oldValue = field + val removedDescs = oldValue.filterNot { newValue.contains(it) }.toSet() + conference.removeAudioSources(removedDescs) + val addedDescs = newValue.filterNot { oldValue.contains(it) }.toSet() + conference.addAudioSources(addedDescs) + field = newValue + audioSourcesChanged(newValue) + } + + private fun setupIceTransport() { + iceTransport.incomingDataHandler = object : IceTransport.IncomingDataHandler { + override fun dataReceived(buffer: Buffer) { + if (looksLikeDtls(buffer.buffer, buffer.offset, buffer.length)) { + // DTLS transport is responsible for making its own copy, because it will manage its own buffers + dtlsTransport.enqueueBuffer(buffer) + } else { + val pktInfo = + PacketInfo(UnparsedPacket(buffer.buffer, buffer.offset, buffer.length)).apply { + this.receivedTime = buffer.receivedTime + } + pktInfo.packetOrigin = PacketOrigin.Routed + transceiver.handleIncomingPacket(pktInfo) + } + } + } + iceTransport.eventHandler = object : IceTransport.EventHandler { + override fun writeable() { + logger.info("ICE connected") + transceiver.setOutgoingPacketHandler(object : PacketHandler { + override fun processPacket(packetInfo: PacketInfo) { + packetInfo.addEvent(SRTP_QUEUE_ENTRY_EVENT) + outgoingSrtpPacketQueue.add(packetInfo) + } + }) + TaskPools.IO_POOL.execute(dtlsTransport::startDtlsHandshake) + } + + override fun connected() { + } + + override fun failed() { + VideobridgeMetrics.endpointsIceFailed.inc() + } + + override fun consentUpdated(time: Instant) { + transceiver.packetIOActivity.lastIceActivityInstant = time + } + } + } + + /** + * Return the list of sources the endpoint has selected as "on stage". We just concatenate with the old + * "onStageEndpoints" since even with the old API we have matching source names. + */ + fun getOnStageSources() = + bitrateController.allocationSettings.onStageEndpoints + bitrateController.allocationSettings.onStageSources + + private fun setupDtlsTransport() { + dtlsTransport.incomingDataHandler = object : DtlsTransport.IncomingDataHandler { + override fun dtlsAppDataReceived(buf: ByteArray, off: Int, len: Int) { + this@Endpoint.dtlsAppPacketReceived(buf, off, len) + } + } + dtlsTransport.outgoingDataHandler = object : DtlsTransport.OutgoingDataHandler { + override fun sendData(buf: ByteArray, off: Int, len: Int) { + iceTransport.send(buf, off, len) + } + } + dtlsTransport.eventHandler = object : DtlsTransport.EventHandler { + override fun handshakeComplete( + chosenSrtpProtectionProfile: Int, + tlsRole: TlsRole, + keyingMaterial: ByteArray + ) { + logger.info("DTLS handshake complete") + transceiver.setSrtpInformation(chosenSrtpProtectionProfile, tlsRole, keyingMaterial, cryptex) + scheduleEndpointMessageTransportTimeout() + } + } + } + + fun updateForceMute(audioForceMuted: Boolean, videoForceMuted: Boolean) { + transceiver.forceMuteAudio(audioForceMuted) + transceiver.forceMuteVideo(videoForceMuted) + } + + override fun addPayloadType(payloadType: PayloadType) { + transceiver.addPayloadType(payloadType) + bitrateController.addPayloadType(payloadType) + } + + /** The Opus payload type negotiated with this endpoint, or null if none. */ + fun getOpusPayloadType(): PayloadType? = transceiver.readOnlyStreamInformationStore.rtpPayloadTypes.values + .firstOrNull { it.encoding == PayloadTypeEncoding.OPUS } + + override fun addRtpExtension(rtpExtension: RtpExtension) = transceiver.addRtpExtension(rtpExtension) + + override fun setExtmapAllowMixed(allow: Boolean) = transceiver.setExtmapAllowMixed(allow) + + fun addReceiveSsrc(ssrc: Long, mediaType: MediaType) { + logger.cdebug { "Adding receive ssrc $ssrc of type $mediaType" } + transceiver.addReceiveSsrc(ssrc, mediaType) + conference.addEndpointSsrc(this, ssrc) + } + + override fun onNewSsrcAssociation( + endpointId: String, + primarySsrc: Long, + secondarySsrc: Long, + type: SsrcAssociationType + ) { + if (endpointId.equals(id, ignoreCase = true)) { + transceiver.addSsrcAssociation(LocalSsrcAssociation(primarySsrc, secondarySsrc, type)) + } else { + transceiver.addSsrcAssociation(RemoteSsrcAssociation(primarySsrc, secondarySsrc, type)) + } + } + + fun setFeature(feature: EndpointDebugFeatures, enabled: Boolean) { + when (feature) { + EndpointDebugFeatures.PCAP_DUMP -> transceiver.setFeature(Features.TRANSCEIVER_PCAP_DUMP, enabled) + + EndpointDebugFeatures.SCTP_PCAP_DUMP -> + if (enabled) { + toggleablePcapWriter.enable() + } else { + toggleablePcapWriter.disable() + } + } + } + + fun isFeatureEnabled(feature: EndpointDebugFeatures): Boolean = when (feature) { + EndpointDebugFeatures.PCAP_DUMP -> transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP) + EndpointDebugFeatures.SCTP_PCAP_DUMP -> toggleablePcapWriter.isEnabled() + } + override val isSendingAudio: Boolean + get() = + // The endpoint is sending audio if we (the transceiver) are receiving audio. + transceiver.isReceivingAudio() + + override val isSendingVideo: Boolean + get() = + // The endpoint is sending video if we (the transceiver) are receiving video. + transceiver.isReceivingVideo() + + private fun doSendSrtp(packetInfo: PacketInfo): Boolean { + packetInfo.addEvent(SRTP_QUEUE_EXIT_EVENT) + + iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) + packetInfo.addEvent(SENT_OVER_ICE_TRANSPORT_EVENT) + PacketTransitStats.packetSent(packetInfo) + ByteBufferPool.returnBuffer(packetInfo.packet.buffer) + packetInfo.sent() + if (timelineLogger.isTraceEnabled && logTimeline()) { + timelineLogger.trace { packetInfo.timeline.toString() } + } + return true + } + + /** + * Notifies this endpoint that the ordered list of last-n endpoints has changed + */ + fun lastNEndpointsChanged() = bitrateController.endpointOrderingChanged() + + /** + * {@inheritDoc} + */ + override fun sendMessage(msg: BridgeChannelMessage) = messageTransport.sendMessage(msg) + + // TODO: this should be part of an EndpointMessageTransport.EventHandler interface + fun endpointMessageTransportConnected() { + sendAllVideoConstraints() + sendForwardedSourcesMessage(bitrateController.forwardedSources) + videoSsrcs.sendAllMappings() + audioSsrcs.sendAllMappings() + } + + private fun sendAllVideoConstraints() { + maxReceiverVideoConstraints.forEach { (sourceName, constraints) -> + sendVideoConstraints(sourceName, constraints) + } + } + + /** + * Handle a DTLS app packet (that is, a packet of some other protocol sent + * over DTLS) which has just been received. + */ + // TODO(brian): change sctp handler to take buf, off, len + fun dtlsAppPacketReceived(data: ByteArray, off: Int, len: Int) = + sctpPipeline.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + + private fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap + ) { + val removedSources = oldEffectiveConstraints.keys.filterNot { it in newEffectiveConstraints.keys } + + // Sources that "this" endpoint no longer receives. + removedSources.map { it.sourceName }.forEach { removedSourceName -> + // Remove ourself as a receiver from that endpoint + conference.findSourceOwner(removedSourceName)?.removeSourceReceiver(removedSourceName, id) + } + + // Added or updated + newEffectiveConstraints.forEach { (source, effectiveConstraints) -> + conference.findSourceOwner(source.sourceName)?.addReceiver(id, source.sourceName, effectiveConstraints) + } + + if (doSsrcRewriting) { + val newActiveSources = + newEffectiveConstraints.entries.filter { + !it.value.isDisabled() && it.key.videoType.isEnabled() + }.map { it.key }.toMutableList() + + // Populate any currently-disabled sources with an enabled video type into the ssrc cache, if there's room, + // so we have mappings for them already if they become enabled later (e.g. because of lastN). + val remainingMapSpace = videoSsrcs.size - newActiveSources.size + if (remainingMapSpace > 0) { + newActiveSources += newEffectiveConstraints.entries.filter { + it.value.isDisabled() && it.key.videoType.isEnabled() + }.take(remainingMapSpace).map { it.key }.toList() + } + + val newActiveSourceNames = newActiveSources.map { it.sourceName }.toSet() + /* safe unlocked access of activeSources. BitrateController will not overlap calls to this method. */ + if (activeSources != newActiveSourceNames) { + activeSources = newActiveSourceNames + videoSsrcs.activate(newActiveSources) + } + } + } + + override fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints) { + // Note that it's up to the client to respect these constraints. + if (findMediaSourceDesc(sourceName) == null) { + logger.warn { + "Suppressing sending a SenderVideoConstraints message, endpoint has no such source: $sourceName" + } + } else { + val senderSourceConstraintsMessage = + SenderSourceConstraintsMessage(sourceName, maxVideoConstraints.maxHeight) + logger.cdebug { "Sender constraints changed: ${senderSourceConstraintsMessage.toJson()}" } + sendMessage(senderSourceConstraintsMessage) + } + } + + /** + * Create an SCTP connection for this Endpoint. If [OPEN_DATA_CHANNEL_LOCALLY] is true, + * we will create the data channel locally, otherwise we will wait for the remote side + * to open it. + */ + fun createSctpConnection() { + if (sctpConfig.enabled) { + logger.cdebug { "Creating SCTP transport" } + sctpTransport = DcSctpTransport(id, logger).also { + it.start(SctpCallbacks(it)) + sctpHandler?.setSctpTransport(it) + } + } else { + logger.error("Not creating SCTP connection, SCTP is disabled in configuration.") + } + } + + /** + * Schedule a timeout to fire log a message and track a stat if we don't + * have an endpoint message transport connected within the timeout. + */ + fun scheduleEndpointMessageTransportTimeout() { + TaskPools.SCHEDULED_POOL.schedule( + { + if (!isExpired) { + if (!messageTransport.isConnected) { + logger.error("EndpointMessageTransport still not connected.") + VideobridgeMetrics.numEndpointsNoMessageTransportAfterDelay.inc() + } + } + }, + 30, + TimeUnit.SECONDS + ) + } + + /** + * Checks whether a WebSocket connection with a specific password string + * should be accepted for this endpoint. + * @param password the + * @return {@code true} iff the password matches. + */ + fun acceptWebSocket(password: String): Boolean { + if (webSocketPassword != password) { + logger.warn("Incoming web socket request with an invalid password.") + return false + } + return true + } + + /** + * Sends a message to this endpoint in order to notify it that the set of media sources for which the bridge + * is sending video has changed. + * + * @param forwardedSources the collection of forwarded media sources (by name). + */ + fun sendForwardedSourcesMessage(forwardedSources: Collection) { + val msg = ForwardedSourcesMessage(forwardedSources) + TaskPools.IO_POOL.execute { + try { + sendMessage(msg) + } catch (t: Throwable) { + logger.warn("Failed to send message:", t) + } + } + } + + /** + * Sets the remote transport information (ICE candidates, DTLS fingerprints). + * + * @param transportInfo the XML extension which contains the remote + * transport information. + */ + fun setTransportInfo(transportInfo: IceUdpTransportPacketExtension) { + val remoteFingerprints = mutableMapOf>() + val fingerprintExtensions = transportInfo.getChildExtensionsOfType(DtlsFingerprintPacketExtension::class.java) + fingerprintExtensions.forEach { fingerprintExtension -> + if (fingerprintExtension.hash != null && fingerprintExtension.fingerprint != null) { + remoteFingerprints.getOrPut(fingerprintExtension.hash.lowercase()) { mutableListOf() } + .add(fingerprintExtension.fingerprint) + } else { + logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toXML()}") + } + if (CryptexConfig.endpoint) { + cryptex = cryptex && fingerprintExtension.cryptex + } + } + dtlsTransport.setRemoteFingerprints(remoteFingerprints) + if (fingerprintExtensions.isNotEmpty()) { + val setup = fingerprintExtensions.first().setup + dtlsTransport.setSetupAttribute(setup) + } + iceTransport.startConnectivityEstablishment(transportInfo) + } + + /** + * Handles an explicit ICE restart request from this endpoint (colibri2 ``). + * + * Creates a new ice4j Agent with freshly rotated local credentials alongside the established one, which + * keeps carrying media until the new one connects (make-before-break). + * + * @return what the caller must signal back to the endpoint: our new transport (returned by + * [describeTransport], which then describes the pending Agent) for [IceRestartResult.STARTED], the + * unchanged established transport for [IceRestartResult.KEEP_EXISTING], or no transport at all for + * [IceRestartResult.UNAVAILABLE]. + */ + fun requestIceRestart(): IceRestartResult = iceTransport.requestIceRestart() + + fun describeTransport(): IceUdpTransportPacketExtension { + val iceUdpTransportPacketExtension = IceUdpTransportPacketExtension() + iceTransport.describe(iceUdpTransportPacketExtension) + dtlsTransport.describe(iceUdpTransportPacketExtension) + colibriWebSocketServiceSupplier.get()?.let { colibriWebsocketService -> + colibriWebsocketService.getColibriWebSocketUrls( + conference.id, + id, + webSocketPassword + ).forEach { wsUrl -> + val wsPacketExtension = WebSocketPacketExtension(wsUrl) + iceUdpTransportPacketExtension.addChildExtension(wsPacketExtension) + } + } + + logger.cdebug { "Transport description:\n${iceUdpTransportPacketExtension.toXML()}" } + + return iceUdpTransportPacketExtension + } + + /** + * Handle incoming RTP packets which have been fully processed by the + * transceiver's incoming pipeline. + */ + fun handleIncomingPacket(packetInfo: PacketInfo) { + if (visitor) { + if (packetInfo.packet !is RtcpFbPliPacket && packetInfo.packet !is RtcpFbFirPacket) { + /* Never forward RTP/RTCP from a visitor, except PLI/FIR. */ + ByteBufferPool.returnBuffer(packetInfo.packet.buffer) + return + } + } + + packetInfo.endpointId = id + conference.handleIncomingPacket(packetInfo) + } + + override fun receivesSsrc(ssrc: Long): Boolean = transceiver.receivesSsrc(ssrc) + + fun doesSsrcRewriting(): Boolean = doSsrcRewriting + + /** + * Returns the mid the bridge stamps on packets forwarded with the given (rewritten) send SSRC, or null if mid + * demux is not enabled for this endpoint or the SSRC is not a rewritten slot SSRC. Invoked per outgoing packet by + * the sender pipeline's [org.jitsi.nlj.transform.node.outgoing.MidStamper]. + */ + fun getMidBySsrc(ssrc: Long): String? { + if (!doMidDemux) { + return null + } + return audioSsrcs.getMidBySsrc(ssrc) ?: videoSsrcs.getMidBySsrc(ssrc) + } + + fun unmapRtcpFbSsrc(packet: RtcpFbPacket) = videoSsrcs.unmapRtcpFbSsrc(packet) + + override val ssrcs + get() = HashSet(transceiver.receiveSsrcs) + + override val lastIncomingActivity + get() = transceiver.packetIOActivity.lastIncomingActivityInstant + + override fun requestKeyframe() = transceiver.requestKeyFrame() + + override fun requestKeyframe(requesterID: String, mediaSsrc: Long) = + transceiver.requestKeyFrame(requesterID, mediaSsrc) + + /** Whether we are currently oversending to this endpoint. */ + fun isOversending(): Boolean = bitrateController.isOversending() + + /** + * Returns how many video sources are currently forwarding to this endpoint. + */ + fun numForwardedSources(): Int = bitrateController.numForwardedSources() + + fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage) { + initialReceiverConstraintsReceived = true + bitrateController.setBandwidthAllocationSettings(message) + } + + /** + * {@inheritDoc} + */ + override fun findVideoSourceProps(ssrc: Long): MediaSourceDesc? { + conference.getEndpointBySsrc(ssrc)?.let { ep -> + ep.mediaSources.forEach { s -> + if (s.findRtpEncodingDesc(ssrc) != null) { + return s + } + } + } + logger.error { "No properties found for SSRC $ssrc." } + return null + } + + /** + * {@inheritDoc} + */ + override fun findAudioSourceProps(ssrc: Long): AudioSourceDesc? { + conference.getEndpointBySsrc(ssrc)?.let { ep -> + return when (ep) { + is Endpoint -> ep.audioSources + is RelayedEndpoint -> ep.audioSources + else -> emptyList() + }.find { it.ssrc == ssrc } + } + logger.error { "No properties found for SSRC $ssrc." } + return null + } + + /** + * {@inheritDoc} + */ + override fun getNextSendSsrc(): Long { + synchronized(sendSsrcs) { + while (true) { + val ssrc = if (useRandomSendSsrcs) { + random.nextLong().and(0xFFFF_FFFFL) + } else { + nextSendSsrc++ + } + if (sendSsrcs.add(ssrc)) { + return ssrc + } + } + } + } + + override fun send(packetInfo: PacketInfo) = transceiver.sendPacket(packetInfo) + + /** + * To find out whether the endpoint should be expired, we check the activity timestamps from the transceiver. + */ + override fun shouldExpire(): Boolean { + if (iceTransport.hasFailed()) { + logger.warn("Allowing to expire because ICE failed.") + return true + } + + val lastActivity = lastIncomingActivity + val now = clock.instant() + + if (lastActivity == NEVER) { + val timeSinceCreation = Duration.between(creationTime, now) + if (timeSinceCreation > epTimeout) { + logger.info( + "Endpoint's ICE connection has neither failed nor connected " + + "after $timeSinceCreation expiring" + ) + return true + } + // We haven't seen any activity yet. If this continues ICE will + // eventually fail (which is handled above). + return false + } + + val expireTimeout = VideobridgeExpireThreadConfig.config.inactivityTimeout + if (Duration.between(lastActivity, now) > expireTimeout) { + logger.info("Allowing to expire because of no activity in over $expireTimeout") + return true + } + return false + } + + override fun otherEndpointExpired(expired: AbstractEndpoint) = super.otherEndpointExpired(expired).also { + if (doSsrcRewriting) { + // Remove the expired endpoint's sources. We don't want duplicate entries in case the endpoint is + // re-created (e.g. after an ICE restart). + audioSsrcs.removeByOwner(expired.id) + videoSsrcs.removeByOwner(expired.id) + } + } + + fun setLastN(lastN: Int) { + bitrateController.lastN = lastN + } + + /** + * Returns true if this endpoint's transport is 'fully' connected (both ICE and DTLS), false otherwise + */ + private fun isTransportConnected(): Boolean = iceTransport.isConnected() && dtlsTransport.isConnected + + fun getRtt(): Double = transceiver.getTransceiverStats().endpointConnectionStats.rtt + + override fun wants(packetInfo: PacketInfo): Boolean { + if (!isTransportConnected()) { + return false + } + + return when (val packet = packetInfo.packet) { + is VideoRtpPacket -> acceptVideo && bitrateController.accept(packetInfo) + + is AudioRtpPacket -> acceptAudio && conference.isEndpointAudioWanted(id, packet.ssrc) + + is RtcpSrPacket -> { + // TODO: For SRs we're only interested in the ntp/rtp timestamp + // association, so we could only accept srs from the main ssrc + bitrateController.accept(packet) + } + + is RtcpFbPliPacket, is RtcpFbFirPacket -> { + // We assume that we are only given PLIs/FIRs destined for this + // endpoint. This is because Conference has to find the target + // endpoint (this endpoint) anyway, and we would essentially be + // performing the same check twice. + true + } + + else -> { + logger.warn("Ignoring an unknown packet type:" + packet.javaClass.simpleName) + false + } + } + } + + /** + * Determine whether to forward endpoint stats from another endpoint to this one. + */ + fun wantsStatsFrom(ep: AbstractEndpoint): Boolean { + // Always forward stats in small conferences + if (conference.endpointCount <= statsFilterThreshold) { + return true + } + logger.debug { + buildString { + append("wantsStatsFrom(${ep.id}): isRecentSpeaker=${conference.speechActivity.isRecentSpeaker(ep)} ") + append("isRankedSpeaker=${conference.isRankedSpeaker(ep)} ") + if (ep.mediaSources.isEmpty()) { + append("(no media sources)") + } + ep.mediaSources.forEach { source -> + val name = source.sourceName + append("isOnStageOrSelected($name)=${bitrateController.isOnStageOrSelected(source)} ") + append("hasNonZeroEffectiveConstraints($name)=") + append("${bitrateController.hasNonZeroEffectiveConstraints(source)} ") + } + } + } + + if (conference.speechActivity.isRecentSpeaker(ep) || conference.isRankedSpeaker(ep)) { + return true + } + return ep.mediaSources.any { source -> + bitrateController.isOnStageOrSelected(source) || + bitrateController.hasNonZeroEffectiveConstraints(source) + } + } + + /** + * Updates the conference statistics with value from this endpoint. Since + * the values are cumulative this should execute only once when the endpoint + * expires. + */ + private fun updateStatsOnExpire() { + val transceiverStats = transceiver.getTransceiverStats() + + val incomingStats = transceiverStats.rtpReceiverStats.packetStreamStats + val outgoingStats = transceiverStats.outgoingPacketStreamStats + VideobridgeMetrics.totalBytesReceived.add(incomingStats.bytes) + VideobridgeMetrics.totalBytesSent.add(outgoingStats.bytes) + VideobridgeMetrics.packetsReceived.addAndGet(incomingStats.packets) + VideobridgeMetrics.packetsSent.addAndGet(outgoingStats.packets) + VideobridgeMetrics.keyframesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numKeyframes.toLong() + ) + VideobridgeMetrics.layeringChangesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numLayeringChanges.toLong() + ) + val durationActiveVideo = transceiverStats.rtpReceiverStats.incomingStats.ssrcStats.values.filter { + it.mediaType == MediaType.VIDEO + }.sumOf { it.durationActive } + VideobridgeMetrics.totalVideoStreamMillisecondsReceived.add(durationActiveVideo.toMillis()) + + if (iceTransport.isConnected() && !dtlsTransport.isConnected) { + logger.info("Expiring an endpoint with ICE connected, but not DTLS.") + VideobridgeMetrics.endpointsDtlsFailed.inc() + } + } + + override fun debugState(mode: DebugStateMode): ObjectNode = super.debugState(mode).apply { + set("bitrate_controller", bitrateController.debugState(mode)) + set("bandwidth_probing", bandwidthProbing.getDebugState()) + put("cryptex", cryptex) + set("ice_transport", iceTransport.getDebugState()) + set("dtls_transport", dtlsTransport.getDebugState()) + set("transceiver", transceiver.debugState(mode)) + put("accept_audio", acceptAudio) + put("accept_video", acceptVideo) + put("visitor", visitor) + put("diarize", diarize) + set("message_transport", messageTransport.debugState) + if (doSsrcRewriting) { + set("audio_ssrcs", audioSsrcs.getDebugState()) + set("video_ssrcs", videoSsrcs.getDebugState()) + } + sctpTransport?.let { + set("sctp", it.getDebugState()) + } + } + + override fun expire() { + if (super.isExpired) { + return + } + super.expire() + + try { + bitrateController.expire() + updateStatsOnExpire() + transceiver.stop() + logger.cdebug { transceiver.debugState(DebugStateMode.FULL).toString() } + logger.cdebug { bitrateController.debugState(DebugStateMode.FULL).toString() } + logger.cdebug { iceTransport.getDebugState().toString() } + logger.cdebug { dtlsTransport.getDebugState().toString() } + + logger.info("Spent ${bitrateController.getTotalOversendingTime().seconds} seconds oversending") + + transceiver.teardown() + messageTransport.close() + sctpHandler?.stop() + sctpTransport?.stop() + } catch (t: Throwable) { + logger.error("Exception while expiring: ", t) + } + + bandwidthProbing.enabled = false + recurringRunnableExecutor.deRegisterRecurringRunnable(bandwidthProbing) + connectionStatsSender?.let { + recurringRunnableExecutor.deRegisterRecurringRunnable(it) + } + conference.encodingsManager.unsubscribe(this) + + dtlsTransport.stop() + iceTransport.stop() + outgoingSrtpPacketQueue.close() + + logger.info("Expired.") + } + + fun setInitialLastN(initialLastN: Int) { + if (initialReceiverConstraintsReceived) { + logger.info("Ignoring initialLastN, message transport already connected.") + } else { + logger.info("Setting initialLastN = $initialLastN") + bitrateController.lastN = initialLastN + } + } + + companion object { + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "srtp_send_queue_dropped_packets", + "Number of packets dropped out of the Endpoint SRTP send queue." + ) + + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "srtp_send_queue_exceptions", + "Number of exceptions from the Endpoint SRTP send queue." + ) + + /** Count the number of dropped packets and exceptions. */ + @JvmField + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } + + /** + * The executor which runs bandwidth probing. + * + * TODO (brian): align the recurringRunnable stuff with whatever we end up + * doing with all the other executors. + */ + private val recurringRunnableExecutor = RecurringRunnableExecutor(Endpoint::class.java.simpleName) + + /** + * How long we'll give an endpoint to either successfully establish + * an ICE connection or fail before we expire it. + */ + // TODO: make this configurable + private val epTimeout = 2.mins + + private val timelineCounter = AtomicLong() + private val timelineFraction: Long by config { + "jmt.debug.packet-timeline.log-fraction".from(JitsiConfig.newConfig) + } + + fun logTimeline() = timelineCounter.getAndIncrement() % timelineFraction == 0L + + private const val SRTP_QUEUE_ENTRY_EVENT = "Entered Endpoint SRTP sender outgoing queue" + private const val SRTP_QUEUE_EXIT_EVENT = "Exited Endpoint SRTP sender outgoing queue" + private const val SENT_OVER_ICE_TRANSPORT_EVENT = "Sent over the ICE transport" + + private val statsFilterThreshold: Int by config { + "videobridge.stats-filter-threshold".from(JitsiConfig.newConfig) + } + + /** + * This can be switched off to ease debugging. + */ + private val useRandomSendSsrcs = true + + /** + * Used for generating send SSRCs. + */ + private val random = SecureRandom() + } + + private inner class SctpCallbacks(transport: DcSctpTransport) : DcSctpBaseCallbacks(transport) { + override fun sendPacketWithStatus(packet: ByteArray): SendPacketStatus { + try { + val newBuf = ByteBufferPool.getBuffer(packet.size) + System.arraycopy(packet, 0, newBuf, 0, packet.size) + + sctpSendPcap.observe(newBuf, 0, packet.size) + dtlsTransport.sendDtlsData(newBuf, 0, packet.size) + + return SendPacketStatus.kSuccess + } catch (e: Throwable) { + logger.warn("Exception sending SCTP packet", e) + return SendPacketStatus.kError + } + } + + override fun OnMessageReceived(message: DcSctpMessage) { + try { + // We assume all data coming over SCTP will be datachannel data + val dataChannelPacket = DataChannelPacket(message) + // Post the rest of the task here because the current context is + // holding a lock inside the SctpSocket which can cause a deadlock + // if two endpoints are trying to send datachannel messages to one + // another (with stats broadcasting it can happen often) + incomingDataChannelMessagesQueue.add(PacketInfo(dataChannelPacket)) + } catch (e: Throwable) { + logger.warn("Exception processing SCTP message", e) + } + } + + override fun OnError(error: ErrorKind, message: String) { + logger.warn("SCTP error $error: $message") + } + + override fun OnAborted(error: ErrorKind, message: String) { + logger.info("SCTP aborted with error $error: $message") + } + + override fun OnConnected() { + try { + logger.info("SCTP connection is ready") + // This handles if the remote side will be opening the data channel + dataChannelStack.onDataChannelStackEvents { dataChannel -> + logger.info("Remote side opened a data channel.") + messageTransport.setDataChannel(dataChannel) + } + logger.info("Will wait for the remote side to open the data channel.") + } catch (e: Throwable) { + logger.warn("Exception processing SCTP connected event", e) + } + } + + override fun OnClosed() { + // I don't think this should happen, except during shutdown. + logger.info("SCTP connection closed") + } + } + + private inner class TransceiverEventHandlerImpl : TransceiverEventHandler { + /** + * Forward audio level events from the Transceiver to the conference. We use the same thread, because this fires + * for every packet and we want to avoid the switch. The conference audio level code must not block. + */ + override fun audioLevelReceived(sourceSsrc: Long, level: Long): Boolean = + conference.levelChanged(this@Endpoint, sourceSsrc, level) + + /** + * Forward bwe events from the Transceiver. + */ + override fun bandwidthEstimationChanged(newValue: Bandwidth) { + logger.cdebug { "Estimated bandwidth is now $newValue" } + latestBandwidth = newValue + bitrateController.bandwidthChanged(newValue.bps) + bandwidthProbing.bandwidthEstimationChanged(newValue) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusConfig.kt new file mode 100644 index 0000000..811200c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusConfig.kt @@ -0,0 +1,47 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import java.time.Duration + +class EndpointConnectionStatusConfig private constructor() { + val firstTransferTimeout: Duration by config { + "org.jitsi.videobridge.EndpointConnectionStatus.FIRST_TRANSFER_TIMEOUT" + .from(JitsiConfig.legacyConfig) + .convertFrom(Duration::ofMillis) + "videobridge.ep-connection-status.first-transfer-timeout".from(JitsiConfig.newConfig) + } + + val maxInactivityLimit: Duration by config { + "org.jitsi.videobridge.EndpointConnectionStatus.MAX_INACTIVITY_LIMIT" + .from(JitsiConfig.legacyConfig) + .convertFrom(Duration::ofMillis) + "videobridge.ep-connection-status.max-inactivity-limit".from(JitsiConfig.newConfig) + } + + val intervalMs: Long by config { + "videobridge.ep-connection-status.check-interval".from(JitsiConfig.newConfig) + .convertFrom(Duration::toMillis) + } + + companion object { + @JvmField + val config = EndpointConnectionStatusConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt new file mode 100644 index 0000000..cc01130 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitor.kt @@ -0,0 +1,169 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.jitsi.utils.NEVER +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.EndpointConnectionStatusConfig.Companion.config +import org.jitsi.videobridge.message.EndpointConnectionStatusMessage +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import java.time.Clock +import java.time.Duration +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +class EndpointConnectionStatusMonitor @JvmOverloads constructor( + private val conference: Conference, + private val executor: ScheduledExecutorService, + parentLogger: Logger, + private val clock: Clock = Clock.systemUTC() +) { + private val logger = createChildLogger(parentLogger) + + /** + * Note that we intentionally do not prune this set when an endpoint expires, because if an endpoint expires and + * is recreated, we need to send an "active" message (the other endpoints in the conference are not aware that the + * object on the bridge was expired and recreated). + * Also note that when an endpoint is moved to another bridge, it will be expired and a RelayedEndpoint with the + * same ID will be created. + */ + private val inactiveEndpointIds = mutableSetOf() + + private val taskHandle = AtomicReference>(null) + + fun start() { + if (taskHandle.compareAndSet( + null, + executor.scheduleWithFixedDelay(::run, config.intervalMs, config.intervalMs, TimeUnit.MILLISECONDS) + ) + ) { + logger.info("Starting connection status monitor") + } else { + logger.warn("Task already started, not starting again") + } + } + + fun stop() { + taskHandle.getAndSet(null)?.cancel(false) + logger.info("Stopped") + } + + private fun run() { + conference.localEndpoints.filter { !it.visitor }.forEach(::monitorEndpointActivity) + } + + private fun monitorEndpointActivity(endpoint: Endpoint) { + val now = clock.instant() + val mostRecentChannelCreatedTime = endpoint.creationTime + val lastActivity = endpoint.lastIncomingActivity + + val active: Boolean + var changed = false + if (lastActivity == NEVER) { + // Here we check if it's taking too long for the endpoint to connect + // We're doing that by checking how much time has elapsed since + // the first endpoint's channel has been created. + val timeSinceCreation = Duration.between(mostRecentChannelCreatedTime, now) + if (timeSinceCreation > config.firstTransferTimeout) { + active = false + synchronized(inactiveEndpointIds) { + val alreadyInactive = inactiveEndpointIds.contains(endpoint.id) + if (!alreadyInactive) { + logger.cdebug { + "${endpoint.id} is having trouble establishing the connection " + + "and will be marked as inactive" + } + inactiveEndpointIds += endpoint.id + changed = true + } + } + } else { + logger.cdebug { "${endpoint.id} not ready for activity checks yet" } + return + } + } else { + val noActivityTime = Duration.between(lastActivity, now) + active = noActivityTime <= config.maxInactivityLimit + synchronized(inactiveEndpointIds) { + val wasActive = !inactiveEndpointIds.contains(endpoint.id) + if (wasActive && !active) { + logger.info { "${endpoint.id} is considered disconnected. No activity for $noActivityTime" } + inactiveEndpointIds += endpoint.id + endpointsDisconnected.inc() + changed = true + } else if (!wasActive && active) { + logger.info { "${endpoint.id} has reconnected" } + inactiveEndpointIds -= endpoint.id + endpointsReconnected.inc() + changed = true + } + } + } + + if (changed) { + notifyStatusChange(endpoint.id, active, null) + } + } + + private fun notifyStatusChange(subjectEpId: String, isConnected: Boolean, receiverEpId: String?) { + val msg = EndpointConnectionStatusMessage(subjectEpId, isConnected) + + if (receiverEpId == null) { + // We broadcast the message also to the endpoint itself for + // debugging purposes, and we also send it to Relays. + conference.broadcastMessage(msg, true) + } else { + conference.getLocalEndpoint(receiverEpId)?.let { + conference.sendMessage(msg, listOf(it), false) + } + } + } + + /** + * Notify this [EndpointConnectionStatusMonitor] that an endpoint in the conference has + * connected + */ + fun endpointConnected(endpointId: String) { + synchronized(inactiveEndpointIds) { + val localEndpointIds = conference.localEndpoints.filter { !it.visitor }.map { it.id } + inactiveEndpointIds.forEach { inactiveEpId -> + // inactiveEndpointIds may contain endpoints that have already expired and/or moved to another bridge. + if (localEndpointIds.contains(inactiveEpId)) { + notifyStatusChange(inactiveEpId, isConnected = false, receiverEpId = endpointId) + } + } + } + } + + companion object { + @JvmField + val endpointsDisconnected = VideobridgeMetricsContainer.instance.registerCounter( + "endpoints_disconnected", + "Endpoints detected as temporarily inactive/disconnected due to inactivity." + ) + + @JvmField + val endpointsReconnected = VideobridgeMetricsContainer.instance.registerCounter( + "endpoints_reconnected", + "Endpoints reconnected after being detected as temporarily inactive/disconnected due to inactivity." + ) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/JvbLastN.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/JvbLastN.kt new file mode 100644 index 0000000..a65716e --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/JvbLastN.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import java.util.function.Supplier + +/** + * A JVB-wide last-n value which will be observed by all endpoints on this bridge. + * + * A value of -1 means no limit is enforced. + */ +class JvbLastN : Supplier { + private val defaultJvbLastN: Int by config("videobridge.cc.jvb-last-n".from(JitsiConfig.newConfig)) + var jvbLastN: Int = defaultJvbLastN + + override fun get(): Int = jvbLastN +} + +@JvmField +val jvbLastNSingleton: JvbLastN = JvbLastN() + +fun calculateLastN(vararg lastN: Int): Int { + val min = lastN.map { if (it == -1) Int.MAX_VALUE else it }.minOrNull() ?: Int.MAX_VALUE + return if (min == Int.MAX_VALUE) -1 else min +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt new file mode 100644 index 0000000..256bae7 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/LoudestConfig.kt @@ -0,0 +1,50 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import java.time.Duration + +class LoudestConfig private constructor() { + companion object { + @JvmStatic + val routeLoudestOnly: Boolean by config( + "videobridge.loudest.route-loudest-only".from(JitsiConfig.newConfig) + ) + + @JvmStatic + val numLoudest: Int by config( + "videobridge.loudest.num-loudest".from(JitsiConfig.newConfig) + ) + + @JvmStatic + val alwaysRouteDominant: Boolean by config( + "videobridge.loudest.always-route-dominant".from(JitsiConfig.newConfig) + ) + + @JvmStatic + val energyExpireTime: Duration by config( + "videobridge.loudest.energy-expire-time".from(JitsiConfig.newConfig) + ) + + @JvmStatic + val energyAlphaPct: Int by config( + "videobridge.loudest.energy-alpha-pct".from(JitsiConfig.newConfig) + ) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt new file mode 100644 index 0000000..485041d --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/Main.kt @@ -0,0 +1,274 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.eclipse.jetty.ee10.servlet.ServletHolder +import org.glassfish.jersey.servlet.ServletContainer +import org.ice4j.ice.harvest.AbstractUdpListener +import org.ice4j.ice.harvest.MappingCandidateHarvesters +import org.ice4j.util.Buffer +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.ConfigException +import org.jitsi.metaconfig.MetaconfigLogger +import org.jitsi.metaconfig.MetaconfigSettings +import org.jitsi.nlj.dtls.DtlsConfig +import org.jitsi.rest.JettyBundleActivatorConfig +import org.jitsi.rest.createServer +import org.jitsi.rest.enableCors +import org.jitsi.rest.isEnabled +import org.jitsi.rest.servletContextHandler +import org.jitsi.rtp.Packet +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.shutdown.ShutdownServiceImpl +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.ice.Harvesters +import org.jitsi.videobridge.metrics.Metrics +import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics +import org.jitsi.videobridge.rest.root.Application +import org.jitsi.videobridge.stats.MucPublisher +import org.jitsi.videobridge.util.ByteBufferPool +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.util.UlimitCheck +import org.jitsi.videobridge.version.JvbVersionService +import org.jitsi.videobridge.websocket.ColibriWebSocketService +import org.jitsi.videobridge.xmpp.XmppConnection +import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig +import sun.misc.Signal +import java.time.Clock +import kotlin.concurrent.thread +import kotlin.system.exitProcess +import org.jitsi.videobridge.websocket.singleton as webSocketServiceSingleton + +fun main() { + val logger = LoggerImpl("org.jitsi.videobridge.Main") + + Thread.setDefaultUncaughtExceptionHandler { thread, exception -> + logger.error("An uncaught exception occurred in thread=$thread", exception) + } + + setupMetaconfigLogger() + + setSystemPropertyDefaults() + + // Some of our dependencies bring in slf4j, which means Jetty will default to using + // slf4j as its logging backend. The version of slf4j brought in, however, is too old + // for Jetty so it throws errors. We use java.util.logging so tell Jetty to use that + // as its logging backend. + // TODO: Instead of setting this here, we should integrate it with the infra/debian scripts + // to be passed. + System.setProperty("org.eclipse.jetty.util.log.class", "org.eclipse.jetty.util.log.JavaUtilLog") + + Metrics.start() + + // Reload the Typesafe config used by ice4j, because the original was initialized before the new system + // properties were set. + JitsiConfig.reloadNewConfig() + + logger.info("Starting jitsi-videobridge version ${JvbVersionService.instance.currentVersion}") + + UlimitCheck.printUlimits() + startIce4j() + + setupBufferPools() + + // Initialize, binding on the main ICE port. + Harvesters.init() + + org.jitsi.videobridge.xmpp.Smack.initialize() + PacketQueue.setEnableStatisticsDefault(true) + + // Trigger an exception early in case the DTLS cipher suites are misconfigured + try { + DtlsConfig.config.cipherSuites + } catch (ce: ConfigException) { + logger.error("Dtls configuration error: $ce") + // According to https://freedesktop.org/software/systemd/man/systemd.exec…html#Process%20Exit%20Code + // 78 means "configuration error" + exitProcess(78) + } + + val xmppConnection = XmppConnection().apply { start() } + val shutdownService = ShutdownServiceImpl() + val videobridge = Videobridge( + xmppConnection, + shutdownService, + JvbVersionService.instance.currentVersion, + Clock.systemUTC() + ) + val videobridgeExpireThread = VideobridgeExpireThread(videobridge).apply { start() } + Metrics.metricsUpdater.addUpdateTask { + VideobridgePeriodicMetrics.update(videobridge) + } + val healthChecker = videobridge.jvbHealthChecker + val presencePublisher = MucPublisher( + TaskPools.SCHEDULED_POOL, + XmppClientConnectionConfig.config.presenceInterval, + xmppConnection + ).apply { start() } + + val publicServerConfig = JettyBundleActivatorConfig( + "org.jitsi.videobridge.rest", + "videobridge.http-servers.public" + ) + val publicHttpServer = if (publicServerConfig.isEnabled()) { + logger.info("Starting public http server") + + val websocketService = ColibriWebSocketService(publicServerConfig.isTls) + webSocketServiceSingleton().setColibriWebSocketService(websocketService) + createServer(publicServerConfig).also { + websocketService.registerServlet(it.servletContextHandler, videobridge) + it.start() + } + } else { + logger.info("Not starting public http server") + null + } + + val privateServerConfig = JettyBundleActivatorConfig( + "org.jitsi.videobridge.rest.private", + "videobridge.http-servers.private" + ) + val privateHttpServer = if (privateServerConfig.isEnabled()) { + logger.info("Starting private http server") + val restApp = Application( + videobridge, + xmppConnection, + JvbVersionService.instance.currentVersion, + healthChecker + ) + createServer(privateServerConfig).also { + it.servletContextHandler.addServlet( + ServletHolder(ServletContainer(restApp)), + "/*" + ) + it.servletContextHandler.enableCors() + it.start() + } + } else { + logger.info("Not starting private http server") + null + } + + var exitStatus = 0 + + /* Catch signals and cause them to trigger a clean shutdown. */ + listOf("TERM", "HUP", "INT").forEach { signalName -> + try { + Signal.handle(Signal(signalName)) { signal -> + exitStatus = signal.number + 128 // Matches java.lang.Terminator + logger.info("Caught signal $signal, shutting down.") + + shutdownService.beginShutdown() + } + } catch (e: IllegalArgumentException) { + /* Unknown signal on this platform, or not allowed to register this signal; that's fine. */ + logger.warn("Unable to register signal '$signalName'", e) + } + } + + // Block here until the bridge shuts down + shutdownService.waitForShutdown() + + logger.info("Bridge shutting down") + healthChecker.stop() + presencePublisher.stop() + xmppConnection.stop() + + try { + publicHttpServer?.stop() + privateHttpServer?.stop() + } catch (t: Throwable) { + logger.error("Error shutting down http servers", t) + } + videobridgeExpireThread.stop() + videobridge.stop() + stopIce4j() + Metrics.stop() + + TaskPools.SCHEDULED_POOL.shutdownNow() + TaskPools.CPU_POOL.shutdownNow() + TaskPools.IO_POOL.shutdownNow() + + exitProcess(exitStatus) +} + +private fun setupMetaconfigLogger() { + val configLogger = LoggerImpl("org.jitsi.config") + MetaconfigSettings.logger = object : MetaconfigLogger { + override fun warn(block: () -> String) = configLogger.warn(block) + override fun error(block: () -> String) = configLogger.error(block) + override fun debug(block: () -> String) = configLogger.debug(block) + } +} + +private fun setSystemPropertyDefaults() { + val defaults = getSystemPropertyDefaults() + + defaults.forEach { (key, value) -> + if (System.getProperty(key) == null) { + System.setProperty(key, value) + } + } +} + +private fun getSystemPropertyDefaults(): Map { + val defaults = mutableMapOf() + + // Make legacy ice4j properties system properties. + val cfg = JitsiConfig.SipCommunicatorProps + val ice4jPropNames = cfg.getPropertyNamesByPrefix("org.ice4j", false) + ice4jPropNames?.forEach { key -> + cfg.getString(key)?.let { value -> + defaults.put(key, value) + } + } + + return defaults +} + +private fun startIce4j() { + AbstractUdpListener.USE_PUSH_API = true + // Start the initialization of the mapping candidate harvesters. + // Asynchronous, because the AWS and STUN harvester may take a long + // time to initialize. + thread(start = true) { + MappingCandidateHarvesters.initialize() + } +} + +private fun stopIce4j() { + // Shut down harvesters. + Harvesters.close() +} + +/** Configure our libraries to use JVB's global [ByteBufferPool] */ +private fun setupBufferPools() { + org.jitsi.rtp.util.BufferPool.getArray = { ByteBufferPool.getBuffer(it) } + org.jitsi.rtp.util.BufferPool.returnArray = { ByteBufferPool.returnBuffer(it) } + org.jitsi.nlj.util.BufferPool.getBuffer = { ByteBufferPool.getBuffer(it) } + org.jitsi.nlj.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it) } + org.ice4j.util.BufferPool.getBuffer = { len -> + val b = ByteBufferPool.getBuffer(len) + Buffer(b, 0, b.size) + } + org.ice4j.util.BufferPool.returnBuffer = { ByteBufferPool.returnBuffer(it.buffer) } + org.ice4j.ice.harvest.AbstractUdpListener.BYTES_TO_LEAVE_AT_START_OF_PACKET = + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + org.ice4j.ice.harvest.AbstractUdpListener.BYTES_TO_LEAVE_AT_END_OF_PACKET = + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/RecentSpeakersList.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/RecentSpeakersList.kt new file mode 100644 index 0000000..34392e3 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/RecentSpeakersList.kt @@ -0,0 +1,71 @@ +/* + * Copyright @ 2021 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import java.util.LinkedList + +/** + * Maintain a list of recent speakers. A speaker can be [promote]d, bumping it to the top of the list. The list of the + * top [numRecentSpeakers], ordered by the time they were promoted, is exposed as [recentSpeakers]. + * + * Removal is possible via [removeAllExcept]. + */ +class RecentSpeakersList( + /** + * The number of speakers to expose via [recentSpeakers]. Internally we store all speakers to allow them to be + * removed without losing information about the order. + */ + private val numRecentSpeakers: Int +) { + /** + * Store the history of all promoted speakers. + */ + val list: MutableList = LinkedList() + + /** + * Promote [endpoint] to the top of the list. If it is already in the list it is moved to the top, otherwise it's + * added to the top. + * + * @return true iff [recentSpeakers] changed as a result of this call. + */ + fun promote(endpoint: T): Boolean { + val index = list.indexOf(endpoint) + if (index >= 0) { + list.removeAt(index) + } + list.add(0, endpoint) + return index != 0 + } + + /** + * Remove all entries from the list except the ones contained in [retain]. + * + * @return true iff [recentSpeakers] changed as a result of this call. + */ + fun removeAllExcept(retain: List): Boolean { + val prevRecentSpeakers = recentSpeakers + list.removeIf { !retain.contains(it) } + return recentSpeakers != prevRecentSpeakers + } + + /** + * The first [numRecentSpeakers] in the list, ordered by the time they were promoted. + */ + val recentSpeakers: List + get() = list.take(numRecentSpeakers) + + fun isRecentSpeaker(endpoint: T) = recentSpeakers.contains(endpoint) +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt new file mode 100644 index 0000000..d5f26a9 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcCache.kt @@ -0,0 +1,786 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.VideoType +import org.jitsi.nlj.codec.vpx.VpxUtils +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.applyTemplateIdDelta +import org.jitsi.nlj.rtp.codec.av1.getTemplateIdDelta +import org.jitsi.nlj.rtp.codec.vp8.Vp8Packet +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.utils.LRUCache +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.message.AudioSourceMapping +import org.jitsi.videobridge.message.AudioSourcesMap +import org.jitsi.videobridge.message.BridgeChannelMessage +import org.jitsi.videobridge.message.VideoSourceMapping +import org.jitsi.videobridge.message.VideoSourcesMap +import org.jitsi.videobridge.relay.AudioSourceDesc +import java.util.concurrent.ConcurrentHashMap + +/** + * Align common fields from different source types. + * Perhaps this could become a base class of those types. + */ +class SourceDesc private constructor( + val name: String, + val owner: String, + val videoType: VideoType, + val ssrc1: Long, + val ssrc2: Long +) { + constructor(s: AudioSourceDesc) : this( + s.sourceName ?: "anon", + s.owner ?: "unknown", + VideoType.DISABLED, + s.ssrc, + -1 + ) + constructor(s: MediaSourceDesc) : this(s.sourceName, s.owner, s.videoType, s.primarySSRC, getRtx(s)) + companion object { + fun getRtx(s: MediaSourceDesc): Long { + /* Ignoring any additional entries for now. */ + return if (s.rtpEncodings.isEmpty()) -1 else s.rtpEncodings[0].getSecondarySsrc(SsrcAssociationType.RTX) + } + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = "$owner:$ssrc1/$ssrc2" +} + +/** + * Some RTP header state to track. + */ +class RtpState { + var lastSequenceNumber = 0 + var lastTimestamp = 0L + var codecState: CodecState? = null + var av1PersistentState: CodecState? = null // TODO? Generalize if needed in the future + var valid = false + + fun update(packet: RtpPacket) { + lastSequenceNumber = packet.sequenceNumber + lastTimestamp = packet.timestamp + codecState = packet.getCodecState() + if (packet is Av1DDPacket) { + av1PersistentState = codecState + } + valid = true + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = if (valid) "$lastSequenceNumber/$lastTimestamp/$codecState" else "-" +} + +/** + * RTP state for received SSRCs. + */ +class ReceiveSsrc(val props: SourceDesc) { + val state = RtpState() + + /** + * True when sequence number and timestamp deltas have been calculated. + * If false, calculate them on the next relayed packet. + */ + var hasDeltas = false + + /** + * {@inheritDoc} + */ + override fun toString(): String = "$state" + if (hasDeltas) "" else " (no \u2206)" +} + +/** + * RTP state for sent SSRCs. + */ +class SendSsrc(val ssrc: Long) { + private val state = RtpState() + private var sequenceNumberDelta = 0 + private var timestampDelta = 0L + private var codecDeltas: CodecDeltas? = null + + /** + * Update RTP state and apply deltas. + */ + fun rewriteRtp(packet: RtpPacket, sending: Boolean, recv: ReceiveSsrc) { + if (sending) { + if (!recv.hasDeltas) { + /* Calculate new deltas the first time a receive ssrc is mapped to a send ssrc. */ + if (state.valid) { + if (recv.state.valid) { + sequenceNumberDelta = + RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, recv.state.lastSequenceNumber) + timestampDelta = + RtpUtils.getTimestampDiff(state.lastTimestamp, recv.state.lastTimestamp) + codecDeltas = state.codecState?.getDeltas( + recv.state.codecState + ) + } else { + val prevSequenceNumber = + RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, -1) + val prevTimestamp = + RtpUtils.applyTimestampDelta(packet.timestamp, -960) // guessing + sequenceNumberDelta = + RtpUtils.getSequenceNumberDelta(state.lastSequenceNumber, prevSequenceNumber) + timestampDelta = + RtpUtils.getTimestampDiff(state.lastTimestamp, prevTimestamp) + codecDeltas = state.codecState?.getDeltas(packet) + } + } + recv.hasDeltas = true + } + + if (packet is Av1DDPacket && state.codecState !is Av1DDCodecState && state.av1PersistentState != null) { + codecDeltas = state.av1PersistentState?.getDeltas(packet) + } + + recv.state.update(packet) + + packet.ssrc = ssrc + packet.sequenceNumber = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, sequenceNumberDelta) + packet.timestamp = RtpUtils.applyTimestampDelta(packet.timestamp, timestampDelta) + codecDeltas?.rewritePacket(packet) + + state.update(packet) + } else { + /* Don't touch send state if we're dropping the packet. */ + recv.state.update(packet) + } + } + + /** + * Fix SSRC and timestamps in an RTCP packet. + * For packets in the same direction as media flow; feedback messages handled separately. + */ + fun rewriteRtcp(packet: RtcpPacket) { + packet.senderSsrc = ssrc + if (packet is RtcpSrPacket) { + packet.senderInfo.rtpTimestamp = RtpUtils.applyTimestampDelta( + packet.senderInfo.rtpTimestamp, + timestampDelta + ) + } + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = "$ssrc{$state,\u2206=$sequenceNumberDelta/$timestampDelta/$codecDeltas}" +} + +/** + * Associates primary and secondary send SSRCs. + * Primary constructor preserves state of the existing send SSRCs. + */ +class SendSource(val props: SourceDesc, val send1: SendSsrc, val send2: SendSsrc, val mid: String? = null) { + + /** + * If false, do not send on this SSRC until a packet with start=true arrives. + */ + private var started = false + + /** + * Create object with new send SSRCs. + */ + constructor(props: SourceDesc, ssrc1: Long, ssrc2: Long) : this(props, SendSsrc(ssrc1), SendSsrc(ssrc2)) + + /** + * Demux to proper SSRC. + */ + private fun getSender(ssrc: Long) = if (ssrc == props.ssrc2) send2 else send1 + + /** + * Update RTP state and apply deltas. + * Returns true if packet should be sent. + */ + fun rewriteRtp(packet: RtpPacket, start: Boolean, recv: ReceiveSsrc): Boolean { + if (start) { + started = true + } + getSender(packet.ssrc).rewriteRtp(packet, started, recv) + return started + } + + /** + * Fix SSRC and timestamps in an RTCP packet. + * For packets in the same direction as media flow; feedback messages handled separately. + */ + fun rewriteRtcp(packet: RtcpPacket) { + getSender(packet.senderSsrc).rewriteRtcp(packet) + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = "$send1, $send2" + if (started) "" else " (not started)" +} + +/** + * Interface between an SsrcCache and its owner. + */ +interface SsrcRewriter { + /** + * Find the properties of the video source indicated by the given SSRC. Returns null if not found. + */ + fun findVideoSourceProps(ssrc: Long): MediaSourceDesc? + + /** + * Find the properties of the audio source indicated by the given SSRC. Returns null if not found. + */ + fun findAudioSourceProps(ssrc: Long): AudioSourceDesc? + + /** + * Get a unique send SSRC. + */ + fun getNextSendSsrc(): Long + + /** + * Sends a specific message to this endpoint over its bridge channel. + */ + fun sendMessage(msg: BridgeChannelMessage) +} + +/** + * Limit the number of local SSRCs used for the given media type to the number specified. + * If there are more sources in the conference than the limit, then the least recently used + * SSRCs are remapped. RTP packets have their header fields rewritten so the stream appears + * to be a continuation of an already advertised SSRC. + */ +abstract class SsrcCache( + val size: Int, + val ep: SsrcRewriter, + /** Whether to assign and stamp a mid per slot for mid-based demuxing on the client. */ + val midDemux: Boolean, + val parentLogger: Logger, + label: String +) { + + private val logger = createChildLogger(parentLogger).apply { addContext("type", label) } + + /** The prefix used for the mids assigned to slots of this cache (e.g. "a" for audio, "v" for video). */ + protected abstract val midPrefix: String + + /** + * Maps a (send-side) SSRC to the mid of the slot it belongs to. Populated when [midDemux] is enabled. Both the + * primary and RTX send SSRCs of a slot map to the same mid. Read from the sender pipeline thread (lock-free) while + * written under the [sendSources] lock, hence the concurrent map. + */ + private val sendSsrcToMid = ConcurrentHashMap() + + /** The next index to use when assigning a mid to a newly-created slot. */ + private var nextMidIndex = 0 + + /** + * All remote SSRCs that have been seen. + */ + private val receivedSsrcs = HashMap() + + /** + * The most recently forwarded remote SSRC groups. If this list is full and a new remote SSRC needs to be + * forwarded to the endpoint, the element at the front of this list will be removed and that element's + * local SSRC will be used. Note: indexed by primary SSRC. + */ + private val sendSources = LRUCache( + size, + // accessOrder + true + ) + + /** + * Whether an incoming RTP packet can automatically activate its source (i.e. acquire a send SSRC). + * If false, sources must be activated using the activate() method. + */ + protected abstract val allowCreateOnPacket: Boolean + + /** + * Number of times an SSRC has changed its mapping. (Does not count initial mappings.) + */ + private var remapCount = 0 + + /** + * Find the properties of the source indicated by the given SSRC. Returns null if not found. + */ + protected abstract fun findSourceProps(ssrc: Long): SourceDesc? + + /** + * Notify new SSRC mappings to the client. The list will contain at least one element. + */ + protected abstract fun notifyMappings(sources: List) + + /** + * Assign a group of send SSRCs to use for the specified source. + * If remapping the send SSRCs from another source, transfer RTP state from the old source. + * Returns null if no current mapping exists and allowCreate is false. + * Otherwise, returns the send source information to use. + */ + private fun getSendSource( + ssrc: Long, + props: SourceDesc, + allowCreate: Boolean, + /* Collect remapped sources into this list, if provided. */ + remappings: MutableList? = null + ): SendSource? { + /* Moves to end of LRU when found. */ + var sendSource = sendSources[ssrc] + + if (sendSource == null) { + if (!allowCreate) { + return null + } + if (sendSources.size == size) { + val eldest = sendSources.eldest() + /* Reuse the eldest slot's send SSRCs *and* its mid: the mid is bound to the slot (i.e. to the send + SSRCs), not to the source, so it stays stable across remaps. */ + sendSource = SendSource(props, eldest.value.send1, eldest.value.send2, eldest.value.mid) + logger.debug { "Remapping SSRC: ${props.ssrc1}->$sendSource. ${eldest.key}->inactive" } + /* Request new deltas on next sent packet */ + receivedSsrcs[props.ssrc1]?.hasDeltas = false + if (props.ssrc2 != -1L) { + receivedSsrcs[props.ssrc2]?.hasDeltas = false + } + ++remapCount + } else { + val ssrc1 = ep.getNextSendSsrc() + val ssrc2 = ep.getNextSendSsrc() + val mid = if (midDemux) "$midPrefix${nextMidIndex++}" else null + sendSource = SendSource(props, SendSsrc(ssrc1), SendSsrc(ssrc2), mid) + logger.debug { "Added send SSRC: ${props.ssrc1}->$sendSource" } + } + sendSources[ssrc] = sendSource + /* Bind both the primary and RTX send SSRCs to the slot's mid so the stamper can resolve either. */ + sendSource.mid?.let { mid -> + sendSsrcToMid[sendSource.send1.ssrc] = mid + sendSsrcToMid[sendSource.send2.ssrc] = mid + } + remappings?.add(sendSource) + } + + return sendSource + } + + /** + * Returns the mid assigned to the slot whose (send-side) SSRC is [ssrc], or null if [ssrc] is not a slot SSRC of + * this cache or mid demux is disabled. + */ + fun getMidBySsrc(ssrc: Long): String? = sendSsrcToMid[ssrc] + + /** + * Remove all send sources owned by [owner]. Does not signal anything to the client, as only additions need to be + * signaled. + */ + fun removeByOwner(owner: String) { + synchronized(sendSources) { + sendSources.values.removeIf { sendSource -> + sendSource.props.owner == owner + } + } + } + + /** + * Assign send SSRCs to the given sources. Any remapped SSRCs will be notified to the client. + */ + fun activate(sources: List) { + val remappings = mutableListOf() + + synchronized(sendSources) { + /* Before creating any new send sources, do a first pass that + touches the already active sources, to prevent them from being + bumped out of the LRU by earlier elements of the list. */ + sources.filter { source -> + sendSources[source.primarySSRC] == null + }.forEach { source -> + getSendSource(source.primarySSRC, SourceDesc(source), allowCreate = true, remappings) + } + + logger.debug { this.toString() } + } + + if (remappings.isNotEmpty()) { + notifyMappings(remappings) + } + } + + /** + * Send all current mappings to the endpoint. + * Can be used to resynchronize after message transport reconnects. + */ + fun sendAllMappings() { + val remappings: List + + synchronized(sendSources) { + remappings = sendSources.values.toList() + } + + if (remappings.isNotEmpty()) { + notifyMappings(remappings) + } + } + + /** + * Rewrite RTP fields for a relayed packet. + * Activates a send SSRC if necessary. + * @param packet the packet about to be sent. + * @param start whether this packet can be the first packet sent on a new SSRC mapping. + * @return whether to send this packet. + */ + fun rewriteRtp(packet: RtpPacket, start: Boolean = true): Boolean { + val remappings = mutableListOf() + var send = false + + try { + synchronized(sendSources) { + var rs = receivedSsrcs[packet.ssrc] + if (rs == null) { + val props = findSourceProps(packet.ssrc) ?: return false + rs = ReceiveSsrc(props) + receivedSsrcs[packet.ssrc] = rs + logger.debug { "Added receive SSRC: ${packet.ssrc}" } + } + + val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreateOnPacket, remappings) + if (ss != null) { + send = ss.rewriteRtp(packet, start, rs) + } + } + + if (remappings.isNotEmpty()) { + notifyMappings(remappings) + } + } catch (e: Exception) { + logger.error("Error rewriting SSRC, packet $packet", e) + send = false + } + + return send + } + + /** + * Fix SSRC and timestamps in an RTCP packet. + * For packets in the same direction as media flow; feedback messages handled separately. + */ + fun rewriteRtcp(packet: RtcpPacket): Boolean { + synchronized(sendSources) { + /* Don't activate a source on RTCP. */ + val rs = receivedSsrcs[packet.senderSsrc] ?: return false + val ss = getSendSource(rs.props.ssrc1, rs.props, allowCreate = false) ?: return false + ss.rewriteRtcp(packet) + return true + } + } + + /** + * Change the SSRC in an RTCP Feedback packet to the receive-side SSRC. + * Return the endpoint identifier of the source owner. + * If the SSRC in the packet does not refer to an active source, + * then do not modify the packet and return null. + */ + fun unmapRtcpFbSsrc(packet: RtcpFbPacket): String? { + val mediaSsrc = packet.mediaSourceSsrc + + synchronized(sendSources) { + val ss = sendSources.values.find { sendSource -> + if (mediaSsrc == sendSource.send1.ssrc) { + packet.mediaSourceSsrc = sendSource.props.ssrc1 + return@find true + } + if (mediaSsrc == sendSource.send2.ssrc) { + packet.mediaSourceSsrc = sendSource.props.ssrc2 + return@find true + } + return@find false + } + + if (ss != null) { + logger.debug { + "Received RTCP FB packet. " + + "Translated send SSRC $mediaSsrc to receive SSRC ${packet.mediaSourceSsrc}. " + + "Owner = ${ss.props.owner}." + } + return ss.props.owner + } else { + logger.debug { "Received RTCP FB packet for SSRC $mediaSsrc. Not active." } + return null + } + } + } + + /** + * Returns JSON statistics useful for debugging. + */ + fun getDebugState(): ObjectNode { + synchronized(sendSources) { + return JsonNodeFactory.instance.objectNode().apply { + put("max", this@SsrcCache.size) + put("received", receivedSsrcs.size) + put("sent", sendSources.size) + put("remappings", remapCount) + } + } + } + + /** + * {@inheritDoc} + */ + override fun toString(): String = "SSRCs: received=" + + receivedSsrcs.entries.joinToString(", ", "[", "]") { + "(${it.key}->${it.value})" + } + + " mappings=" + + sendSources.entries.joinToString(", ", "[", "]") { + "(${it.key}->${it.value})" + } +} + +/** + * SSRC Cache for audio. + */ +class AudioSsrcCache(size: Int, ep: SsrcRewriter, midDemux: Boolean, parentLogger: Logger) : + SsrcCache(size, ep, midDemux, parentLogger, MediaType.AUDIO.toString()) { + + /* Switching occurs on received packets */ + override val allowCreateOnPacket = true + + override val midPrefix = "a" + + /** + * {@inheritDoc} + */ + override fun findSourceProps(ssrc: Long): SourceDesc? { + val p = ep.findAudioSourceProps(ssrc) + return if (p?.sourceName == null || p.owner == null) { + null + } else { + SourceDesc(p) + } + } + + /** + * {@inheritDoc} + */ + override fun notifyMappings(sources: List) { + sources.map { + val props = it.props + AudioSourceMapping(props.name, props.owner, it.send1.ssrc, it.mid) + }.also { + ep.sendMessage(AudioSourcesMap(it)) + } + } +} + +/** + * SSRC Cache for video. + */ +class VideoSsrcCache(size: Int, ep: SsrcRewriter, midDemux: Boolean, parentLogger: Logger) : + SsrcCache(size, ep, midDemux, parentLogger, MediaType.VIDEO.toString()) { + + /* Switching triggered by activate() method only */ + override val allowCreateOnPacket = false + + override val midPrefix = "v" + + /** + * {@inheritDoc} + */ + override fun findSourceProps(ssrc: Long): SourceDesc? = ep.findVideoSourceProps(ssrc)?.let { SourceDesc(it) } + + /** + * {@inheritDoc} + */ + override fun notifyMappings(sources: List) { + sources.map { + val props = it.props + VideoSourceMapping(props.name, props.owner, it.send1.ssrc, it.send2.ssrc, props.videoType, it.mid) + }.also { + ep.sendMessage(VideoSourcesMap(it)) + } + } +} + +/** Codec-specific packet state. */ +interface CodecState { + fun getDeltas(otherState: CodecState?): CodecDeltas? + fun getDeltas(packet: RtpPacket): CodecDeltas? +} + +/** Codec-specific packet deltas. */ +interface CodecDeltas { + fun rewritePacket(packet: RtpPacket) +} + +private class Vp8CodecState(val lastTl0Index: Int) : CodecState { + constructor(packet: Vp8Packet) : this(packet.TL0PICIDX) + + override fun getDeltas(otherState: CodecState?): CodecDeltas? { + if (otherState !is Vp8CodecState) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, otherState.lastTl0Index) + return Vp8CodecDeltas(tl0IndexDelta) + } + + override fun getDeltas(packet: RtpPacket): CodecDeltas? { + if (packet !is Vp8Packet) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta( + lastTl0Index, + VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, -1) + ) + return Vp8CodecDeltas(tl0IndexDelta) + } + + override fun toString() = "[VP8 TL0Idx]$lastTl0Index" +} + +private class Vp8CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas { + override fun rewritePacket(packet: RtpPacket) { + if (packet !is Vp8Packet) { + return + } + packet.TL0PICIDX = VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, tl0IndexDelta) + } + + override fun toString() = "[VP8 TL0Idx]$tl0IndexDelta" +} + +private class Vp9CodecState(val lastTl0Index: Int) : CodecState { + constructor(packet: Vp9Packet) : this(packet.TL0PICIDX) + + override fun getDeltas(otherState: CodecState?): CodecDeltas? { + if (otherState !is Vp9CodecState) { + return null + } + val tl0IndexDelta = VpxUtils.getTl0PicIdxDelta(lastTl0Index, otherState.lastTl0Index) + return Vp9CodecDeltas(tl0IndexDelta) + } + + override fun getDeltas(packet: RtpPacket): CodecDeltas? { + if (packet !is Vp9Packet) { + return null + } + val tl0IndexDelta = if (packet.hasTL0PICIDX) { + VpxUtils.getTl0PicIdxDelta( + lastTl0Index, + VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, -1) + ) + } else { + 0 + } + return Vp9CodecDeltas(tl0IndexDelta) + } + + override fun toString() = "[VP9 TL0Idx]$lastTl0Index" +} + +private class Vp9CodecDeltas(val tl0IndexDelta: Int) : CodecDeltas { + override fun rewritePacket(packet: RtpPacket) { + if (packet !is Vp9Packet) { + return + } + if (packet.hasTL0PICIDX) { + packet.TL0PICIDX = VpxUtils.applyTl0PicIdxDelta(packet.TL0PICIDX, tl0IndexDelta) + } + } + + override fun toString() = "[VP9 TL0Idx]$tl0IndexDelta" +} + +private class Av1DDCodecState : CodecState { + val lastFrameNum: Int + val nextTemplateIdx: Int + constructor(lastFrameNum: Int, lastTemplateIdx: Int) { + this.lastFrameNum = lastFrameNum + this.nextTemplateIdx = lastTemplateIdx + } + + constructor(packet: Av1DDPacket) { + val descriptor = packet.descriptor + requireNotNull(descriptor) { "AV1 Packet being routed must have non-null descriptor" } + this.lastFrameNum = packet.frameNumber + this.nextTemplateIdx = descriptor.structure.templateIdOffset + descriptor.structure.templateCount + } + + override fun getDeltas(otherState: CodecState?): CodecDeltas? { + if (otherState !is Av1DDCodecState) { + return null + } + val frameNumDelta = RtpUtils.getSequenceNumberDelta(lastFrameNum, otherState.lastFrameNum) + val templateIdDelta = getTemplateIdDelta(nextTemplateIdx, otherState.nextTemplateIdx) + return Av1DDCodecDeltas(frameNumDelta, templateIdDelta) + } + + override fun getDeltas(packet: RtpPacket): CodecDeltas? { + if (packet !is Av1DDPacket) { + return null + } + val descriptor = packet.descriptor ?: return null + val frameNumDelta = RtpUtils.getSequenceNumberDelta( + lastFrameNum, + RtpUtils.applySequenceNumberDelta(packet.frameNumber, -1) + ) + val templateIdDelta = getTemplateIdDelta(nextTemplateIdx, descriptor.structure.templateIdOffset) + return Av1DDCodecDeltas(frameNumDelta, templateIdDelta) + } + + override fun toString() = "[Av1DD FrameNum]$lastFrameNum [Av1DD TemplateIdx]$nextTemplateIdx" +} + +private class Av1DDCodecDeltas(val frameNumDelta: Int, val templateIdDelta: Int) : CodecDeltas { + override fun rewritePacket(packet: RtpPacket) { + if (packet !is Av1DDPacket) { + return + } + + val descriptor = packet.descriptor + requireNotNull(descriptor) + + descriptor.frameNumber = RtpUtils.applySequenceNumberDelta(descriptor.frameNumber, frameNumDelta) + descriptor.frameDependencyTemplateId = + applyTemplateIdDelta(descriptor.frameDependencyTemplateId, templateIdDelta) + descriptor.structure.templateIdOffset = + applyTemplateIdDelta(descriptor.structure.templateIdOffset, templateIdDelta) + + packet.reencodeDdExt() + } + + override fun toString() = "[AV1DD FrameNum]$frameNumDelta [Av1DD templateId]$templateIdDelta" +} + +private fun RtpPacket.getCodecState(): CodecState? = when { + this is Vp8Packet && isRewritable() -> Vp8CodecState(this) + this is Vp9Packet -> Vp9CodecState(this) + this is Av1DDPacket -> Av1DDCodecState(this) + else -> null +} + +private fun Vp8Packet.isRewritable(): Boolean = hasTL0PICIDX diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcLimitConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcLimitConfig.kt new file mode 100644 index 0000000..edf4785 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/SsrcLimitConfig.kt @@ -0,0 +1,34 @@ +/* + * Copyright @ 2022 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class SsrcLimitConfig private constructor() { + val maxVideoSsrcs: Int by config { + "videobridge.ssrc-limit.video".from(JitsiConfig.newConfig) + } + + val maxAudioSsrcs: Int by config { + "videobridge.ssrc-limit.audio".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = SsrcLimitConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/TransportConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/TransportConfig.kt new file mode 100644 index 0000000..5997bb8 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/TransportConfig.kt @@ -0,0 +1,27 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from + +class TransportConfig private constructor() { + companion object { + @JvmStatic + val queueSize: Int by config("videobridge.transport.send.queue-size".from(JitsiConfig.newConfig)) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/VersionConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/VersionConfig.kt new file mode 100644 index 0000000..7a11d50 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/VersionConfig.kt @@ -0,0 +1,34 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig.Companion.newConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.metaconfig.optionalconfig + +class VersionConfig private constructor() { + val announceVersion: Boolean by config("videobridge.version.announce".from(newConfig)) + fun announceVersion() = announceVersion + + val release: String? by optionalconfig("videobridge.version.release".from(newConfig)) + + companion object { + @JvmField + val config = VersionConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt new file mode 100644 index 0000000..4e33e3f --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeConfig.kt @@ -0,0 +1,38 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class VideobridgeConfig private constructor() { + companion object { + val initialDrainMode: Boolean by config { + "videobridge.initial-drain-mode".from(JitsiConfig.newConfig) + } + + @JvmStatic + val redactRemoteAddresses: Boolean by config { + "videobridge.redact-remote-addresses".from(JitsiConfig.newConfig) + } + + @JvmStatic + val redactColibriHttpHeaders: Boolean by config { + "videobridge.redact-colibri-http-headers".from(JitsiConfig.newConfig) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeExpireThreadConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeExpireThreadConfig.kt new file mode 100644 index 0000000..e356dfe --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/VideobridgeExpireThreadConfig.kt @@ -0,0 +1,37 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import java.time.Duration + +class VideobridgeExpireThreadConfig private constructor() { + val inactivityTimeout: Duration by config("videobridge.entity-expiration.timeout".from(JitsiConfig.newConfig)) + + val interval: Duration by config { + "org.jitsi.videobridge.EXPIRE_CHECK_SLEEP_SEC".from(JitsiConfig.legacyConfig) + .convertFrom(Duration::ofSeconds) + "videobridge.entity-expiration.check-interval".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = VideobridgeExpireThreadConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/BandwidthProbing.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/BandwidthProbing.kt new file mode 100644 index 0000000..7d2b13c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/BandwidthProbing.kt @@ -0,0 +1,158 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtp.TransportCcEngine +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BitrateTracker +import org.jitsi.nlj.util.bytes +import org.jitsi.utils.concurrent.PeriodicRunnable +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.videobridge.cc.allocation.BitrateControllerStatusSnapshot +import org.jitsi.videobridge.cc.config.BandwidthProbingConfig.Companion.config +import java.util.function.Supplier + +class BandwidthProbing( + private val probingDataSender: ProbingDataSender, + private val statusSnapshotSupplier: Supplier +) : PeriodicRunnable(config.paddingPeriodMs), TransportCcEngine.BandwidthListener { + + /** Whether or not probing is currently enabled */ + var enabled = false + + private var lastTotalNeededBps = 0L + private var lastMaxPaddingBps = 0L + private var lastPaddingBps = 0L + private fun zeroStats() { + lastTotalNeededBps = 0 + lastMaxPaddingBps = 0 + lastPaddingBps = 0 + } + private val probingBitrate = BitrateTracker(5.secs, 100.ms) + + /** + * The number of bytes left over from one run of probing to the next. This + * avoids accumulated rounding errors causing us to under-shoot the probing + * bandwidth, and also handles the use when the number of bytes we want to + * send is less than the size of an RTP header. + */ + private var bytesLeftOver: Double = 0.0 + + private var latestBwe: Long = -1 + + var diagnosticsContext: DiagnosticContext? = null + + override fun bandwidthEstimationChanged(newValue: Bandwidth) { + latestBwe = newValue.bps + } + + override fun run() { + super.run() + if (!enabled) { + zeroStats() + return + } + + // We calculate how much to probe for based on the total target bps + // (what we're able to reach), the total ideal bps (what we want to + // be able to reach) and the total current bps (what we currently send). + val bitrateControllerStatus = statusSnapshotSupplier.get() + + // How much padding do we need? + val totalNeededBps = bitrateControllerStatus.currentIdealBps - bitrateControllerStatus.currentTargetBps + if (totalNeededBps < 1 || !bitrateControllerStatus.hasNonIdealLayer) { + // Don't need to send any probing. + bytesLeftOver = 0.0 + zeroStats() + return + } + + val latestBweCopy = latestBwe + if (bitrateControllerStatus.currentIdealBps <= latestBweCopy) { + zeroStats() + return + } + + // How much padding can we afford? + val maxPaddingBps = latestBweCopy - bitrateControllerStatus.currentTargetBps + val paddingBps = totalNeededBps.coerceAtMost(maxPaddingBps) + + lastTotalNeededBps = totalNeededBps + lastMaxPaddingBps = maxPaddingBps + lastPaddingBps = paddingBps + + var timeSeriesPoint: DiagnosticContext.TimeSeriesPoint? = null + val newBytesNeeded = (config.paddingPeriodMs * paddingBps / 1000.0 / 8.0) + val bytesNeeded = newBytesNeeded + bytesLeftOver + + if (timeSeriesLogger.isTraceEnabled) { + diagnosticsContext?.let { diagnosticsContext -> + timeSeriesPoint = diagnosticsContext + .makeTimeSeriesPoint("sent_padding") + .addField("padding_bps", paddingBps) + .addField("total_ideal_bps", bitrateControllerStatus.currentIdealBps) + .addField("total_target_bps", bitrateControllerStatus.currentTargetBps) + .addField("needed_bps", totalNeededBps) + .addField("max_padding_bps", maxPaddingBps) + .addField("bwe_bps", latestBweCopy) + .addField("bytes_needed", bytesNeeded) + .addField("prev_bytes_left_over", bytesLeftOver) + } + } + + if (bytesNeeded >= 1) { + val bytesSent = probingDataSender.sendProbing(bitrateControllerStatus.activeSsrcs, bytesNeeded.toInt()) + probingBitrate.update(bytesSent.bytes) + bytesLeftOver = (bytesNeeded - bytesSent).coerceAtLeast(0.0) + timeSeriesPoint?.addField("bytes_sent", bytesSent)?.addField("new_bytes_left_over", bytesLeftOver) + } else { + bytesLeftOver = bytesNeeded.coerceAtLeast(0.0) + } + + if (timeSeriesLogger.isTraceEnabled) { + timeSeriesLogger?.trace(timeSeriesPoint) + } + } + + fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("enabled", enabled) + put("latest_bwe", latestBwe) + put("last_total_needed_bps", lastTotalNeededBps) + put("last_max_padding_bps", lastMaxPaddingBps) + put("last_padding_bps", lastPaddingBps) + put("probing_bps", probingBitrate.getRateBps()) + } + + companion object { + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BandwidthProbing::class.java) + } + + interface ProbingDataSender { + /** + * Sends a specific number of bytes with a specific set of SSRCs. + * @param mediaSsrcs the SSRCs + * @param numBytes the number of probing bytes we want to send + * @return the number of bytes of probing data actually sent + */ + fun sendProbing(mediaSsrcs: Collection, numBytes: Int): Int + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt new file mode 100644 index 0000000..e8bccea --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettings.kt @@ -0,0 +1,184 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * Copyright @ 2021 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.util.bps +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config +import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage + +/** + * This class encapsulates all of the client-controlled settings for bandwidth allocation. + */ +data class AllocationSettings @JvmOverloads constructor( + @Deprecated("", ReplaceWith("onStageSources"), DeprecationLevel.WARNING) + val onStageEndpoints: List = emptyList(), + @Deprecated("", ReplaceWith("selectedSources"), DeprecationLevel.WARNING) + val selectedEndpoints: List = emptyList(), + val onStageSources: List = emptyList(), + val selectedSources: List = emptyList(), + val videoConstraints: Map = emptyMap(), + val lastN: Int = defaultInitialLastN, + val defaultConstraints: VideoConstraints, + /** A non-negative value is assumed as the available bandwidth in bps. A negative value is ignored. */ + val assumedBandwidthBps: Long = -1 +) { + fun toJson(): ObjectNode { + val mapper = ObjectMapper() + return JsonNodeFactory.instance.objectNode().apply { + set("on_stage_sources", mapper.valueToTree(onStageSources)) + set("selected_sources", mapper.valueToTree(selectedSources)) + set("video_constraints", mapper.valueToTree(videoConstraints.mapValues { it.value.toString() })) + put("last_n", lastN) + put("default_constraints", defaultConstraints.toString()) + put("assumed_bandwidth_bps", assumedBandwidthBps) + } + } + + override fun toString(): String = toJson().toString() + + fun getConstraints(endpointId: String) = videoConstraints.getOrDefault(endpointId, defaultConstraints) + + companion object { + val defaultInitialLastN: Int by config { + "videobridge.cc.initial-last-n".from(JitsiConfig.newConfig) + } + } +} + +/** + * Maintains an [AllocationSettings] instance and allows fields to be set individually, with an indication of whether + * the overall state changed. + */ +internal class AllocationSettingsWrapper( + parentLogger: Logger = LoggerImpl(AllocationSettingsWrapper::class.java.name) +) { + private val logger = createChildLogger(parentLogger) + + /** + * The last selected sources set signaled by the receiving endpoint. + */ + private var selectedSources = emptyList() + + internal var lastN: Int = AllocationSettings.defaultInitialLastN + + private var videoConstraints: Map = emptyMap() + + private var defaultConstraints: VideoConstraints = VideoConstraints(config.initialMaxHeightPx) + + private var assumedBandwidthBps: Long = -1 + + private var onStageSources: List = emptyList() + + private var allocationSettings = create() + + private var everSet = false + + private fun create(): AllocationSettings = AllocationSettings( + onStageSources = onStageSources, + selectedSources = selectedSources, + videoConstraints = videoConstraints, + defaultConstraints = defaultConstraints, + lastN = lastN, + assumedBandwidthBps = assumedBandwidthBps + ) + + fun get() = allocationSettings + + fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage): Boolean { + var changed = false + + message.lastN?.let { + if (lastN != it) { + lastN = it + changed = true + } + } + message.selectedSources?.let { + if (selectedSources != it) { + selectedSources = it + changed = true + } + } + message.onStageSources?.let { + if (onStageSources != it) { + onStageSources = it + changed = true + } + } + val newDefaultConstraints = message.defaultConstraints ?: if (!everSet) { + defaultDefaultConstraints + } else { + null + } + newDefaultConstraints?.let { + if (defaultConstraints != it) { + defaultConstraints = it + changed = true + } + } + message.constraints?.let { + if (this.videoConstraints != it) { + this.videoConstraints = it + changed = true + } + } + message.assumedBandwidthBps?.let { + config.assumedBandwidthLimit?.let { limit -> + val limited = it.coerceAtMost(limit.bps) + if (assumedBandwidthBps != limited) { + logger.warn("Setting assumed bandwidth ${limited.bps} (receiver asked for $it).") + this.assumedBandwidthBps = limited + changed = true + } + } ?: run { + if (it.bps >= 0.bps) { + logger.info("Ignoring assumed-bandwidth-bps, not allowed in config.") + } + } + } + + if (changed) { + allocationSettings = create() + } + everSet = true + return changed + } + + /** + * Return `true` iff the [AllocationSettings] state changed. + */ + fun setLastN(lastN: Int): Boolean { + if (this.lastN != lastN) { + this.lastN = lastN + allocationSettings = create() + return true + } + return false + } + + companion object { + val defaultDefaultConstraints = VideoConstraints(maxHeight = config.defaultMaxHeightPx) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt new file mode 100644 index 0000000..371bb0e --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocation.kt @@ -0,0 +1,110 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.RtpLayerDesc + +/** + * The result of bandwidth allocation. + */ +class BandwidthAllocation @JvmOverloads constructor( + val allocations: Set, + val oversending: Boolean = false, + val idealBps: Long = -1, + val targetBps: Long = -1, + /** Whether any of the requested sources were suspended (no layer at all was selected) due to BWE. */ + private val suspendedSources: List = emptyList() +) { + val hasSuspendedSources: Boolean = suspendedSources.isNotEmpty() + + val forwardedSources: Set = + allocations.filter { it.isForwarded() }.mapNotNull { it.mediaSource?.sourceName }.toSet() + + /** + * Whether the two allocations have the same endpoints and same layers. + */ + fun isTheSameAs(other: BandwidthAllocation) = allocations.size == other.allocations.size && + oversending == other.oversending && + allocations.all { allocation -> + other.allocations.any { otherAllocation -> + allocation.endpointId == otherAllocation.endpointId && + allocation.mediaSource?.primarySSRC == otherAllocation.mediaSource?.primarySSRC && + allocation.mediaSource?.videoType == otherAllocation.mediaSource?.videoType && + allocation.targetLayer?.index == otherAllocation.targetLayer?.index + } + } + + override fun toString(): String = "oversending=$oversending " + allocations.joinToString() + + val debugState: ObjectNode + get() = JsonNodeFactory.instance.objectNode().apply { + put("idealBps", idealBps) + put("targetBps", targetBps) + put("oversending", oversending) + put("has_suspended_sources", hasSuspendedSources) + put("suspended_sources", suspendedSources.toString()) + val allocationsNode = JsonNodeFactory.instance.objectNode().apply { + allocations.forEach { + val name = it.mediaSource?.sourceName ?: it.endpointId + set(name, it.debugState) + } + } + set("allocations", allocationsNode) + } +} + +/** + * The result of bandwidth allocation for a specific endpoint and a [MediaSourceDesc]. + */ +data class SingleAllocation( + /** The ID of the endpoint which owns the media source */ + val endpointId: String, + /** The media source */ + val mediaSource: MediaSourceDesc?, + /** + * The layer which has been selected to be forwarded. + */ + val targetLayer: RtpLayerDesc? = null, + /** + * The layer which would have been selected without bandwidth constraints. + */ + val idealLayer: RtpLayerDesc? = null +) { + constructor(endpoint: MediaSourceContainer, targetLayer: RtpLayerDesc? = null, idealLayer: RtpLayerDesc? = null) : + this( + endpoint.id, + endpoint.mediaSources.firstOrNull(), + targetLayer, + idealLayer + ) + private val targetIndex: Int + get() = targetLayer?.index ?: -1 + fun isForwarded(): Boolean = targetIndex > -1 + + override fun toString(): String = "[epId=$endpointId sourceName=${mediaSource?.sourceName} " + + "target=${targetLayer?.height}/${targetLayer?.frameRate} " + + "(${targetLayer?.indexString()}) " + + "ideal=${idealLayer?.height}/${idealLayer?.frameRate} (${idealLayer?.indexString()})]" + + val debugState: ObjectNode + get() = JsonNodeFactory.instance.objectNode().apply { + targetLayer?.debugState()?.let { set("target", it) } ?: put("target", null as String?) + idealLayer?.debugState()?.let { set("ideal", it) } ?: put("ideal", null as String?) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt new file mode 100644 index 0000000..3372504 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt @@ -0,0 +1,427 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.util.bps +import org.jitsi.utils.event.EventEmitter +import org.jitsi.utils.event.SyncEventEmitter +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.config.BitrateControllerConfig +import org.jitsi.videobridge.util.TaskPools +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import java.util.function.Supplier +import kotlin.math.abs + +internal class BandwidthAllocator( + eventHandler: EventHandler, + /** + * Provide the current list of endpoints (in no particular order). + * TODO: Simplify to avoid the weird (and slow) flow involving `endpointsSupplier` and `sortedEndpointIds`. + */ + private val endpointsSupplier: Supplier>, + /** + * Whether bandwidth allocation should be constrained to the available bandwidth (when `true`), or assume + * infinite bandwidth (when `false`). + */ + private val trustBwe: Supplier, + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext, + private val clock: Clock +) { + private val logger = createChildLogger(parentLogger) + + /** The estimated available bandwidth in bits per second. */ + private var bweBps: Long = -1 + + /** Whether this bandwidth estimator has been expired. Once expired we stop periodic re-allocation. */ + private var expired = false + + /** + * The "effective" constraints for an endpoint indicate the maximum resolution/fps that this + * [BandwidthAllocator] would allocate for this endpoint given enough bandwidth. + * + * They are the constraints signaled by the receiver, further reduced to 0 when the endpoint is "outside lastN". + * + * Effective constraints are used to signal to video senders to reduce their resolution to the minimum that + * satisfies all receivers. + * + * With the multi-stream support added, the mapping is stored on a per source name basis instead of an endpoint id. + * + * When an endpoint falls out of the last N, the constraints of all the sources of this endpoint are reduced to 0. + * + * TODO Update this description when the endpoint ID signaling is removed from the JVB. + */ + private var effectiveConstraints: EffectiveConstraintsMap = emptyMap() + private val eventEmitter: EventEmitter = SyncEventEmitter().apply { + addHandler(eventHandler) + } + + /** The allocations settings signalled by the receiver. */ + private var allocationSettings = + AllocationSettings( + defaultConstraints = VideoConstraints(BitrateControllerConfig.config.initialMaxHeightPx) + ) + + /** + * The last time [BandwidthAllocator.update] was called. + * Initialized as initialization time to prevent triggering an update immediately, because the settings might not + * have been configured yet. + */ + private var lastUpdateTime: Instant = clock.instant() + + /** The result of the bitrate control algorithm, the last time it ran. */ + var allocation = BandwidthAllocation(emptySet()) + private set + + /** The task scheduled to call [.update]. */ + private var updateTask: ScheduledFuture<*>? = null + + init { + rescheduleUpdate() + } + + /** Gets a JSON representation of the parts of this object's state that are deemed useful for debugging. */ + @get:SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + val debugState: ObjectNode + get() { + val debugState = JsonNodeFactory.instance.objectNode() + debugState.put("trustBwe", trustBwe.get()) + debugState.put("bweBps", bweBps) + debugState.set("allocation", allocation.debugState) + debugState.set("allocationSettings", allocationSettings.toJson()) + debugState.put("effectiveConstraints", effectiveConstraints.mapKeys { it.key.sourceName }.toString()) + return debugState + } + + /** Get the available bandwidth, taking into account the `trustBwe` option. */ + private val availableBandwidth: Long + get() = if (trustBwe.get()) bweBps else Long.MAX_VALUE + + /** + * Notify the [BandwidthAllocator] that the estimated available bandwidth has changed. + * @param newBandwidthBps the newly estimated bandwidth in bps + */ + fun bandwidthChanged(newBandwidthBps: Long) { + if (!bweChangeIsLargerThanThreshold(bweBps, newBandwidthBps)) { + logger.debug { + "New bwe ($newBandwidthBps) is not significantly changed from previous bwe ($bweBps), ignoring." + } + // If this is a "negligible" change in the bandwidth estimation + // wrt the last bandwidth estimation that we reacted to, then + // do not update the bandwidth allocation. The goal is to limit + // the resolution changes due to bandwidth estimation changes, + // as often resolution changes can negatively impact user + // experience, at the risk of clogging the receiver pipe. + } else { + logger.debug { "new bandwidth is $newBandwidthBps, updating" } + bweBps = newBandwidthBps + update() + } + } + + /** + * Updates the allocation settings and calculates a new bitrate [BandwidthAllocation]. + * @param allocationSettings the new allocation settings. + */ + fun update(allocationSettings: AllocationSettings) { + this.allocationSettings = allocationSettings + update() + } + + /** + * Runs the bandwidth allocation algorithm, and fires events if the result is different from the previous result. + */ + @Synchronized + fun update() { + if (expired) { + return + } + lastUpdateTime = clock.instant() + + // Order the sources by selection, followed by Endpoint's speech activity. + val sources = endpointsSupplier.get().flatMap { it.mediaSources.toList() }.toMutableList() + val sortedSources = prioritize(sources, selectedSources) + + // Extract and update the effective constraints. + val oldEffectiveConstraints = effectiveConstraints + val newEffectiveConstraints = getEffectiveConstraints(sortedSources, allocationSettings) + effectiveConstraints = newEffectiveConstraints + + logger.trace { + "Allocating: sortedSources=${sortedSources.map { it.sourceName }}, " + + "effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}" + } + + // Compute the bandwidth allocation. + val newAllocation = allocate(sortedSources) + val allocationChanged = !allocation.isTheSameAs(newAllocation) + val effectiveConstraintsChanged = effectiveConstraints != oldEffectiveConstraints + + if (timeSeriesLogger.isTraceEnabled) { + timeSeriesLogger.trace( + diagnosticContext.makeTimeSeriesPoint("allocator_update", lastUpdateTime) + .addField("target_bps", newAllocation.targetBps) + .addField("ideal_bps", newAllocation.idealBps) + .addField("bwe_bps", bweBps) + .addField("oversending", newAllocation.oversending) + .addField("allocation_changed", allocationChanged) + .addField("effective_constraints_changed", effectiveConstraintsChanged) + ) + } + + if (allocationChanged) { + eventEmitter.fireEvent { allocationChanged(newAllocation) } + } + + allocation = newAllocation + + logger.trace { + "Finished allocation: allocationChanged=$allocationChanged, " + + "effectiveConstraintsChanged=$effectiveConstraintsChanged, " + + "allocation=[$allocation]" + } + if (effectiveConstraintsChanged) { + eventEmitter.fireEvent { + effectiveVideoConstraintsChanged(oldEffectiveConstraints, effectiveConstraints) + } + } + } + + // On-stage sources are considered selected (with higher priority). + private val selectedSources: List + get() { + // On-stage sources are considered selected (with higher priority). + val selectedSources = allocationSettings.onStageSources.toMutableList() + allocationSettings.selectedSources.forEach { + if (!selectedSources.contains(it)) { + selectedSources.add(it) + } + } + return selectedSources + } + + /** + * Implements the bandwidth allocation algorithm for the given ordered list of media sources. + * + * The new version which works with multiple streams per endpoint. + * + * @param conferenceMediaSources the list of endpoint media sources in order of priority to allocate for. + * @return the new [BandwidthAllocation]. + */ + @Synchronized + private fun allocate(conferenceMediaSources: List): BandwidthAllocation { + val sourceBitrateAllocations = createAllocations(conferenceMediaSources) + if (sourceBitrateAllocations.isEmpty()) { + return BandwidthAllocation(emptySet()) + } + val initialBandwidth = if (allocationSettings.assumedBandwidthBps >= 0) { + logger.warn("Allocating with assumed bandwidth ${allocationSettings.assumedBandwidthBps.bps}.") + allocationSettings.assumedBandwidthBps + } else { + availableBandwidth + } + var remainingBandwidth = initialBandwidth + // Only one source is allowed to oversend: the first on-stage source which actually has an oversend layer. + // We can not use the index in the loop below to identify it, because sources with disabled constraints are + // skipped; and we require an oversend layer so that when there is more than one on-stage source (e.g. a + // camera and a screenshare) the slot is not wasted on one which can not use it. + val oversendingIndex = sourceBitrateAllocations.indexOfFirst { + !it.constraints.isDisabled() && it.isOnStage() && it.layers.oversendIndex >= 0 + } + var oldRemainingBandwidth: Long = -1 + var oversending = false + while (oldRemainingBandwidth != remainingBandwidth) { + oldRemainingBandwidth = remainingBandwidth + for (i in sourceBitrateAllocations.indices) { + val sourceBitrateAllocation = sourceBitrateAllocations[i] + if (sourceBitrateAllocation.constraints.isDisabled()) { + continue + } + + // In stage view improve greedily until preferred, in tile view go step-by-step. + remainingBandwidth -= sourceBitrateAllocation.improve(remainingBandwidth, i == oversendingIndex) + if (remainingBandwidth < 0) { + oversending = true + } + + // In stage view, do not allocate bandwidth for thumbnails until the on-stage reaches "preferred". + // This prevents enabling thumbnail only to disable them when bwe slightly increases allowing on-stage + // to take more. + if (sourceBitrateAllocation.isOnStage() && !sourceBitrateAllocation.hasReachedPreferred()) { + break + } + } + } + + // The sources which are in lastN, and are sending video, but were suspended due to bwe. + val suspendedSources = sourceBitrateAllocations.filter { it.isSuspended } + val suspendedIds = suspendedSources.map { it.mediaSource.sourceName }.toList() + if (suspendedIds.isNotEmpty()) { + logger.info { + // The bandwidth we started with is not the BWE if the receiver signaled an assumed bandwidth, so log + // it separately when the two differ. + val budget = if (initialBandwidth != availableBandwidth) "budget=$initialBandwidth bps, " else "" + val maxOversend = BitrateControllerConfig.config.maxOversendBitrate + val sources = suspendedSources.mapIndexed { i, source -> + // Only summarize the first few sources in full. They are in order of priority, so these are the + // ones whose suspension is most likely to be of interest. Name the rest without any detail, to + // bound the size of the message when many sources are suspended at once. + if (i < MAX_SOURCES_SUMMARIZED) source.suspendedSummary() else source.mediaSource.sourceName + } + "Sources suspended due to insufficient bandwidth (bwe=$availableBandwidth bps, $budget" + + "remaining=$remainingBandwidth bps, maxOversend=$maxOversend): " + sources.joinToString() + } + } + val allocations = mutableSetOf() + var targetBps: Long = 0 + var idealBps: Long = 0 + for (sourceBitrateAllocation: SingleSourceAllocation in sourceBitrateAllocations) { + allocations.add(sourceBitrateAllocation.result) + targetBps += sourceBitrateAllocation.targetBitrate + idealBps += sourceBitrateAllocation.idealBitrate + } + return BandwidthAllocation(allocations, oversending, idealBps, targetBps, suspendedIds) + } + + /** + * Query whether the allocator has non-zero effective constraints for the given endpoint or source. + */ + internal fun hasNonZeroEffectiveConstraints(source: MediaSourceDesc): Boolean { + val constraints = effectiveConstraints[source] ?: return false + return !constraints.isDisabled() + } + + @Synchronized + private fun createAllocations(conferenceMediaSources: List): List = + conferenceMediaSources.map { source -> + SingleSourceAllocation( + source.owner, + source, + // Note that we use the effective constraints and not the receiver's constraints + // directly. This means we never even try to allocate bitrate to sources "outside + // lastN". For example, if LastN=1 and the first endpoint sends a non-scalable + // stream with bitrate higher that the available bandwidth, we will forward no + // video at all instead of going to the second endpoint in the list. + // I think this is not desired behavior. However, it is required for the "effective + // constraints" to work as designed. + effectiveConstraints[source]!!, + allocationSettings.onStageSources.contains(source.sourceName), + diagnosticContext, + clock, + logger + ) + }.toList() + + /** + * Expire this bandwidth allocator. + */ + fun expire() { + expired = true + updateTask?.cancel(false) + } + + /** + * Submits a call to `update` in a CPU thread if bandwidth allocation has not been performed recently. + * + * Also, re-schedule the next update in at most `maxTimeBetweenCalculations`. This should only be run + * in the constructor or in the scheduler thread, otherwise it will schedule multiple tasks. + */ + private fun rescheduleUpdate() { + if (expired) { + return + } + val timeSinceLastUpdate = Duration.between(lastUpdateTime, clock.instant()) + val period = BitrateControllerConfig.config.maxTimeBetweenCalculations + val delayMs = if (timeSinceLastUpdate > period) { + logger.debug("Running periodic re-allocation.") + TaskPools.CPU_POOL.execute { this.update() } + period.toMillis() + } else { + period.minus(timeSinceLastUpdate).toMillis() + } + + // Add 5ms to avoid having to re-schedule right away. This increases the average period at which we + // re-allocate by an insignificant amount. + updateTask = TaskPools.SCHEDULED_POOL.schedule( + { rescheduleUpdate() }, + delayMs + 5, + TimeUnit.MILLISECONDS + ) + } + + companion object { + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BandwidthAllocator::class.java) + + /** The number of suspended sources for which we log the full details of the failed allocation. */ + private const val MAX_SOURCES_SUMMARIZED = 3 + } + + interface EventHandler { + fun allocationChanged(allocation: BandwidthAllocation) + fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap + ) + } +} + +/** + * Returns a boolean that indicates whether the current bandwidth estimation (in bps) has changed above the + * configured threshold with respect to the previous bandwidth estimation. + * + * @param previousBwe the previous bandwidth estimation (in bps). + * @param currentBwe the current bandwidth estimation (in bps). + * @return true if the bandwidth has changed above the configured threshold, * false otherwise. + */ +private fun bweChangeIsLargerThanThreshold(previousBwe: Long, currentBwe: Long): Boolean { + if (previousBwe == currentBwe) { // Even if we're "changing" -1 to -1 + return false + } + if (previousBwe == -1L || currentBwe == -1L) { + return true + } + + // We supress re-allocation when BWE has changed less than 15% (by default) of its previous value in order to + // prevent excessive changes during ramp-up. + // When BWE increases it should eventually increase past the threshold because of probing. + // When BWE decreases it is probably above the threshold because of AIMD. It's not clear to me whether we need + // the threshold in this case. + // In any case, there are other triggers for re-allocation, so any suppression we do here will only last up to + // a few seconds. + val deltaBwe = abs(currentBwe - previousBwe) + return deltaBwe > previousBwe * BitrateControllerConfig.config.bweChangeThreshold + + // If, on the other hand, the bwe has decreased, we require at least a 15% drop in order to update the bitrate + // allocation. This is an ugly hack to prevent too many resolution/UI changes in case the bridge produces too + // low bandwidth estimate, at the risk of clogging the receiver's pipe. + // TODO: do we still need this? Do we ever ever see BWE drop by <%15? +} + +typealias EffectiveConstraintsMap = Map diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt new file mode 100644 index 0000000..b2c3260 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt @@ -0,0 +1,344 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.format.PayloadTypeEncoding +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.utils.event.SyncEventEmitter +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config +import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage +import org.jitsi.videobridge.util.BooleanStateTimeTracker +import java.time.Clock +import java.time.Duration +import java.util.function.Supplier + +/** + * [BitrateController] is responsible for controlling the send bitrate to an `Endpoint`. This includes two tasks: + * 1. Decide how to allocate the available bandwidth between the available streams. + * 2. Implement the allocation via a packet handling interface. + * + * Historically both were implemented in a single class, but they are now split between [BandwidthAllocator] and + * [PacketHandler]. This class was introduced as a lightweight shim in order to preserve the previous API. + * + */ +class BitrateController @JvmOverloads constructor( + eventHandler: EventHandler, + endpointsSupplier: Supplier>, + private val diagnosticContext: DiagnosticContext, + parentLogger: Logger, + private val clock: Clock = Clock.systemUTC() +) { + val eventEmitter = SyncEventEmitter() + + private val logger = createChildLogger(parentLogger) + + private val bitrateAllocatorEventHandler = BitrateAllocatorEventHandler() + + /** + * Keep track of the "forwarded" sources, i.e. the media sources for which we are forwarding *some* layer. + */ + var forwardedSources: Set = emptySet() + private set + + /** + * Keep track of how much time we spend knowingly oversending (due to enableOnstageVideoSuspend being false) + */ + val oversendingTimeTracker = BooleanStateTimeTracker() + + val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BitrateController::class.java).let { + if (it.isTraceEnabled) it else null + } + + /** + * NOTE(george): this flag acts as an approximation for determining whether or not adaptivity/probing is + * supported. Eventually we need to scrap this and implement something cleaner, i.e. disable adaptivity if the + * endpoint hasn't signaled `goog-remb` nor `transport-cc`. + * + * Unfortunately the channel iq from jicofo lists `goog-remb` and `transport-cc` support, even tho the jingle from + * firefox doesn't (which is the main use case for wanting to disable adaptivity). + */ + private var supportsRtx = false + + private val packetHandler: PacketHandler = PacketHandler(clock, parentLogger, diagnosticContext, eventEmitter) + private val bandwidthAllocator: BandwidthAllocator = + BandwidthAllocator( + bitrateAllocatorEventHandler, + endpointsSupplier, + Supplier { trustBwe }, + parentLogger, + diagnosticContext, + clock + ) + fun hasSuspendedSources() = bandwidthAllocator.allocation.hasSuspendedSources + + private val allocationSettingsWrapper = AllocationSettingsWrapper(parentLogger) + val allocationSettings + get() = allocationSettingsWrapper.get() + + init { + eventEmitter.addHandler(eventHandler) + } + + private var bweSet = false + + /** + * Ignore the bandwidth estimations for some initial time because the REMBs don't ramp up fast enough. + * This shouldn't be needed for other bandwidth estimation algorithms. + */ + private val trustBwe: Boolean + get() = config.trustBwe && supportsRtx && bweSet && + packetHandler.timeSinceFirstMedia() >= config.initialIgnoreBwePeriod + + // Proxy to the allocator + fun endpointOrderingChanged() = bandwidthAllocator.update() + var lastN: Int + get() = allocationSettingsWrapper.lastN + set(value) { + if (allocationSettingsWrapper.setLastN(value)) { + bandwidthAllocator.update(allocationSettingsWrapper.get()) + } + } + + fun expire() = bandwidthAllocator.expire() + + /** Return the number of sources currently being forwarded. */ + fun numForwardedSources(): Int = forwardedSources.size + fun getTotalOversendingTime(): Duration = oversendingTimeTracker.totalTimeOn() + fun isOversending() = oversendingTimeTracker.state + fun bandwidthChanged(newBandwidthBps: Long) { + bweSet = true + timeSeriesLogger?.logBweChange(newBandwidthBps) + bandwidthAllocator.bandwidthChanged(newBandwidthBps) + } + + // Proxy to the packet handler + fun accept(packetInfo: PacketInfo): Boolean { + if (packetInfo.layeringChanged) { + // This needs to be done synchronously, so it's complete before the accept, below. + logger.debug { + "Layering information changed for packet from ${packetInfo.endpointId}, updating bandwidth allocation" + } + bandwidthAllocator.update() + } + return packetHandler.accept(packetInfo) + } + fun accept(rtcpSrPacket: RtcpSrPacket): Boolean = packetHandler.accept(rtcpSrPacket) + fun transformRtcp(rtcpSrPacket: RtcpSrPacket): Boolean = packetHandler.transformRtcp(rtcpSrPacket) + fun transformRtp(packetInfo: PacketInfo): Boolean = packetHandler.transformRtp(packetInfo) + + fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("bitrate_allocator", bandwidthAllocator.debugState) + set("packet_handler", packetHandler.debugState(mode)) + put("forwarded_sources", forwardedSources.toString()) + put("oversending", oversendingTimeTracker.state) + put("total_oversending_time_secs", oversendingTimeTracker.totalTimeOn().seconds) + put("supports_rtx", supportsRtx) + put("trust_bwe", trustBwe) + } + + fun addPayloadType(payloadType: PayloadType) { + if (payloadType.encoding == PayloadTypeEncoding.RTX) { + supportsRtx = true + } + } + + fun setBandwidthAllocationSettings(message: ReceiverVideoConstraintsMessage) { + if (allocationSettingsWrapper.setBandwidthAllocationSettings(message)) { + bandwidthAllocator.update(allocationSettingsWrapper.get()) + } + } + + /** + * Query whether this source is on stage or selected, as of the most recent + * video constraints + */ + fun isOnStageOrSelected(source: MediaSourceDesc) = allocationSettings.onStageSources.contains(source.sourceName) || + allocationSettings.selectedSources.contains(source.sourceName) + + /** + * Query whether this allocator has non-zero effective constraints for a given source + */ + fun hasNonZeroEffectiveConstraints(source: MediaSourceDesc) = + bandwidthAllocator.hasNonZeroEffectiveConstraints(source) + + /** + * Get the target and ideal bitrate of the current [BandwidthAllocation], as well as the list of SSRCs being + * forwarded, for use in probing. + * + * Note that the ideal layers are calculated with the allocation, and inactive layers are not considered. So when a + * higher layer becomes active, it will not be accounted for until until the allocation updates. Conversely, if the + * ideal layer becomes inactive, it will contribute 0 bps to the total ideal bitrate until the allocation updates + * and a lower layer is selected as idea. + */ + fun getStatusSnapshot(): BitrateControllerStatusSnapshot { + var totalTargetBitrate = 0.bps + var totalIdealBitrate = 0.bps + val activeSsrcs = mutableSetOf() + var hasNonIdealLayer = false + + val nowMs = clock.instant().toEpochMilli() + val allocation = bandwidthAllocator.allocation + allocation.allocations.forEach { singleAllocation -> + val allocationTargetBitrate: Bandwidth? = singleAllocation.targetLayer?.getBitrate(nowMs) + + allocationTargetBitrate?.let { + totalTargetBitrate += it + singleAllocation.mediaSource?.primarySSRC?.let { primarySsrc -> activeSsrcs.add(primarySsrc) } + } + + val allocationIdealBitrate: Bandwidth? = if (config.useVlaTargetBitrate) { + singleAllocation.idealLayer?.targetBitrate ?: singleAllocation.idealLayer?.getBitrate(nowMs) + } else { + singleAllocation.idealLayer?.getBitrate(nowMs) + } + + allocationIdealBitrate?.let { + totalIdealBitrate += it + } + + if (singleAllocation.idealLayer != null && singleAllocation.idealLayer != singleAllocation.targetLayer) { + hasNonIdealLayer = true + } + } + + activeSsrcs.removeIf { it < 0 } + + return BitrateControllerStatusSnapshot( + currentTargetBps = totalTargetBitrate.bps, + currentIdealBps = totalIdealBitrate.bps, + activeSsrcs = activeSsrcs, + hasNonIdealLayer = hasNonIdealLayer + ) + } + + private fun TimeSeriesLogger.logBweChange(newBweBps: Long) { + trace(diagnosticContext.makeTimeSeriesPoint("new_bwe").addField("bwe_bps", newBweBps)) + } + + private fun TimeSeriesLogger.logAllocationChange(allocation: BandwidthAllocation) { + val nowMs = clock.millis() + + var totalTargetBps = 0.0 + var totalIdealBps = 0.0 + var totalTargetSmoothedBps = 0.0 + var totalIdealSmoothedBps = 0.0 + var totalTargetVlaBps = 0.0 + var totalIdealVlaBps = 0.0 + + allocation.allocations.forEach { + it.targetLayer?.getBitrate(nowMs)?.let { bitrate -> totalTargetBps += bitrate.bps } + it.idealLayer?.getBitrate(nowMs)?.let { bitrate -> totalIdealBps += bitrate.bps } + it.targetLayer?.getSmoothedBitrate(nowMs)?.let { bitrate -> totalTargetSmoothedBps += bitrate.bps } + it.idealLayer?.getSmoothedBitrate(nowMs)?.let { bitrate -> totalIdealSmoothedBps += bitrate.bps } + it.targetLayer?.targetBitrate?.let { bitrate -> totalTargetVlaBps += bitrate.bps } + it.idealLayer?.targetBitrate?.let { bitrate -> totalIdealVlaBps += bitrate.bps } + trace( + diagnosticContext + .makeTimeSeriesPoint("allocation_for_source", nowMs) + .addField("remote_endpoint_id", it.endpointId) + .addField("target_idx", it.targetLayer?.index ?: -1) + .addField("ideal_idx", it.idealLayer?.index ?: -1) + .addField("target_bps_measured", it.targetLayer?.getBitrate(nowMs)?.bps ?: -1) + .addField("target_bps_smoothed", it.targetLayer?.getSmoothedBitrate(nowMs)?.bps ?: -1) + .addField("target_bps_vla", it.targetLayer?.targetBitrate?.bps ?: -1) + .addField("ideal_bps_measured", it.idealLayer?.getBitrate(nowMs)?.bps ?: -1) + .addField("ideal_bps_smoothed", it.idealLayer?.getSmoothedBitrate(nowMs)?.bps ?: -1) + .addField("ideal_bps_vla", it.idealLayer?.targetBitrate?.bps ?: -1) + ) + } + + trace( + diagnosticContext + .makeTimeSeriesPoint("allocation", nowMs) + .addField("total_target_measured_bps", totalTargetBps) + .addField("total_ideal_measured_bps", totalIdealBps) + .addField("total_target_smoothed_bps", totalTargetSmoothedBps) + .addField("total_ideal_smoothed_bps", totalIdealSmoothedBps) + .addField("total_target_vla_bps", totalTargetVlaBps) + .addField("total_ideal_vla_bps", totalIdealVlaBps) + ) + } + + interface EventHandler { + fun forwardedSourcesChanged(forwardedSources: Set) + fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap, + ) + fun keyframeNeeded(endpointId: String?, ssrc: Long) + + /** + * This is meant to be internal to BitrateAllocator, but is exposed here temporarily for the purposes of testing. + */ + fun allocationChanged(allocation: BandwidthAllocation) { } + } + + private inner class BitrateAllocatorEventHandler : BandwidthAllocator.EventHandler { + override fun allocationChanged(allocation: BandwidthAllocation) { + timeSeriesLogger?.logAllocationChange(allocation) + // Actually implement the allocation (configure the packet filter to forward the chosen target layers). + packetHandler.allocationChanged(allocation) + + val newForwardedSources = allocation.forwardedSources + if (forwardedSources != newForwardedSources) { + forwardedSources = newForwardedSources + eventEmitter.fireEvent { forwardedSourcesChanged(newForwardedSources) } + } + + oversendingTimeTracker.setState(allocation.oversending) + + // TODO: this is for testing only. Should we change the tests to work with [BitrateAllocator] directly? + eventEmitter.fireEvent { allocationChanged(allocation) } + } + + override fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap + ) { + // Forward to the outer EventHandler. + eventEmitter.fireEvent { + effectiveVideoConstraintsChanged(oldEffectiveConstraints, newEffectiveConstraints) + } + } + } +} + +/** + * Abstracts a media source for the purposes of [BandwidthAllocator]. + */ +interface MediaSourceContainer { + val id: String + val mediaSources: Array +} + +data class BitrateControllerStatusSnapshot( + val currentTargetBps: Long = -1L, + val currentIdealBps: Long = -1L, + val activeSsrcs: Collection = emptyList(), + val hasNonIdealLayer: Boolean +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Layers.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Layers.kt new file mode 100644 index 0000000..e84cb34 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Layers.kt @@ -0,0 +1,77 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import org.jitsi.nlj.RtpLayerDesc + +/** + * Saves the bitrate of a specific [RtpLayerDesc] at a specific point in time. + */ +data class LayerSnapshot( + val layer: RtpLayerDesc, + /** + * The bitrate (in bps) to use for allocation: [vlaTargetBitrate] when `use-vla-target-bitrate` is enabled and the + * sender signals one; otherwise the smoothed measured bitrate for screen sharing (bursty) sources, and + * [measuredBitrate] for everything else. + */ + val bitrate: Long, + /** + * The bitrate (in bps) measured for the layer (and its dependencies) over the short window, i.e. the one which + * matches the bandwidth estimator's. Note that this is not necessarily the bitrate used for allocation. + */ + val measuredBitrate: Long = bitrate, + /** + * The target bitrate (in bps) signaled by the sender in the VLA RTP header extension, or `null` if the sender + * doesn't signal VLA. + */ + val vlaTargetBitrate: Long? = null +) { + /** + * A short description of the layer and its bitrate, for logging. The bitrate which was not used for allocation is + * only mentioned when it differs from the one which was, so in the default configuration (measured bitrates, no + * VLA signaled by the sender) this is just the measured bitrate. + */ + fun summary(): String = buildString { + append("${layer.indexString()}/${layer.height}p/${layer.frameRate}fps=$bitrate") + val other = listOfNotNull( + if (measuredBitrate != bitrate) "measured=$measuredBitrate" else null, + vlaTargetBitrate?.takeIf { it != bitrate }?.let { "vla=$it" } + ) + if (other.isNotEmpty()) append(other.joinToString(prefix = "(", postfix = ")")) + } +} + +/** + * An immutable representation of the layers to be considered when allocating bandwidth for an endpoint. The order is + * ascending by preference (and not necessarily bitrate). + */ +data class Layers( + val layers: List, + /** The index of the "preferred" layer, i.e. the layer up to which we allocate eagerly. */ + val preferredIndex: Int, + /** + * The index of the layer which will be selected if oversending is enabled. If set to -1, oversending is disabled. + */ + val oversendIndex: Int +) : List by layers { + val preferredLayer = layers.getOrNull(preferredIndex) + val oversendLayer = layers.getOrNull(oversendIndex) + val idealLayer = layers.lastOrNull() + + companion object { + val noLayers = Layers(emptyList(), -1, -1) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt new file mode 100644 index 0000000..89917d7 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/PacketHandler.kt @@ -0,0 +1,203 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketInfo.Companion.enablePayloadVerification +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.utils.event.EventEmitter +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.AdaptiveSourceProjection +import org.jitsi.videobridge.cc.RewriteException +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +/** + * Handles packets for a [BitrateController], implementing a specific [BandwidthAllocation] provided via + * [allocationChanged]. Serves as a bridge between [BitrateController] (which decides which layers are to be forwarded) + * and [AdaptiveSourceProjection] which processes packets and maintains the state of a forwarded stream. + * + * Defines "accept" and "transform" functions for video RTP packets and RTCP Sender Reports. + */ +internal class PacketHandler( + private val clock: Clock, + parentLogger: Logger, + private val diagnosticContext: DiagnosticContext, + private val eventEmitter: EventEmitter +) { + private val logger = createChildLogger(parentLogger) + + /** + * The time (in ms) when this instance first transformed any media. This allows to ignore the BWE during the early + * stages of the call. + * + * NOTE This is only meant to be as a temporary hack and ideally should be fixed. + */ + private var firstMedia: Instant? = null + + private val numDroppedPacketsUnknownSsrc = AtomicInteger(0) + + /** + * The [AdaptiveSourceProjection]s that this instance is managing, keyed + * by the SSRCs of the associated [MediaSourceDesc]. + */ + private val adaptiveSourceProjectionMap: MutableMap = ConcurrentHashMap() + + /** + * @return true if the packet was transformed successfully, false otherwise. + */ + fun transformRtp( + /** Contains the video RTP packet to be transformed */ + packetInfo: PacketInfo + ): Boolean { + val videoPacket = packetInfo.packetAs() + + if (firstMedia == null) { + firstMedia = clock.instant() + } + + val adaptiveSourceProjection = adaptiveSourceProjectionMap[videoPacket.ssrc] ?: return false + return try { + adaptiveSourceProjection.rewriteRtp(packetInfo) + + // The rewriteRtp operation must not modify the VP8 payload. + if (enablePayloadVerification) { + val expected = packetInfo.payloadVerification + val actual = videoPacket.payloadVerification + if ("" != expected && expected != actual) { + logger.warn("Payload unexpectedly modified! Expected: $expected, actual: $actual") + } + } + true + } catch (e: RewriteException) { + logger.warn("Failed to rewrite a packet.", e) + false + } + } + + /** + * @return true if a packet should be accepted according to the current configuration. + */ + fun accept(packetInfo: PacketInfo): Boolean { + val videoPacket = packetInfo.packetAs() + val adaptiveSourceProjection = adaptiveSourceProjectionMap[videoPacket.ssrc] + if (adaptiveSourceProjection == null) { + logger.debug { "Dropping an RTP packet for an unknown SSRC: ${videoPacket.ssrc}" } + numDroppedPacketsUnknownSsrc.incrementAndGet() + return false + } + return adaptiveSourceProjection.accept(packetInfo) + } + + /** + * @return true if [rtcpSrPacket] should be accepted, false otherwise. + * + * Filters out packets that match one of the streams that this instance manages, but don't match the target SSRC. + * Allows packets for streams not managed by this instance. + */ + fun accept(rtcpSrPacket: RtcpSrPacket): Boolean { + val adaptiveSourceProjection = adaptiveSourceProjectionMap[rtcpSrPacket.senderSsrc] + + // This is probably for an audio stream. In any case, if it's for a stream which we are not forwarding it + // will be stripped off at a later stage (in RtcpSrUpdater). + return adaptiveSourceProjection == null || + // We only accept SRs for the SSRC that we're forwarding with. + adaptiveSourceProjection.targetSsrc == rtcpSrPacket.senderSsrc + } + + /** + * @return true if the packet was transformed successfully, false otherwise. + */ + fun transformRtcp(rtcpSrPacket: RtcpSrPacket): Boolean = + adaptiveSourceProjectionMap[rtcpSrPacket.senderSsrc]?.rewriteRtcp(rtcpSrPacket) ?: false + + /** + * Utility method that looks-up or creates the adaptive source projection of + * a source. + */ + private fun lookupOrCreateAdaptiveSourceProjection(singleAllocation: SingleAllocation): AdaptiveSourceProjection? { + val source = singleAllocation.mediaSource + val endpointID = singleAllocation.endpointId + if (source == null) { + return null + } + synchronized(adaptiveSourceProjectionMap) { + adaptiveSourceProjectionMap[source.primarySSRC]?.let { return it } + + if (source.rtpEncodings.isEmpty()) { + return null + } + + val adaptiveSourceProjection = AdaptiveSourceProjection( + diagnosticContext, + source, + { + eventEmitter.fireEvent { keyframeNeeded(endpointID, source.primarySSRC) } + }, + logger + ) + logger.debug { "new source projection for $source" } + + // Route all encodings to the specified bitrate controller. + source.rtpEncodings.forEach { + adaptiveSourceProjectionMap[it.primarySSRC] = adaptiveSourceProjection + } + return adaptiveSourceProjection + } + } + + fun timeSinceFirstMedia(): Duration = firstMedia?.let { Duration.between(it, clock.instant()) } ?: Duration.ZERO + + fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("num_dropped_packets_unknown_ssrc", numDroppedPacketsUnknownSsrc.toInt()) + set("adaptive_source_projection_map", adaptiveSourceProjectionMap.debugState(mode)) + } + + private fun Map.debugState(mode: DebugStateMode) = + JsonNodeFactory.instance.objectNode().also { + forEach { (ssrc, adaptiveSourceProjection) -> + it.set(ssrc.toString(), adaptiveSourceProjection.getDebugState(mode)) + } + } + + /** + * Signals to this instance that the allocation chosen by the `BitrateAllocator` has changed. + */ + fun allocationChanged(allocation: BandwidthAllocation) { + if (allocation.allocations.isEmpty()) { + adaptiveSourceProjectionMap.values.forEach { it.setTargetIndex(RtpLayerDesc.SUSPENDED_INDEX) } + } else { + allocation.allocations.forEach { + val sourceTargetIdx = it.targetLayer?.index ?: -1 + + // Review this. + val adaptiveSourceProjection = lookupOrCreateAdaptiveSourceProjection(it) + adaptiveSourceProjection?.setTargetIndex(sourceTargetIdx) + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt new file mode 100644 index 0000000..c6fd9c0 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/Prioritize.kt @@ -0,0 +1,99 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.videobridge.SsrcLimitConfig +import org.jitsi.videobridge.calculateLastN +import org.jitsi.videobridge.jvbLastNSingleton +import org.jitsi.videobridge.load_management.ConferenceSizeLastNLimits.Companion.singleton as conferenceSizeLimits + +fun prioritize( + /** + * All available sources ordered by their endpoint's speech activity and video availability. Note that when an + * endpoint has multiple sources and one of them is disabled it will still sort high because the endpoint has + * video availability. + */ + conferenceSources: MutableList, + selectedSourceNames: List = emptyList() +): List { + val enabledSelectedSources = mutableListOf() + val enabledNonSelectedSources = mutableListOf() + val disabledSources = mutableListOf() + + // conferenceSources can be large, while selectedSourceNames is usually small, so do a single pass over + // conferenceSources. + conferenceSources.forEach { source -> + if (source.videoType.isEnabled()) { + if (selectedSourceNames.contains(source.sourceName)) { + enabledSelectedSources.add(source) + } else { + enabledNonSelectedSources.add(source) + } + } else { + disabledSources.add(source) + } + } + // The enabled selected sources are sorted according to the order in which they are selected and prioritized + // over non-selected. + enabledSelectedSources.sortBy { selectedSourceNames.indexOf(it.sourceName) } + enabledSelectedSources.addAll(enabledNonSelectedSources) + // All disabled sources are sorted last, regardless of whether they are selected. + enabledSelectedSources.addAll(disabledSources) + + return enabledSelectedSources +} + +/** + * Return the "effective" constraints for the given media sources, i.e. the constraints adjusted for LastN. + */ +fun getEffectiveConstraints( + sources: List, + allocationSettings: AllocationSettings +): EffectiveConstraintsMap { + // FIXME figure out before merge - is using source count instead of endpoints + // Add 1 for the receiver endpoint, which is not in the list. + val effectiveLastN = effectiveLastN(allocationSettings.lastN, sources.size + 1) + + // Keep track of the number of sources with non-zero constraints. Once [effectiveLastN] of them have been + // added, all other sources have effectiveConstraints 0, because they would never be forwarded by the + // algorithm. + var sourcesWithNonZeroConstraints = 0 + + return sources.associateWith { source -> + if (!source.videoType.isEnabled() || sourcesWithNonZeroConstraints >= effectiveLastN) { + VideoConstraints.NOTHING + } else { + allocationSettings.getConstraints(source.sourceName).also { + if (!it.isDisabled()) sourcesWithNonZeroConstraints++ + } + } + } +} + +/** + * The LastN value adjusted according to the limits configured on the bridge, or [Int.MAX_VALUE] if LastN is disabled. + */ +private fun effectiveLastN(lastN: Int, conferenceSize: Int): Int { + val adjustedLastN = + calculateLastN( + lastN, + jvbLastNSingleton.jvbLastN, + conferenceSizeLimits.getLastNLimit(conferenceSize), + SsrcLimitConfig.config.maxVideoSsrcs + ) + return if (adjustedLastN < 0) Int.MAX_VALUE else adjustedLastN +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMap.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMap.kt new file mode 100644 index 0000000..3b39695 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMap.kt @@ -0,0 +1,84 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode + +/** + * A Map of Endpoint IDs to their receiver video constraints. Tracks the max height of all + * of those constraints as new constraints are added and removed. + */ +class ReceiverConstraintsMap { + private val map = mutableMapOf() + private val lock = Any() + var maxHeight: Int = 0 + private set + + fun put(key: String, value: VideoConstraints): VideoConstraints? { + synchronized(lock) { + return map.put(key, value).also { removed -> + maxHeight = when { + value.maxHeight == -1 -> value.maxHeight + + maxHeight != -1 && value.maxHeight >= maxHeight -> value.maxHeight + + (maxHeight == -1 || value.maxHeight < maxHeight) && removed?.maxHeight == maxHeight -> + findNextMax(maxHeight) + + else -> maxHeight + } + } + } + } + + fun remove(key: String): VideoConstraints? { + synchronized(lock) { + return map.remove(key)?.also { removed -> + if (removed.maxHeight == maxHeight) { + maxHeight = findNextMax(maxHeight) + } + } + } + } + + /** + * Given the previous max, go through the constraints until we either: + * 1) Find a maxHeight equal to the old one or + * 2) Go through the whole list, and track the highest value we've seen. + * Note: this method must be called with the lock held + */ + private fun findNextMax(oldMaxHeight: Int): Int { + var maxSeen = 0 + map.values.forEach { constraints -> + if (constraints.maxHeight == oldMaxHeight) { + return oldMaxHeight + } else if (maxSeen >= 0 && (constraints.maxHeight > maxSeen || constraints.maxHeight == -1)) { + maxSeen = constraints.maxHeight + } + } + return maxSeen + } + + fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("maxHeight", maxHeight) + set("constraints", ObjectMapper().valueToTree(map.mapValues { it.value.toString() }.toMap())) + } + + override fun toString() = getDebugState().toString() +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt new file mode 100644 index 0000000..85ff6bb --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt @@ -0,0 +1,471 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * Copyright @ 2021 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.VideoType +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config +import java.lang.Integer.max +import java.time.Clock +import kotlin.math.min + +/** + * A bitrate allocation that pertains to a specific source. This is the internal representation used in the allocation + * algorithm, as opposed to [SingleAllocation] which is the end result. + * + * @author George Politis + * @author Pawel Domas + */ +internal class SingleSourceAllocation( + val endpointId: String, + val mediaSource: MediaSourceDesc, + /** The constraints to use while allocating bandwidth to this media source. */ + val constraints: VideoConstraints, + /** Whether the source is on stage. */ + private val onStage: Boolean, + diagnosticContext: DiagnosticContext, + clock: Clock, + parentLogger: Logger = LoggerImpl(SingleSourceAllocation::class.qualifiedName) +) { + val logger = createChildLogger(parentLogger).apply { + addContext(mapOf("remote_endpoint_id" to endpointId)) + } + + /** + * The immutable list of layers to be considered when allocating bandwidth. + */ + val layers: Layers = selectLayers(mediaSource, onStage, constraints, clock.instant().toEpochMilli()) + + /** + * The index (into [layers] of the current target layer). It can be improved in the `improve()` step, if there is + * enough bandwidth. + */ + var targetIdx = -1 + + /** + * The bandwidth (in bps) which was available to this source the last time [improve] was called, or `null` if + * [improve] was never called. The latter means the allocation loop never got as far as this source, because a + * higher priority source had not reached its preferred layer. Note that this is negative when the allocator has + * oversent, so it can not use a negative sentinel for "never called". Only used for logging. + */ + private var lastAvailableBps: Long? = null + + /** Whether oversending was allowed the last time [improve] was called. Only used for logging. */ + private var lastAllowedOversending = false + + init { + if (timeSeriesLogger.isTraceEnabled) { + val ratesTimeSeriesPoint = diagnosticContext.makeTimeSeriesPoint("layers_considered") + .addField("remote_endpoint_id", endpointId) + for (layerSnapshot in layers.layers) { + val l = layerSnapshot.layer + val prefix = "${l.indexString()}_${l.height}p_${l.frameRate}fps" + // Which fields are present must not vary with the data, so that the series can be consumed + // mechanically. Note that the bitrate used for allocation is one of the other two, depending on the + // configuration and on whether the source is bursty. + ratesTimeSeriesPoint.addField("${prefix}_bps", layerSnapshot.bitrate) + ratesTimeSeriesPoint.addField("${prefix}_measured_bps", layerSnapshot.measuredBitrate) + // -1 means the sender does not signal a target bitrate for the layer in the VLA extension. + ratesTimeSeriesPoint.addField("${prefix}_vla_bps", layerSnapshot.vlaTargetBitrate ?: -1) + } + timeSeriesLogger.trace(ratesTimeSeriesPoint) + } + } + + fun isOnStage() = onStage + fun hasReachedPreferred(): Boolean = targetIdx >= layers.preferredIndex + + /** + * Implements an "improve" step, incrementing [.targetIdx] to the next layer if there is sufficient + * bandwidth. Note that this works eagerly up until the "preferred" layer (if any), and as a single step from + * then on. + * + * @param remainingBps the additional bandwidth which is available on top of the bitrate of the current target + * layer. + * @return the bandwidth "consumed" by the method, i.e. the difference between the resulting and initial target + * bitrate. E.g. if the target bitrate goes from 100 to 300 as a result if the method call, it will return 200. + */ + fun improve(remainingBps: Long, allowOversending: Boolean): Long { + val initialTargetBitrate = targetBitrate + val maxBps = remainingBps + initialTargetBitrate + lastAvailableBps = maxBps + lastAllowedOversending = allowOversending + if (layers.isEmpty()) { + return 0 + } + if (targetIdx == -1 && layers.preferredIndex > -1 && onStage) { + // Boost on stage participant to preferred, if there's enough bw. + for (i in layers.indices) { + if (i > layers.preferredIndex || maxBps < layers[i].bitrate) { + break + } + targetIdx = i + } + } else { + // Try the next element in the ratedIndices array. + if (targetIdx + 1 < layers.size && layers[targetIdx + 1].bitrate < maxBps) { + targetIdx++ + } + } + if (targetIdx > -1) { + // If there's a higher layer available with a lower bitrate, skip to it. + // + // For example, if 1080p@15fps is configured as a better subjective quality than 720p@30fps (i.e. it sits + // on a higher index in the ratedIndices array) and the bitrate that we measure for the 1080p stream is less + // than the bitrate that we measure for the 720p stream, then we "jump over" the 720p stream and immediately + // select the 1080p stream. + // + // TODO further: Should we just prune the list of layers we consider to not include such layers? + for (i in layers.size - 1 downTo targetIdx + 1) { + if (layers[i].bitrate <= layers[targetIdx].bitrate) { + targetIdx = i + } + } + } + + // If oversending is allowed, look for a better layer which doesn't exceed maxBps by more than + // `maxOversendBitrate`. + if (allowOversending && layers.oversendIndex >= 0 && targetIdx < layers.oversendIndex) { + for (i in layers.oversendIndex downTo targetIdx + 1) { + if (layers[i].bitrate <= maxBps + config.maxOversendBitrate.bps) { + targetIdx = i + } + } + } + + val resultingTargetBitrate = targetBitrate + return resultingTargetBitrate - initialTargetBitrate + } + + /** + * The source is suspended if we've not selected a layer AND the source has active layers. + * + * TODO: this is not exactly correct because it only looks at the layers we consider. E.g. if the receiver set + * a maxHeight=0 constraint for an endpoint, it will appear suspended. This is not critical, because this val is + * only used for logging. + */ + val isSuspended: Boolean + get() = targetIdx == -1 && layers.isNotEmpty() && layers[0].bitrate > 0 + + /** + * The layer which had to fit in the available bandwidth in order for this source not to be suspended: the first + * layer, which is the one [improve] tries while the source has no target layer. + */ + private val requiredLayer: LayerSnapshot? + get() = layers.firstOrNull() + + /** + * The cheapest of the layers which could have kept this source from being suspended, i.e. of [rescuingLayers]. + * This is normally [requiredLayer], but the layers are not guaranteed to be ordered by bitrate (notably with VP9), + * and when oversending it is the cheapest layer which decides whether the source can be rescued. + */ + private val cheapestLayer: LayerSnapshot? + get() = rescuingLayers.minByOrNull { it.bitrate } + + /** + * The layers which the last [improve] call could have selected while the source had no target layer. That is just + * the first one, unless oversending was allowed, in which case the oversend loop scans everything up to and + * including [Layers.oversendIndex] (but no further). + */ + private val rescuingLayers: List + get() = layers.take( + if (lastAllowedOversending && layers.oversendIndex >= 0) layers.oversendIndex + 1 else 1 + ) + + /** + * A summary of why this source ended up suspended, for logging. Only meaningful once the allocation algorithm has + * completed, and only for a source which is actually suspended (i.e. [isSuspended] is true) -- for any other source + * the fields describe an allocation which did succeed. All bitrates are in bps: + * + * "required" is the layer which had to fit in the available bandwidth, i.e. what the allocator thought the source + * needed; "cheapest" is only present when a different layer could have rescued the source more cheaply; "ideal" is + * the bitrate of the layer we'd have forwarded with unlimited bandwidth, and is omitted when that is the required + * layer; "oversend" describes the layer we'd have used when oversending (capped by + * videobridge.cc.max-oversend-bitrate) -- its bitrate, or "required" when it is the required layer, or "notAllowed" + * when this source was not the one source which is allowed to oversend -- and is absent when the source has no + * oversend layer at all; "available" is the bandwidth which was left for the source when it was last considered, + * and is negative if the allocator had already oversent. The bitrates of the layers which are not called out here + * are only available in the "layers_considered" time series. + */ + fun suspendedSummary(): String = buildString { + append(mediaSource.sourceName) + append("[type=${mediaSource.videoType}") + append(" onStage=$onStage") + append(" required=${requiredLayer?.summary()}") + cheapestLayer?.let { if (it !== requiredLayer) append(" cheapest=${it.summary()}") } + if (idealBitrate != requiredLayer?.bitrate) append(" ideal=$idealBitrate") + layers.oversendLayer?.let { + val oversend = when { + !lastAllowedOversending -> "notAllowed" + it === requiredLayer -> "required" + else -> it.bitrate.toString() + } + append(" oversend=$oversend") + } + append(" available=${lastAvailableBps ?: "none(never considered)"}") + append(" constraints=$constraints]") + } + + /** + * Gets the target bitrate (in bps) for this endpoint allocation, i.e. the bitrate of the currently chosen layer. + */ + val targetBitrate: Long + get() = targetLayer?.bitrate ?: 0 + + private val targetLayer: LayerSnapshot? + get() = layers.getOrNull(targetIdx) + + /** + * Gets the ideal bitrate (in bps) for this endpoint allocation, i.e. the bitrate of the layer the bridge would + * forward if there were no (bandwidth) constraints. + */ + val idealBitrate: Long + get() = layers.idealLayer?.bitrate ?: 0 + + /** + * Exposed for testing only. + */ + val preferredLayer: RtpLayerDesc? + get() = layers.preferredLayer?.layer + + /** + * Exposed for testing only. + */ + val oversendLayer: RtpLayerDesc? + get() = layers.oversendLayer?.layer + + /** + * Creates the final immutable result of this allocation. Should be called once the allocation algorithm has + * completed. + */ + val result: SingleAllocation + get() = SingleAllocation( + endpointId, + mediaSource, + targetLayer?.layer, + layers.idealLayer?.layer + ) + + override fun toString(): String = ( + "[id=" + endpointId + + " constraints=" + constraints + + " ratedPreferredIdx=" + layers.preferredIndex + + " ratedTargetIdx=" + targetIdx + ) + + /** + * Selects from a list of layers the ones which should be considered when allocating bandwidth, as well as the + * "preferred" and "oversend" layers. Logic specific to screensharing: we prioritize resolution over framerate, + * prioritize the full resolution (at the frame rate which is cheapest to send it at) over other endpoints by + * setting it as "preferred", and allow oversending up to the full resolution (with low frame rate). Above the + * preferred layer the screenshare's frame rate competes with the other sources, which may reduce (though not + * disable) their quality as it improves. + */ + private fun selectLayersForScreensharing( + layers: List, + constraints: VideoConstraints, + onStage: Boolean, + /** Whether the sender has asked for the share's frame rate to be prioritized. */ + highFps: Boolean + ): Layers { + var activeLayers = layers.filter { it.bitrate > 0 } + // No active layers usually happens when the source has just been signaled and we haven't received + // any packets yet. Add the layers here, so one gets selected and we can start forwarding sooner. + if (activeLayers.isEmpty()) activeLayers = layers + + // We select all layers that satisfy the constraints. + var selectedLayers = + if (!constraints.heightIsLimited()) { + activeLayers + } else { + activeLayers.filter { it.layer.height <= constraints.maxHeight } + } + // If no layers satisfy the constraints, we use the layers with the lowest resolution. + if (selectedLayers.isEmpty()) { + val minHeight = activeLayers.minOfOrNull { it.layer.height } ?: return Layers.noLayers + selectedLayers = activeLayers.filter { it.layer.height == minHeight } + + // This recognizes the structure used with VP9 (multiple encodings with the same resolution and unknown frame + // rate). In this case, we only want the low quality layer. Unless we're on stage, in which case we should + // consider all layers. + if (!onStage && selectedLayers.isNotEmpty() && selectedLayers[0].layer.frameRate < 0) { + selectedLayers = listOf(selectedLayers[0]) + } + } + + // A source always has at least one layer here (selectLayers returns early otherwise), and the branches above + // never leave selectedLayers empty, so the maxima below exist. + val maxHeight = selectedLayers.maxOf { it.layer.height } + val maxHeightIndices = selectedLayers.indices.filter { selectedLayers[it].layer.height == maxHeight } + // Of all layers with the highest resolution, the one with the lowest bitrate. In case of VP9 the layers are + // not necessarily ordered by bitrate. + val cheapestMaxHeightIdx = maxHeightIndices.minBy { selectedLayers[it].bitrate } + // Since we prioritize resolution over frame rate, "preferred" is the full resolution at the frame rate which + // is cheapest to send it at. Anything above that improves frame rate only, and competes with the other + // sources on equal terms rather than blocking them entirely. There are two exceptions, for which we keep the + // highest layer as "preferred" (as we previously did for all screen sharing): + // - High frame rate screen sharing, where the sender has explicitly asked for frame rate to be prioritized. + // - The structure used with VP9 (multiple encodings with the same resolution and unknown frame rate), where + // the layers above the cheapest one improve quality rather than frame rate, and quality is exactly what + // this policy exists to protect. + val preferredIdx = if (highFps || maxHeightIndices.any { selectedLayers[it].layer.frameRate < 0 }) { + selectedLayers.size - 1 + } else { + cheapestMaxHeightIdx + } + val oversendIdx = if (onStage && config.allowOversendOnStage) cheapestMaxHeightIdx else -1 + return Layers(selectedLayers, preferredIdx, oversendIdx) + } + + /** + * Selects from the layers of a [MediaSourceContainer] the ones which should be considered when allocating bandwidth for + * an endpoint. Also selects the indices of the "preferred" and "oversend" layers. + * + * @param endpoint the [MediaSourceContainer] that describes the available layers. + * @param constraints the constraints signaled for the endpoint. + * @return the ordered list of [endpoint]'s layers which should be considered when allocating bandwidth, as well as the + * indices of the "preferred" and "oversend" layers. + */ + private fun selectLayers( + /** The endpoint which is the source of the stream(s). */ + source: MediaSourceDesc, + onStage: Boolean, + /** The constraints that the receiver specified for [source]. */ + constraints: VideoConstraints, + nowMs: Long + ): Layers { + if (constraints.maxHeight == 0 || !source.hasRtpLayers()) { + return Layers.noLayers + } + // Screen sharing is bursty: while the screen is static the encoder accumulates rate control credit (about a + // second's worth) and then spends it on a single frame, so the bitrate measured over the short window is not + // representative of the bandwidth the source needs. Allocate for it using the longer window instead. + val isBursty = source.videoType.isScreenshare() + val layers = source.rtpLayers.map { + if (isBursty) { + // Only bursty sources need the extra tracker, so it is not created until we ask for it. + it.enableSmoothedBitrate(nowMs) + } else { + // And when a source stops being bursty, drop the tracker so its layers stop paying for it. + it.disableSmoothedBitrate() + } + val measuredBitrate = it.getBitrate(nowMs).bps + val vlaTargetBitrate = it.targetBitrate?.bps + val allocationBitrate = when { + config.useVlaTargetBitrate && vlaTargetBitrate != null -> vlaTargetBitrate + isBursty -> it.getSmoothedBitrate(nowMs).bps + else -> measuredBitrate + } + LayerSnapshot(it, allocationBitrate, measuredBitrate, vlaTargetBitrate) + } + + return when { + source.videoType == VideoType.CAMERA -> selectLayersForCamera(layers, constraints) + + source.videoType.isScreenshare() -> selectLayersForScreensharing( + layers, + constraints, + onStage, + highFps = source.videoType == VideoType.DESKTOP_HIGH_FPS + ) + + else -> Layers.noLayers + } + } + + /** + * Selects from a list of layers the ones which should be considered when allocating bandwidth, as well as the + * "preferred" and "oversend" layers. Logic specific to a camera stream: once the "preferred" height is reached we + * require a high frame rate, with preconfigured values for the "preferred" height and frame rate, and we do not allow + * oversending. + */ + private fun selectLayersForCamera(layers: List, constraints: VideoConstraints): Layers { + val minHeight = layers.minOfOrNull { it.layer.height } ?: return Layers.noLayers + val maxFps = layers.maxOfOrNull { it.layer.frameRate } ?: return Layers.noLayers + val noActiveLayers = layers.none { (_, bitrate) -> bitrate > 0 } + val (preferredHeight, preferredFps) = getPreferred(constraints) + val effectivePreferredFps = if (maxFps > 0) { + min(maxFps, preferredFps) + } else { + preferredFps + } + + val ratesList: MutableList = ArrayList() + // Initialize the list of layers to be considered. These are the layers that satisfy the constraints, with + // a couple of exceptions (see comments below). + for (layerSnapshot in layers) { + val layer = layerSnapshot.layer + val lessThanPreferredHeight = layer.height < preferredHeight + val lessThanOrEqualMaxHeight = layer.height <= constraints.maxHeight || !constraints.heightIsLimited() + // If frame rate is unknown, consider it to be sufficient. + val atLeastPreferredFps = layer.frameRate < 0 || layer.frameRate >= effectivePreferredFps + if (lessThanPreferredHeight || + (lessThanOrEqualMaxHeight && atLeastPreferredFps) || + layer.height == minHeight + ) { + // No active layers usually happens when the source has just been signaled and we haven't received + // any packets yet. Add the layers here, so one gets selected and we can start forwarding sooner. + if (noActiveLayers || layerSnapshot.bitrate > 0) { + ratesList.add(layerSnapshot) + } + } + } + + val effectivePreferredHeight = max(preferredHeight, minHeight) + val preferredIndex = ratesList.lastIndexWhich { it.layer.height <= effectivePreferredHeight } + logger.trace { + "Selected rates list $ratesList, preferred index $preferredIndex " + + "from layers $layers with constraints $constraints" + } + + return Layers(ratesList, preferredIndex, -1) + } + + companion object { + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(BandwidthAllocator::class.java) + } +} + +/** + * Returns the index of the last element of this list which satisfies the given predicate, or -1 if no elements do. + */ +private fun List.lastIndexWhich(predicate: (T) -> Boolean): Int { + var lastIndex = -1 + forEachIndexed { i, e -> if (predicate(e)) lastIndex = i } + return lastIndex +} + +/** + * Gets the "preferred" height and frame rate based on the constraints signaled from the receiver. + * + * For participants with sufficient maxHeight we favor frame rate over resolution. We consider all + * temporal layers for resolutions lower than the preferred, but for resolutions >= preferred, we only + * consider frame rates at least as high as the preferred. In practice this means we consider + * 180p/7.5fps, 180p/15fps, 180p/30fps, 360p/30fps and 720p/30fps. + */ +private fun getPreferred(constraints: VideoConstraints): VideoConstraints = + if (constraints.maxHeight > 180 || !constraints.heightIsLimited()) { + VideoConstraints(config.onstagePreferredHeightPx, config.onstagePreferredFramerate) + } else { + VideoConstraints.UNLIMITED + } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt new file mode 100644 index 0000000..3a6d7de --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/VideoConstraints.kt @@ -0,0 +1,49 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.databind.node.JsonNodeFactory + +@JsonIgnoreProperties(ignoreUnknown = true) +data class VideoConstraints @JvmOverloads constructor( + val maxHeight: Int, + val maxFrameRate: Double = UNLIMITED_FRAME_RATE +) { + init { + require(maxHeight == UNLIMITED_HEIGHT || maxHeight >= 0) { "maxHeight must be either -1, 0, or positive." } + require(maxFrameRate == UNLIMITED_FRAME_RATE || maxFrameRate >= 0.0) { + "maxFrameRate must be either -1, or >= 0" + } + } + override fun toString(): String = JsonNodeFactory.instance.objectNode().apply { + put("maxHeight", maxHeight) + put("maxFrameRate", maxFrameRate) + }.toString() + + fun heightIsLimited() = maxHeight != UNLIMITED_HEIGHT + fun frameRateIsLimited() = maxFrameRate != UNLIMITED_FRAME_RATE + fun isDisabled() = maxHeight == 0 || maxFrameRate == 0.0 + + companion object { + const val UNLIMITED_HEIGHT = -1 + const val UNLIMITED_FRAME_RATE = -1.0 + val NOTHING = VideoConstraints(0) + val UNLIMITED = VideoConstraints(UNLIMITED_HEIGHT, UNLIMITED_FRAME_RATE) + } +} + +fun Map.prettyPrint(): String = entries.joinToString { "${it.key}->${it.value.maxHeight}" } diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt new file mode 100644 index 0000000..0a4904c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt @@ -0,0 +1,694 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex +import org.jitsi.nlj.rtp.codec.av1.getTemplateIdDelta +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtp.header_extensions.DTI +import org.jitsi.rtp.rtp.header_extensions.toShortString +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.AdaptiveSourceProjectionContext +import org.jitsi.videobridge.cc.RewriteException +import org.jitsi.videobridge.cc.RtpState +import java.time.Duration +import java.time.Instant + +class Av1DDAdaptiveSourceProjectionContext( + private val diagnosticContext: DiagnosticContext, + rtpState: RtpState, + persistentState: Any?, + parentLogger: Logger +) : AdaptiveSourceProjectionContext { + private val logger: Logger = createChildLogger(parentLogger) + + /** + * A map that stores the per-encoding AV1 frame maps. + */ + private val av1FrameMaps = HashMap() + + /** + * The [Av1DDQualityFilter] instance that does quality filtering on the + * incoming pictures, to choose encodings and layers to forward. + */ + private val av1QualityFilter = Av1DDQualityFilter(av1FrameMaps, logger) + + init { + require(persistentState is Av1PersistentState?) + } + + private var lastAv1FrameProjection = Av1DDFrameProjection( + diagnosticContext, + rtpState.ssrc, + rtpState.maxSequenceNumber, + rtpState.maxTimestamp, + (persistentState as? Av1PersistentState)?.frameNumber, + (persistentState as? Av1PersistentState)?.templateId + ) + + /** + * The frame number index that started the latest stream resumption. + * We can't send frames with frame number less than this, because we don't have + * space in the projected sequence number/frame number counts. + */ + private var lastFrameNumberIndexResumption = -1L + + override fun accept(packetInfo: PacketInfo, targetIndex: Int): Boolean { + val packet = packetInfo.packet + + if (packet !is Av1DDPacket) { + logger.warn("Packet is not AV1 DD Packet") + return false + } + val incomingEncoding = packet.encodingId + + /* If insertPacketInMap returns null, this is a very old picture, more than Av1FrameMap.PICTURE_MAP_SIZE old, + or something is wrong with the stream. */ + val result = insertPacketInMap(packet) ?: return false + + val frame = result.frame + + if (result.isNewFrame) { + if (packet.isKeyframe && frameIsNewSsrc(frame)) { + /* If we're not currently projecting this SSRC, check if we've + * already decided to drop a subsequent required frame of this SSRC for the DT. + * If we have, we can't turn on the encoding starting from this + * packet, so treat this frame as though it weren't a keyframe. + * Note that this may mean re-analyzing the packets with a now-available template dependency structure. + */ + if (haveSubsequentNonAcceptedChain(frame, incomingEncoding, targetIndex)) { + frame.isKeyframe = false + } + } + val receivedTime = packetInfo.receivedTime + val acceptResult = av1QualityFilter + .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime) + frame.isAccepted = acceptResult.accept && frameIsProjectable(frame) + if (frame.isAccepted) { + val projection: Av1DDFrameProjection + try { + projection = createProjection( + frame = frame, + initialPacket = packet, + isResumption = acceptResult.isResumption, + isReset = result.isReset, + mark = acceptResult.mark, + newDt = acceptResult.newDt, + receivedTime = receivedTime + ) + } catch (e: Exception) { + logger.warn("Failed to create frame projection", e) + /* Make sure we don't have an accepted frame without a projection in the map. */ + frame.isAccepted = false + return false + } + frame.projection = projection + if (projection.earliestProjectedSeqNum isNewerThan lastAv1FrameProjection.latestProjectedSeqNum) { + lastAv1FrameProjection = projection + } + } + } + + val accept = frame.isAccepted && + if (frame.projection?.accept(packet) == true) { + true + } else { + if (frame.projection != null && frame.projection?.closedSeq != -1) { + logger.debug( + "Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}" + ) + } else if (frame.projection == null) { + logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it") + } else { + logger.warn("Not accepting $packet, even though frame projection is not closed") + } + false + } + + if (timeSeriesLogger.isTraceEnabled) { + val pt = diagnosticContext.makeTimeSeriesPoint("rtp_av1") + .addField("ssrc", packet.ssrc) + .addField("timestamp", packet.timestamp) + .addField("seq", packet.sequenceNumber) + .addField("frameNumber", packet.frameNumber) + .addField("templateId", packet.statelessDescriptor.frameDependencyTemplateId) + .addField("hasStructure", packet.descriptor?.newTemplateDependencyStructure != null) + .addField("spatialLayer", packet.frameInfo?.spatialId) + .addField("temporalLayer", packet.frameInfo?.temporalId) + .addField("dti", packet.frameInfo?.dti?.toShortString()) + .addField("hasInterPictureDependency", packet.frameInfo?.hasInterPictureDependency()) + // TODO add more relevant fields from AV1 DD for debugging + .addField("targetIndex", Av1DDRtpLayerDesc.indexString(targetIndex)) + .addField("new_frame", result.isNewFrame) + .addField("accept", accept) + .addField("payload_length", packet.payloadLength) + .addField("packet_length", packet.length) + av1QualityFilter.addDiagnosticContext(pt) + timeSeriesLogger.trace(pt) + } + + return accept + } + + /** Look up an Av1DDFrame for a packet. */ + private fun lookupAv1Frame(av1Packet: Av1DDPacket): Av1DDFrame? = av1FrameMaps[av1Packet.ssrc]?.findFrame(av1Packet) + + /** + * Insert a packet in the appropriate [Av1DDFrameMap]. + */ + private fun insertPacketInMap(av1Packet: Av1DDPacket) = + av1FrameMaps.getOrPut(av1Packet.ssrc) { Av1DDFrameMap(logger) }.insertPacket(av1Packet) + + private fun haveSubsequentNonAcceptedChain(frame: Av1DDFrame, incomingEncoding: Int, targetIndex: Int): Boolean { + val map = av1FrameMaps[frame.ssrc] ?: return false + val structure = frame.structure ?: return false + val dtsToCheck = if (incomingEncoding == getEidFromIndex(targetIndex)) { + setOf(getDtFromIndex(targetIndex)) + } else { + frame.frameInfo?.dtisPresent ?: emptySet() + } + val chainsToCheck = dtsToCheck.mapNotNull { structure.decodeTargetProtectedBy.getOrNull(it) }.toSet() + return map.nextFrameWith(frame) { + if (it.isAccepted) return@nextFrameWith false + if (it.frameInfo == null) { + it.updateParse(structure, logger) + } + return@nextFrameWith chainsToCheck.any { chainIdx -> + it.partOfActiveChain(chainIdx) + } + } != null + } + + private fun Av1DDFrame.partOfActiveChain(chainIdx: Int): Boolean { + val structure = structure ?: return false + val frameInfo = frameInfo ?: return false + for (i in structure.decodeTargetProtectedBy.indices) { + if (structure.decodeTargetProtectedBy[i] != chainIdx) { + continue + } + if (frameInfo.dti[i] == DTI.NOT_PRESENT || frameInfo.dti[i] == DTI.DISCARDABLE) { + return false + } + } + return true + } + + /** + * Calculate the projected sequence number gap between two frames (of the same encoding), + * allowing collapsing for unaccepted frames. + */ + private fun seqGap(frame1: Av1DDFrame, frame2: Av1DDFrame): Int { + var seqGap = RtpUtils.getSequenceNumberDelta( + frame2.earliestKnownSequenceNumber, + frame1.latestKnownSequenceNumber + ) + if (!frame1.isAccepted && !frame2.isAccepted && frame2.isImmediatelyAfter(frame1)) { + /* If neither frame is being projected, and they have consecutive + frame numbers, we don't need to leave any gap. */ + seqGap = 0 + } else { + /* If the earlier frame wasn't projected, and we haven't seen its + * final packet, we know it has to consume at least one more sequence number. */ + if (!frame1.isAccepted && !frame1.seenEndOfFrame && seqGap > 1) { + seqGap-- + } + /* Similarly, if the later frame wasn't projected and we haven't seen + * its first packet. */ + if (!frame2.isAccepted && !frame2.seenStartOfFrame && seqGap > 1) { + seqGap-- + } + /* If the frame wasn't accepted, it has to have consumed at least one sequence number, + * which we can collapse out. */ + if (!frame1.isAccepted && seqGap > 0) { + seqGap-- + } + } + return seqGap + } + + private fun frameIsNewSsrc(frame: Av1DDFrame): Boolean = lastAv1FrameProjection.av1Frame?.matchesSSRC(frame) != true + + private fun frameIsProjectable(frame: Av1DDFrame): Boolean = + frameIsNewSsrc(frame) || frame.index >= lastFrameNumberIndexResumption + + /** + * Find the previous frame before the given one. + */ + @Synchronized + private fun prevFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.prevFrame(frame) + + /** + * Find the next frame after the given one. + */ + @Synchronized + private fun nextFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.nextFrame(frame) + + /** + * Find the previous accepted frame before the given one. + */ + private fun findPrevAcceptedFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.findPrevAcceptedFrame(frame) + + /** + * Find the next accepted frame after the given one. + */ + private fun findNextAcceptedFrame(frame: Av1DDFrame) = av1FrameMaps[frame.ssrc]?.findNextAcceptedFrame(frame) + + /** + * Create a projection for this frame. + */ + private fun createProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + isResumption: Boolean, + isReset: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + if (frameIsNewSsrc(frame)) { + return createEncodingSwitchProjection(frame, initialPacket, mark, newDt, receivedTime) + } else if (isReset) { + return createResetProjection(frame, initialPacket, mark, newDt, receivedTime) + } else if (isResumption) { + return createResumptionProjection(frame, initialPacket, mark, newDt, receivedTime) + } + + return createInEncodingProjection(frame, initialPacket, mark, newDt, receivedTime) + } + + /** + * Create a projection for the first frame after an encoding switch. + */ + private fun createEncodingSwitchProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + // We can only switch on packets that carry a scalability structure, which is the first packet of a keyframe + assert(frame.isKeyframe) + assert(initialPacket.isStartOfFrame) + lastFrameNumberIndexResumption = frame.index + + var projectedSeqGap = 1 + + if (lastAv1FrameProjection.av1Frame?.seenEndOfFrame == false) { + /* Leave a gap to signal to the decoder that the previously routed + frame was incomplete. */ + projectedSeqGap++ + + /* Make sure subsequent packets of the previous projection won't + overlap the new one. (This means the gap, above, will never be + filled in.) + */ + lastAv1FrameProjection.close() + } + + val projectedSeq = + RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.latestProjectedSeqNum, projectedSeqGap) + + // this is a simulcast switch. The typical incremental value = + // 90kHz / 30 = 90,000Hz / 30 = 3000 per frame or per 33ms + val tsDelta = lastAv1FrameProjection.created?.let { created -> + receivedTime?.let { + 3000 * Duration.between(created, receivedTime).dividedBy(33).seconds.coerceAtLeast(1L) + } + } ?: 3000 + val projectedTs = RtpUtils.applyTimestampDelta(lastAv1FrameProjection.timestamp, tsDelta) + + val frameNumber: Int + val templateIdDelta: Int + val nextTemplateId = lastAv1FrameProjection.getNextTemplateId() + if (nextTemplateId != null) { + frameNumber = RtpUtils.applySequenceNumberDelta( + lastAv1FrameProjection.frameNumber, + 1 + ) + val structure = frame.structure + check(structure != null) + templateIdDelta = getTemplateIdDelta(nextTemplateId, structure.templateIdOffset) + } else { + frameNumber = frame.frameNumber + templateIdDelta = 0 + } + + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber), + frameNumber = frameNumber, + templateIdDelta = templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a projection for the first frame after a resumption, i.e. when a source is turned back on. + */ + private fun createResumptionProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + lastFrameNumberIndexResumption = frame.index + + /* These must be non-null because we don't execute this function unless + frameIsNewSsrc and isReset were both false. + */ + val lastFrame = prevFrame(frame)!! + val lastProjectedFrame = lastAv1FrameProjection.av1Frame!! + + /* Project timestamps linearly. */ + val tsDelta = RtpUtils.getTimestampDiff( + lastAv1FrameProjection.timestamp, + lastProjectedFrame.timestamp + ) + val projectedTs = RtpUtils.applyTimestampDelta(frame.timestamp, tsDelta) + + /* Increment frameNumber by 1 from the last projected frame. */ + val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.frameNumber, 1) + + /** If this packet has a template structure, rewrite it to follow after the pre-resumption structure. + * (We could check to see if the structure is unchanged, but that's an unnecessary optimization.) + */ + val templateIdDelta = if (frame.structure != null) { + val nextTemplateId = lastAv1FrameProjection.getNextTemplateId() + + if (nextTemplateId != null) { + val structure = frame.structure + check(structure != null) + getTemplateIdDelta(nextTemplateId, structure.templateIdOffset) + } else { + 0 + } + } else { + 0 + } + + /* Increment sequence numbers based on the last projected frame, but leave a gap + * for packet reordering in case this isn't the first packet of the keyframe. + */ + val seqGap = RtpUtils.getSequenceNumberDelta(initialPacket.sequenceNumber, lastFrame.latestKnownSequenceNumber) + val newSeq = RtpUtils.applySequenceNumberDelta(lastAv1FrameProjection.latestProjectedSeqNum, seqGap) + val seqDelta = RtpUtils.getSequenceNumberDelta(newSeq, initialPacket.sequenceNumber) + + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = seqDelta, + frameNumber = projectedFrameNumber, + templateIdDelta = templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a projection for the first frame after a frame reset, i.e. after a large gap in sequence numbers. + */ + private fun createResetProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + /* This must be non-null because we don't execute this function unless + frameIsNewSsrc has returned false. + */ + val lastFrame = lastAv1FrameProjection.av1Frame!! + + /* Apply the latest projected frame's projections out, linearly. */ + val seqDelta = RtpUtils.getSequenceNumberDelta( + lastAv1FrameProjection.latestProjectedSeqNum, + lastFrame.latestKnownSequenceNumber + ) + val tsDelta = RtpUtils.getTimestampDiff( + lastAv1FrameProjection.timestamp, + lastFrame.timestamp + ) + val frameNumberDelta = RtpUtils.applySequenceNumberDelta( + lastAv1FrameProjection.frameNumber, + lastFrame.frameNumber + ) + + val projectedTs = RtpUtils.applyTimestampDelta(frame.timestamp, tsDelta) + val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(frame.frameNumber, frameNumberDelta) + + /** If this packet has a template structure, rewrite it to follow after the pre-reset structure. + * (We could check to see if the structure is unchanged, but that's an unnecessary optimization.) + */ + val templateIdDelta = if (frame.structure != null) { + val nextTemplateId = lastAv1FrameProjection.getNextTemplateId() + + if (nextTemplateId != null) { + val structure = frame.structure + check(structure != null) + getTemplateIdDelta(nextTemplateId, structure.templateIdOffset) + } else { + 0 + } + } else { + 0 + } + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = seqDelta, + frameNumber = projectedFrameNumber, + templateIdDelta = templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a frame projection for the normal case, i.e. as part of the same encoding as the + * previously-projected frame. + */ + private fun createInEncodingProjection( + frame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + val prevFrame = findPrevAcceptedFrame(frame) + if (prevFrame != null) { + return createInEncodingProjection(frame, prevFrame, initialPacket, mark, newDt, receivedTime) + } + + /* prev frame has rolled off beginning of frame map, try next frame */ + val nextFrame = findNextAcceptedFrame(frame) + if (nextFrame != null) { + return createInEncodingProjection(frame, nextFrame, initialPacket, mark, newDt, receivedTime) + } + + /* Neither previous or next is found. Very big frame? Use previous projected. + (This must be valid because we don't execute this function unless + frameIsNewSsrc has returned false.) + */ + return createInEncodingProjection( + frame, + lastAv1FrameProjection.av1Frame!!, + initialPacket, + mark, + newDt, + receivedTime + ) + } + + /** + * Create a frame projection for the normal case, i.e. as part of the same encoding as the + * previously-projected frame, based on a specific chosen previously-projected frame. + */ + private fun createInEncodingProjection( + frame: Av1DDFrame, + refFrame: Av1DDFrame, + initialPacket: Av1DDPacket, + mark: Boolean, + newDt: Int?, + receivedTime: Instant? + ): Av1DDFrameProjection { + val tsGap = RtpUtils.getTimestampDiff(frame.timestamp, refFrame.timestamp) + val frameNumGap = RtpUtils.getSequenceNumberDelta(frame.frameNumber, refFrame.frameNumber) + var seqGap = 0 + + var f1 = refFrame + var f2: Av1DDFrame? + val refSeq: Int + if (frameNumGap > 0) { + /* refFrame is earlier than frame in decode order. */ + do { + f2 = nextFrame(f1) + checkNotNull(f2) { + "No next frame found after frame with frame number ${f1.frameNumber}, " + + "even though refFrame ${refFrame.frameNumber} is before " + + "frame ${frame.frameNumber}!" + } + seqGap += seqGap(f1, f2) + f1 = f2 + } while (f2 !== frame) + /* refFrame is a projected frame, so it has a projection. */ + refSeq = refFrame.projection!!.latestProjectedSeqNum + } else { + /* refFrame is later than frame in decode order. */ + do { + f2 = prevFrame(f1) + checkNotNull(f2) { + "No previous frame found before frame with frame number ${f1.frameNumber}, " + + "even though refFrame ${refFrame.frameNumber} is after " + + "frame ${frame.frameNumber}!" + } + seqGap += -seqGap(f2, f1) + f1 = f2 + } while (f2 !== frame) + refSeq = refFrame.projection!!.earliestProjectedSeqNum + } + + val projectedSeq = RtpUtils.applySequenceNumberDelta(refSeq, seqGap) + val projectedTs = RtpUtils.applyTimestampDelta(refFrame.projection!!.timestamp, tsGap) + val projectedFrameNumber = RtpUtils.applySequenceNumberDelta(refFrame.projection!!.frameNumber, frameNumGap) + + return Av1DDFrameProjection( + diagnosticContext = diagnosticContext, + av1Frame = frame, + ssrc = lastAv1FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = RtpUtils.getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber), + frameNumber = projectedFrameNumber, + templateIdDelta = lastAv1FrameProjection.templateIdDelta, + dti = newDt?.let { frame.structure?.getDtBitmaskForDt(it) }, + mark = mark, + created = receivedTime + ) + } + + override fun needsKeyframe(): Boolean { + if (av1QualityFilter.needsKeyframe) { + return true + } + + return lastAv1FrameProjection.av1Frame == null + } + + override fun rewriteRtp(packetInfo: PacketInfo) { + if (packetInfo.packet !is Av1DDPacket) { + logger.info("Got a non-AV1 DD packet.") + throw RewriteException("Non-AV1 DD packet in AV1 DD source projection") + } + val av1Packet = packetInfo.packetAs() + + val av1Frame = lookupAv1Frame(av1Packet) + ?: throw RewriteException("Frame not in tracker (aged off?)") + + val av1Projection = av1Frame.projection + ?: throw RewriteException("Frame does not have projection?") + /* Shouldn't happen for an accepted packet whose frame is still known? */ + + logger.trace { "Rewriting packet with structure ${System.identityHashCode(av1Packet.descriptor?.structure)}" } + av1Projection.rewriteRtp(av1Packet) + } + + override fun rewriteRtcp(rtcpSrPacket: RtcpSrPacket): Boolean { + val lastAv1FrameProjectionCopy: Av1DDFrameProjection = lastAv1FrameProjection + if (rtcpSrPacket.senderSsrc != lastAv1FrameProjectionCopy.av1Frame?.ssrc) { + return false + } + + rtcpSrPacket.senderSsrc = lastAv1FrameProjectionCopy.ssrc + + val srcTs = rtcpSrPacket.senderInfo.rtpTimestamp + val delta = RtpUtils.getTimestampDiff( + lastAv1FrameProjectionCopy.timestamp, + lastAv1FrameProjectionCopy.av1Frame.timestamp + ) + + val dstTs = RtpUtils.applyTimestampDelta(srcTs, delta) + + if (srcTs != dstTs) { + rtcpSrPacket.senderInfo.rtpTimestamp = dstTs + } + + return true + } + + override fun getRtpState() = RtpState( + lastAv1FrameProjection.ssrc, + lastAv1FrameProjection.latestProjectedSeqNum, + lastAv1FrameProjection.timestamp + ) + + override fun getPersistentState(): Any = Av1PersistentState( + lastAv1FrameProjection.frameNumber, + lastAv1FrameProjection.getNextTemplateId() ?: 0 + ) + + override fun getDebugState(): ObjectNode { + val debugState = JsonNodeFactory.instance.objectNode() + debugState.put("class", Av1DDAdaptiveSourceProjectionContext::class.java.simpleName) + + val mapSizes: ArrayNode = JsonNodeFactory.instance.arrayNode() + for ((key, value) in av1FrameMaps.entries) { + val sizeInfo = JsonNodeFactory.instance.objectNode() + sizeInfo.put("ssrc", key) + sizeInfo.put("size", value.size()) + mapSizes.add(sizeInfo) + } + debugState.set("av1FrameMaps", mapSizes) + debugState.set("av1QualityFilter", av1QualityFilter.debugState) + + return debugState + } + + companion object { + /** + * The time series logger for this class. + */ + private val timeSeriesLogger = + TimeSeriesLogger.getTimeSeriesLogger(Av1DDAdaptiveSourceProjectionContext::class.java) + } +} + +data class Av1PersistentState( + val frameNumber: Int, + val templateId: Int +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt new file mode 100644 index 0000000..b3727f3 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrame.kt @@ -0,0 +1,303 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyException +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.rtp.rtp.header_extensions.FrameInfo +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging2.Logger + +class Av1DDFrame internal constructor( + /** + * The RTP SSRC of the incoming frame that this instance refers to + * (RFC3550). + */ + val ssrc: Long, + + /** + * The RTP timestamp of the incoming frame that this instance refers to + * (RFC3550). + */ + val timestamp: Long, + + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + earliestKnownSequenceNumber: Int, + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + latestKnownSequenceNumber: Int, + + /** + * A boolean that indicates whether we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + seenStartOfFrame: Boolean, + + /** + * A boolean that indicates whether we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + seenEndOfFrame: Boolean, + + /** + * A boolean that indicates whether we've seen a packet with the marker bit set. + */ + seenMarker: Boolean, + + /** + * AV1 FrameInfo for the frame + */ + frameInfo: FrameInfo?, + + /** + * The AV1 DD Frame Number of this frame. + */ + val frameNumber: Int, + + /** + * The FrameID index (FrameID plus cycles) of this frame. + */ + val index: Long, + + /** + * The template ID of this frame + */ + val templateId: Int, + + /** + * The AV1 Template Dependency Structure in effect for this frame, if known + */ + structure: Av1TemplateDependencyStructure?, + + /** + * A new activeDecodeTargets specified for this frame, if any. + * TODO: is this always specified in all packets of the frame? + */ + val activeDecodeTargets: Int?, + + /** + * A boolean that indicates whether the incoming AV1 frame that this + * instance refers to is a keyframe. + */ + var isKeyframe: Boolean, + + /** + * The raw dependency descriptor included in the packet. Stored if it could not be parsed initially. + */ + val rawDependencyDescriptor: RtpPacket.HeaderExtension? +) { + /** + * AV1 FrameInfo for the frame + */ + var frameInfo = frameInfo + private set + + /** + * The AV1 Template Dependency Structure in effect for this frame, if known + */ + var structure = structure + private set + + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + var earliestKnownSequenceNumber = earliestKnownSequenceNumber + private set + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + var latestKnownSequenceNumber: Int = latestKnownSequenceNumber + private set + + /** + * A boolean that indicates whether we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + var seenStartOfFrame: Boolean = seenStartOfFrame + private set + + /** + * A boolean that indicates whether we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + var seenEndOfFrame: Boolean = seenEndOfFrame + private set + + /** + * A boolean that indicates whether we've seen a packet with the marker bit set. + */ + var seenMarker: Boolean = seenMarker + private set + + /** + * A record of how this frame was projected, or null if not. + */ + var projection: Av1DDFrameProjection? = null + + /** + * A boolean that records whether this frame was accepted, i.e. should be forwarded to the receiver + * given the decoding target currently being forwarded. + */ + var isAccepted = false + + // Validate that the index matches the pictureId + init { + assert((index and 0xffff).toInt() == frameNumber) + } + + constructor(packet: Av1DDPacket, index: Long) : this( + ssrc = packet.ssrc, + timestamp = packet.timestamp, + earliestKnownSequenceNumber = packet.sequenceNumber, + latestKnownSequenceNumber = packet.sequenceNumber, + seenStartOfFrame = packet.isStartOfFrame, + seenEndOfFrame = packet.isEndOfFrame, + seenMarker = packet.isMarked, + frameInfo = packet.frameInfo, + frameNumber = packet.statelessDescriptor.frameNumber, + index = index, + templateId = packet.statelessDescriptor.frameDependencyTemplateId, + structure = packet.descriptor?.structure, + activeDecodeTargets = packet.activeDecodeTargets, + isKeyframe = packet.isKeyframe, + rawDependencyDescriptor = if (packet.frameInfo == null) { + packet.getHeaderExtension(packet.av1DDHeaderExtensionId)?.clone() + } else { + null + } + ) + + /** + * Remember another packet of this frame. + * Note: this assumes every packet is received only once, i.e. a filter + * like [org.jitsi.nlj.transform.node.incoming.PaddingTermination] is in use. + * @param packet The packet to remember. This should be a packet which + * has tested true with [matchesFrame]. + */ + fun addPacket(packet: Av1DDPacket) { + require(matchesFrame(packet)) { "Non-matching packet added to frame" } + val seq = packet.sequenceNumber + if (seq isOlderThan earliestKnownSequenceNumber) { + earliestKnownSequenceNumber = seq + } + if (seq isNewerThan latestKnownSequenceNumber) { + latestKnownSequenceNumber = seq + } + if (packet.isStartOfFrame) { + seenStartOfFrame = true + } + if (packet.isEndOfFrame) { + seenEndOfFrame = true + } + if (packet.isMarked) { + seenMarker = true + } + + if (structure == null && packet.descriptor?.structure != null) { + structure = packet.descriptor?.structure + } + + if (frameInfo == null && packet.frameInfo != null) { + frameInfo = packet.frameInfo + } + } + + fun updateParse(templateDependencyStructure: Av1TemplateDependencyStructure, logger: Logger) { + if (rawDependencyDescriptor == null) { + return + } + val parser = Av1DependencyDescriptorReader(rawDependencyDescriptor) + val descriptor = try { + parser.parse(templateDependencyStructure) + } catch (e: Av1DependencyException) { + logger.warn("Could not parse updated AV1 Dependency Descriptor: ${e.message}") + return + } + structure = descriptor.structure + frameInfo = try { + descriptor.frameInfo + } catch (e: Av1DependencyException) { + logger.warn("Could not extract frame info from updated AV1 Dependency Descriptor: ${e.message}") + null + } + } + + /** + * Small utility method that checks whether the [Av1DDFrame] that is + * specified as a parameter belongs to the same RTP stream as the frame that + * this instance refers to. + * + * @param av1Frame the [Av1DDFrame] to check whether it belongs to the + * same RTP stream as the frame that this instance refers to. + * @return true if the [Av1DDFrame] that is specified as a parameter + * belongs to the same RTP stream as the frame that this instance refers to, + * false otherwise. + */ + fun matchesSSRC(av1Frame: Av1DDFrame): Boolean = ssrc == av1Frame.ssrc + + /** + * Checks whether the specified RTP packet is part of this frame. + * + * @param pkt the RTP packet to check whether it's part of this frame. + * @return true if the specified RTP packet is part of this frame, false + * otherwise. + */ + fun matchesFrame(pkt: Av1DDPacket): Boolean = ssrc == pkt.ssrc && timestamp == pkt.timestamp && + frameNumber == pkt.frameNumber + + fun validateConsistency(pkt: Av1DDPacket) { + if (frameInfo == null) { + return + } + if (frameInfo == pkt.frameInfo) { + return + } + + throw RuntimeException( + buildString { + with(pkt) { + append("Packet ssrc $ssrc, seq $sequenceNumber, frame number $frameNumber, timestamp $timestamp ") + append("packet template ${statelessDescriptor.frameDependencyTemplateId} ") + append("frame info $frameInfo ") + } + append("is not consistent with frame ${this@Av1DDFrame}") + } + ) + } + fun isImmediatelyAfter(otherFrame: Av1DDFrame): Boolean = frameNumber == + applySequenceNumberDelta(otherFrame.frameNumber, 1) + + override fun toString() = buildString { + append("$ssrc, ") + append("seq $earliestKnownSequenceNumber-$latestKnownSequenceNumber ") + append("frame number $frameNumber, timestamp $timestamp: ") + append("frame template $templateId info $frameInfo") + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt new file mode 100644 index 0000000..d848c05 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameMap.kt @@ -0,0 +1,246 @@ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.util.ArrayCache +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import kotlin.math.max +import kotlin.math.min + +/** + * A history of recent frames on a Av1 stream. + */ +class Av1DDFrameMap( + parentLogger: Logger +) { + /** Cache mapping frame IDs to frames. */ + private val frameHistory = FrameHistory(FRAME_MAP_SIZE) + private val logger: Logger = createChildLogger(parentLogger) + + /** Find a frame in the frame map, based on a packet. */ + @Synchronized + fun findFrame(packet: Av1DDPacket): Av1DDFrame? = frameHistory[packet.frameNumber] + + /** Get the current size of the map. */ + fun size(): Int = frameHistory.numCached + + /** Check whether this is a large jump from previous state, so the map should be reset. */ + private fun isLargeJump(packet: Av1DDPacket): Boolean { + val latestFrame: Av1DDFrame = frameHistory.latestFrame ?: return false + val picDelta = RtpUtils.getSequenceNumberDelta(packet.frameNumber, latestFrame.frameNumber) + if (picDelta > FRAME_MAP_SIZE) { + return true + } + val tsDelta: Long = RtpUtils.getTimestampDiff(packet.timestamp, latestFrame.timestamp) + if (picDelta < 0) { + /* if picDelta is negative but timestamp or sequence delta is positive, we've cycled. */ + if (tsDelta > 0) { + return true + } + if (packet.sequenceNumber isNewerThan latestFrame.latestKnownSequenceNumber) { + return true + } + } + + /* If tsDelta is more than twice the frame map size at 1 fps, we've cycled. */ + return tsDelta > FRAME_MAP_SIZE * 90000 * 2 + } + + /** Insert a packet into the frame map. Return a frameInsertionResult + * describing what happened. + * @param packet The packet to insert. + * @return What happened. null if insertion failed. + */ + @Synchronized + fun insertPacket(packet: Av1DDPacket): PacketInsertionResult? { + val frameNumber = packet.frameNumber + + if (isLargeJump(packet)) { + frameHistory.indexTracker.resetAt(frameNumber) + val frame = frameHistory.insert(packet) ?: return null + + return PacketInsertionResult(frame, true, isReset = true) + } + val frame = frameHistory[frameNumber] + if (frame != null) { + if (!frame.matchesFrame(packet)) { + check(frame.frameNumber == frameNumber) { + "frame map returned frame with frame number ${frame.frameNumber} " + "when asked for frame with frame ID $frameNumber" + } + logger.warn( + "Cannot insert packet in frame map: " + + with(frame) { + "frame with ssrc $ssrc, timestamp $timestamp, " + + "and sequence number range $earliestKnownSequenceNumber-$latestKnownSequenceNumber, " + } + + with(packet) { + "and packet $sequenceNumber with ssrc $ssrc, timestamp $timestamp, " + + "and sequence number $sequenceNumber" + } + + " both have frame ID $frameNumber" + ) + return null + } + try { + frame.validateConsistency(packet) + } catch (e: Exception) { + logger.warn(e) + } + + frame.addPacket(packet) + return PacketInsertionResult(frame, isNewFrame = false) + } + + val newframe = frameHistory.insert(packet) ?: return null + + return PacketInsertionResult(newframe, true) + } + + /** Insert a frame. Only used for unit testing. */ + @Synchronized + internal fun insertFrame(frame: Av1DDFrame) { + frameHistory.insert(frame) + } + + @Synchronized + fun getIndex(frameIndex: Long) = frameHistory.getIndex(frameIndex) + + @Synchronized + fun nextFrame(frame: Av1DDFrame): Av1DDFrame? = frameHistory.findAfter(frame) { true } + + @Synchronized + fun nextFrameWith(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? = + frameHistory.findAfter(frame, pred) + + @Synchronized + fun prevFrame(frame: Av1DDFrame): Av1DDFrame? = frameHistory.findBefore(frame) { true } + + @Synchronized + fun prevFrameWith(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? = + frameHistory.findBefore(frame, pred) + + fun findPrevAcceptedFrame(frame: Av1DDFrame): Av1DDFrame? = prevFrameWith(frame) { it.isAccepted } + + fun findNextAcceptedFrame(frame: Av1DDFrame): Av1DDFrame? = nextFrameWith(frame) { it.isAccepted } + + companion object { + const val FRAME_MAP_SIZE = 500 // Matches PacketCache default size. + } +} + +internal class FrameHistory(size: Int) : + ArrayCache( + size, + cloneItem = { k -> k }, + synchronize = false + ) { + var numCached = 0 + var firstIndex = -1L + var indexTracker = RtpSequenceIndexTracker() + + /** + * Gets a frame with a given frame number from the cache. + */ + operator fun get(frameNumber: Int): Av1DDFrame? { + val index = indexTracker.interpret(frameNumber) + return getIndex(index) + } + + /** + * Gets a frame with a given frame number index from the cache. + */ + fun getIndex(index: Long): Av1DDFrame? { + if (index <= lastIndex - size) { + /* We don't want to remember old frames even if they're still + tracked; their neighboring frames may have been evicted, + so findBefore / findAfter will return bogus data. */ + return null + } + val c = getContainer(index) ?: return null + return c.item + } + + /** Get the latest frame in the tracker. */ + val latestFrame: Av1DDFrame? + get() = getIndex(lastIndex) + + fun insert(frame: Av1DDFrame): Boolean { + val ret = super.insertItem(frame, frame.index) + if (ret) { + numCached++ + if (firstIndex == -1L || frame.index < firstIndex) { + firstIndex = frame.index + } + } + return ret + } + + fun insert(packet: Av1DDPacket): Av1DDFrame? { + val index = indexTracker.update(packet.frameNumber) + val frame = Av1DDFrame(packet, index) + return if (insert(frame)) frame else null + } + + /** + * Called when an item in the cache is replaced/discarded. + */ + override fun discardItem(item: Av1DDFrame) { + numCached-- + } + + fun findBefore(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? { + val lastIndex = lastIndex + if (lastIndex == -1L) { + return null + } + val index = frame.index + val searchStartIndex = min(index - 1, lastIndex) + val searchEndIndex = max(lastIndex - size, firstIndex - 1) + return doFind(pred, searchStartIndex, searchEndIndex, -1) + } + + fun findAfter(frame: Av1DDFrame, pred: (Av1DDFrame) -> Boolean): Av1DDFrame? { + val lastIndex = lastIndex + if (lastIndex == -1L) { + return null + } + val index = frame.index + if (index >= lastIndex) { + return null + } + val searchStartIndex = max(index + 1, max(lastIndex - size + 1, firstIndex)) + return doFind(pred, searchStartIndex, lastIndex + 1, 1) + } + + private fun doFind(pred: (Av1DDFrame) -> Boolean, startIndex: Long, endIndex: Long, increment: Int): Av1DDFrame? { + require((increment > 0 && startIndex <= endIndex) || (increment < 0 && startIndex >= endIndex)) { + "Values of startIndex=$startIndex, endIndex=$endIndex, and increment=$increment " + + "could lead to infinite loop" + } + var index = startIndex + while (index != endIndex) { + val frame = getIndex(index) + if (frame != null && pred(frame)) { + return frame + } + index += increment + } + return null + } +} + +/** + * The result of calling [insertPacket] + */ +class PacketInsertionResult( + /** The frame corresponding to the packet that was inserted. */ + val frame: Av1DDFrame, + /** Whether inserting the packet created a new frame. */ + val isNewFrame: Boolean, + /** Whether inserting the packet caused a reset */ + val isReset: Boolean = false +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt new file mode 100644 index 0000000..ab81601 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt @@ -0,0 +1,246 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.applyTemplateIdDelta +import org.jitsi.rtp.rtp.header_extensions.toShortString +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import java.time.Instant + +/** + * Represents an AV1 DD frame projection. It puts together all the necessary bits + * and pieces that are useful when projecting an accepted AV1 DD frame. A + * projection is responsible for rewriting a AV1 DD packet. Instances of this class + * are thread-safe. + */ +class Av1DDFrameProjection internal constructor( + /** + * The diagnostic context for this instance. + */ + private val diagnosticContext: DiagnosticContext, + /** + * The projected [Av1DDFrame]. + */ + val av1Frame: Av1DDFrame?, + /** + * The RTP SSRC of the projection (RFC7667, RFC3550). + */ + val ssrc: Long, + /** + * The RTP timestamp of the projection (RFC7667, RFC3550). + */ + val timestamp: Long, + /** + * The sequence number delta for packets of this frame. + */ + private val sequenceNumberDelta: Int, + /** + * The AV1 frame number of the projection. + */ + val frameNumber: Int, + /** + * The template ID delta for this frame. This applies both to the frame's own template ID, and to + * the template ID offset in the dependency structure, if present. + */ + val templateIdDelta: Int, + /** + * The decode target indication to set on this frame, if any. + */ + val dti: Int?, + /** + * Whether to add a marker bit to the last packet of this frame. + * (Note this will not clear already-existing marker bits.) + */ + val mark: Boolean, + /** + * A timestamp of when this instance was created. It's used to calculate + * RTP timestamps when we switch encodings. + */ + val created: Instant? +) { + + /** + * -1 if this projection is still "open" for new, later packets. + * Projections can be closed when we switch away from their encodings. + */ + var closedSeq = -1 + private set + + /** + * Ctor. + * + * @param ssrc the SSRC of the destination AV1 frame. + * @param timestamp The RTP timestamp of the projected frame that this + * instance refers to (RFC3550). + * @param sequenceNumberDelta The starting RTP sequence number of the + * projected frame that this instance refers to (RFC3550). + */ + internal constructor( + diagnosticContext: DiagnosticContext, + ssrc: Long, + sequenceNumberDelta: Int, + timestamp: Long, + frameNumber: Int?, + templateId: Int? + ) : this( + diagnosticContext = diagnosticContext, + av1Frame = null, + ssrc = ssrc, + timestamp = timestamp, + sequenceNumberDelta = sequenceNumberDelta, + frameNumber = frameNumber ?: 0, + templateIdDelta = templateId ?: -1, + dti = null, + mark = false, + created = null + ) + + fun rewriteSeqNo(seq: Int): Int = applySequenceNumberDelta(seq, sequenceNumberDelta) + + fun rewriteTemplateId(id: Int): Int = applyTemplateIdDelta(id, templateIdDelta) + + /** + * Rewrites an RTP packet. + * + * @param pkt the RTP packet to rewrite. + */ + fun rewriteRtp(pkt: Av1DDPacket) { + val sequenceNumber = rewriteSeqNo(pkt.sequenceNumber) + val templateId = rewriteTemplateId(pkt.statelessDescriptor.frameDependencyTemplateId) + if (timeSeriesLogger.isTraceEnabled) { + timeSeriesLogger.trace( + diagnosticContext + .makeTimeSeriesPoint("rtp_av1_rewrite") + .addField("orig.rtp.ssrc", pkt.ssrc) + .addField("orig.rtp.timestamp", pkt.timestamp) + .addField("orig.rtp.seq", pkt.sequenceNumber) + .addField("orig.av1.framenum", pkt.frameNumber) + .addField("orig.av1.dti", pkt.frameInfo?.dti?.toShortString() ?: "-") + .addField("orig.av1.templateid", pkt.statelessDescriptor.frameDependencyTemplateId) + .addField("proj.rtp.ssrc", ssrc) + .addField("proj.rtp.timestamp", timestamp) + .addField("proj.rtp.seq", sequenceNumber) + .addField("proj.av1.framenum", frameNumber) + .addField("proj.av1.dti", dti ?: -1) + .addField("proj.av1.templateid", templateId) + .addField("proj.rtp.mark", mark) + ) + } + + // update ssrc, sequence number, timestamp, frameNumber, and templateID + pkt.ssrc = ssrc + pkt.timestamp = timestamp + pkt.sequenceNumber = sequenceNumber + if (mark && pkt.isEndOfFrame) pkt.isMarked = true + + val descriptor = pkt.descriptor + if (descriptor != null && ( + frameNumber != pkt.frameNumber || templateId != descriptor.frameDependencyTemplateId || + dti != null + ) + ) { + descriptor.frameNumber = frameNumber + descriptor.frameDependencyTemplateId = templateId + val structure = descriptor.structure + check( + descriptor.newTemplateDependencyStructure == null || + descriptor.newTemplateDependencyStructure === descriptor.structure + ) + + structure.templateIdOffset = rewriteTemplateId(structure.templateIdOffset) + if (dti != null && ( + descriptor.newTemplateDependencyStructure == null || + dti != (1 shl structure.decodeTargetCount) - 1 + ) + ) { + descriptor.activeDecodeTargetsBitmask = dti + } + + pkt.descriptor = descriptor + pkt.reencodeDdExt() + } + } + + /** + * Determines whether a packet can be forwarded as part of this + * [Av1DDFrameProjection] instance. The check is based on the sequence + * of the incoming packet and whether or not the [Av1DDFrameProjection] + * has been "closed" or not. + * + * @param rtpPacket the [Av1DDPacket] that will be examined. + * @return true if the packet can be forwarded as part of this + * [Av1DDFrameProjection], false otherwise. + */ + fun accept(rtpPacket: Av1DDPacket): Boolean { + if (av1Frame?.matchesFrame(rtpPacket) != true) { + // The packet does not belong to this AV1 picture. + return false + } + synchronized(av1Frame) { + return if (closedSeq < 0) { + true + } else { + rtpPacket.sequenceNumber isOlderThan closedSeq + } + } + } + + val earliestProjectedSeqNum: Int + get() { + if (av1Frame == null) { + return sequenceNumberDelta + } + synchronized(av1Frame) { return rewriteSeqNo(av1Frame.earliestKnownSequenceNumber) } + } + + val latestProjectedSeqNum: Int + get() { + if (av1Frame == null) { + return sequenceNumberDelta + } + synchronized(av1Frame) { return rewriteSeqNo(av1Frame.latestKnownSequenceNumber) } + } + + /** + * Prevents the max sequence number of this frame to grow any further. + */ + fun close() { + if (av1Frame != null) { + synchronized(av1Frame) { closedSeq = av1Frame.latestKnownSequenceNumber } + } + } + + /** + * Get the next template ID that would come after the template IDs in this projection's structure + */ + fun getNextTemplateId(): Int? { + if (av1Frame == null && templateIdDelta != -1) { + return templateIdDelta + } + return av1Frame?.structure?.let { rewriteTemplateId(it.templateIdOffset + it.templateCount) } + } + + companion object { + /** + * The time series logger for this class. + */ + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(Av1DDFrameProjection::class.java) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt new file mode 100644 index 0000000..7af43f1 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt @@ -0,0 +1,470 @@ +/* + * Copyright @ 2019 - present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_ENCODING_ID +import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_INDEX +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.SUSPENDED_DT +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getDtFromIndex +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getIndex +import org.jitsi.nlj.rtp.codec.av1.containsDecodeTarget +import org.jitsi.rtp.rtp.header_extensions.DTI +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import java.time.Duration +import java.time.Instant + +/** + * This class is responsible for dropping AV1 simulcast/svc packets based on + * their quality, i.e. packets that correspond to qualities that are above a + * given quality target. Instances of this class are thread-safe. + */ +internal class Av1DDQualityFilter( + val av1FrameMap: Map, + parentLogger: Logger +) { + /** + * The [Logger] to be used by this instance to print debug + * information. + */ + private val logger: Logger = createChildLogger(parentLogger) + + /** + * Holds the arrival time of the most recent keyframe group. + * Reading/writing of this field is synchronized on this instance. + */ + private var mostRecentKeyframeGroupArrivalTime: Instant? = null + + /** + * A boolean flag that indicates whether a keyframe is needed, due to an + * encoding or (in some cases) a decode target switch. + */ + var needsKeyframe = false + private set + + /** + * The encoding ID that this instance tries to achieve. Upon + * receipt of a packet, we check whether encoding in the externalTargetIndex + * (that's specified as an argument to the + * [acceptFrame] method) is set to something different, + * in which case we set [needsKeyframe] equal to true and + * update. + */ + private var internalTargetEncoding = SUSPENDED_ENCODING_ID + + /** + * The decode target that this instance tries to achieve. + */ + private var internalTargetDt = SUSPENDED_DT + + /** + * The layer index that we're currently forwarding. [SUSPENDED_INDEX] + * indicates that we're not forwarding anything. Reading/writing of this + * field is synchronized on this instance. + */ + private var currentIndex = SUSPENDED_INDEX + + /** + * Determines whether to accept or drop an AV1 frame. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param frame the AV1 frame. + * @param incomingEncoding The encoding ID of the incoming packet + * @param externalTargetIndex the target quality index that the user of this + * instance wants to achieve. + * @param receivedTime the current time (as an Instant) + * @return true to accept the AV1 frame, otherwise false. + */ + @Synchronized + fun acceptFrame( + frame: Av1DDFrame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): AcceptResult { + val prevIndex = currentIndex + val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime) + val currentDt = getDtFromIndex(currentIndex) + val mark = currentDt != SUSPENDED_DT && + (frame.frameInfo?.spatialId == frame.structure?.decodeTargetLayers?.getOrNull(currentDt)?.spatialId) + val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX) + if (isResumption) { + check(accept) { + // Every code path that can turn off SUSPENDED_INDEX also accepts + "isResumption=$isResumption but accept=$accept for frame ${frame.frameNumber}, " + + "frameInfo=${frame.frameInfo}" + } + } + val dtChanged = (prevIndex != currentIndex) + if (dtChanged && currentDt != SUSPENDED_DT) { + check(accept) { + // Every code path that changes DT also accepts + "dtChanged=$dtChanged but accept=$accept for frame ${frame.frameNumber}, frameInfo=${frame.frameInfo}" + } + } + val newDt = if (dtChanged || frame.activeDecodeTargets != null) currentDt else null + return AcceptResult(accept = accept, isResumption = isResumption, mark = mark, newDt = newDt) + } + + private fun doAcceptFrame( + frame: Av1DDFrame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): Boolean { + val externalTargetEncoding = getEidFromIndex(externalTargetIndex) + val currentEncoding = getEidFromIndex(currentIndex) + + if (externalTargetEncoding != internalTargetEncoding) { + // The externalEncodingIdTarget has changed since accept last + // ran; perhaps we should request a keyframe. + internalTargetEncoding = externalTargetEncoding + if (externalTargetEncoding != SUSPENDED_ENCODING_ID && + externalTargetEncoding != currentEncoding + ) { + needsKeyframe = true + } + } + if (externalTargetEncoding == SUSPENDED_ENCODING_ID) { + // We stop forwarding immediately. We will need a keyframe in order + // to resume. + currentIndex = SUSPENDED_INDEX + return false + } + return if (frame.isKeyframe) { + logger.debug { + "Quality filter got keyframe for stream ${frame.ssrc}" + } + acceptKeyframe(frame, incomingEncoding, externalTargetIndex, receivedTime) + } else if (currentEncoding != SUSPENDED_ENCODING_ID) { + if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding)) { + // XXX(george) i've noticed some "rogue" base layer keyframes + // that trigger this. what happens is the client sends a base + // layer key frame, the bridge switches to that layer because + // for all it knows it may be the only keyframe sent by the + // client engine. then the bridge notices that packets from the + // higher quality streams are flowing and execution ends-up + // here. it is a mystery why the engine is "leaking" base layer + // key frames + needsKeyframe = true + } + if (incomingEncoding != currentEncoding) { + // for non-keyframes, we can't route anything but the current encoding + return false + } + + /** Logic to forward a non-keyframe: + * If the frame does not have FrameInfo, reject and set needsKeyframe (we couldn't decode its templates). + * If we're trying to switch DTs in the current encoding, check the template structure to ensure that + * there is at least one template which is SWITCH for the target DT and not NOT_PRESENT for the current DT. + * If there is not, request a keyframe. + * If the current frame is SWITCH for the target DT, and we've forwarded all the frames on which (by + * its fdiffs) it depends, forward it, and change the current DT to the target DT. + * In normal circumstances (when the target index == the current index), or when we're trying to switch + * *up* encodings, forward all frames whose DT for the current DT is not NOT_PRESENT. + * If we're trying to switch *down* encodings, only forward frames which are REQUIRED or SWITCH for the + * current DT. + */ + val frameInfo = frame.frameInfo ?: run { + needsKeyframe = true + return@doAcceptFrame false + } + var currentDt = getDtFromIndex(currentIndex) + val externalTargetDt = if (currentEncoding == externalTargetEncoding) { + getDtFromIndex(externalTargetIndex) + } else { + currentDt + } + + if ( + frame.activeDecodeTargets != null && + !frame.activeDecodeTargets.containsDecodeTarget(externalTargetDt) + ) { + /* This shouldn't happen, because we should have set layeringChanged for this packet. */ + logger.warn { + "External target DT $externalTargetDt not present in current decode targets 0x" + + Integer.toHexString(frame.activeDecodeTargets) + " for frame $frame." + } + return false + } + + if (currentDt != externalTargetDt) { + val frameMap = av1FrameMap[frame.ssrc] + if (frameInfo.dti.getOrNull(externalTargetDt) == null) { + logger.warn { "Target DT $externalTargetDt not present for frame $frame [frameInfo $frameInfo]" } + } + if (frameInfo.dti[externalTargetDt] == DTI.SWITCH && + frameMap != null && + frameInfo.fdiff.all { + frameMap.getIndex(frame.index - it)?.isAccepted == true + } + ) { + logger.debug { "Switching to DT $externalTargetDt from $currentDt" } + currentDt = externalTargetDt + currentIndex = externalTargetIndex + } else { + if (frame.structure?.canSwitchWithoutKeyframe(currentDt, externalTargetDt) != true) { + logger.debug { + "Want to switch to DT $externalTargetDt from $currentDt, requesting keyframe" + } + needsKeyframe = true + } + } + } + + val currentFrameDti = frameInfo.dti[currentDt] + if (currentEncoding > externalTargetEncoding) { + (currentFrameDti == DTI.SWITCH || currentFrameDti == DTI.REQUIRED) + } else { + (currentFrameDti != DTI.NOT_PRESENT) + } + } else { + // In this branch we're not processing a keyframe and the + // currentEncoding is in suspended state, which means we need + // a keyframe to start streaming again. + + // We should have already requested a keyframe, either above or when the + // internal target encoding was first moved off SUSPENDED_ENCODING. + false + } + } + + /** + * Returns a boolean that indicates whether we are in layer switching phase + * or not. + * + * @param receivedTime the time the latest frame was received + * @return false if we're in layer switching phase, true otherwise. + */ + @Synchronized + private fun isOutOfSwitchingPhase(receivedTime: Instant?): Boolean { + if (receivedTime == null) { + return false + } + if (mostRecentKeyframeGroupArrivalTime == null) { + return true + } + val delta = Duration.between(mostRecentKeyframeGroupArrivalTime, receivedTime) + return delta > MIN_KEY_FRAME_WAIT + } + + /** + * @return true if it looks like we can re-scale (see implementation of + * method for specific details). + */ + @Synchronized + private fun isPossibleToSwitch(incomingEncoding: Int): Boolean { + val currentEncoding = getEidFromIndex(currentIndex) + + if (incomingEncoding == SUSPENDED_ENCODING_ID) { + // We failed to resolve the spatial/quality layer of the packet. + return false + } + return when { + incomingEncoding > currentEncoding && currentEncoding < internalTargetEncoding -> + // It looks like upscaling is possible + true + + incomingEncoding < currentEncoding && currentEncoding > internalTargetEncoding -> + // It looks like downscaling is possible. + true + + else -> + false + } + } + + /** + * Determines whether to accept or drop an AV1 keyframe. This method updates + * the encoding id. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param receivedTime the time the frame was received + * @return true to accept the AV1 keyframe, otherwise false. + */ + @Synchronized + private fun acceptKeyframe( + frame: Av1DDFrame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): Boolean { + // This branch writes the {@link #currentSpatialLayerId} and it + // determines whether or not we should switch to another simulcast + // stream. + if (incomingEncoding < 0) { + // something went terribly wrong, normally we should be able to + // extract the layer id from a keyframe. + logger.error("unable to get layer id from keyframe") + return false + } + val frameInfo = frame.frameInfo ?: run { + // something went terribly wrong, normally we should be able to + // extract the frame info from a keyframe. + logger.error("unable to get frame info from keyframe") + return@acceptKeyframe false + } + logger.debug { + "Received a keyframe of encoding: $incomingEncoding" + } + + val currentEncoding = getEidFromIndex(currentIndex) + val externalTargetEncoding = getEidFromIndex(externalTargetIndex) + + val indexIfSwitched = when { + incomingEncoding == externalTargetEncoding -> externalTargetIndex + + incomingEncoding == internalTargetEncoding && internalTargetDt != -1 -> + getIndex(currentEncoding, internalTargetDt) + + else -> frameInfo.dtisPresent.maxOrNull()!! + } + val dtIfSwitched = getDtFromIndex(indexIfSwitched) + val dtiIfSwitched = frameInfo.dti[dtIfSwitched] + val acceptIfSwitched = dtiIfSwitched != DTI.NOT_PRESENT + + // The keyframe request has been fulfilled at this point, regardless of + // whether we'll be able to achieve the internalEncodingIdTarget. + needsKeyframe = false + return if (isOutOfSwitchingPhase(receivedTime)) { + // During the switching phase we always project the first + // keyframe because it may very well be the only one that we + // receive (i.e. the endpoint is sending low quality only). Then + // we try to approach the target. + mostRecentKeyframeGroupArrivalTime = receivedTime + logger.debug { + "First keyframe in this kf group " + + "currentEncodingId: $incomingEncoding. " + + "Target is $internalTargetEncoding" + } + if (incomingEncoding <= internalTargetEncoding) { + // If the target is 180p and the first keyframe of a group of + // keyframes is a 720p keyframe we don't project it. If we + // receive a 720p keyframe, we know that there MUST be a 180p + // keyframe shortly after. + if (acceptIfSwitched) { + currentIndex = indexIfSwitched + } + acceptIfSwitched + } else { + false + } + } else { + // We're within the 300ms window since the reception of the + // first key frame of a key frame group, let's check whether an + // upscale/downscale is possible. + when { + currentEncoding <= incomingEncoding && + incomingEncoding <= internalTargetEncoding -> { + // upscale or current quality case + if (acceptIfSwitched) { + currentIndex = indexIfSwitched + logger.debug { + "Upscaling to encoding $incomingEncoding. " + + "The target is $internalTargetEncoding" + } + } + acceptIfSwitched + } + + incomingEncoding <= internalTargetEncoding && + internalTargetEncoding < currentEncoding -> { + // downscale case + if (acceptIfSwitched) { + currentIndex = indexIfSwitched + logger.debug { + "Downscaling to encoding $incomingEncoding. " + + "The target is $internalTargetEncoding" + } + } + acceptIfSwitched + } + + else -> { + false + } + } + } + } + + /** + * Adds internal state to a diagnostic context time series point. + */ + @SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) { + pt.addField("qf.currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex)) + .addField("qf.internalTargetEncoding", internalTargetEncoding) + .addField("qf.internalTargetDt", internalTargetDt) + .addField("qf.needsKeyframe", needsKeyframe) + .addField( + "qf.mostRecentKeyframeGroupArrivalTimeMs", + mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 + ) + /* TODO any other fields necessary */ + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + @get:SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + val debugState: ObjectNode + get() { + val debugState = JsonNodeFactory.instance.objectNode() + debugState.put( + "mostRecentKeyframeGroupArrivalTimeMs", + mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 + ) + debugState.put("needsKeyframe", needsKeyframe) + debugState.put("internalTargetEncoding", internalTargetEncoding) + debugState.put("internalTargetDt", internalTargetDt) + debugState.put("currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex)) + return debugState + } + + data class AcceptResult( + val accept: Boolean, + val isResumption: Boolean, + val mark: Boolean, + val newDt: Int? + ) + + companion object { + /** + * The default maximum frequency at which the media engine + * generates key frame. + */ + private val MIN_KEY_FRAME_WAIT = Duration.ofMillis(300) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BandwidthProbingConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BandwidthProbingConfig.kt new file mode 100644 index 0000000..f63dd15 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BandwidthProbingConfig.kt @@ -0,0 +1,37 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.config + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import java.time.Duration + +class BandwidthProbingConfig private constructor() { + /** + * How often we check to send probing data + */ + val paddingPeriodMs: Long by config { + "org.jitsi.videobridge.PADDING_PERIOD_MS".from(JitsiConfig.legacyConfig) + "videobridge.cc.padding-period" + .from(JitsiConfig.newConfig).convertFrom { it.toMillis() } + } + + companion object { + @JvmField + val config = BandwidthProbingConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt new file mode 100644 index 0000000..72cd88d --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/config/BitrateControllerConfig.kt @@ -0,0 +1,142 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.config + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.metaconfig.optionalconfig +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorConfig +import org.jitsi.nlj.rtp.bandwidthestimation.BandwidthEstimatorEngine +import org.jitsi.nlj.rtp.bandwidthestimation.GoogleCcEstimator +import org.jitsi.nlj.rtp.bandwidthestimation2.GoogCcTransportCcEngine +import org.jitsi.nlj.util.Bandwidth +import java.time.Duration + +class BitrateControllerConfig private constructor() { + /** + * The bandwidth estimation threshold. + * + * In order to limit the resolution changes due to bandwidth changes we only react to bandwidth changes greater + * than {@code bweChangeThreshold * last_bandwidth_estimation}. + */ + val bweChangeThreshold: Double by config { + "org.jitsi.videobridge.BWE_CHANGE_THRESHOLD_PCT".from(JitsiConfig.legacyConfig) + .transformedBy { it / 100.0 } + // This is an old version, include for backward compat. + "videobridge.cc.bwe-change-threshold-pct".from(JitsiConfig.newConfig) + .transformedBy { it / 100.0 } + "videobridge.cc.bwe-change-threshold".from(JitsiConfig.newConfig) + } + + val initialMaxHeightPx: Int by config { + // Support the old property name if the user has overridden it. + "videobridge.cc.thumbnail-max-height-px".from(JitsiConfig.newConfig) + .softDeprecated("use videobridge.cc.initial-max-height-px") + "videobridge.cc.initial-max-height-px".from(JitsiConfig.newConfig) + } + val defaultMaxHeightPx: Int by config { + // Support the old property name if the user has overridden it. + "videobridge.cc.thumbnail-max-height-px".from(JitsiConfig.newConfig) + .softDeprecated("use videobridge.cc.default-max-height-px") + "videobridge.cc.default-max-height-px".from(JitsiConfig.newConfig) + } + + /** + * The default preferred resolution to allocate for the onstage participant, + * before allocating bandwidth for the thumbnails. + */ + val onstagePreferredHeightPx: Int by config { + "org.jitsi.videobridge.ONSTAGE_PREFERRED_HEIGHT".from(JitsiConfig.legacyConfig) + "videobridge.cc.onstage-preferred-height-px".from(JitsiConfig.newConfig) + } + + /** + * The preferred frame rate to allocate for the onstage participant. + */ + val onstagePreferredFramerate: Double by config { + "org.jitsi.videobridge.ONSTAGE_PREFERRED_FRAME_RATE".from(JitsiConfig.legacyConfig) + "videobridge.cc.onstage-preferred-framerate".from(JitsiConfig.newConfig) + } + + /** + * Whether or not we are allowed to oversend (exceed available bandwidth) for the video of the on-stage + * participant. + */ + val allowOversendOnStage: Boolean by config { + "org.jitsi.videobridge.ENABLE_ONSTAGE_VIDEO_SUSPEND".from(JitsiConfig.legacyConfig).transformedBy { !it } + "videobridge.cc.enable-onstage-video-suspend".from(JitsiConfig.newConfig).transformedBy { !it } + "videobridge.cc.allow-oversend-onstage".from(JitsiConfig.newConfig) + } + + /** + * The maximum bitrate by which the bridge may exceed the estimated available bandwidth when oversending. + */ + val maxOversendBitrate: Bandwidth by config { + "videobridge.cc.max-oversend-bitrate".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + + /** + * Whether or not we should trust the bandwidth + * estimations. If this is se to false, then we assume a bandwidth + * estimation of Long.MAX_VALUE. + */ + val trustBwe: Boolean by config { + "org.jitsi.videobridge.TRUST_BWE".from(JitsiConfig.legacyConfig) + "videobridge.cc.trust-bwe".from(JitsiConfig.newConfig) + } + + /** + * The maximum amount of time we'll run before recalculating which streams we'll + * forward. + */ + val maxTimeBetweenCalculations: Duration by config( + "videobridge.cc.max-time-between-calculations".from(JitsiConfig.newConfig) + ) + + val assumedBandwidthLimit: Bandwidth? by optionalconfig { + "videobridge.cc.assumed-bandwidth-limit".from(JitsiConfig.newConfig) + .convertFrom { Bandwidth.fromString(it) } + } + + val useVlaTargetBitrate: Boolean by config { + "videobridge.cc.use-vla-target-bitrate".from(JitsiConfig.newConfig) + } + + private val _initialIgnoreBwePeriod: Duration? by optionalconfig( + "videobridge.cc.initial-ignore-bwe-period".from(JitsiConfig.newConfig) + ) + + /** The default initial ignore BWE period if not set in jvb.conf, based on the BWE algorithm in use.*/ + private fun defaultInitialIgnoreBwePeriod(): Duration = when (BandwidthEstimatorConfig.engine) { + BandwidthEstimatorEngine.GoogleCc -> GoogleCcEstimator.defaultInitialIgnoreBwePeriod + BandwidthEstimatorEngine.GoogleCc2 -> GoogCcTransportCcEngine.defaultInitialIgnoreBwePeriod + } + + /** + * How long after first media to ignore bandwidth estimation. Needed for BWE + * algorithms that ramp up slowly; should be set to zero if this isn't a problem. + */ + val initialIgnoreBwePeriod: Duration + get() = _initialIgnoreBwePeriod ?: defaultInitialIgnoreBwePeriod() + + companion object { + @JvmField + val config = BitrateControllerConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt new file mode 100644 index 0000000..1a577cf --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt @@ -0,0 +1,636 @@ +/* + * Copyright @ 2019-present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc.Companion.indexString +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyExtendedPictureIdDelta +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyTl0PicIdxDelta +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getExtendedPictureIdDelta +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getTl0PicIdxDelta +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.RtpUtils.Companion.applyTimestampDelta +import org.jitsi.rtp.util.RtpUtils.Companion.getSequenceNumberDelta +import org.jitsi.rtp.util.RtpUtils.Companion.getTimestampDiff +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.cc.AdaptiveSourceProjectionContext +import org.jitsi.videobridge.cc.RewriteException +import org.jitsi.videobridge.cc.RtpState +import java.time.Duration +import java.time.Instant + +/** + * This class represents a projection of a VP9 RTP stream + * and it is the main entry point for VP9 simulcast/svc RTP/RTCP rewriting. Read + * svc.md for implementation details. Instances of this class are thread-safe. + */ +class Vp9AdaptiveSourceProjectionContext( + private val diagnosticContext: DiagnosticContext, + rtpState: RtpState, + parentLogger: Logger +) : AdaptiveSourceProjectionContext { + private val logger: Logger = createChildLogger(parentLogger) + + /** + * A map that stores the per-encoding VP9 picture maps. + */ + private val vp9PictureMaps = HashMap() + + /** + * The [Vp9QualityFilter] instance that does quality filtering on the + * incoming pictures, to choose encodings and layers to forward. + */ + private val vp9QualityFilter = Vp9QualityFilter(logger) + + private var lastVp9FrameProjection = Vp9FrameProjection( + diagnosticContext, + rtpState.ssrc, + rtpState.maxSequenceNumber, + rtpState.maxTimestamp + ) + + /** + * The picture ID index that started the latest stream resumption. + * We can't send frames with picIdIdx less than this, because we don't have + * space in the projected sequence number/picId/tl0PicIdx counts. + */ + private var lastPicIdIndexResumption = -1L + + @Synchronized + override fun accept(packetInfo: PacketInfo, targetIndex: Int): Boolean { + val packet = packetInfo.packet + if (packet !is Vp9Packet) { + logger.warn("Packet is not Vp9 packet") + return false + } + val incomingEncoding = packet.encodingId + + /* If insertPacketInMap returns null, this is a very old picture, more than Vp9PictureMap.PICTURE_MAP_SIZE old, + or something is wrong with the stream. */ + val result = insertPacketInMap(packet) ?: return false + + val frame = result.frame + + if (result.isNewFrame) { + if (packet.isKeyframe && frameIsNewSsrc(frame)) { + /* If we're not currently projecting this SSRC, check if we've + * already decided to drop a subsequent base TL0 frame of this SSRC. + * If we have, we can't turn on the encoding starting from this + * packet, so treat this frame as though it weren't a keyframe. + */ + val f: Vp9Frame? = findNextBaseTl0(frame) + if (f != null && !f.isAccepted) { + frame.isKeyframe = false + } + } + val receivedTime = packetInfo.receivedTime + val acceptResult = vp9QualityFilter + .acceptFrame(frame, incomingEncoding, targetIndex, receivedTime) + frame.isAccepted = acceptResult.accept && frameIsProjectable(frame) + if (frame.isAccepted) { + val projection: Vp9FrameProjection + try { + projection = createProjection( + frame = frame, + initialPacket = packet, + isResumption = acceptResult.isResumption, + isReset = result.isReset, + mark = acceptResult.mark, + receivedTime = receivedTime + ) + } catch (e: Exception) { + logger.warn("Failed to create frame projection", e) + /* Make sure we don't have an accepted frame without a projection in the map. */ + frame.isAccepted = false + return false + } + frame.projection = projection + if (projection.earliestProjectedSeqNum isNewerThan lastVp9FrameProjection.latestProjectedSeqNum) { + lastVp9FrameProjection = projection + } + } + } + + val accept = frame.isAccepted && + if (frame.projection?.accept(packet) == true) { + true + } else { + if (frame.projection != null && frame.projection?.closedSeq != -1) { + logger.debug( + "Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}" + ) + } else if (frame.projection == null) { + logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it") + } else { + logger.warn("Not accepting $packet, even though frame projection is not closed") + } + false + } + + if (timeSeriesLogger.isTraceEnabled) { + val pt = diagnosticContext.makeTimeSeriesPoint("rtp_vp9") + .addField("ssrc", packet.ssrc) + .addField("timestamp", packet.timestamp) + .addField("seq", packet.sequenceNumber) + .addField("pictureId", packet.pictureId) + .addField("pictureIdIndex", frame.index) + .addField("encoding", incomingEncoding) + .addField("keyframe", packet.isKeyframe) + .addField("spatialLayer", packet.spatialLayerIndex) + .addField("temporalLayer", packet.temporalLayerIndex) + .addField("isInterPicturePredicted", packet.isInterPicturePredicted) + .addField("usesInterLayerDependency", packet.usesInterLayerDependency) + .addField("isUpperLevelReference", packet.isUpperLevelReference) + .addField("startOfFrame", packet.isStartOfFrame) + .addField("endOfFrame", packet.isEndOfFrame) + .addField("mark", packet.isMarked) + .addField("targetIndex", indexString(targetIndex)) + .addField("new_frame", result.isNewFrame) + .addField("reset", result.isReset) + .addField("accept", accept) + vp9QualityFilter.addDiagnosticContext(pt) + timeSeriesLogger.trace(pt) + } + + return accept + } + + /** Look up a Vp9Frame for a packet. */ + private fun lookupVp9Frame(vp9Packet: Vp9Packet): Vp9Frame? = + vp9PictureMaps[vp9Packet.ssrc]?.findPicture(vp9Packet)?.frame(vp9Packet) + + /** + * Insert a packet in the appropriate Vp9FrameMap. + */ + private fun insertPacketInMap(vp9Packet: Vp9Packet) = + vp9PictureMaps.getOrPut(vp9Packet.ssrc) { Vp9PictureMap(logger) }.insertPacket(vp9Packet) + + /** + * Calculate the projected sequence number gap between two frames (of the same encoding), + * allowing collapsing for unaccepted frames. + */ + private fun seqGap(frame1: Vp9Frame, frame2: Vp9Frame): Int { + var seqGap = getSequenceNumberDelta( + frame2.earliestKnownSequenceNumber, + frame1.latestKnownSequenceNumber + ) + if (!frame1.isAccepted && !frame2.isAccepted && frame2.isImmediatelyAfter(frame1)) { + /* If neither frame is being projected, and they have consecutive + picture IDs, we don't need to leave any gap. */ + seqGap = 0 + } else { + /* If the earlier frame wasn't projected, and we haven't seen its + * final packet, we know it has to consume at least one more sequence number. */ + if (!frame1.isAccepted && !frame1.seenEndOfFrame && seqGap > 1) { + seqGap-- + } + /* Similarly, if the later frame wasn't projected and we haven't seen + * its first packet. */ + if (!frame2.isAccepted && !frame2.seenStartOfFrame && seqGap > 1) { + seqGap-- + } + /* If the frame wasn't accepted, it has to have consumed at least one sequence number, + * which we can collapse out. */ + if (!frame1.isAccepted && seqGap > 0) { + seqGap-- + } + } + return seqGap + } + + private fun frameIsNewSsrc(frame: Vp9Frame): Boolean = lastVp9FrameProjection.vp9Frame?.matchesSSRC(frame) != true + + private fun frameIsProjectable(frame: Vp9Frame): Boolean = + frameIsNewSsrc(frame) || frame.index >= lastPicIdIndexResumption + + /** + * Find the previous frame before the given one. + */ + @Synchronized + private fun prevFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.prevFrame(frame) + + /** + * Find the next frame after the given one. + */ + @Synchronized + private fun nextFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.nextFrame(frame) + + /** + * Find the previous accepted frame before the given one. + */ + private fun findPrevAcceptedFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.findPrevAcceptedFrame(frame) + + /** + * Find the next accepted frame after the given one. + */ + private fun findNextAcceptedFrame(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.findNextAcceptedFrame(frame) + + /** + * Find a subsequent base-layer TL0 frame after the given frame + * @param frame The frame to query + * @return A subsequent base-layer TL0 frame, or null + */ + private fun findNextBaseTl0(frame: Vp9Frame) = vp9PictureMaps.get(frame.ssrc)?.findNextBaseTl0(frame) + + /** + * Create a projection for this frame. + */ + private fun createProjection( + frame: Vp9Frame, + initialPacket: Vp9Packet, + mark: Boolean, + isResumption: Boolean, + isReset: Boolean, + receivedTime: Instant? + ): Vp9FrameProjection { + if (frameIsNewSsrc(frame)) { + return createEncodingSwitchProjection(frame, initialPacket, mark, receivedTime) + } else if (isReset) { + return createResetProjection(frame, initialPacket, mark, receivedTime) + } else if (isResumption) { + return createResumptionProjection(frame, initialPacket, mark, receivedTime) + } + + return createInEncodingProjection(frame, initialPacket, mark, receivedTime) + } + + /** + * Create a projection for the first frame after an encoding switch. + */ + private fun createEncodingSwitchProjection( + frame: Vp9Frame, + initialPacket: Vp9Packet, + mark: Boolean, + receivedTime: Instant? + ): Vp9FrameProjection { + assert(frame.isKeyframe) + lastPicIdIndexResumption = frame.index + + var projectedSeqGap = if (!initialPacket.isStartOfFrame) { + val f = prevFrame(frame) + if (f != null) { + /* Leave enough of a gap to fill in the earlier packets of the keyframe */ + seqGap(f, frame) + } else { + /* If this is the first packet we've seen on this encoding, and it's not the start of a frame, + * we have a problem - we don't know where this packet might fall in the frame. + * Guess a reasonably-sized guess for the number of packets that might have been dropped, and hope + * we don't end up reusing sequence numbers. + */ + 16 + } + } else { + 1 + } + + if (lastVp9FrameProjection.vp9Frame?.seenEndOfFrame == false) { + /* Leave a gap to signal to the decoder that the previously routed + frame was incomplete. */ + projectedSeqGap++ + + /* Make sure subsequent packets of the previous projection won't + overlap the new one. (This means the gap, above, will never be + filled in.) + */ + lastVp9FrameProjection.close() + } + + val projectedSeq = applySequenceNumberDelta(lastVp9FrameProjection.latestProjectedSeqNum, projectedSeqGap) + + // this is a simulcast switch. The typical incremental value = + // 90kHz / 30 = 90,000Hz / 30 = 3000 per frame or per 33ms + val tsDelta: Long = lastVp9FrameProjection.created?.let { created -> + receivedTime?.let { + 3000 * Math.max(1, Duration.between(created, receivedTime).dividedBy(33).seconds) + } + } ?: run { + 3000 + } + val projectedTs = applyTimestampDelta(lastVp9FrameProjection.timestamp, tsDelta) + + val picId: Int + val tl0PicIdx: Int + if (lastVp9FrameProjection.vp9Frame != null) { + picId = applyExtendedPictureIdDelta( + lastVp9FrameProjection.pictureId, + 1 + ) + tl0PicIdx = applyTl0PicIdxDelta( + lastVp9FrameProjection.tl0PICIDX, + 1 + ) + } else { + picId = frame.pictureId + tl0PicIdx = frame.tl0PICIDX + } + + return Vp9FrameProjection( + diagnosticContext = diagnosticContext, + vp9Frame = frame, + ssrc = lastVp9FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber), + pictureId = picId, + tl0PICIDX = tl0PicIdx, + mark = mark, + created = receivedTime + ) + } + + /** + * Create a projection for the first frame after a resumption, i.e. when a source is turned back on. + */ + private fun createResumptionProjection( + frame: Vp9Frame, + initialPacket: Vp9Packet, + mark: Boolean, + receivedTime: Instant? + ): Vp9FrameProjection { + lastPicIdIndexResumption = frame.index + + /* These must be non-null because we don't execute this function unless + frameIsNewSsrc and isReset were both false. + */ + val lastFrame = prevFrame(frame)!! + val lastProjectedFrame = lastVp9FrameProjection.vp9Frame!! + + /* Project timestamps linearly. */ + val tsDelta = getTimestampDiff( + lastVp9FrameProjection.timestamp, + lastProjectedFrame.timestamp + ) + val projectedTs = applyTimestampDelta(frame.timestamp, tsDelta) + + /* Increment picId and tl0picidx by 1 from the last projected frame. */ + val projectedPicId = applyExtendedPictureIdDelta(lastVp9FrameProjection.pictureId, 1) + val projectedTl0PicIdx = applyTl0PicIdxDelta(lastVp9FrameProjection.tl0PICIDX, 1) + + /* Increment sequence numbers based on the last projected frame, but leave a gap + * for packet reordering in case this isn't the first packet of the keyframe. + */ + val seqGap = getSequenceNumberDelta(initialPacket.sequenceNumber, lastFrame.latestKnownSequenceNumber) + val newSeq = applySequenceNumberDelta(lastVp9FrameProjection.latestProjectedSeqNum, seqGap) + val seqDelta = getSequenceNumberDelta(newSeq, initialPacket.sequenceNumber) + + return Vp9FrameProjection( + diagnosticContext, + frame, lastVp9FrameProjection.ssrc, projectedTs, + seqDelta, + projectedPicId, projectedTl0PicIdx, mark, receivedTime + ) + } + + /** + * Create a projection for the first frame after a frame reset, i.e. after a large gap in sequence numbers. + */ + private fun createResetProjection( + frame: Vp9Frame, + initialPacket: Vp9Packet, + mark: Boolean, + receivedTime: Instant? + ): Vp9FrameProjection { + /* This must be non-null because we don't execute this function unless + frameIsNewSsrc has returned false. + */ + val lastFrame = lastVp9FrameProjection.vp9Frame!! + + /* Apply the latest projected frame's projections out, linearly. */ + val seqDelta = getSequenceNumberDelta( + lastVp9FrameProjection.latestProjectedSeqNum, + lastFrame.latestKnownSequenceNumber + ) + val tsDelta = getTimestampDiff( + lastVp9FrameProjection.timestamp, + lastFrame.timestamp + ) + val picIdDelta = getExtendedPictureIdDelta( + lastVp9FrameProjection.pictureId, + lastFrame.pictureId + ) + val tl0PicIdxDelta = getTl0PicIdxDelta( + lastVp9FrameProjection.tl0PICIDX, + lastFrame.tl0PICIDX + ) + + val projectedTs = applyTimestampDelta(frame.timestamp, tsDelta) + val projectedPicId = applyExtendedPictureIdDelta(frame.pictureId, picIdDelta) + val projectedTl0PicIdx = applyTl0PicIdxDelta(frame.tl0PICIDX, tl0PicIdxDelta) + + return Vp9FrameProjection( + diagnosticContext, + frame, lastVp9FrameProjection.ssrc, projectedTs, + seqDelta, + projectedPicId, projectedTl0PicIdx, mark, receivedTime + ) + } + + /** + * Create a frame projection for the normal case, i.e. as part of the same encoding as the + * previously-projected frame. + */ + private fun createInEncodingProjection( + frame: Vp9Frame, + initialPacket: Vp9Packet, + mark: Boolean, + receivedTime: Instant? + ): Vp9FrameProjection { + val prevFrame = findPrevAcceptedFrame(frame) + if (prevFrame != null) { + return createInEncodingProjection(frame, prevFrame, initialPacket, mark, receivedTime) + } + + /* prev frame has rolled off beginning of frame map, try next frame */ + val nextFrame = findNextAcceptedFrame(frame) + if (nextFrame != null) { + return createInEncodingProjection(frame, nextFrame, initialPacket, mark, receivedTime) + } + + /* Neither previous or next is found. Very big frame? Use previous projected. + (This must be valid because we don't execute this function unless + frameIsNewSsrc has returned false.) + */ + return createInEncodingProjection( + frame, + lastVp9FrameProjection.vp9Frame!!, + initialPacket, + mark, + receivedTime + ) + } + + /** + * Create a frame projection for the normal case, i.e. as part of the same encoding as the + * previously-projected frame, based on a specific chosen previously-projected frame. + */ + private fun createInEncodingProjection( + frame: Vp9Frame, + refFrame: Vp9Frame, + initialPacket: Vp9Packet, + mark: Boolean, + receivedTime: Instant? + ): Vp9FrameProjection { + val tsGap = getTimestampDiff(frame.timestamp, refFrame.timestamp) + val tl0Gap = getTl0PicIdxDelta(frame.tl0PICIDX, refFrame.tl0PICIDX) + val picGap = getExtendedPictureIdDelta(frame.pictureId, refFrame.pictureId) + val layerGap = frame.spatialLayer - refFrame.spatialLayer + var seqGap = 0 + + var f1 = refFrame + var f2: Vp9Frame? + val refSeq: Int + if (picGap > 0 || (picGap == 0 && layerGap > 0)) { + /* refFrame is earlier than frame in decode order. */ + do { + f2 = nextFrame(f1) + checkNotNull(f2) { + "No next frame found after frame with picId ${f1.pictureId} layer ${f1.spatialLayer}, " + + "even though refFrame ${refFrame.pictureId}/${refFrame.spatialLayer} is before " + + "frame ${frame.pictureId}/${frame.spatialLayer}!" + } + seqGap += seqGap(f1, f2) + f1 = f2 + } while (f2 !== frame) + /* refFrame is a projected frame, so it has a projection. */ + refSeq = refFrame.projection!!.latestProjectedSeqNum + } else { + /* refFrame is later than frame in decode order. */ + do { + f2 = prevFrame(f1) + checkNotNull(f2) { + "No previous frame found before frame with picId ${f1.pictureId} layer ${f1.spatialLayer}, " + + "even though refFrame ${refFrame.pictureId}/${refFrame.spatialLayer} is after " + + "frame ${frame.pictureId}/${frame.spatialLayer}!" + } + seqGap += -seqGap(f2, f1) + f1 = f2 + } while (f2 !== frame) + refSeq = refFrame.projection!!.earliestProjectedSeqNum + } + + val projectedSeq = applySequenceNumberDelta(refSeq, seqGap) + val projectedTs = applyTimestampDelta(refFrame.projection!!.timestamp, tsGap) + val projectedPicId = applyExtendedPictureIdDelta(refFrame.projection!!.pictureId, picGap) + val projectedTl0PicIdx = applyTl0PicIdxDelta(refFrame.projection!!.tl0PICIDX, tl0Gap) + + return Vp9FrameProjection( + diagnosticContext = diagnosticContext, + vp9Frame = frame, + ssrc = lastVp9FrameProjection.ssrc, + timestamp = projectedTs, + sequenceNumberDelta = getSequenceNumberDelta(projectedSeq, initialPacket.sequenceNumber), + pictureId = projectedPicId, + tl0PICIDX = projectedTl0PicIdx, + mark = mark, + created = receivedTime + ) + } + + override fun needsKeyframe(): Boolean { + if (vp9QualityFilter.needsKeyframe) { + return true + } + + return lastVp9FrameProjection.vp9Frame == null + } + + @Throws(RewriteException::class) + override fun rewriteRtp(packetInfo: PacketInfo) { + if (packetInfo.packet !is Vp9Packet) { + logger.info("Got a non-VP9 packet.") + throw RewriteException("Non-VP9 packet in VP9 source projection") + } + val vp9Packet = packetInfo.packetAs() + + if (vp9Packet.pictureId == -1) { + /* Should have been routed to generic projection context. */ + logger.info("VP9 packet does not have picture ID, cannot track in frame map.") + throw RewriteException("VP9 packet without picture ID in VP9 source projection") + } + + val vp9Frame: Vp9Frame = lookupVp9Frame(vp9Packet) + ?: throw RewriteException("Frame not in tracker (aged off?)") + + val vp9Projection = vp9Frame.projection + ?: throw RewriteException("Frame does not have projection?") + /* Shouldn't happen for an accepted packet whose frame is still known? */ + + vp9Projection.rewriteRtp(vp9Packet) + } + + override fun rewriteRtcp(rtcpSrPacket: RtcpSrPacket): Boolean { + val lastVp9FrameProjectionCopy: Vp9FrameProjection = lastVp9FrameProjection + if (rtcpSrPacket.senderSsrc != lastVp9FrameProjectionCopy.vp9Frame?.ssrc) { + return false + } + + rtcpSrPacket.senderSsrc = lastVp9FrameProjectionCopy.ssrc + + val srcTs = rtcpSrPacket.senderInfo.rtpTimestamp + val delta = getTimestampDiff( + lastVp9FrameProjectionCopy.timestamp, + lastVp9FrameProjectionCopy.vp9Frame.timestamp + ) + + val dstTs = applyTimestampDelta(srcTs, delta) + + if (srcTs != dstTs) { + rtcpSrPacket.senderInfo.rtpTimestamp = dstTs + } + + return true + } + + override fun getRtpState() = RtpState( + lastVp9FrameProjection.ssrc, + lastVp9FrameProjection.latestProjectedSeqNum, + lastVp9FrameProjection.timestamp + ) + + @Synchronized + override fun getDebugState(): ObjectNode { + val debugState = JsonNodeFactory.instance.objectNode() + debugState.put("class", Vp9AdaptiveSourceProjectionContext::class.java.simpleName) + + val mapSizes: ArrayNode = JsonNodeFactory.instance.arrayNode() + for ((key, value) in vp9PictureMaps.entries) { + val sizeInfo = JsonNodeFactory.instance.objectNode() + sizeInfo.put("ssrc", key) + sizeInfo.put("size", value.size()) + mapSizes.add(sizeInfo) + } + debugState.set("vp9FrameMaps", mapSizes) + debugState.set("vp9QualityFilter", vp9QualityFilter.debugState) + + return debugState + } + + companion object { + /** + * The time series logger for this class. + */ + private val timeSeriesLogger = + TimeSeriesLogger.getTimeSeriesLogger(Vp9AdaptiveSourceProjectionContext::class.java) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt new file mode 100644 index 0000000..c4237fc --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Frame.kt @@ -0,0 +1,400 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getExtendedPictureIdDelta +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isOlderThan + +/** + * Groups together some RTP/VP9 fields that refer to a specific incoming VP9 + * frame. Most of those fields are final and cannot be changed, with the + * exception of the ending and max sequence number that may be unknown at the + * time of the creation of this instance. + * + * Instances of this class are *NOT* thread safe. While most internal state of + * this class instances is final, the sequence number ranges, haveStart/haveEnd, + * and isKeyframe are not. + * + * @author Jonathan Lennox + */ +class Vp9Frame internal constructor( + /** + * The RTP SSRC of the incoming frame that this instance refers to + * (RFC3550). + */ + val ssrc: Long, + + /** + * The RTP timestamp of the incoming frame that this instance refers to + * (RFC3550). + */ + val timestamp: Long, + + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + earliestKnownSequenceNumber: Int, + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + latestKnownSequenceNumber: Int, + + /** + * A boolean that indicates whether or not we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + seenStartOfFrame: Boolean, + + /** + * A boolean that indicates whether or not we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + seenEndOfFrame: Boolean, + + /** + * A boolean that indicates whether we've seen a packet with the marker bit set. + */ + seenMarker: Boolean, + + /** + * The temporal layer of this frame. + */ + val temporalLayer: Int, + + /** + * The spatial layer of this frame. + */ + val spatialLayer: Int, + + /** + * Whether the frame is used as an upper level reference. + */ + val isUpperLevelReference: Boolean, + + /** + * Whether the frame is a temporal switching-up point. + */ + val isSwitchingUpPoint: Boolean, + + /** + * Whether the frame uses inter-layer dependency. + */ + val usesInterLayerDependency: Boolean, + + /** + * Whether the frame is inter-picture predicted. + */ + val isInterPicturePredicted: Boolean, + + /** + * The VP9 PictureID of the incoming VP9 frame that this instance refers to. + */ + val pictureId: Int, + + /** + * The PictureID index (PictureID plus cycles) of this frame. + */ + val index: Long, + + /** + * The VP9 TL0PICIDX of the incoming VP9 frame that this instance refers to + * (RFC7741). + */ + val tl0PICIDX: Int, + + /** + * A boolean that indicates whether the incoming VP9 frame that this + * instance refers to is a keyframe. + */ + var isKeyframe: Boolean, + + /** + * The number of spatial layers reported by this frame's scalability structure, + * if it has one, otherwise -1. + */ + var numSpatialLayers: Int +) { + /** + * The earliest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + var earliestKnownSequenceNumber = earliestKnownSequenceNumber + private set + + /** + * The latest RTP sequence number seen of the incoming frame that this instance + * refers to (RFC3550). + */ + var latestKnownSequenceNumber: Int = latestKnownSequenceNumber + private set + + /** + * A boolean that indicates whether or not we've seen the first packet of the frame. + * If so, its sequence is earliestKnownSequenceNumber. + */ + var seenStartOfFrame: Boolean = seenStartOfFrame + private set + + /** + * A boolean that indicates whether or not we've seen the last packet of the frame. + * If so, its sequence is latestKnownSequenceNumber. + */ + var seenEndOfFrame: Boolean = seenEndOfFrame + private set + + /** + * A boolean that indicates whether we've seen a packet with the marker bit set. + */ + var seenMarker: Boolean = seenMarker + private set + + /** + * A record of how this frame was projected, or null if not. + */ + var projection: Vp9FrameProjection? = null + + /** + * A boolean that records whether this frame was accepted, i.e. should be forwarded to the receiver + * given the layer currently being forwarded. + */ + var isAccepted = false + + /** + * The "effective" spatial layer of the frame, i.e. the spatial layer, or 0 if the frame does not have layer information. + */ + val effectiveSpatialLayer: Int + get() = if (spatialLayer >= 0) spatialLayer else 0 + + // Validate that the index matches the pictureId + init { + assert((index and 0x7fff).toInt() == pictureId) + } + + constructor(packet: Vp9Packet, index: Long) : this( + ssrc = packet.ssrc, + timestamp = packet.timestamp, + earliestKnownSequenceNumber = packet.sequenceNumber, + latestKnownSequenceNumber = packet.sequenceNumber, + seenStartOfFrame = packet.isStartOfFrame, + seenEndOfFrame = packet.isEndOfFrame, + seenMarker = packet.isMarked, + temporalLayer = packet.temporalLayerIndex, + spatialLayer = packet.spatialLayerIndex, + isUpperLevelReference = packet.isUpperLevelReference, + isSwitchingUpPoint = packet.isSwitchingUpPoint, + usesInterLayerDependency = packet.usesInterLayerDependency, + isInterPicturePredicted = packet.isInterPicturePredicted, + pictureId = packet.pictureId, + index = index, + tl0PICIDX = packet.TL0PICIDX, + isKeyframe = packet.isKeyframe, + numSpatialLayers = packet.scalabilityStructureNumSpatial + ) + + /** + * Remember another packet of this frame. + * Note: this assumes every packet is received only once, i.e. a filter + * like [org.jitsi.nlj.transform.node.incoming.PaddingTermination] is in use. + * @param packet The packet to remember. This should be a packet which + * has tested true with [matchesFrame]. + */ + fun addPacket(packet: Vp9Packet) { + require(matchesFrame(packet)) { "Non-matching packet added to frame" } + val seq = packet.sequenceNumber + if (seq isOlderThan earliestKnownSequenceNumber) { + earliestKnownSequenceNumber = seq + } + if (seq isNewerThan latestKnownSequenceNumber) { + latestKnownSequenceNumber = seq + } + if (packet.isStartOfFrame) { + seenStartOfFrame = true + } + if (packet.isEndOfFrame) { + seenEndOfFrame = true + } + if (packet.isMarked) { + seenMarker = true + } + if (packet.hasScalabilityStructure) { + numSpatialLayers = packet.scalabilityStructureNumSpatial + } + } + + /** + * @return true if this is a base temporal layer frame, false otherwise + * @note We treat unknown temporal layer frames as TL0. + */ + val isTL0: Boolean + get() = temporalLayer <= 0 + + /** + * Small utility method that checks whether the [Vp9Frame] that is + * specified as a parameter belongs to the same RTP stream as the frame that + * this instance refers to. + * + * @param vp9Frame the [Vp9Frame] to check whether it belongs to the + * same RTP stream as the frame that this instance refers to. + * @return true if the [Vp9Frame] that is specified as a parameter + * belongs to the same RTP stream as the frame that this instance refers to, + * false otherwise. + */ + fun matchesSSRC(vp9Frame: Vp9Frame): Boolean = ssrc == vp9Frame.ssrc + + /** + * Determines whether the [VideoRtpPacket] that is specified as an + * argument is part of the VP9 picture that is represented by this + * [Vp9Frame] instance. + * + * @param pkt the [VideoRtpPacket] instance to check whether it's part + * of the VP9 picture that is represented by this [Vp9Frame] + * instance. + * @return true if the [VideoRtpPacket] that is specified as an + * argument is part of the VP9 picture that is represented by this + * [Vp9Frame] instance, false otherwise. + */ + private fun matchesSSRC(pkt: VideoRtpPacket): Boolean = ssrc == pkt.ssrc + + /** + * Checks whether the specified RTP packet is part of this frame. + * + * @param pkt the RTP packet to check whether it's part of this frame. + * @return true if the specified RTP packet is part of this frame, false + * otherwise. + */ + fun matchesFrame(pkt: Vp9Packet): Boolean = matchesSSRC(pkt) && timestamp == pkt.timestamp && + spatialLayer == pkt.spatialLayerIndex + + /** + * Validates that the specified RTP packet consistently matches all the + * parameters of this frame. + * + * This can be useful for diagnosing invalid streams if this fails when + * [matchesFrame] is true. + * + * @param pkt the RTP packet to check whether its parameters match this frame. + * @throws RuntimeException if the specified RTP packet is inconsistent with this frame + */ + fun validateConsistency(pkt: Vp9Packet) { + if (temporalLayer == pkt.temporalLayerIndex && + tl0PICIDX == pkt.TL0PICIDX && + pictureId == pkt.pictureId && + isSwitchingUpPoint == pkt.isSwitchingUpPoint && + isUpperLevelReference == pkt.isUpperLevelReference && + usesInterLayerDependency == pkt.usesInterLayerDependency && + isInterPicturePredicted == pkt.isInterPicturePredicted + // TODO: also check start, end, seq nums? + ) { + return + } + throw RuntimeException( + buildString { + with(pkt) { + append("Packet ssrc $ssrc, seq $sequenceNumber, picture id $pictureId, timestamp $timestamp ") + } + append("is not consistent with frame $ssrc, ") + append("seq $earliestKnownSequenceNumber-$latestKnownSequenceNumber ") + append("picture id $pictureId, timestamp $timestamp: ") + + var complained = false + if (temporalLayer != pkt.temporalLayerIndex) { + append("packet temporal layer ${pkt.temporalLayerIndex} != frame temporal layer $temporalLayer") + complained = true + } + if (tl0PICIDX != pkt.TL0PICIDX) { + if (complained) { + append("; ") + } + append("packet TL0PICIDX ${pkt.TL0PICIDX} != frame TL0PICIDX $tl0PICIDX") + complained = true + } + if (pictureId != pkt.pictureId) { + if (complained) { + append("; ") + } + append("packet PictureID ${pkt.pictureId} != frame PictureID $pictureId") + complained = true + } + if (isSwitchingUpPoint != pkt.isSwitchingUpPoint) { + if (complained) { + append("; ") + } + append( + "packet switchingUpPoint ${pkt.isSwitchingUpPoint} != " + + "frame switchingUpPoint $isSwitchingUpPoint" + ) + complained = true + } + if (isUpperLevelReference != pkt.isUpperLevelReference) { + if (complained) { + append("; ") + } + append( + "packet upperLevelReference ${pkt.isUpperLevelReference} != " + + "frame upperLevelReference $isUpperLevelReference" + ) + complained = true + } + if (usesInterLayerDependency != pkt.usesInterLayerDependency) { + if (complained) { + append("; ") + } + append( + "packet usesInterLayerDepencency ${pkt.usesInterLayerDependency} != " + + "frame usesInterLayerDepencency $usesInterLayerDependency" + ) + complained = true + } + if (isInterPicturePredicted != pkt.isInterPicturePredicted) { + if (complained) { + append("; ") + } + append( + "packet isInterPicturePredicted ${pkt.isInterPicturePredicted} != " + + "frame isInterPicturePredicted $isInterPicturePredicted" + ) + complained = true + } + } + ) + } + + /** + * Check whether this frame is immediately after another one, according + * to their extended picture IDs and spatial layers + */ + fun isImmediatelyAfter(otherFrame: Vp9Frame): Boolean { + val delta = getExtendedPictureIdDelta(otherFrame.pictureId, pictureId) + + return when { + delta == 0 -> + spatialLayer == otherFrame.spatialLayer + 1 + + delta == 1 -> + spatialLayer == 0 && otherFrame.spatialLayer == 2 + + // ??? + else -> + false + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt new file mode 100644 index 0000000..1f4202c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt @@ -0,0 +1,214 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.rtp.util.RtpUtils.Companion.applySequenceNumberDelta +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging.TimeSeriesLogger +import java.time.Instant + +/** + * Represents a VP9 frame projection. It puts together all the necessary bits + * and pieces that are useful when projecting an accepted VP9 frame. A + * projection is responsible for rewriting a VP9 packet. Instances of this class + * are thread-safe. + */ +class Vp9FrameProjection +/* + * Ctor. + * + * @param vp9Frame The [Vp9Frame] that's projected. + * @param ssrc The RTP SSRC of the projected frame that this instance refers + * to (RFC3550). + * @param timestamp The RTP timestamp of the projected frame that this + * instance refers to (RFC3550). + * @param sequenceNumberDelta The starting RTP sequence number of the + * projected frame that this instance refers to (RFC3550). + * @param extendedPictureId The VP9 extended picture ID of the projected VP9 + * frame that this instance refers to. + * @param tl0PICIDX The VP9 TL0PICIDX of the projected VP9 frame that this + * instance refers to. + */ +internal constructor( + /** + * The diagnostic context for this instance. + */ + private val diagnosticContext: DiagnosticContext, + /** + * The projected [Vp9Frame]. + */ + val vp9Frame: Vp9Frame?, + /** + * The RTP SSRC of the projection (RFC7667, RFC3550). + */ + val ssrc: Long, + /** + * The RTP timestamp of the projection (RFC7667, RFC3550). + */ + val timestamp: Long, + /** + * The sequence number delta for packets of this frame. + */ + private val sequenceNumberDelta: Int, + /** + * The VP9 picture ID of the projection. + */ + val pictureId: Int, + /** + * The VP9 TL0PICIDX of the projection. + */ + val tl0PICIDX: Int, + /** + * Whether to add a marker bit to the last packet of this frame. + * (Note this will not clear already-existing marker bits.) + */ + val mark: Boolean, + /** + * A timestamp of when this instance was created. It's used to calculate + * RTP timestamps when we switch encodings. + */ + val created: Instant? +) { + + /** + * -1 if this projection is still "open" for new, later packets. + * Projections can be closed when we switch away from their encodings. + */ + var closedSeq = -1 + private set + + /** + * Ctor. + * + * @param ssrc the SSRC of the destination VP9 picture. + * @param timestamp The RTP timestamp of the projected frame that this + * instance refers to (RFC3550). + * @param sequenceNumberDelta The starting RTP sequence number of the + * projected frame that this instance refers to (RFC3550). + */ + internal constructor( + diagnosticContext: DiagnosticContext, + ssrc: Long, + sequenceNumberDelta: Int, + timestamp: Long + ) : this( + diagnosticContext = diagnosticContext, + vp9Frame = null, + ssrc = ssrc, + timestamp = timestamp, + sequenceNumberDelta = sequenceNumberDelta, + pictureId = 0, + tl0PICIDX = 0, + mark = false, + created = null + ) + + fun rewriteSeqNo(seq: Int): Int = applySequenceNumberDelta(seq, sequenceNumberDelta) + + /** + * Rewrites an RTP packet. + * + * @param pkt the RTP packet to rewrite. + */ + fun rewriteRtp(pkt: Vp9Packet) { + val sequenceNumber = rewriteSeqNo(pkt.sequenceNumber) + if (timeSeriesLogger.isTraceEnabled) { + timeSeriesLogger.trace( + diagnosticContext + .makeTimeSeriesPoint("rtp_vp9_rewrite") + .addField("orig.rtp.ssrc", pkt.ssrc) + .addField("orig.rtp.timestamp", pkt.timestamp) + .addField("orig.rtp.seq", pkt.sequenceNumber) + .addField("orig.vp9.pictureid", pkt.pictureId) + .addField("orig.vp9.tl0picidx", pkt.TL0PICIDX) + .addField("proj.rtp.ssrc", ssrc) + .addField("proj.rtp.timestamp", timestamp) + .addField("proj.rtp.seq", sequenceNumber) + .addField("proj.vp9.pictureid", pictureId) + .addField("proj.vp9.tl0picidx", tl0PICIDX) + .addField("proj.rtp.mark", mark) + ) + } + + // update ssrc, sequence number, timestamp, pictureId and tl0picidx + pkt.ssrc = ssrc + pkt.timestamp = timestamp + pkt.sequenceNumber = sequenceNumber + if (pkt.hasTL0PICIDX) { + pkt.TL0PICIDX = tl0PICIDX + } + pkt.pictureId = pictureId + if (mark && pkt.isEndOfFrame) pkt.isMarked = true + } + + /** + * Determines whether a packet can be forwarded as part of this + * [Vp9FrameProjection] instance. The check is based on the sequence + * of the incoming packet and whether or not the [Vp9FrameProjection] + * has been "closed" or not. + * + * @param rtpPacket the [Vp9Packet] that will be examined. + * @return true if the packet can be forwarded as part of this + * [Vp9FrameProjection], false otherwise. + */ + fun accept(rtpPacket: Vp9Packet): Boolean { + if (vp9Frame?.matchesFrame(rtpPacket) != true) { + // The packet does not belong to this VP9 picture. + return false + } + synchronized(vp9Frame) { + return if (closedSeq < 0) { + true + } else { + rtpPacket.sequenceNumber isOlderThan closedSeq + } + } + } + + val earliestProjectedSeqNum: Int + get() { + if (vp9Frame == null) { + return sequenceNumberDelta + } + synchronized(vp9Frame) { return rewriteSeqNo(vp9Frame.earliestKnownSequenceNumber) } + } + + val latestProjectedSeqNum: Int + get() { + if (vp9Frame == null) { + return sequenceNumberDelta + } + synchronized(vp9Frame) { return rewriteSeqNo(vp9Frame.latestKnownSequenceNumber) } + } + + /** + * Prevents the max sequence number of this frame to grow any further. + */ + fun close() { + if (vp9Frame != null) { + synchronized(vp9Frame) { closedSeq = vp9Frame.latestKnownSequenceNumber } + } + } + + companion object { + /** + * The time series logger for this class. + */ + private val timeSeriesLogger = TimeSeriesLogger.getTimeSeriesLogger(Vp9FrameProjection::class.java) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt new file mode 100644 index 0000000..5d7b30b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9Picture.kt @@ -0,0 +1,212 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyExtendedPictureIdDelta +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.nlj.util.setAndExtend +import kotlin.collections.ArrayList + +/** + * Groups together some RTP/VP9 fields that refer to a specific incoming VP9 + * picture, which may consist of multiple frames (of different spatial layers). + * + * Instances of this class are *NOT* thread safe. + * + * @author Jonathan Lennox + */ +class Vp9Picture(packet: Vp9Packet, index: Long) { + val frames = ArrayList() + + init { + val sid = packet.effectiveSpatialLayerIndex + + setFrameAtSid(Vp9Frame(packet, index), sid) + } + + fun frame(sid: Int) = frames.getOrNull(sid) + + fun frame(packet: Vp9Packet) = frame(packet.effectiveSpatialLayerIndex) + + private fun setFrameAtSid(frame: Vp9Frame, sid: Int) = frames.setAndExtend(sid, frame, null) + + /** + * Return the first (lowest-sid, earliest in decoding order) frame that we've received so far. + * A valid picture must have at least one frame, so this will always return one. + */ + private fun firstFrame(): Vp9Frame { + val f = frames.find { f -> f != null } + check(f != null) { "Picture must have at least one frame" } + return f + } + + /** + * Return the last (highest-sid, earliest in decoding order) frame that we've received so far. + * A valid picture must have at least one frame, so this will always return one. + */ + private fun lastFrame(): Vp9Frame { + val f = frames.findLast { f -> f != null } + check(f != null) { "Picture must have at least one frame" } + return f + } + + val ssrc: Long + get() = firstFrame().ssrc + + val timestamp: Long + get() = firstFrame().timestamp + + val temporalLayer: Int + get() = firstFrame().temporalLayer + + val earliestKnownSequenceNumber: Int + get() = firstFrame().earliestKnownSequenceNumber + + val latestKnownSequenceNumber: Int + get() = lastFrame().latestKnownSequenceNumber + + val pictureId: Int + get() = firstFrame().pictureId + + val index: Long + get() = firstFrame().index + + val tl0PICIDX: Int + get() = firstFrame().tl0PICIDX + + /** + * Remember another packet of this frame. + * Note: this assumes every packet is received only once, i.e. a filter + * like [org.jitsi.nlj.transform.node.incoming.PaddingTermination] is in use. + * @param packet The packet to remember. This should be a packet which + * has tested true with [matchesPicture]. + */ + fun addPacket(packet: Vp9Packet): PacketInsertionResult { + require(matchesPicture(packet)) { "Non-matching packet added to picture" } + + val sid = packet.effectiveSpatialLayerIndex + + val f = frame(packet) + + if (f != null) { + f.addPacket(packet) + return PacketInsertionResult(f, this, false) + } + + val newF = Vp9Frame(packet, index) + + setFrameAtSid(newF, sid) + + return PacketInsertionResult(newF, this, true) + } + + /** + * Determines whether the [VideoRtpPacket] that is specified as an + * argument is part of the VP9 picture that is represented by this + * [Vp9Picture] instance. + * + * @param pkt the [VideoRtpPacket] instance to check whether it's part + * of the VP9 picture that is represented by this [Vp9Picture] + * instance. + * @return true if the [VideoRtpPacket] that is specified as an + * argument is part of the VP9 picture that is represented by this + * [Vp9Picture] instance, false otherwise. + */ + private fun matchesSSRC(pkt: VideoRtpPacket): Boolean = ssrc == pkt.ssrc + + /** + * Checks whether the specified RTP packet is part of this frame. + * + * @param pkt the RTP packet to check whether it's part of this frame. + * @return true if the specified RTP packet is part of this frame, false + * otherwise. + */ + fun matchesPicture(pkt: Vp9Packet): Boolean = matchesSSRC(pkt) && timestamp == pkt.timestamp + + /** + * Validates that the specified RTP packet consistently matches all the + * parameters of this picture (or the appropriate frame). + * + * This can be useful for diagnosing invalid streams if this fails when + * [matchesPicture] is true. + * + * @param pkt the RTP packet to check whether its parameters match this frame. + * @throws RuntimeException if the specified RTP packet is inconsistent with this frame + */ + fun validateConsistency(pkt: Vp9Packet) { + val f = frame(pkt) + if (f != null) { + f.validateConsistency(pkt) + return + } + + if (temporalLayer == pkt.temporalLayerIndex && tl0PICIDX == pkt.TL0PICIDX && pictureId == pkt.pictureId) { + /* TODO: also check start, end, seq nums? */ + return + } + throw RuntimeException( + buildString { + with(pkt) { + append("Packet ssrc $ssrc, seq $sequenceNumber, picture id $pictureId, timestamp $timestamp ") + } + append("is not consistent with picture $ssrc, ") + append("seq $earliestKnownSequenceNumber-$latestKnownSequenceNumber ") + append("picture id $pictureId, timestamp $timestamp: ") + + var complained = false + if (temporalLayer != pkt.temporalLayerIndex) { + append("packet temporal layer ${pkt.temporalLayerIndex} != frame temporal layer $temporalLayer") + complained = true + } + if (tl0PICIDX != pkt.TL0PICIDX) { + if (complained) { + append("; ") + } + append("packet TL0PICIDX ${pkt.TL0PICIDX} != frame TL0PICIDX $tl0PICIDX") + complained = true + } + if (pictureId != pkt.pictureId) { + if (complained) { + append("; ") + } + append("packet PictureID ${pkt.pictureId} != frame PictureID $pictureId") + } + } + ) + } + + /** + * Check whether this picture is immediately after another one, according + * to their extended picture IDs. + */ + fun isImmediatelyAfter(otherPicture: Vp9Picture): Boolean = pictureId == + applyExtendedPictureIdDelta(otherPicture.pictureId, 1) +} + +/** + * The result of calling [insertPacket] + */ +class PacketInsertionResult( + /** The frame corresponding to the packet that was inserted. */ + val frame: Vp9Frame, + /** The picture corresponding to the packet that was inserted. */ + val picture: Vp9Picture, + /** Whether inserting the packet created a new frame. */ + val isNewFrame: Boolean, + /** Whether inserting the packet caused a reset */ + val isReset: Boolean = false +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt new file mode 100644 index 0000000..4558a99 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9PictureMap.kt @@ -0,0 +1,274 @@ +/* + * Copyright @ 2019 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getExtendedPictureIdDelta +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.nlj.util.ArrayCache +import org.jitsi.nlj.util.PictureIdIndexTracker +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import kotlin.math.max +import kotlin.math.min + +/** + * A history of recent pictures on a VP9 stream. + */ +class Vp9PictureMap( + parentLogger: Logger +) { + /** Cache mapping picture IDs to pictures. */ + private val pictureHistory = PictureHistory(PICTURE_MAP_SIZE) + private val logger: Logger = createChildLogger(parentLogger) + + /** Find a picture in the picture map, based on a packet. */ + @Synchronized + fun findPicture(packet: Vp9Packet): Vp9Picture? = pictureHistory[packet.pictureId] + + /** Get the current size of the map. */ + fun size(): Int = pictureHistory.numCached + + /** Check whether this is a large jump from previous state, so the map should be reset. */ + private fun isLargeJump(packet: Vp9Packet): Boolean { + val latestPicture: Vp9Picture = pictureHistory.latestPicture ?: return false + val picDelta = getExtendedPictureIdDelta(packet.pictureId, latestPicture.pictureId) + if (picDelta > PICTURE_MAP_SIZE) { + return true + } + val tsDelta: Long = RtpUtils.getTimestampDiff(packet.timestamp, latestPicture.timestamp) + if (picDelta < 0) { + /* if picDelta is negative but timestamp or sequence delta is positive, we've cycled. */ + if (tsDelta > 0) { + return true + } + if (RtpUtils.getSequenceNumberDelta(packet.sequenceNumber, latestPicture.latestKnownSequenceNumber) > 0) { + return true + } + } + + /* If tsDelta is more than twice the picture map size at 1 fps, we've cycled. */ + return tsDelta > PICTURE_MAP_SIZE * 90000 * 2 + } + + /** Insert a packet into the picture map. Return a PictureInsertionResult + * describing what happened. + * @param packet The packet to insert. + * @return What happened. null if insertion failed. + */ + @Synchronized + fun insertPacket(packet: Vp9Packet): PacketInsertionResult? { + val pictureId = packet.pictureId + if (pictureId == -1) { + /* Picture map indexes by picture ID. All supported browsers should currently be setting it. */ + /* Log message will have been logged by Vp9Parser in jmt. */ + return null + } + if (isLargeJump(packet)) { + pictureHistory.indexTracker.resetAt(pictureId) + val picture = pictureHistory.insert(packet) ?: return null + + return PacketInsertionResult(picture.frame(packet)!!, picture, true, isReset = true) + } + val picture = pictureHistory[pictureId] + if (picture != null) { + if (!picture.matchesPicture(packet)) { + check(picture.pictureId == pictureId) { + "Picture map returned picture with picture ID ${picture.pictureId} " + "when asked for picture with picture ID $pictureId" + } + logger.warn( + "Cannot insert packet in picture map: " + + with(picture) { + "picture with ssrc $ssrc, timestamp $timestamp, " + + "and sequence number range $earliestKnownSequenceNumber-$latestKnownSequenceNumber, " + } + + with(packet) { + "and packet $sequenceNumber with ssrc $ssrc, timestamp $timestamp, " + + "and sequence number $sequenceNumber" + } + + " both have picture ID $pictureId" + ) + return null + } + try { + picture.validateConsistency(packet) + } catch (e: Exception) { + logger.warn(e) + } + + return picture.addPacket(packet) + } + + val newPicture = pictureHistory.insert(packet) ?: return null + + return PacketInsertionResult(newPicture.frame(packet)!!, newPicture, true) + } + + @Synchronized + fun nextFrame(frame: Vp9Frame): Vp9Frame? = pictureHistory.findAfter(frame) { true } + + @Synchronized + fun nextFrameWith(frame: Vp9Frame, pred: (Vp9Frame) -> Boolean): Vp9Frame? = pictureHistory.findAfter(frame, pred) + + @Synchronized + fun prevFrame(frame: Vp9Frame): Vp9Frame? = pictureHistory.findBefore(frame) { true } + + @Synchronized + fun prevFrameWith(frame: Vp9Frame, pred: (Vp9Frame) -> Boolean): Vp9Frame? = pictureHistory.findBefore(frame, pred) + + fun findPrevAcceptedFrame(frame: Vp9Frame): Vp9Frame? = prevFrameWith(frame) { it.isAccepted } + + fun findNextAcceptedFrame(frame: Vp9Frame): Vp9Frame? = nextFrameWith(frame) { it.isAccepted } + + fun findNextBaseTl0(frame: Vp9Frame): Vp9Frame? = nextFrameWith(frame) { + it.spatialLayer <= 0 && it.temporalLayer <= 0 + } + + companion object { + const val PICTURE_MAP_SIZE = 500 // Matches PacketCache default size. + } +} + +internal class PictureHistory +constructor(size: Int) : ArrayCache( + size, + cloneItem = { k -> k }, + synchronize = false +) { + var numCached = 0 + var firstIndex = -1L + val indexTracker = PictureIdIndexTracker() + + /** + * Gets a picture with a given VP9 picture ID from the cache. + */ + operator fun get(pictureId: Int): Vp9Picture? { + val index = indexTracker.interpret(pictureId) + return getIndex(index) + } + + /** + * Gets a picture with a given VP9 picture ID index from the cache. + */ + private fun getIndex(index: Long): Vp9Picture? { + if (index <= lastIndex - size) { + /* We don't want to remember old pictures even if they're still + tracked; their neighboring pictures may have been evicted, + so findBefore / findAfter will return bogus data. */ + return null + } + val c = getContainer(index) ?: return null + return c.item + } + + /** Get the latest picture in the tracker. */ + val latestPicture: Vp9Picture? + get() = getIndex(lastIndex) + + /** Insert a new picture for the given Vp9 packet */ + fun insert(packet: Vp9Packet): Vp9Picture? { + val pictureId = packet.pictureId + val index = indexTracker.update(pictureId) + val picture = Vp9Picture(packet, index) + val inserted = super.insertItem(picture, index) + if (inserted) { + numCached++ + if (firstIndex == -1L || index < firstIndex) { + firstIndex = index + } + return picture + } + return null + } + + /** + * Called when an item in the cache is replaced/discarded. + */ + override fun discardItem(item: Vp9Picture) { + numCached-- + } + + fun findBefore(frame: Vp9Frame, pred: (Vp9Frame) -> Boolean): Vp9Frame? { + val lastIndex = lastIndex + if (lastIndex == -1L) { + return null + } + val index = indexTracker.interpret(frame.pictureId) + val searchStartIndex = min(index, lastIndex) + val searchEndIndex = max(lastIndex - size, firstIndex - 1) + return doFind( + pred = pred, + startIndex = searchStartIndex, + endIndex = searchEndIndex, + startLayer = frame.effectiveSpatialLayer - 1, + increment = -1 + ) + } + + fun findAfter(frame: Vp9Frame, pred: (Vp9Frame) -> Boolean): Vp9Frame? { + val lastIndex = lastIndex + if (lastIndex == -1L) { + return null + } + val index = indexTracker.interpret(frame.pictureId) + if (index > lastIndex) { + return null + } + val searchStartIndex = max(index, max(lastIndex - size + 1, firstIndex)) + return doFind( + pred = pred, + startIndex = searchStartIndex, + endIndex = lastIndex + 1, + startLayer = frame.effectiveSpatialLayer + 1, + increment = 1 + ) + } + + private fun doFind( + pred: (Vp9Frame) -> Boolean, + startIndex: Long, + endIndex: Long, + startLayer: Int, + increment: Int + ): Vp9Frame? { + require((increment > 0 && startIndex <= endIndex) || (increment < 0 && startIndex >= endIndex)) { + "Values of startIndex=$startIndex, endIndex=$endIndex, and increment=$increment " + + "could lead to infinite loop" + } + var index = startIndex + var layer = startLayer + var firstPicture = true + while (index != endIndex) { + val picture = getIndex(index) + if (picture != null) { + if (!firstPicture) { + layer = if (increment < 0) picture.frames.size - 1 else 0 + } + while (layer in picture.frames.indices) { + val frame = picture.frames[layer] + if (frame != null && pred(frame)) { + return frame + } + layer += increment + } + } + index += increment + firstPicture = false + } + return null + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt new file mode 100644 index 0000000..27e6bad --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt @@ -0,0 +1,488 @@ +/* + * Copyright @ 2019 - present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_ENCODING_ID +import org.jitsi.nlj.RtpLayerDesc.Companion.SUSPENDED_INDEX +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.getSidFromIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.getTidFromIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.indexString +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import java.time.Duration +import java.time.Instant + +/** + * This class is responsible for dropping VP9 simulcast/svc packets based on + * their quality, i.e. packets that correspond to qualities that are above a + * given quality target. Instances of this class are thread-safe. + */ +internal class Vp9QualityFilter(parentLogger: Logger) { + /** + * The [Logger] to be used by this instance to print debug + * information. + */ + private val logger: Logger = createChildLogger(parentLogger) + + /** + * Holds the arrival time of the most recent keyframe group. + * Reading/writing of this field is synchronized on this instance. + */ + private var mostRecentKeyframeGroupArrivalTime: Instant? = null + + /** + * A boolean flag that indicates whether a keyframe is needed, due to an + * encoding or (in some cases) a spatial layer switch. + */ + var needsKeyframe = false + private set + + /** + * The encoding ID that this instance tries to achieve. Upon + * receipt of a packet, we check whether encoding in the externalTargetIndex + * (that's specified as an argument to the + * [acceptFrame] method) is set to something different, + * in which case we set [needsKeyframe] equal to true and + * update. + */ + private var internalTargetEncoding = SUSPENDED_ENCODING_ID + + /** + * The spatial layer that this instance tries to achieve. + */ + private var internalTargetSpatialId = SUSPENDED_ENCODING_ID + + /** + * The layer index that we're currently forwarding. [SUSPENDED_INDEX] + * indicates that we're not forwarding anything. Reading/writing of this + * field is synchronized on this instance. + */ + private var currentIndex = SUSPENDED_INDEX + + /** + * Which spatial layers are currently being forwarded. + */ + private val layers = BooleanArray(MAX_VP9_LAYERS) + + /** + * Determines whether to accept or drop a VP9 frame. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param frame the VP9 frame. + * @param incomingEncoding the encoding of the incoming RTP packet + * @param externalTargetIndex the target quality index that the user of this + * instance wants to achieve. + * @param receivedTime the current time (as an Instant) + * @return true to accept the VP9 frame, otherwise false. + */ + @Synchronized + fun acceptFrame( + frame: Vp9Frame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): AcceptResult { + val prevIndex = currentIndex + val accept = doAcceptFrame(frame, incomingEncoding, externalTargetIndex, receivedTime) + val mark = if (frame.isInterPicturePredicted) { + frame.spatialLayer.coerceAtLeast(0) == getSidFromIndex(currentIndex) + } else { + /* This is wrong if the stream isn't actually currently encoding the target index's spatial layer */ + /* However, in that case the final (lower) spatial layer should have the marker bit set by the encoder, so + I think this shouldn't be a problem? */ + frame.spatialLayer.coerceAtLeast(0) == getSidFromIndex(externalTargetIndex) + } + val isResumption = (prevIndex == SUSPENDED_INDEX && currentIndex != SUSPENDED_INDEX) + if (isResumption) assert(accept) // Every code path that can turn off SUSPENDED_INDEX also accepts + return AcceptResult(accept = accept, isResumption = isResumption, mark = mark) + } + + private fun doAcceptFrame( + frame: Vp9Frame, + incomingEncoding: Int, + externalTargetIndex: Int, + receivedTime: Instant? + ): Boolean { + val externalTargetEncoding = getEidFromIndex(externalTargetIndex) + val currentEncoding = getEidFromIndex(currentIndex) + + if (externalTargetEncoding != internalTargetEncoding) { + // The externalEncodingIdTarget has changed since accept last + // ran; perhaps we should request a keyframe. + internalTargetEncoding = externalTargetEncoding + if (externalTargetEncoding != SUSPENDED_ENCODING_ID && + externalTargetEncoding != currentEncoding + ) { + needsKeyframe = true + } + } + if (externalTargetEncoding == SUSPENDED_ENCODING_ID) { + // We stop forwarding immediately. We will need a keyframe in order + // to resume. + currentIndex = SUSPENDED_INDEX + return false + } + // If temporal scalability is not enabled, pretend that this is the base temporal layer. + val temporalLayerIdOfFrame = frame.temporalLayer.coerceAtLeast(0) + return if (frame.isKeyframe) { + logger.debug { + "Quality filter got keyframe for stream ${frame.ssrc}" + } + val accept = acceptKeyframe(frame, incomingEncoding, receivedTime) + if (accept) { + // Keyframes reset layer forwarding, whether or not they're an encoding switch + for (i in layers.indices) { + layers[i] = (i == 0) + } + } + accept + } else if (currentEncoding != SUSPENDED_ENCODING_ID) { + if (isOutOfSwitchingPhase(receivedTime) && isPossibleToSwitch(incomingEncoding)) { + // XXX(george) i've noticed some "rogue" base layer keyframes + // that trigger this. what happens is the client sends a base + // layer key frame, the bridge switches to that layer because + // for all it knows it may be the only keyframe sent by the + // client engine. then the bridge notices that packets from the + // higher quality streams are flowing and execution ends-up + // here. it is a mystery why the engine is "leaking" base layer + // key frames + needsKeyframe = true + } + if (incomingEncoding != currentEncoding) { + // for non-keyframes, we can't route anything but the current encoding + return false + } + + /* Logic to forward spatial layer: + * Can forward layer: If layer is being sent, or frame is not inter-picture predicted; and + * if layer below is being sent, or if frame does not use inter-picture dependency. + * Switching layers: If layer of frame is closer to target than current, set current to target + * if can forward, otherwise request keyframe. + * Want to forward layer: If layer of frame is equal to currentLayer, or is less than currentLayer + * and isUpperLayerReference. + * If wantToForward and !canForward, something's wrong, request keyFrame. + * accept = wantToForward && canForward + * if (tid == 0) + * layers[layerOfFrame] = accept + * return accept + */ + + val spatialLayerOfFrame = frame.spatialLayer.coerceAtLeast(0) + var externalTargetSpatialId = getSidFromIndex(externalTargetIndex) + var currentSpatialLayer = getSidFromIndex(currentIndex) + + /* If the stream includes a new SS which doesn't list the target spatial layer, lower the target. */ + /* The target should change the next time the BitrateController runs, but that may be some time. */ + if (frame.numSpatialLayers != -1 && externalTargetSpatialId >= frame.numSpatialLayers) { + externalTargetSpatialId = frame.numSpatialLayers - 1 + } + + val canForwardLayer = (!frame.isInterPicturePredicted || layers[spatialLayerOfFrame]) && + (!frame.usesInterLayerDependency || layers[spatialLayerOfFrame - 1]) + + /* TODO: this logic is fragile in the presence of frame reordering. */ + val wantToSwitch = + (spatialLayerOfFrame > currentSpatialLayer && spatialLayerOfFrame <= externalTargetSpatialId) || + (spatialLayerOfFrame < currentSpatialLayer && spatialLayerOfFrame >= externalTargetSpatialId) || + (frame.numSpatialLayers != -1 && currentSpatialLayer >= frame.numSpatialLayers) + + if (wantToSwitch) { + if (canForwardLayer) { + logger.debug { "Switching to spatial layer $externalTargetSpatialId from $currentSpatialLayer" } + currentIndex = RtpLayerDesc.getIndex(incomingEncoding, frame.spatialLayer, frame.temporalLayer) + currentSpatialLayer = spatialLayerOfFrame + } else { + if (internalTargetSpatialId != externalTargetSpatialId) { + logger.debug { + "Want to switch to spatial layer $externalTargetSpatialId from $currentSpatialLayer, " + + "requesting keyframe" + } + } + needsKeyframe = true + } + internalTargetSpatialId = externalTargetSpatialId + } + + val wantToForwardLayer = + (spatialLayerOfFrame == currentSpatialLayer) || + (spatialLayerOfFrame < currentSpatialLayer && frame.isUpperLevelReference) + + if (wantToForwardLayer && !canForwardLayer) { + logger.warn( + "Want to forward ${indexString(currentIndex)} frame, but can't! " + + "layers=${layers.joinToString()}, currentIndex=${indexString(currentIndex)}, " + + "isInterPicturePredicted=${frame.isInterPicturePredicted}, " + + "usesInterLayerDependency=${frame.usesInterLayerDependency}." + ) + } + + val accept = wantToForwardLayer && canForwardLayer + + if (temporalLayerIdOfFrame == 0) { + layers[spatialLayerOfFrame] = accept + } + + if (!accept) { + return false + } + + // This branch reads the {@link #currentEncodingId} and it + // filters packets based on their temporal layer. + /* TODO: pay attention to isSwitchingUpPoint. (I believe the current VP9 encoders we deal with always + * have it set, however.) */ + if (currentEncoding > externalTargetEncoding || currentSpatialLayer > externalTargetSpatialId) { + // pending downscale, decrease the frame rate until we + // downscale. + temporalLayerIdOfFrame < 1 + } else if (currentEncoding < externalTargetEncoding || currentSpatialLayer < externalTargetSpatialId) { + // pending upscale, increase the frame rate until we upscale. + true + } else { + // The currentEncoding exactly matches the externalTargetEncoding. + val externalTargetTemporalId = getTidFromIndex(externalTargetIndex) + val currentTemporalLayer = getTidFromIndex(currentIndex) + + val acceptTemporal = temporalLayerIdOfFrame <= externalTargetTemporalId + if (acceptTemporal && temporalLayerIdOfFrame > currentTemporalLayer) { + currentIndex = RtpLayerDesc.getIndex(currentEncoding, currentSpatialLayer, temporalLayerIdOfFrame) + } + acceptTemporal + } + } else { + // In this branch we're not processing a keyframe and the + // currentEncoding is in suspended state, which means we need + // a keyframe to start streaming again. + + // We should have already requested a keyframe, either above or when the + // internal target encoding was first moved off SUSPENDED_ENCODING. + + false + } + } + + /** + * Returns a boolean that indicates whether we are in layer switching phase + * or not. + * + * @param receivedTime the time the latest frame was received + * @return false if we're in layer switching phase, true otherwise. + */ + @Synchronized + private fun isOutOfSwitchingPhase(receivedTime: Instant?): Boolean { + if (receivedTime == null) { + return false + } + if (mostRecentKeyframeGroupArrivalTime == null) { + return true + } + val delta = Duration.between(mostRecentKeyframeGroupArrivalTime, receivedTime) + return delta > MIN_KEY_FRAME_WAIT + } + + /** + * @return true if it looks like we can re-scale (see implementation of + * method for specific details). + */ + @Synchronized + private fun isPossibleToSwitch(incomingEncoding: Int): Boolean { + val currentEncoding = getEidFromIndex(currentIndex) + + if (incomingEncoding == SUSPENDED_ENCODING_ID) { + // We failed to resolve the spatial/quality layer of the packet. + return false + } + return when { + incomingEncoding > currentEncoding && currentEncoding < internalTargetEncoding -> + // It looks like upscaling is possible + true + + incomingEncoding < currentEncoding && currentEncoding > internalTargetEncoding -> + // It looks like downscaling is possible. + true + + else -> + false + } + } + + /** + * Determines whether to accept or drop a VP9 keyframe. This method updates + * the encoding id. + * + * Note that, at the time of this writing, there's no practical need for a + * synchronized keyword because there's only one thread accessing this + * method at a time. + * + * @param receivedTime the time the frame was received + * @return true to accept the VP9 keyframe, otherwise false. + */ + @Synchronized + private fun acceptKeyframe(frame: Vp9Frame, incomingEncoding: Int, receivedTime: Instant?): Boolean { + // This branch writes the {@link #currentSpatialLayerId} and it + // determines whether or not we should switch to another simulcast + // stream. + if (incomingEncoding < 0) { + // something went terribly wrong, normally we should be able to + // extract the layer id from a keyframe. + logger.error("invalid encoding id for keyframe") + return false + } + // Keyframes have to be sid 0, tid 0, unless something screwy is going on. + // The layers can also be -1 if the layers aren't known + if (frame.spatialLayer > 0 || frame.temporalLayer > 0) { + logger.warn("Surprising layers S${frame.spatialLayer}T${frame.temporalLayer} on keyframe") + } + logger.debug { + "Received a keyframe of encoding: $incomingEncoding" + } + val incomingIndex = RtpLayerDesc.getIndex(incomingEncoding, frame.spatialLayer, frame.temporalLayer) + + // The keyframe request has been fulfilled at this point, regardless of + // whether we'll be able to achieve the internalEncodingIdTarget. + needsKeyframe = false + return if (isOutOfSwitchingPhase(receivedTime)) { + // During the switching phase we always project the first + // keyframe because it may very well be the only one that we + // receive (i.e. the endpoint is sending low quality only). Then + // we try to approach the target. + mostRecentKeyframeGroupArrivalTime = receivedTime + logger.debug { + "First keyframe in this kf group " + + "currentEncodingId: $incomingEncoding. " + + "Target is $internalTargetEncoding" + } + if (incomingEncoding <= internalTargetEncoding) { + val currentEncoding = getEidFromIndex(currentIndex) + // If the target is 180p and the first keyframe of a group of + // keyframes is a 720p keyframe we don't project it. If we + // receive a 720p keyframe, we know that there MUST be a 180p + // keyframe shortly after. + if (currentEncoding != incomingEncoding) { + currentIndex = incomingIndex + } + true + } else { + false + } + } else { + // We're within the 300ms window since the reception of the + // first key frame of a key frame group, let's check whether an + // upscale/downscale is possible. + val currentEncoding = getEidFromIndex(currentIndex) + when { + currentEncoding <= incomingEncoding && + incomingEncoding <= internalTargetEncoding -> { + // upscale or current quality case + if (currentEncoding != incomingEncoding) { + currentIndex = incomingIndex + } + logger.debug { + "Upscaling to encoding $incomingEncoding. " + + "The target is $internalTargetEncoding" + } + true + } + + incomingEncoding <= internalTargetEncoding && + internalTargetEncoding < currentEncoding -> { + // downscale case + currentIndex = incomingIndex + logger.debug { + "Downscaling to encoding $incomingEncoding. " + + "The target is $internalTargetEncoding" + } + true + } + + else -> { + false + } + } + } + } + + /** + * Adds internal state to a diagnostic context time series point. + */ + @SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) { + pt.addField("qf.currentIndex", indexString(currentIndex)) + .addField("qf.internalTargetEncoding", internalTargetEncoding) + .addField("qf.internalTargetSpatialId", internalTargetSpatialId) + .addField("qf.needsKeyframe", needsKeyframe) + .addField( + "qf.mostRecentKeyframeGroupArrivalTimeMs", + mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 + ) + for (i in layers.indices) { + pt.addField("qf.layer.$i", layers[i]) + } + } + + /** + * Gets a JSON representation of the parts of this object's state that + * are deemed useful for debugging. + */ + @get:SuppressFBWarnings( + value = ["IS2_INCONSISTENT_SYNC"], + justification = "We intentionally avoid synchronizing while reading fields only used in debug output." + ) + val debugState: ObjectNode + get() { + val debugState = JsonNodeFactory.instance.objectNode() + debugState.put( + "mostRecentKeyframeGroupArrivalTimeMs", + mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1 + ) + debugState.put("needsKeyframe", needsKeyframe) + debugState.put("internalTargetEncoding", internalTargetEncoding) + debugState.put("internalTargetSpatialId", internalTargetSpatialId) + debugState.put("currentIndex", indexString(currentIndex)) + debugState.put("layersForwarded", layers.map { toString().first() }.joinToString(separator = "")) + return debugState + } + + data class AcceptResult( + val accept: Boolean, + val isResumption: Boolean, + val mark: Boolean + ) + + companion object { + /** + * The default maximum frequency at which the media engine + * generates key frame. + */ + private val MIN_KEY_FRAME_WAIT = Duration.ofMillis(300) + + /** + * The maximum possible number of VP9 spatial layers. + */ + private const val MAX_VP9_LAYERS = 8 + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt new file mode 100644 index 0000000..f5d2b32 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2ConferenceHandler.kt @@ -0,0 +1,574 @@ +/* + * Copyright @ 2022 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.colibri2 + +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType.Companion.createFromUri +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.util.LocalSsrcAssociation +import org.jitsi.nlj.util.SsrcAssociation +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.AbstractEndpoint +import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.dcsctp.DcSctpTransport +import org.jitsi.videobridge.relay.AudioSourceDesc +import org.jitsi.videobridge.relay.Relay +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.videobridge.sctp.SctpConfig +import org.jitsi.videobridge.transport.ice.IceRestartResult +import org.jitsi.videobridge.util.PayloadTypeUtil.Companion.create +import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig +import org.jitsi.videobridge.xmpp.MediaSourceFactory +import org.jitsi.xmpp.extensions.colibri.SourcePacketExtension +import org.jitsi.xmpp.extensions.colibri2.Capability +import org.jitsi.xmpp.extensions.colibri2.Colibri2Endpoint +import org.jitsi.xmpp.extensions.colibri2.Colibri2Relay +import org.jitsi.xmpp.extensions.colibri2.ConferenceModifiedIQ +import org.jitsi.xmpp.extensions.colibri2.ConferenceModifyIQ +import org.jitsi.xmpp.extensions.colibri2.Media +import org.jitsi.xmpp.extensions.colibri2.MediaSource +import org.jitsi.xmpp.extensions.colibri2.Sctp +import org.jitsi.xmpp.extensions.colibri2.Sources +import org.jitsi.xmpp.extensions.colibri2.Transport +import org.jitsi.xmpp.extensions.jingle.RTPHdrExtPacketExtension +import org.jitsi.xmpp.extensions.jingle.SourceGroupPacketExtension +import org.jitsi.xmpp.util.createError +import org.jivesoftware.smack.packet.IQ +import org.jivesoftware.smack.packet.StanzaError.Condition +import org.jivesoftware.smackx.muc.MUCRole + +class Colibri2ConferenceHandler( + private val conference: Conference, + parentLogger: Logger +) { + private val logger = createChildLogger(parentLogger) + + /** + * @return A pair with an IQ to be sent as a response, and a boolean indicating if the conference needs to be + * expired. + */ + fun handleConferenceModifyIQ(conferenceModifyIQ: ConferenceModifyIQ): Pair = try { + validateRequest(conferenceModifyIQ) + + val ignoreUnknownEndpoints = shouldIgnoreUnknownEndpoints(conferenceModifyIQ) + val responseBuilder = + ConferenceModifiedIQ.builder(ConferenceModifiedIQ.Builder.createResponse(conferenceModifyIQ)) + var expire = conferenceModifyIQ.expire.also { + if (it) logger.info("Received request to expire conference.") + } + + for (e in conferenceModifyIQ.endpoints) { + responseBuilder.addEndpoint(handleColibri2Endpoint(e, ignoreUnknownEndpoints)) + } + conferenceModifyIQ.connects?.let { conference.applyConnects(it.getConnects()) } + for (r in conferenceModifyIQ.relays) { + if (!RelayConfig.config.enabled) { + throw IqProcessingException(Condition.feature_not_implemented, "Octo is disabled in configuration.") + } + if (!WebsocketServiceConfig.config.enabled && !SctpConfig.config.enabled) { + logger.warn( + "Can not use a colibri2 relay, because neither SCTP nor colibri web sockets are enabled. See " + + "https://github.com/jitsi/jitsi-videobridge/blob/master/doc/relay.md" + ) + throw UnsupportedOperationException( + "Colibri websockets or SCTP need to be enabled to use a colibri2 relay." + ) + } + responseBuilder.addRelay(handleColibri2Relay(r, ignoreUnknownEndpoints)) + } + + // Include feedback sources with any "create conference" or "create endpoint" request. This allows async + // handling of responses in jicofo without potentially losing the feedback sources. + // TODO: perhaps colibri clients should be required to process responses in order? + if (conferenceModifyIQ.create || conferenceModifyIQ.endpoints.any { it.create }) { + responseBuilder.setSources(buildFeedbackSources()) + } + + if (!conferenceModifyIQ.create && conference.endpointCount == 0 && conference.relayCount == 0) { + logger.info("All endpoints and relays removed, expiring.") + expire = true + } + + Pair(responseBuilder.build(), expire) + } catch (e: UnknownEndpointException) { + logger.warn("Unknown Endpoint during processing conference-modify IQ: $e") + val error = createEndpointNotFoundError(conferenceModifyIQ, e.endpointId) + Pair(error, false) + } catch (e: FeatureNotImplementedException) { + logger.warn("Unsupported request (${e.message}): ${conferenceModifyIQ.toXML()}") + Pair(createFeatureNotImplementedError(conferenceModifyIQ, e.message), false) + } catch (e: IqProcessingException) { + // Item not found conditions are assumed to be less critical, as they often happen in case a request + // arrives late for an expired endpoint. + if (Condition.item_not_found == e.condition) { + logger.warn("Error processing conference-modify IQ: $e") + } else { + logger.error("Error processing conference-modify IQ: $e") + } + Pair(createError(conferenceModifyIQ, e.condition, e.message), false) + } + + private fun buildFeedbackSources(): Sources = Sources.getBuilder().apply { + addMediaSource( + MediaSource.getBuilder() + .setType(MediaType.AUDIO) + .setId("jvb-a0") + .addSource( + SourcePacketExtension().apply { + ssrc = conference.localAudioSsrc + name = "jvb-a0" + } + ) + .build() + ) + addMediaSource( + MediaSource.getBuilder() + .setType(MediaType.VIDEO) + .setId("jvb-v0") + .addSource( + SourcePacketExtension().apply { + ssrc = conference.localVideoSsrc + name = "jvb-v0" + } + ) + .build() + ) + }.build() + + /** + * Process a colibri2 Endpoint in a conference-modify, return the response to be put in + * the conference-modified. + */ + @Throws(IqProcessingException::class) + private fun handleColibri2Endpoint( + c2endpoint: Colibri2Endpoint, + ignoreUnknownEndpoints: Boolean + ): Colibri2Endpoint { + val respBuilder = Colibri2Endpoint.getBuilder().apply { setId(c2endpoint.id) } + if (c2endpoint.expire) { + conference.getLocalEndpoint(c2endpoint.id)?.expire() + respBuilder.setExpire(true) + return respBuilder.build() + } + + val endpoint = if (c2endpoint.create) { + if (conference.getLocalEndpoint(c2endpoint.id) != null) { + throw IqProcessingException(Condition.conflict, "Endpoint with ID ${c2endpoint.id} already exists") + } + val transport = c2endpoint.transport ?: throw IqProcessingException( + Condition.bad_request, + "Attempt to create endpoint ${c2endpoint.id} with no " + ) + if (!c2endpoint.hasCapability(Capability.CAP_SOURCE_NAME_SUPPORT)) { + throw IqProcessingException(Condition.bad_request, "Source name support is mandatory.") + } + + val ssrcRewriting = c2endpoint.hasCapability(Capability.CAP_SSRC_REWRITING_SUPPORT) + val privateAddresses = c2endpoint.hasCapability(Capability.CAP_PRIVATE_ADDRESS_CONNECTIVITY) + val midDemux = ssrcRewriting && c2endpoint.hasCapability(Capability.CAP_RTP_MID_DEMUX_SUPPORT) + conference.createLocalEndpoint( + c2endpoint.id, + transport.iceControlling, + ssrcRewriting, + midDemux, + c2endpoint.mucRole == MUCRole.visitor, + privateAddresses, + c2endpoint.diarize == true + ).apply { + c2endpoint.statsId?.let { + statsId = it + } + transport.sctp?.let { sctp -> + if (!SctpConfig.config.enabled) { + throw IqProcessingException( + Condition.feature_not_implemented, + "SCTP support is not configured" + ) + } + if (sctp.role != null && sctp.role != Sctp.Role.SERVER) { + throw IqProcessingException( + Condition.feature_not_implemented, + "Unsupported SCTP role: ${sctp.role}" + ) + } + if (sctp.port != null && sctp.port != DcSctpTransport.DEFAULT_SCTP_PORT) { + throw IqProcessingException( + Condition.bad_request, + "Specific SCTP port requested, not supported." + ) + } + + createSctpConnection() + } + } + } else { + conference.getLocalEndpoint(c2endpoint.id) + } + + if (endpoint == null) { + if (ignoreUnknownEndpoints) { + return respBuilder.build() + } else { + throw UnknownEndpointException(c2endpoint.id) + } + } + + for (media in c2endpoint.media) { + // TODO: support removing payload types/header extensions + media.payloadTypes.forEach { ptExt -> + create(ptExt, media.type)?.let { endpoint.addPayloadType(it) } + ?: logger.warn("Ignoring unrecognized payload type extension: ${ptExt.toXML()}") + } + + media.rtpHdrExts.forEach { rtpHdrExt -> + rtpHdrExt.toRtpExtension()?.let { endpoint.addRtpExtension(it) } + ?: logger.warn("Ignoring unrecognized RTP header extension: ${rtpHdrExt.toXML()}") + } + + endpoint.setExtmapAllowMixed(media.extmapAllowMixed != null) + + /* No need to put media in conference-modified. */ + } + + endpoint.acceptAudio = endpoint.transceiver.readOnlyStreamInformationStore.rtpPayloadTypes.values.any { + it.mediaType == MediaType.AUDIO + } + endpoint.acceptVideo = endpoint.transceiver.readOnlyStreamInformationStore.rtpPayloadTypes.values.any { + it.mediaType == MediaType.VIDEO + } + + c2endpoint.initialLastN?.value?.let { + endpoint.setInitialLastN(it) + } + + c2endpoint.transport?.iceUdpTransport?.let { endpoint.setTransportInfo(it) } + + c2endpoint.sources?.let { sources -> + if (endpoint.visitor && sources.mediaSources.isNotEmpty()) { + throw IqProcessingException( + Condition.bad_request, + "Attempt to set sources for visitor endpoint ${c2endpoint.id}" + ) + } + + sources.mediaSources.forEach { mediaSource -> + mediaSource.sources.forEach { + endpoint.addReceiveSsrc(it.ssrc, mediaSource.type) + } + // TODO: remove any old associations for this endpoint + mediaSource.ssrcGroups.mapNotNull { it.toSsrcAssociation() }.forEach { + addSsrcAssociation(endpoint.id, it) + } + } + + // Assume a message can only contain one source per media type. + // If "sources" was signaled, but it didn't contain any video sources, clear the endpoint's video sources + val newMediaSources = sources.mediaSources.filter { it.type == MediaType.VIDEO }.mapNotNull { + MediaSourceFactory.createMediaSource(it.sources, it.ssrcGroups, c2endpoint.id, it.id, it.isSynthetic) + } + endpoint.mediaSources = newMediaSources.toTypedArray() + + val audioSources: ArrayList = ArrayList() + sources.mediaSources.filter { it.type == MediaType.AUDIO }.forEach { + it.sources.forEach { s -> + audioSources.add(AudioSourceDesc(s.ssrc, c2endpoint.id, it.id, it.isSynthetic)) + } + } + endpoint.audioSources = audioSources + } + + c2endpoint.forceMute?.let { + endpoint.updateForceMute(it.audio, it.video) + } + + // An explicit ICE restart request. Handled after setTransportInfo, so any credentials included in this + // same request still belong to (and are applied to) the pre-restart Agent. The restart rotates our own + // ICE credentials, so the endpoint has to be told the new ones — its connectivity checks are addressed + // to them — even though this is not a create. + // + // Handled last, once nothing else in this request can throw: a restart is armed as a side effect (it + // rotates the credentials we advertise and starts the timeout), so failing the request after arming it + // would leave the bridge waiting out that timeout for an answer the endpoint was never asked for. + val iceRestartResult = if (c2endpoint.transport?.iceRestart == true) { + endpoint.requestIceRestart().also { + if (it != IceRestartResult.STARTED) { + logger.warn("Did not restart ICE for endpoint ${c2endpoint.id}: $it.") + } + } + } else { + null + } + + // How the refusal of a restart is signaled: with the *absence* of a in the + // conference-modified for this endpoint. There is no explicit "refused" flag, and the request is not + // failed with an error, because an error would fail the whole conference-modify and take every other + // endpoint's updates with it. Jicofo pairs a request it sent with the answer it gets back: a + // means the restart happened and is relayed to the endpoint, no means it did + // not and jicofo falls back to a re-invite. + // + // So a transport is signaled back for a restart that started (the new Agent's rotated credentials) and + // for one that kept the existing Agent (its unchanged credentials, so the endpoint keeps the connection + // it has, with no re-invite). Only IceRestartResult.UNAVAILABLE signals nothing. + if (c2endpoint.create || iceRestartResult == IceRestartResult.STARTED || + iceRestartResult == IceRestartResult.KEEP_EXISTING + ) { + val transBuilder = Transport.getBuilder() + transBuilder.setIceUdpExtension(endpoint.describeTransport()) + if (c2endpoint.transport?.sctp != null) { + transBuilder.setSctp( + Sctp.Builder() + .setPort(DcSctpTransport.DEFAULT_SCTP_PORT) + .setRole(Sctp.Role.SERVER) + .build() + ) + } + respBuilder.setTransport(transBuilder.build()) + } + + return respBuilder.build() + } + + private fun addSsrcAssociation(endpointId: String, ssrcAssociation: SsrcAssociation) { + conference.encodingsManager.addSsrcAssociation( + endpointId, + ssrcAssociation.primarySsrc, + ssrcAssociation.secondarySsrc, + ssrcAssociation.type + ) + } + + private fun SourceGroupPacketExtension.toSsrcAssociation(): SsrcAssociation? { + if (sources.size < 2) { + logger.warn("Ignoring source group with <2 sources: ${toXML()}") + return null + } + + val type = semantics.parseAssociationType() ?: return null + if (type == SsrcAssociationType.SIM) return null + + return LocalSsrcAssociation(sources[0].ssrc, sources[1].ssrc, type) + } + + private fun String.parseAssociationType(): SsrcAssociationType? = when { + this.equals(SourceGroupPacketExtension.SEMANTICS_FID, ignoreCase = true) -> SsrcAssociationType.RTX + this.equals(SourceGroupPacketExtension.SEMANTICS_SIMULCAST, ignoreCase = true) -> SsrcAssociationType.SIM + this.equals(SourceGroupPacketExtension.SEMANTICS_FEC, ignoreCase = true) -> SsrcAssociationType.FEC + else -> null + } + + private fun RTPHdrExtPacketExtension.toRtpExtension(): RtpExtension? { + val type = createFromUri(uri.toString()) ?: return null + return RtpExtension(java.lang.Byte.valueOf(id), type) + } + + /** + * Process a colibri2 Relay in a conference-modify, return the response to be put in + * the conference-modified. + */ + @Throws(IqProcessingException::class) + private fun handleColibri2Relay(c2relay: Colibri2Relay, ignoreUnknownRelays: Boolean): Colibri2Relay { + val respBuilder = Colibri2Relay.getBuilder() + respBuilder.setId(c2relay.id) + if (c2relay.expire) { + conference.getRelay(c2relay.id)?.expire() + respBuilder.setExpire(true) + return respBuilder.build() + } + + val relay: Relay? + if (c2relay.create) { + if (conference.getRelay(c2relay.id) != null) { + throw IqProcessingException(Condition.conflict, "Relay with ID ${c2relay.id} already exists") + } + val transport = c2relay.transport ?: throw IqProcessingException( + Condition.bad_request, + "Attempt to create relay ${c2relay.id} with no " + ) + + relay = conference.createRelay( + c2relay.id, + c2relay.meshId, + transport.iceControlling, + transport.useUniquePort + ) + } else { + relay = conference.getRelay(c2relay.id) + } + + if (relay == null) { + if (ignoreUnknownRelays) { + return respBuilder.build() + } else { + // TODO: this should be Condition.item_not_found but this conflicts with some error codes from the Muc. + throw IqProcessingException(Condition.bad_request, "Unknown relay ${c2relay.id}") + } + } + + c2relay.transport?.sctp?.let { sctp -> + if (!SctpConfig.config.enabled) { + throw IqProcessingException( + Condition.feature_not_implemented, + "SCTP support is not configured" + ) + } + if (sctp.port != null && sctp.port != DcSctpTransport.DEFAULT_SCTP_PORT) { + throw IqProcessingException( + Condition.bad_request, + "Specific SCTP port requested, not supported." + ) + } + + relay.createSctpConnection(sctp) + } + + c2relay.transport?.iceUdpTransport?.let { relay.setTransportInfo(it) } + if (c2relay.create) { + val transBuilder = Transport.getBuilder() + transBuilder.setIceUdpExtension(relay.describeTransport()) + c2relay.transport?.sctp?.let { + val role = if (it.role == Sctp.Role.CLIENT) { + Sctp.Role.SERVER + } else { + Sctp.Role.CLIENT + } + transBuilder.setSctp( + Sctp.Builder() + .setPort(DcSctpTransport.DEFAULT_SCTP_PORT) + .setRole(role) + .build() + ) + } + respBuilder.setTransport(transBuilder.build()) + } + + for (media: Media in c2relay.media) { + // TODO: support removing payload types/header extensions + media.payloadTypes.forEach { ptExt -> + create(ptExt, media.type)?.let { relay.addPayloadType(it) } + ?: logger.warn("Ignoring unrecognized payload type extension: ${ptExt.toXML()}") + } + + media.rtpHdrExts.forEach { rtpHdrExt -> + rtpHdrExt.toRtpExtension()?.let { relay.addRtpExtension(it) } + ?: logger.warn("Ignoring unrecognized RTP header extension: ${rtpHdrExt.toXML()}") + } + + relay.setExtmapAllowMixed(media.extmapAllowMixed != null) + + /* No need to put media in conference-modified. */ + } + + // Calls to conference.addEndpoint re-run bandwidth allocation for the existing endpoints in the conference, + // so call it only once. + val newEndpoints = mutableSetOf() + c2relay.endpoints?.endpoints?.forEach { endpoint -> + if (endpoint.expire) { + relay.removeRemoteEndpoint(endpoint.id) + } else { + val sources = endpoint.parseSourceDescs() + if (endpoint.create) { + relay.addRemoteEndpoint(endpoint.id, endpoint.statsId, sources.first, sources.second)?.let { + newEndpoints.add(it) + } + } else { + relay.updateRemoteEndpoint(endpoint.id, sources.first, sources.second) + } + + // TODO: remove any old associations for this endpoint + endpoint.sources?.mediaSources?.forEach { mediaSource -> + mediaSource.ssrcGroups.mapNotNull { it.toSsrcAssociation() }.forEach { + addSsrcAssociation(endpoint.id, it) + } + } + } + } + conference.addEndpoints(newEndpoints) + + /* TODO: handle the rest of the relay's fields: feedback sources. */ + return respBuilder.build() + } + + private fun Colibri2Endpoint.parseSourceDescs(): Pair, List> { + val audioSources: MutableList = ArrayList() + val videoSources: MutableList = ArrayList() + sources?.let { + it.mediaSources.forEach { m -> + if (m.type == MediaType.AUDIO) { + if (m.sources.isEmpty()) { + logger.warn( + "Ignoring audio source ${m.id} in endpoint $id of a relay (no SSRCs): ${toXML()}" + ) + } else { + m.sources.forEach { audioSources.add(AudioSourceDesc(it.ssrc, id, m.id, m.isSynthetic)) } + } + } else if (m.type == MediaType.VIDEO) { + val desc = MediaSourceFactory.createMediaSource(m.sources, m.ssrcGroups, id, m.id, m.isSynthetic) + if (desc != null) { + videoSources.add(desc) + } + } else { + logger.warn("Ignoring source ${m.id} in endpoint $id of a relay: unsupported type ${m.type}") + } + } + } + return Pair(audioSources, videoSources) + } + + /** + * Check if the request satisfies the additional restrictions that jitsi-videobridge imposes on colibri2 requests + * and throw [FeatureNotImplementedException] if it doesn't. + * + * Mostly, we don't support requests that contain arbitrary multiple [Colibri2Endpoint]s or [Colibri2Relay]s. We do + * support them when they expire a set of endpoints, or solely update the "force-mute" state of a set of endpoints, + * because in these cases it's reasonable to ignore errors when endpoints expired. + */ + @kotlin.jvm.Throws(FeatureNotImplementedException::class) + private fun validateRequest(iq: ConferenceModifyIQ) { + if (iq.endpoints.size > 0 && iq.relays.size > 0) { + throw FeatureNotImplementedException("Using both 'relay' and 'endpoint' in a request") + } + + if (iq.endpoints.size > 1) { + if (iq.endpoints.any { it.create || it.media.size > 0 || it.transport != null || it.sources != null }) { + throw FeatureNotImplementedException( + "Creating or updating media, sources or transport for more than one endpoint in a request." + ) + } + } + + if (iq.relays.size > 1) { + throw FeatureNotImplementedException("Updating more than one 'relay' in a request.") + } + } +} + +/** + * Return true if "unknown endpoint" and "unknown relay" errors should be ignored when handling [iq]. We ignore errors + * if the request simply expires endpoints or a relay (since presumably the entity is already expired), or if it is a + * batch update of force-mute state (for the same reason, in order to simplify error handling). + */ +private fun shouldIgnoreUnknownEndpoints(iq: ConferenceModifyIQ): Boolean { + if (iq.endpoints.any { it.create || it.media.size > 0 || it.transport != null || it.sources != null }) { + return false + } + if (iq.relays.any { !it.expire }) { + return false + } + return true +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt new file mode 100644 index 0000000..526b345 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/Colibri2Util.kt @@ -0,0 +1,60 @@ +/* + * Copyright @ 2022 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.colibri2 + +import org.jitsi.xmpp.extensions.colibri2.Colibri2Endpoint +import org.jitsi.xmpp.extensions.colibri2.Colibri2Error +import org.jitsi.xmpp.util.createError +import org.jivesoftware.smack.packet.IQ +import org.jivesoftware.smack.packet.StanzaError + +fun createConferenceAlreadyExistsError(iq: IQ, conferenceId: String) = createError( + iq, + StanzaError.Condition.conflict, + "Conference already exists for ID: $conferenceId", + Colibri2Error(Colibri2Error.Reason.CONFERENCE_ALREADY_EXISTS) +) + +fun createConferenceNotFoundError(iq: IQ, conferenceId: String) = createError( + iq, + StanzaError.Condition.item_not_found, + "Conference not found for ID: $conferenceId", + Colibri2Error(Colibri2Error.Reason.CONFERENCE_NOT_FOUND) +) + +fun createGracefulShutdownErrorResponse(iq: IQ): IQ = createError( + iq, + StanzaError.Condition.service_unavailable, + "In graceful shutdown", + Colibri2Error(Colibri2Error.Reason.GRACEFUL_SHUTDOWN) +) + +fun createEndpointNotFoundError(iq: IQ, endpointId: String) = createError( + iq, + StanzaError.Condition.item_not_found, + "Endpoint not found for ID: $endpointId", + listOf( + Colibri2Error(Colibri2Error.Reason.UNKNOWN_ENDPOINT), + Colibri2Endpoint.getBuilder().setId(endpointId).build() + ) +) + +fun createFeatureNotImplementedError(iq: IQ, message: String?) = createError( + iq, + StanzaError.Condition.feature_not_implemented, + message ?: "", + Colibri2Error(Colibri2Error.Reason.FEATURE_NOT_IMPLEMENTED), +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt new file mode 100644 index 0000000..4464338 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/colibri2/IqProcessingException.kt @@ -0,0 +1,36 @@ +/* + * Copyright @ 2021-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.colibri2 + +import org.jivesoftware.smack.packet.StanzaError +import java.lang.Exception + +internal open class IqProcessingException( + val condition: StanzaError.Condition, + message: String +) : Exception(message) { + override fun toString() = "$condition $message" +} + +internal class UnknownEndpointException(val endpointId: String) : IqProcessingException( + StanzaError.Condition.item_not_found, + "Unknown endpoint $endpointId" +) + +internal class FeatureNotImplementedException(message: String) : IqProcessingException( + StanzaError.Condition.feature_not_implemented, + message +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt new file mode 100644 index 0000000..f2cf2d7 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpHandler.kt @@ -0,0 +1,69 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.dcsctp + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.NodeStatsBlock +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.videobridge.sctp.SctpConfig +import org.jitsi.videobridge.util.TaskPools +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.atomic.AtomicLong + +/** + * A node which can be placed in the pipeline to cache SCTP packets until + * the DcSctpTransport is ready to handle them. + */ +class DcSctpHandler : ConsumerNode("SCTP handler") { + private val sctpTransportLock = Any() + private var sctpTransport: DcSctpTransport? = null + private val numCachedSctpPackets = AtomicLong(0) + private val cachedSctpPackets = LinkedBlockingQueue(100) + + override fun consume(packetInfo: PacketInfo) { + synchronized(sctpTransportLock) { + if (SctpConfig.config.enabled) { + sctpTransport?.handleIncomingSctp(packetInfo) ?: run { + numCachedSctpPackets.incrementAndGet() + cachedSctpPackets.add(packetInfo) + } + } + } + } + + override fun getNodeStats(): NodeStatsBlock = super.getNodeStats().apply { + addNumber("num_cached_packets", numCachedSctpPackets.get()) + } + + fun setSctpTransport(sctpTransport: DcSctpTransport) { + // Submit this to the pool since we wait on the lock and process any + // cached packets here as well + TaskPools.IO_POOL.execute { + // We grab the lock here so that we can set the SCTP transport and + // process any previously-cached packets as an atomic operation. + // It also prevents another thread from coming in via + // #doProcessPackets and processing packets at the same time in + // another thread, which would be a problem. + synchronized(sctpTransportLock) { + this.sctpTransport = sctpTransport + cachedSctpPackets.forEach { sctpTransport.handleIncomingSctp(it) } + cachedSctpPackets.clear() + } + } + } + + override fun trace(f: () -> Unit) = f.invoke() +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt new file mode 100644 index 0000000..bb48356 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/dcsctp/DcSctpTransport.kt @@ -0,0 +1,204 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.dcsctp + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.dcsctp4j.DcSctpMessage +import org.jitsi.dcsctp4j.DcSctpOptions +import org.jitsi.dcsctp4j.DcSctpSocketCallbacks +import org.jitsi.dcsctp4j.DcSctpSocketFactory +import org.jitsi.dcsctp4j.DcSctpSocketInterface +import org.jitsi.dcsctp4j.SendOptions +import org.jitsi.dcsctp4j.SendStatus +import org.jitsi.dcsctp4j.Timeout +import org.jitsi.nlj.PacketInfo +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.sctp.SctpConfig +import org.jitsi.videobridge.util.TaskPools +import java.lang.ref.WeakReference +import java.time.Clock +import java.time.Instant +import java.util.concurrent.Future +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.ThreadLocalRandom +import java.util.concurrent.TimeUnit + +class DcSctpTransport( + val name: String, + parentLogger: Logger +) { + val logger = createChildLogger(parentLogger) + private val lock = Any() + private var socket: DcSctpSocketInterface? = null + + fun start(callbacks: DcSctpSocketCallbacks, options: DcSctpOptions = DEFAULT_SOCKET_OPTIONS) { + synchronized(lock) { + socket = factory.create(name, callbacks, null, options) + } + } + + fun handleIncomingSctp(packetInfo: PacketInfo) { + val packet = packetInfo.packet + synchronized(lock) { + socket?.receivePacket(packet.getBuffer(), packet.getOffset(), packet.getLength()) + } + } + + fun stop() { + synchronized(lock) { + socket?.close() + socket = null + } + } + + fun connect() { + synchronized(lock) { + socket?.connect() + } + } + + fun send(message: DcSctpMessage, options: SendOptions): SendStatus { + synchronized(lock) { + return socket?.send(message, options) ?: SendStatus.kErrorShuttingDown + } + } + + fun handleTimeout(timeoutId: Long) { + synchronized(lock) { + socket?.handleTimeout(timeoutId) + } + } + + fun getDebugState(): ObjectNode { + val metrics = synchronized(lock) { + socket?.metrics + } + return JsonNodeFactory.instance.objectNode().apply { + if (metrics != null) { + put("tx_packets_count", metrics.txPacketsCount) + put("tx_messages_count", metrics.txMessagesCount) + put("rtx_packets_count", metrics.rtxPacketsCount) + put("rtx_bytes_count", metrics.rtxBytesCount) + put("cwnd_bytes", metrics.cwndBytes) + put("srtt_ms", metrics.srttMs) + put("unack_data_count", metrics.unackDataCount) + put("rx_packets_count", metrics.rxPacketsCount) + put("rx_messages_count", metrics.rxMessagesCount) + put("peer_rwnd_bytes", metrics.peerRwndBytes) + put("peer_implementation", metrics.peerImplementation.name) + put("uses_message_interleaving", metrics.usesMessageInterleaving()) + put("uses_zero_checksum", metrics.usesZeroChecksum()) + put("negotiated_maximum_incoming_streams", metrics.negotiatedMaximumIncomingStreams) + put("negotiated_maximum_outgoing_streams", metrics.negotiatedMaximumOutgoingStreams) + } + } + } + + companion object { + private val factory by lazy { + check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } + DcSctpSocketFactory() + } + + /* Copying value set by Chrome's dcsctp_transport. */ + const val DEFAULT_MAX_TIMER_DURATION = 3000L + + val DEFAULT_SOCKET_OPTIONS by lazy { + check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } + DcSctpOptions().apply { + maxTimerBackoffDuration = DEFAULT_MAX_TIMER_DURATION + // Because we're making retransmits faster, we need to allow unlimited retransmits + // or SCTP can time out (which we don't handle). Peer connection timeouts are handled at + // a higher layer. + maxRetransmissions = null + maxInitRetransmits = null + } + } + + val DEFAULT_SEND_OPTIONS by lazy { + check(SctpConfig.config.enabled()) { "SCTP is disabled in configuration" } + SendOptions() + } + + const val DEFAULT_SCTP_PORT: Int = 5000 + } +} + +abstract class DcSctpBaseCallbacks( + val transport: DcSctpTransport, + val clock: Clock = Clock.systemUTC() +) : DcSctpSocketCallbacks { + /* Methods we can usefully implement for every JVB socket */ + override fun createTimeout(p0: DcSctpSocketCallbacks.DelayPrecision): Timeout = ATimeout(transport) + + override fun Now(): Instant = clock.instant() + + override fun getRandomInt(low: Long, high: Long): Long = ThreadLocalRandom.current().nextLong(low, high) + + /* Methods we wouldn't normally expect to be called for a JVB SCTP socket. */ + override fun OnConnectionRestarted() { + transport.logger.info("Surprising SCTP callback: connection restarted") + } + + override fun OnStreamsResetFailed(outgoingStreams: ShortArray, reason: String) { + transport.logger.info( + "Surprising SCTP callback: streams ${outgoingStreams.joinToString()} reset failed: $reason" + ) + } + + override fun OnStreamsResetPerformed(outgoingStreams: ShortArray) { + // This is normal following a call to close(), which is a hard-close (as opposed to shutdown() which is + // soft-close) + transport.logger.info("Outgoing streams ${outgoingStreams.joinToString()} reset") + } + + override fun OnIncomingStreamsReset(incomingStreams: ShortArray) { + /* Does Chrome ever reset streams? */ + transport.logger.info("Surprising SCTP callback: incoming streams ${incomingStreams.joinToString()} reset") + } + + private class ATimeout(transport: DcSctpTransport) : Timeout { + // This holds a weak reference to the transport, to break JNI reference cycles + private val weakTransport = WeakReference(transport) + private val transport: DcSctpTransport? + get() = weakTransport.get() + private var timeoutId: Long = 0 + private var scheduledFuture: ScheduledFuture<*>? = null + private var future: Future<*>? = null + override fun start(duration: Long, timeoutId: Long) { + try { + this.timeoutId = timeoutId + scheduledFuture = TaskPools.SCHEDULED_POOL.schedule({ + /* Execute it on the IO_POOL, because a timer may trigger sending new SCTP packets. */ + future = TaskPools.IO_POOL.submit { + transport?.handleTimeout(timeoutId) + } + }, duration, TimeUnit.MILLISECONDS) + } catch (e: Throwable) { + transport?.logger?.warn("Exception scheduling DCSCTP timeout", e) + } + } + + override fun stop() { + scheduledFuture?.cancel(false) + future?.cancel(false) + scheduledFuture = null + future = null + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/export/Exporter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/export/Exporter.kt new file mode 100644 index 0000000..1496d7b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/export/Exporter.kt @@ -0,0 +1,554 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.eclipse.jetty.websocket.api.Callback +import org.eclipse.jetty.websocket.api.Session +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest +import org.eclipse.jetty.websocket.client.WebSocketClient +import org.jitsi.config.JitsiConfig +import org.jitsi.mediajson.Event +import org.jitsi.mediajson.InfoEvent +import org.jitsi.mediajson.MediaEvent +import org.jitsi.mediajson.PingEvent +import org.jitsi.mediajson.PongEvent +import org.jitsi.mediajson.SessionEndEvent +import org.jitsi.mediajson.SourcesEvent +import org.jitsi.mediajson.StartEvent +import org.jitsi.mediajson.StopEvent +import org.jitsi.mediajson.TranscriptionResultEvent +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.optionalconfig +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.util.PacketInfoQueue +import org.jitsi.utils.logging2.Logger +import org.jitsi.videobridge.PotentialPacketHandler +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.videobridge.util.ByteBufferPool +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.version.JvbVersionService +import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig +import org.jitsi.xmpp.extensions.colibri2.Connect +import java.net.URI +import java.time.Duration +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import kotlin.math.min +import kotlin.math.pow + +internal class Exporter( + private val url: URI, + private val httpHeaders: Map, + val logger: Logger, + /** The conference-side sinks and SSRC lookups this exporter calls back into. */ + private val eventHandler: ExporterEventHandler, + private val pingEnabled: Boolean = false, + private val pingIntervalMs: Int = 0, + private val pingTimeoutMs: Int = 0, + private val type: Connect.Types = Connect.Types.RECORDER, + /** Source names to export (send out), communicated to the peer via a [SourcesEvent]. */ + @Volatile private var exports: List = emptyList(), + /** Source names requested (to receive back), communicated to the peer via a [SourcesEvent]. */ + @Volatile private var requests: List = emptyList() +) : PotentialPacketHandler { + private val isShuttingDown = AtomicBoolean(false) + private val reconnectAttempts = AtomicInteger(0) + private var reconnectFuture: ScheduledFuture<*>? = null + + private var connectionOpenedAtMs: Long = 0 + + // Ping/pong state + private var pingScheduledFuture: ScheduledFuture<*>? = null + private var pingTimeoutFuture: ScheduledFuture<*>? = null + private val nextPingId = AtomicInteger(0) + private val lastPingSentId = AtomicInteger(0) + private val lastPongReceivedMs = AtomicLong(0) + + // Instance-level counters for debugState + private val instancePacketsSent = AtomicLong(0) + private val instanceWebSocketFailures = AtomicLong(0) + private val instanceWebSocketInternalErrors = AtomicLong(0) + private val instanceStarts = AtomicLong(0) + private val instanceTranscriptsReceived = AtomicLong(0) + private val instanceMediaEventsReceived = AtomicLong(0) + private val instanceSendingChangesReceived = AtomicLong(0) + private val instanceOtherMessagesReceived = AtomicLong(0) + private val instanceParseFailures = AtomicLong(0) + private val instanceInfoReceived = AtomicLong(0) + + val queue: PacketInfoQueue by lazy { + PacketInfoQueue( + "${javaClass.simpleName}-packet-queue", + TaskPools.IO_POOL, + this::doHandlePacket, + 1024 + ) + } + inner class RecorderWebSocket : Session.Listener.AutoDemanding { + @Volatile + var session: Session? = null + + val isConnected: Boolean + get() = session != null + + override fun onWebSocketClose(statusCode: Int, reason: String?) { + val openedAt = connectionOpenedAtMs + val stableConnection = openedAt > 0 && + System.currentTimeMillis() - openedAt > stableConnectionThreshold.toMillis() + if (stableConnection) { + reconnectAttempts.set(0) + connectionOpenedAtMs = 0 + } + + session = null + logger.info("Websocket closed with status $statusCode, reason: $reason") + stopPing() + val internalError = statusCode == 1011 + if (internalError) { + webSocketInternalErrors.inc() + instanceWebSocketInternalErrors.incrementAndGet() + } + if (!isShuttingDown.get()) { + scheduleReconnect() + } + } + + override fun onWebSocketOpen(session: Session) { + this.session = session + connectionOpenedAtMs = System.currentTimeMillis() + logger.info("Websocket connected: $isConnected") + serializer = initSerializer() + cancelReconnect() + startPing() + // Declare our exported/requested sources to the peer, if any, now that the connection is up. + if (exports.isNotEmpty() || requests.isNotEmpty()) { + sendSources() + } + sendInfo() + } + + override fun onWebSocketError(cause: Throwable?) { + logger.error("Websocket error", cause) + webSocketFailures.inc() + instanceWebSocketFailures.incrementAndGet() + if (!isShuttingDown.get()) { + scheduleReconnect() + } + } + + override fun onWebSocketText(message: String) { + handleIncomingMessage(message) + } + } + + private val recorderWebSocket = RecorderWebSocket() + + private var serializer: MediaJsonSerializer? = null + + private fun initSerializer() = MediaJsonSerializer(eventHandler::getAudioSourceName, eventHandler::getDiarize) { + val session = recorderWebSocket.session + if (session != null) { + session.sendText(it.toJson(), Callback.NOOP) + } else { + logger.warn("Not connected, cannot send event: $it") + } + } + + fun isConnected() = recorderWebSocket.isConnected + + /** + * Whether to forward this packet to this exporter: it must be audio from a source we export. An empty [exports] + * list means "export all audio"; otherwise only audio from a source named in [exports] is wanted. Either way the + * SSRC must resolve to a signaled source: audio from an unknown (e.g. not-yet-signaled) SSRC isn't routed to + * receivers, so it isn't exported either. + */ + override fun wants(packet: PacketInfo): Boolean { + if (!isConnected()) return false + val audioPacket = packet.packet as? AudioRtpPacket ?: return false + val sourceName = eventHandler.getAudioSourceName(audioPacket.ssrc) ?: return false + return exports.isEmpty() || sourceName in exports + } + + /** + * Apply an update to the connect with this exporter's URL. Only the exported/requested source names may change + * on a running exporter; changes to other parameters are rejected by [ExporterWrapper] before reaching here. + */ + fun update(exports: List, requests: List) { + logger.info("Updating exports=$exports requests=$requests") + this.exports = exports + this.requests = requests + // If we're connected, propagate the change now; otherwise it will be sent when the connection opens. + sendSources() + } + + /** Send the current exported/requested source names to the peer. No-op if not currently connected. */ + private fun sendSources() { + val session = recorderWebSocket.session ?: return + logger.info("Sending sources: exports=$exports requests=$requests") + session.sendText(SourcesEvent(exports, requests).toJson(), Callback.NOOP) + } + + /** Internal (rather than private) so [ExporterTest] can drive the event dispatch without opening a socket. */ + internal fun handleIncomingMessage(message: String) { + try { + val event = Event.parse(message) + logger.debug { "Received message from websocket: ${event.toJson()}" } + + when (event) { + is TranscriptionResultEvent -> { + transcriptsReceivedCount.inc() + instanceTranscriptsReceived.incrementAndGet() + eventHandler.handleTranscriptionResult(event) + } + + is PongEvent -> { + val expectedId = lastPingSentId.get() + if (event.id == expectedId) { + logger.debug { "Received pong with matching id=${event.id}" } + lastPongReceivedMs.set(System.currentTimeMillis()) + // Cancel any pending timeout + pingTimeoutFuture?.cancel(false) + pingTimeoutFuture = null + } else { + logger.warn("Received pong with id=${event.id}, expected id=$expectedId") + } + } + + is MediaEvent -> { + mediaEventsReceivedCount.inc() + instanceMediaEventsReceived.incrementAndGet() + eventHandler.handleMediaEvent(event) + } + + is StartEvent -> { + // A `start` with a talk timestamp brackets the beginning of a "talk" (translated audio); one + // without is a plain stream-start announcement (the bridge->peer direction), which the bridge + // does not act on when received. + val timestamp = event.start.timestamp + if (timestamp != null) { + sendingChangesReceivedCount.inc() + instanceSendingChangesReceived.incrementAndGet() + eventHandler.handleSendingChange(event.start.tag, true, timestamp) + } else { + otherMessagesReceivedCount.inc() + instanceOtherMessagesReceived.incrementAndGet() + } + } + + is StopEvent -> { + // A `stop` with a talk timestamp brackets the end of a "talk"; a plain stop without one is a + // stream-end the bridge does not act on when received. + val timestamp = event.stop.timestamp + if (timestamp != null) { + sendingChangesReceivedCount.inc() + instanceSendingChangesReceived.incrementAndGet() + eventHandler.handleSendingChange(event.stop.tag, false, timestamp) + } else { + otherMessagesReceivedCount.inc() + instanceOtherMessagesReceived.incrementAndGet() + } + } + + is InfoEvent -> { + infoReceivedCount.inc() + instanceInfoReceived.incrementAndGet() + logger.info("Received InfoEvent: ${event.toJson()}") + } + + else -> { + otherMessagesReceivedCount.inc() + instanceOtherMessagesReceived.incrementAndGet() + } + } + } catch (e: Exception) { + logger.warn("Failed to parse incoming websocket message: $message", e) + parseFailuresCount.inc() + instanceParseFailures.incrementAndGet() + } + } + + /** + * Send an informational message describing this application to the peer, once the connection + * is up. Used for runtime observability (the peer logs what it received). No-op if disconnected. + */ + private fun sendInfo() { + val session = recorderWebSocket.session ?: return + val info = InfoEvent() + .put("application", "jitsi-videobridge") + .put("version", JvbVersionService.instance.currentVersion.toString()) + RelayConfig.config.region?.let { info.put("region", it) } + try { + logger.info("Sending info to transcriber: ${info.toJson()}") + session.sendText(info.toJson(), Callback.NOOP) + } catch (e: Exception) { + logger.warn("Failed to send info message", e) + } + } + + private fun sendPing() { + if (!recorderWebSocket.isConnected || isShuttingDown.get()) { + return + } + + val pingId = nextPingId.incrementAndGet() + val pingEvent = PingEvent(pingId) + + try { + recorderWebSocket.session?.sendText(pingEvent.toJson(), Callback.NOOP) + lastPingSentId.set(pingId) + logger.debug { "Sent ping with id=$pingId" } + + // Schedule timeout check + pingTimeoutFuture = TaskPools.SCHEDULED_POOL.schedule({ + handlePingTimeout() + }, pingTimeoutMs.toLong(), TimeUnit.MILLISECONDS) + } catch (e: Exception) { + logger.warn("Failed to send ping message", e) + } + } + + private fun handlePingTimeout() { + logger.warn("Ping timeout, reconnecting websocket") + + // Force reconnect on timeout + stopPing() + recorderWebSocket.session?.close(1000, "Ping timeout", Callback.NOOP) + scheduleReconnect() + } + + private fun startPing() { + if (!pingEnabled || pingIntervalMs <= 0) { + return + } + + logger.info("Starting ping with interval=$pingIntervalMs ms, timeout=$pingTimeoutMs ms") + stopPing() + + // Schedule recurring ping + pingScheduledFuture = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate( + { sendPing() }, + pingIntervalMs.toLong(), + pingIntervalMs.toLong(), + TimeUnit.MILLISECONDS + ) + } + + private fun stopPing() { + pingScheduledFuture?.cancel(false) + pingScheduledFuture = null + pingTimeoutFuture?.cancel(false) + pingTimeoutFuture = null + } + + /** Run inside the queue thread, handle a packet. */ + private fun doHandlePacket(packet: PacketInfo): Boolean { + if (recorderWebSocket.isConnected) { + serializer?.encode(packet) + packetsSentCount.inc() + instancePacketsSent.incrementAndGet() + } + ByteBufferPool.returnBuffer(packet.packet.buffer) + return true + } + + override fun send(packet: PacketInfo) { + if (recorderWebSocket.isConnected) { + queue.add(packet) + } else { + ByteBufferPool.returnBuffer(packet.packet.buffer) + } + } + + private fun scheduleReconnect() { + if (isShuttingDown.get()) { + return + } + + val attempt = reconnectAttempts.incrementAndGet() + maxReconnectAttempts?.let { + if (attempt > it) { + logger.warn("Max reconnection attempts ($it) reached, giving up") + return + } + } + + val delayMs = getDelayMs(attempt) + logger.info("Scheduling reconnection attempt $attempt in $delayMs ms") + + cancelReconnect() + reconnectFuture = TaskPools.SCHEDULED_POOL.schedule({ + if (!isShuttingDown.get()) { + try { + logger.info("Attempting reconnection (attempt $attempt)") + webSocketClient.connect(recorderWebSocket, url, createUpgradeRequest()) + } catch (e: Exception) { + logger.warn("Reconnection attempt $attempt failed", e) + scheduleReconnect() + } + } + }, delayMs, TimeUnit.MILLISECONDS) + } + + private fun cancelReconnect() { + reconnectFuture?.cancel(false) + reconnectFuture = null + } + + private fun createUpgradeRequest() = ClientUpgradeRequest().apply { + httpHeaders.forEach { (name, value) -> + setHeader(name, value) + } + } + + fun start() { + isShuttingDown.set(false) + startsCount.inc() + instanceStarts.incrementAndGet() + webSocketClient.connect(recorderWebSocket, url, createUpgradeRequest()) + } + + fun stop() { + isShuttingDown.set(true) + stopPing() + cancelReconnect() + if (recorderWebSocket.isConnected) { + recorderWebSocket.session?.sendText(SessionEndEvent().toJson(), Callback.NOOP) + } + recorderWebSocket.session?.close( + org.eclipse.jetty.websocket.core.CloseStatus.SHUTDOWN, + "closing", + Callback.NOOP + ) + recorderWebSocket.session?.disconnect() + queue.close() + } + + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("url", url.toString()) + put("type", type.toString().lowercase()) + putArray("exports").apply { exports.forEach { add(it) } } + putArray("requests").apply { requests.forEach { add(it) } } + put("is_connected", isConnected()) + put("is_shutting_down", isShuttingDown.get()) + put("reconnect_attempts", reconnectAttempts.get()) + put("packets_sent", instancePacketsSent.get()) + put("websocket_failures", instanceWebSocketFailures.get()) + put("websocket_internal_errors", instanceWebSocketInternalErrors.get()) + put("starts", instanceStarts.get()) + put("transcripts_received", instanceTranscriptsReceived.get()) + put("media_events_received", instanceMediaEventsReceived.get()) + put("sending_changes_received", instanceSendingChangesReceived.get()) + put("other_messages_received", instanceOtherMessagesReceived.get()) + put("parse_failures", instanceParseFailures.get()) + put("info_received", instanceInfoReceived.get()) + put("queue_size", queue.size()) + put("ping_enabled", pingEnabled) + if (pingEnabled) { + put("ping_interval_ms", pingIntervalMs) + put("ping_timeout_ms", pingTimeoutMs) + put("last_pong_received_ms", lastPongReceivedMs.get()) + } + } + + companion object { + private val webSocketClient = WebSocketClient().apply { + idleTimeout = WebsocketServiceConfig.config.idleTimeout + start() + } + + private val webSocketFailures = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_websocket_failures", + "Number of websocket connection failures from Exporter" + ) + + private val webSocketInternalErrors = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_websocket_internal_errors", + "Number of websocket connection which were connected but closed with code 1011" + ) + + private val startsCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_starts", + "Number of times Exporter has been started" + ) + + private val packetsSentCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_packets_sent", + "Number of packets sent by Exporter" + ) + + private val transcriptsReceivedCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_transcripts_received", + "Number of transcription results received by Exporter" + ) + + private val mediaEventsReceivedCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_media_events_received", + "Number of media (translated audio) events received by Exporter" + ) + + private val sendingChangesReceivedCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_sending_changes_received", + "Number of synthetic-source sending-change (start/stop) events received by Exporter" + ) + + private val otherMessagesReceivedCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_other_messages_received", + "Number of non-transcription messages received by Exporter" + ) + + private val parseFailuresCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_parse_failures", + "Number of messages that failed to parse" + ) + + private val infoReceivedCount = VideobridgeMetricsContainer.instance.registerCounter( + "exporter_info_received", + "Number of info messages received by Exporter" + ) + + private val maxReconnectAttempts: Int? by optionalconfig { + "videobridge.exporter.max-reconnect-attempts".from(JitsiConfig.newConfig) + } + + private val baseDelay: Duration by config { + "videobridge.exporter.base-delay".from(JitsiConfig.newConfig) + } + + private val maxDelay: Duration by config { + "videobridge.exporter.max-delay".from(JitsiConfig.newConfig) + } + + val stableConnectionThreshold: Duration by config { + "videobridge.exporter.stable-connection-threshold".from(JitsiConfig.newConfig) + } + + // 0, base, base * 2, max at 30 seconds. The min is computed in Double space and converted afterwards so that + // a large attempt count (where 2.0.pow overflows to Infinity) clamps to maxDelay instead of overflowing Long + // and producing a negative (immediate) delay. + private fun getDelayMs(attempt: Int) = if (attempt == 1) { + 0L + } else { + min(baseDelay.toMillis() * 2.0.pow(attempt - 2), maxDelay.toMillis().toDouble()).toLong() + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/export/ExporterEventHandler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/export/ExporterEventHandler.kt new file mode 100644 index 0000000..223ceeb --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/export/ExporterEventHandler.kt @@ -0,0 +1,49 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import org.jitsi.mediajson.MediaEvent +import org.jitsi.mediajson.TranscriptionResultEvent + +/** + * The conference-side operations an [Exporter] (and the [ExporterWrapper] that owns it) depends on: sinks for the + * events a peer sends back, plus lookups resolving an audio SSRC to conference state. Implemented by the conference + * and passed in, so the exporter subsystem doesn't depend on the conference directly and can be driven by a fake in + * tests. + */ +interface ExporterEventHandler { + /** Handles a transcription result received back from a peer. */ + fun handleTranscriptionResult(event: TranscriptionResultEvent) + + /** Handles a translated-audio media event received back from a peer. */ + fun handleMediaEvent(event: MediaEvent) + + /** + * Handles a synthetic source's sending-state change, derived from the `start`/`stop` mediajson events a peer sends + * to bracket a "talk". + * + * @param sourceName the synthetic source whose sending state changed. + * @param sending true at the start of a talk, false at its end. + * @param timestamp the talk boundary's RTP timestamp (48 kHz); for a stop, one past the end of the run. + */ + fun handleSendingChange(sourceName: String, sending: Boolean, timestamp: Long) + + /** Resolves an audio SSRC to its source name, used to filter outbound audio by a connect's exports. */ + fun getAudioSourceName(ssrc: Long): String? + + /** Resolves an audio SSRC to whether diarization is requested for its endpoint (colibri2 `diarize` attribute). */ + fun getDiarize(ssrc: Long): Boolean +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/export/ExporterWrapper.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/export/ExporterWrapper.kt new file mode 100644 index 0000000..87b06e5 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/export/ExporterWrapper.kt @@ -0,0 +1,284 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.PotentialPacketHandler +import org.jitsi.videobridge.colibri2.FeatureNotImplementedException +import org.jitsi.videobridge.colibri2.IqProcessingException +import org.jitsi.xmpp.extensions.colibri2.Connect +import org.jivesoftware.smack.packet.StanzaError +import java.net.URI +import java.util.concurrent.ConcurrentHashMap + +class ExporterWrapper internal constructor( + parentLogger: Logger, + /** The conference-side sinks and SSRC lookups each [Exporter] calls back into. */ + private val eventHandler: ExporterEventHandler, + /** Creates (and starts) the [Exporter] for a connect. Overridable for testing; defaults to the real one. */ + exporterFactory: ((Connect) -> Exporter)? +) { + constructor( + parentLogger: Logger, + eventHandler: ExporterEventHandler + ) : this( + parentLogger, + eventHandler, + null + ) + + val logger = createChildLogger(parentLogger) + + @Volatile + var started = false + + /** Set once [stop] has run (the conference expired), so a racing [applyConnects] can't start a new exporter + * that nothing would ever stop. */ + private var stopped = false + + private val exporterFactory: (Connect) -> Exporter = exporterFactory ?: ::createExporter + + /** + * One running [Exporter] per requested connect, keyed by the connect's id. Mutated only under the object lock + * ([applyConnects] runs on the conference's colibri queue or, for colibri-over-REST, a Jetty thread; [stop] on + * the conference-expiry thread), but iterated without it by [debugState] on the debug REST thread, so it must + * also be concurrent. + */ + private val exporters = ConcurrentHashMap() + + /** + * The running exporters exposed as [PotentialPacketHandler]s for the conference's send path. Each exporter is an + * independent handler, so the conference's send loop clones a packet only when it actually goes to more than one + * handler (and not at all for an exporter that filters the packet out via its own [PotentialPacketHandler.wants]). + * Cached and recomputed only when the set of exporters changes, so it allocates nothing on the per-packet path. + */ + @Volatile + private var packetHandlers: List = emptyList() + + /** The current exporters as packet handlers, for the conference's send path. */ + fun getPacketHandlers(): List = packetHandlers + + /** + * Apply a delta of connects, using the connect's [Connect.id] as its identity. Connects not mentioned are left + * running unchanged. Each connect is one of: + * - `expire=true`: stop and remove the exporter with that id (a no-op with a warning if it isn't running), + * - `create=true`: start a new exporter; rejected if an exporter with that id is already running, + * - neither: update the already-running exporter with that id; rejected if none is running (i.e. a connect for a + * new id must set `create`). Only the exported/requested source names can change live; any other change is + * rejected. + * + * Expires are applied before creates, so a single delta may both expire and re-create the same id (replacing a + * connect, e.g. to change its URL, in one request); any other repeated mention of an id is rejected. + * + * The whole delta is validated before anything is applied, so a rejected connect doesn't disturb running exporters. + */ + @Synchronized + fun applyConnects(connects: List) { + if (stopped) { + throw IqProcessingException(StanzaError.Condition.bad_request, "The conference's exporter is stopped") + } + // Validate up front so a rejected connect doesn't disturb the already-running exporters. + connects.forEach { validate(it) } + + // Pass 1: classify each connect and fail before mutating anything. Expires are collected first so that, + // regardless of the order within the delta, a create is checked against the map as it will be after the + // expires are applied. + val toExpire = mutableSetOf() + connects.filter { it.expire }.forEach { connect -> + if (!toExpire.add(connect.id)) { + throw IqProcessingException( + StanzaError.Condition.bad_request, + "Duplicate expire for id=${connect.id}" + ) + } + } + val toCreate = mutableListOf() + val toUpdate = mutableListOf>() + connects.filterNot { it.expire }.forEach { connect -> + val id = connect.id + if (toCreate.any { it.id == id } || toUpdate.any { it.first.id == id }) { + throw IqProcessingException( + StanzaError.Condition.bad_request, + "Multiple connects for id=$id" + ) + } + if (connect.create) { + if (exporters.containsKey(id) && id !in toExpire) { + throw IqProcessingException( + StanzaError.Condition.bad_request, + "Connect with create=true for an already-existing id=$id" + ) + } + toCreate += connect + } else { + if (id in toExpire) { + throw IqProcessingException( + StanzaError.Condition.bad_request, + "Connect update for id=$id expired in the same request" + ) + } + val running = exporters[id] ?: throw IqProcessingException( + StanzaError.Condition.bad_request, + "Connect for unknown id=$id without create=true" + ) + val params = ConnectParams(connect) + val unsupported = running.params.nonUpdatableChangesTo(params) + if (unsupported.isNotEmpty()) { + throw FeatureNotImplementedException( + "Updating connect ${unsupported.joinToString()} for id=$id" + ) + } + toUpdate += connect to params + } + } + + // Pass 2: apply (expires before creates, so an expire+create of the same id replaces it). + toExpire.forEach { id -> + val removed = exporters.remove(id) + if (removed != null) { + logger.info("Stopping exporter id=$id") + removed.exporter.stop() + } else { + logger.warn("Received expire for unknown connect id=$id") + } + } + toCreate.forEach { connect -> + exporters[connect.id] = Entry(exporterFactory(connect), ConnectParams(connect)) + } + toUpdate.forEach { (connect, params) -> + val existing = exporters.getValue(connect.id) + if (existing.params != params) { + logger.info("Updating exporter id=${connect.id}") + existing.exporter.update(params.exports, params.requests) + existing.params = params + } + } + + started = exporters.isNotEmpty() + packetHandlers = exporters.values.map { it.exporter } + } + + @Synchronized + fun stop() { + if (started) { + logger.info("Stopping.") + } + stopped = true + started = false + exporters.values.forEach { it.exporter.stop() } + exporters.clear() + packetHandlers = emptyList() + } + + private fun validate(connect: Connect) { + if (connect.video) throw FeatureNotImplementedException("Video") + if (connect.protocol != Connect.Protocols.MEDIAJSON) { + throw FeatureNotImplementedException("Protocol ${connect.protocol}") + } + } + + private fun createExporter(connect: Connect): Exporter { + logger.info("Starting with url=${connect.url}") + val httpHeaders = connect.getHttpHeaders().associate { header -> + header.name to header.value + } + + // Extract ping configuration if present + val ping = connect.getPing() + val pingEnabled = ping != null + // Default values in case ping is enabled, but no values are specified. + val pingIntervalMs = ping?.interval ?: 10000 + val pingTimeoutMs = ping?.timeout ?: 3000 + + // The source names to export (send out) and request (receive back). Not yet acted upon. + val exports = connect.getExports() + val requests = connect.getRequests() + + return Exporter( + connect.url, + httpHeaders, + logger, + eventHandler, + pingEnabled, + pingIntervalMs, + pingTimeoutMs, + connect.type, + exports, + requests + ).apply { + start() + } + } + + fun debugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("started", started) + val exportersArray = putArray("exporters") + // Tag each exporter's debug info with its connect id (the identity it is stored and signaled under) so debug + // entries can be correlated to a specific connect. + exporters.forEach { (id, entry) -> exportersArray.add(entry.exporter.debugState().put("tag", id)) } + } + + /** A running exporter together with the connect parameters it was last (re)configured with. */ + private class Entry(val exporter: Exporter, var params: ConnectParams) + + /** + * The parameters of a [Connect] other than its id (which is its identity). Used to detect whether a re-signaled + * connect for an already-running id has changed and therefore needs to be passed to the exporter as an update. + * Source-name lists are normalized (sorted) so reordering alone isn't treated as a change. + */ + private data class ConnectParams( + val url: URI, + val protocol: Connect.Protocols, + val type: Connect.Types, + val audio: Boolean, + val video: Boolean, + val headers: Map, + val pingInterval: Int?, + val pingTimeout: Int?, + val exports: List, + val requests: List + ) { + constructor(connect: Connect) : this( + connect.url, + connect.protocol, + connect.type, + connect.audio, + connect.video, + connect.getHttpHeaders().associate { it.name to it.value }, + connect.getPing()?.interval, + connect.getPing()?.timeout, + connect.getExports().sorted(), + connect.getRequests().sorted() + ) + + /** + * The names of the parameters that differ from [other] and cannot be applied to an already-running exporter. + * Everything except [exports]/[requests] is connection-level and requires a reconnect to change, so a change + * to any of them is reported here (and rejected). protocol/video aren't included: [validate] already + * constrains them. + */ + fun nonUpdatableChangesTo(other: ConnectParams): List = buildList { + if (url != other.url) add("url") + if (type != other.type) add("type") + if (audio != other.audio) add("audio") + if (headers != other.headers) add("headers") + if (pingInterval != other.pingInterval || pingTimeout != other.pingTimeout) add("ping") + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/export/MediaJsonSerializer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/export/MediaJsonSerializer.kt new file mode 100644 index 0000000..78beb2e --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/export/MediaJsonSerializer.kt @@ -0,0 +1,154 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import org.jitsi.mediajson.CustomParameters +import org.jitsi.mediajson.Event +import org.jitsi.mediajson.Media +import org.jitsi.mediajson.MediaEvent +import org.jitsi.mediajson.MediaFormat +import org.jitsi.mediajson.Start +import org.jitsi.mediajson.StartEvent +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.AudioPayloadType +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.nlj.util.RtpTimestampIndexTracker +import org.jitsi.utils.logging2.createLogger +import java.time.Clock +import java.time.Duration +import java.time.Instant +import kotlin.io.encoding.Base64 +import kotlin.io.encoding.ExperimentalEncodingApi + +/** + * Encodes the media in a conference into a mediajson format. Maintains state for each SSRC in order to maintain a + * common space for timestamps. + * + */ +class MediaJsonSerializer( + /** Resolves an audio SSRC to its source name, which is used as the media-json tag. */ + private val getSourceName: (Long) -> String?, + /** Resolves an audio SSRC to whether diarization is requested for its endpoint (colibri2 `diarize` attribute). */ + private val getDiarize: (Long) -> Boolean, + /** Encoded mediajson events are sent to this function */ + private val handleEvent: (Event) -> Unit +) { + /** Reference time, timestamps are set relative to this instant. **/ + private val ref: Instant = Clock.systemUTC().instant() + private val ssrcsStarted = mutableMapOf() + + /** Global sequence number for all events */ + var seq = 0 + + val logger = createLogger() + fun encode(packetInfo: PacketInfo) = synchronized(ssrcsStarted) { + val p: AudioRtpPacket = packetInfo.packetAs() + val epId = packetInfo.endpointId ?: run { + logger.info("Ignoring packet without endpoint ID, SSRC ${p.ssrc}") + return@synchronized + } + val payloadType = packetInfo.payloadType ?: run { + logger.info("Ignoring packet without payloadType, SSRC ${p.ssrc}") + return@synchronized + } + + var state = ssrcsStarted[p.ssrc] + if (state == null) { + // Tag the stream with its source name. If the SSRC doesn't resolve to a signaled source (media racing + // colibri signaling), drop the packet: such audio isn't routed to receivers either, and starting the + // stream with a placeholder tag would strand its media events once the name resolves. The tag is kept + // in the state, so it stays consistent between the stream's start event and its media events. + val tag = getSourceName(p.ssrc) ?: run { + logger.info("Ignoring packet for SSRC ${p.ssrc} with no known source") + return@synchronized + } + state = createSsrcState(p.timestamp, payloadType, tag) + ssrcsStarted[p.ssrc] = state + logger.info("Starting SSRC ${p.ssrc} for endpoint $epId ") + handleEvent(createStart(tag, epId, payloadType, p.ssrc)) + } + + if (payloadType.pt != state.payloadType.pt) { + logger.info("SSRC ${p.ssrc} changed payload type from ${state.payloadType} to $payloadType.") + // An SSRC's source name doesn't change, so the new state (announced by a new start event) keeps the tag. + state = createSsrcState(p.timestamp, payloadType, state.tag) + ssrcsStarted[p.ssrc] = state + handleEvent(createStart(state.tag, epId, payloadType, p.ssrc)) + } + + handleEvent(encodeMedia(p, state, packetInfo.audioLevel, packetInfo.vad)) + } + + private fun createSsrcState(initialRtpTimestamp: Long, payloadType: PayloadType, tag: String): SsrcState { + val now = Clock.systemUTC().instant() + return SsrcState( + initialRtpTimestamp, + (Duration.between(ref, now).toNanos() * payloadType.clockRate.toDouble() * 1e-9).toLong(), + payloadType, + tag + ) + } + + private fun createStart(tag: String, epId: String, payloadType: PayloadType, ssrc: Long) = StartEvent( + ++seq, + Start( + tag, + MediaFormat( + payloadType.encodingName(), + payloadType.clockRate, + (payloadType as? AudioPayloadType)?.channels ?: 1, + payloadType.parameters + ), + CustomParameters(endpointId = epId), + // Only set the flag when diarization is on; when off, leave it null so the field is omitted from the wire + // format (jackson drops nulls) and the transcriber proxy falls back to its global config. + diarize = if (getDiarize(ssrc)) true else null + ) + ) + + @OptIn(ExperimentalEncodingApi::class) + private fun encodeMedia(p: AudioRtpPacket, state: SsrcState, audioLevel: Int?, vad: Boolean?): Event { + ++seq + return MediaEvent( + seq, + media = Media( + state.tag, + state.getSequenceNumber(p.sequenceNumber), + state.getTimestamp(p.timestamp), + Base64.encode(p.buffer, p.payloadOffset, p.payloadOffset + p.payloadLength), + audioLevel, + vad + ) + ) + } + + private class SsrcState( + initialRtpTimestamp: Long, + // Offset of this SSRC since the start time in RTP units + startOffset: Long, + val payloadType: PayloadType, + /** The stream's mediajson tag, as announced in its start event. */ + val tag: String + ) { + private val seqIndexTracker = RtpSequenceIndexTracker() + private val timestampIndexTracker = RtpTimestampIndexTracker().apply { update(initialRtpTimestamp) } + private val offset = startOffset - initialRtpTimestamp + fun getTimestamp(rtpTimestamp: Long): Long = offset + timestampIndexTracker.update(rtpTimestamp) + fun getSequenceNumber(seq: Int) = seqIndexTracker.update(seq).toInt() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt new file mode 100644 index 0000000..500103b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/JvbHealthChecker.kt @@ -0,0 +1,83 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.health + +import org.ice4j.ice.harvest.MappingCandidateHarvesters +import org.jitsi.health.HealthCheckService +import org.jitsi.health.HealthChecker +import org.jitsi.health.Result +import org.jitsi.videobridge.health.config.HealthConfig.Companion.config +import org.jitsi.videobridge.ice.Harvesters +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import java.net.InetAddress + +class JvbHealthChecker : HealthCheckService { + private val healthChecker = HealthChecker( + config.interval, + config.timeout, + config.maxCheckDuration, + config.stickyFailures, + healthCheckFunc = ::checkAndUpdateMetric + ) + + fun start() = healthChecker.start() + fun stop() = healthChecker.stop() + + private fun checkAndUpdateMetric(): Result = check().also { + healthyMetric.set(it.success) + } + + private fun check(): Result { + if (config.requireValidAddress && !hasValidAddress()) { + return Result(success = false, message = "No valid IP addresses available for harvesting.") + } + if (config.requireStun && MappingCandidateHarvesters.stunDiscoveryFailed) { + return Result(success = false, message = "Address discovery through STUN failed") + } + if (!Harvesters.INSTANCE.healthy) { + return Result(success = false, message = "Failed to bind single-port") + } + + // TODO: check if XmppConnection is configured and connected. + + return Result(success = true) + } + + private fun InetAddress.isValid(): Boolean = + !this.isSiteLocalAddress && !this.isLinkLocalAddress && !this.isLoopbackAddress + + private fun hasValidAddress(): Boolean { + if (Harvesters.INSTANCE.singlePortHarvesters.any { it.localAddress.address.isValid() }) { + return true + } + if (MappingCandidateHarvesters.getHarvesters().any { it.mask?.address?.isValid() == true }) { + return true + } + return false + } + + override val result: Result + get() = healthChecker.result + + companion object { + val healthyMetric = VideobridgeMetricsContainer.instance.registerBooleanMetric( + "healthy", + "Whether the Videobridge instance is healthy or not.", + true + ) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt new file mode 100644 index 0000000..db2a935 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/health/config/HealthConfig.kt @@ -0,0 +1,56 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.health.config + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import java.time.Duration + +class HealthConfig private constructor() { + val interval: Duration by config { + "org.jitsi.videobridge.health.INTERVAL" + .from(JitsiConfig.legacyConfig).convertFrom(Duration::ofMillis) + "videobridge.health.interval".from(JitsiConfig.newConfig) + } + + val timeout: Duration by config { + "org.jitsi.videobridge.health.TIMEOUT" + .from(JitsiConfig.legacyConfig).convertFrom(Duration::ofMillis) + "videobridge.health.timeout".from(JitsiConfig.newConfig) + } + + val maxCheckDuration: Duration by config("videobridge.health.max-check-duration".from(JitsiConfig.newConfig)) + + val stickyFailures: Boolean by config { + "org.jitsi.videobridge.health.STICKY_FAILURES".from(JitsiConfig.legacyConfig) + "videobridge.health.sticky-failures".from(JitsiConfig.newConfig) + } + + val requireStun: Boolean by config { + "videobridge.health.require-stun".from(JitsiConfig.newConfig) + } + + val requireValidAddress: Boolean by config { + "videobridge.health.require-valid-address".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = HealthConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt new file mode 100644 index 0000000..eb9253b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/Harvesters.kt @@ -0,0 +1,49 @@ +/* +* Copyright @ 2018 - present 8x8, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.jitsi.videobridge.ice + +import org.ice4j.ice.harvest.SinglePortUdpHarvester +import org.jitsi.utils.logging2.createLogger + +class Harvesters private constructor(val singlePortHarvesters: List) { + /* We're unhealthy if there are no single port harvesters. */ + val healthy: Boolean + get() = singlePortHarvesters.isNotEmpty() + + private fun close() { + singlePortHarvesters.forEach { it.close() } + } + + companion object { + private val logger = createLogger() + + fun init() { + // Trigger the lazy init. + INSTANCE + } + + fun close() = INSTANCE.close() + + val INSTANCE: Harvesters by lazy { + val singlePortHarvesters = SinglePortUdpHarvester.createHarvesters(IceConfig.config.port) + if (singlePortHarvesters.isEmpty()) { + logger.warn("No single-port harvesters created.") + } + + Harvesters(singlePortHarvesters) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt new file mode 100644 index 0000000..2119e8b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/ice/IceConfig.kt @@ -0,0 +1,106 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.ice + +import org.ice4j.ice.KeepAliveStrategy +import org.ice4j.ice.NominationStrategy +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.metaconfig.optionalconfig +import java.time.Duration + +class IceConfig private constructor() { + /** + * The ICE UDP port. + */ + val port: Int by config { + "org.jitsi.videobridge.SINGLE_PORT_HARVESTER_PORT".from(JitsiConfig.legacyConfig) + "videobridge.ice.udp.port".from(JitsiConfig.newConfig) + } + + /** + * The prefix to STUN username fragments we generate. + */ + val ufragPrefix: String? by optionalconfig { + "org.jitsi.videobridge.ICE_UFRAG_PREFIX".from(JitsiConfig.legacyConfig) + "videobridge.ice.ufrag-prefix".from(JitsiConfig.newConfig) + } + + val keepAliveStrategy: KeepAliveStrategy by config { + "org.jitsi.videobridge.KEEP_ALIVE_STRATEGY" + .from(JitsiConfig.legacyConfig) + .convertFrom { KeepAliveStrategy.fromString(it) } + "videobridge.ice.keep-alive-strategy" + .from(JitsiConfig.newConfig) + .convertFrom { KeepAliveStrategy.fromString(it) } + } + + val resolveRemoteCandidates: Boolean by config( + "videobridge.ice.resolve-remote-candidates".from(JitsiConfig.newConfig) + ) + + /** + * The ice4j nomination strategy policy. + */ + val nominationStrategy: NominationStrategy by config { + "videobridge.ice.nomination-strategy" + .from(JitsiConfig.newConfig) + .convertFrom { NominationStrategy.fromString(it) } + } + + /** + * Whether to advertise ICE candidates with private IP addresses (RFC1918 IPv4 addresses and + * fec0::/10 or fc00::/7 IPv6 addresses) even to endpoints that have not signaled support for private addresses. + */ + val advertisePrivateCandidates: Boolean by config( + "videobridge.ice.advertise-private-candidates".from(JitsiConfig.newConfig) + ) + + /** + * Whether ICE restarts are enabled: when an endpoint explicitly requests one (colibri2 + * ``), create a second [org.ice4j.ice.Agent] with rotated local credentials + * and run it alongside the established one, instead of rejecting the request. + */ + val restartEnabled: Boolean by config( + "videobridge.ice.restart.enabled".from(JitsiConfig.newConfig) + ) + + /** + * How long the old [org.ice4j.ice.Agent] is kept alive after an ICE restart has cut over to the new one. + * Both Agents keep their sockets during this window, and ice4j routes each packet by the address it came + * from, so the endpoint's old-generation checks — which come from its old address — are answered by the old + * Agent rather than dropped. + */ + val restartTransitionWindow: Duration by config( + "videobridge.ice.restart.transition-window".from(JitsiConfig.newConfig) + ) + + /** + * How long to wait for the new [org.ice4j.ice.Agent] of an ICE restart to connect before giving up on the + * restart and keeping the existing Agent. A value that is not positive disables ICE restarts: the new Agent + * would be freed before the endpoint could answer it. + */ + val restartTimeout: Duration by config( + "videobridge.ice.restart.timeout".from(JitsiConfig.newConfig) + ) + + companion object { + @JvmField + val config = IceConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/ConferenceSizeLastNLimits.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/ConferenceSizeLastNLimits.kt new file mode 100644 index 0000000..0704fdb --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/ConferenceSizeLastNLimits.kt @@ -0,0 +1,36 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import com.typesafe.config.ConfigObject +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import java.util.TreeMap + +class ConferenceSizeLastNLimits { + private val lastNLimits: TreeMap by config { + "videobridge.load-management.conference-last-n-limits".from(JitsiConfig.newConfig) + .convertFrom { cfg -> + TreeMap(cfg.entries.map { it.key.toInt() to it.value.unwrapped() as Int }.toMap()) + } + } + + fun getLastNLimit(conferenceSize: Int): Int = lastNLimits.floorEntry(conferenceSize)?.value ?: -1 + + companion object { + val singleton = ConferenceSizeLastNLimits() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt new file mode 100644 index 0000000..11b98aa --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuLoadSampler.kt @@ -0,0 +1,26 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import com.sun.management.OperatingSystemMXBean +import java.lang.management.ManagementFactory + +class CpuLoadSampler(private val newMeasurementHandler: (CpuMeasurement) -> Unit) : Runnable { + override fun run() { + val osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean::class.java) + newMeasurementHandler(CpuMeasurement(osBean.systemCpuLoad)) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt new file mode 100644 index 0000000..6b021c2 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/CpuMeasurement.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class CpuMeasurement(private val value: Double) : JvbLoadMeasurement { + override fun getLoad(): Double = value + + override fun div(other: JvbLoadMeasurement): Double { + if (other !is CpuMeasurement) { + throw UnsupportedOperationException("Can only divide load measurements of same type") + } + return value / other.value + } + + override fun toString(): String = "CPU usage ${String.format("%.2f", value * 100)}%" + + companion object { + val loadThreshold: CpuMeasurement by config { + "${JvbLoadMeasurement.CONFIG_BASE}.cpu-usage.load-threshold" + .from(JitsiConfig.newConfig) + .convertFrom(::CpuMeasurement) + } + val recoverThreshold: CpuMeasurement by config { + "${JvbLoadMeasurement.CONFIG_BASE}.cpu-usage.recovery-threshold" + .from(JitsiConfig.newConfig) + .convertFrom(::CpuMeasurement) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt new file mode 100644 index 0000000..26b47e7 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadManager.kt @@ -0,0 +1,211 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.utils.NEVER +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.jvbLastNSingleton +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.util.TaskPools +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import java.util.logging.Level + +open class JvbLoadManager @JvmOverloads constructor( + private val jvbLoadThreshold: T, + private val jvbRecoveryThreshold: T, + private val loadReducer: JvbLoadReducer?, + private val clock: Clock = Clock.systemUTC() +) { + protected val logger = createLogger(minLogLevel = Level.ALL) + + private var lastReducerTime: Instant = NEVER + + private var state: State = State.NOT_OVERLOADED + + private var mostRecentLoadMeasurement: T? = null + + private var loadSamplerTask: ScheduledFuture<*>? = null + + protected fun startSampler(sampler: Runnable) { + loadSamplerTask = TaskPools.SCHEDULED_POOL.scheduleAtFixedRate(sampler, 0, 10, TimeUnit.SECONDS) + } + + fun stop() { + loadSamplerTask?.cancel(true) + loadSamplerTask = null + } + + fun loadUpdate(loadMeasurement: T) { + logger.cdebug { "Got a load measurement of $loadMeasurement" } + mostRecentLoadMeasurement = loadMeasurement + val now = clock.instant() + if (loadMeasurement.getLoad() >= jvbLoadThreshold.getLoad()) { + state = State.OVERLOADED + loadReducer?.let { + logger.info("Load measurement $loadMeasurement is above threshold of $jvbLoadThreshold") + if (canRunReducer(now)) { + logger.info("Running load reducer") + it.reduceLoad() + lastReducerTime = now + } else { + logger.info( + "Load reducer ran at $lastReducerTime, which is within " + + "${it.impactTime()} of now, not running reduce" + ) + } + } + } else { + state = State.NOT_OVERLOADED + loadReducer?.let { + if (loadMeasurement.getLoad() < jvbRecoveryThreshold.getLoad()) { + if (canRunReducer(now)) { + if (it.recover()) { + logger.info( + "Recovery ran after a load measurement of $loadMeasurement (which was " + + "below threshold of $jvbRecoveryThreshold) was received" + ) + lastReducerTime = now + } else { + logger.cdebug { "Recovery had no work to do" } + } + } else { + logger.cdebug { + "Load measurement $loadMeasurement is below recovery threshold, but load reducer " + + "ran at $lastReducerTime, which is within ${it.impactTime()} of now, " + + "not running recover" + } + } + } + } + } + } + + fun getCurrentStressLevel(): Double = mostRecentLoadMeasurement?.div(jvbLoadThreshold) ?: 0.0 + + fun getStats() = JsonNodeFactory.instance.objectNode().apply { + put("state", state.toString()) + put("stress", getCurrentStressLevel().toString()) + put("reducer_enabled", reducerEnabled.toString()) + loadReducer?.let { + set("reducer", it.getStats()) + } + } + + private fun canRunReducer(now: Instant): Boolean { + loadReducer?.let { + return Duration.between(lastReducerTime, now) >= it.impactTime() + } + return false + } + + enum class State { + OVERLOADED, + NOT_OVERLOADED + } + + companion object { + val averageParticipantStress: Double by config { + "videobridge.load-management.average-participant-stress".from(JitsiConfig.newConfig) + } + + val loadMeasurement: String by config { + "videobridge.load-management.load-measurements.load-measurement".from(JitsiConfig.newConfig) + } + + val reducerEnabled: Boolean by config("videobridge.load-management.reducer-enabled".from(JitsiConfig.newConfig)) + + const val PACKET_RATE_MEASUREMENT = "packet-rate" + const val CPU_USAGE_MEASUREMENT = "cpu-usage" + + @JvmStatic + fun create(videobridge: Videobridge): JvbLoadManager<*> { + val reducer = if (reducerEnabled) LastNReducer({ videobridge.conferences }, jvbLastNSingleton) else null + + return when (loadMeasurement) { + PACKET_RATE_MEASUREMENT -> PacketRateLoadManager( + PacketRateMeasurement.loadedThreshold, + PacketRateMeasurement.recoveryThreshold, + reducer, + videobridge + ) + + CPU_USAGE_MEASUREMENT -> CpuUsageLoadManager( + CpuMeasurement.loadThreshold, + CpuMeasurement.recoverThreshold, + reducer + ) + + else -> throw IllegalArgumentException( + "Invalid configuration for load measurement type: $loadMeasurement" + ) + } + } + } +} + +class PacketRateLoadManager( + loadThreshold: PacketRateMeasurement, + recoveryThreshold: PacketRateMeasurement, + loadReducer: JvbLoadReducer?, + videobridge: Videobridge +) : JvbLoadManager(loadThreshold, recoveryThreshold, loadReducer) { + + init { + val sampler = PacketRateLoadSampler(videobridge) { loadMeasurement -> + loadUpdate(loadMeasurement) + VideobridgeMetrics.stressLevel.set(getCurrentStressLevel()) + } + + startSampler(sampler) + } +} + +class CpuUsageLoadManager( + loadThreshold: CpuMeasurement, + recoveryThreshold: CpuMeasurement, + loadReducer: JvbLoadReducer? +) : JvbLoadManager(loadThreshold, recoveryThreshold, loadReducer) { + init { + val sampler = CpuLoadSampler { loadMeasurement -> + val stealMeasurement = (if (detectCpuSteal) StealDetection.instance?.update() else null) + ?: CpuMeasurement(0.0) + + loadUpdate(CpuMeasurement(loadMeasurement.getLoad() + stealMeasurement.getLoad())) + VideobridgeMetrics.stressLevel.set(getCurrentStressLevel()) + } + + startSampler(sampler) + } + + companion object { + val detectCpuSteal: Boolean by config { + "${JvbLoadMeasurement.CONFIG_BASE}.cpu-usage.detect-cpu-steal" + .from(JitsiConfig.newConfig) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadMeasurement.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadMeasurement.kt new file mode 100644 index 0000000..90a06aa --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadMeasurement.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +/** + * A measurement of load on the JVB. Implementations should implement a description of the + * measurement in the [toString] method. + */ +interface JvbLoadMeasurement { + fun getLoad(): Double + + operator fun div(other: JvbLoadMeasurement): Double + + companion object { + const val CONFIG_BASE = "videobridge.load-management.load-measurements" + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadReducer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadReducer.kt new file mode 100644 index 0000000..365da01 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/JvbLoadReducer.kt @@ -0,0 +1,43 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import com.fasterxml.jackson.databind.node.ObjectNode +import java.time.Duration + +interface JvbLoadReducer { + fun reduceLoad() + + /** + * Run this [JvbLoadReducer]'s recovery logic, returning true if the logic + * *did* run, false otherwise (if, for example, no limit was currently + * in place so no more 'recovery' was needed) + */ + fun recover(): Boolean + + /** + * How long to wait for load reduction / recovery operations to take effect + * before doing another iteration. + */ + fun impactTime(): Duration + + fun getStats(): ObjectNode + + companion object { + const val CONFIG_BASE = "videobridge.load-management.load-reducers" + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt new file mode 100644 index 0000000..f1aca71 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/LastNReducer.kt @@ -0,0 +1,130 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.from +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.cinfo +import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.JvbLastN +import java.lang.Integer.max +import java.time.Duration +import java.util.function.Supplier + +/** + * A [JvbLoadReducer] which samples the amount of currently forwarded endpoints across + * the entire bridge and sets a bridge-wide last-n value (via [JvbLastN]) to a number + * less than that (based on [reductionScale]). + */ +class LastNReducer( + private val conferencesSupplier: Supplier>, + private val jvbLastN: JvbLastN +) : JvbLoadReducer { + private val logger = createLogger() + + private val reductionScale: Double by config( + "${JvbLoadReducer.CONFIG_BASE}.last-n.reduction-scale".from(JitsiConfig.newConfig) + ) + + private val recoverScale: Double by config( + "${JvbLoadReducer.CONFIG_BASE}.last-n.recover-scale".from(JitsiConfig.newConfig) + ) + + private val impactTime: Duration by config( + "${JvbLoadReducer.CONFIG_BASE}.last-n.impact-time".from(JitsiConfig.newConfig) + ) + + private val minLastN: Int by config( + "${JvbLoadReducer.CONFIG_BASE}.last-n.minimum-last-n-value".from(JitsiConfig.newConfig) + ) + + private val maxEnforcedLastN: Int by config( + "${JvbLoadReducer.CONFIG_BASE}.last-n.maximum-enforced-last-n-value".from(JitsiConfig.newConfig) + ) + + init { + logger.cinfo { this.toString() } + } + + private fun getMaxForwardedEps(): Int? = conferencesSupplier.get() + .flatMap { it.endpoints } + .asSequence() + .filterIsInstance() + .map { + it.numForwardedSources() + } + .maxOrNull() + + override fun reduceLoad() { + // Find the highest amount of endpoints any endpoint on this bridge is forwarding video for + // so we can set a new last-n number to something lower + val maxForwardedEps = getMaxForwardedEps() ?: run { + logger.info("No endpoints with video being forwarded, can't reduce load by reducing last n") + return + } + + val newLastN = max(minLastN, (maxForwardedEps * reductionScale).toInt()) + logger.info( + "Largest number of forwarded videos was $maxForwardedEps, A last-n value of $newLastN is " + + "being enforced to reduce bridge load" + ) + + jvbLastN.jvbLastN = newLastN + } + + override fun recover(): Boolean { + val currLastN = jvbLastN.jvbLastN + if (currLastN == -1) { + logger.cdebug { "No recovery necessary, no JVB last-n is set" } + return false + } + // We want to make sure the last-n value increases by at least 1 + val newLastN = maxOf(currLastN + 1, (currLastN * recoverScale).toInt()) + if (newLastN >= maxEnforcedLastN) { + logger.info( + "JVB last-n was $currLastN, increasing to $newLastN which is beyond the max enforced value" + + "of $maxEnforcedLastN, removing limit completely" + ) + jvbLastN.jvbLastN = -1 + } else { + logger.info("JVB last-n was $currLastN, increasing to $newLastN as part of load recovery") + jvbLastN.jvbLastN = newLastN + } + + return true + } + + override fun impactTime(): Duration = impactTime + + override fun getStats() = JsonNodeFactory.instance.objectNode().apply { + put("jvbLastN", jvbLastN.jvbLastN) + } + + override fun toString(): String = buildString { + append("LastNReducer with") + append(" reductionScale: $reductionScale") + append(" recoverScale: $recoverScale") + append(" impactTime: $impactTime") + append(" minLastN: $minLastN") + append(" maxEnforcedLastN: $maxEnforcedLastN") + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateLoadSampler.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateLoadSampler.kt new file mode 100644 index 0000000..cdef318 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateLoadSampler.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import org.jitsi.videobridge.Videobridge + +class PacketRateLoadSampler( + private val videobridge: Videobridge, + private val newMeasurementHandler: (PacketRateMeasurement) -> Unit +) : Runnable { + + override fun run() { + var totalPacketRate: Long = 0 + videobridge.conferences.forEach { conf -> + conf.localEndpoints.forEach { ep -> + with(ep.transceiver.getTransceiverStats()) { + totalPacketRate += rtpReceiverStats.packetStreamStats.packetRate + totalPacketRate += outgoingPacketStreamStats.packetRate + } + } + conf.relays.forEach { relay -> + totalPacketRate += relay.incomingPacketRate + totalPacketRate += relay.outgoingPacketRate + } + } + newMeasurementHandler(PacketRateMeasurement(totalPacketRate)) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt new file mode 100644 index 0000000..584405c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurement.kt @@ -0,0 +1,56 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class PacketRateMeasurement(private val packetRate: Long) : JvbLoadMeasurement { + override fun getLoad(): Double = packetRate.toDouble() + + override fun div(other: JvbLoadMeasurement): Double { + if (other !is PacketRateMeasurement) { + throw UnsupportedOperationException("Can only divide load measurements of same type") + } + return packetRate / other.packetRate.toDouble() + } + + override fun toString(): String = "RTP packet rate (up + down) of $packetRate pps" + + companion object { + private val config = PacketRateMeasurementConfig() + + @JvmStatic + val loadedThreshold = config.loadThreshold + + @JvmStatic + val recoveryThreshold = config.recoverThreshold + } +} + +private class PacketRateMeasurementConfig { + val loadThreshold: PacketRateMeasurement by config { + "${JvbLoadMeasurement.CONFIG_BASE}.packet-rate.load-threshold" + .from(JitsiConfig.newConfig) + .convertFrom(::PacketRateMeasurement) + } + val recoverThreshold: PacketRateMeasurement by config { + "${JvbLoadMeasurement.CONFIG_BASE}.packet-rate.recovery-threshold" + .from(JitsiConfig.newConfig) + .convertFrom(::PacketRateMeasurement) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/StealDetection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/StealDetection.kt new file mode 100644 index 0000000..e7277ca --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/load_management/StealDetection.kt @@ -0,0 +1,78 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import org.jitsi.metrics.DoubleGaugeMetric +import java.io.File +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +class StealDetection internal constructor(private val file: File) { + + private var previous = TotalAndSteal(0, 0) + + fun update(): CpuMeasurement { + val current = try { + readFromFile() ?: return CpuMeasurement(0.0) + } catch (e: Exception) { + return CpuMeasurement(0.0) + } + + if (previous.steal == 0L && previous.total == 0L) { + previous = current + return CpuMeasurement(0.0) + } + + val totalDelta = current.total - previous.total + val stealDelta = current.steal - previous.steal + val currentSteal = if (totalDelta > 0 && stealDelta > 0) { + stealDelta.toDouble() / totalDelta + } else { + 0.0 + } + + previous = current + stealMetric.set(currentSteal) + + return CpuMeasurement(currentSteal) + } + + private fun readFromFile(): TotalAndSteal? { + val cpuLine = file.readLines() + .firstOrNull { it.startsWith("cpu ") } + ?.split("\\s+".toRegex()) + ?.drop(1) + ?.mapNotNull { it.toLongOrNull() } + ?: return null + + if (cpuLine.size < 8) return null + return TotalAndSteal(total = cpuLine.sum(), steal = cpuLine[7]) + } + + companion object { + private val stealMetric: DoubleGaugeMetric by lazy { + metricsContainer.registerDoubleGauge( + "cpu_steal", + "CPU steal fraction (from 0 to 1)" + ) + } + + private val linux = System.getProperty("os.name").lowercase().contains("linux") + + val instance = if (linux) StealDetection(File("/proc/stat")) else null + } +} + +private data class TotalAndSteal(val total: Long, val steal: Long) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt new file mode 100644 index 0000000..817d7e1 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/message/BridgeChannelMessage.kt @@ -0,0 +1,609 @@ +/* + * Copyright @ 2020-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.message + +import com.fasterxml.jackson.annotation.JsonAnyGetter +import com.fasterxml.jackson.annotation.JsonAnySetter +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.MapperFeature +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import org.jitsi.nlj.VideoType +import org.jitsi.utils.ResettableLazy +import org.jitsi.videobridge.cc.allocation.VideoConstraints +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong + +/** + * Represent a message sent over the "bridge channel" between an endpoint (or "client") and jitsi-videobridge, or + * between two jitsi-videobridge instances over a relay connection. + * + * The messages are formatted in JSON with a required "colibriClass" field, which indicates the message type. Different + * message types have different (if any) additional fields. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = BridgeChannelMessage.TYPE_PROPERTY_NAME) +@JsonSubTypes( + JsonSubTypes.Type(value = ClientHelloMessage::class, name = ClientHelloMessage.TYPE), + JsonSubTypes.Type(value = ServerHelloMessage::class, name = ServerHelloMessage.TYPE), + JsonSubTypes.Type(value = EndpointMessage::class, name = EndpointMessage.TYPE), + JsonSubTypes.Type(value = EndpointStats::class, name = EndpointStats.TYPE), + JsonSubTypes.Type(value = LastNMessage::class, name = LastNMessage.TYPE), + JsonSubTypes.Type(value = DominantSpeakerMessage::class, name = DominantSpeakerMessage.TYPE), + JsonSubTypes.Type(value = EndpointConnectionStatusMessage::class, name = EndpointConnectionStatusMessage.TYPE), + JsonSubTypes.Type(value = ForwardedSourcesMessage::class, name = ForwardedSourcesMessage.TYPE), + JsonSubTypes.Type(value = VideoSourcesMap::class, name = VideoSourcesMap.TYPE), + JsonSubTypes.Type(value = AudioSourcesMap::class, name = AudioSourcesMap.TYPE), + JsonSubTypes.Type(value = SenderSourceConstraintsMessage::class, name = SenderSourceConstraintsMessage.TYPE), + JsonSubTypes.Type(value = AddReceiverMessage::class, name = AddReceiverMessage.TYPE), + JsonSubTypes.Type(value = RemoveReceiverMessage::class, name = RemoveReceiverMessage.TYPE), + JsonSubTypes.Type(value = ReceiverVideoConstraintsMessage::class, name = ReceiverVideoConstraintsMessage.TYPE), + JsonSubTypes.Type(value = SourceVideoTypeMessage::class, name = SourceVideoTypeMessage.TYPE), + JsonSubTypes.Type(value = ConnectionStats::class, name = ConnectionStats.TYPE), + JsonSubTypes.Type(value = VideoTypeMessage::class, name = VideoTypeMessage.TYPE), + JsonSubTypes.Type(value = ReceiverAudioSubscriptionMessage::class, name = ReceiverAudioSubscriptionMessage.TYPE), + JsonSubTypes.Type( + value = SyntheticSourceSendingChangeEvent::class, + name = SyntheticSourceSendingChangeEvent.TYPE + ) +) +sealed class BridgeChannelMessage { + private val jsonCacheDelegate = ResettableLazy { createJson() } + + /** + * Caches the JSON string representation of this object. Note that after any changes to state (e.g. vars being set) + * the cache needs to be invalidated via [resetJsonCache]. + */ + private val jsonCache: String by jsonCacheDelegate + protected fun resetJsonCache() = jsonCacheDelegate.reset() + + /** + * Get a JSON representation of this [BridgeChannelMessage]. + */ + fun toJson(): String = jsonCache + + /** + * Serialize this [BridgeChannelMessage] to a string in JSON format. Note that this default implementation can be + * slow, which is why some of the messages that we serialize often override it with a custom optimized version. + */ + protected open fun createJson(): String = mapper.writeValueAsString(this) + + companion object { + private val mapper = jacksonObjectMapper().apply { + enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS) + enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) + } + + @JvmStatic + @Throws(JsonProcessingException::class, JsonMappingException::class) + fun parse(string: String): BridgeChannelMessage = mapper.readValue(string) + const val TYPE_PROPERTY_NAME = "colibriClass" + } +} + +open class MessageHandler { + private val receivedCounts = ConcurrentHashMap() + + /** + * Handles a [BridgeChannelMessage] that was received. Returns an optional response. + */ + fun handleMessage(message: BridgeChannelMessage): BridgeChannelMessage? { + receivedCounts.computeIfAbsent(message::class.java.simpleName) { AtomicLong() }.incrementAndGet() + + return when (message) { + is ClientHelloMessage -> clientHello(message) + is ServerHelloMessage -> serverHello(message) + is EndpointMessage -> endpointMessage(message) + is EndpointStats -> endpointStats(message) + is LastNMessage -> lastN(message) + is DominantSpeakerMessage -> dominantSpeaker(message) + is EndpointConnectionStatusMessage -> endpointConnectionStatus(message) + is ForwardedSourcesMessage -> forwardedSources(message) + is VideoSourcesMap -> videoSourcesMap(message) + is AudioSourcesMap -> audioSourcesMap(message) + is SenderSourceConstraintsMessage -> senderSourceConstraints(message) + is AddReceiverMessage -> addReceiver(message) + is RemoveReceiverMessage -> removeReceiver(message) + is ReceiverVideoConstraintsMessage -> receiverVideoConstraints(message) + is SourceVideoTypeMessage -> sourceVideoType(message) + is ConnectionStats -> connectionStats(message) + is VideoTypeMessage -> videoType(message) + is ReceiverAudioSubscriptionMessage -> receiverAudioSubscription(message) + is SyntheticSourceSendingChangeEvent -> syntheticSourceSendingChange(message) + } + } + + open fun unhandledMessage(message: BridgeChannelMessage) {} + private fun unhandledMessageReturnNull(message: BridgeChannelMessage): BridgeChannelMessage? { + unhandledMessage(message) + return null + } + + open fun clientHello(message: ClientHelloMessage) = unhandledMessageReturnNull(message) + open fun serverHello(message: ServerHelloMessage) = unhandledMessageReturnNull(message) + open fun endpointMessage(message: EndpointMessage) = unhandledMessageReturnNull(message) + open fun endpointStats(message: EndpointStats) = unhandledMessageReturnNull(message) + open fun lastN(message: LastNMessage) = unhandledMessageReturnNull(message) + open fun dominantSpeaker(message: DominantSpeakerMessage) = unhandledMessageReturnNull(message) + open fun endpointConnectionStatus(message: EndpointConnectionStatusMessage) = unhandledMessageReturnNull(message) + open fun forwardedSources(message: ForwardedSourcesMessage) = unhandledMessageReturnNull(message) + open fun videoSourcesMap(message: VideoSourcesMap) = unhandledMessageReturnNull(message) + open fun audioSourcesMap(message: AudioSourcesMap) = unhandledMessageReturnNull(message) + open fun senderSourceConstraints(message: SenderSourceConstraintsMessage) = unhandledMessageReturnNull(message) + open fun addReceiver(message: AddReceiverMessage) = unhandledMessageReturnNull(message) + open fun removeReceiver(message: RemoveReceiverMessage) = unhandledMessageReturnNull(message) + open fun receiverVideoConstraints(message: ReceiverVideoConstraintsMessage) = unhandledMessageReturnNull(message) + open fun sourceVideoType(message: SourceVideoTypeMessage) = unhandledMessageReturnNull(message) + open fun connectionStats(message: ConnectionStats) = unhandledMessageReturnNull(message) + open fun videoType(message: VideoTypeMessage) = unhandledMessageReturnNull(message) + open fun receiverAudioSubscription(message: ReceiverAudioSubscriptionMessage) = unhandledMessageReturnNull(message) + open fun syntheticSourceSendingChange(message: SyntheticSourceSendingChangeEvent) = + unhandledMessageReturnNull(message) + + fun getReceivedCounts() = receivedCounts.mapValues { it.value.get() } +} + +/** + * A message sent from a client to a bridge in the beginning of a session. + */ +class ClientHelloMessage : BridgeChannelMessage() { + companion object { + const val TYPE = "ClientHello" + } +} + +/** + * A message sent from a bridge to a client in response to a [ClientHelloMessage] or when a websocket is accepted. + */ +class ServerHelloMessage @JvmOverloads constructor( + @JsonInclude(JsonInclude.Include.NON_NULL) + val version: String? = null +) : BridgeChannelMessage() { + + override fun createJson(): String = + if (version == null) JSON_STRING_NO_VERSION else """{"colibriClass":"$TYPE","version":"$version"}""" + companion object { + const val TYPE = "ServerHello" + const val JSON_STRING_NO_VERSION: String = """{"colibriClass":"$TYPE"}""" + } +} + +/** + * An endpoint-to-endpoint message, which originates from one endpoint and is routed by the bridge. It can be targeted + * to a specific endpoint (when [to] is not-blank) or broadcast (when [to] is blank). + * + * The message contains custom fields, which are to be preserved (via [otherFields]). + * + * The same class is used regardless of whether the specific message is received from a client, received from a + * bridge, sent to a client or sent to a bridge. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +class EndpointMessage(val to: String) : BridgeChannelMessage() { + @JsonInclude(JsonInclude.Include.NON_NULL) + var from: String? = null + set(value) { + field = value + resetJsonCache() + } + + @get:JsonAnyGetter + val otherFields = mutableMapOf() + + /** + * Whether this message is to be broadcast or targeted to a specific endpoint. + */ + @JsonIgnore + fun isBroadcast(): Boolean = to.isBlank() + + @JsonAnySetter + fun put(key: String, value: Any?) { + if (key != TYPE_PROPERTY_NAME) { + otherFields[key] = value + } + } + + companion object { + const val TYPE = "EndpointMessage" + } +} + +/** + * An endpoint statistics message, which originates from one endpoint and is routed by the bridge. + * When received from a client it is filtered depending on which endpoints are "interesting" to other clients. + * + * The message contains custom fields, which are to be preserved (via [otherFields]). + * + * The same class is used regardless of whether the specific message is received from a client, received from a + * bridge, sent to a client or sent to a bridge. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +class EndpointStats : BridgeChannelMessage() { + @JsonInclude(JsonInclude.Include.NON_NULL) + var from: String? = null + set(value) { + field = value + resetJsonCache() + } + + @get:JsonAnyGetter + val otherFields = mutableMapOf() + + @JsonAnySetter + fun put(key: String, value: Any?) { + if (key != TYPE_PROPERTY_NAME) { + otherFields[key] = value + } + } + + companion object { + const val TYPE = "EndpointStats" + } +} + +/** + * A message sent from a client, indicating that it wishes to change its "lastN" (i.e. the maximum number of video + * streams to be received). + */ +class LastNMessage(val lastN: Int) : BridgeChannelMessage() { + companion object { + const val TYPE = "LastNChangedEvent" + } +} + +/** + * A message sent from the bridge to a client, indicating that the dominant speaker in the conference changed. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +class DominantSpeakerMessage @JvmOverloads constructor( + val dominantSpeakerEndpoint: String, + val previousSpeakers: List? = null, + val silence: Boolean = false +) : BridgeChannelMessage() { + /** + * Construct a message from a list of speakers with the dominant speaker on top. The list must have at least one + * element. + */ + constructor(previousSpeakers: List, silence: Boolean) : this( + previousSpeakers[0], + previousSpeakers.drop(1), + silence + ) + companion object { + const val TYPE = "DominantSpeakerEndpointChangeEvent" + } +} + +/** + * A message sent from the bridge to a client, indicating that the bridge's connection to another endpoint changed + * status. + */ +class EndpointConnectionStatusMessage( + val endpoint: String, + activeBoolean: Boolean +) : BridgeChannelMessage() { + + // For whatever reason we encode the boolean in JSON as a string. + val active: String = activeBoolean.toString() + + /** + * Serialize manually because it's faster than Jackson. + */ + override fun createJson(): String = """{"colibriClass":"$TYPE","endpoint":"$endpoint","active":"$active"}""" + + companion object { + const val TYPE = "EndpointConnectivityStatusChangeEvent" + } +} + +/** + * A message sent from the bridge to a client, indicating the set of media sources that are currently being forwarded. + */ +class ForwardedSourcesMessage( + /** + * The set of media sources for which the bridge is currently sending video. + */ + val forwardedSources: Collection +) : BridgeChannelMessage() { + companion object { + const val TYPE = "ForwardedSources" + } +} + +/** + * A mapping of a video source to SSRCs. + */ +data class VideoSourceMapping( + /** The name of the source being mapped. */ + val source: String, + /** The endpoint that is the sender of the source. */ + val owner: String?, + /** The primary SSRC of the source being mapped. */ + val ssrc: Long, + /** The RTX SSRC of the source being mapped. */ + val rtx: Long, + /** The video type of the source being mapped. */ + val videoType: VideoType, + /** The mid the bridge stamps on this source's packets, when mid-based demuxing is enabled. */ + @JsonInclude(JsonInclude.Include.NON_NULL) + val mid: String? = null +) + +/** + * A message sent from the bridge to a client to indicate video source mappings. + */ +class VideoSourcesMap( + /* The current list of maps of sources to ssrcs. */ + val mappedSources: Collection +) : BridgeChannelMessage() { + companion object { + const val TYPE = "VideoSourcesMap" + } +} + +/** + * A mapping of an audio source to SSRCs. + */ +data class AudioSourceMapping( + /** The name of the source being mapped. */ + val source: String, + /** The endpoint that is the sender of the source. */ + val owner: String?, + /** The SSRC of the source being mapped. */ + val ssrc: Long, + /** The mid the bridge stamps on this source's packets, when mid-based demuxing is enabled. */ + @JsonInclude(JsonInclude.Include.NON_NULL) + val mid: String? = null +) + +/** + * A message sent from the bridge to a client to indicate audio source mappings. + */ +class AudioSourcesMap( + /* The current list of maps of sources to ssrcs. */ + val mappedSources: Collection +) : BridgeChannelMessage() { + companion object { + const val TYPE = "AudioSourcesMap" + } +} + +/** + * A message sent from the bridge to a client indicating that a synthetic source (audio generated by the bridge + * rather than sent by an endpoint, e.g. translated audio) started or stopped sending. It is derived from the + * `start`/`stop` mediajson events a translator sends to bracket a "talk". One source per message; the flat shape + * matches lib-jitsi-meet's `BridgeChannel` validation of this colibriClass. + */ +class SyntheticSourceSendingChangeEvent( + /** The name of the synthetic source whose sending state changed (e.g. "55555555-a0.hi"). */ + val sourceName: String, + /** True if the source started sending, false if it stopped. */ + val sending: Boolean, + /** + * The RTP timestamp (in the source's media clock, i.e. 48000 Hz for Opus) at which the change takes effect, on + * the same timeline as the source's media, so a receiver can align it with the audio it receives. A 32-bit value + * that wraps at 2^32 (lib-jitsi-meet validates 0..0xFFFFFFFF). For a start it is the first packet's timestamp; + * for a stop it is one past the last packet (the timestamp the next contiguous packet would carry), so a sending + * run spans [start, stop). + */ + val timestamp: Long +) : BridgeChannelMessage() { + companion object { + const val TYPE = "SyntheticSourceSendingChangeEvent" + } +} + +/** + * A message sent from the bridge to a client (sender), indicating constraints for the sender's video stream. + */ +class SenderSourceConstraintsMessage( + val sourceName: String, + val maxHeight: Int +) : BridgeChannelMessage() { + + /** + * Serialize manually because it's faster than Jackson. + */ + override fun createJson(): String = + """{"colibriClass":"$TYPE", "sourceName":"$sourceName", "maxHeight":$maxHeight}""" + + companion object { + const val TYPE = "SenderSourceConstraints" + } +} + +/** + * A message sent from one bridge to another (via a relay connection) indicating that the first bridge wishes to + * receive video streams from the specified endpoint with the specified constraints. + */ +class AddReceiverMessage( + val bridgeId: String, + val sourceName: String, + val videoConstraints: VideoConstraints +) : BridgeChannelMessage() { + /** + * Serialize manually because it's faster than Jackson. + */ + override fun createJson(): String = "{\"colibriClass\":\"$TYPE\",\"bridgeId\":\"$bridgeId\"," + + "\"sourceName\":\"$sourceName\"," + + "\"videoConstraints\":$videoConstraints}" + + companion object { + const val TYPE = "AddReceiver" + } +} + +/** + * A message sent from one bridge to another (via a relay connection) indicating that it no longer wishes to receive + * video streams from the specified endpoint. + */ +class RemoveReceiverMessage( + val bridgeId: String, + val endpointId: String +) : BridgeChannelMessage() { + /** + * Serialize manually because it's faster than Jackson. + */ + override fun createJson(): String = """{"colibriClass":"$TYPE","bridgeId":"$bridgeId","endpointId":"$endpointId"}""" + + companion object { + const val TYPE = "RemoveReceiver" + } +} + +class ReceiverVideoConstraintsMessage( + val lastN: Int? = null, + val selectedSources: List? = null, + val onStageSources: List? = null, + val defaultConstraints: VideoConstraints? = null, + val constraints: Map? = null, + val assumedBandwidthBps: Long? = null +) : BridgeChannelMessage() { + companion object { + const val TYPE = "ReceiverVideoConstraints" + } +} + +/** + * A message signaling the video type of the media source. + */ +class SourceVideoTypeMessage( + val videoType: VideoType, + sourceName: String, + /** + * The endpoint ID that the message relates to, or null. When null, the ID is inferred from the channel the + * message was received on (non-null values are needed only for Relays). + */ + endpointId: String? = null +) : BridgeChannelMessage() { + @JsonInclude(JsonInclude.Include.NON_NULL) + var endpointId: String? = endpointId + set(value) { + field = value + resetJsonCache() + } + + @JsonInclude(JsonInclude.Include.ALWAYS) + var sourceName: String = sourceName + set(value) { + field = value + resetJsonCache() + } + + companion object { + const val TYPE = "SourceVideoTypeMessage" + } +} + +/* + * A message sent from a client to the bridge to indicate which audio sources it wants to receive. + */ +data class ReceiverAudioSubscriptionMessage( + /** + * The default for any source not named in [include] or [exclude]: if true the endpoint receives it (except + * synthetic sources, which are received only when explicitly named in [include]); if false it does not. + */ + @JsonProperty("all") val all: Boolean = false, + /** + * Sources to always receive, in addition to whatever [all] selects. This is the only way to receive a synthetic + * source. Allows e.g. "all regular audio plus these synthetic sources" via [all] = true with a non-empty list. + */ + @JsonProperty("include") val include: List = emptyList(), + /** Sources to never receive. Takes precedence over [all] and [include]. */ + @JsonProperty("exclude") val exclude: List = emptyList() +) : BridgeChannelMessage() { + companion object { + const val TYPE = "ReceiverAudioSubscription" + + /** + * Deserializes both the current `all`/`include`/`exclude` form and the legacy `mode`/`list` form, translating + * the latter into the new semantics so that no code outside the parser needs to be aware of the old syntax. + * + * Legacy mapping: + * - `mode: "All"` -> [all] = true + * - `mode: "None"` -> [all] = false + * - `mode: "Include"` -> [all] = false, [include] = `list` + * - `mode: "Exclude"` -> [all] = true, [exclude] = `list` + */ + @JvmStatic + @JsonCreator + fun fromJson( + @JsonProperty("all") all: Boolean?, + @JsonProperty("include") include: List?, + @JsonProperty("exclude") exclude: List?, + @JsonProperty("mode") mode: String?, + @JsonProperty("list") list: List? + ): ReceiverAudioSubscriptionMessage { + if (mode != null) { + return when (mode) { + "All" -> ReceiverAudioSubscriptionMessage(all = true) + "None" -> ReceiverAudioSubscriptionMessage(all = false) + "Include" -> ReceiverAudioSubscriptionMessage(all = false, include = list ?: emptyList()) + "Exclude" -> ReceiverAudioSubscriptionMessage(all = true, exclude = list ?: emptyList()) + else -> throw IllegalArgumentException("Unknown ReceiverAudioSubscription mode: $mode") + } + } + return ReceiverAudioSubscriptionMessage( + all = all ?: false, + include = include ?: emptyList(), + exclude = exclude ?: emptyList() + ) + } + } +} + +/** + * A message from the bridge to an endpoint giving information about the connection to the bridge + * (that the client can't determine on its own) + */ +class ConnectionStats( + val estimatedDownlinkBandwidth: Long, +) : BridgeChannelMessage() { + companion object { + const val TYPE = "ConnectionStats" + } +} + +/** + * A signaling the type of video stream an endpoint has available. + */ +@Deprecated("", ReplaceWith("SourceVideoTypeMessage"), DeprecationLevel.WARNING) +class VideoTypeMessage( + val videoType: VideoType, + /** + * The endpoint ID that the message relates to, or null. When null, the ID is inferred from the channel the + * message was received on (non-null values are needed only for Relays). + */ + endpointId: String? = null +) : BridgeChannelMessage() { + @JsonInclude(JsonInclude.Include.NON_NULL) + var endpointId: String? = endpointId + set(value) { + field = value + resetJsonCache() + } + + companion object { + const val TYPE = "VideoTypeMessage" + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt new file mode 100644 index 0000000..b9cca88 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/JvmMetrics.kt @@ -0,0 +1,106 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import com.sun.management.UnixOperatingSystemMXBean +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.utils.logging2.createLogger +import java.lang.management.ManagementFactory +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +class JvmMetrics private constructor() { + val logger = createLogger() + + private val gcType = ManagementFactory.getGarbageCollectorMXBeans().firstOrNull()?.name.let { + when { + it?.contains("shenandoah", ignoreCase = true) == true -> GcType.Shenandoah + it?.contains("zgc", ignoreCase = true) == true -> GcType.Zgc + it?.contains("g1", ignoreCase = true) == true -> GcType.G1 + else -> GcType.Other + } + }.also { + logger.info("Detected GC type $it") + } + + fun update() { + threadCount.set(ManagementFactory.getThreadMXBean().threadCount.toLong()) + gcCount.set( + ManagementFactory.getGarbageCollectorMXBeans().sumOf { it.collectionCount } + ) + gcTime.set( + ManagementFactory.getGarbageCollectorMXBeans().sumOf { it.collectionTime } + ) + (ManagementFactory.getOperatingSystemMXBean() as? UnixOperatingSystemMXBean)?.let { + openFdCount.set(it.openFileDescriptorCount) + } + if (gcType != GcType.Other) { + ManagementFactory.getMemoryPoolMXBeans().find { it.name == gcType.memoryPoolName }?.let { + heapUsed.set(it.usage.used) + heapCommitted.set(it.usage.committed) + } + } + } + + val threadCount = metricsContainer.registerLongGauge( + "thread_count", + "Current number of JVM threads." + ) + + private val gcCount = metricsContainer.registerLongGauge( + "jvm_gc_count", + "Garbage collection count." + ) + + private val gcTime = metricsContainer.registerLongGauge( + "jvm_gc_time", + "Garbage collection time." + ) + + private val heapCommitted = metricsContainer.registerLongGauge( + "jvm_heap_committed", + "Capacity of the main memory pool for the heap (GC type specific)." + ) + + private val heapUsed = metricsContainer.registerLongGauge( + "jvm_heap_used", + "Usage of the main memory pool for the heap (GC type specific)." + ) + + private val openFdCount = metricsContainer.registerLongGauge( + "jvm_open_fd_count", + "Number of open file descriptors." + ) + + private enum class GcType( + /** The name of the memory pool we're interested with this type of GC */ + val memoryPoolName: String? + ) { + G1("G1 Old Gen"), + Zgc("ZHeap"), + Shenandoah("Shenandoah"), + Other(null) + } + + companion object { + val enable: Boolean by config { + "videobridge.stats.jvm.enabled".from(JitsiConfig.newConfig) + } + + val INSTANCE = if (enable) JvmMetrics() else null + fun update() = INSTANCE?.update() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt new file mode 100644 index 0000000..e7eb2cf --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/Metrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metrics.MetricsUpdater +import org.jitsi.utils.concurrent.CustomizableThreadFactory +import java.time.Duration +import java.util.concurrent.Executors + +object Metrics { + val interval: Duration by config { + "videobridge.stats.interval".from(JitsiConfig.newConfig) + } + + /** Updating the metrics shouldn't block anywhere, but use a separate executor just in case. */ + private val executor = Executors.newSingleThreadScheduledExecutor( + CustomizableThreadFactory("MetricsUpdater-scheduled", false) + ) + val metricsUpdater = MetricsUpdater(executor, interval) + + /** + * The lock which is used when metrics are updated or queried. The [MetricsUpdater] internally uses itself as the + * lock, so we reuse it here. + */ + val lock: Any + get() = metricsUpdater + + fun start() { + if (JvmMetrics.enable) { + metricsUpdater.addUpdateTask { JvmMetrics.update() } + } + QueueMetrics.init() + } + fun stop() { + metricsUpdater.stop() + executor.shutdown() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt new file mode 100644 index 0000000..ceab845 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/QueueMetrics.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.nlj.RtpReceiverImpl +import org.jitsi.nlj.RtpSenderImpl +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +object QueueMetrics { + private val rtpReceiverDroppedPackets = metricsContainer.registerCounter( + "rtp_receiver_dropped_packets", + "Number of packets dropped out of the RTP receiver queue." + ) + private val rtpReceiverExceptions = metricsContainer.registerCounter( + "rtp_receiver_exceptions", + "Number of exceptions from the RTP receiver queue." + ) + private val rtpSenderDroppedPackets = metricsContainer.registerCounter( + "rtp_sender_dropped_packets", + "Number of packets dropped out of the RTP sender queue." + ) + private val rtpSenderExceptions = metricsContainer.registerCounter( + "rtp_sender_exceptions", + "Number of exceptions from the RTP sender queue." + ) + + val droppedPackets = metricsContainer.registerCounter( + "queue_dropped_packets", + "Number of packets dropped from any of the queues." + ) + val exceptions = metricsContainer.registerCounter( + "queue_exceptions", + "Number of exceptions from any of the queues." + ) + + fun init() { + RtpReceiverImpl.queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + rtpReceiverDroppedPackets.inc() + droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + rtpReceiverExceptions.inc() + exceptions.inc() + } + } + + RtpSenderImpl.queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + rtpSenderDroppedPackets.inc() + droppedPackets.inc() + } + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + rtpSenderExceptions.inc() + exceptions.inc() + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt new file mode 100644 index 0000000..a4a6357 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetrics.kt @@ -0,0 +1,307 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.metrics.CounterMetric +import org.jitsi.videobridge.VideobridgeConfig +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +object VideobridgeMetrics { + val gracefulShutdown = metricsContainer.registerBooleanMetric( + "graceful_shutdown", + "Whether the bridge is in graceful shutdown mode (not accepting new conferences)." + ) + val shuttingDown = metricsContainer.registerBooleanMetric( + "shutting_down", + "Whether the bridge is shutting down." + ) + val drainMode = metricsContainer.registerBooleanMetric( + "drain_mode", + "Whether the bridge is in drain shutdown mode.", + VideobridgeConfig.initialDrainMode + ) + + @JvmField + val conferencesCompleted = metricsContainer.registerCounter( + "conferences_completed", + "The total number of conferences completed/expired on the Videobridge." + ) + + @JvmField + val conferencesCreated = metricsContainer.registerCounter( + "conferences_created", + "The total number of conferences created on the Videobridge." + ) + + @JvmField + val dataChannelMessagesReceived = metricsContainer.registerCounter( + "data_channel_messages_received", + "Number of messages received from the data channels of the endpoints of this conference." + ) + + @JvmField + val dataChannelMessagesSent = metricsContainer.registerCounter( + "data_channel_messages_sent", + "Number of messages sent via the data channels of the endpoints of this conference." + ) + + @JvmField + val colibriWebSocketMessagesReceived: CounterMetric = metricsContainer.registerCounter( + "colibri_web_socket_messages_received", + "Number of messages received from the data channels of the endpoints of this conference." + ) + + @JvmField + val colibriWebSocketMessagesSent = metricsContainer.registerCounter( + "colibri_web_socket_messages_sent", + "Number of messages sent via the data channels of the endpoints of this conference." + ) + + @JvmField + val colibriWebSocketCloseNormal = metricsContainer.registerCounter( + "colibri_web_socket_close_normal", + "Number of times a colibri web socket was closed normally." + ) + + @JvmField + val colibriWebSocketCloseAbnormal = metricsContainer.registerCounter( + "colibri_web_socket_close_abnormal", + "Number of times a colibri web socket was closed abnormally." + ) + + @JvmField + val colibriWebSocketErrors = metricsContainer.registerCounter( + "colibri_web_socket_error", + "Number of times a colibri web socket reported an error." + ) + + @JvmField + val packetsReceived = metricsContainer.registerCounter( + "packets_received", + "Number of RTP packets received in conferences on this videobridge." + ) + + @JvmField + val packetsSent = metricsContainer.registerCounter( + "packets_sent", + "Number of RTP packets sent in conferences on this videobridge." + ) + + @JvmField + val relayPacketsReceived = metricsContainer.registerCounter( + "relay_packets_received", + "Number of RTP packets received by relays in conferences on this videobridge." + ) + + @JvmField + val relayPacketsSent = metricsContainer.registerCounter( + "relay_packets_sent", + "Number of RTP packets sent by relays in conferences on this videobridge." + ) + + @JvmField + val totalEndpoints = metricsContainer.registerCounter( + "endpoints", + "The total number of endpoints created." + ) + + @JvmField + val totalVisitors = metricsContainer.registerCounter( + "visitors", + "The total number of visitor endpoints created." + ) + + @JvmField + val numEndpointsNoMessageTransportAfterDelay = metricsContainer.registerCounter( + "endpoints_no_message_transport_after_delay", + "Number of endpoints which had not established a relay message transport even after some delay." + ) + + @JvmField + val totalRelays = metricsContainer.registerCounter( + "relays", + "The total number of relays created." + ) + + @JvmField + val numRelaysNoMessageTransportAfterDelay = metricsContainer.registerCounter( + "relays_no_message_transport_after_delay", + "Number of relays which had not established a relay message transport even after some delay." + ) + + @JvmField + val dominantSpeakerChanges = metricsContainer.registerCounter( + "dominant_speaker_changes", + "Number of times the dominant speaker in any conference changed." + ) + + @JvmField + val endpointsIceFailed = metricsContainer.registerCounter( + "endpoints_ice_failed", + "Number of endpoints for which ICE failed to establish." + ) + + @JvmField + val endpointsDtlsFailed = metricsContainer.registerCounter( + "endpoints_dtls_failed", + "Number of endpoints whose ICE connection was established, but DTLS wasn't (at time of expiration)." + ) + + @JvmField + val relaysIceFailed = metricsContainer.registerCounter( + "relays_ice_failed", + "Number of relays for which ICE failed to establish." + ) + + @JvmField + val relaysDtlsFailed = metricsContainer.registerCounter( + "relays_dtls_failed", + "Number of relays whose ICE connection was established, but DTLS wasn't (at time of expiration)." + ) + + @JvmField + val stressLevel = metricsContainer.registerDoubleGauge( + "stress", + "Current stress (between 0 and 1)." + ) + + @JvmField + val preemptiveKeyframeRequestsSent = metricsContainer.registerCounter( + "preemptive_keyframe_requests_sent", + "Number of preemptive keyframe requests that were sent." + ) + + @JvmField + val preemptiveKeyframeRequestsSuppressed = metricsContainer.registerCounter( + "preemptive_keyframe_requests_suppressed", + "Number of preemptive keyframe requests that were not sent because no endpoints were in stage view." + ) + + @JvmField + val keyframesReceived = metricsContainer.registerCounter( + "keyframes_received", + "Number of keyframes that were received (updated on endpoint expiration)." + ) + + @JvmField + val layeringChangesReceived = metricsContainer.registerCounter( + "layering_changes_received", + "Number of times the layering of an incoming video stream changed (updated on endpoint expiration)." + ) + + @JvmField + val currentLocalEndpoints = metricsContainer.registerLongGauge( + "local_endpoints", + "Number of local endpoints that exist currently." + ) + + @JvmField + val currentVisitors = metricsContainer.registerLongGauge( + "current_visitors", + "Number of visitor endpoints." + ) + + @JvmField + val currentConferences = metricsContainer.registerLongGauge( + "conferences", + "Current number of conferences." + ) + + @JvmField + val totalConferenceSeconds = metricsContainer.registerCounter( + "conference_seconds", + "The total duration in seconds of all completed conferences." + ) + + @JvmField + val totalBytesReceived = metricsContainer.registerCounter( + "bytes_received", + "The total number of bytes received in RTP packets." + ) + + @JvmField + val totalBytesSent = metricsContainer.registerCounter( + "bytes_sent", + "The total number of bytes sent in RTP packets." + ) + + @JvmField + val totalRelayBytesReceived = metricsContainer.registerCounter( + "relay_bytes_received", + "The total number of bytes received by relays in RTP packets." + ) + + @JvmField + val totalRelayBytesSent = metricsContainer.registerCounter( + "relay_bytes_sent", + "The total number of bytes sent to relays in RTP packets." + ) + + /** + * The total duration, in milliseconds, of video streams (SSRCs) that were received. For example, if an + * endpoint sends simulcast with 3 SSRCs for 1 minute it would contribute a total of 3 minutes. Suspended + * streams do not contribute to this duration. + * + * This is updated on endpoint expiration. + */ + @JvmField + val totalVideoStreamMillisecondsReceived = metricsContainer.registerCounter( + "video_milliseconds_received", + "Total duration of video received, in milliseconds (each SSRC counts separately)." + ) + + val xmppDisconnects = metricsContainer.registerCounter( + "xmpp_disconnects", + "The number of times one of the XMPP connections has disconnected." + ) + + private val tossedPacketsEnergyBuckets = + listOf(0, 7, 15, 23, 31, 39, 47, 55, 63, 71, 79, 87, 95, 103, 111, 119, 127).map { it.toDouble() } + .toDoubleArray() + + @JvmField + val tossedPacketsEnergy = metricsContainer.registerHistogram( + "tossed_packets_energy", + "Distribution of energy scores for discarded audio packets.", + *tossedPacketsEnergyBuckets + ) + + /** The currently configured region, if any. */ + val regionInfo = if (RelayConfig.config.region != null) { + metricsContainer.registerInfo( + ColibriStatsExtension.REGION, + "The currently configured region.", + RelayConfig.config.region!! + ) + } else { + null + } + + /** Just set once to allow detection of restarts */ + private val startupTime = metricsContainer.registerLongGauge( + "startup_time", + "The startup time of the service.", + System.currentTimeMillis() + ) + + @JvmField + val rejectedDataChannels = metricsContainer.registerCounter( + "rejected_data_channels", + "Number of data channels rejected by the bridge." + ) +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt new file mode 100644 index 0000000..e13c0b1 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgeMetricsContainer.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2022 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.metrics.MetricsContainer + +/** + * The [MetricsContainer] instance for jitsi-videobridge where all Prometheus metrics are registered. + */ +class VideobridgeMetricsContainer private constructor() : MetricsContainer(namespace = "jitsi_jvb") { + + /** Acquire Metrics.lock to make sure we don't race with the updater. */ + override fun getPrometheusMetrics(contentType: String): String { + synchronized(Metrics.lock) { + return super.getPrometheusMetrics(contentType) + } + } + + /** Acquire Metrics.lock to make sure we don't race with the updater. */ + override val jsonString: String + get() = synchronized(Metrics.lock) { + super.jsonString + } + + companion object { + /** + * The singleton instance of `MetricsContainer`. + */ + @JvmStatic + val instance = VideobridgeMetricsContainer() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt new file mode 100644 index 0000000..cdd7ac4 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/metrics/VideobridgePeriodicMetrics.kt @@ -0,0 +1,370 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.metrics + +import org.jitsi.nlj.rtcp.RembHandler.Companion.endpointsWithSpuriousRemb +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.stats.ConferencePacketStats +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer.Companion.instance as metricsContainer + +/** + * Holds gauge metrics that need to be updated periodically. + */ +object VideobridgePeriodicMetrics { + val incomingLoss = metricsContainer.registerDoubleGauge( + "incoming_loss_fraction", + "Fraction of incoming RTP packets that are lost." + ) + val outgoingLoss = metricsContainer.registerDoubleGauge( + "outgoing_loss_fraction", + "Fraction of outgoing RTP packets that are lost." + ) + val loss = metricsContainer.registerDoubleGauge( + "loss_fraction", + "Fraction of RTP packets that are lost (incoming and outgoing combined)." + ) + + val relayIncomingBitrate = metricsContainer.registerLongGauge( + "relay_incoming_bitrate", + "Incoming RTP/RTCP bitrate from relays in bps." + ) + val relayOutgoingBitrate = metricsContainer.registerLongGauge( + "relay_outgoing_bitrate", + "Outgoing RTP/RTCP bitrate to relays in bps." + ) + val relayIncomingPacketRate = metricsContainer.registerLongGauge( + "relay_incoming_packet_rate", + "Incoming RTP/RTCP packet rate from relays in pps." + ) + val relayOutgoingPacketRate = metricsContainer.registerLongGauge( + "relay_outgoing_packet_rate", + "Outgoing RTP/RTCP packet rate to relays in pps." + ) + val incomingBitrate = metricsContainer.registerLongGauge( + "incoming_bitrate", + "Incoming RTP/RTCP bitrate in bps." + ) + val outgoingBitrate = metricsContainer.registerLongGauge( + "outgoing_bitrate", + "Outgoing RTP/RTCP bitrate in bps." + ) + val incomingPacketRate = metricsContainer.registerLongGauge( + "incoming_packet_rate", + "Incoming RTP/RTCP packet rate in pps." + ) + val outgoingPacketRate = metricsContainer.registerLongGauge( + "outgoing_packet_rate", + "Outgoing RTP/RTCP packet rate in pps." + ) + + val averageRtt = metricsContainer.registerDoubleGauge( + "average_rtt", + "Average RTT across all local endpoints in ms." + ) + + val largestConference = metricsContainer.registerLongGauge( + "largest_conference", + "The size of the largest conference (number of endpoints)." + ) + + val endpoints = metricsContainer.registerLongGauge( + "current_endpoints", + "Number of current endpoints (local and relayed)." + ) + val tccUnmatchedFeedback = metricsContainer.registerLongGauge( + "tcc_unmatched_feedback", + "Total transport-cc feedback reports that could not be matched to the send-time history. " + + "A sustained flood of these is the marker of a feedback-starvation saturation lock." + ) + val endpointsWithHighOutgoingLoss = metricsContainer.registerLongGauge( + "endpoints_with_high_outgoing_loss", + "Number of endpoints that have high outgoing loss (>10%)." + ) + val activeEndpoints = metricsContainer.registerLongGauge( + "active_endpoints", + "The number of active local endpoints (in a conference where at least one endpoint sends audio or video)." + ) + val endpointsSendingAudio = metricsContainer.registerLongGauge( + "endpoints_sending_audio", + "The number of local endpoints sending audio." + ) + val endpointsSendingVideo = metricsContainer.registerLongGauge( + "endpoints_sending_video", + "The number of local endpoints sending video." + ) + val endpointsRelayed = metricsContainer.registerLongGauge( + "endpoints_relayed", + "Number of relayed endpoints." + ) + val endpointsOversending = metricsContainer.registerLongGauge( + "endpoints_oversending", + "Number of endpoints that we are oversending to." + ) + val endpointsReceiveOnly = metricsContainer.registerLongGauge( + "endpoints_recvonly", + "Number of endpoints that are not sending audio or video (but are receiveing)." + ) + val endpointsInactive = metricsContainer.registerLongGauge( + "endpoints_inactive", + "Number of endpoints in inactive conferences (where no endpoint sends audio or video)." + ) + val endpointsWithSpuriousRemb = metricsContainer.registerLongGauge( + "endpoints_with_spurious_remb", + "Number of endpoints that have send a REMB packet even though REMB wasn't configured." + ) + val endpointsWithSuspendedSources = metricsContainer.registerLongGauge( + "endpoints_with_suspended_sources", + "Number of endpoints that that we have suspended sending some video streams to because of bwe." + ) + + val conferencesInactive = metricsContainer.registerLongGauge( + "conferences_inactive", + "Number of inactive conferences (no endpoint is sending audio or video)." + ) + val conferencesP2p = metricsContainer.registerLongGauge( + "conferences_p2p", + "Number of p2p conferences (inactive with 2 endpoints)." + ) + val conferencesWithRelay = metricsContainer.registerLongGauge( + "conferences_with_relay", + "Number of conferences with one or more relays." + ) + + private val conferenceSizeBuckets = (0..20).map { it.toDouble() }.toList().toDoubleArray() + val conferencesBySize = metricsContainer.registerHistogram( + "conferences_by_size", + "Histogram of conferences by total number of endpoints.", + *conferenceSizeBuckets + ) + val conferencesByAudioSender = metricsContainer.registerHistogram( + "conferences_by_audio_sender", + "Histogram of conferences by number of local endpoints sending audio.", + *conferenceSizeBuckets + ) + val conferencesByVideoSender = metricsContainer.registerHistogram( + "conferences_by_video_sender", + "Histogram of conferences by number of local endpoints sending video.", + *conferenceSizeBuckets + ) + + fun update(videobridge: Videobridge) { + var endpoints = 0L + var localEndpoints = 0L + var octoEndpoints = 0L + + var octoConferences = 0L + + var bitrateDownloadBps = 0.0 + var bitrateUploadBps = 0.0 + var packetRateUpload: Long = 0 + var packetRateDownload: Long = 0 + + var relayBitrateIncomingBps = 0.0 + var relayBitrateOutgoingBps = 0.0 + var relayPacketRateOutgoing: Long = 0 + var relayPacketRateIncoming: Long = 0 + + // Packets we received + var incomingPacketsReceived: Long = 0 + // Packets we should have received but were lost + var incomingPacketsLost: Long = 0 + // Packets we sent that were reported received + var outgoingPacketsReceived: Long = 0 + // Packets we sent that were reported lost + var outgoingPacketsLost: Long = 0 + + var rttSumMs = 0.0 + var rttCount: Long = 0 + var largestConferenceSize = 0L + var inactiveConferences = 0L + var p2pConferences = 0L + var inactiveEndpoints = 0L + var receiveOnlyEndpoints = 0L + var numAudioSenders = 0L + var numVideoSenders = 0L + // The number of endpoints to which we're "oversending" (which can occur when + // enableOnstageVideoSuspend is false) + var numOversending = 0L + var endpointsWithHighOutgoingLoss = 0L + var tccUnmatchedFeedback = 0L + var numLocalActiveEndpoints = 0L + var endpointsWithSuspendedSources = 0L + + val conferences = videobridge.conferences + val conferenceSizesList = ArrayList(conferences.size) + val audioSendersList = ArrayList(conferences.size) + val videoSendersList = ArrayList(conferences.size) + + for (conference in conferences) { + var conferenceBitrate: Long = 0 + var conferencePacketRate: Long = 0 + if (conference.isP2p) { + p2pConferences++ + } + val inactive = conference.isInactive + if (inactive) { + inactiveConferences++ + inactiveEndpoints += conference.endpointCount + } else { + numLocalActiveEndpoints += conference.localEndpointCount + } + if (conference.hasRelays()) { + octoConferences++ + } + val numConferenceEndpoints = conference.endpointCount.toLong() + val numLocalEndpoints = conference.localEndpointCount + localEndpoints += numLocalEndpoints + if (numConferenceEndpoints > largestConferenceSize) { + largestConferenceSize = numConferenceEndpoints + } + conferenceSizesList.add(numConferenceEndpoints) + endpoints += numConferenceEndpoints + octoEndpoints += numConferenceEndpoints - numLocalEndpoints + var conferenceAudioSenders = 0L + var conferenceVideoSenders = 0L + for (endpoint in conference.localEndpoints) { + if (endpoint.isOversending()) { + numOversending++ + } + val sendingAudio = endpoint.isSendingAudio + val sendingVideo = endpoint.isSendingVideo + if (sendingAudio) { + conferenceAudioSenders++ + } + if (sendingVideo) { + conferenceVideoSenders++ + } + if (!sendingAudio && !sendingVideo && !inactive) { + receiveOnlyEndpoints++ + } + if (endpoint.hasSuspendedSources()) { + endpointsWithSuspendedSources++ + } + val (endpointConnectionStats, rtpReceiverStats, _, outgoingStats, tccEngineStats) = + endpoint.transceiver.getTransceiverStats() + tccUnmatchedFeedback += tccEngineStats.unmatchedFeedback + val incomingStats = rtpReceiverStats.incomingStats + val incomingPacketStreamStats = rtpReceiverStats.packetStreamStats + bitrateDownloadBps += incomingPacketStreamStats.getBitrateBps() + packetRateDownload += incomingPacketStreamStats.packetRate + conferenceBitrate = conferenceBitrate + incomingPacketStreamStats.getBitrateBps() + conferencePacketRate += incomingPacketStreamStats.packetRate + bitrateUploadBps += outgoingStats.getBitrateBps() + packetRateUpload += outgoingStats.packetRate + conferenceBitrate = conferenceBitrate + outgoingStats.getBitrateBps() + conferencePacketRate += outgoingStats.packetRate + val endpointRtt = endpointConnectionStats.rtt + if (endpointRtt > 0) { + rttSumMs += endpointRtt + rttCount++ + } + incomingPacketsReceived += endpointConnectionStats.incomingLossStats.packetsReceived + incomingPacketsLost += endpointConnectionStats.incomingLossStats.packetsLost + val endpointOutgoingPacketsReceived = endpointConnectionStats.outgoingLossStats.packetsReceived + val endpointOutgoingPacketsLost = endpointConnectionStats.outgoingLossStats.packetsLost + outgoingPacketsReceived += endpointOutgoingPacketsReceived + outgoingPacketsLost += endpointOutgoingPacketsLost + if (!inactive && endpointOutgoingPacketsLost + endpointOutgoingPacketsReceived > 0) { + val endpointOutgoingFractionLost = ( + endpointOutgoingPacketsLost.toDouble() / + (endpointOutgoingPacketsLost + endpointOutgoingPacketsReceived) + ) + if (endpointOutgoingFractionLost > 0.1) { + endpointsWithHighOutgoingLoss++ + } + } + } + for (relay in conference.relays) { + relayBitrateIncomingBps += relay.incomingBitrateBps + relayPacketRateIncoming += relay.incomingPacketRate + conferenceBitrate = conferenceBitrate + relay.incomingBitrateBps + conferencePacketRate += relay.incomingPacketRate + relayBitrateOutgoingBps += relay.outgoingBitrateBps + relayPacketRateOutgoing += relay.outgoingPacketRate + conferenceBitrate = conferenceBitrate + relay.outgoingBitrateBps + conferencePacketRate += relay.outgoingPacketRate + + /* TODO: report Relay RTT and loss, like we do for Endpoints? */ + } + audioSendersList.add(conferenceAudioSenders) + numAudioSenders += conferenceAudioSenders + videoSendersList.add(conferenceVideoSenders) + numVideoSenders += conferenceVideoSenders + ConferencePacketStats.stats.addValue( + numConferenceEndpoints.toInt(), + conferencePacketRate, + conferenceBitrate + ) + } + + // RTT_AGGREGATE + val rttAggregate: Double = if (rttCount > 0) rttSumMs / rttCount else 0.0 + + var incomingLoss = 0.0 + if (incomingPacketsReceived + incomingPacketsLost > 0) { + incomingLoss = incomingPacketsLost.toDouble() / (incomingPacketsReceived + incomingPacketsLost) + } + var outgoingLoss = 0.0 + if (outgoingPacketsReceived + outgoingPacketsLost > 0) { + outgoingLoss = outgoingPacketsLost.toDouble() / (outgoingPacketsReceived + outgoingPacketsLost) + } + var overallLoss = 0.0 + if (incomingPacketsReceived + incomingPacketsLost + outgoingPacketsReceived + outgoingPacketsLost > 0) { + overallLoss = ( + (outgoingPacketsLost + incomingPacketsLost).toDouble() / + (incomingPacketsReceived + incomingPacketsLost + outgoingPacketsReceived + outgoingPacketsLost) + ) + } + + VideobridgePeriodicMetrics.incomingLoss.set(incomingLoss) + VideobridgePeriodicMetrics.outgoingLoss.set(outgoingLoss) + VideobridgePeriodicMetrics.loss.set(overallLoss) + VideobridgePeriodicMetrics.endpoints.set(endpoints) + VideobridgePeriodicMetrics.endpointsWithHighOutgoingLoss.set(endpointsWithHighOutgoingLoss) + VideobridgePeriodicMetrics.tccUnmatchedFeedback.set(tccUnmatchedFeedback) + VideobridgePeriodicMetrics.endpointsWithSpuriousRemb.set(endpointsWithSpuriousRemb().toLong()) + VideobridgePeriodicMetrics.activeEndpoints.set(numLocalActiveEndpoints) + VideobridgePeriodicMetrics.incomingBitrate.set(bitrateDownloadBps.toLong()) + VideobridgePeriodicMetrics.outgoingBitrate.set(bitrateUploadBps.toLong()) + VideobridgePeriodicMetrics.incomingPacketRate.set(packetRateDownload) + VideobridgePeriodicMetrics.outgoingPacketRate.set(packetRateUpload) + VideobridgePeriodicMetrics.averageRtt.set(rttAggregate) + VideobridgePeriodicMetrics.largestConference.set(largestConferenceSize) + VideobridgePeriodicMetrics.endpointsSendingAudio.set(numAudioSenders) + VideobridgePeriodicMetrics.endpointsSendingVideo.set(numVideoSenders) + VideobridgePeriodicMetrics.endpointsRelayed.set(octoEndpoints) + VideobridgePeriodicMetrics.endpointsOversending.set(numOversending) + VideobridgePeriodicMetrics.endpointsReceiveOnly.set(receiveOnlyEndpoints) + VideobridgePeriodicMetrics.endpointsInactive.set(inactiveEndpoints) + VideobridgePeriodicMetrics.endpointsWithSuspendedSources.set(endpointsWithSuspendedSources) + VideobridgePeriodicMetrics.conferencesInactive.set(inactiveConferences) + VideobridgePeriodicMetrics.conferencesP2p.set(p2pConferences) + VideobridgePeriodicMetrics.conferencesWithRelay.set(octoConferences) + VideobridgePeriodicMetrics.relayIncomingBitrate.set(relayBitrateIncomingBps.toLong()) + VideobridgePeriodicMetrics.relayOutgoingBitrate.set(relayBitrateOutgoingBps.toLong()) + VideobridgePeriodicMetrics.relayIncomingPacketRate.set(relayPacketRateIncoming) + VideobridgePeriodicMetrics.relayOutgoingPacketRate.set(relayPacketRateOutgoing) + + conferencesBySize.histogram.clear() + conferenceSizesList.forEach { conferencesBySize.histogram.observe(it.toDouble()) } + + conferencesByAudioSender.histogram.clear() + audioSendersList.forEach { conferencesByAudioSender.histogram.observe(it.toDouble()) } + + conferencesByVideoSender.histogram.clear() + videoSendersList.forEach { conferencesByVideoSender.histogram.observe(it.toDouble()) } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/AudioSourceDesc.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/AudioSourceDesc.kt new file mode 100644 index 0000000..5680560 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/AudioSourceDesc.kt @@ -0,0 +1,28 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.relay + +/** + * Description of an audio source, parallel to [MediaSourceDesc] which is just for video. + * TODO: these should be made more parallel and share a common base class, probably. + */ +data class AudioSourceDesc( + val ssrc: Long, + val owner: String? = null, + val sourceName: String? = null, + /** Whether this is a synthetic source, i.e. one generated by the bridge rather than sent by an endpoint. */ + val synthetic: Boolean = false, +) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt new file mode 100644 index 0000000..ee789c1 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/Relay.kt @@ -0,0 +1,1227 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.relay + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.ice4j.util.Buffer +import org.jitsi.dcsctp4j.DcSctpMessage +import org.jitsi.dcsctp4j.ErrorKind +import org.jitsi.dcsctp4j.SendPacketStatus +import org.jitsi.dcsctp4j.SendStatus +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.Features +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.PacketOrigin +import org.jitsi.nlj.Transceiver +import org.jitsi.nlj.TransceiverEventHandler +import org.jitsi.nlj.VideoType +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtcp.RtcpListener +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.rtp.RtpExtensionType +import org.jitsi.nlj.rtp.SsrcAssociationType +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.srtp.SrtpUtil +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.nlj.stats.EndpointConnectionStats +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.transform.node.ToggleablePcapWriter +import org.jitsi.nlj.transform.pipeline +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.BufferPool +import org.jitsi.nlj.util.LocalSsrcAssociation +import org.jitsi.nlj.util.PacketInfoQueue +import org.jitsi.nlj.util.RemoteSsrcAssociation +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.extensions.looksLikeRtcp +import org.jitsi.rtp.extensions.looksLikeRtp +import org.jitsi.rtp.rtcp.CompoundRtcpPacket +import org.jitsi.rtp.rtcp.RtcpByePacket +import org.jitsi.rtp.rtcp.RtcpHeader +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.RtcpRrPacket +import org.jitsi.rtp.rtcp.RtcpSdesPacket +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket +import org.jitsi.rtp.rtp.RtpHeader +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.sumOf +import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.CryptexConfig +import org.jitsi.videobridge.EncodingsManager +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.PotentialPacketHandler +import org.jitsi.videobridge.TransportConfig +import org.jitsi.videobridge.datachannel.DataChannelStack +import org.jitsi.videobridge.datachannel.protocol.DataChannelPacket +import org.jitsi.videobridge.datachannel.protocol.DataChannelProtocolConstants +import org.jitsi.videobridge.dcsctp.DcSctpBaseCallbacks +import org.jitsi.videobridge.dcsctp.DcSctpHandler +import org.jitsi.videobridge.dcsctp.DcSctpTransport +import org.jitsi.videobridge.message.BridgeChannelMessage +import org.jitsi.videobridge.message.SourceVideoTypeMessage +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.rest.root.debug.EndpointDebugFeatures +import org.jitsi.videobridge.stats.PacketTransitStats +import org.jitsi.videobridge.transport.dtls.DtlsTransport +import org.jitsi.videobridge.transport.ice.IceTransport +import org.jitsi.videobridge.util.ByteBufferPool +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.videobridge.util.looksLikeDtls +import org.jitsi.videobridge.websocket.colibriWebSocketServiceSupplier +import org.jitsi.videobridge.websocket.generateColibriWebSocketPassword +import org.jitsi.xmpp.extensions.colibri.WebSocketPacketExtension +import org.jitsi.xmpp.extensions.colibri2.Sctp +import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import java.nio.ByteBuffer +import java.time.Clock +import java.time.Instant +import java.util.* +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import kotlin.collections.ArrayList +import kotlin.collections.HashMap +import kotlin.collections.HashSet +import kotlin.collections.sumOf +import org.jitsi.videobridge.sctp.SctpConfig.Companion.config as sctpConfig + +/** + * Models a relay (remote videobridge) in a [Conference]. + */ +/* TODO: figure out how best to share code between this and [Endpoint], without multiple inheritance. */ +class Relay @JvmOverloads constructor( + /** + * The unique identifier of this [Relay] + */ + val id: String, + /** + * The [Conference] this [Relay] belongs to. + */ + val conference: Conference, + parentLogger: Logger, + /** + * The ID of the mesh to which this [Relay] connection belongs. + * (Note that a bridge can be a member of more than one mesh, but each relay link will belong to only one.) + */ + val meshId: String?, + /** + * True if the ICE agent for this [Relay] will be initialized to serve as a controlling ICE agent, false otherwise. + */ + iceControlling: Boolean, + useUniquePort: Boolean, + clock: Clock = Clock.systemUTC() +) : EncodingsManager.EncodingsUpdateListener, PotentialPacketHandler { + + /** + * The [Logger] used by the [Relay] class to print debug information. + */ + private val logger = createChildLogger(parentLogger).apply { addContext("relayId", id) } + + /** + * A cache of the signaled payload types, since these are only signaled + * at the top level but apply to all relayed endpoints + */ + private val payloadTypes: MutableList = ArrayList() + + /** + * A cache of the signaled rtp extensions, since these are only signaled + * at the top level but apply to all relayed endpoints + */ + private val rtpExtensions: MutableList = ArrayList() + + /** + * A cache of extmap-allow-mixed + */ + private var extmapAllowMixed = false + + /** + * The indicator which determines whether [expire] has been called on this [Relay]. + */ + private var expired = false + + private val sctpHandler = if (sctpConfig.enabled) DcSctpHandler() else null + + /** The [DcSctpTransport] instance we'll use to manage the SCTP connection */ + private var sctpTransport: DcSctpTransport? = null + + /** The role we'll play in the SCTP handshake, if negotiated */ + private var sctpRole: Sctp.Role? = null + + private var dataChannelStack = DataChannelStack( + { data, sid, ppid -> + val message = DcSctpMessage(sid.toShort(), ppid, data.array()) + val status = sctpTransport?.send(message, DcSctpTransport.DEFAULT_SEND_OPTIONS) + return@DataChannelStack if (status == SendStatus.kSuccess) { + 0 + } else { + logger.error("Error sending to SCTP: $status") + -1 + } + }, + logger + ) + + private val toggleablePcapWriter = ToggleablePcapWriter(logger, "$id-sctp") + private val sctpRecvPcap = toggleablePcapWriter.newObserverNode(outbound = false, suffix = "rx_sctp") + private val sctpSendPcap = toggleablePcapWriter.newObserverNode(outbound = true, suffix = "tx_sctp") + + private val sctpPipeline = pipeline { + node(sctpRecvPcap) + sctpHandler?.let { + node(it) + } + } + + /** + * The password which authenticates the colibri WebSocket of this relay. It is independent of ICE and stays + * the same for the lifetime of the relay, because the peer re-dials the URL which contains it each time the + * WebSocket reconnects. See [generateColibriWebSocketPassword]. + */ + private val webSocketPassword = generateColibriWebSocketPassword() + + private val iceTransport = IceTransport( + id = id, + controlling = iceControlling, + useUniquePort = useUniquePort, + // There's no good reason to disable private addresses. + advertisePrivateAddresses = true, + parentLogger = logger, + clock = clock + ) + + private val dtlsTransport = DtlsTransport(logger, id).also { it.cryptex = CryptexConfig.relay } + + private var cryptex = CryptexConfig.relay + + private val diagnosticContext = conference.newDiagnosticContext().apply { + put("relay_id", id) + } + + private val timelineLogger = logger.createChildLogger("timeline.${this.javaClass.name}") + + private val relayedEndpoints = HashMap() + private val endpointsBySsrc = HashMap() + private val endpointsLock = Any() + + private val senders = ConcurrentHashMap() + + val statistics = Statistics() + + /** + * Listen for RTT updates from [transceiver] and update the ICE stats the first time an RTT is available. Note that + * the RTT is measured via RTCP, since we don't expose response time for STUN requests. + */ + private val rttListener: EndpointConnectionStats.EndpointConnectionStatsListener = + object : EndpointConnectionStats.EndpointConnectionStatsListener { + override fun onRttUpdate(newRttMs: Double) { + if (newRttMs > 0) { + transceiver.removeEndpointConnectionStatsListener(this) + iceTransport.updateStatsOnInitialRtt(newRttMs) + } + } + } + + /* This transceiver is only for packets that are not handled by [RelayedEndpoint]s + * or [RelayEndpointSender]s */ + val transceiver = Transceiver( + id, + TaskPools.CPU_POOL, + TaskPools.CPU_POOL, + TaskPools.SCHEDULED_POOL, + diagnosticContext, + logger, + TransceiverEventHandlerImpl(), + clock + ).apply { + setIncomingPacketHandler(object : ConsumerNode("receiver chain handler") { + override fun consume(packetInfo: PacketInfo) { + this@Relay.handleIncomingPacket(packetInfo) + } + + override fun trace(f: () -> Unit) = f.invoke() + }) + addEndpointConnectionStatsListener(rttListener) + setLocalSsrc(MediaType.AUDIO, conference.localAudioSsrc) + setLocalSsrc(MediaType.VIDEO, conference.localVideoSsrc) + rtcpEventNotifier.addRtcpEventListener( + object : RtcpListener { + override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) { + this@Relay.rtcpPacketReceived(packet, receivedTime, null) + } + override fun rtcpPacketSent(packet: RtcpPacket) { + this@Relay.rtcpPacketSent(packet, null) + } + }, + external = true + ) + addRtpExtensionToRetain(RtpExtensionType.VLA) + } + + /** + * The instance which manages the Colibri messaging (over web sockets). + */ + private val messageTransport = RelayMessageTransport( + this, + conference, + logger + ) + + init { + conference.encodingsManager.subscribe(this) + setupIceTransport() + setupDtlsTransport() + + VideobridgeMetrics.totalRelays.inc() + } + + fun getMessageTransport(): RelayMessageTransport = messageTransport + + /** + * The queue we put outgoing SRTP packets onto so they can be sent + * out via the [IceTransport] on an IO thread. This queue is only + * for packets that are not handled by [RelayEndpointSender]s. + */ + private val outgoingSrtpPacketQueue = PacketInfoQueue( + "${javaClass.simpleName}-outgoing-packet-queue", + TaskPools.IO_POOL, + this::doSendSrtp, + TransportConfig.queueSize + ).apply { + setErrorHandler(queueErrorCounter) + } + + /** + * The queue which enforces sequential processing of incoming data channel messages + * to maintain processing order. + */ + private val incomingDataChannelMessagesQueue = PacketInfoQueue( + "${javaClass.simpleName}-incoming-data-channel-queue", + TaskPools.IO_POOL, + { packetInfo -> + packetInfo.packetAs().let { packet -> + dataChannelStack.onIncomingDataChannelPacket( + ByteBuffer.wrap(packet.buffer), + packet.sid, + packet.ppid + ) + } + true + }, + TransportConfig.queueSize + ) + + fun debugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("ice_transport", iceTransport.getDebugState()) + set("dtls_transport", dtlsTransport.getDebugState()) + set("transceiver", transceiver.debugState(mode)) + put("mesh_id", meshId) + set("message_transport", messageTransport.debugState) + sctpTransport?.let { + set("sctp", it.getDebugState()) + } + + if (mode == DebugStateMode.FULL) { + val remoteEndpoints = JsonNodeFactory.instance.objectNode() + val endpointsBySsrcMap = JsonNodeFactory.instance.objectNode() + synchronized(endpointsLock) { + for (r in relayedEndpoints.values) { + remoteEndpoints.set(r.id, r.debugState(mode)) + } + for ((s, e) in endpointsBySsrc) { + endpointsBySsrcMap.put(s.toString(), e.id) + } + } + set("remote_endpoints", remoteEndpoints) + set("endpoints_by_ssrc", endpointsBySsrcMap) + val endpointSenders = JsonNodeFactory.instance.objectNode() + for (s in senders.values) { + endpointSenders.set(s.id, s.getDebugState(mode)) + } + set("senders", endpointSenders) + } + } + + private fun setupIceTransport() { + iceTransport.incomingDataHandler = object : IceTransport.IncomingDataHandler { + override fun dataReceived(buffer: Buffer) { + // DTLS data will be handled by the DtlsTransport, but SRTP data can go + // straight to the transceiver + if (looksLikeDtls(buffer.buffer, buffer.offset, buffer.length)) { + // DTLS transport is responsible for making its own copy, because it will manage its own buffers + dtlsTransport.enqueueBuffer(buffer) + } else { + val pktInfo = + RelayedPacketInfo( + UnparsedPacket(buffer.buffer, buffer.offset, buffer.length), + meshId + ).apply { + this.receivedTime = buffer.receivedTime + } + pktInfo.packetOrigin = PacketOrigin.Routed + handleMediaPacket(pktInfo) + } + } + } + iceTransport.eventHandler = object : IceTransport.EventHandler { + override fun writeable() { + logger.info("ICE connected") + transceiver.setOutgoingPacketHandler(object : PacketHandler { + override fun processPacket(packetInfo: PacketInfo) { + packetInfo.addEvent(SRTP_QUEUE_ENTRY_EVENT) + outgoingSrtpPacketQueue.add(packetInfo) + } + }) + TaskPools.IO_POOL.execute(dtlsTransport::startDtlsHandshake) + } + + override fun connected() {} + + override fun failed() { + VideobridgeMetrics.relaysIceFailed.inc() + } + + override fun consentUpdated(time: Instant) { + transceiver.packetIOActivity.lastIceActivityInstant = time + } + } + } + + private fun setupDtlsTransport() { + dtlsTransport.incomingDataHandler = object : DtlsTransport.IncomingDataHandler { + override fun dtlsAppDataReceived(buf: ByteArray, off: Int, len: Int) { + dtlsAppPacketReceived(buf, off, len) + } + } + dtlsTransport.outgoingDataHandler = object : DtlsTransport.OutgoingDataHandler { + override fun sendData(buf: ByteArray, off: Int, len: Int) { + iceTransport.send(buf, off, len) + } + } + dtlsTransport.eventHandler = object : DtlsTransport.EventHandler { + override fun handshakeComplete( + chosenSrtpProtectionProfile: Int, + tlsRole: TlsRole, + keyingMaterial: ByteArray + ) { + logger.info("DTLS handshake complete") + setSrtpInformation(chosenSrtpProtectionProfile, tlsRole, keyingMaterial) + scheduleRelayMessageTransportTimeout() + if (sctpConfig.enabled) { + if (sctpRole == Sctp.Role.CLIENT) { + sctpTransport!!.connect() + } + } + } + } + } + + private var srtpTransformers: SrtpTransformers? = null + + private fun setSrtpInformation(chosenSrtpProtectionProfile: Int, tlsRole: TlsRole, keyingMaterial: ByteArray) { + val srtpProfileInfo = + SrtpUtil.getSrtpProfileInformationFromSrtpProtectionProfile(chosenSrtpProtectionProfile) + logger.cdebug { + "Transceiver $id creating transformers with:\n" + + "profile info:\n$srtpProfileInfo\n" + + "tls role: $tlsRole" + } + val srtpTransformers = SrtpUtil.initializeTransformer( + srtpProfileInfo, + keyingMaterial, + tlsRole, + cryptex, + logger + ) + this.srtpTransformers = srtpTransformers + + transceiver.setSrtpInformation(srtpTransformers) + synchronized(endpointsLock) { + relayedEndpoints.values.forEach { it.setSrtpInformation(srtpTransformers) } + } + + senders.values.forEach { it.setSrtpInformation(srtpTransformers) } + } + + /** + * Create an SCTP connection for this Relay. If [sctpDesc.role] is [Sctp.Role.CLIENT], + * we will create the data channel locally, otherwise we will wait for the remote side + * to open it. + */ + fun createSctpConnection(sctpDesc: Sctp) { + if (sctpConfig.enabled) { + createDcSctpConnection(sctpDesc) + } else { + logger.error("Not creating SCTP connection, SCTP is disabled in configuration.") + } + } + + private fun createDcSctpConnection(sctpDesc: Sctp) { + sctpRole = sctpDesc.role + + logger.cdebug { "Creating SCTP transport" } + sctpTransport = DcSctpTransport(id, logger).also { + it.start(SctpCallbacks(it)) + sctpHandler!!.setSctpTransport(it) + if (dtlsTransport.isConnected && sctpDesc.role == Sctp.Role.CLIENT) { + it.connect() + } + } + } + + /** + * Sets the remote transport information (ICE candidates, DTLS fingerprints). + * + * @param transportInfo the XML extension which contains the remote + * transport information. + */ + fun setTransportInfo(transportInfo: IceUdpTransportPacketExtension) { + val remoteFingerprints = mutableMapOf>() + val fingerprintExtensions = transportInfo.getChildExtensionsOfType(DtlsFingerprintPacketExtension::class.java) + fingerprintExtensions.forEach { fingerprintExtension -> + if (fingerprintExtension.hash != null && fingerprintExtension.fingerprint != null) { + remoteFingerprints.getOrPut(fingerprintExtension.hash.lowercase()) { mutableListOf() } + .add(fingerprintExtension.fingerprint) + } else { + logger.info("Ignoring empty DtlsFingerprint extension: ${transportInfo.toXML()}") + } + + if (CryptexConfig.relay) { + cryptex = cryptex && fingerprintExtension.cryptex + } + } + dtlsTransport.setRemoteFingerprints(remoteFingerprints) + if (fingerprintExtensions.isNotEmpty()) { + val setup = fingerprintExtensions.first().setup + dtlsTransport.setSetupAttribute(setup) + } + iceTransport.startConnectivityEstablishment(transportInfo) + + val websocketExtension = transportInfo.getFirstChildOfType(WebSocketPacketExtension::class.java) + websocketExtension?.url?.let { messageTransport.connectToWebsocket(it) } + } + + fun describeTransport(): IceUdpTransportPacketExtension { + val iceUdpTransportPacketExtension = IceUdpTransportPacketExtension() + iceTransport.describe(iceUdpTransportPacketExtension) + dtlsTransport.describe(iceUdpTransportPacketExtension) + + if (sctpTransport == null) { + /* TODO: this should be dependent on videobridge.websockets.enabled, if we support that being + * disabled for relay. + */ + if (messageTransport.isActive) { + iceUdpTransportPacketExtension.addChildExtension( + WebSocketPacketExtension().apply { active = true } + ) + } else { + colibriWebSocketServiceSupplier.get()?.let { colibriWebsocketService -> + val urls = colibriWebsocketService.getColibriRelayWebSocketUrls( + conference.id, + id, + webSocketPassword + ) + if (urls.isEmpty()) { + logger.warn("No colibri relay URLs configured") + } + urls.forEach { + iceUdpTransportPacketExtension.addChildExtension( + WebSocketPacketExtension().apply { + url = it + } + ) + } + } + } + } + + logger.cdebug { "Transport description:\n${iceUdpTransportPacketExtension.toXML()}" } + + return iceUdpTransportPacketExtension + } + + fun setFeature(feature: EndpointDebugFeatures, enabled: Boolean) { + when (feature) { + EndpointDebugFeatures.PCAP_DUMP -> { + transceiver.setFeature(Features.TRANSCEIVER_PCAP_DUMP, enabled) + synchronized(endpointsLock) { + relayedEndpoints.values.forEach { e -> e.setFeature(Features.TRANSCEIVER_PCAP_DUMP, enabled) } + } + senders.values.forEach { s -> s.setFeature(Features.TRANSCEIVER_PCAP_DUMP, enabled) } + } + + EndpointDebugFeatures.SCTP_PCAP_DUMP -> + if (enabled) { + toggleablePcapWriter.enable() + } else { + toggleablePcapWriter.disable() + } + } + } + + fun isFeatureEnabled(feature: EndpointDebugFeatures): Boolean = when (feature) { + EndpointDebugFeatures.PCAP_DUMP -> transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP) + EndpointDebugFeatures.SCTP_PCAP_DUMP -> toggleablePcapWriter.isEnabled() + } + + /** + * Handle media packets that have arrived, using the appropriate endpoint's transceiver. + */ + private fun handleMediaPacket(packetInfo: RelayedPacketInfo) { + if (packetInfo.packet.looksLikeRtp()) { + val ssrc = RtpHeader.getSsrc(packetInfo.packet.buffer, packetInfo.packet.offset) + val ep = getEndpointBySsrc(ssrc) + if (ep != null) { + ep.handleIncomingPacket(packetInfo) + return + } else { + logger.warn { "RTP Packet received for unknown endpoint SSRC $ssrc" } + BufferPool.returnBuffer(packetInfo.packet.buffer) + } + } else if (packetInfo.packet.looksLikeRtcp()) { + val ssrc = RtcpHeader.getSenderSsrc(packetInfo.packet.buffer, packetInfo.packet.offset) + val ep = getEndpointBySsrc(ssrc) + if (ep != null) { + ep.handleIncomingPacket(packetInfo) + return + } else { + /* Handle RTCP from non-endpoint senders on the generic transceiver - it's probably + * from a feedback source. + */ + transceiver.handleIncomingPacket(packetInfo) + } + } + } + + /** + * Handle incoming RTP packets which have been fully processed by the + * transceiver's incoming pipeline. + */ + fun handleIncomingPacket(packetInfo: PacketInfo) { + val packet = packetInfo.packet + if (packet is RtpPacket) { + val ep = synchronized(endpointsLock) { endpointsBySsrc[packet.ssrc] } + if (ep != null) { + packetInfo.endpointId = ep.id + } + } + /* RTCP packets are left without an endpointId. Keyframe requests relayed from another bridge can not be + * attributed to a single receiver, so the KeyframeRequester applies only the source-wide rate limit to them. */ + conference.handleIncomingPacket(packetInfo) + } + + override fun onNewSsrcAssociation( + endpointId: String, + primarySsrc: Long, + secondarySsrc: Long, + type: SsrcAssociationType + ) { + if (synchronized(endpointsLock) { relayedEndpoints.containsKey(endpointId) }) { + transceiver.addSsrcAssociation(LocalSsrcAssociation(primarySsrc, secondarySsrc, type)) + } else { + transceiver.addSsrcAssociation(RemoteSsrcAssociation(primarySsrc, secondarySsrc, type)) + } + } + + fun doSendSrtp(packetInfo: PacketInfo): Boolean { + packetInfo.addEvent(SRTP_QUEUE_EXIT_EVENT) + + iceTransport.send(packetInfo.packet.buffer, packetInfo.packet.offset, packetInfo.packet.length) + PacketTransitStats.packetSent(packetInfo) + ByteBufferPool.returnBuffer(packetInfo.packet.buffer) + packetInfo.sent() + if (timelineLogger.isTraceEnabled && Endpoint.logTimeline()) { + timelineLogger.trace { packetInfo.timeline.toString() } + } + return true + } + + /** + * Sends a specific message to the remote side. + */ + fun sendMessage(msg: BridgeChannelMessage) = messageTransport.sendMessage(msg) + + fun relayMessageTransportConnected() { + relayedEndpoints.values.forEach { e -> e.relayMessageTransportConnected() } + conference.endpoints.forEach { e -> + if (e is Endpoint || (e is RelayedEndpoint && e.relay.meshId != meshId)) { + e.mediaSources.forEach { msd: MediaSourceDesc -> + // Do not send the initial value for CAMERA, because it's the default + if (msd.videoType != VideoType.CAMERA) { + val videoTypeMsg = SourceVideoTypeMessage( + msd.videoType, + msd.sourceName, + e.id + ) + sendMessage(videoTypeMsg) + } + } + } + } + } + + /** + * Handle a DTLS app packet (that is, a packet of some other protocol sent + * over DTLS) which has just been received. + */ + // TODO(brian): change sctp handler to take buf, off, len + fun dtlsAppPacketReceived(data: ByteArray, off: Int, len: Int) = + sctpPipeline.processPacket(PacketInfo(UnparsedPacket(data, off, len))) + + /** + * Return the newly created endpoint, or null if an endpoint with that ID already existed. Note that the new + * endpoint has to be added to the [Conference] separately. + */ + fun addRemoteEndpoint( + id: String, + statsId: String?, + audioSources: Collection, + videoSources: Collection + ): RelayedEndpoint? { + val ep: RelayedEndpoint + synchronized(endpointsLock) { + if (relayedEndpoints.containsKey(id)) { + logger.warn("Relay already contains remote endpoint with ID $id") + updateRemoteEndpoint(id, audioSources, videoSources) + return null + } + ep = RelayedEndpoint( + conference, + this, + id, + logger, + conference.newDiagnosticContext().apply { + put("relay_id", this@Relay.id) + put("endpoint_id", id) + } + ) + ep.statsId = statsId + ep.audioSources = audioSources.toList() + ep.mediaSources = videoSources.toTypedArray() + + relayedEndpoints[id] = ep + + ep.ssrcs.forEach { ssrc -> endpointsBySsrc[ssrc] = ep } + } + + srtpTransformers?.let { ep.setSrtpInformation(it) } + payloadTypes.forEach { payloadType -> ep.addPayloadType(payloadType) } + rtpExtensions.forEach { rtpExtension -> ep.addRtpExtension(rtpExtension) } + ep.setExtmapAllowMixed(extmapAllowMixed) + + setEndpointMediaSources(ep, audioSources, videoSources) + + ep.setFeature(Features.TRANSCEIVER_PCAP_DUMP, transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP)) + return ep + } + + fun updateRemoteEndpoint( + id: String, + audioSources: Collection, + videoSources: Collection + ) { + val ep: RelayedEndpoint + synchronized(endpointsLock) { + ep = relayedEndpoints[id] ?: run { + logger.warn("Endpoint with ID $id not found in relay") + return + } + val oldSsrcs = ep.ssrcs + + ep.audioSources = audioSources.toList() + ep.mediaSources = videoSources.toTypedArray() + + val newSsrcs = ep.ssrcs + val removedSsrcs = oldSsrcs.minus(newSsrcs) + val addedSsrcs = newSsrcs.minus(oldSsrcs) + + endpointsBySsrc.keys.removeAll(removedSsrcs) + addedSsrcs.forEach { ssrc -> endpointsBySsrc[ssrc] = ep } + } + setEndpointMediaSources(ep, audioSources, videoSources) + } + + fun removeRemoteEndpoint(id: String) { + val ep: RelayedEndpoint? + synchronized(endpointsLock) { + ep = relayedEndpoints.remove(id) + if (ep != null) { + endpointsBySsrc.keys.removeAll(ep.ssrcs) + } + } + ep?.expire() + } + + private fun getOrCreateRelaySender(endpointId: String): RelayEndpointSender { + synchronized(senders) { + senders[endpointId]?.let { return it } + + val s = RelayEndpointSender( + this, + endpointId, + logger, + conference.newDiagnosticContext().apply { + put("relay_id", id) + put("endpoint_id", endpointId) + } + ) + + srtpTransformers?.let { s.setSrtpInformation(it) } + payloadTypes.forEach { payloadType -> s.addPayloadType(payloadType) } + rtpExtensions.forEach { rtpExtension -> s.addRtpExtension(rtpExtension) } + s.setExtmapAllowMixed(extmapAllowMixed) + s.setFeature(Features.TRANSCEIVER_PCAP_DUMP, transceiver.isFeatureEnabled(Features.TRANSCEIVER_PCAP_DUMP)) + + senders[endpointId] = s + + return s + } + } + + fun addPayloadType(payloadType: PayloadType) { + transceiver.addPayloadType(payloadType) + payloadTypes.add(payloadType) + synchronized(endpointsLock) { + relayedEndpoints.values.forEach { ep -> ep.addPayloadType(payloadType) } + } + senders.values.forEach { s -> s.addPayloadType(payloadType) } + } + + fun addRtpExtension(rtpExtension: RtpExtension) { + /* We don't want to do any BWE for relay-relay channels; also, the sender split confuses things. */ + if (rtpExtension.type == RtpExtensionType.TRANSPORT_CC || + rtpExtension.type == RtpExtensionType.ABS_SEND_TIME + ) { + return + } + transceiver.addRtpExtension(rtpExtension) + rtpExtensions.add(rtpExtension) + synchronized(endpointsLock) { + relayedEndpoints.values.forEach { ep -> ep.addRtpExtension(rtpExtension) } + } + senders.values.forEach { s -> s.addRtpExtension(rtpExtension) } + } + + fun setExtmapAllowMixed(allow: Boolean) { + transceiver.setExtmapAllowMixed(allow) + extmapAllowMixed = allow + + synchronized(endpointsLock) { + relayedEndpoints.values.forEach { ep -> ep.setExtmapAllowMixed(allow) } + } + senders.values.forEach { s -> s.setExtmapAllowMixed(allow) } + } + + private fun setEndpointMediaSources( + ep: RelayedEndpoint, + audioSources: Collection, + videoSources: Collection + ) { + ep.audioSources = audioSources.toList() + ep.mediaSources = videoSources.toTypedArray() + } + + fun getEndpoint(id: String): RelayedEndpoint? = synchronized(endpointsLock) { relayedEndpoints[id] } + + fun getEndpointBySsrc(ssrc: Long): RelayedEndpoint? = synchronized(endpointsLock) { endpointsBySsrc[ssrc] } + + /** + * Schedule a timeout to fire log a message and track a stat if we don't + * have a relay message transport connected within the timeout. + */ + fun scheduleRelayMessageTransportTimeout() { + TaskPools.SCHEDULED_POOL.schedule( + { + if (!expired) { + if (!messageTransport.isConnected) { + logger.error("RelayMessageTransport still not connected.") + VideobridgeMetrics.numRelaysNoMessageTransportAfterDelay.inc() + } + } + }, + 30, + TimeUnit.SECONDS + ) + } + + /** + * Checks whether a WebSocket connection with a specific password string + * should be accepted for this relay. + * @param password the + * @return {@code true} iff the password matches. + */ + fun acceptWebSocket(password: String): Boolean { + if (webSocketPassword != password) { + logger.warn("Incoming web socket request with an invalid password.") + return false + } + return true + } + + /** + * Get a collection of all of the SSRCs mentioned in an RTCP packet. + */ + private fun getRtcpSsrcs(packet: RtcpPacket): Collection { + val ssrcs = HashSet() + ssrcs.add(packet.senderSsrc) + when (packet) { + is CompoundRtcpPacket -> packet.packets.forEach { ssrcs.addAll(getRtcpSsrcs(it)) } + + is RtcpFbFirPacket -> ssrcs.add(packet.mediaSenderSsrc) + + // TODO: support multiple FIRs in a packet + is RtcpFbPacket -> ssrcs.add(packet.mediaSourceSsrc) + + is RtcpSrPacket -> packet.reportBlocks.forEach { ssrcs.add(it.ssrc) } + + is RtcpRrPacket -> packet.reportBlocks.forEach { ssrcs.add(it.ssrc) } + + is RtcpSdesPacket -> packet.sdesChunks.forEach { ssrcs.add(it.ssrc) } + + is RtcpByePacket -> ssrcs.addAll(packet.ssrcs) + } + return ssrcs + } + + /** + * Call a callback on an [RtcpEventNotifier] once per [RelayedEndpoint] or [RelayEndpointSender] whose + * SSRC is mentioned in an RTCP packet, plus on the Relay's local [Transceiver]. + * Do not call the event notifier which was the source of this packet, based on the [endpointId]. + */ + private fun doRtcpCallbacks(packet: RtcpPacket, endpointId: String?, callback: (RtcpEventNotifier) -> Unit) { + val ssrcs = getRtcpSsrcs(packet) + + val eps = HashSet() + ssrcs.forEach { conference.getEndpointBySsrc(it)?.let { eps.add(it.id) } } + endpointId?.let { eps.remove(it) } + + eps.forEach { epId -> + /* Any given ID should only be in one or the other of these sets. */ + getEndpoint(epId)?.let { callback(it.rtcpEventNotifier) } + senders[epId]?.let { callback(it.rtcpEventNotifier) } + } + + if (endpointId != null) { + callback(transceiver.rtcpEventNotifier) + } + } + + fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?, endpointId: String?) { + doRtcpCallbacks(packet, endpointId) { it.notifyRtcpReceived(packet, receivedTime, external = true) } + } + + fun rtcpPacketSent(packet: RtcpPacket, endpointId: String?) { + doRtcpCallbacks(packet, endpointId) { it.notifyRtcpSent(packet, external = true) } + } + + /** + * Returns true if this endpoint's transport is 'fully' connected (both ICE and DTLS), false otherwise + */ + private fun isTransportConnected(): Boolean = iceTransport.isConnected() && dtlsTransport.isConnected + + /* If we're connected, forward everything that didn't come in over a relay or that came from a different + relay mesh. + TODO: worry about bandwidth limits on relay links? */ + override fun wants(packet: PacketInfo): Boolean { + if (!isTransportConnected()) { + return false + } + if (packet is RelayedPacketInfo && packet.meshId == meshId) { + return false + } + + return when (packet.packet) { + is VideoRtpPacket, is AudioRtpPacket, is RtcpSrPacket, + is RtcpFbPliPacket, is RtcpFbFirPacket -> { + // We assume that we are only given PLIs/FIRs destined for this + // endpoint. This is because Conference has to find the target + // endpoint (belonging to this relay) anyway, and we would essentially be + // performing the same check twice. + true + } + + else -> { + logger.warn("Ignoring an unknown packet type:" + packet.packet.javaClass.simpleName) + false + } + } + } + + override fun send(packet: PacketInfo) { + packet.endpointId?.let { + getOrCreateRelaySender(it).sendPacket(packet) + } ?: run { + transceiver.sendPacket(packet) + } + } + + fun endpointExpired(id: String) { + val s = senders.remove(id) + s?.expire() + } + + val incomingBitrateBps: Long + get() = transceiver.getTransceiverStats().rtpReceiverStats.packetStreamStats.getBitrateBps() + + synchronized(endpointsLock) { + relayedEndpoints.values.sumOf { it.getIncomingStats().getBitrateBps() } + } + + val incomingPacketRate: Long + get() = transceiver.getTransceiverStats().rtpReceiverStats.packetStreamStats.packetRate + + synchronized(endpointsLock) { + relayedEndpoints.values.sumOf { it.getIncomingStats().packetRate } + } + + val outgoingBitrateBps: Long + get() = transceiver.getTransceiverStats().outgoingPacketStreamStats.getBitrateBps() + + senders.values.sumOf { it.getOutgoingStats().getBitrateBps() } + + val outgoingPacketRate: Long + get() = transceiver.getTransceiverStats().outgoingPacketStreamStats.packetRate + + senders.values.sumOf { it.getOutgoingStats().packetRate } + + /** + * Updates the conference statistics with value from this endpoint. Since + * the values are cumulative this should execute only once when the endpoint + * expires. + */ + private fun updateStatsOnExpire() { + val transceiverStats = transceiver.getTransceiverStats() + + // Add stats from the local transceiver + val incomingStats = transceiverStats.rtpReceiverStats.packetStreamStats + val outgoingStats = transceiverStats.outgoingPacketStreamStats + + statistics.bytesReceived.getAndAdd(incomingStats.bytes) + statistics.packetsReceived.getAndAdd(incomingStats.packets) + statistics.bytesSent.getAndAdd(outgoingStats.bytes) + statistics.packetsSent.getAndAdd(outgoingStats.packets) + + VideobridgeMetrics.totalRelayBytesReceived.add(statistics.bytesReceived.get()) + VideobridgeMetrics.totalRelayBytesSent.add(statistics.bytesSent.get()) + VideobridgeMetrics.relayPacketsReceived.add(statistics.packetsReceived.get()) + VideobridgeMetrics.relayPacketsSent.add(statistics.packetsSent.get()) + + VideobridgeMetrics.keyframesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numKeyframes.toLong() + ) + VideobridgeMetrics.layeringChangesReceived.addAndGet( + transceiverStats.rtpReceiverStats.videoParserStats.numLayeringChanges.toLong() + ) + val durationActiveVideo = transceiverStats.rtpReceiverStats.incomingStats.ssrcStats.values.filter { + it.mediaType == MediaType.VIDEO + }.sumOf { it.durationActive } + VideobridgeMetrics.totalVideoStreamMillisecondsReceived.add(durationActiveVideo.toMillis()) + + if (iceTransport.isConnected() && !dtlsTransport.isConnected) { + logger.info("Expiring a relay with ICE connected, but not DTLS.") + VideobridgeMetrics.relaysDtlsFailed.inc() + } + } + + fun expire() { + expired = true + logger.info("Expiring.") + synchronized(endpointsLock) { + relayedEndpoints.values.forEach { it.expire() } + } + senders.values.forEach { it.expire() } + conference.relayExpired(this) + + try { + updateStatsOnExpire() + transceiver.stop() + srtpTransformers?.close() + logger.cdebug { transceiver.debugState(DebugStateMode.FULL).toString() } + logger.cdebug { iceTransport.getDebugState().toString() } + logger.cdebug { dtlsTransport.getDebugState().toString() } + + transceiver.teardown() + messageTransport.close() + sctpHandler?.stop() + sctpTransport?.stop() + } catch (t: Throwable) { + logger.error("Exception while expiring: ", t) + } + + conference.encodingsManager.unsubscribe(this) + + dtlsTransport.stop() + iceTransport.stop() + outgoingSrtpPacketQueue.close() + + logger.info("Expired.") + } + + companion object { + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_srtp_send_queue_dropped_packets", + "Number of packets dropped out of the Relay SRTP send queue." + ) + + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_srtp_send_queue_exceptions", + "Number of exceptions from the Relay SRTP send queue." + ) + + /** Count the number of dropped packets and exceptions. */ + @JvmField + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } + + private const val SRTP_QUEUE_ENTRY_EVENT = "Entered Relay SRTP sender outgoing queue" + private const val SRTP_QUEUE_EXIT_EVENT = "Exited Relay SRTP sender outgoing queue" + } + + class Statistics { + val bytesReceived = AtomicLong(0) + val packetsReceived = AtomicLong(0) + val bytesSent = AtomicLong(0) + val packetsSent = AtomicLong(0) + + private fun getJson(): ObjectNode { + val jsonObject = JsonNodeFactory.instance.objectNode() + jsonObject.put("bytes_received", bytesReceived.get()) + jsonObject.put("bytes_sent", bytesSent.get()) + jsonObject.put("packets_received", packetsReceived.get()) + jsonObject.put("packets_sent", packetsSent.get()) + return jsonObject + } + } + + private inner class SctpCallbacks(transport: DcSctpTransport) : DcSctpBaseCallbacks(transport) { + override fun sendPacketWithStatus(packet: ByteArray): SendPacketStatus { + try { + val newBuf = ByteBufferPool.getBuffer(packet.size) + System.arraycopy(packet, 0, newBuf, 0, packet.size) + + sctpSendPcap.observe(newBuf, 0, packet.size) + dtlsTransport.sendDtlsData(newBuf, 0, packet.size) + + return SendPacketStatus.kSuccess + } catch (e: Throwable) { + logger.warn("Exception sending SCTP packet", e) + return SendPacketStatus.kError + } + } + + override fun OnMessageReceived(message: DcSctpMessage) { + try { + // We assume all data coming over SCTP will be datachannel data + val dataChannelPacket = DataChannelPacket(message) + // Post the rest of the task here because the current context is + // holding a lock inside the SctpSocket which can cause a deadlock + // if two endpoints are trying to send datachannel messages to one + // another (with stats broadcasting it can happen often) + incomingDataChannelMessagesQueue.add(PacketInfo(dataChannelPacket)) + } catch (e: Throwable) { + logger.warn("Exception processing SCTP message", e) + } + } + + override fun OnError(error: ErrorKind, message: String) { + logger.warn("SCTP error $error: $message") + } + + override fun OnAborted(error: ErrorKind, message: String) { + logger.info("SCTP aborted with error $error: $message") + } + + override fun OnConnected() { + try { + logger.info("SCTP connection is ready") + // This handles if the remote side will be opening the data channel + dataChannelStack.onDataChannelStackEvents { dataChannel -> + logger.info("Remote side opened a data channel.") + messageTransport.setDataChannel(dataChannel) + } + if (sctpRole == Sctp.Role.CLIENT) { + // This logic is for opening the data channel locally + logger.info("Will open the data channel.") + val dataChannel = dataChannelStack.createDataChannel( + DataChannelProtocolConstants.RELIABLE, + 0, + 0, + 0, + "default" + ) + messageTransport.setDataChannel(dataChannel) + dataChannel.open() + } else { + logger.info("Will wait for the remote side to open the data channel.") + } + } catch (e: Throwable) { + logger.warn("Exception processing SCTP connected event", e) + } + } + + override fun OnClosed() { + // I don't think this should happen, except during shutdown. + logger.info("SCTP connection closed") + } + } + + private inner class TransceiverEventHandlerImpl : TransceiverEventHandler { + /** + * Forward audio level events from the Transceiver to the conference. We use the same thread, because this fires + * for every packet and we want to avoid the switch. The conference audio level code must not block. + */ + override fun audioLevelReceived(sourceSsrc: Long, level: Long): Boolean { + /* We shouldn't receive audio levels from the local transceiver, since all media should be + * processed by the media endpoints. + */ + logger.warn { "Audio level reported by relay transceiver for source $sourceSsrc" } + return false + } + + /** + * Forward bwe events from the Transceiver. + */ + override fun bandwidthEstimationChanged(newValue: Bandwidth) { + logger.cdebug { "Estimated bandwidth is now $newValue" } + /* We don't use BWE for relay connections. */ + } + } + + interface IncomingRelayPacketHandler { + fun handleIncomingPacket(packetInfo: RelayedPacketInfo) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayConfig.kt new file mode 100644 index 0000000..df5a74f --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayConfig.kt @@ -0,0 +1,46 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.relay + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.optionalconfig + +class RelayConfig private constructor() { + val enabled: Boolean by config { + "videobridge.octo.enabled".from(JitsiConfig.newConfig) + "videobridge.relay.enabled".from(JitsiConfig.newConfig) + } + + val region: String? by optionalconfig { + "org.jitsi.videobridge.REGION".from(JitsiConfig.legacyConfig) + "videobridge.octo.region".from(JitsiConfig.newConfig) + "videobridge.relay.region".from(JitsiConfig.newConfig) + } + + val relayId: String by config { + "videobridge.octo.relay-id".from(JitsiConfig.newConfig) + "videobridge.relay.relay-id".from(JitsiConfig.newConfig) + // Fallback to the legacy "bind-address" property. + "videobridge.octo.bind-address".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = RelayConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt new file mode 100644 index 0000000..747d6a3 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayEndpointSender.kt @@ -0,0 +1,184 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.relay + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.Features +import org.jitsi.nlj.PacketHandler +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpSender +import org.jitsi.nlj.RtpSenderImpl +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtcp.RtcpListener +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.util.PacketInfoQueue +import org.jitsi.nlj.util.StreamInformationStore +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.videobridge.TransportConfig +import org.jitsi.videobridge.metrics.QueueMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.transport.ice.IceTransport +import org.jitsi.videobridge.util.TaskPools +import java.time.Instant + +/** + * An object that sends media from a single local endpoint to a single remote relay. + */ +class RelayEndpointSender( + val relay: Relay, + val id: String, + parentLogger: Logger, + diagnosticContext: DiagnosticContext +) { + private val logger = createChildLogger(parentLogger) + + private val streamInformationStore: StreamInformationStore = StreamInformationStoreImpl() + + val rtcpEventNotifier = RtcpEventNotifier().apply { + addRtcpEventListener( + object : RtcpListener { + override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?): Unit = + throw IllegalStateException("got rtcpPacketReceived callback from a sender") + override fun rtcpPacketSent(packet: RtcpPacket) { + relay.rtcpPacketSent(packet, id) + } + }, + external = true + ) + } + + private val rtpSender: RtpSender = RtpSenderImpl( + "${relay.id}-$id", + rtcpEventNotifier, + TaskPools.CPU_POOL, + TaskPools.SCHEDULED_POOL, + streamInformationStore, + logger, + diagnosticContext + ).apply { + onOutgoingPacket(object : PacketHandler { + override fun processPacket(packetInfo: PacketInfo) { + packetInfo.addEvent(SRTP_QUEUE_ENTRY_EVENT) + outgoingSrtpPacketQueue.add(packetInfo) + } + }) + } + + /** + * The queue we put outgoing SRTP packets onto so they can be sent + * out via the Relay's [IceTransport] on an IO thread. + */ + private val outgoingSrtpPacketQueue = PacketInfoQueue( + "${javaClass.simpleName}-outgoing-packet-queue", + TaskPools.IO_POOL, + { packet -> relay.doSendSrtp(packet) }, + TransportConfig.queueSize + ).apply { + setErrorHandler(queueErrorCounter) + } + + private var expired = false + + fun addPayloadType(payloadType: PayloadType) = streamInformationStore.addRtpPayloadType(payloadType) + fun addRtpExtension(rtpExtension: RtpExtension) = streamInformationStore.addRtpExtensionMapping(rtpExtension) + + fun setExtmapAllowMixed(allow: Boolean) = streamInformationStore.setExtmapAllowMixed(allow) + + fun setSrtpInformation(srtpTransformers: SrtpTransformers) { + rtpSender.setSrtpTransformers(srtpTransformers) + } + + fun sendPacket(packetInfo: PacketInfo) = rtpSender.processPacket(packetInfo) + + fun setFeature(feature: Features, enabled: Boolean) { + rtpSender.setFeature(feature, enabled) + } + + fun getOutgoingStats() = rtpSender.getPacketStreamStats() + + fun getDebugState(mode: DebugStateMode): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("expired", expired) + put("id", id) + set("sender", rtpSender.debugState(mode)) + set("stream_information_store", streamInformationStore.debugState(mode)) + } + + fun expire() { + if (expired) { + return + } + expired = true + + try { + updateStatsOnExpire() + rtpSender.stop() + logger.cdebug { getDebugState(DebugStateMode.FULL).toString() } + rtpSender.tearDown() + } catch (t: Throwable) { + logger.error("Exception while expiring: ", t) + } + outgoingSrtpPacketQueue.close() + } + + private fun updateStatsOnExpire() { + val relayStats = relay.statistics + val outgoingStats = rtpSender.getPacketStreamStats() + + relayStats.apply { + bytesSent.getAndAdd(outgoingStats.bytes) + packetsSent.getAndAdd(outgoingStats.packets) + } + } + + companion object { + private val droppedPacketsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_endpoint_srtp_send_queue_dropped_packets", + "Number of packets dropped out of the Relay SRTP send queue." + ) + + private val exceptionsMetric = VideobridgeMetricsContainer.instance.registerCounter( + "relay_endpoint_srtp_send_queue_exceptions", + "Number of exceptions from the Relay SRTP send queue." + ) + + /** Count the number of dropped packets and exceptions. */ + @JvmField + val queueErrorCounter = object : CountingErrorHandler() { + override fun packetDropped() = super.packetDropped().also { + droppedPacketsMetric.inc() + QueueMetrics.droppedPackets.inc() + } + + override fun packetHandlingFailed(t: Throwable?) = super.packetHandlingFailed(t).also { + exceptionsMetric.inc() + QueueMetrics.exceptions.inc() + } + } + + private const val SRTP_QUEUE_ENTRY_EVENT = "Entered RelayEndpointSender SRTP sender outgoing queue" + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt new file mode 100644 index 0000000..218cfce --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayMessageTransport.kt @@ -0,0 +1,523 @@ +/* + * Copyright @ 2017 Atlassian Pty Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.relay + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.eclipse.jetty.websocket.api.Callback +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest +import org.eclipse.jetty.websocket.client.WebSocketClient +import org.eclipse.jetty.websocket.core.CloseStatus +import org.jitsi.utils.logging2.Logger +import org.jitsi.videobridge.AbstractEndpointMessageTransport +import org.jitsi.videobridge.VersionConfig +import org.jitsi.videobridge.datachannel.DataChannel +import org.jitsi.videobridge.datachannel.DataChannelStack.DataChannelMessageListener +import org.jitsi.videobridge.datachannel.protocol.DataChannelMessage +import org.jitsi.videobridge.datachannel.protocol.DataChannelStringMessage +import org.jitsi.videobridge.message.AddReceiverMessage +import org.jitsi.videobridge.message.BridgeChannelMessage +import org.jitsi.videobridge.message.ClientHelloMessage +import org.jitsi.videobridge.message.EndpointConnectionStatusMessage +import org.jitsi.videobridge.message.EndpointMessage +import org.jitsi.videobridge.message.EndpointStats +import org.jitsi.videobridge.message.ServerHelloMessage +import org.jitsi.videobridge.message.SourceVideoTypeMessage +import org.jitsi.videobridge.message.SyntheticSourceSendingChangeEvent +import org.jitsi.videobridge.message.VideoTypeMessage +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.websocket.ColibriWebSocket +import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig +import java.lang.ref.WeakReference +import java.net.URI +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong + +/** + * Handles the functionality related to sending and receiving COLIBRI messages + * for a [Relay]. Supports two underlying transport mechanisms -- + * WebRTC data channels and {@code WebSocket}s. + */ +class RelayMessageTransport( + private val relay: Relay, + private val eventHandler: EndpointMessageTransportEventHandler, + parentLogger: Logger +) : AbstractEndpointMessageTransport(parentLogger), ColibriWebSocket.EventHandler, DataChannelMessageListener { + /** + * The last connected/accepted web-socket by this instance, if any. + */ + private var webSocket: ColibriWebSocket? = null + + /** + * For active websockets, the URL that was connected to. + */ + private var url: String? = null + + /** + * Use to synchronize access to [webSocket] + */ + private val webSocketSyncRoot = Any() + + /** + * Whether the last active transport channel (i.e. the last to receive a + * message from the remote endpoint) was the web socket (if `true`), + * or the WebRTC data channel (if `false`). + */ + private var webSocketLastActive = false + + private var dataChannel = WeakReference(null) + + private val numOutgoingMessagesDropped = AtomicInteger(0) + + /** + * The number of sent message by type. + */ + private val sentMessagesCounts: MutableMap = ConcurrentHashMap() + + /** + * Connect the bridge channel message to the websocket URL specified + */ + fun connectToWebsocket(url: String) { + if (this.url != null && this.url == url) { + return + } + this.url = url + + doConnect() + } + + private fun doConnect() { + val url = this.url ?: throw IllegalStateException("Cannot connect Relay transport when no URL set") + + webSocket?.let { + logger.warn("Re-connecting while webSocket != null, possible leak.") + webSocket = null + } + + ColibriWebSocket(relay.id, this).also { + webSocketClient.connect(it, URI(url), ClientUpgradeRequest()) + synchronized(webSocketSyncRoot) { + webSocket = it + } + } + } + + /** + * {@inheritDoc} + */ + override fun notifyTransportChannelConnected() { + relay.relayMessageTransportConnected() + } + + /** + * {@inheritDoc} + */ + override fun clientHello(message: ClientHelloMessage): BridgeChannelMessage? { + // ClientHello was introduced for functional testing purposes. It + // triggers a ServerHello response from Videobridge. The exchange + // reveals (to the peer) that the transport channel between the + // remote relay and the Videobridge is operational. + // We take care to send the reply using the same transport channel on + // which we received the request.. + return createServerHello() + } + + override fun serverHello(message: ServerHelloMessage): BridgeChannelMessage? { + if (message.version?.equals(relay.conference.videobridge.version.toString()) != true) { + logger.warn { + "Received ServerHelloMessage with version ${message.version}, but " + + "this server is version ${relay.conference.videobridge.version}" + } + } else { + logger.info { "Received ServerHelloMessage, version ${message.version}" } + } + return null + } + + /** + * This message indicates that a remote bridge wishes to receive video + * with certain constraints for a specific endpoint. + * @param message + * @return + */ + override fun addReceiver(message: AddReceiverMessage): BridgeChannelMessage? { + val sourceName = message.sourceName + val ep = relay.conference.findSourceOwner(sourceName) ?: run { + logger.warn("Received AddReceiverMessage for unknown or non-local: $sourceName") + return null + } + + ep.addReceiver(relay.id, sourceName, message.videoConstraints) + return null + } + + override fun videoType(message: VideoTypeMessage): BridgeChannelMessage? { + logger.error("Relay: unexpected video type message: ${message.toJson()}") + return null + } + + override fun sourceVideoType(message: SourceVideoTypeMessage): BridgeChannelMessage? { + val epId = message.endpointId + if (epId == null) { + logger.warn("Received SourceVideoTypeMessage over relay channel with no endpoint ID") + return null + } + + val ep = relay.getEndpoint(epId) + + if (ep == null) { + logger.warn("Received SourceVideoTypeMessage for unknown epId $epId") + return null + } + + ep.setVideoType(message.sourceName, message.videoType) + + relay.conference.sendMessageFromRelay(message, false, relay.meshId) + + return null + } + + override fun unhandledMessage(message: BridgeChannelMessage) { + logger.warn("Received a message with an unexpected type: ${message.javaClass.simpleName}") + } + + /** + * Sends a string via a particular transport channel. + * @param dst the transport channel. + * @param message the message to send. + */ + override fun sendMessage(dst: Any?, message: BridgeChannelMessage) { + super.sendMessage(dst, message) // Log message + if (dst is ColibriWebSocket) { + sendMessage(dst, message) + } else if (dst is DataChannel) { + sendMessage(dst, message) + } else { + throw IllegalArgumentException("unknown transport:$dst") + } + } + + /** + * Sends a string via a particular [DataChannel]. + * @param dst the data channel to send through. + * @param message the message to send. + */ + private fun sendMessage(dst: DataChannel, message: BridgeChannelMessage) { + dst.sendString(message.toJson()) + VideobridgeMetrics.dataChannelMessagesSent.inc() + } + + /** + * Sends a string via a particular [ColibriWebSocket] instance. + * @param dst the [ColibriWebSocket] through which to send the message. + * @param message the message to send. + */ + private fun sendMessage(dst: ColibriWebSocket, message: BridgeChannelMessage) { + dst.sendString(message.toJson()) + VideobridgeMetrics.colibriWebSocketMessagesSent.inc() + } + + override fun onDataChannelMessage(dataChannelMessage: DataChannelMessage?) { + webSocketLastActive = false + VideobridgeMetrics.dataChannelMessagesReceived.inc() + if (dataChannelMessage is DataChannelStringMessage) { + onMessage(dataChannel.get(), dataChannelMessage.data) + } + } + + /** + * {@inheritDoc} + */ + public override fun sendMessage(msg: BridgeChannelMessage) { + val dst = getActiveTransportChannel() + if (dst == null) { + logger.debug("No available transport channel, can't send a message") + numOutgoingMessagesDropped.incrementAndGet() + } else { + sentMessagesCounts.computeIfAbsent(msg.javaClass.simpleName) { AtomicLong() }.incrementAndGet() + sendMessage(dst, msg) + } + } + + /** + * @return the active transport channel for this + * [RelayMessageTransport] (either the [.webSocket], or + * the WebRTC data channel represented by a [DataChannel]). + * + * The "active" channel is determined based on what channels are available, + * and which one was the last to receive data. That is, if only one channel + * is available, it will be returned. If two channels are available, the + * last one to have received data will be returned. Otherwise, `null` + * will be returned. + */ + // TODO(brian): seems like it'd be nice to have the websocket and datachannel + // share a common parent class (or, at least, have a class that is returned + // here and provides a common API but can wrap either a websocket or + // datachannel) + private fun getActiveTransportChannel(): Any? { + val dataChannel = dataChannel.get() + val webSocket = webSocket + var dst: Any? = null + if (webSocketLastActive) { + dst = webSocket + } + + // Either the socket was not the last active channel, + // or it has been closed. + if (dst == null) { + if (dataChannel != null && dataChannel.isReady) { + dst = dataChannel + } + } + + // Maybe the WebRTC data channel is the last active, but it is not + // currently available. If so, and a web-socket is available -- use it. + if (dst == null && webSocket != null) { + dst = webSocket + } + return dst + } + + override val isConnected: Boolean + get() = getActiveTransportChannel() != null + + val isActive: Boolean + get() = url != null + + /** + * {@inheritDoc} + */ + override fun webSocketConnected(ws: ColibriWebSocket) { + synchronized(webSocketSyncRoot) { + // If we already have a web-socket, discard it and use the new one. + if (ws != webSocket) { + if (webSocket != null) { + logger.info("Replacing an existing websocket.") + webSocket?.session?.close(CloseStatus.NORMAL, "replaced", Callback.NOOP) + } + webSocketLastActive = true + webSocket = ws + sendMessage(ws, createServerHello()) + } else { + logger.warn("Websocket already connected.") + } + } + try { + notifyTransportChannelConnected() + } catch (e: Exception) { + logger.warn("Caught an exception in notifyTransportConnected", e) + } + } + + private fun createServerHello(): ServerHelloMessage = if (VersionConfig.config.announceVersion()) { + ServerHelloMessage(relay.conference.videobridge.version.toString()) + } else { + ServerHelloMessage() + } + + /** + * {@inheritDoc} + */ + override fun webSocketClosed(ws: ColibriWebSocket, statusCode: Int, reason: String) { + synchronized(webSocketSyncRoot) { + if (ws == webSocket) { + webSocket = null + webSocketLastActive = false + logger.debug { "Web socket closed, statusCode $statusCode ( $reason)." } + // 1000 is normal, 1001 is e.g. a tab closing. 1005 is "No Status Rcvd" and we see the majority of + // sockets close this way. + if (statusCode == 1000 || statusCode == 1001 || statusCode == 1005) { + VideobridgeMetrics.colibriWebSocketCloseNormal.inc() + } else { + VideobridgeMetrics.colibriWebSocketCloseAbnormal.inc() + } + } + } + + // This check avoids trying to establish a new WS when the closing of the existing WS races the signaling to + // expire the relay. 1001 with RELAY_CLOSED means that the remote side willingly closed the socket. + if (statusCode != 1001 || reason != RELAY_CLOSED) { + doConnect() + } + } + + override fun webSocketError(ws: ColibriWebSocket, cause: Throwable) { + logger.error("Colibri websocket error: ${cause.message}") + VideobridgeMetrics.colibriWebSocketErrors.inc() + } + + /** + * {@inheritDoc} + */ + override fun close() { + synchronized(webSocketSyncRoot) { + if (webSocket != null) { + // 410 Gone indicates that the resource requested is no longer + // available and will not be available again. + webSocket?.session?.close(CloseStatus.SHUTDOWN, RELAY_CLOSED, Callback.NOOP) + webSocket = null + logger.debug { "Relay expired, closed colibri web-socket." } + } + } + } + + /** + * {@inheritDoc} + */ + override fun webSocketTextReceived(ws: ColibriWebSocket, message: String) { + if (ws != webSocket) { + logger.warn("Received text from an unknown web socket.") + return + } + VideobridgeMetrics.colibriWebSocketMessagesReceived.inc() + webSocketLastActive = true + onMessage(ws, message) + } + + /** + * Sets the data channel for this endpoint. + * @param dataChannel the [DataChannel] to use for this transport + */ + fun setDataChannel(dataChannel: DataChannel) { + val prevDataChannel = this.dataChannel.get() + if (prevDataChannel == null) { + this.dataChannel = WeakReference(dataChannel) + // We install the handler first, otherwise the 'ready' might fire after we check it but before we + // install the handler + dataChannel.onDataChannelEvents { notifyTransportChannelConnected() } + if (dataChannel.isReady) { + notifyTransportChannelConnected() + } + dataChannel.onDataChannelMessage(this) + } else if (prevDataChannel === dataChannel) { + // TODO: i think we should be able to ensure this doesn't happen, + // so throwing for now. if there's a good + // reason for this, we can make this a no-op + throw Error("Re-setting the same data channel") + } else { + throw Error("Overwriting a previous data channel!") + } + } + + override val debugState: ObjectNode + get() { + val debugState = super.debugState + debugState.put("numOutgoingMessagesDropped", numOutgoingMessagesDropped.get()) + val sentCounts = JsonNodeFactory.instance.objectNode() + sentMessagesCounts.forEach { (k, v) -> sentCounts.put(k, v.get()) } + debugState.set("sent_counts", sentCounts) + return debugState + } + + /** + * Handles an opaque message received on the Relay channel. The message originates from an endpoint with an ID of + * `message.getFrom`, as verified by the remote bridge sending the message. + * + * @param message the message that was received from the endpoint. + */ + override fun endpointMessage(message: EndpointMessage): BridgeChannelMessage? { + // We trust the "from" field, because it comes from another bridge, not an endpoint. + val conference = relay.conference + if (conference.isExpired) { + logger.warn("Unable to send EndpointMessage, conference is expired") + return null + } + if (message.isBroadcast()) { + conference.sendMessageFromRelay(message, true, relay.meshId) + } else { + // 1:1 message + val to = message.to + val targetEndpoint = conference.getLocalEndpoint(to) + if (targetEndpoint == null) { + logger.warn("Unable to find endpoint to send EndpointMessage to: $to") + return null + } + + conference.sendMessage( + message, + listOf(targetEndpoint), + // sendToRelays + false + ) + } + return null + } + + /** + * Handles an endpoint statistics message on the Relay channel that should be forwarded to + * local endpoints as appropriate. + * + * @param message the message that was received from the endpoint. + */ + override fun endpointStats(message: EndpointStats): BridgeChannelMessage? { + // We trust the "from" field, because it comes from another bridge, not an endpoint. + val conference = relay.conference + if (conference.isExpired) { + logger.warn("Unable to send EndpointStats, conference is null or expired") + return null + } + if (message.from == null) { + logger.warn("Unable to send EndpointStats, missing from") + return null + } + val from = conference.getEndpoint(message.from!!) + if (from == null) { + logger.warn("Unable to send EndpointStats, unknown endpoint " + message.from) + return null + } + conference.localEndpoints.filter { it.wantsStatsFrom(from) }.forEach { it.sendMessage(message) } + conference.relays.filter { it.meshId != relay.meshId }.forEach { it.sendMessage(message) } + return null + } + + override fun endpointConnectionStatus(message: EndpointConnectionStatusMessage): BridgeChannelMessage? { + val conference = relay.conference + if (conference.isExpired) { + logger.warn("Unable to send EndpointConnectionStatusMessage, conference is expired") + return null + } + conference.sendMessageFromRelay(message, true, relay.meshId) + return null + } + + override fun syntheticSourceSendingChange(message: SyntheticSourceSendingChangeEvent): BridgeChannelMessage? { + // The synthetic source's audio is forwarded across relays, so its sending-change notification is too: + // forward to local endpoints and on to other-mesh relays, like endpointConnectionStatus above. + val conference = relay.conference + if (conference.isExpired) { + logger.warn("Unable to send SyntheticSourceSendingChangeEvent, conference is expired") + return null + } + conference.sendMessageFromRelay(message, true, relay.meshId) + return null + } + + companion object { + /** + * The single [WebSocketClient] instance that all [Relay]s use to initiate a web socket connection. + */ + val webSocketClient = WebSocketClient().apply { + idleTimeout = WebsocketServiceConfig.config.idleTimeout + start() + } + + /** + * Reason to use when closing a WS due to the relay being expired. + */ + const val RELAY_CLOSED = "relay_closed" + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt new file mode 100644 index 0000000..31a0803 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/relay/RelayedEndpoint.kt @@ -0,0 +1,260 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.relay + +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.DebugStateMode +import org.jitsi.nlj.Features +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.MediaSources +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpReceiverEventHandler +import org.jitsi.nlj.RtpReceiverImpl +import org.jitsi.nlj.SetLocalSsrcEvent +import org.jitsi.nlj.SetMediaSourcesEvent +import org.jitsi.nlj.copy +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.rtcp.RtcpEventNotifier +import org.jitsi.nlj.rtcp.RtcpListener +import org.jitsi.nlj.rtp.RtpExtension +import org.jitsi.nlj.srtp.SrtpTransformers +import org.jitsi.nlj.transform.node.ConsumerNode +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.StreamInformationStore +import org.jitsi.nlj.util.StreamInformationStoreImpl +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.utils.MediaType +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.videobridge.AbstractEndpoint +import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.cc.allocation.VideoConstraints +import org.jitsi.videobridge.message.AddReceiverMessage +import org.jitsi.videobridge.util.TaskPools +import java.time.Instant + +/** + * An object that handles media received from a single remote endpoint to a relay. + */ +class RelayedEndpoint( + conference: Conference, + val relay: Relay, + id: String, + parentLogger: Logger, + diagnosticContext: DiagnosticContext +) : AbstractEndpoint(conference, id, parentLogger), Relay.IncomingRelayPacketHandler { + override var audioSources: List = listOf() + set(value) { + val oldValue = field + field = value + audioSourcesChanged(value) + // Register relayed sources with the conference's audio subscription manager, so local clients can + // subscribe to them (Include/Exclude resolve their names) and relayed synthetic sources are suppressed + // for "All" subscriptions, the same as local sources. + conference.removeAudioSources(oldValue.filterNot { value.contains(it) }.toSet()) + conference.addAudioSources(value.filterNot { oldValue.contains(it) }.toSet()) + value.forEach { + streamInformationStore.addReceiveSsrc(it.ssrc, MediaType.AUDIO) + conference.addEndpointSsrc(this, it.ssrc) + } + } + + private val streamInformationStore: StreamInformationStore = StreamInformationStoreImpl() + + val rtcpEventNotifier = RtcpEventNotifier().apply { + addRtcpEventListener( + object : RtcpListener { + override fun rtcpPacketReceived(packet: RtcpPacket, receivedTime: Instant?) { + relay.rtcpPacketReceived(packet, receivedTime, id) + } + override fun rtcpPacketSent(packet: RtcpPacket): Unit = + throw IllegalStateException("got rtcpPacketSent callback from a receiver") + }, + external = true + ) + } + + private val rtpReceiver = RtpReceiverImpl( + "${relay.id}-$id", + { rtcpPacket -> + if (rtcpPacket.length >= 1500) { + logger.warn( + "Sending large locally-generated RTCP packet of size ${rtcpPacket.length}, " + + "first packet of type ${rtcpPacket.packetType} rc ${rtcpPacket.reportCount}." + ) + } + /* TODO */ + relay.transceiver.sendPacket(PacketInfo(rtcpPacket)) + }, + rtcpEventNotifier, + TaskPools.CPU_POOL, + TaskPools.SCHEDULED_POOL, + streamInformationStore, + RtpReceiverEventHandlerImpl(), + logger, + diagnosticContext + ).apply { + packetHandler = object : ConsumerNode("receiver chain handler") { + override fun consume(packetInfo: PacketInfo) { + packetInfo.endpointId = id + conference.handleIncomingPacket(packetInfo) + } + + override fun trace(f: () -> Unit) = f.invoke() + } + handleEvent(SetLocalSsrcEvent(MediaType.AUDIO, conference.localAudioSsrc)) + handleEvent(SetLocalSsrcEvent(MediaType.VIDEO, conference.localVideoSsrc)) + } + + override fun receivesSsrc(ssrc: Long): Boolean = streamInformationStore.receiveSsrcs.contains(ssrc) + + override val ssrcs + get() = HashSet(streamInformationStore.receiveSsrcs) + + // Visitors are never advertised between relays, so relayed endpoints are never visitors. + override val visitor = false + + /** Relayed endpoints are not automatically expired. **/ + override fun shouldExpire(): Boolean = false + + override fun requestKeyframe(requesterID: String, mediaSsrc: Long) = + relay.transceiver.requestKeyFrame(requesterID, mediaSsrc) + + override fun requestKeyframe() = relay.transceiver.requestKeyFrame(null, mediaSource?.primarySSRC) + + override val isSendingAudio + get() = rtpReceiver.isReceivingAudio() + override val isSendingVideo + get() = rtpReceiver.isReceivingVideo() + + override fun addPayloadType(payloadType: PayloadType) = streamInformationStore.addRtpPayloadType(payloadType) + override fun addRtpExtension(rtpExtension: RtpExtension) = + streamInformationStore.addRtpExtensionMapping(rtpExtension) + + override fun setExtmapAllowMixed(allow: Boolean) = streamInformationStore.setExtmapAllowMixed(allow) + + override fun sendVideoConstraints(sourceName: String, maxVideoConstraints: VideoConstraints) { + relay.sendMessage( + AddReceiverMessage( + RelayConfig.config.relayId, + sourceName, + maxVideoConstraints + ) + ) + } + + fun relayMessageTransportConnected() { + maxReceiverVideoConstraints.forEach { (sourceName, constraints) -> + sendVideoConstraints(sourceName, constraints) + } + } + + private val _mediaSources = MediaSources() + + override var mediaSources: Array + get() = _mediaSources.getMediaSources() + set(value) { + applyVideoTypeCache(value) + val changed = _mediaSources.setMediaSources(value) + val mergedMediaSources = _mediaSources.getMediaSources() + val signaledMediaSources = value.copy() + if (changed) { + val setMediaSourcesEvent = SetMediaSourcesEvent(mergedMediaSources, signaledMediaSources) + + rtpReceiver.handleEvent(setMediaSourcesEvent) + mediaSources.forEach { + it.rtpEncodings.forEach { + it.ssrcs.forEach { + streamInformationStore.addReceiveSsrc(it, MediaType.VIDEO) + conference.addEndpointSsrc(this, it) + } + } + } + } + } + + fun setSrtpInformation(srtpTransformers: SrtpTransformers) { + rtpReceiver.setSrtpTransformers(srtpTransformers) + } + + override fun handleIncomingPacket(packetInfo: RelayedPacketInfo) = rtpReceiver.enqueuePacket(packetInfo) + + fun setFeature(feature: Features, enabled: Boolean) { + rtpReceiver.setFeature(feature, enabled) + } + + fun getIncomingStats() = rtpReceiver.getStats().packetStreamStats + + override fun debugState(mode: DebugStateMode): ObjectNode = super.debugState(mode).apply { + if (mode == DebugStateMode.FULL) { + set("stream_information_store", streamInformationStore.debugState(mode)) + set("receiver", rtpReceiver.debugState(mode)) + set("media_sources", _mediaSources.debugState()) + } + } + + private fun updateStatsOnExpire() { + val relayStats = relay.statistics + val rtpReceiverStats = rtpReceiver.getStats() + + // Add stats from the local transceiver + val incomingStats = rtpReceiverStats.packetStreamStats + + relayStats.apply { + bytesReceived.getAndAdd(incomingStats.bytes) + packetsReceived.getAndAdd(incomingStats.packets) + } + } + + override fun expire() { + if (super.isExpired) { + return + } + super.expire() + + // Remove our sources from the conference's audio subscription manager. + conference.removeAudioSources(audioSources.toSet()) + + try { + updateStatsOnExpire() + rtpReceiver.stop() + logger.cdebug { debugState(DebugStateMode.FULL).toString() } + rtpReceiver.tearDown() + } catch (t: Throwable) { + logger.error("Exception while expiring: ", t) + } + + logger.info("Expired.") + } + + private inner class RtpReceiverEventHandlerImpl : RtpReceiverEventHandler { + /** + * Forward audio level events from the Transceiver to the conference. We use the same thread, because this fires + * for every packet and we want to avoid the switch. The conference audio level code must not block. + */ + override fun audioLevelReceived(sourceSsrc: Long, level: Long): Boolean = + conference.levelChanged(this@RelayedEndpoint, sourceSsrc, level) + + /** + * Forward bwe events from the Transceiver. + */ + override fun bandwidthEstimationChanged(newValue: Bandwidth) { + logger.cdebug { "Estimated bandwidth is now $newValue" } + /* We don't use BWE for relay connections. */ + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestApis.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestApis.kt new file mode 100644 index 0000000..65c03a5 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestApis.kt @@ -0,0 +1,27 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.rest + +enum class RestApis { + COLIBRI, + DEBUG, + DRAIN, + HEALTH, + PROMETHEUS, + RTCSTATS, + SHUTDOWN, + VERSION +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt new file mode 100644 index 0000000..4c87f8f --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/RestConfig.kt @@ -0,0 +1,116 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class RestConfig private constructor() { + private val colibriRestEnabled: Boolean by config { + // If the value was passed via a command line arg, we set it as a system + // variable at this path, which the new config will pick up + "videobridge.apis.rest.enabled".from(JitsiConfig.newConfig) + } + private val legacyColibriRestEnabled: Boolean by config { + "org.jitsi.videobridge.ENABLE_REST_COLIBRI".from(JitsiConfig.legacyConfig) + "default" { true } + } + + /** + * Due to historical reasons the COLIBRI REST API is controlled in multiple ways. There is: + * 1. A command line argument (--apis=rest) + * 2. A new config property (videobridge.apis.rest.enabled) + * 3. A legacy config property ENABLE_REST_COLIBRI + * + * The last one defaults to `true` so it is simply an old way to force the API to be disabled. It is implemented + * in [legacyColibriRestEnabled]. The first two are implemented in [colibriRestEnabled], and this property + * implements the combined logic. + */ + private val colibriEnabled + get() = colibriRestEnabled && legacyColibriRestEnabled + + /** + * The debug API (query conference state, enable/disable debug features). It currently defaults to being enabled, + * but that might change in the future. + */ + private val debugEnabled: Boolean by config { + "videobridge.rest.debug.enabled".from(JitsiConfig.newConfig) + } + + /** + * The health-check API. + */ + private val healthEnabled: Boolean by config { + "videobridge.rest.health.enabled".from(JitsiConfig.newConfig) + } + + /** + * The property which enables/disables the graceful shutdown API. + */ + private val shutdownEnabledProperty: Boolean by config { + "org.jitsi.videobridge.ENABLE_REST_SHUTDOWN".from(JitsiConfig.legacyConfig) + "videobridge.rest.shutdown.enabled".from(JitsiConfig.newConfig) + } + + /** + * Due to historical reasons the shutdown API is only enabled when the COLIBRI API is enabled. + */ + private val shutdownEnabled + get() = colibriEnabled && shutdownEnabledProperty + + /** + * The drain API. + */ + private val drainEnabled: Boolean by config { + "videobridge.rest.drain.enabled".from(JitsiConfig.newConfig) + } + + private val rtcstatsEnabled: Boolean by config { + "videobridge.rest.rtcstats.enabled".from(JitsiConfig.newConfig) + } + + private val versionEnabled: Boolean by config { + "videobridge.rest.version.enabled".from(JitsiConfig.newConfig) + } + + private val prometheusEnabled: Boolean by config { + "videobridge.rest.prometheus.enabled".from(JitsiConfig.newConfig) + } + + /** + * Whether any of the REST APIs are enabled by the configuration. If there aren't, the HTTP server doesn't need to + * be started at all. + */ + fun isEnabled() = colibriEnabled || debugEnabled || healthEnabled || + shutdownEnabled || drainEnabled || versionEnabled || prometheusEnabled + + fun isEnabled(api: RestApis) = when (api) { + RestApis.COLIBRI -> colibriEnabled + RestApis.DEBUG -> debugEnabled + RestApis.DRAIN -> drainEnabled + RestApis.HEALTH -> healthEnabled + RestApis.PROMETHEUS -> prometheusEnabled + RestApis.RTCSTATS -> rtcstatsEnabled + RestApis.SHUTDOWN -> shutdownEnabled + RestApis.VERSION -> versionEnabled + } + + companion object { + @JvmField + val config = RestConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt new file mode 100644 index 0000000..6a41558 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/rest/binders/ServiceBinder.kt @@ -0,0 +1,34 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.binders + +import org.glassfish.hk2.utilities.binding.AbstractBinder +import org.jitsi.health.HealthCheckService +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.xmpp.XmppConnection + +class ServiceBinder( + private val videobridge: Videobridge, + private val xmppConnection: XmppConnection, + private val healthChecker: HealthCheckService +) : AbstractBinder() { + override fun configure() { + bind(videobridge).to(Videobridge::class.java) + bind(xmppConnection).to(XmppConnection::class.java) + bind(healthChecker).to(HealthCheckService::class.java) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt new file mode 100644 index 0000000..3bab072 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/sctp/SctpConfig.kt @@ -0,0 +1,33 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.sctp + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config + +class SctpConfig private constructor() { + /** Whether SCTP should be signaled or used when signaled to us */ + val enabled: Boolean by config { "videobridge.sctp.enabled".from(JitsiConfig.newConfig) } + fun enabled() = enabled + + val maxChannels: Int by config { "videobridge.sctp.max-channels".from(JitsiConfig.newConfig) } + + companion object { + @JvmField + val config = SctpConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownConfig.kt new file mode 100644 index 0000000..1b427e5 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownConfig.kt @@ -0,0 +1,40 @@ +/* + * Copyright @ 2022 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.shutdown + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import java.time.Duration + +class ShutdownConfig { + private constructor() + + val gracefulShutdownMaxDuration: Duration by config { + "videobridge.shutdown.graceful-shutdown-max-duration".from(JitsiConfig.newConfig) + } + val gracefulShutdownMinParticipants: Int by config { + "videobridge.shutdown.graceful-shutdown-min-participants".from(JitsiConfig.newConfig) + } + val shuttingDownDelay: Duration by config { + "videobridge.shutdown.shutting-down-delay".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = ShutdownConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt new file mode 100644 index 0000000..1ff6601 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownManager.kt @@ -0,0 +1,109 @@ +/* + * Copyright @ 2022 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.shutdown + +import org.jitsi.meet.ShutdownService +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.shutdown.ShutdownConfig.Companion.config +import org.jitsi.videobridge.shutdown.ShutdownState.GRACEFUL_SHUTDOWN +import org.jitsi.videobridge.shutdown.ShutdownState.RUNNING +import org.jitsi.videobridge.shutdown.ShutdownState.SHUTTING_DOWN +import org.jitsi.videobridge.util.TaskPools +import java.util.concurrent.TimeUnit + +/** + * Implements the graceful shutdown logic. + * + * When graceful shutdown is initiated the state changes to [GRACEFUL_SHUTDOWN], and this is advertised in presence. + * In this mode new conferences are rejected, but existing conferences continue to run and accept new participants. We + * stay in this state until either [config.gracefulShutdownMaxDuration] has passed, or the number of participants + * drops below [config.gracefulShutdownMinParticipants]. + * + * Then the state transitions to [SHUTTING_DOWN], and this state is advertised in presence. We expect jicofo to + * actively move participants off this bridge. We stay in [SHUTTING_DOWN] for [config.shuttingDownDelay] before we + * actually perform the shutdown. + */ +class ShutdownManager( + private val shutdownService: ShutdownService, + parentLogger: Logger +) { + val logger = createChildLogger(parentLogger) + + var state: ShutdownState = RUNNING + private set + + fun initiateShutdown(graceful: Boolean) { + logger.info("Received shutdown request, graceful=$graceful") + + if (graceful) { + if (state == RUNNING) { + state = GRACEFUL_SHUTDOWN + VideobridgeMetrics.gracefulShutdown.set(true) + logger.info( + "Entered graceful shutdown mode, will stay in this mode for up to " + + config.gracefulShutdownMaxDuration + ) + TaskPools.SCHEDULED_POOL.schedule( + { + if (state == GRACEFUL_SHUTDOWN) { + logger.info("Graceful shutdown period expired, changing to SHUTTING_DOWN.") + changeToShuttingDown() + } + }, + config.gracefulShutdownMaxDuration.toMillis(), + TimeUnit.MILLISECONDS + ) + } else { + logger.info("Graceful shutdown already initiated mode.") + } + } else { + changeToShuttingDown() + } + } + + /** + * Change to SHUTTING_DOWN if the number of endpoints has dropped below the threshold. + */ + fun maybeShutdown(endpointCount: Long) { + // If we're RUNNING we don't want to shutdown. If we're SHUTTING_DOWN we've already done this. + if (state == GRACEFUL_SHUTDOWN) { + if (endpointCount <= config.gracefulShutdownMinParticipants) { + logger.info("Entering SHUTTING_DOWN with $endpointCount participants.") + changeToShuttingDown() + } + } + } + + private fun changeToShuttingDown() { + if (state == SHUTTING_DOWN) { + return + } + + logger.info("Will shut down in ${config.shuttingDownDelay}") + state = SHUTTING_DOWN + VideobridgeMetrics.shuttingDown.set(true) + TaskPools.SCHEDULED_POOL.schedule( + { + logger.info("Videobridge is shutting down NOW") + shutdownService.beginShutdown() + }, + config.shuttingDownDelay.toMillis(), + TimeUnit.MILLISECONDS + ) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownState.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownState.kt new file mode 100644 index 0000000..f82fb3c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/shutdown/ShutdownState.kt @@ -0,0 +1,22 @@ +/* + * Copyright @ 2022 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.shutdown + +enum class ShutdownState { + RUNNING, + GRACEFUL_SHUTDOWN, + SHUTTING_DOWN +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt new file mode 100644 index 0000000..cd062a8 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/ConferencePacketStats.kt @@ -0,0 +1,83 @@ +/* + * Copyright @ 2023 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.videobridge.metrics.Metrics +import java.util.concurrent.atomic.AtomicLong + +/** + * Saves statistics for packets and bytes based on conference size. + * + * We assume the stats are updated periodically at an interval of [periodSeconds] seconds. This way we can convert + * the rates passed to [addValue] to number of packets and bytes. + */ +class ConferencePacketStats private constructor() { + private val periodSeconds = Metrics.interval.toMillis().toDouble() / 1000 + + /** Maps a conference size to a [Stats] instance that sums the added packet and bit rates. */ + private val stats: Map = mutableMapOf().apply { + (0..MAX_SIZE).forEach { + this[it] = Stats() + } + } + + val totalPackets = AtomicLong() + val totalBytes = AtomicLong() + + fun addValue(conferenceSize: Int, packetRatePps: Long, bitrateBps: Long) { + stats[conferenceSize.coerceAtMost(MAX_SIZE)]?.let { + val packets = (packetRatePps * periodSeconds).toLong() + val bytes = (bitrateBps * periodSeconds / 8).toLong() + + it.packets.addAndGet(packets) + it.bytes.addAndGet(bytes) + totalPackets.addAndGet(packets) + totalBytes.addAndGet(bytes) + } + } + + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + val packetRates: ArrayNode = JsonNodeFactory.instance.arrayNode() + val bitrates: ArrayNode = JsonNodeFactory.instance.arrayNode() + + (0..MAX_SIZE).forEach { conferenceSize -> + stats[conferenceSize]?.let { + packetRates.add(it.packets.get()) + bitrates.add(it.bytes.get()) + } + } + + set("packets", packetRates) + set("bytes", bitrates) + put("total_packets", totalPackets.get()) + put("total_bytes", totalBytes.get()) + } + + companion object { + const val MAX_SIZE = 500 + + @JvmField + val stats = ConferencePacketStats() + } + + private class Stats { + val packets = AtomicLong() + val bytes = AtomicLong() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt new file mode 100644 index 0000000..be20d34 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/MucPublisher.kt @@ -0,0 +1,60 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.xmpp.XmppConnection +import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension +import java.time.Duration +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + +class MucPublisher( + val executor: ScheduledExecutorService, + val interval: Duration, + val xmppConnection: XmppConnection +) { + val logger = createLogger() + + var task: ScheduledFuture<*>? = null + fun start() { + logger.info("Starting with interval $interval.") + // We read the stats from the prometheus metrics and send updated presence. None of these are blocking. + task = executor.scheduleAtFixedRate( + { publishPresence() }, + 0, + interval.toMillis(), + TimeUnit.MILLISECONDS + ) + } + fun stop() { + task?.cancel(true) + task = null + } + + private fun publishPresence() { + val statsExt: ColibriStatsExtension = if (XmppClientConnectionConfig.config.statsFilterEnabled) { + VideobridgeStatisticsShim.getColibriStatsExtension(XmppClientConnectionConfig.config.statsWhitelist) + } else { + VideobridgeStatisticsShim.getColibriStatsExtension() + } + logger.debug { "Publishing statistics in presence: $statsExt" } + + xmppConnection.setPresenceExtension(statsExt) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt new file mode 100644 index 0000000..6fe7600 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/PacketTransitStats.kt @@ -0,0 +1,149 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.stats.BridgeJitterStats +import org.jitsi.nlj.stats.PacketDelayStats +import org.jitsi.rtp.extensions.looksLikeRtcp +import org.jitsi.rtp.extensions.looksLikeRtp +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.stats.BucketStats +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import java.time.Clock +import java.time.Duration + +/** + * Track how long it takes for all RTP and RTCP packets to make their way through the bridge. + * [Endpoint] and [Relay] are the 'last place' that is aware of [PacketInfo] in the outgoing + * chains; they track these stats here. Since they're static, these members will track the delay + * for packets going out to all endpoints. + */ +object PacketTransitStats { + private val jsonEnabled: Boolean by config { + "videobridge.stats.transit-time.enable-json".from(JitsiConfig.newConfig) + } + private val prometheusEnabled: Boolean by config { + "videobridge.stats.transit-time.enable-prometheus".from(JitsiConfig.newConfig) + } + private val jitterEnabled: Boolean by config { + "videobridge.stats.transit-time.enable-jitter".from(JitsiConfig.newConfig) + } + private val enabled = jsonEnabled || prometheusEnabled || jitterEnabled + + private val logger = createLogger() + private val clock: Clock = Clock.systemUTC() + + init { + logger.info( + "Initializing, jsonEnabled=$jsonEnabled, prometheusEnabled=$prometheusEnabled, " + + "jitterEnabled=$jitterEnabled" + ) + } + + private val rtpPacketDelayStats = if (jsonEnabled) PacketDelayStats() else null + private val rtcpPacketDelayStats = if (jsonEnabled) PacketDelayStats() else null + private val prometheusRtpDelayStats = if (prometheusEnabled) { + PrometheusPacketDelayStats("rtp_transit_time") + } else { + null + } + private val prometheusRtcpDelayStats = if (prometheusEnabled) { + PrometheusPacketDelayStats("rtcp_transit_time") + } else { + null + } + + private val bridgeJitterStats = if (jitterEnabled) BridgeJitterStats() else null + + @JvmStatic + fun packetSent(packetInfo: PacketInfo) { + if (!enabled) { + return + } + + val delayMs = packetInfo.receivedTime?.let { Duration.between(it, clock.instant()).toMillis() } + if (packetInfo.packet.looksLikeRtp()) { + if (delayMs != null) { + rtpPacketDelayStats?.addDelay(delayMs) + prometheusRtpDelayStats?.addDelay(delayMs) + bridgeJitterStats?.packetSent(packetInfo) + } else { + rtpPacketDelayStats?.addUnknown() + prometheusRtpDelayStats?.addUnknown() + } + } else if (packetInfo.packet.looksLikeRtcp()) { + if (delayMs != null) { + rtcpPacketDelayStats?.addDelay(delayMs) + prometheusRtcpDelayStats?.addDelay(delayMs) + } else { + rtcpPacketDelayStats?.addUnknown() + prometheusRtcpDelayStats?.addUnknown() + } + } + } + + @JvmStatic + val statsJson: ObjectNode + get() { + val stats = JsonNodeFactory.instance.objectNode() + stats.set("e2e_packet_delay", getPacketDelayStats()) + bridgeJitterStats?.let { + stats.put("overall_bridge_jitter", it.jitter) + } + return stats + } + + @JvmStatic + val bridgeJitter + get() = bridgeJitterStats?.jitter + + private fun getPacketDelayStats() = JsonNodeFactory.instance.objectNode().apply { + rtpPacketDelayStats?.let { + put("rtp", it.toJson(format = BucketStats.Format.CumulativeRight)) + } + rtcpPacketDelayStats?.let { + put("rtcp", it.toJson(format = BucketStats.Format.CumulativeRight)) + } + } +} + +class PrometheusPacketDelayStats(name: String) { + private val histogram = VideobridgeMetricsContainer.instance.registerHistogram( + name, + "Packet delay stats for $name", + 0.0, + 5.0, + 50.0, + 500.0 + ) + private val numPacketsWithoutTimestamps = VideobridgeMetricsContainer.instance.registerCounter( + "${name}_unknown_delay", + "Number of packets without an unknown delay ($name)" + ) + + fun addUnknown() { + numPacketsWithoutTimestamps.inc() + } + fun addDelay(delayMs: Long) { + histogram.histogram.observe(delayMs.toDouble()) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt new file mode 100644 index 0000000..17633de --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/QueueStats.kt @@ -0,0 +1,75 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.RtpReceiverImpl +import org.jitsi.nlj.RtpSenderImpl +import org.jitsi.utils.queue.CountingErrorHandler +import org.jitsi.utils.queue.QueueStatistics.Companion.getStatistics +import org.jitsi.videobridge.AbstractEndpointMessageTransport +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.relay.Relay +import org.jitsi.videobridge.relay.RelayEndpointSender + +object QueueStats { + /** Gets statistics for the different `PacketQueue`s that this bridge uses. */ + @JvmStatic + fun getQueueStats(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + getJsonFromQueueStatisticsAndErrorHandler( + Endpoint.queueErrorCounter, + "Endpoint-outgoing-packet-queue" + )?.let { set("srtp_send_queue", it) } + getJsonFromQueueStatisticsAndErrorHandler( + Relay.queueErrorCounter, + "Relay-outgoing-packet-queue" + )?.let { set("relay_srtp_send_queue", it) } + getJsonFromQueueStatisticsAndErrorHandler( + RelayEndpointSender.queueErrorCounter, + "RelayEndpointSender-outgoing-packet-queue" + )?.let { set("relay_endpoint_sender_srtp_send_queue", it) } + getJsonFromQueueStatisticsAndErrorHandler( + RtpReceiverImpl.queueErrorCounter, + "rtp-receiver-incoming-packet-queue" + )?.let { set("rtp_receiver_queue", it) } + getJsonFromQueueStatisticsAndErrorHandler( + RtpSenderImpl.queueErrorCounter, + "rtp-sender-incoming-packet-queue" + )?.let { set("rtp_sender_queue", it) } + getStatistics().get("colibri-queue")?.let { set("colibri_queue", it) } + getJsonFromQueueStatisticsAndErrorHandler( + null, + AbstractEndpointMessageTransport.INCOMING_MESSAGE_QUEUE_ID + )?.let { set(AbstractEndpointMessageTransport.INCOMING_MESSAGE_QUEUE_ID, it) } + } + + private fun getJsonFromQueueStatisticsAndErrorHandler( + countingErrorHandler: CountingErrorHandler?, + queueName: String + ): ObjectNode? { + var json = getStatistics().get(queueName) as? ObjectNode + if (countingErrorHandler != null) { + if (json == null) { + json = JsonNodeFactory.instance.objectNode() + json.put("dropped_packets", countingErrorHandler.numPacketsDropped) + } + json.put("exceptions", countingErrorHandler.numExceptions) + } + + return json + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt new file mode 100644 index 0000000..4315a1b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/stats/VideobridgeStatisticsShim.kt @@ -0,0 +1,248 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.stats + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.rtcp.RembHandler +import org.jitsi.videobridge.EndpointConnectionStatusMonitor +import org.jitsi.videobridge.VersionConfig +import org.jitsi.videobridge.health.JvbHealthChecker +import org.jitsi.videobridge.load_management.JvbLoadManager +import org.jitsi.videobridge.metrics.JvmMetrics +import org.jitsi.videobridge.metrics.Metrics +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgePeriodicMetrics +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.videobridge.transport.ice.IceTransport +import org.jitsi.videobridge.version.JvbVersionService +import org.jitsi.videobridge.xmpp.XmppConnection +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.BITRATE_DOWNLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.BITRATE_UPLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.DRAIN +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.ENDPOINTS_SENDING_AUDIO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.ENDPOINTS_SENDING_VIDEO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.INACTIVE_CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.INACTIVE_ENDPOINTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.LARGEST_CONFERENCE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_ENDPOINTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_RECEIVE_BITRATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_RECEIVE_PACKET_RATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_SEND_BITRATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.OCTO_SEND_PACKET_RATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.P2P_CONFERENCES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.PACKET_RATE_DOWNLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.PACKET_RATE_UPLOAD +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.PARTICIPANTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RECEIVE_ONLY_ENDPOINTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.REGION +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RELAY_ID +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RELEASE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.RTT_AGGREGATE +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.SHUTDOWN_IN_PROGRESS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.Stat +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.THREADS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TIMESTAMP +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_RECEIVED_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_BYTES_SENT_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_CONFERENCES_COMPLETED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_CONFERENCES_CREATED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_CONFERENCE_SECONDS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DATA_CHANNEL_MESSAGES_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DATA_CHANNEL_MESSAGES_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_DOMINANT_SPEAKER_CHANGES +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_FAILED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_ICE_SUCCEEDED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_RECEIVED +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_RECEIVED_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_SENT +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PACKETS_SENT_OCTO +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.TOTAL_PARTICIPANTS +import org.jitsi.xmpp.extensions.colibri.ColibriStatsExtension.VERSION +import java.text.SimpleDateFormat +import java.util.Date +import java.util.TimeZone + +/** + * A shim layer which translates the Prometheus metrics (from [VideobridgeMetricsContainer]) to the legacy + * [ColibriStatsExtension] suitable to be added to XMPP presence or converted to JSON for the response to the legacy + * /colibri/stats + */ +object VideobridgeStatisticsShim { + fun getStatsJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + getStats().forEach { (k, v) -> + when (v) { + is Boolean -> put(k, v) + is Long -> put(k, v) + is Int -> put(k, v) + is Double -> put(k, v) + is Float -> put(k, v) + else -> put(k, v.toString()) + } + } + } + + /** + * Formats statistics in ColibriStatsExtension object + * @param statistics the statistics instance + * @return the ColibriStatsExtension instance. + */ + fun getColibriStatsExtension() = ColibriStatsExtension().apply { + getStats().forEach { (key, value) -> + addStat(Stat(key, value)) + } + } + + /** + * Formats statistics in ColibriStatsExtension object + * @param statistics the statistics instance + * @param whitelist which of the statistics to use + * @return the ColibriStatsExtension instance. + */ + fun getColibriStatsExtension(whitelist: List) = ColibriStatsExtension().apply { + val allStats = getStats() + whitelist.forEach { whitelistedKey -> + val value = allStats[whitelistedKey] + if (value != null) { + addStat(Stat(whitelistedKey, value)) + } + } + } + + private val timestampFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + private fun getStats(): Map = synchronized(Metrics.lock) { + return buildMap { + put("incoming_loss", VideobridgePeriodicMetrics.incomingLoss.get()) + put("outgoing_loss", VideobridgePeriodicMetrics.outgoingLoss.get()) + put("overall_loss", VideobridgePeriodicMetrics.loss.get()) + put("endpoints_with_high_outgoing_loss", VideobridgePeriodicMetrics.endpointsWithHighOutgoingLoss.get()) + put("tcc_unmatched_feedback", VideobridgePeriodicMetrics.tccUnmatchedFeedback.get()) + put("local_active_endpoints", VideobridgePeriodicMetrics.activeEndpoints.get()) + put(BITRATE_DOWNLOAD, VideobridgePeriodicMetrics.incomingBitrate.get() / 1000) + put(BITRATE_UPLOAD, VideobridgePeriodicMetrics.outgoingBitrate.get() / 1000) + put(PACKET_RATE_DOWNLOAD, VideobridgePeriodicMetrics.incomingPacketRate.get()) + put(PACKET_RATE_UPLOAD, VideobridgePeriodicMetrics.outgoingPacketRate.get()) + put(RTT_AGGREGATE, VideobridgePeriodicMetrics.averageRtt.get()) + put("num_eps_oversending", VideobridgePeriodicMetrics.endpointsOversending.get()) + put(OCTO_CONFERENCES, VideobridgePeriodicMetrics.conferencesWithRelay.get()) + put(INACTIVE_CONFERENCES, VideobridgePeriodicMetrics.conferencesInactive.get()) + put(P2P_CONFERENCES, VideobridgePeriodicMetrics.conferencesP2p.get()) + put("endpoints", VideobridgePeriodicMetrics.endpoints.get()) + put(PARTICIPANTS, VideobridgePeriodicMetrics.endpoints.get()) + put(RECEIVE_ONLY_ENDPOINTS, VideobridgePeriodicMetrics.endpointsReceiveOnly.get()) + put(INACTIVE_ENDPOINTS, VideobridgePeriodicMetrics.endpointsInactive.get()) + put(OCTO_ENDPOINTS, VideobridgePeriodicMetrics.endpointsRelayed.get()) + put(ENDPOINTS_SENDING_AUDIO, VideobridgePeriodicMetrics.endpointsSendingAudio.get()) + put(ENDPOINTS_SENDING_VIDEO, VideobridgePeriodicMetrics.endpointsSendingVideo.get()) + put(LARGEST_CONFERENCE, VideobridgePeriodicMetrics.largestConference.get()) + put(OCTO_RECEIVE_BITRATE, VideobridgePeriodicMetrics.relayIncomingBitrate.get()) + put(OCTO_RECEIVE_PACKET_RATE, VideobridgePeriodicMetrics.relayIncomingPacketRate.get()) + put(OCTO_SEND_BITRATE, VideobridgePeriodicMetrics.relayOutgoingBitrate.get()) + put(OCTO_SEND_PACKET_RATE, VideobridgePeriodicMetrics.relayOutgoingPacketRate.get()) + put("endpoints_with_suspended_sources", VideobridgePeriodicMetrics.endpointsWithSuspendedSources.get()) + + put(TOTAL_CONFERENCES_CREATED, VideobridgeMetrics.conferencesCreated.get()) + put(TOTAL_CONFERENCES_COMPLETED, VideobridgeMetrics.conferencesCompleted.get()) + put(TOTAL_CONFERENCE_SECONDS, VideobridgeMetrics.totalConferenceSeconds.get()) + put(TOTAL_PARTICIPANTS, VideobridgeMetrics.totalEndpoints.get()) + put("total_visitors", VideobridgeMetrics.totalVisitors.get()) + put( + "num_eps_no_msg_transport_after_delay", + VideobridgeMetrics.numEndpointsNoMessageTransportAfterDelay.get() + ) + put("total_relays", VideobridgeMetrics.totalRelays.get()) + put( + "num_relays_no_msg_transport_after_delay", + VideobridgeMetrics.numRelaysNoMessageTransportAfterDelay.get() + ) + put("total_keyframes_received", VideobridgeMetrics.keyframesReceived.get()) + put("total_layering_changes_received", VideobridgeMetrics.layeringChangesReceived.get()) + put( + "total_video_stream_milliseconds_received", + VideobridgeMetrics.totalVideoStreamMillisecondsReceived.get() + ) + put("stress_level", VideobridgeMetrics.stressLevel.get()) + put(CONFERENCES, VideobridgeMetrics.currentConferences.get()) + put("visitors", VideobridgeMetrics.currentVisitors.get()) + put("local_endpoints", VideobridgeMetrics.currentLocalEndpoints.get()) + put(TOTAL_DATA_CHANNEL_MESSAGES_RECEIVED, VideobridgeMetrics.dataChannelMessagesReceived.get()) + put(TOTAL_DATA_CHANNEL_MESSAGES_SENT, VideobridgeMetrics.dataChannelMessagesSent.get()) + put(TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_RECEIVED, VideobridgeMetrics.colibriWebSocketMessagesReceived.get()) + put(TOTAL_COLIBRI_WEB_SOCKET_MESSAGES_SENT, VideobridgeMetrics.colibriWebSocketMessagesSent.get()) + put(TOTAL_BYTES_RECEIVED, VideobridgeMetrics.totalBytesReceived.get()) + put("endpoints_ice_failed", VideobridgeMetrics.endpointsIceFailed.get()) + put("dtls_failed_endpoints", VideobridgeMetrics.endpointsDtlsFailed.get()) + put("relays_ice_failed", VideobridgeMetrics.relaysIceFailed.get()) + put("dtls_failed_relays", VideobridgeMetrics.relaysDtlsFailed.get()) + put(TOTAL_BYTES_SENT, VideobridgeMetrics.totalBytesSent.get()) + put(TOTAL_PACKETS_RECEIVED, VideobridgeMetrics.packetsReceived.get()) + put(TOTAL_PACKETS_SENT, VideobridgeMetrics.packetsSent.get()) + put(TOTAL_BYTES_RECEIVED_OCTO, VideobridgeMetrics.totalRelayBytesReceived.get()) + put(TOTAL_BYTES_SENT_OCTO, VideobridgeMetrics.totalRelayBytesSent.get()) + put(TOTAL_PACKETS_RECEIVED_OCTO, VideobridgeMetrics.relayPacketsReceived.get()) + put(TOTAL_PACKETS_SENT_OCTO, VideobridgeMetrics.relayPacketsSent.get()) + put(TOTAL_DOMINANT_SPEAKER_CHANGES, VideobridgeMetrics.dominantSpeakerChanges.get()) + put("preemptive_kfr_sent", VideobridgeMetrics.preemptiveKeyframeRequestsSent.get()) + put("preemptive_kfr_suppressed", VideobridgeMetrics.preemptiveKeyframeRequestsSuppressed.get()) + + put(TOTAL_ICE_FAILED, IceTransport.iceFailed.get()) + put(TOTAL_ICE_SUCCEEDED, IceTransport.iceSucceeded.get()) + put("total_ice_succeeded_relayed", IceTransport.iceSucceededRelayed.get()) + + put("average_participant_stress", JvbLoadManager.averageParticipantStress) + + JvmMetrics.INSTANCE?.threadCount?.let { + put(THREADS, it.get()) + } + + put(SHUTDOWN_IN_PROGRESS, VideobridgeMetrics.gracefulShutdown.get()) + put("shutting_down", VideobridgeMetrics.shuttingDown.get()) + put(DRAIN, VideobridgeMetrics.drainMode.get()) + + put(TIMESTAMP, timestampFormat.format(Date())) + if (RelayConfig.config.enabled) { + put(RELAY_ID, RelayConfig.config.relayId) + } + put("muc_clients_configured", XmppConnection.mucClientsConfigured.get()) + put("muc_clients_connected", XmppConnection.mucClientsConnected.get()) + put("mucs_configured", XmppConnection.mucsConfigured.get()) + put("mucs_joined", XmppConnection.mucsJoined.get()) + + put("endpoints_with_spurious_remb", RembHandler.endpointsWithSpuriousRemb()) + put("healthy", JvbHealthChecker.healthyMetric.get()) + put("endpoints_disconnected", EndpointConnectionStatusMonitor.endpointsDisconnected.get()) + put("endpoints_reconnected", EndpointConnectionStatusMonitor.endpointsReconnected.get()) + + put(VERSION, JvbVersionService.instance.currentVersion.toString()) + VersionConfig.config.release?.let { + put(RELEASE, it) + } + VideobridgeMetrics.regionInfo?.let { + put(REGION, it.get()) + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt new file mode 100644 index 0000000..f1cf572 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/dtls/DtlsTransport.kt @@ -0,0 +1,269 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.transport.dtls + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.ice4j.util.Buffer +import org.jitsi.nlj.dtls.DtlsClient +import org.jitsi.nlj.dtls.DtlsServer +import org.jitsi.nlj.dtls.DtlsStack +import org.jitsi.nlj.srtp.TlsRole +import org.jitsi.nlj.util.BufferPool +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.utils.queue.PacketQueue +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Transport layer which negotiates a DTLS connection and supports + * decrypting and encrypting data. + * + * Incoming DTLS data should be fed into this layer via [dtlsDataReceived], + * and decrypted DTLS application data will be passed to the + * [incomingDataHandler], which should be set by an interested party. + * + * Outgoing data can be sent via [sendDtlsData] and the encrypted data will + * be passed to the [outgoingDataHandler], which should be set by an + * interested party. + */ +class DtlsTransport(parentLogger: Logger, id: String) { + private val logger = createChildLogger(parentLogger) + + private val running = AtomicBoolean(true) + + @JvmField + var incomingDataHandler: IncomingDataHandler? = null + + @JvmField + var outgoingDataHandler: OutgoingDataHandler? = null + + @JvmField + var eventHandler: EventHandler? = null + private var dtlsHandshakeComplete = false + + val isConnected: Boolean + get() = dtlsHandshakeComplete + + private val stats = Stats() + + /** Whether to advertise cryptex to peers. */ + var cryptex = false + + val dtlsQueue = object : PacketQueue( + 128, + null, + "dtls-queue-$id", + { buffer: Buffer -> + try { + dtlsDataReceived(buffer.buffer, buffer.offset, buffer.length) + true + } catch (e: Exception) { + logger.warn("Failed to handle DTLS data", e) + false + } + }, + TaskPools.IO_POOL, + ) { + override fun releasePacket(buffer: Buffer) { + BufferPool.returnBuffer(buffer.buffer) + } + } + + /** + * The DTLS stack instance + */ + private val dtlsStack = DtlsStack(logger).also { + // Install a handler for when the DTLS stack has decrypted application data available + it.incomingDataHandler = object : DtlsStack.IncomingDataHandler { + override fun dataReceived(data: ByteArray, off: Int, len: Int) { + stats.numPacketsReceived++ + incomingDataHandler?.dtlsAppDataReceived(data, off, len) ?: run { + stats.numIncomingPacketsDroppedNoHandler++ + } + } + } + + // Install a handler to allow the DTLS stack to send out encrypted data + it.outgoingDataHandler = object : DtlsStack.OutgoingDataHandler { + override fun sendData(data: ByteArray, off: Int, len: Int) { + outgoingDataHandler?.sendData(data, off, len)?.also { + stats.numPacketsSent++ + } ?: run { + stats.numOutgoingPacketsDroppedNoHandler++ + } + } + } + + // Handle DTLS stack events + it.eventHandler = object : DtlsStack.EventHandler { + override fun handshakeComplete( + chosenSrtpProtectionProfile: Int, + tlsRole: TlsRole, + keyingMaterial: ByteArray + ) { + dtlsHandshakeComplete = true + eventHandler?.handshakeComplete(chosenSrtpProtectionProfile, tlsRole, keyingMaterial) + } + } + } + + /** + * Start a DTLS handshake. The 'role' should have been set before calling this + * (via [setSetupAttribute] + */ + fun startDtlsHandshake() { + logger.info("Starting DTLS handshake, role=${dtlsStack.role?.javaClass?.simpleName}") + if (dtlsStack.role == null) { + logger.warn("Starting the DTLS stack before it knows its role") + } + try { + dtlsStack.start() + } catch (t: Throwable) { + // TODO: we're not doing anything here, should we? or change the log? + logger.error("Error during DTLS negotiation, closing this transport manager", t) + } + } + + /** + * Applies the peer's `setup` attribute. This is called for every transport update that carries a fingerprint, + * including the ones that have nothing to do with DTLS (an ICE restart, a trickled candidate), so re-applying + * the role we already have does nothing and is not logged. + */ + fun setSetupAttribute(setupAttr: String?) { + if (setupAttr.isNullOrEmpty()) { + return + } + when (setupAttr.lowercase()) { + "active" -> { + if (dtlsStack.actAsServer()) { + logger.info("The remote side is acting as DTLS client, we'll act as server") + } + } + + "passive" -> { + if (dtlsStack.actAsClient()) { + logger.info("The remote side is acting as DTLS server, we'll act as client") + } + } + + else -> { + logger.error( + "The remote side sent an unrecognized DTLS setup value: " + + setupAttr + ) + } + } + } + + fun setRemoteFingerprints(remoteFingerprints: Map>) { + // Don't pass an empty list to the stack in order to avoid wiping + // certificates that were contained in a previous request. + if (remoteFingerprints.isEmpty()) { + return + } + + dtlsStack.remoteFingerprints = remoteFingerprints + } + + /** + * Describe the properties of this [DtlsTransport] into the given + * [IceUdpTransportPacketExtension] + */ + fun describe(iceUdpTransportPe: IceUdpTransportPacketExtension) { + val fingerprintPE = iceUdpTransportPe.getFirstChildOfType(DtlsFingerprintPacketExtension::class.java) ?: run { + DtlsFingerprintPacketExtension().also { iceUdpTransportPe.addChildExtension(it) } + } + fingerprintPE.setup = when (dtlsStack.role) { + is DtlsServer -> "passive" + is DtlsClient -> "active" + null -> "actpass" + else -> throw IllegalStateException("Cannot describe role ${dtlsStack.role}") + } + fingerprintPE.fingerprint = dtlsStack.localFingerprint + fingerprintPE.hash = dtlsStack.localFingerprintHashFunction + if (cryptex) { + fingerprintPE.cryptex = true + } + } + + fun enqueueBuffer(buffer: Buffer) = dtlsQueue.add(buffer) + + /** + * Notify this layer that DTLS data has been received from the network + */ + private fun dtlsDataReceived(data: ByteArray, off: Int, len: Int) = + dtlsStack.processIncomingProtocolData(data, off, len) + + /** + * Send out DTLS data + */ + fun sendDtlsData(data: ByteArray, off: Int, len: Int) = dtlsStack.sendApplicationData(data, off, len) + + fun stop() { + if (running.compareAndSet(true, false)) { + logger.info("Stopping") + dtlsStack.close() + } + } + + fun getDebugState(): ObjectNode = stats.toJson().apply { + put("running", running.get()) + put("role", dtlsStack.role?.javaClass?.simpleName ?: "null") + put("is_connected", isConnected) + } + + private data class Stats( + var numPacketsReceived: Int = 0, + var numIncomingPacketsDroppedNoHandler: Int = 0, + var numPacketsSent: Int = 0, + var numOutgoingPacketsDroppedNoHandler: Int = 0 + ) { + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("num_packets_received", numPacketsReceived) + put("num_incoming_packets_dropped_no_handler", numIncomingPacketsDroppedNoHandler) + put("num_packets_sent", numPacketsSent) + put("num_outgoing_packets_dropped_no_handler", numOutgoingPacketsDroppedNoHandler) + } + } + + /** + * A handler for when [DtlsTransport] wants to send data out + * onto the network + */ + interface OutgoingDataHandler { + fun sendData(buf: ByteArray, off: Int, len: Int) + } + + /** + * A handler for when [DtlsTransport] has received DTLS application + * data + */ + interface IncomingDataHandler { + fun dtlsAppDataReceived(buf: ByteArray, off: Int, len: Int) + } + + /** + * A handler for [DtlsTransport] events + */ + interface EventHandler { + fun handshakeComplete(chosenSrtpProtectionProfile: Int, tlsRole: TlsRole, keyingMaterial: ByteArray) + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceStatistics.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceStatistics.kt new file mode 100644 index 0000000..baf3654 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceStatistics.kt @@ -0,0 +1,69 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.transport.ice + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.utils.stats.BucketStats +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.DoubleAdder + +/** + * Keep ICE-related statistics (currently only RTT). + */ +class IceStatistics { + /** Stats for all harvesters */ + private val combinedStats = Stats() + + /** Stats by harvester name. */ + private val statsByHarvesterName = ConcurrentHashMap() + + /** Add a round-trip-time measurement for a specific harvester */ + fun add(harvesterName: String, rttMs: Double) { + combinedStats.add(rttMs = rttMs) + statsByHarvesterName.computeIfAbsent(harvesterName) { Stats() }.add(rttMs = rttMs) + } + + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("all", combinedStats.toJson()) + statsByHarvesterName.forEach { (harvesterName, stats) -> + set(harvesterName, stats.toJson()) + } + } + + companion object { + val stats = IceStatistics() + } + + private class Stats { + /** Histogram of RTTs */ + val buckets = BucketStats(listOf(0, 10, 20, 40, 60, 80, 100, 150, 200, 250, 300, 500, 1000, Long.MAX_VALUE)) + var sum = DoubleAdder() + var count = AtomicInteger() + + fun add(rttMs: Double) { + sum.add(rttMs) + count.incrementAndGet() + buckets.addValue((rttMs + 0.5).toLong()) + } + + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("average", sum.sum() / count.toDouble()) + set("buckets", buckets.toJson()) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt new file mode 100755 index 0000000..80772b9 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/transport/ice/IceTransport.kt @@ -0,0 +1,1196 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.transport.ice + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import com.google.common.net.InetAddresses +import org.ice4j.Transport +import org.ice4j.TransportAddress +import org.ice4j.ice.Agent +import org.ice4j.ice.CandidateType +import org.ice4j.ice.Component +import org.ice4j.ice.HostCandidate +import org.ice4j.ice.IceMediaStream +import org.ice4j.ice.IceProcessingState +import org.ice4j.ice.LocalCandidate +import org.ice4j.ice.RemoteCandidate +import org.ice4j.ice.harvest.MappingCandidateHarvesters +import org.ice4j.util.Buffer +import org.ice4j.util.BufferHandler +import org.jitsi.rtp.Packet +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createChildLogger +import org.jitsi.videobridge.ice.Harvesters +import org.jitsi.videobridge.ice.IceConfig +import org.jitsi.videobridge.ice.TransportUtils +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.util.ByteBufferPool +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.xmpp.extensions.jingle.CandidatePacketExtension +import org.jitsi.xmpp.extensions.jingle.IceCandidatePacketExtension +import org.jitsi.xmpp.extensions.jingle.IceRtcpmuxPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import java.beans.PropertyChangeEvent +import java.beans.PropertyChangeListener +import java.io.IOException +import java.net.DatagramPacket +import java.net.Inet6Address +import java.time.Clock +import java.time.Duration +import java.time.Instant +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.LongAdder + +/** + * The outcome of an ICE restart request, which decides what the caller signals back to the peer. + */ +enum class IceRestartResult { + /** A new Agent was created. Signal its transport, so the peer moves its checks to it. */ + STARTED, + + /** + * No new Agent was created, but the established one is still usable. Signal its transport unchanged: the + * peer keeps the connection it has, instead of escalating to a full re-invite over a transport that is + * merely still connecting. + */ + KEEP_EXISTING, + + /** + * This bridge can not restart ICE. Signal no transport at all, which is how the peer learns to fall back to + * a full re-invite. + */ + UNAVAILABLE +} + +class IceTransport @JvmOverloads constructor( + id: String, + /** + * Whether the ICE agent created by this transport should be the + * 'controlling' role. + */ + controlling: Boolean, + /** + * Whether the ICE agent created by this transport should use + * unique local ports, rather than the configured port. + */ + val useUniquePort: Boolean, + /** + * Use private addresses for this [IceTransport] even if [IceConfig.advertisePrivateCandidates] is false. + */ + private val advertisePrivateAddresses: Boolean, + parentLogger: Logger, + private val clock: Clock = Clock.systemUTC(), + /** + * Creates the ice4j [Agent]s of this transport, with their candidate harvesters configured. Only replaced + * in tests, which have no use for real Agents (they bind ports and start threads). + */ + agentFactory: ((Logger) -> Agent)? = null +) { + private val logger = createChildLogger(parentLogger) + + private val agentFactory: (Logger) -> Agent = agentFactory ?: { agentLogger -> + Agent(IceConfig.config.ufragPrefix, agentLogger).apply { + if (useUniquePort) { + setUseDynamicPorts(true) + } else { + appendHarvesters(this) + } + } + } + + /** + * The handler which will be invoked when data is received. + * This field should be set by some other entity which wishes to handle the incoming data + * received over the ICE connection. + */ + @JvmField + var incomingDataHandler: IncomingDataHandler? = null + + /** + * The handler which will be invoked when events fired by [IceTransport] + * occur. This field should be set by another entity who wishes to handle + * the events. Handlers will only be notified of events which occur + * *after* the handler has been set. + */ + @JvmField + var eventHandler: EventHandler? = null + + /** + * Whether or not it is possible to write to this [IceTransport]. + * + * This happens as soon as any candidate pair is validated, and happens (usually) before iceConnected. + */ + private val iceWriteable = AtomicBoolean(false) + + /** + * Whether or not this [IceTransport] has connected. + */ + private val iceConnected = AtomicBoolean(false) + + /** + * Whether or not this [IceTransport] has failed to connect. + */ + private val iceFailed = AtomicBoolean(false) + + fun hasFailed(): Boolean = iceFailed.get() + + fun isWriteable(): Boolean = iceWriteable.get() + + fun isConnected(): Boolean = iceConnected.get() + + /** + * Whether or not this transport is 'running'. If it is not + * running, no more data will be read from the socket or sent out. + */ + private val running = AtomicBoolean(true) + + private val packetStats = PacketStats() + + /** + * Whether the ice4j [Agent]s created by this transport take the 'controlling' role. + */ + private val controlling = controlling + + /** + * Guards transitions between [currentBundle] and [pendingBundle]. + */ + private val restartLock = Any() + + /** + * The [AgentBundle] that media is currently sent on and whose credentials we advertise. Replaced (via + * [cutOver]) when an ICE restart's new Agent connects. + */ + @Volatile + private var currentBundle: AgentBundle + + /** + * The [AgentBundle] of an ICE restart that is in progress but has not connected yet, if any. It runs + * alongside [currentBundle], which keeps sending until the cutover (make-before-break). + */ + @Volatile + private var pendingBundle: AgentBundle? = null + + /** + * The [AgentBundle] we cut over from, for as long as its transition window lasts. Held here, and not only + * by the task that frees it, so that [stop] can free it right away instead of leaving an Agent alive after + * the transport is gone. + * + * Guarded by [restartLock]. + */ + private var retiringBundle: AgentBundle? = null + + /** + * The task that abandons [pendingBundle] if it does not connect in time, if one is scheduled. + * + * Guarded by [restartLock]. + */ + private var restartTimeoutTask: ScheduledFuture<*>? = null + + /** + * The task that frees [retiringBundle] when its transition window elapses, if one is scheduled. + * + * Guarded by [restartLock]. + */ + private var transitionWindowTask: ScheduledFuture<*>? = null + + /** + * The ICE generation of the most recent restart we started. The bridge owns this counter: each accepted + * restart request gets the next value, it is advertised on the transport we return (as `ice-generation`), + * and the peer echoes it on the transport-info carrying its own new credentials. That lets us match the + * peer's response to the right pending bundle and discard responses from a superseded round. + * + * Restart generations start at 1. Generation 0 means "the initial allocation, never restarted" and + * [IceUdpTransportPacketExtension.GENERATION_UNSPECIFIED] means the attribute is absent altogether: the initial + * bundle carries the latter, so nothing is stamped on the wire until a restart actually happens. Clients reject + * a restart transport whose generation is not >= 1, so this must not start any lower. + * + * Guarded by [restartLock]. + */ + private var restartGeneration = 0 + + init { + currentBundle = createAgentBundle(IceUdpTransportPacketExtension.GENERATION_UNSPECIFIED) + ?: throw IllegalStateException("Failed to create the initial ICE Agent") + logger.addContext("local_ufrag", currentBundle.agent.localUfrag) + } + + /** + * Creates an [Agent] with its stream and component, and wraps them in an [AgentBundle]. + * + * @return the new bundle, or null if ice4j failed to create it (creating a component harvests candidates + * and can fail to bind). Anything that was already created is freed, so a failure leaks neither an Agent + * nor its ufrag in the single port harvester. + */ + private fun createAgentBundle(generation: Int): AgentBundle? { + var agent: Agent? = null + return try { + val newAgent = agentFactory(logger).apply { + isControlling = this@IceTransport.controlling + performConsentFreshness = true + nominationStrategy = IceConfig.config.nominationStrategy + } + agent = newAgent + val stream = newAgent.createMediaStream("stream") + val component = newAgent.createComponent(stream, IceConfig.config.keepAliveStrategy, false) + AgentBundle(generation, newAgent, stream, component) + } catch (t: Throwable) { + logger.error("Failed to create an ICE Agent (generation=$generation).", t) + agent?.free() + null + } + } + + /** + * An ice4j [Agent] together with the single stream and component we create on it and the listeners we + * attach to them. An [IceTransport] has exactly one of these normally, and briefly two while an ICE + * restart is in flight: the established one (which keeps sending) and the new one (which is running + * connectivity checks with freshly rotated local credentials). + * + * Created by [createAgentBundle], which is also where a failure to create the ice4j objects is handled. + * + * @param generation the `ice-generation` of the restart round that created this bundle, or + * [IceUdpTransportPacketExtension.GENERATION_UNSPECIFIED] for the initial bundle. + */ + private inner class AgentBundle( + val generation: Int, + val agent: Agent, + val stream: IceMediaStream, + val component: Component + ) { + private val stateChangeListener = PropertyChangeListener { ev -> iceStateChanged(this@AgentBundle, ev) } + private val pairChangeListener = PropertyChangeListener { ev -> iceStreamPairChanged(this@AgentBundle, ev) } + + /** When this bundle was created, used to report how long a restart took. */ + val createdAt: Instant = clock.instant() + + /** + * Whether connectivity establishment has been started on [agent]. For a restart's bundle this only + * happens once the peer's new remote credentials arrive, and it must happen at most once. + */ + val checksStarted = AtomicBoolean(false) + + init { + agent.addStateChangeListener(stateChangeListener) + stream.addPairChangeListener(pairChangeListener) + component.setBufferCallback(object : BufferHandler { + override fun handleBuffer(buffer: Buffer) { + incomingDataHandler?.dataReceived(buffer) ?: run { + packetStats.numIncomingPacketsDroppedNoHandler.increment() + ByteBufferPool.returnBuffer(buffer.buffer) + } + } + }) + } + + private val freed = AtomicBoolean(false) + + fun free() { + if (!freed.compareAndSet(false, true)) { + return + } + agent.removeStateChangeListener(stateChangeListener) + stream.removePairStateChangeListener(pairChangeListener) + agent.free() + } + + override fun toString(): String = "AgentBundle[generation=$generation, ufrag=${agent.localUfrag}]" + } + + /** + * Tell this [IceTransport] to start ICE connectivity establishment. + */ + fun startConnectivityEstablishment(transportPacketExtension: IceUdpTransportPacketExtension) { + if (!running.get()) { + logger.warn("Not starting connectivity establishment, transport is not running") + return + } + + // An update tagged with an `ice-generation` is the peer answering an ICE restart with the new + // credentials that our new Agent needs in order to address its own connectivity checks. Route it to the + // pending bundle instead of the established one. + // + // Only a tagged update: the peer stamps the generation on the restart answer and on nothing else, so an + // untagged update arriving while a restart is in flight is an ordinary transport update (a trickled + // candidate, or an old one still on the wire) and belongs to the established bundle. Applying such an + // update to the pending bundle would give it the peer's *old* password, whose checks the peer rejects. + // + // Note that the peer's own trickled candidates never reach the pending bundle this way, and the + // candidates of a tagged update never reach the established one. Neither matters for the bridge: it + // signals no candidates of its own that the peer must answer, and it discovers the peer's address + // peer-reflexively from the peer's incoming checks. + val pending = pendingBundle + if (pending != null && + transportPacketExtension.iceGeneration != IceUdpTransportPacketExtension.GENERATION_UNSPECIFIED + ) { + applyRemoteCredentialsToPendingRestart(pending, transportPacketExtension) + return + } + + val bundle = currentBundle + if (bundle.agent.state.isEstablished) { + logger.cdebug { "Connection already established" } + return + } + logger.cdebug { "Starting ICE connectivity establishment" } + + // Set the remote ufrag/password + bundle.stream.remoteUfrag = transportPacketExtension.ufrag + bundle.stream.remotePassword = transportPacketExtension.password + + // If ICE is running already, we try to update the checklists with the + // candidates. Note that this is a best effort. + val iceAgentStateIsRunning = IceProcessingState.RUNNING == bundle.agent.state + + val remoteCandidates = transportPacketExtension.getChildExtensionsOfType(CandidatePacketExtension::class.java) + if (iceAgentStateIsRunning && remoteCandidates.isEmpty()) { + logger.cdebug { + "Ignoring transport extensions with no candidates, " + + "the Agent is already running." + } + return + } + + val remoteCandidateCount = addRemoteCandidates(bundle, remoteCandidates, iceAgentStateIsRunning) + if (iceAgentStateIsRunning) { + when (remoteCandidateCount) { + 0 -> { + // XXX Effectively, the check above but realizing that all + // candidates were ignored: + // iceAgentStateIsRunning && candidates.isEmpty(). + } + + else -> bundle.component.updateRemoteCandidates() + } + } else if (remoteCandidateCount != 0) { + // Once again, because the ICE Agent does not support adding + // candidates after the connectivity establishment has been started + // and because multiple transport-info JingleIQs may be used to send + // the whole set of transport candidates from the remote peer to the + // local peer, do not really start the connectivity establishment + // until we have at least one remote candidate per ICE Component. + if (bundle.component.remoteCandidateCount > 0) { + logger.debug("Starting the agent with remote candidates.") + bundle.agent.startConnectivityEstablishment() + bundle.checksStarted.set(true) + } + } else if (bundle.stream.remoteUfragAndPasswordKnown()) { + // We don't have any remote candidates, but we already know the + // remote ufrag and password, so we can start ICE. + logger.debug("Starting the Agent without remote candidates.") + bundle.agent.startConnectivityEstablishment() + bundle.checksStarted.set(true) + } else { + logger.cdebug { "Not starting ICE, no ufrag and pwd yet. ${transportPacketExtension.toXML()}" } + } + } + + /** + * Starts an ICE restart, as explicitly requested by the peer (colibri2 ``). + * + * We create a second [Agent] with freshly rotated local credentials of our own (RFC 8445 section 9, so that + * checks belonging to different restart generations can be told apart by their credentials) and assign it + * the next [restartGeneration]. Connectivity checks are deliberately *not* started yet: we are the + * controlling agent, so our own checks need the peer's new remote credentials to build their USERNAME and + * MESSAGE-INTEGRITY, and those only arrive later (see [applyRemoteCredentialsToPendingRestart]). The new + * Agent does answer incoming checks in the meantime — ice4j starts its connectivity check server when the + * component is created and queues pre-RUNNING checks until we start. + * + * The established Agent keeps its selected pair and keeps sending throughout, so media is not interrupted + * (make-before-break); we only [cutOver] once the new Agent connects, and keep the old one alive for + * [IceConfig.restartTransitionWindow] after that so old-generation checks still in flight are answered. + * + * This assumes the peer's address changes, which is what a restart is for: ice4j's single port harvester + * demultiplexes on the remote address (see [cutOver]), so the new Agent is only reached by checks from an + * address the old Agent's socket is not already bound to. Checks from an unchanged address are delivered to + * the old Agent instead, which drops them because their local ufrag is not its own, and the new Agent — + * which has no signalled remote candidates and discovers the peer peer-reflexively — never connects. The + * restart is then abandoned and the established Agent is kept, which is the same outcome as any other + * failed restart. + * + * @return [IceRestartResult.STARTED] if a restart was started, in which case the caller must signal our new + * transport (rotated credentials plus the new `ice-generation`) back to the peer. + * [IceRestartResult.KEEP_EXISTING] if no restart was started but the established transport is still usable, + * in which case the caller must signal that transport unchanged. [IceRestartResult.UNAVAILABLE] if this + * bridge can not restart ICE at all, in which case the caller must signal no transport so that the peer + * falls back to a full re-invite. + */ + fun requestIceRestart(): IceRestartResult { + if (!running.get()) { + logger.warn("Can not restart ICE: the transport is not running.") + iceRestartsRejected.inc() + return IceRestartResult.UNAVAILABLE + } + if (!IceConfig.config.restartEnabled) { + logger.warn( + "Can not restart ICE: ICE restarts are disabled (videobridge.ice.restart.enabled=false)." + ) + iceRestartsRejected.inc() + return IceRestartResult.UNAVAILABLE + } + val timeout = IceConfig.config.restartTimeout + if (timeout.isZero || timeout.isNegative) { + // The restart would be abandoned before the peer could even answer, and the peer would be left + // holding credentials of an Agent we had already freed. Treat it the same as being disabled. + logger.warn( + "Can not restart ICE: videobridge.ice.restart.timeout is not positive ($timeout)." + ) + iceRestartsRejected.inc() + return IceRestartResult.UNAVAILABLE + } + if (!currentBundle.agent.state.isEstablished) { + // There is no established connectivity to preserve, so there is nothing for a make-before-break + // restart to do. The initial Agent is still gathering/checking and the peer should keep using it. + logger.warn( + "Not restarting ICE: the transport is not established yet " + + "(state=${currentBundle.agent.state}). Keeping the existing Agent." + ) + iceRestartsNotEstablished.inc() + return IceRestartResult.KEEP_EXISTING + } + + val newBundle: AgentBundle + val generation: Int + synchronized(restartLock) { + // A repeated request for a restart that has not started its checks yet (the peer has not answered + // with its own credentials) is answered with the bundle we already have. Clients do fire duplicate + // network-change events, and rolling a new generation here would free a bundle the peer may already + // be checking against, discard its progress and restart the timeout. + pendingBundle?.let { pending -> + if (!pending.checksStarted.get()) { + logger.info("An ICE restart is already pending and has not started checks: $pending.") + return IceRestartResult.STARTED + } + } + + // Create the new Agent before touching any state, so that a failure leaves the restart in flight + // (if there is one) alone rather than freeing a bundle the peer may already be checking against. + generation = restartGeneration + 1 + newBundle = createAgentBundle(generation) ?: run { + // A resource problem (no port to bind, most likely), so this bridge can not restart ICE at all + // right now. Escalate rather than keep the endpoint on a transport it just told us it can no + // longer reach: the established Agent still works, but its path is probably already dead. + logger.error("Can not restart ICE: failed to create the new Agent.") + iceRestartsAgentCreationFailed.inc() + return IceRestartResult.UNAVAILABLE + } + + // A newer restart supersedes one that has not connected yet. + pendingBundle?.let { superseded -> + logger.info("Superseding an in-flight ICE restart: $superseded") + iceRestartsSuperseded.inc() + TaskPools.IO_POOL.submit { superseded.free() } + } + restartTimeoutTask?.cancel(false) + restartTimeoutTask = null + + restartGeneration = generation + pendingBundle = newBundle + } + + logger.info( + "ICE restart requested (generation=$generation): created a pending Agent with local ufrag=" + + "${newBundle.agent.localUfrag} (current local ufrag=${currentBundle.agent.localUfrag}). " + + "Waiting for the peer's new remote credentials before starting connectivity checks." + ) + iceRestartsStarted.inc() + + val timeoutTask = TaskPools.SCHEDULED_POOL.schedule( + { abandonPendingRestart(newBundle, "it did not connect within $timeout") }, + timeout.toMillis(), + TimeUnit.MILLISECONDS + ) + synchronized(restartLock) { + // Unless the restart is already over, in which case there is nothing left to abandon. + if (pendingBundle === newBundle) { + restartTimeoutTask = timeoutTask + } else { + timeoutTask.cancel(false) + } + } + + return IceRestartResult.STARTED + } + + /** + * Handles a generation-tagged transport update that arrived while an ICE restart is pending. This is step + * two of a restart: the peer has applied the transport we returned from [requestIceRestart] and is now + * signalling its own new ICE credentials, tagged with the `ice-generation` we advertised. Apply them to the + * pending Agent and start its connectivity checks, which is the first moment we can — the checks we send as + * the controlling agent are authenticated with the peer's password. + */ + private fun applyRemoteCredentialsToPendingRestart( + pending: AgentBundle, + transportPacketExtension: IceUdpTransportPacketExtension + ) { + val generation = transportPacketExtension.iceGeneration + if (generation != pending.generation) { + // The peer answered a round we have since moved on from. This is the normal outcome of a superseded + // restart, not a rejected request, so it is not counted as one. + logger.info( + "Ignoring remote credentials for ICE generation $generation, the pending ICE restart is " + + "generation ${pending.generation}." + ) + return + } + + val ufrag = transportPacketExtension.ufrag + val password = transportPacketExtension.password + if (ufrag == null || password == null) { + logger.warn( + "Ignoring a transport update with no ufrag/pwd while ICE restart generation " + + "${pending.generation} is pending." + ) + return + } + + // Claim the round before applying anything. Everything that can make an update unusable has been + // checked by now (the credentials are present, they carry our generation, and the restart is still + // pending), so claiming here can not burn the round on an update we then reject. Applying first would + // instead let a repeated update overwrite the credentials of an Agent whose checks are already running: + // ice4j reads them live off the stream to sign its checks and to validate the peer's + // (ConnectivityCheckServer.getRemoteKey). + synchronized(restartLock) { + // Re-check under the lock: the restart may have been superseded or abandoned since we read it. + if (pendingBundle !== pending) { + logger.info( + "ICE restart generation ${pending.generation} is no longer pending, dropping the " + + "remote credentials that arrived for it." + ) + return + } + if (!pending.checksStarted.compareAndSet(false, true)) { + logger.info( + "Already started connectivity checks for ICE restart generation ${pending.generation}, " + + "ignoring a repeated transport update (remote ufrag=$ufrag)." + ) + return + } + pending.stream.remoteUfrag = ufrag + pending.stream.remotePassword = password + } + + // Add any signalled remote candidates. Normally there are none: clients do not signal candidates to the + // bridge and are discovered peer-reflexively from their incoming checks, which is also how a new address + // after a real network change is learned. + val remoteCandidates = transportPacketExtension.getChildExtensionsOfType(CandidatePacketExtension::class.java) + val remoteCandidateCount = addRemoteCandidates(pending, remoteCandidates, iceAgentIsRunning = false) + + logger.info( + "Applied the peer's new remote credentials to the pending ICE restart " + + "(generation=${pending.generation}, remote ufrag=$ufrag, remoteCandidates=" + + "$remoteCandidateCount, local ufrag=${pending.agent.localUfrag}), starting connectivity checks." + ) + pending.agent.startConnectivityEstablishment() + } + + /** + * Switches [currentBundle] over to [newBundle], which has just connected, and schedules the old bundle to be + * freed after [IceConfig.restartTransitionWindow]. + * + * Both bundles keep their sockets during that window, and both keep accepting whatever arrives on them: + * connectivity checks, and media. What routes a packet to one or the other is the peer's address, not the + * local ufrag it carries — ice4j's single port harvester looks the remote address up in its socket map + * first and only parses the ufrag for an address it has never seen (`AbstractUdpListener`). So the window + * is useful because the peer's old-generation checks come *from the old address*, which is still mapped to + * the old Agent's socket, and are answered there rather than dropped. See [requestIceRestart] for what this + * demultiplexing means for the new Agent. + */ + private fun cutOver(newBundle: AgentBundle) { + val transitionWindow = IceConfig.config.restartTransitionWindow + val keepOldBundle = !transitionWindow.isZero && !transitionWindow.isNegative + val oldBundle = synchronized(restartLock) { + if (pendingBundle !== newBundle) { + // Superseded or abandoned while it was connecting. + return + } + val old = currentBundle + currentBundle = newBundle + pendingBundle = null + restartTimeoutTask?.cancel(false) + restartTimeoutTask = null + + // A bundle still retiring from an earlier cutover has been superseded twice over by now. Free it + // rather than let two old Agents linger. + retiringBundle?.let { previous -> + transitionWindowTask?.cancel(false) + transitionWindowTask = null + TaskPools.IO_POOL.submit { previous.free() } + } + retiringBundle = if (keepOldBundle) old else null + old + } + + val elapsedMs = Duration.between(newBundle.createdAt, clock.instant()).toMillis() + logger.info( + "ICE restart (generation=${newBundle.generation}) connected after ${elapsedMs}ms, cutting over " + + "from local ufrag ${oldBundle.agent.localUfrag} to ${newBundle.agent.localUfrag}. Freeing the " + + "old Agent in $transitionWindow." + ) + iceRestartsSucceeded.inc() + + if (useUniquePort) { + // ice4j's push API only works with the single port harvester, so with unique ports we have to read + // from the new Agent's socket ourselves. The old bundle's reader exits when its socket is closed. + TaskPools.IO_POOL.submit { startReadingData(newBundle) } + } + + if (!keepOldBundle) { + TaskPools.IO_POOL.submit { oldBundle.free() } + return + } + + val freeTask = TaskPools.SCHEDULED_POOL.schedule( + { + logger.info( + "ICE restart (generation=${newBundle.generation}) transition window elapsed, freeing " + + "the old Agent with local ufrag ${oldBundle.agent.localUfrag}." + ) + synchronized(restartLock) { + if (retiringBundle === oldBundle) { + retiringBundle = null + transitionWindowTask = null + } + } + // Not inline: SCHEDULED_POOL is a single thread shared by the whole bridge, and + // Agent.free() shuts down the StunStack, closes sockets and joins threads. + TaskPools.IO_POOL.submit { oldBundle.free() } + }, + transitionWindow.toMillis(), + TimeUnit.MILLISECONDS + ) + synchronized(restartLock) { + // Unless the bundle is already gone, in which case there is nothing left to free. + if (retiringBundle === oldBundle) { + transitionWindowTask = freeTask + } else { + freeTask.cancel(false) + } + } + } + + /** + * Gives up on an ICE restart whose new Agent never connected (it failed, or the timeout elapsed), keeping + * the established Agent in place — the transport itself does not fail. A no-op if the restart already cut + * over or was superseded. + */ + private fun abandonPendingRestart(bundle: AgentBundle, reason: String) { + synchronized(restartLock) { + if (pendingBundle !== bundle) { + return + } + pendingBundle = null + restartTimeoutTask?.cancel(false) + restartTimeoutTask = null + } + val elapsedMs = Duration.between(bundle.createdAt, clock.instant()).toMillis() + logger.warn( + "Abandoning ICE restart (generation=${bundle.generation}, local ufrag=${bundle.agent.localUfrag}) " + + "after ${elapsedMs}ms: $reason. Keeping the established Agent with local ufrag " + + "${currentBundle.agent.localUfrag}." + ) + iceRestartsFailed.inc() + // Not inline: this can run on the Agent's own state-change notification thread. + TaskPools.IO_POOL.submit { bundle.free() } + } + + private fun startReadingData(bundle: AgentBundle) { + logger.cdebug { "Starting to read incoming data" } + val socket = bundle.component.selectedPair.iceSocketWrapper + val receiveBuf = ByteArray(1500) + val packet = DatagramPacket(receiveBuf, 0, receiveBuf.size) + var receivedTime: Instant + + while (running.get()) { + try { + socket.receive(packet) + receivedTime = clock.instant() + } catch (e: IOException) { + logger.warn("Stopping reader", e) + break + } + packetStats.numPacketsReceived.increment() + try { + val b = ByteBufferPool.getBuffer( + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + packet.length + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET + ) + System.arraycopy( + packet.data, + packet.offset, + b, + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + packet.length + ) + val buffer = Buffer(b, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, packet.length, receivedTime) + + incomingDataHandler?.dataReceived(buffer) ?: run { + logger.cdebug { "Data handler is null, dropping data" } + packetStats.numIncomingPacketsDroppedNoHandler.increment() + } + } catch (e: Throwable) { + logger.error("Uncaught exception processing packet", e) + } + } + logger.info("No longer running, stopped reading packets") + } + + /** + * Send data out via this transport + */ + fun send(data: ByteArray, off: Int, length: Int) { + if (running.get()) { + try { + // Always the established bundle: during a restart the new Agent is still running checks and we + // keep sending on the old selected pair until cutOver() swaps it in (make-before-break). + currentBundle.component.send(data, off, length) + packetStats.numPacketsSent.increment() + } catch (e: IOException) { + logger.error("Error sending packet", e) + throw RuntimeException() + } + } else { + packetStats.numOutgoingPacketsDroppedStopped.increment() + } + } + + fun stop() { + if (running.compareAndSet(true, false)) { + logger.info("Stopping") + val bundles = synchronized(restartLock) { + restartTimeoutTask?.cancel(false) + restartTimeoutTask = null + transitionWindowTask?.cancel(false) + transitionWindowTask = null + buildList { + add(currentBundle) + pendingBundle?.let { add(it) } + // A bundle inside its transition window would otherwise outlive the transport, until the + // task that frees it fires. + retiringBundle?.let { add(it) } + pendingBundle = null + retiringBundle = null + } + } + bundles.forEach { it.free() } + } + } + + fun getDebugState(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("keepAliveStrategy", IceConfig.config.keepAliveStrategy.toString()) + put("nominationStrategy", IceConfig.config.nominationStrategy.toString()) + put("advertisePrivateCandidates", IceConfig.config.advertisePrivateCandidates) + put("closed", !running.get()) + put("iceWriteable", iceWriteable.get()) + put("iceConnected", iceConnected.get()) + put("iceFailed", iceFailed.get()) + put("localUfrag", currentBundle.agent.localUfrag) + put("iceGeneration", currentBundle.generation) + val pending = pendingBundle + put("restartPending", pending != null) + if (pending != null) { + put("pendingIceGeneration", pending.generation) + put("pendingLocalUfrag", pending.agent.localUfrag) + put("pendingChecksStarted", pending.checksStarted.get()) + } + setAll(packetStats.toJson()) + } + + fun describe(pe: IceUdpTransportPacketExtension) { + if (!running.get()) { + logger.warn("Not describing, transport is not running") + } + // Prefer a pending restart's bundle: once we have rolled a new Agent, its credentials are the ones the + // peer must address its connectivity checks to (the USERNAME and MESSAGE-INTEGRITY of a check are + // built from the *peer's* view of our ufrag/password). Describing the old ones would send the peer + // checking against an Agent we are about to retire. With no restart in flight this is the established + // bundle, so the initial allocation path is unchanged. + val pending = pendingBundle + val bundle = pending ?: currentBundle + with(pe) { + password = bundle.agent.localPassword + ufrag = bundle.agent.localUfrag + if (bundle === pending) { + // Stamp the generation of the restart round these credentials belong to. The peer echoes it + // back on the transport-info carrying its own new credentials, which is how we match its + // answer to this bundle and discard answers from a round we have since moved on from. + // + // Only for a pending bundle: the generation marks a transport the peer must restart against, + // and describing the established bundle (which keeps the generation it connected with) means + // the opposite — use what is already there. The peer rejects a generation that is not newer + // than the last one it saw, so stamping one here would have it drop the transport. + iceGeneration = bundle.generation + } + bundle.component.localCandidates?.forEach { cand -> + cand.toCandidatePacketExtension(advertisePrivateAddresses)?.let { pe.addChildExtension(it) } + } + addChildExtension(IceRtcpmuxPacketExtension()) + } + } + + /** + * @return the number of network reachable remote candidates contained in + * the given list of candidates. + */ + private fun addRemoteCandidates( + bundle: AgentBundle, + remoteCandidates: List, + iceAgentIsRunning: Boolean + ): Int { + var remoteCandidateCount = 0 + // Sort the remote candidates (host < reflexive < relayed) in order to + // create first the host, then the reflexive, the relayed candidates and + // thus be able to set the relative-candidate matching the + // rel-addr/rel-port attribute. + remoteCandidates.sorted().forEach { candidate -> + // Is the remote candidate from the current generation of the + // iceAgent? + if (candidate.generation != bundle.agent.generation) { + return@forEach + } + if (candidate.ipNeedsResolution() && !IceConfig.config.resolveRemoteCandidates) { + logger.cdebug { "Ignoring remote candidate with non-literal address: ${candidate.ip}" } + return@forEach + } + val component = bundle.stream.getComponent(candidate.component) + val remoteCandidate = RemoteCandidate( + TransportAddress(candidate.ip, candidate.port, Transport.parse(candidate.protocol)), + component, + CandidateType.parse(candidate.type.toString()), + candidate.foundation, + candidate.priority.toLong(), + null + ) + // XXX IceTransport harvests host candidates only and the + // ICE Components utilize the UDP protocol/transport only at the + // time of this writing. The ice4j library will, of course, check + // the theoretical reachability between the local and the remote + // candidates. However, we would like (1) to not mess with a + // possibly running iceAgent and (2) to return a consistent return + // value. + if (!TransportUtils.canReach(component, remoteCandidate)) { + return@forEach + } + if (iceAgentIsRunning) { + component.addUpdateRemoteCandidates(remoteCandidate) + } else { + component.addRemoteCandidate(remoteCandidate) + } + remoteCandidateCount++ + } + + return remoteCandidateCount + } + + private fun iceStateChanged(bundle: AgentBundle, ev: PropertyChangeEvent) { + val oldState = ev.oldValue as IceProcessingState + val newState = ev.newValue as IceProcessingState + val transition = IceProcessingStateTransition(oldState, newState) + + val isPending = bundle === pendingBundle + val isCurrent = bundle === currentBundle + logger.debug( + "ICE state changed old=$oldState new=$newState for $bundle (pending=$isPending, current=$isCurrent)" + ) + + if (!isPending && !isCurrent) { + // A bundle we have already moved on from: either the pre-restart Agent inside its transition + // window, or one that was superseded and is being freed. It no longer speaks for the transport, so + // in particular it must not fail it. + logger.debug("Ignoring an ICE state change from a retired $bundle") + return + } + + when { + transition.completed() -> { + if (isPending) { + // A restart's new Agent connected: swap it in and retire the old one. + cutOver(bundle) + } else if (iceConnected.compareAndSet(false, true)) { + eventHandler?.connected() + if (useUniquePort) { + // ice4j's push API only works with the single port harvester. With unique ports we still need + // to read from the socket. + TaskPools.IO_POOL.submit { + startReadingData(bundle) + } + } + if (bundle.component.selectedPair.remoteCandidate.type == CandidateType.RELAYED_CANDIDATE || + bundle.component.selectedPair.localCandidate.type == CandidateType.RELAYED_CANDIDATE + ) { + iceSucceededRelayed.inc() + } + iceSucceeded.inc() + } + } + + transition.failed() -> { + if (isPending) { + // Only the restart failed. The established Agent is untouched, so keep using it rather + // than failing the whole transport. + abandonPendingRestart(bundle, "the new Agent failed to connect") + } else if (iceFailed.compareAndSet(false, true)) { + eventHandler?.failed() + Companion.iceFailed.inc() + } + } + } + } + + /** Update IceStatistics once an initial round-trip-time measurement is available. */ + fun updateStatsOnInitialRtt(rttMs: Double) { + val selectedPair = currentBundle.component.selectedPair + val localCandidate = selectedPair?.localCandidate ?: return + val harvesterName = if (localCandidate is HostCandidate) { + "host" + } else { + MappingCandidateHarvesters.findHarvesterForAddress(localCandidate.transportAddress)?.name ?: "other" + } + + IceStatistics.stats.add(harvesterName, rttMs) + } + + private fun iceStreamPairChanged(bundle: AgentBundle, ev: PropertyChangeEvent) { + // Only the bundle we actually send on speaks for the connection's liveness. A lingering old bundle (in + // its transition window) or a pending one still running checks must not report writeability or refresh + // consent on the transport's behalf. + if (bundle !== currentBundle) { + return + } + if (IceMediaStream.PROPERTY_PAIR_VALIDATED == ev.propertyName) { + if (iceWriteable.compareAndSet(false, true)) { + eventHandler?.writeable() + } + } else if (IceMediaStream.PROPERTY_PAIR_CONSENT_FRESHNESS_CHANGED == ev.propertyName) { + /* TODO: Currently ice4j only triggers this event for the selected + * pair, but should we double-check the pair anyway? + */ + val time = Instant.ofEpochMilli(ev.newValue as Long) + eventHandler?.consentUpdated(time) + } + } + + companion object { + fun appendHarvesters(iceAgent: Agent) { + Harvesters.INSTANCE.singlePortHarvesters.forEach(iceAgent::addCandidateHarvester) + } + + /** + * The total number of times an ICE Agent failed to establish + * connectivity. + */ + val iceFailed = VideobridgeMetricsContainer.instance.registerCounter( + "ice_failed", + "Number of times an ICE Agent failed to establish connectivity." + ) + + /** + * The total number of times an ICE Agent succeeded. + */ + val iceSucceeded = VideobridgeMetricsContainer.instance.registerCounter( + "ice_succeeded", + "Number of times an ICE Agent succeeded." + ) + + /** + * The total number of times an ICE Agent succeeded and the selected + * candidate pair included a relayed candidate. + */ + val iceSucceededRelayed = VideobridgeMetricsContainer.instance.registerCounter( + "ice_succeeded_relayed", + "Number of times an ICE Agent succeeded and the selected pair included a relayed candidate." + ) + + /** + * The total number of ICE restarts started (a new Agent was created for a peer-requested restart). + */ + val iceRestartsStarted = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_started", + "Number of ICE restarts started." + ) + + /** + * The total number of ICE restart requests this bridge could not honour at all: the feature is + * disabled, its timeout is misconfigured, or the transport is stopped. The endpoint is told to fall + * back to a full re-invite. + */ + val iceRestartsRejected = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_rejected", + "Number of ICE restart requests rejected because this bridge does not do ICE restarts (the " + + "feature is disabled or misconfigured, or the transport is stopped)." + ) + + /** + * The total number of ICE restart requests for a transport that had not connected yet, for which the + * existing Agent was kept. Ordinary traffic: the endpoint keeps a session that is still being + * negotiated, and nothing is lost. + */ + val iceRestartsNotEstablished = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_not_established", + "Number of ICE restart requests for a transport that was not established yet, for which the " + + "existing Agent was kept." + ) + + /** + * The total number of ICE restarts that could not create their new Agent, which means ice4j could not + * harvest or bind. Unlike the other two this indicates a problem with the bridge itself. + */ + val iceRestartsAgentCreationFailed = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_agent_creation_failed", + "Number of ICE restarts that failed because a new ICE Agent could not be created." + ) + + /** + * The total number of ICE restarts whose new Agent connected and was cut over to. + */ + val iceRestartsSucceeded = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_succeeded", + "Number of ICE restarts whose new Agent connected and was cut over to." + ) + + /** + * The total number of ICE restarts superseded by a newer one before they connected. Together with + * [iceRestartsSucceeded] and [iceRestartsFailed] this accounts for every restart in + * [iceRestartsStarted], except the ones still in flight. + */ + val iceRestartsSuperseded = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_superseded", + "Number of ICE restarts superseded by a newer restart before they connected." + ) + + /** + * The total number of ICE restarts abandoned because the new Agent failed or timed out. The + * established Agent is kept in these cases, so this does not imply the endpoint lost connectivity. + */ + val iceRestartsFailed = VideobridgeMetricsContainer.instance.registerCounter( + "ice_restarts_failed", + "Number of ICE restarts abandoned because the new Agent failed to connect." + ) + } + + private class PacketStats { + val numPacketsReceived = LongAdder() + val numIncomingPacketsDroppedNoHandler = LongAdder() + val numPacketsSent = LongAdder() + val numOutgoingPacketsDroppedStopped = LongAdder() + + fun toJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + put("num_packets_received", numPacketsReceived.sum()) + put("num_incoming_packets_dropped_no_handler", numIncomingPacketsDroppedNoHandler.sum()) + put("num_packets_sent", numPacketsSent.sum()) + put("num_outgoing_packets_dropped_stopped", numOutgoingPacketsDroppedStopped.sum()) + } + } + + interface IncomingDataHandler { + /** + * Notify the handler that data was received (contained + * within [data] at [offset] with [length]) at [receivedTime] + */ + fun dataReceived(buffer: Buffer) + } + + interface EventHandler { + /** + * Notify the event handler that it is possible to write to the ICE stack + */ + fun writeable() + + /** + * Notify the event handler that ICE connected successfully + */ + fun connected() + + /** + * Notify the event handler that ICE failed to connect + */ + fun failed() + + /** + * Notify the event handler that ICE consent was updated + */ + fun consentUpdated(time: Instant) + } +} + +/** + * Models a transition from one ICE state to another and provides convenience + * functions to test the transition. + */ +private data class IceProcessingStateTransition( + val oldState: IceProcessingState, + val newState: IceProcessingState +) { + // We should be using newState.isEstablished() here, but we see + // transitions from RUNNING to TERMINATED, which can happen if the Agent is + // free prior to being started, so we handle that case separately below. + fun completed(): Boolean = newState == IceProcessingState.COMPLETED + + fun failed(): Boolean = newState == IceProcessingState.FAILED || + (oldState == IceProcessingState.RUNNING && newState == IceProcessingState.TERMINATED) +} + +private fun IceMediaStream.remoteUfragAndPasswordKnown(): Boolean = remoteUfrag != null && remotePassword != null + +private fun CandidatePacketExtension.ipNeedsResolution(): Boolean = !InetAddresses.isInetAddress(ip) + +private fun TransportAddress.isPrivateAddress(): Boolean = address.isSiteLocalAddress || + /* 0xfc00::/7 */ + ((address is Inet6Address) && ((addressBytes[0].toInt() and 0xfe) == 0xfc)) + +private fun generateCandidateId(candidate: LocalCandidate): String = buildString { + append(java.lang.Long.toHexString(hashCode().toLong())) + append(java.lang.Long.toHexString(candidate.parentComponent.parentStream.parentAgent.hashCode().toLong())) + append(java.lang.Long.toHexString(candidate.parentComponent.parentStream.parentAgent.generation.toLong())) + append(java.lang.Long.toHexString(candidate.hashCode().toLong())) +} + +private fun LocalCandidate.toCandidatePacketExtension(advertisePrivateAddresses: Boolean): CandidatePacketExtension? { + if (transportAddress.isPrivateAddress() && + !advertisePrivateAddresses && + !IceConfig.config.advertisePrivateCandidates + ) { + return null + } + val cpe = IceCandidatePacketExtension() + cpe.component = parentComponent.componentID + cpe.foundation = foundation + cpe.generation = parentComponent.parentStream.parentAgent.generation + cpe.id = generateCandidateId(this) + cpe.network = 0 + cpe.setPriority(priority) + + cpe.protocol = transport.toString() + cpe.type = org.jitsi.xmpp.extensions.jingle.CandidateType.valueOf(type.toString()) + cpe.ip = transportAddress.hostAddress + cpe.port = transportAddress.port + + relatedAddress?.let { + if (!IceConfig.config.advertisePrivateCandidates && it.isPrivateAddress()) { + cpe.relAddr = "0.0.0.0" + cpe.relPort = 9 + } else { + cpe.relAddr = it.hostAddress + cpe.relPort = it.port + } + } + + return cpe +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/BooleanStateTimeTracker.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/BooleanStateTimeTracker.kt new file mode 100644 index 0000000..7fc706b --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/BooleanStateTimeTracker.kt @@ -0,0 +1,72 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.util + +import java.time.Clock +import java.time.Duration + +class BooleanStateTimeTracker( + initialState: Boolean = false, + private val clock: Clock = Clock.systemUTC() +) { + private var totalTimeOn: Duration = Duration.ofMillis(0) + private var totalTimeOff: Duration = Duration.ofMillis(0) + var state: Boolean = initialState + private set + private var mostRecentStateChange = clock.instant() + private val lock = Any() + + fun totalTimeOn(): Duration { + synchronized(lock) { + if (state) { + return totalTimeOn + Duration.between(mostRecentStateChange, clock.instant()) + } + return totalTimeOn + } + } + + fun totalTimeOff(): Duration { + synchronized(lock) { + if (!state) { + return totalTimeOff + Duration.between(mostRecentStateChange, clock.instant()) + } + return totalTimeOff + } + } + + fun setState(state: Boolean) = if (state) on() else off() + + fun on() { + synchronized(lock) { + if (!state) { + state = true + totalTimeOff += Duration.between(mostRecentStateChange, clock.instant()) + mostRecentStateChange = clock.instant() + } + } + } + + fun off() { + synchronized(lock) { + if (state) { + state = false + totalTimeOn += Duration.between(mostRecentStateChange, clock.instant()) + mostRecentStateChange = clock.instant() + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/MultiStreamCompatibility.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/MultiStreamCompatibility.kt new file mode 100644 index 0000000..dc560d4 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/MultiStreamCompatibility.kt @@ -0,0 +1,3 @@ +package org.jitsi.videobridge.util + +fun endpointIdToSourceName(endpointId: String): String = "$endpointId-v0" diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PacketUtils.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PacketUtils.kt new file mode 100644 index 0000000..7000528 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PacketUtils.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:JvmName("PacketUtils") + +package org.jitsi.videobridge.util + +import org.jitsi.rtp.extensions.unsigned.toPositiveInt + +private val DTLS_RANGE = 20..63 + +// TODO(brian): move this to RTP, and have the Packet extension +// function leverage it +fun looksLikeDtls(buf: ByteArray, off: Int, len: Int): Boolean { + if (len < 1) { + return false + } + return buf[off].toPositiveInt() in DTLS_RANGE +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt new file mode 100644 index 0000000..6f6a397 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/util/PayloadTypeUtil.kt @@ -0,0 +1,125 @@ +/* + * Copyright @ 2019-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.util + +import org.jitsi.nlj.format.AudioRedPayloadType +import org.jitsi.nlj.format.Av1PayloadType +import org.jitsi.nlj.format.H264PayloadType +import org.jitsi.nlj.format.OpusPayloadType +import org.jitsi.nlj.format.OtherAudioPayloadType +import org.jitsi.nlj.format.OtherVideoPayloadType +import org.jitsi.nlj.format.PayloadType +import org.jitsi.nlj.format.PayloadTypeEncoding.AV1 +import org.jitsi.nlj.format.PayloadTypeEncoding.Companion.createFrom +import org.jitsi.nlj.format.PayloadTypeEncoding.H264 +import org.jitsi.nlj.format.PayloadTypeEncoding.OPUS +import org.jitsi.nlj.format.PayloadTypeEncoding.OTHER +import org.jitsi.nlj.format.PayloadTypeEncoding.RED +import org.jitsi.nlj.format.PayloadTypeEncoding.RTX +import org.jitsi.nlj.format.PayloadTypeEncoding.TELEPHONE_EVENT +import org.jitsi.nlj.format.PayloadTypeEncoding.VP8 +import org.jitsi.nlj.format.PayloadTypeEncoding.VP9 +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.format.TelephoneEventPayloadType +import org.jitsi.nlj.format.VideoRedPayloadType +import org.jitsi.nlj.format.Vp8PayloadType +import org.jitsi.nlj.format.Vp9PayloadType +import org.jitsi.utils.MediaType +import org.jitsi.utils.MediaType.AUDIO +import org.jitsi.utils.MediaType.VIDEO +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.xmpp.extensions.jingle.PayloadTypePacketExtension +import java.util.concurrent.ConcurrentHashMap + +/** + * Utilities to deserialize [PayloadTypePacketExtension] into a + * {PayloadType}. This is currently in `jitsi-videobridge` in order to + * avoid adding the XML extensions as a dependency to + * `jitsi-media-transform`. + * + * @author Boris Grozev + * @author Brian Baldino + */ +class PayloadTypeUtil { + companion object { + private val logger: Logger = LoggerImpl(PayloadTypeUtil::class.java.name) + + /** + * Creates a [PayloadType] for the payload type described in the + * given [PayloadTypePacketExtension]. + * @param ext the XML extension which describes the payload type. + */ + @JvmStatic + fun create(ext: PayloadTypePacketExtension, mediaType: MediaType): PayloadType? { + val parameters: MutableMap = ConcurrentHashMap() + ext.parameters.forEach { parameter -> + // In SDP, format parameters don't necessarily come in name=value pairs (see e.g. the format used in + // RFC2198). However XEP-0167 requires a name and a value. Our SDP-to-Jingle implementation in + // lib-jitsi-meet translates a non-name=value SDP string into a parameter extension with a value but no + // name. Here we'll just ignore such parameters, because we don't currently support any and changing the + // implementation would be inconvenient (we store them mapped by name). + if (parameter.name != null) { + parameters[parameter.name] = parameter.value + } else { + logger.warn("Ignoring a format parameter with no name: " + parameter.toXML()) + } + } + + val rtcpFeedbackSet = ext.rtcpFeedbackTypeList.map { rtcpExtension -> + buildString { + append(rtcpExtension.feedbackType) + if (!rtcpExtension.feedbackSubtype.isNullOrBlank()) { + append(" ${rtcpExtension.feedbackSubtype}") + } + } + }.toSet() + + val id = ext.id.toByte() + val encoding = createFrom(ext.name) + val clockRate = ext.clockrate + val channels = ext.channels + + return when (encoding) { + VP8 -> Vp8PayloadType(id, parameters, rtcpFeedbackSet) + + VP9 -> Vp9PayloadType(id, parameters, rtcpFeedbackSet) + + AV1 -> Av1PayloadType(id, parameters, rtcpFeedbackSet) + + H264 -> H264PayloadType(id, parameters, rtcpFeedbackSet) + + RTX -> RtxPayloadType(id, parameters) + + OPUS -> OpusPayloadType(id, clockRate, channels, parameters) + + TELEPHONE_EVENT -> TelephoneEventPayloadType(id, clockRate, channels, parameters) + + RED -> when (mediaType) { + AUDIO -> AudioRedPayloadType(id, clockRate, channels, parameters) + VIDEO -> VideoRedPayloadType(id, clockRate, parameters, rtcpFeedbackSet) + else -> null + } + + OTHER -> when (mediaType) { + AUDIO -> OtherAudioPayloadType(id, ext.name, clockRate, channels, parameters) + VIDEO -> OtherVideoPayloadType(id, ext.name, clockRate, parameters) + else -> null + } + } + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt new file mode 100644 index 0000000..0ac6a7c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/version/JvbVersionService.kt @@ -0,0 +1,62 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.version + +import org.jitsi.utils.version.Version +import org.jitsi.utils.version.VersionImpl +import org.jitsi.utils.version.VersionService +import java.util.regex.Matcher +import java.util.regex.Pattern + +class JvbVersionService : VersionService { + private val versionString: String? by lazy { + JvbVersionService::class.java.`package`.implementationVersion + } + + private val version: Version by lazy { + parseVersionString(versionString) + } + + override fun getCurrentVersion(): Version = version + + override fun parseVersionString(version: String?): Version { + val matcher = Pattern.compile("(\\d*)\\.(\\d*)-(.*)").matcher(version ?: "").apply { find() } + val majorVersion = matcher.groupOrNull(1)?.toInt() ?: DEFAULT_MAJOR_VERSION + val minorVersion = matcher.groupOrNull(2)?.toInt() ?: DEFAULT_MINOR_VERSION + val buildId = matcher.groupOrNull(3) ?: defaultBuildId + + return VersionImpl( + "JVB", + majorVersion, + minorVersion, + buildId + ) + } + + companion object { + private const val DEFAULT_MAJOR_VERSION = 2 + private const val DEFAULT_MINOR_VERSION = 1 + private val defaultBuildId: String? = null + val instance = JvbVersionService() + } +} + +private fun Matcher.groupOrNull(groupId: Int): String? = try { + group(groupId) +} catch (t: Throwable) { + null +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketPassword.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketPassword.kt new file mode 100644 index 0000000..c27dc62 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketPassword.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2026 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.websocket + +import java.security.SecureRandom + +private val random = SecureRandom() +private const val CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +/** + * Generate a password which authenticates a colibri WebSocket. The password is used in a URL query parameter, so + * it only contains URL-safe characters. + * + * The peer receives the URL once, when we describe its transport, and re-dials that same URL each time the + * WebSocket reconnects. Because of this, the password must stay the same for the lifetime of the endpoint or + * relay which owns it. + */ +fun generateColibriWebSocketPassword(length: Int = 24) = + String(CharArray(length) { CHARS[random.nextInt(CHARS.length)] }) diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt new file mode 100644 index 0000000..bdce3e2 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketService.kt @@ -0,0 +1,123 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.websocket + +import org.eclipse.jetty.ee10.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletHolder +import org.eclipse.jetty.ee10.websocket.server.config.JettyWebSocketServletContainerInitializer +import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.relay.RelayConfig +import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig.Companion.config + +class ColibriWebSocketService( + webserverIsTls: Boolean +) { + private val baseUrls: List + private val relayBaseUrls: List + + init { + // We default to matching the protocol used by the local jetty + // instance, but we allow for the configuration via properties + // to override it since certain use-cases require it. + if (config.enabled) { + val useTls = config.useTls ?: webserverIsTls + val protocol = if (useTls) "wss" else "ws" + baseUrls = config.domains.map { + "$protocol://$it/$COLIBRI_WS_ENDPOINT/${config.serverId}" + } + relayBaseUrls = if (RelayConfig.config.enabled) { + config.relayDomains.map { + "$protocol://$it/$COLIBRI_RELAY_WS_ENDPOINT/${config.serverId}" + } + } else { + emptyList() + } + if (baseUrls.isEmpty() || relayBaseUrls.isEmpty()) { + logger.warn("Websockets enabled, but no domains specified: URLs=$baseUrls, Relay URLs=$relayBaseUrls") + } else { + logger.info("Base URL: $baseUrls Relay URL: $relayBaseUrls") + } + } else { + logger.info("WebSockets are not enabled") + baseUrls = emptyList() + relayBaseUrls = emptyList() + } + } + + /** + * Return a String representing the URL for an endpoint with ID [endpointId] in a conference with + * ID [conferenceId] to use to connect to the websocket with password [pwd] or null if the + * [ColibriWebSocketService] is not enabled. + */ + fun getColibriWebSocketUrls(conferenceId: String, endpointId: String, pwd: String): List { + // "wss://example.com/colibri-ws/server-id/conf-id/endpoint-id?pwd=123 + return baseUrls.map { "$it/$conferenceId/$endpointId?pwd=$pwd" } + } + + /** + * Return a String representing the URL for a relay with ID [relayId] in a conference with + * ID [conferenceId] to use to connect to the websocket with password [pwd] or null if the + * [ColibriWebSocketService] is not enabled. + */ + fun getColibriRelayWebSocketUrls(conferenceId: String, relayId: String, pwd: String): List { + // "wss://example.com/colibri-relay-ws/server-id/conf-id/relay-id?pwd=123 + return relayBaseUrls.map { "$it/$conferenceId/$relayId?pwd=$pwd" } + } + + fun registerServlet(servletContextHandler: ServletContextHandler, videobridge: Videobridge) { + if (config.enabled) { + logger.info("Registering servlet with baseUrls = $baseUrls, relayBaseUrls = $relayBaseUrls") + val holder = ServletHolder().apply { + servlet = ColibriWebSocketServlet(config.serverId, videobridge) + } + servletContextHandler.addServlet(holder, "/$COLIBRI_WS_ENDPOINT/*") + servletContextHandler.addServlet(holder, "/$COLIBRI_RELAY_WS_ENDPOINT/*") + /* TODO, if you want to register additional Websocket servlets elsewhere: + factor this out, it should only be called once. */ + JettyWebSocketServletContainerInitializer.configure(servletContextHandler, null) + } else { + logger.info("Disabled, not registering servlet") + } + } + + companion object { + private val logger = createLogger() + + /** + * The root path of the HTTP endpoint for COLIBRI WebSockets. + */ + private const val COLIBRI_WS_ENDPOINT = "colibri-ws" + + /** + * The root path of the HTTP endpoint for COLIBRI Relay-to-Relay (Octo) WebSockets. + */ + private const val COLIBRI_RELAY_WS_ENDPOINT = "colibri-relay-ws" + + /** + * Code elsewhere needs the value with the leading and trailing slashes, but when + * building URLs above, it's more readable to have the slashes be part of the String + * being built, so the separation is obvious. + */ + const val COLIBRI_WS_PATH = "/$COLIBRI_WS_ENDPOINT/" + + /** + * Similarly for relay + */ + const val COLIBRI_RELAY_WS_PATH = "/$COLIBRI_RELAY_WS_ENDPOINT/" + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketServiceSupplier.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketServiceSupplier.kt new file mode 100644 index 0000000..a3497fb --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/ColibriWebSocketServiceSupplier.kt @@ -0,0 +1,34 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.websocket + +import java.util.concurrent.atomic.AtomicReference +import java.util.function.Supplier + +class ColibriWebSocketServiceSupplier : Supplier { + private val colibriWebSocketService = AtomicReference(null) + + fun setColibriWebSocketService(colibriWebSocketService: ColibriWebSocketService) { + this.colibriWebSocketService.compareAndSet(null, colibriWebSocketService) + } + + override fun get(): ColibriWebSocketService? = colibriWebSocketService.get() +} + +val colibriWebSocketServiceSupplier = ColibriWebSocketServiceSupplier() + +fun singleton() = colibriWebSocketServiceSupplier diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt new file mode 100644 index 0000000..e5e1a0c --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfig.kt @@ -0,0 +1,136 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.websocket.config + +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.config +import org.jitsi.metaconfig.optionalconfig +import java.time.Duration + +class WebsocketServiceConfig private constructor() { + /** + * Whether [org.jitsi.videobridge.websocket.ColibriWebSocketService] is enabled + */ + val enabled: Boolean by config { + // The old property is named 'disable', while the new one + // is 'enable', so invert the old value + "org.jitsi.videobridge.rest.COLIBRI_WS_DISABLE" + .from(JitsiConfig.legacyConfig).transformedBy { !it } + "videobridge.websockets.enabled".from(JitsiConfig.newConfig) + } + + private val domainProp: String? by optionalconfig { + "org.jitsi.videobridge.rest.COLIBRI_WS_DOMAIN".from(JitsiConfig.legacyConfig) + "videobridge.websockets.domain".from(JitsiConfig.newConfig) + } + + private val domainsProp: List by config { + "videobridge.websockets.domains".from(JitsiConfig.newConfig) + } + + /** + * The list of domains to advertise (advertise a separate URL for each domain). + * Constructed at get() time to allow underlying config changes. + */ + val domains: List + get() = domainsProp.toMutableList().apply { + domainProp?.let { + if (!contains(it)) { + add(0, it) + } + } + } + + private val relayDomainProp: String? by optionalconfig { + "videobridge.websockets.relay-domain".from(JitsiConfig.newConfig) + } + + private val relayDomainsProp: List by config { + "videobridge.websockets.relay-domains".from(JitsiConfig.newConfig) + } + + /** + * The list of domains to advertise (advertise a separate URL for each domain) for relays. + * Constructed at get() time to allow underlying config changes. + */ + val relayDomains: List + get() { + val relayDomainProp = relayDomainProp + return if (relayDomainProp != null) { + relayDomainsProp.toMutableList().apply { + if (!contains(relayDomainProp)) { + add(0, relayDomainProp) + } + } + } else if (relayDomainsProp.isNotEmpty()) { + relayDomainsProp.toList() + } else { + domains + } + } + + /** + * Whether the "wss" or "ws" protocol should be used for websockets + */ + val useTls: Boolean? by optionalconfig { + onlyIf("Websockets are enabled", ::enabled) { + "org.jitsi.videobridge.rest.COLIBRI_WS_TLS".from(JitsiConfig.legacyConfig) + "videobridge.websockets.tls".from(JitsiConfig.newConfig) + } + } + + /** + * Whether compression (permessage-deflate) should be used for websockets + */ + private val enableCompression: Boolean by config { + onlyIf("Websockets are enabled", ::enabled) { + "videobridge.websockets.enable-compression".from(JitsiConfig.newConfig) + } + } + + fun shouldEnableCompression() = enableCompression + + /** + * The server ID used in URLs advertised for COLIBRI WebSockets. + */ + val serverId: String by config { + onlyIf("Websockets are enabled", ::enabled) { + "org.jitsi.videobridge.rest.COLIBRI_WS_SERVER_ID".from(JitsiConfig.legacyConfig) + "videobridge.websockets.server-id".from(JitsiConfig.newConfig) + } + } + + /** Whether keepalive pings are enabled */ + val sendKeepalivePings: Boolean by config { + "videobridge.websockets.send-keepalive-pings".from(JitsiConfig.newConfig) + } + + /** The time interval for keepalive pings */ + val keepalivePingInterval: Duration by config { + "videobridge.websockets.keepalive-ping-interval".from(JitsiConfig.newConfig) + } + + /** The time interval for websocket timeouts */ + val idleTimeout: Duration by config { + "videobridge.websockets.idle-timeout".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = WebsocketServiceConfig() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt new file mode 100644 index 0000000..436cc96 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/Smack.kt @@ -0,0 +1,77 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.xmpp + +import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig +import org.jitsi.xmpp.extensions.DefaultPacketExtensionProvider +import org.jitsi.xmpp.extensions.colibri.ColibriStatsIqProvider +import org.jitsi.xmpp.extensions.colibri.ForcefulShutdownIqProvider +import org.jitsi.xmpp.extensions.colibri.GracefulShutdownIqProvider +import org.jitsi.xmpp.extensions.colibri2.IqProviderUtils +import org.jitsi.xmpp.extensions.health.HealthCheckIQProvider +import org.jitsi.xmpp.extensions.jingle.DtlsFingerprintPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceCandidatePacketExtension +import org.jitsi.xmpp.extensions.jingle.IceRtcpmuxPacketExtension +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import org.jivesoftware.smack.provider.ProviderManager +import org.jxmpp.stringprep.XmppStringPrepUtil + +object Smack { + fun initialize() { + org.jitsi.xmpp.Smack.initialize() + + XmppStringPrepUtil.setMaxCacheSizes(XmppClientConnectionConfig.config.jidCacheSize) + + registerProviders() + } + + private fun registerProviders() { + // + ForcefulShutdownIqProvider.registerIQProvider() + // + GracefulShutdownIqProvider.registerIQProvider() + // + ColibriStatsIqProvider() // registers itself with Smack + // ICE-UDP + ProviderManager.addExtensionProvider( + IceUdpTransportPacketExtension.ELEMENT, + IceUdpTransportPacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(IceUdpTransportPacketExtension::class.java) + ) + // ICE-UDP + ProviderManager.addExtensionProvider( + IceCandidatePacketExtension.ELEMENT, + IceCandidatePacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(IceCandidatePacketExtension::class.java) + ) + // ICE + ProviderManager.addExtensionProvider( + IceRtcpmuxPacketExtension.ELEMENT, + IceRtcpmuxPacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(IceRtcpmuxPacketExtension::class.java) + ) + // DTLS-SRTP + ProviderManager.addExtensionProvider( + DtlsFingerprintPacketExtension.ELEMENT, + DtlsFingerprintPacketExtension.NAMESPACE, + DefaultPacketExtensionProvider(DtlsFingerprintPacketExtension::class.java) + ) + // Health-check + HealthCheckIQProvider.registerIQProvider() + // Colibri2 + IqProviderUtils.registerProviders() + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt new file mode 100644 index 0000000..ea41bad --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/XmppConnection.kt @@ -0,0 +1,345 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.xmpp + +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import org.jitsi.nlj.stats.DelayStats +import org.jitsi.utils.logging2.cdebug +import org.jitsi.utils.logging2.createLogger +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer +import org.jitsi.videobridge.xmpp.config.XmppClientConnectionConfig.Companion.config +import org.jitsi.xmpp.extensions.colibri.ForcefulShutdownIQ +import org.jitsi.xmpp.extensions.colibri.GracefulShutdownIQ +import org.jitsi.xmpp.extensions.colibri2.ConferenceModifyIQ +import org.jitsi.xmpp.extensions.health.HealthCheckIQ +import org.jitsi.xmpp.mucclient.ConnectionStateListener +import org.jitsi.xmpp.mucclient.IQListener +import org.jitsi.xmpp.mucclient.MucClient +import org.jitsi.xmpp.mucclient.MucClientConfiguration +import org.jitsi.xmpp.mucclient.MucClientManager +import org.jitsi.xmpp.util.createError +import org.jivesoftware.smack.packet.ExtensionElement +import org.jivesoftware.smack.packet.IQ +import org.jivesoftware.smack.packet.StanzaError +import org.jivesoftware.smackx.iqversion.packet.Version +import java.util.concurrent.atomic.AtomicBoolean + +/** + * The XMPP client connection for the videobridge + */ +class XmppConnection : IQListener { + private val logger = createLogger() + + /** + * The [MucClientManager] which manages the XMPP user connections and the MUCs. + */ + val mucClientManager = MucClientManager(FEATURES) + + private val running = AtomicBoolean(false) + + var eventHandler: EventHandler? = null + + fun start() { + if (running.compareAndSet(false, true)) { + mucClientManager.apply { + registerIQ(HealthCheckIQ()) + registerIQ(Version()) + registerIQ(ForcefulShutdownIQ()) + registerIQ(GracefulShutdownIQ()) + registerIQ(ConferenceModifyIQ.ELEMENT, ConferenceModifyIQ.NAMESPACE, false) + setIQListener(this@XmppConnection) + addConnectionStateListener(object : ConnectionStateListener { + override fun connected(mucClient: MucClient) {} + override fun closed(mucClient: MucClient) = VideobridgeMetrics.xmppDisconnects.inc() + override fun closedOnError(mucClient: MucClient) = VideobridgeMetrics.xmppDisconnects.inc() + override fun reconnecting(mucClient: MucClient) {} + override fun reconnectionFailed(mucClient: MucClient) {} + override fun pingFailed(mucClient: MucClient) {} + }) + } + + config.clientConfigs.forEach { cfg -> mucClientManager.addMucClient(cfg) } + org.jitsi.videobridge.metrics.Metrics.metricsUpdater.addUpdateTask { updateMetrics() } + } else { + logger.info("Already started") + } + } + + fun stop() { + if (running.compareAndSet(true, false)) { + mucClientManager.stop() + } + } + + fun updateMetrics() { + mucClientsConfigured.set(mucClientManager.clientCount) + mucClientsConnected.set(mucClientManager.clientConnectedCount) + mucsConfigured.set(mucClientManager.mucCount) + mucsJoined.set(mucClientManager.mucJoinedCount) + } + + /** + * Adds an [ExtensionElement] to our presence, and removes any other + * extensions with the same element name and namespace, if any exists. + * @param extension the extension to add. + */ + fun setPresenceExtension(extension: ExtensionElement) { + mucClientManager.setPresenceExtension(extension) + } + + /** + * Adds a new [MucClient] with configuration described in JSON. + * @param jsonObject the JSON which describes the configuration of the + * client. + *

+ *

{@code
+     * The expected JSON format is:
+     * {
+     *     "id": "muc-client-id",
+     *     "key": "value"
+     * }
+     * }
+ * The [key, value] pairs are interpreted as property names and values to + * set for the client's configuration (see {@link MucClientConfiguration}). + * + * @return {@code true} if the request was successful (i.e. the JSON + * is in the required format and either a new {@link MucClient} was added + * or a client with the same ID already existed). + */ + fun addMucClient(jsonObject: ObjectNode): Boolean { + val idNode = jsonObject.get("id") + if (idNode == null || !idNode.isTextual) { + return false + } + val config = MucClientConfiguration(idNode.asText()) + jsonObject.fields().forEach { (key, value) -> + if (value.isTextual && key != "id") { + config.setProperty(key, value.asText()) + } + } + if (!config.isComplete) { + logger.info("Not adding a MucClient, configuration incomplete.") + return false + } + + mucClientManager.getMucClient(config.id)?.let { existingMucClient -> + if (!existingMucClient.config.matches(config)) { + logger.warn( + "Config for ${config.id} has changed, removing the old MucClient. Existing " + + "config=${existingMucClient.config}, new config=$config" + ) + mucClientManager.removeMucClient(config.id) + } else { + logger.info( + "Ignoring request to add a MucClient that matches an existing one. Existing " + + "config=${existingMucClient.config}, new config=$config" + ) + } + } + + logger.info("Adding MucClient for ${config.id}") + mucClientManager.addMucClient(config) + return true + } + + private fun MucClientConfiguration.matches(other: MucClientConfiguration) = hostname == other.hostname && + port == other.port && + domain == other.domain && + username == other.username + + /** + * Returns ids of [MucClient] that have been added. + * @return JSON string of the list of ids + */ + fun getMucClientIds(): String { + val arr: ArrayNode = JsonNodeFactory.instance.arrayNode() + mucClientManager.mucClientIds.forEach { arr.add(it) } + return arr.toString() + } + + /** + * Removes a {@link MucClient} with an ID described in JSON. + * @param jsonObject the JSON which contains the ID of the client to remove. + *

+ *
{@code
+     * The expected JSON format is:
+     * {
+     *     "id": "muc-client-id",
+     * }
+     * }
+ * + * @return {@code true} if the MUC client with the specified ID was removed. + * Otherwise (if instance has not been initialized, if the JSON is not in + * the expected format, or if no MUC client with the specified ID exists), + * returns {@code false}. + */ + fun removeMucClient(jsonObject: ObjectNode): Boolean { + val idNode = jsonObject.get("id") + if (idNode == null || !idNode.isTextual) { + logger.info("Invalid ID: $idNode") + return false + } + logger.info("Removing muc client ${idNode.asText()}") + return mucClientManager.removeMucClient(idNode.asText()) + } + + /** + * Process an incoming IQ + */ + override fun handleIq(iq: IQ?, mucClient: MucClient): IQ? { + if (iq == null) { + return null + } + // colibri2 requests are logged at the conference level. + if (iq !is ConferenceModifyIQ) { + logger.cdebug { "RECV: ${iq.toXML()}" } + } + + return when (iq.type) { + IQ.Type.get, IQ.Type.set -> handleIqRequest(iq, mucClient)?.also { + logger.cdebug { "SENT: ${it.toXML()}" } + } + + else -> null + } + } + + private fun handleIqRequest(iq: IQ, mucClient: MucClient): IQ? { + val handler = eventHandler ?: return createError( + iq, + StanzaError.Condition.service_unavailable, + "Service unavailable" + ) + val response = when (iq) { + is Version -> measureDelay(versionDelayStats, { iq.toXML() }) { + handler.versionIqReceived(iq) + } + + is ConferenceModifyIQ -> { + // Colibri IQs are handled async. + handler.colibriRequestReceived( + ColibriRequest(iq, colibriDelayStats, colibriProcessingDelayStats) { response -> + response.setResponseTo(iq) + mucClient.sendStanza(response) + } + ) + null + } + + is HealthCheckIQ -> measureDelay(healthDelayStats, { iq.toXML() }) { + handler.healthCheckIqReceived(iq) + } + + else -> createError( + iq, + StanzaError.Condition.service_unavailable, + "Unsupported IQ request ${iq.childElementName}" + ) + } + + return response?.setResponseTo(iq) + } + + private fun IQ.setResponseTo(request: IQ) = apply { + from = request.to + stanzaId = request.stanzaId + to = request.from + } + + private fun measureDelay(delayStats: DelayStats, context: () -> CharSequence, block: () -> T): T { + val start = System.currentTimeMillis() + val result = block() + val delayMs = System.currentTimeMillis() - start + delayStats.addDelay(delayMs) + if (delayMs > 100) { + logger.warn("Took $delayMs ms to handle IQ: ${context()}") + } + return result + } + + interface EventHandler { + fun colibriRequestReceived(request: ColibriRequest) + fun versionIqReceived(iq: Version): IQ + fun healthCheckIqReceived(iq: HealthCheckIQ): IQ + } + + data class ColibriRequest( + /** + * The IQ which was received. + */ + val request: ConferenceModifyIQ, + /** + * The [DelayStats] instance which is to be updated with the total time it took to handle the request + * (including queueing delay). + */ + val totalDelayStats: DelayStats, + /** + * The [DelayStats] instance which is to be updated with the time it took to process the request once it was + * picked from the queue. + */ + val processingDelayStats: DelayStats, + val receiveTime: Long = System.currentTimeMillis(), + /** + * The callback to use to send the response. + */ + val callback: (IQ) -> Unit + ) + + companion object { + private val FEATURES = arrayOf( + HealthCheckIQ.NAMESPACE, + "urn:xmpp:jingle:apps:dtls:0", + "urn:xmpp:jingle:transports:ice-udp:1", + Version.NAMESPACE + ) + + private val delayThresholds = listOf(0, 5, 50, 100, 1000, Long.MAX_VALUE) + + private val colibriProcessingDelayStats = DelayStats(delayThresholds) + private val colibriDelayStats = DelayStats(delayThresholds) + private val healthDelayStats = DelayStats(delayThresholds) + private val versionDelayStats = DelayStats(delayThresholds) + + val mucClientsConfigured = VideobridgeMetricsContainer.instance.registerLongGauge( + "muc_clients_configured", + "Number of MUC clients that are configured." + ) + val mucClientsConnected = VideobridgeMetricsContainer.instance.registerLongGauge( + "muc_clients_connected", + "Number of MUC clients that are connected." + ) + val mucsConfigured = VideobridgeMetricsContainer.instance.registerLongGauge( + "mucs_connected", + "Number of MUCs that are configured." + ) + val mucsJoined = VideobridgeMetricsContainer.instance.registerLongGauge( + "mucs_joined", + "Number of MUCs that are joined." + ) + + @JvmStatic + fun getStatsJson(): ObjectNode = JsonNodeFactory.instance.objectNode().apply { + set("colibri", colibriDelayStats.toJson()) + set("colibri_processing", colibriProcessingDelayStats.toJson()) + set("health", healthDelayStats.toJson()) + set("version", versionDelayStats.toJson()) + } + } +} diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt new file mode 100644 index 0000000..8830233 --- /dev/null +++ b/jvb/src/main/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfig.kt @@ -0,0 +1,99 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.xmpp.config + +import com.typesafe.config.ConfigObject +import com.typesafe.config.ConfigValue +import org.jitsi.config.JitsiConfig +import org.jitsi.metaconfig.ConfigException +import org.jitsi.metaconfig.config +import org.jitsi.xmpp.mucclient.MucClientConfiguration +import java.time.Duration + +class XmppClientConnectionConfig private constructor() { + val clientConfigs: List by config { + "org.jitsi.videobridge.xmpp.user." + .from(JitsiConfig.legacyConfig) + .convertFrom> { propsMap -> + MucClientConfiguration.loadFromMap(propsMap, "org.jitsi.videobridge.xmpp.user.", true) + .toList() + .apply { forEach { it.applyDefaultIqHandlerMode() } } + .takeIf { it.isNotEmpty() } ?: throw ConfigException.UnableToRetrieve.NotFound("no configs found") + } + "videobridge.apis.xmpp-client.configs".from(JitsiConfig.newConfig) + .convertFrom { cfg -> + cfg.entries + .map { it.toMucClientConfiguration() } + .filter { it.isComplete } + } + } + + val presenceInterval: Duration by config { + "videobridge.apis.xmpp-client.presence-interval".from(JitsiConfig.newConfig) + } + + /** + * Whether to filter the statistics. + */ + val statsFilterEnabled: Boolean by config { + "videobridge.apis.xmpp-client.stats-filter.enabled".from(JitsiConfig.newConfig) + } + + /** + * The set of statistics to send, when filtering is enabled. + */ + val statsWhitelist: List by config { + "videobridge.apis.xmpp-client.stats-filter.whitelist".from(JitsiConfig.newConfig) + } + + /** + * The size to set for Smack's JID cache + */ + val jidCacheSize: Int by config { + "videobridge.apis.xmpp-client.jid-cache-size".from(JitsiConfig.newConfig) + } + + companion object { + @JvmField + val config = XmppClientConnectionConfig() + } +} + +/** + * We want the bridge to default to using "sync" as the IQ handler mode (unless the config actually overrides it). + */ +private fun MucClientConfiguration.applyDefaultIqHandlerMode() { + if (this.iqHandlerMode == null) { + this.iqHandlerMode = "sync" + } +} + +private fun MutableMap.MutableEntry.toMucClientConfiguration(): MucClientConfiguration { + val config = MucClientConfiguration(this.key) + (value as? ConfigObject)?.let { + it.forEach { (propName, propValue) -> + config.setProperty(propName, propValue.unwrapped().toString()) + } + } ?: run { + throw Exception( + "Invalid muc client configuration. " + + "Expected type ConfigObject but got ${value.unwrapped()::class.java}" + ) + } + + return config.also { it.applyDefaultIqHandlerMode() } +} diff --git a/jvb/src/main/resources/application.conf b/jvb/src/main/resources/application.conf new file mode 100644 index 0000000..09d391d --- /dev/null +++ b/jvb/src/main/resources/application.conf @@ -0,0 +1,23 @@ +# This file contains overrides for libraries JVB uses +ice4j { + consent-freshness { + // Sends "consent freshness" check every 5 seconds. + interval = 5 seconds + // Retry max 5 times which will take up to 2500ms, that is before the next "consent freshness" transaction starts. + max-retransmissions = 5 + } + harvest { + // In the majority of use-cases the clients which connect to Jitsi Videobridge are not in the same network, so we + // don't need to advertise link-local addresses. + use-link-local-addresses = false + + udp { + // Configure the receive buffer size for the sockets used for the + // single-port mode to be 10MB. + receive-buffer-size = 10485760 + + use-dynamic-ports = false + } + } + redact-remote-addresses = ${videobridge.redact-remote-addresses} +} diff --git a/jvb/src/main/resources/reference.conf b/jvb/src/main/resources/reference.conf new file mode 100644 index 0000000..6d0d29d --- /dev/null +++ b/jvb/src/main/resources/reference.conf @@ -0,0 +1,464 @@ +videobridge { + entity-expiration { + # If an entity has no activity after this timeout, it is expired + timeout = 1 minute + + # The interval at which the videobridge will check for expired entities + check-interval = ${videobridge.entity-expiration.timeout} + } + health { + # The interval between health checks + interval = 1 minute + + # The timeout for a health check. This needs to be higher than [interval], otherwise health checks timeout because + # none were scheduled. + timeout = 90 seconds + + # If performing a health check takes longer than this, it is considered unsuccessful. + max-check-duration = 3 seconds + + # Whether or not health check failures should be 'sticky' + # (i.e. once the bridge becomes unhealthy, it will never + # go back to a healthy state) + sticky-failures = false + + # If enabled, health checks fail when there are no valid addresses available for ICE. Here "valid" means not + # site local, link local or loopback. + require-valid-address = true + + # If enabled, health checks will fail when a STUN mapping harvester is configured, but fails to get a mapping (even + # if there is a valid address available through other mappings or locally). + require-stun = false + } + ep-connection-status { + # How long we'll wait for an endpoint to *start* sending + # data before we consider it 'inactive' + first-transfer-timeout = 15 seconds + + # How long an endpoint can be 'inactive' before it will + # be considered disconnected + max-inactivity-limit = 8 seconds + + # How often we check endpoint's connectivity status + check-interval = 500 milliseconds + } + # Congestion Control + cc { + bwe-change-threshold = 0.15 + # The default constraint to use if the receiver signals ReceiverConstraints with missing defaultConstraints. + default-max-height-px = 180 + # The constraint to use for the initial allocation, before the receiver has signaled ReceiverConstraints. + # This should be set to non-zero only if you have clients that do not always send ReceiverConstraints. + # Otherwise this leads to a race condition where video media is sent before clients are ready to receive + # it, resulting in excessive PLIs and delays displaying video. + initial-max-height-px = 0 + onstage-preferred-height-px = 360 + onstage-preferred-framerate = 30 + // Whether the bridge is allowed to oversend (send the lowest layer regardless of BWE) for on-stage endpoints. If + // allowed, it's only used when an endpoint is screensharing. + allow-oversend-onstage = true + // The maximum bitrate by which the estimation will be exceeded when oversending (if oversending is allowed). + max-oversend-bitrate = 500 kbps + trust-bwe = true + + # How often we check to send probing data + padding-period = 15ms + + # How often we'll run bandwidth re-allocation. This is in addition to re-allocating when otherwise required (e.g. + # a new endpoint joins or the available layers change). + max-time-between-calculations = 5 seconds + + # A JVB-wide last-n value, observed by all endpoints. Endpoints + # will take the minimum of their setting and this one (-1 implies + # no last-n limit) + jvb-last-n = -1 + + // The initial value for the client-configured last-n setting. This applies before it is overriden by colibri or + // data channel signaling (-1 implies no last-n limit). + initial-last-n = -1 + + # If set allows receivers to override bandwidth estimation (BWE) with a specific value signaled over the bridge + # channel (limited to the configured value). If not set, receivers are not allowed to override BWE. + // assumed-bandwidth-limit = 10 Mbps + + # Whether to use the target bitrate signaled in the VLA extension for allocation. When disabled we use the measured + # bitrate instead (preserving previous behavior). + use-vla-target-bitrate = false + + # How long after first media to ignore bandwidth estimation for the purposes of allocation. Needed + # for bwe algorithms that ramp up slowly; should be set to zero if this isn't a problem. + + # By default this value will depend on the BWE algorithm chosen based on values set under + # jmt.bwe.estimator, but they can also be specified explicitly. + + # initial-ignore-bwe-period = 10 seconds + } + # Whether to indicate support for cryptex header extension encryption (RFC 9335) + cryptex { + # To endpoints + endpoint = false + # To other relays + relay = false + } + # The APIs by which the JVB can be controlled + apis { + xmpp-client { + # The interval at which presence is published in the configured MUCs. + presence-interval = ${videobridge.stats.interval} + + # Controls which statistics are sent. + stats-filter { + # Whether to filter the statistics. + # If true, send whitelisted keys only. If false, send all statistics. + enabled = false + + # Which statistics to send, when filter is enabled. + # Ignored if filter is disabled. + whitelist = ["average_participant_stress", "colibri2", "current_timestamp", "drain", "graceful_shutdown", + "healthy", "region", "relay_id", "release", "shutting_down", "stress_level", "version"] + } + + # The size of the Smack JID cache + jid-cache-size = 1000 + + configs { + # example-connection-id { + # For the properties which should be + # filled out here, see MucClientConfiguration + # } + } + } + # The COLIBRI REST API + rest { + enabled = false + } + } + # Configuration of the different REST APIs. + # Note that the COLIBRI REST API is configured under videobridge.apis.rest instead. + rest { + debug { + enabled = true + } + drain { + enabled = true + } + health { + enabled = true + } + prometheus { + enabled = true + } + rtcstats { + enabled = true + } + shutdown { + # Note that the shutdown API requires the COLIBRI API to also be enabled. + enabled = false + } + version { + enabled = true + } + } + http-servers { + # The HTTP server which hosts services intended for 'public' use + # (e.g. websockets for the bridge channel connection) + public { + # See JettyBundleActivatorConfig in Jicoco for values + port = -1 + tls-port = -1 + } + # The HTTP server which hosts services intended for 'private' use + # (e.g. health or debug stats) + private { + # See JettyBundleActivatorConfig in Jicoco for values + host = 127.0.0.1 + } + } + relay { + # Whether or not relays (octo) are enabled + enabled = false + + # A string denoting the 'region' of this JVB. This region will be used by Jicofo in the selection of a bridge for + # a client by comparing it to the client's region. + # Must be set when 'enabled' is true. + #region = "us-west-1" + + # The unique identifier of the jitsi-videobridge instance as a relay. + # Must be set when 'enabled' is true. + #relay-id = "jvb-1234" + } + load-management { + # Whether or not the reducer will be enabled to take actions to mitigate load + reducer-enabled = false + load-measurements { + packet-rate { + # The packet rate at which we'll consider the bridge overloaded + load-threshold = 50000 + # The packet rate at which we'll consider the bridge 'underloaded' enough + # to start recovery + recovery-threshold = 40000 + } + cpu-usage { + load-threshold = 0.9 + recovery-threshold = 0.72 + # If enabled, CPU steal will be detected by reading /proc/stat (supported only on linux) and will be added to + # the CPU usage metric before stress is calculated. + detect-cpu-steal = true + } + + # Which of the available measurements to use, either "packet-rate" or "cpu-usage". This determines how the stress + # metric is calculated, which is just the measured load divided by the configured load-threshold. + # The packet-rate metric load-threshold needs to be configured based on the machine that the bridge is running on, + # while the cpu-usage metric should behave well with the default configuration on any machine. + load-measurement = "cpu-usage" + } + load-reducers { + last-n { + # The factor by which we'll reduce the current last-n when trying to reduce load + reduction-scale = .75 + # The factor by which we'll increase the current last-n when trying to recover + recover-scale = 1.25 + # The minimum time in between runs of the last-n reducer to reduce or recover from load + impact-time = 1 minute + # The lowest value we'll set for last-n + minimum-last-n-value = 1 + # The highest last-n value we'll enforce. Once the enforced last-n exceeds this value + # we'll remove the limit entirely + maximum-enforced-last-n-value = 40 + } + } + // LastN limits based on conference size. Maps a conference size to the maximum number of streams that will be + // forwarded to endpoints in a conference of that size or larger (up until the next entry). + conference-last-n-limits { + // With these example values conferences with size<20 endpoints can have arbitrary lastN, conferences with + // 20 < size <= 29 can have lastN at most 20, conferences 30 < size <= 50 can have lastN at most 15. + #20 = 20, + #30 = 15, + #50 = 10, + #90 = 2 + } + // An estimation of the stress an endpoint adds to the bridge when ramped up. This is communicated to jicofo + // together with the stress level, where it is used to calculate an augmented stress value for the bridge, which + // takes into account recently added endpoints: + // augmented_stress = jvb_reported_stress + number_recently_added_participants * average-participant-stress + average-participant-stress = 0.01 + } + sctp { + // Whether SCTP data channels are enabled. + enabled = true + + // Maximum number of data channels per connection + max-channels = 1 + } + stats { + // The interval at which stats are gathered. + interval = 5 seconds + + // Statistics about the transit time of RTP/RTCP packets. Note that the collection code for the JSON and Prometheus + // outputs is different, and each has a slight performance impact, so the format(s) that are not needed should be + // kept disabled. + transit-time { + // Enable collection of transit time stats in JSON format. Available through /debug/jvb/stats/transit-time + enable-json = true + // Enable collection of transit time stats in Prometheus. Available through /metrics. + enable-prometheus = false + // Enable collection of internal jitter (difference in processing time). Available through + // /debug/jvb/stats/transit-time + enable-jitter = false + } + jvm { + // Whether to enable collection of JVM metrics (thread count, garbage collection) + enabled = true + } + } + websockets { + enabled = false + server-id = "default-id" + // Whether to negotiate WebSocket compression (permessage-deflate) + enable-compression = true + // Whether to send keepalive pings on idle websockets + send-keepalive-pings = true + // The keepalive ping interval + keepalive-ping-interval = 15 seconds + // The websocket idle timeout + idle-timeout = 60 seconds + + // Optional, even when 'enabled' is set to true + #tls = true + // The domains used when advertising a colibri-ws URL. Must be set when enabled = true + domains = [] + // The domain used when advertising a colibri-relay-ws URL. If empty defaults to the value of `domains`. + relay-domains = [] + } + ice { + udp { + # The port for ICE/UDP. + port = 10000 + } + + # An optional prefix to include in STUN username fragments generated by the bridge. + #ufrag-prefix = "jvb-123:" + + # Which candidate pairs to keep alive. The accepted values are defined in ice4j's KeepAliveStrategy: + # "selected_only", or "all_succeeded". + keep-alive-strategy = "selected_only" + + # Whether to attempt DNS resolution for remote candidates that contain a non-literal address. When set to 'false' + # such candidates will be ignored. + resolve-remote-candidates = false + + # The nomination strategy to use for ICE. THe accepted values are defined in ice4j's NominationStrategy: + # "NominateFirstValid", "NominateHighestPriority", "NominateFirstHostOrReflexiveValid", or "NominateBestRTT". + nomination-strategy = "NominateFirstHostOrReflexiveValid" + + # Whether to advertise private ICE candidates, i.e. RFC 1918 IPv4 addresses and fec0::/10 and fc00::/7 IPv6 + # addresses even for endpoints that have not signaled support for private addresses. + # Note: Jicofo signals support for private addresses for jigasi and jibri. + advertise-private-candidates = true + + # ICE restarts: when an endpoint explicitly requests one (colibri2 ), + # create a second ice4j Agent with freshly rotated local credentials and run it alongside the existing + # one. The old Agent keeps the selected pair and keeps sending until the new one connects + # (make-before-break), then we cut over to the new Agent. If disabled, such requests are rejected and + # the established Agent is left alone. + restart { + enabled = true + + # How long the old Agent is kept alive after we cut over to the new one. During this window both + # Agents keep their sockets, and ice4j routes each packet by the address it came from, so the + # endpoint's old-generation checks are answered by the old Agent rather than dropped. Set to 0 to + # free the old Agent immediately on cutover. + transition-window = 3 seconds + + # How long to wait for a new Agent to connect before giving up on the restart and keeping the + # existing Agent. Counted from when the Agent was created, so this covers the whole signaling + # round trip (bridge to jicofo to the endpoint and back), the endpoint gathering its own new + # candidates, and ICE itself -- on an endpoint whose network has just changed. A value that is + # not positive disables ICE restarts: the new Agent would be freed before the endpoint could + # answer it. + timeout = 20 seconds + } + } + + transport { + send { + # The size of the dtls-transport outgoing queue. This is a per-participant + # queue. Packets from the egress end-up in this queue right before + # transmission by the outgoing srtp pipeline (which mainly consists of the + # packet sender). + # + # Its size needs to be of the same order of magnitude as the rtp sender + # queue. In a 100 participant call, assuming 300pps for the on-stage and + # 100pps for low-definition, last-n 20 and 2 participants talking, so + # 2*50pps for audio, this queue is fed 300+19*100+2*50 = 2300pps, so its + # size in terms of millis is 1024/2300*1000 ~= 445ms. + queue-size = 1024 + } + } + + speech-activity { + # The number of speakers to include in the list of recent speakers sent with dominant speaker change + # notifications. + recent-speakers-count = 10 + + # Whether to enable silence detection. When silence detection is enabled and there is no speech activity for a + # certain time (see silence-detection-timeout below) we fire a "dominant speaker changed" event notifying endpoints + # that we entered a period of silence. + enable-silence-detection = false + + # How long to wait for lack of speech activity before a period of silence begins. + silence-detection-timeout = 3 seconds + } + + loudest { + # Whether to route only the loudest speakers. If false, all audio is forwarded. + route-loudest-only = false + + # The number of current loudest speakers to route audio packets for. + # Ignored if route-loudest-only = false. + num-loudest = 3 + + # Whether to route dominant speaker when it is not among the current loudest speakers. + # Ignored if route-loudest-only = false. + always-route-dominant = true + + # Time after which speaker is removed from loudest list if + # no new audio packets have been received from that speaker. + energy-expire-time = 150 milliseconds + + # Alpha factor for exponential smoothing of energy values, multiplied by 100. + energy-alpha-pct = 50 + } + + version { + # Whether to announce the jitsi-videobridge version to clients in the ServerHello message. + announce = false + + # A release identifier. When set, this will be appended to the version string + # that is associated with this bridge when selecting bridges for a conference. + # This allows finer-grained control of the selection process. + # Note that multiple bridges must have the same version and release identifier + # in order to share conferences between them. + #release = "123" + } + + shutdown { + # The maximum amount of time to stay in GRACEFUL_SHUTDOWN before going into SHUTTING_DOWN. See ShutdownManager. + graceful-shutdown-max-duration = 24 hours + + # The minimum number of participants required to stay in GRACEFUL_SHUTDOWN. If the number of participants falls + # at or below this threshold the bridge will transition to SHUTTING_DOWN. See ShutdownManager. + graceful-shutdown-min-participants = 0 + + # The amount of time to stay in SHUTTING_DOWN before acutally shutting down. Gives jicofo time to actively move + # participants off this bridge. + shutting-down-delay = 1 minutes + } + + # Whether to start the videobridge in drain mode. + initial-drain-mode = false + + # In small conferences we disable the stats filter and always forward stats. This sets the minimum conference + # size for which the filter is enabled. + stats-filter-threshold = 20 + + # Whether to redact remote endpoint IP addresses from logs + redact-remote-addresses = true + + # Whether to redact the values of HTTP headers specified in incoming colibri requests. + redact-colibri-http-headers = true + + + connection-stats { + # Whether to periodically send connection stats (currently, estimated downlink bandwidth) to endpoints + enabled = true + # How often to send these stats + interval = 10 seconds + } + + ssrc-limit { + # maximum number of SSRCs to send to an endpoint for video + video = 50 + + # maximum number of SSRCs to send to an endpoint for audio + audio = 50 + } + // Options for exporting conference media through a websocket. + exporter { + // How many attempts to make to reconnect in case of a disconnection. By default there is no limit. + // max-reconnect-attempts = 10 + // The base delay between reconnection attempts. The first attempt is made with no delay, then it starts at base + // and doubles at each step maxing at max-delay. + base-delay = 1 second + max-delay = 30 seconds + // How long a connection must stay open before it is considered stable. Only stable connections reset the + // reconnect attempt counter (and thus the backoff). Prevents tight reconnect loops when the remote closes + // the connection immediately after it opens. + stable-connection-threshold = 30 seconds + } +} + +tracing { + enabled = false + service-name = "jvb" +} diff --git a/jvb/src/test/java/org/jitsi/videobridge/Util.java b/jvb/src/test/java/org/jitsi/videobridge/Util.java new file mode 100644 index 0000000..5625965 --- /dev/null +++ b/jvb/src/test/java/org/jitsi/videobridge/Util.java @@ -0,0 +1,97 @@ +/* + * Copyright @ 2015 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge; + +import java.util.function.*; + +import static junit.framework.TestCase.assertEquals; + +/** + * Test utilities. + * + * @author Boris Grozev + */ +public class Util +{ + /** + * Tries to take a nap, but isn't grumpy if it's woken up early. + * @param millis the number of milliseconds to sleep. + */ + public static void sleepNoExceptions(long millis) + { + try + { + Thread.sleep(millis); + } + catch (InterruptedException ie) + { + } + } + + /** + * Retries a function call ({@code f}) up to {@code attempts} times until + * it produces the {@code expected} result, sleeping for + * {@code millisToWait} in between attempts. If the function doesn't produce + * the expected result after {@code attempts} attempts, throws an exception. + * + * @param millisToWait the number of milliseconds to wait in total. + * @param attempts + * @param message The message to throw in case of failure. + * @param expected The expected result. + * @param f The function which produces the actual result. + */ + public static void waitForEquals( + long millisToWait, + int attempts, + String message, + Object expected, + Supplier f) + { + Object actual; + int i = 0; + + do { + actual = f.get(); + if (expected.equals(actual)) + { + return; + } + + sleepNoExceptions(millisToWait); + i++; + } while (i < attempts); + + assertEquals(message, expected, actual); + } + + /** + * Retries a function call ({@code f}) up to 5 times until + * it produces the {@code expected} result, sleeping for 500 milliseconds + * in between attempts. If the function doesn't produce the expected result + * after {@code attempts} attempts, throws an exception. + * + * @param message The message to throw in case of failure. + * @param expected The expected result. + * @param f The function which produces the actual result. + */ + public static void waitForEquals( + String message, + Object expected, + Supplier f) + { + waitForEquals(500, 5, message, expected, f); + } +} diff --git a/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java b/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java new file mode 100644 index 0000000..736c1f1 --- /dev/null +++ b/jvb/src/test/java/org/jitsi/videobridge/cc/vp8/VP8AdaptiveSourceProjectionTest.java @@ -0,0 +1,1208 @@ +/* + * Copyright @ 2019 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp8; + +import org.jitsi.nlj.*; +import org.jitsi.nlj.codec.vpx.*; +import org.jitsi.nlj.rtp.codec.vp8.*; +import org.jitsi.nlj.util.*; +import org.jitsi.rtp.rtcp.*; +import org.jitsi.rtp.rtp.*; +import org.jitsi.rtp.util.*; +import org.jitsi.utils.logging.DiagnosticContext; +import org.jitsi.utils.logging2.*; +import org.jitsi.videobridge.cc.*; +import org.jitsi_modified.impl.neomedia.codec.video.vp8.*; +import org.junit.*; + +import javax.xml.bind.*; +import java.time.*; +import java.util.*; + +import static org.junit.Assert.*; + +public class VP8AdaptiveSourceProjectionTest +{ + private final Logger logger = new LoggerImpl(getClass().getName()); + + @Test + public void singlePacketProjectionTest() throws RewriteException + { + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "singlePacketProjectionTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + int targetIndex = RtpLayerDesc.getIndex(0, 0, 0); + + assertTrue(context.accept(packetInfo, targetIndex)); + + context.rewriteRtp(packetInfo); + + assertEquals(10001, packet.getSequenceNumber()); + assertEquals(1003000, packet.getTimestamp()); + assertEquals(0, packet.getPictureId()); + assertEquals(0, packet.getTemporalLayerIndex()); + } + + private void runInOrderTest(Vp8PacketGenerator generator, int targetTid) + throws RewriteException + { + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", Thread.currentThread().getStackTrace()[2].getMethodName()); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + int targetIndex = RtpLayerDesc.getIndex(0, 0, targetTid); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, + initialState, logger); + + int expectedSeq = 10001; + long expectedTs = 1003000; + int expectedPicId = 0; + int expectedTl0PicIdx = 0; + + for (int i = 0; i < 100000; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + boolean accepted = context.accept(packetInfo, targetIndex); + + if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + if (packet.getTemporalLayerIndex() <= targetIndex) + { + assertTrue(accepted); + + context.rewriteRtp(packetInfo); + + assertEquals(expectedSeq, packet.getSequenceNumber()); + assertEquals(expectedTs, packet.getTimestamp()); + assertEquals(expectedPicId, packet.getPictureId()); + assertEquals(expectedTl0PicIdx, packet.getTL0PICIDX()); + + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + if (packet.isEndOfFrame()) + { + expectedPicId = VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + else + { + assertFalse(accepted); + } + if (packet.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + } + } + } + + private static class ProjectedPacket + { + final Vp8Packet packet; + final int origSeq; + final long extOrigSeq; + final boolean nearOldest; + + ProjectedPacket(Vp8Packet p, int s, long e, boolean o) + { + packet = p; + origSeq = s; + extOrigSeq = e; + nearOldest = o; + } + } + + /** Run an out-of-order test on a single stream, randomized order except for the first packet. */ + private void doRunOutOfOrderTest(Vp8PacketGenerator generator, int targetTid, + long seed) + throws RewriteException + { + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", Thread.currentThread().getStackTrace()[2].getMethodName()); + + int targetIndex = RtpLayerDesc.getIndex(0, 0, targetTid); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + final long expectedInitialTs = RtpUtils.applyTimestampDelta(initialState.maxTimestamp, 3000); + final long expectedTsOffset = RtpUtils.getTimestampDiff(expectedInitialTs, generator.ts); + + final int reorderSize = 64; + ArrayList buffer = new ArrayList<>(reorderSize); + + for (int i = 0; i < reorderSize; i++) + { + buffer.add(generator.nextPacket()); + } + + Random random = new Random(seed); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, + initialState, logger); + + int latestSeq = buffer.get(0).packetAs().getSequenceNumber(); + + TreeMap projectedPackets = new TreeMap<>(); + RtpSequenceIndexTracker origSeqIdxTracker = new RtpSequenceIndexTracker(); + RtpSequenceIndexTracker newSeqIdxTracker = new RtpSequenceIndexTracker(); + + for (int i = 0; i < 100000; i++) + { + PacketInfo packetInfo = buffer.get(0); + Vp8Packet packet = packetInfo.packetAs(); + + int origSeq = packet.getSequenceNumber(); + long origTs = packet.getTimestamp(); + int origTl0PicIdx = packet.getTL0PICIDX(); + + if (RtpUtils.isOlderSequenceNumberThan(latestSeq, origSeq)) + { + latestSeq = origSeq; + } + boolean accepted = context.accept(packetInfo, targetIndex); + + int oldestValidSeq = RtpUtils.applySequenceNumberDelta(latestSeq, -((VP8FrameMap.FRAME_MAP_SIZE - 1) * generator.packetsPerFrame)); + + if (RtpUtils.isOlderSequenceNumberThan(origSeq, oldestValidSeq) && !accepted) + { + /* This is fine; packets that are too old get ignored. */ + /* Note we don't want assertFalse(accepted) here because slightly-too-old packets + * that are part of an existing accepted frame will be accepted. + */ + } + else if (packet.getTemporalLayerIndex() <= targetIndex) + { + assertTrue(accepted); + + /* There's an edge condition in frame projection where a packet + of a frame can be projected, then the frame can be forgotten + for being too old, then a later packet of the frame (which is + just barely not too old) can be projected, at which point it + can potentially get assigned different sequence number/TL0PICIDX + values. + + This is an unlikely enough case in real life that it's not worth + worrying about; but the incredibly aggressive packet randomizer + used by the unit tests can trigger it, so explicitly allow it. + */ + boolean nearOldest = RtpUtils.getSequenceNumberDelta(origSeq, oldestValidSeq) < generator.packetsPerFrame; + + context.rewriteRtp(packetInfo); + + assertEquals(RtpUtils.applyTimestampDelta(origTs, expectedTsOffset), packet.getTimestamp()); + assertEquals(origTl0PicIdx, packet.getTL0PICIDX()); + int newSeq = packet.getSequenceNumber(); + long extNewSeq = newSeqIdxTracker.update(newSeq); + long extOrigSeq = origSeqIdxTracker.update(origSeq); + assertFalse(projectedPackets.containsKey(extNewSeq)); + projectedPackets.put(extNewSeq, new ProjectedPacket(packet, origSeq, extOrigSeq, nearOldest)); + } + else + { + assertFalse(accepted); + } + + buffer.set(0, generator.nextPacket()); + Collections.shuffle(buffer, random); + } + + Iterator iter = projectedPackets.keySet().iterator(); + + ProjectedPacket prevPacket = projectedPackets.get(iter.next()); + + while (iter.hasNext()) + { + ProjectedPacket packet = projectedPackets.get(iter.next()); + + assertTrue(RtpUtils.isNewerSequenceNumberThan(packet.origSeq, prevPacket.origSeq)); + if (RtpUtils.isOlderTimestampThan(prevPacket.packet.getTimestamp(), packet.packet.getTimestamp())) + { + assertTrue(VpxUtils.getExtendedPictureIdDelta(prevPacket.packet.getPictureId(), packet.packet.getPictureId()) < 0); + } + else + { + assertEquals(prevPacket.packet.getTimestamp(), packet.packet.getTimestamp()); + assertTrue(prevPacket.packet.getPictureId() == packet.packet.getPictureId() || packet.nearOldest); + } + + prevPacket = packet; + } + + /* Overall, we should not have expanded sequence numbers. */ + ProjectedPacket firstPacket = projectedPackets.firstEntry().getValue(); + ProjectedPacket lastPacket = projectedPackets.lastEntry().getValue(); + + long origDelta = lastPacket.extOrigSeq - firstPacket.extOrigSeq; + long projDelta = projectedPackets.lastKey() - projectedPackets.firstKey(); + assertTrue(projDelta <= origDelta); + } + + /** Run multiple instances of out-of-order test on a single stream, with different + * random seeds. */ + private void runOutOfOrderTest(Vp8PacketGenerator generator, int targetIndex) + throws RewriteException + { + /* Seeds that have triggered problems in the past, plus a random one. */ + long[] seeds = { 1576267371838L, 1578347926155L, 1579620018479L, System.currentTimeMillis()}; + + for (long seed: seeds) + { + try + { + doRunOutOfOrderTest(generator, targetIndex, seed); + } + catch (Throwable e) + { + logger.error( + "Exception thrown in randomized test, seed = " + seed, e); + throw e; + } + + generator.reset(); + } + } + + @Test + public void simpleProjectionTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + runInOrderTest(generator, 2); + } + + @Test + public void filteredProjectionTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + runInOrderTest(generator, 0); + } + + @Test + public void largerFrameProjectionTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + + runInOrderTest(generator, 2); + } + + @Test + public void largerFrameFilteredTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + + runInOrderTest(generator, 0); + } + + @Test + public void hugeFrameTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(200); + + runInOrderTest(generator, 0); + } + + @Test + public void simpleOutOfOrderTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + runOutOfOrderTest(generator, 2); + } + + @Test + public void largerOutOfOrderTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + + runOutOfOrderTest(generator, 2); + } + + @Test + public void filteredOutOfOrderTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + runOutOfOrderTest(generator, 0); + } + + @Test + public void largerFilteredOutOfOrderTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + + runOutOfOrderTest(generator, 0); + } + + @Test + public void slightlyDelayedKeyframeTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "slightlyDelayedKeyframeTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + PacketInfo firstPacketInfo = generator.nextPacket(); + Vp8Packet firstPacket = firstPacketInfo.packetAs(); + + int targetIndex = RtpLayerDesc.getIndex(0, 0, 2); + + for (int i = 0; i < 3; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertFalse(context.accept(packetInfo, targetIndex)); + } + + assertTrue(context.accept(firstPacketInfo, targetIndex)); + context.rewriteRtp(firstPacketInfo); + + for (int i = 0; i < 9996; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertTrue(context.accept(packetInfo, targetIndex)); + context.rewriteRtp(packetInfo); + } + } + + @Test + public void veryDelayedKeyframeTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "veryDelayedKeyframeTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + PacketInfo firstPacketInfo = generator.nextPacket(); + Vp8Packet firstPacket = firstPacketInfo.packetAs(); + + int targetIndex = RtpLayerDesc.getIndex(0, 0, 2); + + for (int i = 0; i < 4; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertFalse(context.accept(packetInfo, targetIndex)); + } + + assertFalse(context.accept(firstPacketInfo, targetIndex)); + + for (int i = 0; i < 10; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertFalse(context.accept(packetInfo, targetIndex)); + } + + generator.requestKeyframe(); + + for (int i = 0; i < 9996; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertTrue(context.accept(packetInfo, targetIndex)); + context.rewriteRtp(packetInfo); + } + } + + @Test + public void delayedPartialKeyframeTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "delayedPartialKeyframeTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + PacketInfo firstPacketInfo = generator.nextPacket(); + Vp8Packet firstPacket = firstPacketInfo.packetAs(); + + int targetIndex = RtpLayerDesc.getIndex(0, 0, 2); + + for (int i = 0; i < 11; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertFalse(context.accept(packetInfo, targetIndex)); + } + + assertFalse(context.accept(firstPacketInfo, 2)); + + for (int i = 0; i < 30; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertFalse(context.accept(packetInfo, targetIndex)); + } + + generator.requestKeyframe(); + + for (int i = 0; i < 9958; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + assertTrue(context.accept(packetInfo, targetIndex)); + context.rewriteRtp(packetInfo); + } + } + + @Test + public void twoStreamsNoSwitchingTest() throws RewriteException + { + Vp8PacketGenerator generator1 = new Vp8PacketGenerator(3, 1); + Vp8PacketGenerator generator2 = new Vp8PacketGenerator(3, 0); + generator2.setSsrc(0xdeadbeefL); + + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "twoStreamsNoSwitchingTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + int targetIndex = RtpLayerDesc.getIndex(1, 0, 2); + + int expectedSeq = 10001; + long expectedTs = 1003000; + for (int i = 0; i < 10000; i++) + { + PacketInfo packetInfo1 = generator1.nextPacket(); + Vp8Packet packet1 = packetInfo1.packetAs(); + + assertTrue(context.accept(packetInfo1, targetIndex)); + + PacketInfo packetInfo2 = generator2.nextPacket(); + Vp8Packet packet2 = packetInfo2.packetAs(); + + assertFalse(context.accept(packetInfo2, targetIndex)); + + context.rewriteRtp(packetInfo1); + + assertEquals(expectedSeq, packet1.getSequenceNumber()); + assertEquals(expectedTs, packet1.getTimestamp()); + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + + if (packet1.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + } + } + } + + @Test + public void twoStreamsSwitchingTest() throws RewriteException + { + Vp8PacketGenerator generator1 = new Vp8PacketGenerator(3, 0); + Vp8PacketGenerator generator2 = new Vp8PacketGenerator(3, 1); + generator2.setSsrc(0xdeadbeefL); + + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "twoStreamsSwitchingTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + int expectedSeq = 10001; + long expectedTs = 1003000; + int expectedPicId = 0; + int expectedTl0PicIdx = 0; + + int targetIndex = RtpLayerDesc.getIndex(0, 0, 2); + + /* Start by wanting spatial layer 0 */ + for (int i = 0; i < 900; i++) + { + RtcpSrPacket srPacket1 = generator1.getSrPacket(); + PacketInfo packetInfo1 = generator1.nextPacket(); + Vp8Packet packet1 = packetInfo1.packetAs(); + + if (packet1.isStartOfFrame() && packet1.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + assertTrue(context.accept(packetInfo1, targetIndex)); + + context.rewriteRtp(packetInfo1); + + assertTrue(context.rewriteRtcp(srPacket1)); + assertEquals(packet1.getSsrc(), srPacket1.getSenderSsrc()); + assertEquals(packet1.getTimestamp(), srPacket1.getSenderInfo().getRtpTimestamp()); + + RtcpSrPacket srPacket2 = generator2.getSrPacket(); + PacketInfo packetInfo2 = generator2.nextPacket(); + Vp8Packet packet2 = packetInfo2.packetAs(); + + assertFalse(context.accept(packetInfo2, targetIndex)); + assertFalse(context.rewriteRtcp(srPacket2)); + + assertEquals(expectedSeq, packet1.getSequenceNumber()); + assertEquals(expectedTs, packet1.getTimestamp()); + assertEquals(expectedPicId, packet1.getPictureId()); + assertEquals(expectedTl0PicIdx, packet1.getTL0PICIDX()); + + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + if (packet1.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + expectedPicId = VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + + /* Switch to wanting spatial layer 1, but don't send a keyframe. We should stay at the higher layer. */ + targetIndex = RtpLayerDesc.getIndex(1, 0, 2); + for (int i = 0; i < 90; i++) + { + RtcpSrPacket srPacket1 = generator1.getSrPacket(); + PacketInfo packetInfo1 = generator1.nextPacket(); + Vp8Packet packet1 = packetInfo1.packetAs(); + + if (packet1.isStartOfFrame() && packet1.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + assertTrue(context.accept(packetInfo1, targetIndex)); + + context.rewriteRtp(packetInfo1); + + assertTrue(context.rewriteRtcp(srPacket1)); + assertEquals(packet1.getSsrc(), srPacket1.getSenderSsrc()); + assertEquals(packet1.getTimestamp(), srPacket1.getSenderInfo().getRtpTimestamp()); + + RtcpSrPacket srPacket2 = generator2.getSrPacket(); + PacketInfo packetInfo2 = generator2.nextPacket(); + Vp8Packet packet2 = packetInfo2.packetAs(); + + assertFalse(context.accept(packetInfo2, targetIndex)); + assertFalse(context.rewriteRtcp(srPacket2)); + + assertEquals(expectedSeq, packet1.getSequenceNumber()); + assertEquals(expectedTs, packet1.getTimestamp()); + assertEquals(expectedPicId, packet1.getPictureId()); + assertEquals(expectedTl0PicIdx, packet1.getTL0PICIDX()); + + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + + if (packet1.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + expectedPicId = VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + + generator1.requestKeyframe(); + generator2.requestKeyframe(); + + /* After a keyframe we should accept spatial layer 1 */ + for (int i = 0; i < 9000; i++) + { + RtcpSrPacket srPacket1 = generator1.getSrPacket(); + PacketInfo packetInfo1 = generator1.nextPacket(); + Vp8Packet packet1 = packetInfo1.packetAs(); + + if (i == 0 && packet1.isStartOfFrame() && packet1.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ + assertEquals(i == 0, context.accept(packetInfo1, targetIndex)); + assertEquals(i == 0, context.rewriteRtcp(srPacket1)); + + if (i == 0) + { + context.rewriteRtp(packetInfo1); + assertEquals(packet1.getSsrc(), srPacket1.getSenderSsrc()); + assertEquals(packet1.getTimestamp(), srPacket1.getSenderInfo().getRtpTimestamp()); + } + + RtcpSrPacket srPacket2 = generator2.getSrPacket(); + PacketInfo packetInfo2 = generator2.nextPacket(); + Vp8Packet packet2 = packetInfo2.packetAs(); + + if (packet2.isStartOfFrame() && packet2.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + assertTrue(context.accept(packetInfo2, targetIndex)); + + context.rewriteRtp(packetInfo2); + + assertTrue(context.rewriteRtcp(srPacket2)); + assertEquals(packet2.getSsrc(), srPacket2.getSenderSsrc()); + assertEquals(packet2.getTimestamp(), srPacket2.getSenderInfo().getRtpTimestamp()); + + if (i == 0) + { + /* We leave a 2-packet gap for the layer 0 keyframe. */ + expectedSeq += 2; + /* ts will advance by an extra 3000 samples for the extra frame. */ + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + /* pid id and tl0picidx will advance by 1 for the extra keyframe. */ + expectedPicId = VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + + assertEquals(expectedSeq, packet2.getSequenceNumber()); + assertEquals(expectedTs, packet2.getTimestamp()); + assertEquals(expectedPicId, packet2.getPictureId()); + assertEquals(expectedTl0PicIdx, packet2.getTL0PICIDX()); + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + + if (packet2.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + expectedPicId = VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + } + + @Test + public void temporalLayerSwitchingTest() throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", "temporalLayerSwitchingTest"); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + int targetTid = 0; + int decodableTid = 0; + + int targetIndex = RtpLayerDesc.getIndex(0, 0, targetTid); + + int expectedSeq = 10001; + long expectedTs = 1003000; + int expectedPicId = 0; + int expectedTl0PicIdx = 0; + + for (int i = 0; i < 10000; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + boolean accepted = context.accept(packetInfo, targetIndex); + + if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + if (accepted) + { + if (decodableTid < packet.getTemporalLayerIndex()) + { + decodableTid = packet.getTemporalLayerIndex(); + } + } + else + { + if (decodableTid > packet.getTemporalLayerIndex() - 1) + { + decodableTid = packet.getTemporalLayerIndex() - 1; + } + } + + if (packet.getTemporalLayerIndex() <= decodableTid) + { + assertTrue(accepted); + + context.rewriteRtp(packetInfo); + + assertEquals(expectedSeq, packet.getSequenceNumber()); + assertEquals(expectedTs, packet.getTimestamp()); + assertEquals(expectedPicId, packet.getPictureId()); + assertEquals(expectedTl0PicIdx, packet.getTL0PICIDX()); + + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + if (packet.isEndOfFrame()) + { + expectedPicId = VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + else + { + assertFalse(accepted); + } + if (packet.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + + if (i % 97 == 0) /* Prime number so it's out of sync with packet cycles. */ + { + targetTid = (targetTid + 2) % 3; + targetIndex = RtpLayerDesc.getIndex(0, 0, targetTid); + } + } + } + } + + private void runLargeDropoutTest(Vp8PacketGenerator generator, + int targetIndex) + throws RewriteException + { + DiagnosticContext diagnosticContext = new DiagnosticContext(); + diagnosticContext.put("test", Thread.currentThread().getStackTrace()[2].getMethodName()); + + RtpState initialState = + new RtpState(1, 10000, 1000000); + + VP8AdaptiveSourceProjectionContext context = + new VP8AdaptiveSourceProjectionContext(diagnosticContext, initialState, logger); + + int expectedSeq = 10001; + long expectedTs = 1003000; + int expectedPicId = 0; + int expectedTl0PicIdx = 0; + + for (int i = 0; i < 1000; i++) + { + PacketInfo packetInfo = generator.nextPacket(); + Vp8Packet packet = packetInfo.packetAs(); + + boolean accepted = + context.accept(packetInfo, targetIndex); + + if (packet.isStartOfFrame() && packet.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = + VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + if (packet.getTemporalLayerIndex() <= targetIndex) + { + assertTrue(accepted); + + context.rewriteRtp(packetInfo); + + assertEquals(expectedSeq, packet.getSequenceNumber()); + assertEquals(expectedTs, packet.getTimestamp()); + assertEquals(expectedPicId, packet.getPictureId()); + assertEquals(expectedTl0PicIdx, packet.getTL0PICIDX()); + + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + if (packet.isEndOfFrame()) + { + expectedPicId = + VpxUtils.applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + else + { + assertFalse(accepted); + } + if (packet.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + } + } + + for (int gap = 64; gap < 65536; gap *= 2) + { + for (int i = 0; i < gap; i++) + { + generator.nextPacket(); + } + + PacketInfo packetInfo; + Vp8Packet packet; + + do + { + packetInfo = generator.nextPacket(); + packet = packetInfo.packetAs(); + } + while (packet.getTemporalLayerIndex() > targetIndex); + + assertTrue(context.accept(packetInfo, targetIndex)); + context.rewriteRtp(packetInfo); + + /* Allow any values after a gap. */ + expectedSeq = + RtpUtils.applySequenceNumberDelta(packet.getSequenceNumber(), 1); + expectedTs = packet.getTimestamp(); + expectedPicId = packet.getPictureId(); + expectedTl0PicIdx = packet.getTL0PICIDX(); + + if (packet.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + expectedPicId = VpxUtils + .applyExtendedPictureIdDelta(expectedPicId, 1); + } + + for (int i = 0; i < 1000; i++) + { + packetInfo = generator.nextPacket(); + packet = packetInfo.packetAs(); + + boolean accepted = context + .accept(packetInfo, targetIndex); + + if (packet.isStartOfFrame() + && packet.getTemporalLayerIndex() == 0) + { + expectedTl0PicIdx = + VpxUtils.applyTl0PicIdxDelta(expectedTl0PicIdx, 1); + } + + if (packet.getTemporalLayerIndex() <= targetIndex) + { + assertTrue(accepted); + + context.rewriteRtp(packetInfo); + + assertEquals(expectedSeq, packet.getSequenceNumber()); + assertEquals(expectedTs, packet.getTimestamp()); + assertEquals(expectedPicId, packet.getPictureId()); + assertEquals(expectedTl0PicIdx, packet.getTL0PICIDX()); + + expectedSeq = + RtpUtils.applySequenceNumberDelta(expectedSeq, 1); + if (packet.isEndOfFrame()) + { + expectedPicId = VpxUtils + .applyExtendedPictureIdDelta(expectedPicId, 1); + } + } + else + { + assertFalse(accepted); + } + if (packet.isEndOfFrame()) + { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000); + } + } + } + } + + @Test + public void largeDropoutTest() + throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + runLargeDropoutTest(generator, 2); + } + + @Test + public void filteredLargeDropoutTest() + throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(1); + runLargeDropoutTest(generator, 0); + } + + @Test + public void largeFrameDropoutTest() + throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + runLargeDropoutTest(generator, 2); + } + + @Test + public void filteredLargeFrameDropoutTest() + throws RewriteException + { + Vp8PacketGenerator generator = new Vp8PacketGenerator(3); + runLargeDropoutTest(generator, 0); + } + + + private static class Vp8PacketGenerator { + private static final byte[] vp8PacketTemplate = + DatatypeConverter.parseHexBinary( + /* RTP Header */ + "80" + /* V, P, X, CC */ + "60" + /* M, PT */ + "0000" + /* Seq */ + "00000000" + /* TS */ + "cafebabe" + /* SSRC */ + /* VP8 Payload descriptor */ + "90" + /* First byte, X, S set, PID = 0 */ + "e0" + /* X byte, I, L, T set */ + "8000" + /* I byte (ext pic id), M set */ + "00" + /* L byte (tl0 pic idx) */ + "00" + /* T/K byte (tid) */ + /* VP8 payload header */ + "00" + /* P = 0. */ + /* Rest of payload is "don't care" for this code, except + keyframe frame size, so make this a valid-ish keyframe header. + */ + "0000" + /* Length = 0. */ + "9d012a" + /* Keyframe startcode */ + "0050D002" /* 1280 × 720 (little-endian) */ + ); + + final int packetsPerFrame; + + final int encodingId; + + Vp8PacketGenerator(int packetsPerFrame, int encodingId) + { + this.packetsPerFrame = packetsPerFrame; + this.encodingId = encodingId; + + reset(); + } + + Vp8PacketGenerator(int packetsPerFrame) { + this(packetsPerFrame, 0); + } + + private int seq; + private long ts; + private int picId ; + private int tl0picidx ; + private int packetOfFrame ; + private boolean keyframe; + private boolean keyframeRequested; + private int tidCycle; + private long ssrc; + + private int packetCount; + private int octetCount; + private int frameCount; + + private static final Instant baseReceivedTime = Instant.ofEpochMilli(1577836800000L); /* 2020-01-01 00:00:00 UTC */ + private Instant receivedTime; + + void reset() + { + boolean useRandom = true; // switch off to ease debugging + long seed = System.currentTimeMillis(); + Random random = new Random(seed); + + seq = useRandom ? random.nextInt() % 0x10000 : 0; + ts = useRandom ? random.nextLong() % 0x100000000L : 0; + + picId = 0; + tl0picidx = 0; + packetOfFrame = 0; + keyframe = true; + keyframeRequested = false; + tidCycle = 0; + ssrc = 0xcafebabeL; + + packetCount = 0; + octetCount = 0; + frameCount = 0; + + receivedTime = baseReceivedTime; + } + + + public void setSsrc(long ssrc) + { + this.ssrc = ssrc; + } + + public PacketInfo nextPacket() + { + int tid; + switch(tidCycle % 4) + { + case 0: + tid = 0; + break; + case 2: + tid = 1; + break; + case 1: + case 3: + tid = 2; + break; + default: + /* Convince Java that yes, tid is always initialized. */ + assert(false); /* Math is broken */ + tid = -1; + } + + boolean startOfFrame = (packetOfFrame == 0); + boolean endOfFrame = (packetOfFrame == packetsPerFrame - 1); + + if (startOfFrame && tid == 0) { + tl0picidx = VpxUtils.applyTl0PicIdxDelta(tl0picidx, 1); + } + + byte[] buffer = vp8PacketTemplate.clone(); + + RtpPacket rtpPacket = new RtpPacket(buffer,0, buffer.length); + + rtpPacket.setSsrc(ssrc); + rtpPacket.setSequenceNumber(seq); + rtpPacket.setTimestamp(ts); + + /* Do VP8 manipulations on buffer before constructing Vp8Packet, because + Vp8Packet computes values at construct-time. */ + DePacketizer.VP8PayloadDescriptor.setStartOfPartition(rtpPacket.buffer, + rtpPacket.getPayloadOffset(), startOfFrame); + assertTrue(DePacketizer.VP8PayloadDescriptor.setTemporalLayerIndex(rtpPacket.buffer, + rtpPacket.getPayloadOffset(), rtpPacket.getPayloadLength(), tid)); + + if (startOfFrame) { + int szVP8PayloadDescriptor = DePacketizer + .VP8PayloadDescriptor.getSize(rtpPacket.buffer, rtpPacket.getPayloadOffset(), rtpPacket.getPayloadLength()); + + DePacketizer.VP8PayloadHeader.setKeyFrame(rtpPacket.buffer, + rtpPacket.getPayloadOffset() + szVP8PayloadDescriptor, keyframe); + } + rtpPacket.setMarked(endOfFrame); + + Vp8Packet vp8Packet = rtpPacket.toOtherType(Vp8Packet::new); + vp8Packet.setEncodingId(encodingId); + + /* Make sure our manipulations of the raw buffer were correct. */ + assertEquals(startOfFrame, vp8Packet.isStartOfFrame()); + assertEquals(tid, vp8Packet.getTemporalLayerIndex()); + if (startOfFrame) + { + assertEquals(keyframe, vp8Packet.isKeyframe()); + } + + vp8Packet.setPictureId(picId); + vp8Packet.setTL0PICIDX(tl0picidx); + + PacketInfo info = new PacketInfo(vp8Packet); + info.setReceivedTime(receivedTime); + + seq = RtpUtils.applySequenceNumberDelta(seq, 1); + packetCount++; + octetCount += vp8Packet.length; + + if (endOfFrame) + { + packetOfFrame = 0; + ts = RtpUtils.applyTimestampDelta(ts, 3000); + picId = VpxUtils.applyExtendedPictureIdDelta(picId, 1); + tidCycle++; + keyframe = keyframeRequested; + keyframeRequested = false; + if (keyframe) + { + tidCycle = 0; + } + frameCount++; + receivedTime = baseReceivedTime.plus(Duration.ofMillis(frameCount * 100L / 3)); + } + else + { + packetOfFrame++; + } + + return info; + } + + private void requestKeyframe() + { + if (packetOfFrame == 0) + { + keyframe = true; + keyframeRequested = false; + tidCycle = 0; + } + else + { + keyframeRequested = true; + } + } + + /* TODO: move this to jitsi-rtp */ + public static void setSIBuilderNtp(SenderInfoBuilder siBuilder, long wallTime) + { + final long JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L; + + long wallSecs = wallTime / 1000; + long wallMs = wallTime % 1000; + + siBuilder.setNtpTimestampMsw(wallSecs + JAVA_TO_NTP_EPOCH_OFFSET_SECS); + siBuilder.setNtpTimestampLsw(wallMs * (1L << 32) / 1000); + } + + public RtcpSrPacket getSrPacket() + { + RtcpSrPacketBuilder srPacketBuilder = new RtcpSrPacketBuilder(); + + srPacketBuilder.getRtcpHeader().setSenderSsrc(ssrc); + + SenderInfoBuilder siBuilder = srPacketBuilder.getSenderInfo(); + setSIBuilderNtp(srPacketBuilder.getSenderInfo(), receivedTime.toEpochMilli()); + siBuilder.setRtpTimestamp(ts); + siBuilder.setSendersOctetCount(packetCount); + siBuilder.setSendersOctetCount(octetCount); + + return srPacketBuilder.build(); + } + } +} diff --git a/jvb/src/test/java/org/jitsi/videobridge/rest/MockBinder.java b/jvb/src/test/java/org/jitsi/videobridge/rest/MockBinder.java new file mode 100644 index 0000000..5a32f05 --- /dev/null +++ b/jvb/src/test/java/org/jitsi/videobridge/rest/MockBinder.java @@ -0,0 +1,43 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest; + +import org.glassfish.hk2.utilities.binding.*; + +/** + * MockBinder is a helper for binding mock instances to resource classes + * which are injected to REST resources + * @param the type to inject + * @param the type which the injected type should be bound to + */ +public class MockBinder extends AbstractBinder +{ + protected T classToBind; + protected Class classToBindTo; + + public MockBinder(T classToBind, Class classToBindTo) + { + this.classToBind = classToBind; + this.classToBindTo = classToBindTo; + } + + @Override + protected void configure() + { + bind(classToBind).to(classToBindTo); + } +} diff --git a/jvb/src/test/java/org/jitsi/videobridge/rest/root/colibri/ColibriResourceTest.java b/jvb/src/test/java/org/jitsi/videobridge/rest/root/colibri/ColibriResourceTest.java new file mode 100644 index 0000000..31b3479 --- /dev/null +++ b/jvb/src/test/java/org/jitsi/videobridge/rest/root/colibri/ColibriResourceTest.java @@ -0,0 +1,52 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri; + +import org.jitsi.videobridge.rest.annotations.*; +import org.junit.*; +import org.reflections.*; +import org.reflections.scanners.*; +import org.reflections.util.*; + +import jakarta.ws.rs.*; + +import static org.junit.Assert.*; + +public class ColibriResourceTest +{ + /** + * The /colibri path is enabled via a config parameter, so all resources on that path must extend be annotated + * with @EnabledByConfig. Note that the /colibri/shutdown has its own config property, so not all paths under + * /colibri have the same annotation value. + */ + @Test + public void testAllResourcesAreBehindConfig() + { + Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage("org.jitsi.videobridge.rest.root.colibri")) + .filterInputsBy(new FilterBuilder().includePackage("org.jitsi.videobridge.rest.root.colibri")) + .setScanners(new SubTypesScanner(false), new TypeAnnotationsScanner())); + + for (Class clazz : reflections.getTypesAnnotatedWith(Path.class)) + { + assertTrue( + "Class " + clazz.getSimpleName() + " must be annotated with @EnabledByConfig", + clazz.isAnnotationPresent(EnabledByConfig.class)); + } + } + +} \ No newline at end of file diff --git a/jvb/src/test/java/org/jitsi/videobridge/xmpp/MediaSourceFactoryTest.java b/jvb/src/test/java/org/jitsi/videobridge/xmpp/MediaSourceFactoryTest.java new file mode 100644 index 0000000..5a2c45b --- /dev/null +++ b/jvb/src/test/java/org/jitsi/videobridge/xmpp/MediaSourceFactoryTest.java @@ -0,0 +1,301 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.xmpp; + +import org.jitsi.nlj.*; +import org.jitsi.nlj.rtp.SsrcAssociationType; +import org.jitsi.xmpp.extensions.colibri.*; +import org.jitsi.xmpp.extensions.jingle.*; +import org.jitsi.xmpp.extensions.jitsimeet.SSRCInfoPacketExtension; +import org.junit.*; +import org.jxmpp.jid.impl.JidCreate; +import org.jxmpp.stringprep.XmppStringprepException; + +import java.util.*; + +import static org.junit.Assert.*; + +public class MediaSourceFactoryTest +{ + private SourceGroupPacketExtension createGroup( + String semantics, SourcePacketExtension... sources) + { + SourceGroupPacketExtension sgpe = new SourceGroupPacketExtension(); + sgpe.setSemantics(semantics); + if (sources.length > 0) + { + sgpe.addSources(Arrays.asList(sources)); + } + + return sgpe; + } + + private SourcePacketExtension createSource(Long ssrc) + { + SourcePacketExtension spe = new SourcePacketExtension(); + if (ssrc != null) + { + spe.setSSRC(ssrc); + } + + return spe; + } + + // 1 video stream -> 1 source, 1 layer + @Test + public void createMediaSource() + { + long videoSsrc = 12345; + + SourcePacketExtension videoSource = createSource(videoSsrc); + + MediaSourceDesc source = + MediaSourceFactory.createMediaSource( + Collections.singletonList(videoSource), Collections.emptyList(), "E1", "S1"); + + assertNotNull(source); + assertEquals(3, source.numRtpLayers()); + assertEquals(videoSsrc, source.getPrimarySSRC()); + } + + // 1 video stream, 1 rtx -> 1 source, 1 layer + @Test + public void createMediaSources1() + { + long videoSsrc = 12345; + long rtxSsrc = 54321; + + SourcePacketExtension videoSource = createSource(videoSsrc); + SourcePacketExtension rtx = createSource(rtxSsrc); + + SourceGroupPacketExtension rtxGroup + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource, rtx); + + MediaSourceDesc source = + MediaSourceFactory.createMediaSource( + Arrays.asList(videoSource, rtx), Collections.singletonList(rtxGroup), "E1", "S1"); + + assertEquals(3, source.numRtpLayers()); + assertEquals(videoSsrc, source.getPrimarySSRC()); + assertEquals(rtxSsrc, source.getRtpEncodings()[0].getSecondarySsrc(SsrcAssociationType.RTX)); + } + + // 3 sim streams, 3 rtx -> 1 source, 9 layers + @Test + public void createMediaSources2() + { + long videoSsrc1 = 12345; + long videoSsrc2 = 23456; + long videoSsrc3 = 34567; + long rtxSsrc1 = 54321; + long rtxSsrc2 = 43215; + long rtxSsrc3 = 32154; + + SourcePacketExtension videoSource1 = createSource(videoSsrc1); + SourcePacketExtension videoSource2 = createSource(videoSsrc2); + SourcePacketExtension videoSource3 = createSource(videoSsrc3); + SourcePacketExtension rtx1 = createSource(rtxSsrc1); + SourcePacketExtension rtx2 = createSource(rtxSsrc2); + SourcePacketExtension rtx3 = createSource(rtxSsrc3); + + SourceGroupPacketExtension simGroup + = createGroup( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST, + videoSource1, + videoSource2, + videoSource3); + SourceGroupPacketExtension rtxGroup1 + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource1, rtx1); + SourceGroupPacketExtension rtxGroup2 + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource2, rtx2); + SourceGroupPacketExtension rtxGroup3 + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource3, rtx3); + + MediaSourceDesc source = + MediaSourceFactory.createMediaSource( + Arrays.asList( + videoSource1, videoSource2, videoSource3, rtx1, rtx2, rtx3), + Arrays.asList(simGroup, rtxGroup1, rtxGroup2, rtxGroup3), + "E1", + "S1" + ); + + assertNotNull(source); + assertEquals(9, source.numRtpLayers()); + assertEquals(videoSsrc1, source.getPrimarySSRC()); + } + + // 3 sim streams, svc enabled, 3 rtx -> 1 source, 3 layers + @Test + public void createMediaSources3() + { + long videoSsrc1 = 12345; + long videoSsrc2 = 23456; + long videoSsrc3 = 34567; + long rtxSsrc1 = 54321; + long rtxSsrc2 = 43215; + long rtxSsrc3 = 32154; + + SourcePacketExtension videoSource1 = createSource(videoSsrc1); + SourcePacketExtension videoSource2 = createSource(videoSsrc2); + SourcePacketExtension videoSource3 = createSource(videoSsrc3); + SourcePacketExtension rtx1 = createSource(rtxSsrc1); + SourcePacketExtension rtx2 = createSource(rtxSsrc2); + SourcePacketExtension rtx3 = createSource(rtxSsrc3); + + SourceGroupPacketExtension simGroup + = createGroup( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST, + videoSource1, videoSource2, videoSource3); + SourceGroupPacketExtension rtxGroup1 + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource1, rtx1); + SourceGroupPacketExtension rtxGroup2 + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource2, rtx2); + SourceGroupPacketExtension rtxGroup3 + = createGroup( + SourceGroupPacketExtension.SEMANTICS_FID, videoSource3, rtx3); + + MediaSourceDesc source = + MediaSourceFactory.createMediaSource( + Arrays.asList( + videoSource1, videoSource2, videoSource3, rtx1, rtx2, rtx3), + Arrays.asList(simGroup, rtxGroup1, rtxGroup2, rtxGroup3), + "E1", + "S1" + ); + + assertNotNull(source); + assertEquals(9, source.numRtpLayers()); + + RtpEncodingDesc[] encodings = source.getRtpEncodings(); + assertNotNull(encodings); + assertEquals(3, encodings.length); + + assertEquals(videoSsrc1, encodings[0].getPrimarySSRC()); + assertEquals(rtxSsrc1, encodings[0].getSecondarySsrc(SsrcAssociationType.RTX)); + + assertEquals(videoSsrc2, encodings[1].getPrimarySSRC()); + assertEquals(rtxSsrc2, encodings[1].getSecondarySsrc(SsrcAssociationType.RTX)); + + assertEquals(videoSsrc3, encodings[2].getPrimarySSRC()); + assertEquals(rtxSsrc3, encodings[2].getSecondarySsrc(SsrcAssociationType.RTX)); + } + + @Test + public void testEmptySimGroup() + { + long videoSsrc1 = 12345; + + SourcePacketExtension videoSource1 = createSource(videoSsrc1); + + SourceGroupPacketExtension simGroup + = createGroup( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST); + + MediaSourceDesc source = + MediaSourceFactory.createMediaSource( + Collections.singletonList(videoSource1), + Collections.singletonList(simGroup), + "E1", + "S1" + ); + + assertNotNull(source); + assertEquals(3, source.numRtpLayers()); + } + + @Test + public void testEmptySource() + { + SourcePacketExtension videoSource1 = createSource(null); + + MediaSourceDesc source = + MediaSourceFactory.createMediaSource( + Collections.singletonList(videoSource1), + Collections.emptyList(), + "E1", + "S1" + ); + + assertNull(source); + } + + @Test + public void testSourceNameWithSingleSource() + throws XmppStringprepException + { + String testName = "endpoint1-v0"; + SourcePacketExtension videoSource1 = createSource(1L); + + videoSource1.setName(testName); + + SSRCInfoPacketExtension ssrcInfoPacketExtension = new SSRCInfoPacketExtension(); + ssrcInfoPacketExtension.setOwner(JidCreate.from("chat@server.com/nick1")); + videoSource1.addChildExtension(ssrcInfoPacketExtension); + + MediaSourceDesc[] sources = + MediaSourceFactory.createMediaSources( + Collections.singletonList(videoSource1), + Collections.emptyList() + ); + + assertEquals(testName, sources[0].getSourceName()); + } + + @Test + public void testSourceNameThreeSources() throws XmppStringprepException { + String testName = "endpoint1-v0"; + long videoSsrc1 = 12345; + long videoSsrc2 = 23456; + long videoSsrc3 = 34567; + + SourcePacketExtension videoSource1 = createSource(videoSsrc1); + SourcePacketExtension videoSource2 = createSource(videoSsrc2); + SourcePacketExtension videoSource3 = createSource(videoSsrc3); + + SourceGroupPacketExtension simGroup + = createGroup( + SourceGroupPacketExtension.SEMANTICS_SIMULCAST, + videoSource1, + videoSource2, + videoSource3); + + // Only primary SSRC matters + videoSource1.setName(testName); + videoSource2.setName("something else"); + videoSource3.setName("something3"); + + SSRCInfoPacketExtension ssrcInfoPacketExtension = new SSRCInfoPacketExtension(); + ssrcInfoPacketExtension.setOwner(JidCreate.from("chat@server.com/nick1")); + videoSource1.addChildExtension(ssrcInfoPacketExtension); + + MediaSourceDesc[] sources = + MediaSourceFactory.createMediaSources( + Arrays.asList( + videoSource1, videoSource2, videoSource3), + Arrays.asList(simGroup)); + + assertEquals(1, sources.length); + assertEquals(testName, sources[0].getSourceName()); + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/ConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/ConfigTest.kt new file mode 100644 index 0000000..b686bea --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/ConfigTest.kt @@ -0,0 +1,27 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec + +/** + * A helper class for testing configuration properties + */ +abstract class ConfigTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf +} diff --git a/jvb/src/test/kotlin/org/jitsi/TestReadOnlyConfigurationService.kt b/jvb/src/test/kotlin/org/jitsi/TestReadOnlyConfigurationService.kt new file mode 100644 index 0000000..f8a6978 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/TestReadOnlyConfigurationService.kt @@ -0,0 +1,30 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi + +import org.jitsi.config.AbstractReadOnlyConfigurationService +import java.util.Properties + +class TestReadOnlyConfigurationService( + override var properties: Properties = Properties() +) : AbstractReadOnlyConfigurationService() { + + val props: Properties + get() = properties + + override fun reloadConfiguration() {} +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/AudioSubscriptionManagerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/AudioSubscriptionManagerTest.kt new file mode 100644 index 0000000..722b77c --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/AudioSubscriptionManagerTest.kt @@ -0,0 +1,646 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.videobridge.message.ReceiverAudioSubscriptionMessage +import org.jitsi.videobridge.relay.AudioSourceDesc + +class AudioSubscriptionManagerTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + val audioSourceDescs: List = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3") + ) + + // The manager owns the authoritative set of known sources; register the standard ones up front (as the conference + // would via colibri) so subscriptions can resolve their names. Synthetic-source tests add more via onSourcesAdded. + private val manager = AudioSubscriptionManager().apply { onSourcesAdded(audioSourceDescs.toSet()) } + + init { + context("Basic subscription management") { + should("handle single endpoint subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + // Endpoint should want included sources + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // source1 + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe true // source2 + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe false // source3 not included + } + + should("handle multiple endpoint subscriptions") { + val subscription1 = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1")) + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source2", "source3")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + + // endpoint1 subscriptions + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // source1 + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false // source2 + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe false // source3 + + // endpoint2 subscriptions + manager.isEndpointAudioWanted("endpoint2", 1001L) shouldBe false // source1 + manager.isEndpointAudioWanted("endpoint2", 1002L) shouldBe true // source2 + manager.isEndpointAudioWanted("endpoint2", 1003L) shouldBe true // source3 + } + + should("handle All subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = true) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe true + // An SSRC that corresponds to no known source is not routed, even under "all". + manager.isEndpointAudioWanted("endpoint1", 9999L) shouldBe false + } + + should("handle None subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = false) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe false + } + + should("handle Exclude subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = true, exclude = listOf("source2")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // source1 not excluded + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false // source2 excluded + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe true // source3 not excluded + } + } + + context("Subscribed local audio sources tracking") { + should("track explicitly subscribed sources with Include subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source3")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe false + } + + should("not track sources for All subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = true) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("not track sources for None subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = false) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("not track sources for Exclude subscription") { + val subscription = ReceiverAudioSubscriptionMessage(all = true, exclude = listOf("source2")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("track sources from multiple endpoints with Include subscriptions") { + val subscription1 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source2", "source3")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + } + + should("handle overlapping subscriptions correctly") { + val subscription1 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source3")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + + // source1 should be tracked (subscribed by both endpoints) + // source2 should be tracked (subscribed by endpoint1) + // source3 should be tracked (subscribed by endpoint2) + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + } + + should("update tracked sources when subscription changes") { + val subscription1 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe false + + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source2", "source3")) + manager.setEndpointAudioSubscription("endpoint1", subscription2) + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + } + } + + context("Endpoint removal") { + should("remove endpoint from subscribed sources tracking") { + val subscription1 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source3")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + + val audioSources: Set = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint1", "source2") + ) + manager.removeEndpoint("endpoint1") + manager.removeSources(audioSources) + // Only source3 should remain (from endpoint2) + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe true + } + + should("handle removal of non-existent endpoint") { + manager.removeEndpoint("non-existent") + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("clean up empty source entries") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + + manager.removeEndpoint("endpoint1") + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("remove audio sources from subscribed sources when endpoint is removed") { + val subscription1 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2", "source3")) + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source2", "source3")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + + val audioSources: Set = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint1", "source2") + ) + manager.removeEndpoint("endpoint1") + manager.removeSources(audioSources) + + // Only source3 should remain (from endpoint2) + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe true + } + + should("notify remaining subscriptions about removed sources") { + val subscription1 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + val subscription2 = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source3")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + + // Both endpoints should initially want source1 + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint2", 1001L) shouldBe true + + val audioSources: Set = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + manager.removeEndpoint("endpoint1") + manager.removeSources(audioSources) + + // endpoint2 should no longer want source1 since it was removed from the conference + manager.isEndpointAudioWanted("endpoint2", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint2", 1003L) shouldBe true // source3 should still be wanted + } + } + + context("Source lifecycle management") { + should("handle source addition") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source4", "source5")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + // Initially, these sources don't exist, so they're not wanted + manager.isEndpointAudioWanted("endpoint1", 1004L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1005L) shouldBe false + + val newSources = setOf( + AudioSourceDesc(1004L, "endpoint4", "source4"), + AudioSourceDesc(1005L, "endpoint5", "source5"), + AudioSourceDesc(1006L, "endpoint6", "source6") + ) + manager.onSourcesAdded(newSources) + + // Now the included sources should be wanted + manager.isEndpointAudioWanted("endpoint1", 1004L) shouldBe true // source4 included + manager.isEndpointAudioWanted("endpoint1", 1005L) shouldBe true // source5 included + manager.isEndpointAudioWanted("endpoint1", 1006L) shouldBe false // source6 not included + } + + should("resolve a subscription naming a source that was signaled before it") { + // Regression: a ReceiverAudioSubscription can arrive after a source is signaled via colibri but + // referencing it by name. The subscription must resolve against the conference's known sources, not + // a snapshot captured by the caller, otherwise the source is silently never forwarded. + val lateSource = AudioSourceDesc(1007L, "endpoint7", "source7") + manager.onSourcesAdded(setOf(lateSource)) + + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source7")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isEndpointAudioWanted("endpoint1", 1007L) shouldBe true + } + + should("handle source removal") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe true + + val removedSources = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + manager.removeSources(removedSources) + + // Removed source should no longer be wanted + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe true // still wanted + } + + should("clean up subscribed sources when sources are removed") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source2")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe true + + val removedSources = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + manager.removeSources(removedSources) + + // source1 should be removed from subscribed sources + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe true + } + } + + context("Complex scenarios") { + should("handle multiple endpoints with mixed subscription types") { + val allSub = ReceiverAudioSubscriptionMessage(all = true) + val noneSub = ReceiverAudioSubscriptionMessage(all = false) + val includeSub = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "source3")) + val excludeSub = ReceiverAudioSubscriptionMessage(all = true, exclude = listOf("source2")) + + manager.setEndpointAudioSubscription("endpoint-all", allSub) + manager.setEndpointAudioSubscription("endpoint-none", noneSub) + manager.setEndpointAudioSubscription("endpoint-include", includeSub) + manager.setEndpointAudioSubscription("endpoint-exclude", excludeSub) + + // endpoint-all should want everything + manager.isEndpointAudioWanted("endpoint-all", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint-all", 1002L) shouldBe true + manager.isEndpointAudioWanted("endpoint-all", 1003L) shouldBe true + + // endpoint-none should want nothing + manager.isEndpointAudioWanted("endpoint-none", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint-none", 1002L) shouldBe false + manager.isEndpointAudioWanted("endpoint-none", 1003L) shouldBe false + + // endpoint-include should want only included sources + manager.isEndpointAudioWanted("endpoint-include", 1001L) shouldBe true // source1 + manager.isEndpointAudioWanted("endpoint-include", 1002L) shouldBe false // source2 + manager.isEndpointAudioWanted("endpoint-include", 1003L) shouldBe true // source3 + + // endpoint-exclude should want all except excluded sources + manager.isEndpointAudioWanted("endpoint-exclude", 1001L) shouldBe true // source1 + manager.isEndpointAudioWanted("endpoint-exclude", 1002L) shouldBe false // source2 (excluded) + manager.isEndpointAudioWanted("endpoint-exclude", 1003L) shouldBe true // source3 + + // Only include subscription should contribute to subscribed sources + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source3") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe false + } + + should("handle subscription changes across different types") { + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true) + ) + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1")) + ) + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false + manager.isExplicitlySubscribed("source1") shouldBe true + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = false) + ) + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe false + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true, exclude = listOf("source2")) + ) + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe true + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("maintain independent endpoint subscriptions") { + val subscription1 = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1")) + val subscription2 = ReceiverAudioSubscriptionMessage(all = false, include = listOf("source2")) + + manager.setEndpointAudioSubscription("endpoint1", subscription1) + manager.setEndpointAudioSubscription("endpoint2", subscription2) + + // Verify independent subscriptions + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false + manager.isEndpointAudioWanted("endpoint2", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint2", 1002L) shouldBe true + + // Update one subscription shouldn't affect the other + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true) + ) + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe true // now wants all + manager.isEndpointAudioWanted("endpoint2", 1001L) shouldBe false // unchanged + manager.isEndpointAudioWanted("endpoint2", 1002L) shouldBe true // unchanged + + manager.removeEndpoint("endpoint1") + manager.isEndpointAudioWanted("endpoint2", 1002L) shouldBe true // unchanged + } + } + + context("Edge cases") { + should("handle empty include list") { + val subscription = ReceiverAudioSubscriptionMessage(all = false, include = emptyList()) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe false + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("handle empty exclude list") { + val subscription = ReceiverAudioSubscriptionMessage(all = true, exclude = emptyList()) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + // Nothing excluded means everything is wanted + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe true + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe true + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + + should("handle subscription to non-existent sources") { + val subscription = + ReceiverAudioSubscriptionMessage(all = false, include = listOf("non-existent-source")) + manager.setEndpointAudioSubscription("endpoint1", subscription) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1002L) shouldBe false + manager.isEndpointAudioWanted("endpoint1", 1003L) shouldBe false + + // Should still track the explicitly subscribed source name + manager.isExplicitlySubscribed("non-existent-source") shouldBe true + manager.isExplicitlySubscribed("source1") shouldBe false + manager.isExplicitlySubscribed("source2") shouldBe false + manager.isExplicitlySubscribed("source3") shouldBe false + } + } + + context("Synthetic sources") { + // ssrc 2001 / "syntheticSource" is synthetic; the regular audioSourceDescs (1001-1003) are not. + val syntheticSsrc = 2001L + val syntheticDesc = AudioSourceDesc(syntheticSsrc, "endpointS", "syntheticSource", synthetic = true) + + should("not be wanted under an All subscription") { + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true) + ) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // non-synthetic still wanted + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false + } + + should("not be wanted by an endpoint with no subscription (default)") { + manager.onSourcesAdded(setOf(syntheticDesc)) + + // No subscription set for this endpoint: defaults to wanting everything except synthetic. + manager.isEndpointAudioWanted("no-subscription", 1001L) shouldBe true + manager.isEndpointAudioWanted("no-subscription", syntheticSsrc) shouldBe false + } + + should("not be wanted under an Exclude subscription that doesn't name it") { + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true, exclude = listOf("source2")) + ) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // non-synthetic, not excluded + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false + } + + should("be wanted only when explicitly included by name") { + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = false, include = listOf("syntheticSource")) + ) + + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe true + } + + should("be wanted under an All subscription that also includes it") { + // The motivating case: "all regular audio plus this synthetic source", which the old + // mode-based message couldn't express. + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true, include = listOf("syntheticSource")) + ) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // regular audio via `all` + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe true // synthetic via `include` + } + + should("not be wanted by an Include subscription that names a different source") { + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1")) + ) + + manager.isEndpointAudioWanted("endpoint1", 1001L) shouldBe true // explicitly included + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false + } + + should("be resolvable by name only while known, and only when synthetic") { + manager.findSyntheticSource("syntheticSource") shouldBe null // not added yet + manager.findSyntheticSource("source1") shouldBe null // known but not synthetic + + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.findSyntheticSource("syntheticSource") shouldBe syntheticDesc + manager.findSyntheticSource("source1") shouldBe null // still not synthetic + + manager.removeSources(setOf(syntheticDesc)) + manager.findSyntheticSource("syntheticSource") shouldBe null // gone after removal + } + + should("not be routed once removed, even to an All subscriber (no leak as ordinary audio)") { + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true) + ) + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false + + manager.removeSources(setOf(syntheticDesc)) + // Once removed the SSRC corresponds to no known source, so it must not be routed -- in particular it + // must not revert to being delivered as ordinary audio under an "all" subscription. This guards + // against late synthetic packets (translator media arriving just after the source was unsubscribed) + // leaking to non-subscribers. + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false + } + + should("not be routed before it is known, even to an All subscriber (early-packet leak)") { + // The mirror of the late case: media for a synthetic SSRC that arrives before colibri has signaled + // the source must not be forwarded as ordinary audio. + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true) + ) + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false // not yet known + + manager.onSourcesAdded(setOf(syntheticDesc)) + // Now known and synthetic: still gated to explicit (Include) subscribers only, not "all". + manager.isEndpointAudioWanted("endpoint1", syntheticSsrc) shouldBe false + } + } + + context("Debug state") { + val syntheticSsrc = 2001L + val syntheticDesc = AudioSourceDesc(syntheticSsrc, "endpointS", "syntheticSource", synthetic = true) + + should("report known sources, subscriptions, and explicit subscribers") { + manager.onSourcesAdded(setOf(syntheticDesc)) + manager.setEndpointAudioSubscription( + "endpoint1", + ReceiverAudioSubscriptionMessage(all = true, exclude = listOf("source2")) + ) + manager.setEndpointAudioSubscription( + "endpoint2", + ReceiverAudioSubscriptionMessage(all = false, include = listOf("source1", "syntheticSource")) + ) + + val state = manager.debugState() + + // known_sources lists every known source with its synthetic flag (regulars + the synthetic one). + val knownSsrcs = state.get("known_sources").map { it.get("ssrc").asLong() }.toSet() + knownSsrcs shouldBe setOf(1001L, 1002L, 1003L, syntheticSsrc) + val syntheticEntry = state.get("known_sources").single { it.get("ssrc").asLong() == syntheticSsrc } + syntheticEntry.get("synthetic").asBoolean() shouldBe true + + // subscribed_local_sources reflects only Include subscriptions. + val subscribed = state.get("subscribed_local_sources") + subscribed.has("source1") shouldBe true + subscribed.has("syntheticSource") shouldBe true + subscribed.has("source2") shouldBe false // only excluded, not an explicit subscription + + // Per-endpoint subscription state, including resolved SSRCs. + val ep1 = state.get("subscriptions").get("endpoint1") + ep1.get("all").asBoolean() shouldBe true + ep1.get("excluded_ssrcs").map { it.asLong() } shouldBe listOf(1002L) + val ep2 = state.get("subscriptions").get("endpoint2") + ep2.get("all").asBoolean() shouldBe false + ep2.get("included_ssrcs").map { it.asLong() }.toSet() shouldBe setOf(1001L, syntheticSsrc) + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/AudioSubscriptionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/AudioSubscriptionTest.kt new file mode 100644 index 0000000..509619f --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/AudioSubscriptionTest.kt @@ -0,0 +1,398 @@ +/* + * Copyright @ 2025 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.videobridge.message.ReceiverAudioSubscriptionMessage +import org.jitsi.videobridge.relay.AudioSourceDesc + +class AudioSubscriptionTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val audioSubscription = AudioSubscription() + + init { + context("Mode=None subscription") { + should("return false for any SSRC when subscription is None") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = false), emptyList()) + audioSubscription.isSsrcWanted(1001L) shouldBe false + audioSubscription.isSsrcWanted(1002L) shouldBe false + audioSubscription.isSsrcWanted(9999L) shouldBe false + } + + should("return false for any SSRC after adding conference sources") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = false), emptyList()) + val newSources = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + audioSubscription.onConferenceSourceAdded(newSources) + audioSubscription.isSsrcWanted(1001L) shouldBe false + audioSubscription.isSsrcWanted(1002L) shouldBe false + } + + should("handle source removal without affecting None behavior") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = false), emptyList()) + val sourcesToRemove = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + audioSubscription.onConferenceSourceRemoved(sourcesToRemove) + audioSubscription.isSsrcWanted(1001L) shouldBe false + audioSubscription.isSsrcWanted(1002L) shouldBe false + } + } + + context("Mode=All subscription") { + should("return true for any SSRC when subscription is All") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), emptyList()) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true + audioSubscription.isSsrcWanted(9999L) shouldBe true + } + + should("return true for any SSRC after adding conference sources") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), emptyList()) + val newSources = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + audioSubscription.onConferenceSourceAdded(newSources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true + } + + should("handle source removal without affecting All behavior") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), emptyList()) + val sourcesToRemove = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + audioSubscription.onConferenceSourceRemoved(sourcesToRemove) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true + } + } + + context("Mode=Include subscription") { + should("return true only for sources in include list") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1", "source3") + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 included + audioSubscription.isSsrcWanted(1002L) shouldBe false // source2 not included + audioSubscription.isSsrcWanted(1003L) shouldBe true // source3 included + audioSubscription.isSsrcWanted(9999L) shouldBe false // unknown SSRC + } + + should("handle empty include list") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = emptyList() + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe false + audioSubscription.isSsrcWanted(1002L) shouldBe false + } + } + + context("Mode=Exclude subscription") { + should("return true for sources not in exclude list") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3") + ) + val excludeSubscription = ReceiverAudioSubscriptionMessage( + all = true, + exclude = listOf("source2") + ) + audioSubscription.updateSubscription(excludeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 not excluded + audioSubscription.isSsrcWanted(1002L) shouldBe false // source2 excluded + audioSubscription.isSsrcWanted(1003L) shouldBe true // source3 not excluded + } + + should("handle empty exclude list") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + val excludeSubscription = ReceiverAudioSubscriptionMessage( + all = true, + exclude = emptyList() + ) + audioSubscription.updateSubscription(excludeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // nothing excluded + audioSubscription.isSsrcWanted(1002L) shouldBe true // nothing excluded + } + } + + context("onConferenceSourceAdded with Include subscription") { + should("add new sources that match include criteria") { + val initialSources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1", "source2", "source3") + ) + audioSubscription.updateSubscription(includeSubscription, initialSources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 included + audioSubscription.isSsrcWanted(1002L) shouldBe false // source2 not yet added + val newSources = setOf( + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3"), + AudioSourceDesc(1004L, "endpoint4", "source4") + ) + audioSubscription.onConferenceSourceAdded(newSources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 still included + audioSubscription.isSsrcWanted(1002L) shouldBe true // source2 now included + audioSubscription.isSsrcWanted(1003L) shouldBe true // source3 now included + audioSubscription.isSsrcWanted(1004L) shouldBe false // source4 not in include list + } + + should("not add sources for None subscription") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = false), emptyList()) + val newSources = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + audioSubscription.onConferenceSourceAdded(newSources) + audioSubscription.isSsrcWanted(1001L) shouldBe false + } + + should("not change behavior for All subscription") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), emptyList()) + val newSources = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + audioSubscription.onConferenceSourceAdded(newSources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + } + } + + context("onConferenceSourceAdded with Exclude subscription") { + should("add new sources that don't match exclude criteria") { + val initialSources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + val excludeSubscription = ReceiverAudioSubscriptionMessage( + all = true, + exclude = listOf("source2") + ) + audioSubscription.updateSubscription(excludeSubscription, initialSources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 not excluded + val newSources = setOf( + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3") + ) + audioSubscription.onConferenceSourceAdded(newSources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 still not excluded + audioSubscription.isSsrcWanted(1002L) shouldBe false // source2 excluded + audioSubscription.isSsrcWanted(1003L) shouldBe true // source3 not excluded + } + } + + context("onConferenceSourceRemoved") { + should("remove sources from wanted list for Include subscription") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1", "source2") + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true + val sourcesToRemove = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + audioSubscription.onConferenceSourceRemoved(sourcesToRemove) + audioSubscription.isSsrcWanted(1001L) shouldBe false // removed from wanted list + audioSubscription.isSsrcWanted(1002L) shouldBe true // still wanted + } + + should("remove sources from excluded set for Exclude subscription") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + val excludeSubscription = ReceiverAudioSubscriptionMessage( + all = true, + exclude = listOf("source1") + ) + audioSubscription.updateSubscription(excludeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe false // excluded + audioSubscription.isSsrcWanted(1002L) shouldBe true // not excluded, wanted via all + val sourcesToRemove = setOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + audioSubscription.onConferenceSourceRemoved(sourcesToRemove) + // source1 is no longer in the conference, so it drops out of the excluded set; under `all` an + // unknown SSRC is wanted again. + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true // still wanted + } + + should("handle removal of non-existent sources") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1") + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + val sourcesToRemove = setOf( + AudioSourceDesc(9999L, "endpoint999", "source999") + ) + audioSubscription.onConferenceSourceRemoved(sourcesToRemove) + audioSubscription.isSsrcWanted(1001L) shouldBe true // unaffected + } + } + + context("Endpoint with multiple sources") { + should("handle subscription for endpoint with multiple audio sources") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint1", "source2"), + AudioSourceDesc(1003L, "endpoint1", "source3"), + AudioSourceDesc(1004L, "endpoint2", "source4") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1", "source3", "source4"), + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // source1 included + audioSubscription.isSsrcWanted(1002L) shouldBe false // source2 not included + audioSubscription.isSsrcWanted(1003L) shouldBe true // source3 included + audioSubscription.isSsrcWanted(1004L) shouldBe true // source4 included + } + + should("handle All subscription for endpoint with multiple sources") { + val sources = listOf( + AudioSourceDesc(2001L, "endpoint1", "source1"), + AudioSourceDesc(2002L, "endpoint1", "source2"), + AudioSourceDesc(2003L, "endpoint1", "source3") + ) + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), sources) + audioSubscription.isSsrcWanted(2001L) shouldBe true + audioSubscription.isSsrcWanted(2002L) shouldBe true + audioSubscription.isSsrcWanted(2003L) shouldBe true + } + + should("handle None subscription for endpoint with multiple sources") { + val sources = listOf( + AudioSourceDesc(3001L, "endpoint1", "source1"), + AudioSourceDesc(3002L, "endpoint1", "source2"), + AudioSourceDesc(3003L, "endpoint1", "source3") + ) + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = false), sources) + audioSubscription.isSsrcWanted(3001L) shouldBe false + audioSubscription.isSsrcWanted(3002L) shouldBe false + audioSubscription.isSsrcWanted(3003L) shouldBe false + } + } + + context("Subscription updates") { + should("update from None to All") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = false), emptyList()) + audioSubscription.isSsrcWanted(1001L) shouldBe false + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), emptyList()) + audioSubscription.isSsrcWanted(1001L) shouldBe true + } + + should("update from All to Include") { + audioSubscription.updateSubscription(ReceiverAudioSubscriptionMessage(all = true), emptyList()) + audioSubscription.isSsrcWanted(1001L) shouldBe true + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1") + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe false + } + + should("update from Include to Exclude") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3") + ) + val includeSubscription = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1", "source2") + ) + audioSubscription.updateSubscription(includeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true + audioSubscription.isSsrcWanted(1003L) shouldBe false + val excludeSubscription = ReceiverAudioSubscriptionMessage( + all = true, + exclude = listOf("source2") + ) + audioSubscription.updateSubscription(excludeSubscription, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true // not excluded + audioSubscription.isSsrcWanted(1002L) shouldBe false // excluded + audioSubscription.isSsrcWanted(1003L) shouldBe true // not excluded + } + + should("update Include subscription with different list") { + val sources = listOf( + AudioSourceDesc(1001L, "endpoint1", "source1"), + AudioSourceDesc(1002L, "endpoint2", "source2"), + AudioSourceDesc(1003L, "endpoint3", "source3") + ) + val includeSubscription1 = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source1", "source2") + ) + audioSubscription.updateSubscription(includeSubscription1, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe true + audioSubscription.isSsrcWanted(1002L) shouldBe true + audioSubscription.isSsrcWanted(1003L) shouldBe false + val includeSubscription2 = ReceiverAudioSubscriptionMessage( + all = false, + include = listOf("source2", "source3") + ) + audioSubscription.updateSubscription(includeSubscription2, sources) + audioSubscription.isSsrcWanted(1001L) shouldBe false // not in new include list + audioSubscription.isSsrcWanted(1002L) shouldBe true // in new include list + audioSubscription.isSsrcWanted(1003L) shouldBe true // in new include list + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt new file mode 100644 index 0000000..35cc680 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/ConferenceTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright @ 2020 - Present, 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import io.kotest.matchers.shouldBe +import io.mockk.mockk +import org.jitsi.ConfigTest +import org.jitsi.nlj.DebugStateMode +import org.jxmpp.jid.impl.JidCreate + +/** + * This is a high-level test for [Conference] and related functionality. + */ +class ConferenceTest : ConfigTest() { + private val videobridge = mockk(relaxed = true) + + init { + val name = JidCreate.entityBareFrom("roomName@somedomain.com") + + context("Adding local endpoints should work") { + with(Conference(videobridge, "id", name, null, false)) { + endpointCount shouldBe 0 + // TODO cover the case when they're true + createLocalEndpoint("abcdabcd", true, false, false, false, false, false) + endpointCount shouldBe 1 + DebugStateMode.entries.forEach { mode -> + getDebugState(mode, null).shouldBeValidJsonConf() + } + } + } + context("Creating relays should work") { + with(Conference(videobridge, "id", name, null, false)) { + hasRelays() shouldBe false + createRelay("relay-id", "mesh-id", true, true) + hasRelays() shouldBe true + DebugStateMode.entries.forEach { mode -> + getDebugState(mode, null).shouldBeValidJsonConf() + } + } + } + } +} + +fun ObjectNode.shouldBeValidJsonConf() = ObjectMapper().readTree(this.toString()) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt new file mode 100644 index 0000000..355700b --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/EndpointConnectionStatusMonitorTest.kt @@ -0,0 +1,230 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.inspectors.forAll +import io.kotest.inspectors.forAny +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import org.jitsi.utils.NEVER +import org.jitsi.utils.concurrent.FakeScheduledExecutorService +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.mins +import org.jitsi.utils.secs +import org.jitsi.videobridge.message.EndpointConnectionStatusMessage + +class EndpointConnectionStatusMonitorTest : ShouldSpec({ + isolationMode = IsolationMode.InstancePerLeaf + + val executor = FakeScheduledExecutorService() + val localEp1: Endpoint = mockk { + every { id } returns "1" + every { visitor } returns false + } + val localEp2: Endpoint = mockk { + every { id } returns "2" + every { visitor } returns false + } + val localEp3: Endpoint = mockk { + every { id } returns "3" + every { visitor } returns true + } + val eps = listOf(localEp1, localEp2, localEp3) + + val broadcastMessage = slot() + val broadcastSendToRelays = slot() + val broadcastCalls = mutableListOf>() + + val sendMessageMessage = slot() + val sendMessageDestinationEps = slot>() + val sendMessageSendToRelays = slot() + val sendMessageCalls = mutableListOf, Boolean>>() + + val conference: Conference = mockk { + every { localEndpoints } returns eps + every { broadcastMessage(capture(broadcastMessage), capture(broadcastSendToRelays)) } answers { + broadcastCalls += Pair(broadcastMessage.captured, broadcastSendToRelays.captured) + } + every { + sendMessage( + capture(sendMessageMessage), + capture(sendMessageDestinationEps), + capture(sendMessageSendToRelays) + ) + } answers { + sendMessageCalls += + Triple( + sendMessageMessage.captured, + sendMessageDestinationEps.captured, + sendMessageSendToRelays.captured + ) + } + } + + val clock = executor.clock + + val monitor = EndpointConnectionStatusMonitor(conference, executor, LoggerImpl("test"), clock).apply { + start() + } + + context("EndpointConnectionStatusMonitor") { + context("when endpoints have no activity") { + eps.forEach { + every { it.lastIncomingActivity } returns NEVER + } + context("but haven't been around longer than first transfer timeout") { + eps.forEach { + every { it.creationTime } returns clock.instant() + } + executor.runOne() + should("not fire any events") { + sendMessageCalls.shouldBeEmpty() + broadcastCalls.shouldBeEmpty() + } + } + context("and have been around longer than first transfer timeout") { + eps.forEach { + every { it.creationTime } returns clock.instant() + } + clock.elapse(1.mins) + executor.runOne() + should("fire broadcast events for the non-visitor local endpoints") { + sendMessageCalls.shouldBeEmpty() + broadcastCalls shouldHaveSize 2 + broadcastCalls.forAny { (msg, sendToRelays) -> + sendToRelays && msg.endpoint == "1" && msg.active == "false" + sendToRelays shouldBe true + msg.endpoint shouldBe "1" + msg.active shouldBe "false" + } + broadcastCalls.forAny { (msg, sendToRelays) -> + sendToRelays shouldBe true + msg.endpoint shouldBe "2" + msg.active shouldBe "false" + } + broadcastCalls.forAll { (msg, sendToOcto) -> + msg.endpoint shouldNotBe "3" + } + } + context("and then become active") { + clock.elapse(30.secs) + eps.forEach { + every { it.lastIncomingActivity } returns clock.instant() + } + executor.runOne() + should("fire broadcast active events for the local endpoints") { + sendMessageCalls.shouldBeEmpty() + // 2 from the messages when it went inactive, and 2 more now for going active + broadcastCalls shouldHaveSize 4 + broadcastCalls.forAny { (msg, sendToRelays) -> + sendToRelays shouldBe true + msg.endpoint shouldBe "1" + msg.active shouldBe "true" + } + broadcastCalls.forAny { (msg, sendToRelays) -> + sendToRelays shouldBe true + msg.endpoint shouldBe "2" + msg.active shouldBe "true" + } + broadcastCalls.forAll { (msg, sendToOcto) -> + msg.endpoint shouldNotBe "3" + } + } + } + } + } + context("when the endpoints have had activity") { + eps.forEach { + every { it.creationTime } returns clock.instant() + every { it.lastIncomingActivity } returns clock.instant() + } + context("that is within maxInactivityLimit") { + clock.elapse(1.secs) + executor.runOne() + should("not fire any events") { + sendMessageCalls.shouldBeEmpty() + broadcastCalls.shouldBeEmpty() + } + } + context("but not within maxInactivityLimit") { + clock.elapse(1.mins) + executor.runOne() + should("fire inactive events for non-visitor endpoints") { + sendMessageCalls.shouldBeEmpty() + broadcastCalls shouldHaveSize 2 + broadcastCalls.forAny { (msg, sendToRelays) -> + sendToRelays shouldBe true + msg.endpoint shouldBe "1" + msg.active shouldBe "false" + } + broadcastCalls.forAny { (msg, sendToRelays) -> + sendToRelays shouldBe true + msg.endpoint shouldBe "2" + msg.active shouldBe "false" + } + broadcastCalls.forAll { (msg, sendToOcto) -> + msg.endpoint shouldNotBe "3" + } + } + context("but then one becomes active") { + every { localEp1.lastIncomingActivity } returns clock.instant() + clock.elapse(1.secs) + executor.runOne() + should("fire an active event for that ep") { + sendMessageCalls.shouldBeEmpty() + broadcastCalls shouldHaveSize 3 + broadcastCalls.last().let { (msg, sendToRelays) -> + sendToRelays shouldBe true + msg.endpoint shouldBe "1" + msg.active shouldBe "true" + } + } + } + context("and then a new ep joins") { + every { conference.getLocalEndpoint("4") } returns mockk { every { id } returns "4" } + monitor.endpointConnected("4") + should("update the new endpoint of the other non-visitor endpoints' statuses") { + sendMessageCalls shouldHaveSize 2 + sendMessageCalls.forAll { (_, destEps, sendToRelays) -> + destEps shouldHaveSize 1 + destEps.first().id shouldBe "4" + sendToRelays shouldBe false + } + sendMessageCalls.forAny { (msg, _, _) -> + msg.endpoint shouldBe "1" + msg.active shouldBe "false" + } + sendMessageCalls.forAny { (msg, _, _) -> + msg.endpoint shouldBe "2" + msg.active shouldBe "false" + } + sendMessageCalls.forAll { (msg, _, _) -> + msg.endpoint shouldNotBe "3" + } + } + } + } + } + } +}) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/JvbLastNKtTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/JvbLastNKtTest.kt new file mode 100644 index 0000000..1968a21 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/JvbLastNKtTest.kt @@ -0,0 +1,35 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class JvbLastNKtTest : ShouldSpec({ + context("calculateLastN") { + should("return the correct result") { + calculateLastN(-1, -1) shouldBe -1 + calculateLastN(-1, 10) shouldBe 10 + calculateLastN(10, -1) shouldBe 10 + calculateLastN(2, 3) shouldBe 2 + calculateLastN(-1, -1, -1) shouldBe -1 + calculateLastN(-1, 10, -1) shouldBe 10 + calculateLastN(2, 3, 3) shouldBe 2 + calculateLastN(2, 3, 33) shouldBe 2 + } + } +}) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt new file mode 100644 index 0000000..1ad89dc --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/KotestProjectConfig.kt @@ -0,0 +1,36 @@ +/* + * Copyright @ 2022 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge + +import io.kotest.core.config.AbstractProjectConfig +import org.jitsi.metaconfig.MetaconfigSettings +import org.jitsi.videobridge.metrics.VideobridgeMetricsContainer + +class KotestProjectConfig : AbstractProjectConfig() { + override suspend fun beforeProject() = super.beforeProject().also { + // The only purpose of config caching is performance. We always want caching disabled in tests (so we can + // freely modify the config without affecting other tests executing afterwards). + MetaconfigSettings.cacheEnabled = false + + // By default, MetricsContainer throws an exception when attempting to register a metric with the same name + // as an existing metric. MetricsContainer is a singleton with a static instance and is preserved across tests. + // We want this flag disabled so that instancing classes that register metrics (such as Videobridge) does not + // result in runtime exceptions. Note that metrics with matching name and type are returned this way. + // As such, tests that rely on metric values changing may require additional steps. + VideobridgeMetricsContainer.instance.checkForNameConflicts = false + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/MediaSourceFactoryTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/MediaSourceFactoryTest.kt new file mode 100644 index 0000000..e896e0b --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/MediaSourceFactoryTest.kt @@ -0,0 +1,44 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * Copyright @ 2021 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.videobridge.xmpp.MediaSourceFactory +import org.jitsi.xmpp.extensions.colibri.SourcePacketExtension + +fun createSource(ssrc: Long) = SourcePacketExtension().apply { this.ssrc = ssrc } + +// TODO port MediaSourceFactoryTest.java to kotlin and unify with this class +class MediaSourceFactoryTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + init { + context("MediaSourceFactory") { + context("should throw an exception if there's no source name in the packet extension") { + val videoSource: SourcePacketExtension = createSource(1) + + val exception = shouldThrow { + MediaSourceFactory.createMediaSources(listOf(videoSource), emptyList()) + } + exception.message shouldBe "The 'name' is missing in the source description" + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/RecentSpeakersListTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/RecentSpeakersListTest.kt new file mode 100644 index 0000000..18648e4 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/RecentSpeakersListTest.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2021 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class RecentSpeakersListTest : ShouldSpec() { + init { + context("Adding") { + RecentSpeakersList(3).apply { + promote("a") shouldBe true + recentSpeakers shouldBe listOf("a") + + promote("a") shouldBe false + recentSpeakers shouldBe listOf("a") + + promote("b") shouldBe true + recentSpeakers shouldBe listOf("b", "a") + + promote("a") shouldBe true + recentSpeakers shouldBe listOf("a", "b") + + promote("c") shouldBe true + recentSpeakers shouldBe listOf("c", "a", "b") + + promote("b") shouldBe true + recentSpeakers shouldBe listOf("b", "c", "a") + + promote("d") shouldBe true + recentSpeakers shouldBe listOf("d", "b", "c") + + promote("d") shouldBe false + recentSpeakers shouldBe listOf("d", "b", "c") + } + } + context("Removing") { + RecentSpeakersList(3).apply { + promote("a") shouldBe true + promote("b") shouldBe true + promote("c") shouldBe true + promote("d") shouldBe true + promote("e") shouldBe true + recentSpeakers shouldBe listOf("e", "d", "c") + + removeAllExcept(listOf("a", "b", "c", "d", "e", "and another one")) shouldBe false + recentSpeakers shouldBe listOf("e", "d", "c") + + removeAllExcept(listOf("a", "b", "c", "d")) shouldBe true + recentSpeakers shouldBe listOf("d", "c", "b") + + promote("e") shouldBe true + recentSpeakers shouldBe listOf("e", "d", "c") + + removeAllExcept(listOf("a", "b")) shouldBe true + recentSpeakers shouldBe listOf("b", "a") + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/SpeechActivityTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/SpeechActivityTest.kt new file mode 100644 index 0000000..a531200 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/SpeechActivityTest.kt @@ -0,0 +1,125 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.collections.shouldContainInOrder +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk + +class SpeechActivityTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val a = mockEndpoint("a") + private val b = mockEndpoint("b") + private val c = mockEndpoint("c") + private val d = mockEndpoint("d") + private val conferenceSpeechActivity = ConferenceSpeechActivity(object : ConferenceSpeechActivity.Listener { + override fun lastNEndpointsChanged() {} + override fun recentSpeakersChanged( + recentSpeakers: List, + dominantSpeakerChanged: Boolean, + silence: Boolean + ) {} + }) + + init { + context("Should maintain correct order when everyone has video") { + conferenceSpeechActivity.endpointsChanged(listOf(a, b, c)) + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(a, b, c) + + conferenceSpeechActivity.activeSpeakerChanged(b.id) + conferenceSpeechActivity.dominantEndpoint shouldBe b + + conferenceSpeechActivity.activeSpeakerChanged(c.id) + conferenceSpeechActivity.dominantEndpoint shouldBe c + + conferenceSpeechActivity.activeSpeakerChanged(a.id) + conferenceSpeechActivity.dominantEndpoint shouldBe a + + conferenceSpeechActivity.orderedEndpoints shouldContainInOrder listOf(a, c, b) + + conferenceSpeechActivity.endpointsChanged(listOf(b, c)) + conferenceSpeechActivity.orderedEndpoints shouldContainInOrder listOf(c, b) + + conferenceSpeechActivity.activeSpeakerChanged(c.id) + conferenceSpeechActivity.dominantEndpoint shouldBe c + + conferenceSpeechActivity.activeSpeakerChanged(b.id) + conferenceSpeechActivity.dominantEndpoint shouldBe b + + conferenceSpeechActivity.endpointsChanged(listOf(d, b, c)) + conferenceSpeechActivity.dominantEndpoint shouldBe b + + conferenceSpeechActivity.activeSpeakerChanged(d.id) + conferenceSpeechActivity.dominantEndpoint shouldBe d + conferenceSpeechActivity.orderedEndpoints shouldContainInOrder listOf(d, b, c) + } + + context("Should maintain correct order when endpoints disable/enable video") { + conferenceSpeechActivity.endpointsChanged(listOf(a, b, c, d)) + conferenceSpeechActivity.activeSpeakerChanged(d.id) + conferenceSpeechActivity.activeSpeakerChanged(c.id) + conferenceSpeechActivity.activeSpeakerChanged(b.id) + conferenceSpeechActivity.activeSpeakerChanged(a.id) + + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(a, b, c, d) + + every { a.hasVideoAvailable() } returns false + conferenceSpeechActivity.updateLastNEndpoints() + conferenceSpeechActivity.dominantEndpoint shouldBe a + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(b, c, d, a) + + every { a.hasVideoAvailable() } returns true + conferenceSpeechActivity.updateLastNEndpoints() + conferenceSpeechActivity.dominantEndpoint shouldBe a + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(a, b, c, d) + + every { b.hasVideoAvailable() } returns false + conferenceSpeechActivity.updateLastNEndpoints() + conferenceSpeechActivity.dominantEndpoint shouldBe a + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(a, c, d, b) + + conferenceSpeechActivity.activeSpeakerChanged(b.id) + conferenceSpeechActivity.dominantEndpoint shouldBe b + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(a, c, d, b) + + every { b.hasVideoAvailable() } returns true + conferenceSpeechActivity.updateLastNEndpoints() + conferenceSpeechActivity.dominantEndpoint shouldBe b + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(b, a, c, d) + + every { a.hasVideoAvailable() } returns false + every { b.hasVideoAvailable() } returns false + every { c.hasVideoAvailable() } returns false + every { d.hasVideoAvailable() } returns false + conferenceSpeechActivity.updateLastNEndpoints() + conferenceSpeechActivity.activeSpeakerChanged(a.id) + conferenceSpeechActivity.dominantEndpoint shouldBe a + conferenceSpeechActivity.orderedEndpoints shouldContainExactly listOf(a, b, c, d) + } + } + + companion object { + fun mockEndpoint(endpointId: String) = mockk().apply { + every { id } returns endpointId + every { hasVideoAvailable() } returns true + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/SsrcCacheTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/SsrcCacheTest.kt new file mode 100644 index 0000000..56a67ad --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/SsrcCacheTest.kt @@ -0,0 +1,162 @@ +/* + * Copyright @ 2020 - Present, 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.ints.shouldBeLessThanOrEqual +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.rtp.AudioRtpPacket +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.videobridge.message.AudioSourcesMap +import org.jitsi.videobridge.message.BridgeChannelMessage +import org.jitsi.videobridge.relay.AudioSourceDesc +import java.util.Random + +class SsrcCacheTest : ShouldSpec() { + private val limit = 50 + private val ep = Ep() + private val logger: Logger = LoggerImpl(javaClass.name) + private val audioSsrcCache = AudioSsrcCache(limit, ep, false, logger) + + private val seed = System.currentTimeMillis() + private val random = Random(seed) + private val streams = ArrayList() + private val checker = PacketChecker() + + init { + for (i in 0 until 4 * limit) { + val ssrc = random.nextLong().and(0xFFFF_FFFFL) + streams.add(PacketGenerator(ssrc, random)) + } + + for (i in 0 until 100000) { + /* Generate packets for the highest ranked streams */ + for (j in 0 until 10) { + val packet = streams[j].nextPacket() + audioSsrcCache.rewriteRtp(packet) + checker.checkPacket(packet) + } + + /* Randomly select a stream and move to head of the list */ + val pos = random.nextInt(streams.size) + val nextSpeaker = streams.removeAt(pos) + streams.add(0, nextSpeaker) + } + + checker.size() shouldBeLessThanOrEqual limit + + should("assign a stable mid per slot when mid demux is enabled") { + val midEp = Ep() + val cache = AudioSsrcCache(2, midEp, midDemux = true, logger) + + cache.rewriteRtp(PacketGenerator(1001L, random).nextPacket()) + cache.rewriteRtp(PacketGenerator(1002L, random).nextPacket()) + // The pool (size 2) is now full; a third source remaps the least-recently-used slot. + cache.rewriteRtp(PacketGenerator(1003L, random).nextPacket()) + + val msgs = midEp.sentMessages.filterIsInstance() + msgs.size shouldBe 3 + val first = msgs[0].mappedSources.single() + val second = msgs[1].mappedSources.single() + val third = msgs[2].mappedSources.single() + + first.mid shouldBe "a0" + second.mid shouldBe "a1" + // The remap reuses the evicted slot, so it keeps that slot's mid and send SSRC. + third.mid shouldBe "a0" + third.ssrc shouldBe first.ssrc + + cache.getMidBySsrc(first.ssrc) shouldBe "a0" + cache.getMidBySsrc(second.ssrc) shouldBe "a1" + } + + should("not assign mids when mid demux is disabled") { + val plainEp = Ep() + val cache = AudioSsrcCache(2, plainEp, midDemux = false, logger) + + cache.rewriteRtp(PacketGenerator(2001L, random).nextPacket()) + + val mapping = plainEp.sentMessages.filterIsInstance() + .flatMap { it.mappedSources }.single() + mapping.mid shouldBe null + cache.getMidBySsrc(mapping.ssrc) shouldBe null + } + } +} + +private class Ep : SsrcRewriter { + + private var nextSendSsrc = 1L + + override fun findVideoSourceProps(ssrc: Long): MediaSourceDesc? = null + + override fun findAudioSourceProps(ssrc: Long): AudioSourceDesc? = + AudioSourceDesc(ssrc, "anon-$ssrc", "anon-$ssrc-a0") + + val sentMessages = mutableListOf() + + override fun getNextSendSsrc(): Long { + synchronized(this) { + return nextSendSsrc++ + } + } + + override fun sendMessage(msg: BridgeChannelMessage) { + sentMessages.add(msg) + } +} + +private class PacketGenerator(val ssrc: Long, random: Random) { + private var seq = random.nextInt(0x1_0000) + private var ts = random.nextLong().and(0xFFFF_FFFFL) + private var frameCount = 0 + + fun nextPacket(): AudioRtpPacket { + val buffer = ByteArray(28) + + val audioPacket = AudioRtpPacket(buffer, 0, buffer.size) + audioPacket.ssrc = ssrc + audioPacket.sequenceNumber = seq + audioPacket.timestamp = ts + audioPacket.isMarked = true + + seq = RtpUtils.applySequenceNumberDelta(seq, 1) + ts = RtpUtils.applyTimestampDelta(ts, 960) + ++frameCount + + return audioPacket + } +} + +private class PacketChecker() { + + private val ssrcs = HashMap() + + fun checkPacket(packet: RtpPacket) { + val last = ssrcs.get(packet.ssrc) + if (last != null) { + packet.sequenceNumber shouldBe RtpUtils.applySequenceNumberDelta(last.sequenceNumber, 1) + packet.timestamp shouldBe RtpUtils.applyTimestampDelta(last.timestamp, 960L) + } + ssrcs.put(packet.ssrc, packet) + } + + fun size(): Int = ssrcs.size +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt new file mode 100644 index 0000000..de23dc1 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/VideobridgeTest.kt @@ -0,0 +1,100 @@ +/* + * Copyright @ 2020 - Present, 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.core.test.TestCase +import io.kotest.core.test.TestResult +import io.kotest.matchers.shouldBe +import io.mockk.mockk +import io.mockk.verify +import org.jitsi.config.withNewConfig +import org.jitsi.nlj.DebugStateMode +import org.jitsi.shutdown.ShutdownServiceImpl +import org.jitsi.utils.concurrent.FakeScheduledExecutorService +import org.jitsi.videobridge.metrics.VideobridgeMetrics +import org.jitsi.videobridge.shutdown.ShutdownConfig +import org.jitsi.videobridge.shutdown.ShutdownState +import org.jitsi.videobridge.util.TaskPools + +class VideobridgeTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + private val shutdownService: ShutdownServiceImpl = mockk(relaxed = true) + private val fakeExecutor = FakeScheduledExecutorService() + private val videobridge = Videobridge(null, shutdownService, mockk(), fakeExecutor.clock) + + override suspend fun beforeAny(testCase: TestCase) = super.beforeAny(testCase).also { + TaskPools.SCHEDULED_POOL = fakeExecutor + } + + override suspend fun afterAny(testCase: TestCase, result: TestResult) = super.afterAny(testCase, result).also { + TaskPools.resetScheduledPool() + // we need this to test shutdown behavior since metrics are preserved across tests + VideobridgeMetrics.currentLocalEndpoints.set(0) + } + + init { + DebugStateMode.entries.forEach { mode -> + context("Debug state should be JSON (mode=$mode)") { + videobridge.getDebugState(null, null, mode).shouldBeValidJson() + } + } + context("Shutdown") { + context("when a conference is active") { + withNewConfig("videobridge.shutdown.graceful-shutdown-min-participants=10") { + repeat(15) { videobridge.localEndpointCreated(false) } + context("starting a graceful shutdown") { + videobridge.shutdown(true) + should("report that shutdown is in progress") { + videobridge.isInGracefulShutdown shouldBe true + videobridge.shutdownState shouldBe ShutdownState.GRACEFUL_SHUTDOWN + } + should("not have started the shutdown yet") { + verify(exactly = 0) { shutdownService.beginShutdown() } + } + context("When the number of participants drops below the threshold") { + repeat(10) { videobridge.localEndpointExpired(false) } + videobridge.shutdownState shouldBe ShutdownState.SHUTTING_DOWN + fakeExecutor.clock.elapse(ShutdownConfig.config.shuttingDownDelay) + fakeExecutor.run() + verify(exactly = 1) { shutdownService.beginShutdown() } + } + context("When the graceful shutdown period expires") { + fakeExecutor.clock.elapse(ShutdownConfig.config.gracefulShutdownMaxDuration) + fakeExecutor.run() + should("go to SHUTTING_DOWN") { + videobridge.shutdownState shouldBe ShutdownState.SHUTTING_DOWN + } + should("and then shut down after shuttingDownDelay") { + fakeExecutor.clock.elapse(ShutdownConfig.config.shuttingDownDelay) + fakeExecutor.run() + verify(exactly = 1) { shutdownService.beginShutdown() } + } + } + } + } + } + } + } +} + +fun ObjectNode.shouldBeValidJson() { + ObjectMapper().readTree(this.toString()) +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt new file mode 100644 index 0000000..018f0f2 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/AllocationSettingsTest.kt @@ -0,0 +1,125 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.config.withNewConfig +import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage + +class AllocationSettingsTest : ShouldSpec() { + init { + context("computeVideoConstraints") { + context("With client which supports source names") { + context("no conversion from endpoint to source takes place") { + val allocationSettings = AllocationSettingsWrapper() + allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage( + onStageSources = listOf("S1", "S2"), + selectedSources = listOf("S3", "S4"), + constraints = mapOf( + "S1" to VideoConstraints(720), + "E1" to VideoConstraints(360) + ) + ) + ) + + allocationSettings.get().onStageSources shouldBe listOf("S1", "S2") + allocationSettings.get().selectedSources shouldBe listOf("S3", "S4") + + allocationSettings.get().onStageEndpoints shouldBe emptyList() + allocationSettings.get().selectedEndpoints shouldBe emptyList() + + allocationSettings.get().videoConstraints shouldBe mapOf( + "S1" to VideoConstraints(720), + // There's no error when endpoint is used in constraint + "E1" to VideoConstraints(360) + ) + } + } + context("with initial constraints different from default default constraints") { + withNewConfig( + """ + videobridge.cc.initial-max-height-px = 0 + videobridge.cc.default-max-height-px = 180 + """.trimIndent() + ) { + should("consider an empty receiver video constraints a change") { + val allocationSettings = AllocationSettingsWrapper() + val changed = + allocationSettings.setBandwidthAllocationSettings(ReceiverVideoConstraintsMessage()) + changed shouldBe true + } + should("consider the default receiver video constraints a change") { + val allocationSettings = AllocationSettingsWrapper() + val changed = + allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage(defaultConstraints = VideoConstraints(maxHeight = 180)) + ) + changed shouldBe true + } + should("not consider the initial receiver video constraints a change") { + val allocationSettings = AllocationSettingsWrapper() + val changed = + allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage(defaultConstraints = VideoConstraints(maxHeight = 0)) + ) + changed shouldBe false + } + should("not apply the default default constraints after they've been set explicitly") { + val allocationSettings = AllocationSettingsWrapper() + allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage(defaultConstraints = VideoConstraints(maxHeight = 720)) + ) + val changed = allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage() + ) + changed shouldBe false + allocationSettings.get().defaultConstraints shouldBe VideoConstraints(maxHeight = 720) + } + } + } + context("With an assumed bandwidth limit set") { + withNewConfig("videobridge.cc.assumed-bandwidth-limit = 10 Mbps") { + should("limit an assumed bandwidth to the desired rate") { + val allocationSettings = AllocationSettingsWrapper() + val changed = allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage(assumedBandwidthBps = 20_000_000) + ) + changed shouldBe true + allocationSettings.get().assumedBandwidthBps shouldBe 10_000_000 + } + should("Not limit a value less than the configured limit") { + val allocationSettings = AllocationSettingsWrapper() + val changed = allocationSettings.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage(assumedBandwidthBps = 2_000_000) + ) + changed shouldBe true + allocationSettings.get().assumedBandwidthBps shouldBe 2_000_000 + } + should("Not consider the default value to be a change") { + val allocationSettings = AllocationSettingsWrapper() + // Set this twice because of the initial vs. default max height difference above. + val constraints = ReceiverVideoConstraintsMessage(assumedBandwidthBps = -1) + allocationSettings.setBandwidthAllocationSettings(constraints) + val changed = allocationSettings.setBandwidthAllocationSettings(constraints) + changed shouldBe false + } + } + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt new file mode 100644 index 0000000..68fc088 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerPerfTest.kt @@ -0,0 +1,172 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import org.jitsi.utils.nanos +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage +import org.jitsi.videobridge.util.endpointIdToSourceName +import java.util.function.Supplier +import kotlin.random.Random +import kotlin.time.ExperimentalTime + +/** + * Keep track of the performance of the bandwidth allocation code. + * + * The following results were obtained on an AWS t3.large instance. The tests for stage and tile view were run + * separately (only one was enabled when the command was run). Shown is the "microseconds per speaker switch", averaged + * across the 10 runs (warmup excluded). The test was run with `mvn clean test -Dtest=BitrateControllerPerfTest`. + * + * Tested three versions of the code: + * 1. Pre-refactoring: on top of 3bf235fb9a2c9ce8caa52de9544f0adcd3933752 + * 2. After refactoring (PR #1530): 98af5107c99e057f51fdf0f60acc2d8aa9b364bb + * 3. After refactoring and optimization (PR #1557): ef237f692677d3e5c81b7050b17103a001d3edec + * + * Tile View: + * pre-ref: 4.16 + * after refactoring: 5.84 + * after optimization: 4.44 + * + * Stage View: + * pre-ref: 4.15 + * after refactoring: 6.56 + * after optimization: 4.48 + * + * + */ +@ExperimentalTime +class BitrateControllerPerfTest : StringSpec() { + private val logger = createLogger() + private val clock = FakeClock() + private val random = Random(93232) + + private val endpointIds = mutableListOf("A", "B", "C", "D", "E", "F", "G", "H", "I", "J") + private val endpoints: MutableList = createEndpoints(*endpointIds.toTypedArray()) + private val bc = BitrateController( + object : BitrateController.EventHandler { + override fun forwardedSourcesChanged(forwardedSources: Set) { } + override fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap + ) { } + override fun keyframeNeeded(endpointId: String?, ssrc: Long) { } + override fun allocationChanged(allocation: BandwidthAllocation) { } + }, + Supplier { endpoints.toList() }, + DiagnosticContext(), + createLogger(), + clock, + ).apply { + // The BC only starts working 10 seconds after it first received media, so fake that. + transformRtp(PacketInfo(VideoRtpPacket(ByteArray(100), 0, 100))) + clock.elapse(15.secs) + + // Adaptivity is disabled when RTX support is not signalled. + addPayloadType(RtxPayloadType(123, mapOf("apt" to "124"))) + } + + init { + "Tile view".config(enabled = false) { + repeat(5) { + run("Warmup", listOf("A", "B", "C", "D", "E"), 180) + } + repeat(10) { + run("Tile view", listOf("A", "B", "C", "D", "E"), 180) + } + } + "Stage view".config(enabled = false) { + repeat(5) { + run("Warmup", listOf("A"), 720) + } + repeat(10) { + run("Stage view", listOf("A"), 720) + } + } + } + + private fun run(testName: String, selectedEndpoints: List, maxFrameHeight: Int) { + val start = System.nanoTime() + bc.lastN = 7 + + // Ramp-up to 5mbps + for (bwe in 0..5_000_000 step 10_000) { + bc.bandwidthChanged(bwe.toLong()) + clock.elapse(100.ms) + } + + bc.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage( + selectedSources = selectedEndpoints.map { endpointIdToSourceName(it) }, + defaultConstraints = VideoConstraints(maxFrameHeight) + ) + ) + bc.endpointOrderingChanged() + + // Change the dominant speaker just a couple of times. + repeat(NUM_SPEAKER_CHANGES) { + endpoints.selectNewDominantSpeaker() + bc.endpointOrderingChanged() + clock.elapse(2.secs) + } + + val stop = System.nanoTime() + val totalDuration = (stop - start).nanos + val microsPerSpeakerChange = ((stop - start).toDouble() / NUM_SPEAKER_CHANGES) / 1000.0 + logger.info("$testName took a total of $totalDuration, $microsPerSpeakerChange µs per speaker change.") + } + + private fun MutableList.selectNewDominantSpeaker() { + val newDominantSpeakerIdx = 1 + random.nextInt(size - 1) + val newDominantSpeaker = this.removeAt(newDominantSpeakerIdx) + this.add(0, newDominantSpeaker) + } +} + +const val NUM_SPEAKER_CHANGES = 1_000_000 + +fun BandwidthAllocation.shouldMatch(other: BandwidthAllocation) { + allocations.size shouldBe other.allocations.size + allocations.forEach { thisSingleAllocation -> + withClue("Allocation for ${thisSingleAllocation.endpointId}") { + val otherSingleAllocation = other.allocations.find { it.endpointId == thisSingleAllocation.endpointId } + otherSingleAllocation.shouldNotBeNull() + thisSingleAllocation.targetLayer?.height shouldBe otherSingleAllocation.targetLayer?.height + thisSingleAllocation.targetLayer?.frameRate shouldBe otherSingleAllocation.targetLayer?.frameRate + } + } +} + +fun List>.shouldMatchInOrder(vararg events: Event) { + size shouldBe events.size + events.forEachIndexed { i, it -> + this[i].bwe shouldBe it.bwe + withClue("bwe=${it.bwe}") { + this[i].event.shouldMatch(it.event) + } + // Ignore this.time + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt new file mode 100644 index 0000000..7e86b83 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTest.kt @@ -0,0 +1,1692 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * Copyright @ 2021 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.Spec +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainInOrder +import io.kotest.matchers.longs.shouldBeWithinPercentageOf +import io.kotest.matchers.shouldBe +import io.mockk.CapturingSlot +import io.mockk.every +import io.mockk.mockk +import org.jitsi.config.setNewConfig +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.VideoType +import org.jitsi.nlj.format.RtxPayloadType +import org.jitsi.nlj.rtp.VideoRtpPacket +import org.jitsi.nlj.util.Bandwidth +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.kbps +import org.jitsi.nlj.util.mbps +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.createLogger +import org.jitsi.utils.ms +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import org.jitsi.videobridge.cc.config.BitrateControllerConfig +import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage +import org.jitsi.videobridge.util.TaskPools +import java.time.Instant +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import java.util.function.Supplier + +class BitrateControllerTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + private val logger = createLogger() + private val clock = FakeClock() + private val bc = BitrateControllerWrapper(createEndpoints("A", "B", "C", "D"), clock = clock) + private val a = bc.endpoints.find { it.id == "A" }!! as TestEndpoint + private val b = bc.endpoints.find { it.id == "B" }!! as TestEndpoint + private val c = bc.endpoints.find { it.id == "C" }!! as TestEndpoint + private val d = bc.endpoints.find { it.id == "D" }!! as TestEndpoint + + override suspend fun beforeSpec(spec: Spec) = super.beforeSpec(spec).also { + // We disable the threshold, causing [BandwidthAllocator] to make a new decision every time BWE changes. This is + // because these tests are designed to test the decisions themselves and not necessarily when they are made. + setNewConfig( + """ + videobridge.cc { + bwe-change-threshold = 0 + // Effectively disable periodic updates. + max-time-between-calculations = 1 hour + } + """.trimIndent(), + true + ) + } + + override suspend fun afterSpec(spec: Spec) = super.afterSpec(spec).also { + bc.bc.expire() + setNewConfig("", true) + } + + init { + context("Expire") { + val captureDelay = CapturingSlot() + val captureDelayTimeunit = CapturingSlot() + val captureCancel = CapturingSlot() + val executor: ScheduledExecutorService = mockk { + every { schedule(any(), capture(captureDelay), capture(captureDelayTimeunit)) } returns mockk { + every { cancel(capture(captureCancel)) } returns true + } + } + TaskPools.SCHEDULED_POOL = executor + val bc = BitrateControllerWrapper(createEndpoints(), clock = clock) + val delayMs = TimeUnit.MILLISECONDS.convert(captureDelay.captured, captureDelayTimeunit.captured) + + delayMs.shouldBeWithinPercentageOf( + BitrateControllerConfig.config.maxTimeBetweenCalculations.toMillis(), + 10.0 + ) + + captureCancel.isCaptured shouldBe false + bc.bc.expire() + captureCancel.isCaptured shouldBe true + + TaskPools.resetScheduledPool() + } + context("Prioritization") { + context("Without selection") { + val sources = createSources("s6", "s5", "s4", "s3", "s2", "s1") + val ordered = prioritize(sources) + ordered.map { it.sourceName } shouldBe listOf("s6", "s5", "s4", "s3", "s2", "s1") + } + context("With one selected") { + val sources = createSources("s6", "s5", "s4", "s3", "s2", "s1") + val ordered = prioritize(sources, listOf("s2")) + ordered.map { it.sourceName } shouldBe listOf("s2", "s6", "s5", "s4", "s3", "s1") + } + context("With multiple selected") { + val sources = createSources("s6", "s5", "s4", "s3", "s2", "s1") + val ordered = prioritize(sources, listOf("s2", "s1", "s5")) + ordered.map { it.sourceName } shouldBe listOf("s2", "s1", "s5", "s6", "s4", "s3") + } + } + + context("Allocation") { + context("Stage view") { + context("When LastN is not set") { + context("and the dominant speaker is on stage") { + listOf(true, false).forEach { screensharing -> + context("With ${if (screensharing) "screensharing" else "camera"}") { + if (screensharing) { + a.mediaSources[0].videoType = VideoType.DESKTOP + } + bc.setEndpointOrdering(a, b, c, d) + bc.setStageView("A-v0") + + bc.bc.allocationSettings.lastN shouldBe -1 + bc.bc.allocationSettings.selectedSources shouldBe emptyList() + bc.bc.allocationSettings.onStageSources shouldBe listOf("A-v0") + + runBweLoop() + + verifyStageView(screensharing) + } + } + } + context("and a non-dominant speaker is on stage") { + bc.setEndpointOrdering(b, a, c, d) + bc.setStageView("A-v0") + + bc.bc.allocationSettings.lastN shouldBe -1 + bc.bc.allocationSettings.selectedSources shouldBe emptyList() + bc.bc.allocationSettings.onStageSources shouldBe listOf("A-v0") + runBweLoop() + + verifyStageView() + } + } + context("and the on-stage screenshare does not fit in the available bandwidth") { + // A screenshare with a single expensive layer, i.e. with nothing cheaper for the allocator to + // fall back to. This is what low frame rate screensharing looks like, because the sender only + // activates its highest resolution encoding. + val screenshare = TestEndpoint( + "S", + arrayOf( + MediaSourceDesc( + arrayOf( + RtpEncodingDesc( + 100L, + arrayOf( + MockRtpLayerDesc( + tid = 0, + eid = 0, + height = 1080, + frameRate = 5.0, + bitrate = 2000.kbps + ) + ) + ) + ), + sourceName = "S-v0", + owner = "S", + videoType = VideoType.DESKTOP + ) + ) + ) + val bc = BitrateControllerWrapper(listOf(screenshare, b, c), clock = clock) + bc.setStageView("S-v0") + + // Not enough for the screenshare even with the oversend allowance, though plenty for thumbnails. + bc.bwe = 300.kbps + + // Nothing is forwarded: bandwidth is reserved for the on-stage source until it reaches its + // "preferred" layer, even when it can not be forwarded at all. This is deliberate, to prevent + // flapping while the estimate ramps up (thumbnails would be enabled here only to be disabled + // when the estimate improves enough for the on-stage source to claim the whole budget). + bc.allocationHistory.last().event.forwardedSources shouldBe emptySet() + } + context("When LastN=0") { + // LastN=0 is used when the client goes in "audio-only" mode. + bc.setEndpointOrdering(a, b, c, d) + bc.setStageView("A", lastN = 0) + + bc.bc.allocationSettings.lastN shouldBe 0 + bc.bc.allocationSettings.selectedSources shouldBe emptyList() + bc.bc.allocationSettings.onStageSources shouldBe listOf("A") + + runBweLoop() + + verifyLastN0() + } + context("When LastN=1") { + // LastN=1 is used when the client goes in "audio-only" mode, but someone starts a screenshare. + context("and the dominant speaker is on-stage") { + bc.setEndpointOrdering(a, b, c, d) + bc.setStageView("A-v0", lastN = 1) + + bc.bc.allocationSettings.lastN shouldBe 1 + bc.bc.allocationSettings.selectedSources shouldBe emptyList() + bc.bc.allocationSettings.onStageSources shouldBe listOf("A-v0") + + runBweLoop() + + verifyStageViewLastN1() + } + context("and a non-dominant speaker is on stage") { + bc.setEndpointOrdering(b, a, c, d) + bc.setStageView("A-v0", lastN = 1) + + bc.bc.allocationSettings.lastN shouldBe 1 + bc.bc.allocationSettings.selectedSources shouldBe emptyList() + bc.bc.allocationSettings.onStageSources shouldBe listOf("A-v0") + + runBweLoop() + + verifyStageViewLastN1() + } + } + } + context("Tile view") { + bc.setEndpointOrdering(a, b, c, d) + bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0") + + bc.bc.allocationSettings.lastN shouldBe -1 + bc.bc.allocationSettings.selectedSources shouldBe + listOf("A-v0", "B-v0", "C-v0", "D-v0") + + context("When LastN is not set") { + runBweLoop() + + verifyTileView() + } + context("When LastN=0") { + bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0", lastN = 0) + runBweLoop() + + verifyLastN0() + } + context("When LastN=1") { + bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0", lastN = 1) + runBweLoop() + + verifyTileViewLastN1() + } + } + context("Tile view 360p") { + bc.setEndpointOrdering(a, b, c, d) + bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0", maxFrameHeight = 360) + + bc.bc.allocationSettings.lastN shouldBe -1 + // The legacy API (currently used by jitsi-meet) uses "selected count > 0" to infer TileView, + // and in tile view we do not use selected endpoints. + bc.bc.allocationSettings.selectedSources shouldBe + listOf("A-v0", "B-v0", "C-v0", "D-v0") + + context("When LastN is not set") { + runBweLoop() + + verifyTileView360p() + } + context("When LastN=0") { + bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0", lastN = 0, maxFrameHeight = 360) + runBweLoop() + + verifyLastN0() + } + context("When LastN=1") { + bc.setTileView("A-v0", "B-v0", "C-v0", "D-v0", lastN = 1, maxFrameHeight = 360) + runBweLoop() + + verifyTileViewLastN1(360) + } + } + context("Selected sources should override the dominant speaker (with new signaling)") { + // A is dominant speaker, A and B are selected. With LastN=2 we should always forward the selected + // sources regardless of who is speaking. + // The exact flow of this scenario was taken from a (non-jitsi-meet) client. + bc.setEndpointOrdering(a, b, c, d) + bc.bc.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage( + selectedSources = listOf("A-v0", "B-v0"), + constraints = mapOf("A-v0" to VideoConstraints(720), "B-v0" to VideoConstraints(720)) + ) + ) + + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(720), + "B-v0" to VideoConstraints(720), + "C-v0" to VideoConstraints(180), + "D-v0" to VideoConstraints(180) + ) + + bc.bc.setBandwidthAllocationSettings(ReceiverVideoConstraintsMessage(lastN = 2)) + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(720), + "B-v0" to VideoConstraints(720), + "C-v0" to VideoConstraints(0), + "D-v0" to VideoConstraints(0) + ) + + bc.bc.allocationSettings.lastN shouldBe 2 + bc.bc.allocationSettings.selectedSources shouldBe listOf("A-v0", "B-v0") + + clock.elapse(20.secs) + bc.bwe = 10.mbps + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + clock.elapse(2.secs) + // B becomes dominant speaker. + bc.setEndpointOrdering(b, a, c, d) + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + clock.elapse(2.secs) + bc.bc.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage( + constraints = mapOf("A-v0" to VideoConstraints(360), "B-v0" to VideoConstraints(360)) + ) + ) + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(360), + "B-v0" to VideoConstraints(360), + "C-v0" to VideoConstraints(0), + "D-v0" to VideoConstraints(0) + ) + + clock.elapse(2.secs) + // This should change nothing, the selection didn't change. + bc.bc.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage(selectedSources = listOf("A-v0", "B-v0")) + ) + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + clock.elapse(2.secs) + bc.bc.setBandwidthAllocationSettings(ReceiverVideoConstraintsMessage(lastN = -1)) + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(360), + "B-v0" to VideoConstraints(360), + "C-v0" to VideoConstraints(180), + "D-v0" to VideoConstraints(180) + ) + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0", "C-v0", "D-v0")) + + bc.bc.setBandwidthAllocationSettings(ReceiverVideoConstraintsMessage(lastN = 2)) + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(360), + "B-v0" to VideoConstraints(360), + "C-v0" to VideoConstraints(0), + "D-v0" to VideoConstraints(0) + ) + clock.elapse(2.secs) + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + clock.elapse(2.secs) + // D is now dominant speaker, but it should not override the selected endpoints. + bc.setEndpointOrdering(d, b, a, c) + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + bc.bwe = 10.mbps + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + clock.elapse(2.secs) + bc.bwe = 0.mbps + clock.elapse(2.secs) + bc.bwe = 10.mbps + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + + clock.elapse(2.secs) + // C is now dominant speaker, but it should not override the selected endpoints. + bc.setEndpointOrdering(c, d, a, b) + bc.forwardedSourcesHistory.last().event.shouldBe(setOf("A-v0", "B-v0")) + } + } + } + + private fun runBweLoop() { + for (bwe in 0..5_000_000 step 10_000) { + bc.bwe = bwe.bps + clock.elapse(100.ms) + } + logger.info("Forwarded sources history: ${bc.forwardedSourcesHistory}") + logger.info("Effective constraints history: ${bc.effectiveConstraintsHistory}") + logger.info("Allocation history: ${bc.allocationHistory}") + } + + private fun verifyStageViewScreensharing() { + // At this stage the purpose of this is just to document current behavior. + // TODO: The results with bwe==-1 are wrong. + bc.forwardedSourcesHistory.map { it.event }.shouldContainInOrder( + setOf("A-v0"), + setOf("A-v0", "B-v0"), + setOf("A-v0", "B-v0", "C-v0"), + setOf("A-v0", "B-v0", "C-v0", "D-v0") + ) + + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(720), + "B-v0" to VideoConstraints(180), + "C-v0" to VideoConstraints(180), + "D-v0" to VideoConstraints(180) + ) + + // At this stage the purpose of this is just to document current behavior. + // TODO: the allocations for bwe=-1 are wrong. + bc.allocationHistory.removeIf { it.bwe < 0.bps } + + // Note that the thumbnails are enabled once the screenshare has reached full resolution at its cheapest + // frame rate (720p/7.5 at 660 kbps; the first thumbnail fits alongside it at bwe=710 kbps), and from then + // on the screenshare's frame rate competes with them. + bc.allocationHistory.shouldMatchInOrder( + Event( + 0.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ), + oversending = true + ) + ), + Event( + 160.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ), + oversending = true + ) + ), + Event( + 660.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 710.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 760.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 810.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 860.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 910.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 960.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 1010.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 1060.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 1120.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd7_5), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 1470.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 1520.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 1570.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 1620.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 1670.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 1720.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 1780.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd15), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 2300.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 2350.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 2400.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 2460.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ) + ) + } + + private fun verifyStageView(screensharing: Boolean = false) { + when (screensharing) { + true -> verifyStageViewScreensharing() + false -> verifyStageViewCamera() + } + } + + private fun verifyStageViewCamera() { + // At this stage the purpose of this is just to document current behavior. + // TODO: The results with bwe==-1 are wrong. + bc.forwardedSourcesHistory.removeIf { it.bwe < 0.bps } + bc.forwardedSourcesHistory.map { it.event }.shouldContainInOrder( + setOf("A-v0"), + setOf("A-v0", "B-v0"), + setOf("A-v0", "B-v0", "C-v0"), + setOf("A-v0", "B-v0", "C-v0", "D-v0") + ) + // TODO add forwarded sources history here + + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(720), + "B-v0" to VideoConstraints(180), + "C-v0" to VideoConstraints(180), + "D-v0" to VideoConstraints(180) + ) + + // At this stage the purpose of this is just to document current behavior. + // TODO: the allocations for bwe=-1 are wrong. + bc.allocationHistory.removeIf { it.bwe < 0.bps } + + bc.allocationHistory.shouldMatchInOrder( + Event( + 50.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ), + oversending = false + ) + ), + Event( + 100.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 150.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 500.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 550.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 600.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 650.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 700.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 750.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 800.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 850.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 900.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 960.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 2150.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 2200.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 2250.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 2300.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 2350.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 2400.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 2460.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ) + ) + } + + private fun verifyLastN0() { + // No video forwarded even with high BWE. + bc.forwardedSourcesHistory.size shouldBe 0 + + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(0), + "B-v0" to VideoConstraints(0), + "C-v0" to VideoConstraints(0), + "D-v0" to VideoConstraints(0) + ) + + // TODO: The history contains 3 identical elements, which is probably a bug. + bc.allocationHistory.last().event.shouldMatch( + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ) + } + + private fun verifyStageViewLastN1() { + // At this stage the purpose of this is just to document current behavior. + // TODO: The results with bwe==-1 are wrong. + bc.forwardedSourcesHistory.removeIf { it.bwe < 0.bps } + + bc.forwardedSourcesHistory.map { it.event }.shouldContainInOrder( + setOf("A-v0") + ) + + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(720), + "B-v0" to VideoConstraints(0), + "C-v0" to VideoConstraints(0), + "D-v0" to VideoConstraints(0) + ) + + // At this stage the purpose of this is just to document current behavior. + // TODO: the allocations for bwe=-1 are wrong. + bc.allocationHistory.removeIf { it.bwe < 0.bps } + + bc.allocationHistory.shouldMatchInOrder( + Event( + 50.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 100.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 150.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 500.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 2010.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = hd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ) + ) + } + + private fun verifyTileView() { + // At this stage the purpose of this is just to document current behavior. + // TODO: The results with bwe==-1 are wrong. + bc.forwardedSourcesHistory.removeIf { it.bwe < 0.bps } + bc.forwardedSourcesHistory.map { it.event }.shouldContainInOrder( + setOf("A-v0"), + setOf("A-v0", "B-v0"), + setOf("A-v0", "B-v0", "C-v0"), + setOf("A-v0", "B-v0", "C-v0", "D-v0") + ) + + bc.allocationHistory.shouldMatchInOrder( + Event( + (-1).bps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 50.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 100.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 150.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 200.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 250.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 300.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 350.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 400.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 450.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 500.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 550.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 610.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ) + ) + } + + private fun verifyTileView360p() { + // At this stage the purpose of this is just to document current behavior. + // TODO: The results with bwe==-1 are wrong. + bc.forwardedSourcesHistory.removeIf { it.bwe < 0.bps } + bc.forwardedSourcesHistory.map { it.event }.shouldContainInOrder( + setOf("A-v0"), + setOf("A-v0", "B-v0"), + setOf("A-v0", "B-v0", "C-v0"), + setOf("A-v0", "B-v0", "C-v0", "D-v0") + ) + + bc.allocationHistory.shouldMatchInOrder( + Event( + (-1).bps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 50.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ), + oversending = false + ) + ), + Event( + 100.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 150.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 200.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 250.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld7_5), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 300.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld7_5), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 350.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld7_5) + ) + ) + ), + Event( + 400.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 450.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld15), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 500.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld15), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 550.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld15) + ) + ) + ), + Event( + 610.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 960.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = ld30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 1310.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = sd30), + SingleAllocation(c, targetLayer = ld30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 1660.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = sd30), + SingleAllocation(c, targetLayer = sd30), + SingleAllocation(d, targetLayer = ld30) + ) + ) + ), + Event( + 2010.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = sd30), + SingleAllocation(c, targetLayer = sd30), + SingleAllocation(d, targetLayer = sd30) + ) + ) + ) + ) + } + + private fun verifyTileViewLastN1(maxFrameHeight: Int = 180) { + // At this stage the purpose of this is just to document current behavior. + // TODO: The results with bwe==-1 are wrong. + bc.forwardedSourcesHistory.removeIf { it.bwe < 0.bps } + bc.forwardedSourcesHistory.map { it.event }.shouldContainInOrder( + setOf("A-v0") + ) + + bc.effectiveConstraintsHistory.last().event shouldBe mapOf( + "A-v0" to VideoConstraints(maxFrameHeight), + "B-v0" to VideoConstraints(0), + "C-v0" to VideoConstraints(0), + "D-v0" to VideoConstraints(0) + ) + + val expectedAllocationHistory = mutableListOf( + // TODO: do we want to oversend in tile view? + Event( + (-1).bps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = noVideo), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ), + oversending = true + ) + ), + Event( + 50.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld7_5), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + 100.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld15), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ), + Event( + // TODO: why 160 instead of 150? weird. + 160.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = ld30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ) + ) + if (maxFrameHeight > 180) { + expectedAllocationHistory.addAll( + listOf( + Event( + 510.kbps, + BandwidthAllocation( + setOf( + SingleAllocation(a, targetLayer = sd30), + SingleAllocation(b, targetLayer = noVideo), + SingleAllocation(c, targetLayer = noVideo), + SingleAllocation(d, targetLayer = noVideo) + ) + ) + ) + ) + ) + } + bc.allocationHistory.shouldMatchInOrder(*expectedAllocationHistory.toTypedArray()) + } +} + +class BitrateControllerWrapper(initialEndpoints: List, val clock: FakeClock = FakeClock()) { + var endpoints: List = initialEndpoints + val logger = createLogger() + + var bwe = (-1).bps + set(value) { + logger.debug("Setting bwe=$value") + field = value + bc.bandwidthChanged(value.bps) + } + + // Save the output. + val effectiveConstraintsHistory: History> = mutableListOf() + val forwardedSourcesHistory: History> = mutableListOf() + val allocationHistory: History = mutableListOf() + + val bc = BitrateController( + object : BitrateController.EventHandler { + override fun forwardedSourcesChanged(forwardedSources: Set) { + Event(bwe, forwardedSources, clock.instant()).apply { + logger.info("Forwarded sources changed: $this") + forwardedSourcesHistory.add(this) + } + } + + override fun effectiveVideoConstraintsChanged( + oldEffectiveConstraints: EffectiveConstraintsMap, + newEffectiveConstraints: EffectiveConstraintsMap + ) { + Event(bwe, newEffectiveConstraints.mapKeys { it.key.sourceName }, clock.instant()).apply { + logger.info("Effective constraints changed: $this") + effectiveConstraintsHistory.add(this) + } + } + + override fun keyframeNeeded(endpointId: String?, ssrc: Long) {} + + override fun allocationChanged(allocation: BandwidthAllocation) { + Event( + bwe, + allocation, + clock.instant() + ).apply { + logger.info("Allocation changed: $this") + allocationHistory.add(this) + } + } + }, + Supplier { endpoints }, + DiagnosticContext(), + logger, + clock + ).apply { bandwidthChanged(bwe.bps) } // TODO: handle the -1 bps case better + + fun setEndpointOrdering(vararg endpoints: TestEndpoint) { + logger.info("Set endpoints ${endpoints.map{ it.id }.joinToString(",")}") + this.endpoints = endpoints.toList() + bc.endpointOrderingChanged() + } + + fun setStageView(onStageSource: String, lastN: Int? = null) { + bc.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage( + lastN = lastN, + onStageSources = listOf(onStageSource), + constraints = mapOf(onStageSource to VideoConstraints(720)) + ) + ) + } + + fun setTileView(vararg selectedSources: String, maxFrameHeight: Int = 180, lastN: Int? = null) { + bc.setBandwidthAllocationSettings( + ReceiverVideoConstraintsMessage( + lastN = lastN, + selectedSources = listOf(*selectedSources), + constraints = selectedSources.map { it to VideoConstraints(maxFrameHeight) }.toMap() + ) + ) + } + + init { + // The BC only starts working 10 seconds after it first received media, so fake that. + bc.transformRtp(PacketInfo(VideoRtpPacket(ByteArray(100), 0, 100))) + clock.elapse(15.secs) + + // Adaptivity is disabled when RTX support is not signalled. + bc.addPayloadType(RtxPayloadType(123, mapOf("apt" to "124"))) + } +} + +class TestEndpoint( + override val id: String, + override val mediaSources: Array = emptyArray() +) : MediaSourceContainer + +fun createEndpoints(vararg ids: String): MutableList = MutableList(ids.size) { i -> + TestEndpoint( + ids[i], + arrayOf( + createSourceDesc( + 3 * i + 1, + 3 * i + 2, + 3 * i + 3, + ids[i] + "-v0", + ids[i] + ) + ) + ) +} + +fun createSources(vararg ids: String): MutableList = MutableList(ids.size) { i -> + createSourceDesc( + 3 * i + 1, + 3 * i + 2, + 3 * i + 3, + ids[i], + ids[i] + ) +} + +fun createSourceDesc(ssrc1: Int, ssrc2: Int, ssrc3: Int, sourceName: String, owner: String): MediaSourceDesc = + MediaSourceDesc( + arrayOf( + RtpEncodingDesc(ssrc1.toLong(), arrayOf(ld7_5, ld15, ld30)), + RtpEncodingDesc(ssrc2.toLong(), arrayOf(sd7_5, sd15, sd30)), + RtpEncodingDesc(ssrc3.toLong(), arrayOf(hd7_5, hd15, hd30)) + ), + sourceName = sourceName, + owner = owner + ) + +val bitrateLd = 150.kbps +val bitrateSd = 500.kbps +val bitrateHd = 2000.kbps + +val ld7_5 + get() = MockRtpLayerDesc(tid = 0, eid = 0, height = 180, frameRate = 7.5, bitrate = bitrateLd * 0.33) +val ld15 + get() = MockRtpLayerDesc(tid = 1, eid = 0, height = 180, frameRate = 15.0, bitrate = bitrateLd * 0.66) +val ld30 + get() = MockRtpLayerDesc(tid = 2, eid = 0, height = 180, frameRate = 30.0, bitrate = bitrateLd) + +val sd7_5 + get() = MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = bitrateSd * 0.33) +val sd15 + get() = MockRtpLayerDesc(tid = 1, eid = 1, height = 360, frameRate = 15.0, bitrate = bitrateSd * 0.66) +val sd30 + get() = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = bitrateSd) + +val hd7_5 + get() = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = bitrateHd * 0.33) +val hd15 + get() = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = bitrateHd * 0.66) +val hd30 + get() = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = bitrateHd) + +val noVideo: RtpLayerDesc? = null + +/** + * An [RtpLayerDesc] whose bitrate can be set externally. We inherit directly from [RtpLayerDesc], because mocking with + * mockk absolutely kills the performance. + */ +class MockRtpLayerDesc( + tid: Int, + eid: Int, + height: Int, + frameRate: Double, + /** + * Note: this mock impl does not model the dependency layers, so the cumulative bitrate should be provided. + */ + var bitrate: Bandwidth, + sid: Int = -1 +) : RtpLayerDesc(eid, tid, sid, height, frameRate) { + override fun copy(height: Int, tid: Int, inherit: Boolean): RtpLayerDesc { + TODO("Not yet implemented") + } + + override val layerId = getIndex(0, sid, tid) + override val index = getIndex(eid, sid, tid) + + override fun getBitrate(nowMs: Long): Bandwidth = bitrate + override fun getSmoothedBitrate(nowMs: Long): Bandwidth = bitrate + override fun hasZeroBitrate(nowMs: Long): Boolean = bitrate == 0.bps + override fun indexString() = indexString(index) +} + +typealias History = MutableList> +data class Event( + val bwe: Bandwidth, + val event: T, + val time: Instant = Instant.MIN +) { + override fun toString(): String = "\n[time=${time.toEpochMilli()} bwe=$bwe] $event" + override fun equals(other: Any?): Boolean { + if (other !is Event<*>) return false + // Ignore this.time + return bwe == other.bwe && event == other.event + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt new file mode 100644 index 0000000..88b8003 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/BitrateControllerTraceTest.kt @@ -0,0 +1,138 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import io.kotest.assertions.fail +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.ints.shouldBeLessThan +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.util.bps +import org.jitsi.utils.time.FakeClock +import java.time.Instant + +/** + * This test reads a trace file containing bandwidth estimations and bitrates for individual layers captured from a + * "real" conference. The conference had a total of 7 endpoints (one receiver and 6 senders), and the bridge->receiver + * BWE was artificially reset to 0 just prior to taking the trace, simulating a ramp-up from 0. + * + * While [BitrateControllerTest] tests the individual decisions made by [BandwidthAllocator] when the bitrates of the + * layers are fixed, here we test for the number of allocation changes which are visible to the user as changes in the + * number and/or quality of the videos being shown. Specifically, we want to limit the "flickering" which happens as + * a result of fluctuations in the bitrates of the layers and the changes in BWE. + */ +class BitrateControllerTraceTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + private val clock = FakeClock() + private val a = Endpoint("A") + private val b = Endpoint("B") + private val c = Endpoint("C") + private val d = Endpoint("D") + private val e = Endpoint("E") + private val f = Endpoint("F") + private val bc = BitrateControllerWrapper(listOf(a, b, c, d, e, f), clock = clock).apply { + bc.endpointOrderingChanged() + } + + init { + val bweEvents = javaClass.getResource("/bwe-events.csv") ?: fail("Can not read bwe-events.csv") + val parsedLines = bweEvents.readText().lines().drop(1).dropLast(1).map { ParsedLine(it) }.toList() + + context("Number of allocation changes") { + + println("Read ${parsedLines.size} events.") + parsedLines.forEach { line -> + clock.setTime(line.time) + a.layer7.bitrate = line.bps_a_7.bps + a.layer30.bitrate = line.bps_a_30.bps + b.layer7.bitrate = line.bps_b_7.bps + b.layer30.bitrate = line.bps_b_30.bps + c.layer7.bitrate = line.bps_c_7.bps + c.layer30.bitrate = line.bps_c_30.bps + d.layer7.bitrate = line.bps_d_7.bps + d.layer30.bitrate = line.bps_d_30.bps + e.layer7.bitrate = line.bps_e_7.bps + e.layer30.bitrate = line.bps_e_30.bps + f.layer7.bitrate = line.bps_f_7.bps + f.layer30.bitrate = line.bps_f_30.bps + bc.bwe = line.bwe.bps + } + + println("Finished. Made a total of ${bc.allocationHistory.size} allocation changes.") + // Ideally as the BWE ramps up from 0 we'd see the endpoints being added sequentially, with some changes + // from 7 fps to 30 fps. + bc.allocationHistory.size shouldBeLessThan 20 + } + } + + class Endpoint( + override val id: String, + override val mediaSources: Array = emptyArray() + ) : MediaSourceContainer { + val layer7 = MockRtpLayerDesc(tid = 0, eid = 0, height = 180, frameRate = 7.5, 0.bps) + val layer30 = MockRtpLayerDesc(tid = 2, eid = 0, height = 180, frameRate = 30.0, bitrate = 0.bps) + } +} + +/** + * Parse a timestamp from the CSV file as an [Instant]. The actual instant doesn't matter as long as the relationships + * between the timestamps in the file are preserved. + */ +private fun parseTs( + /** + * A timestamp in this format: 19:52:49.234 + */ + ts: String +) = Instant.parse("2020-01-01T${ts}Z") + +private data class ParsedLine( + val time: Instant, + val bwe: Long, + val bps_a_7: Long, + val bps_a_30: Long, + val bps_b_7: Long, + val bps_b_30: Long, + val bps_c_7: Long, + val bps_c_30: Long, + val bps_d_7: Long, + val bps_d_30: Long, + val bps_e_7: Long, + val bps_e_30: Long, + val bps_f_7: Long, + val bps_f_30: Long +) { + /** + * ts,bwe,bps_a_7,bps_a_30,bps_b_7,bps_b_30,bps_c_7,bps_c_30,bps_d_7,bps_d_30,bps_e_7,bps_e_30,bps_f_7,bps_f_30 + * 19:52:49.234,33400,93560,157626,86717,160533,54922,135183,93405,156291,92150,155582,92397,152019 + */ + constructor(line: String) : this( + parseTs(line.split(",")[0]), + line.split(",")[1].toLong(), + line.split(",")[2].toLong(), + line.split(",")[3].toLong(), + line.split(",")[4].toLong(), + line.split(",")[5].toLong(), + line.split(",")[6].toLong(), + line.split(",")[7].toLong(), + line.split(",")[8].toLong(), + line.split(",")[9].toLong(), + line.split(",")[10].toLong(), + line.split(",")[11].toLong(), + line.split(",")[12].toLong(), + line.split(",")[13].toLong() + ) +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt new file mode 100644 index 0000000..075e2e6 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/EffectiveConstraintsTest.kt @@ -0,0 +1,180 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * Copyright @ 2021 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.allocation + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.VideoType + +fun testSource(endpointId: String, sourceName: String, videoType: VideoType = VideoType.CAMERA): MediaSourceDesc = + MediaSourceDesc( + emptyArray(), + endpointId, + sourceName, + videoType + ) + +@Suppress("NAME_SHADOWING") +class EffectiveConstraintsTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + init { + val s1 = testSource("e1", "s1") + val s2 = testSource("e1", "s2") + val s3 = testSource("e1", "s3") + val s4 = testSource("e1", "s4", videoType = VideoType.DISABLED) + val s5 = testSource("e1", "s5", videoType = VideoType.DISABLED) + val s6 = testSource("e1", "s6", videoType = VideoType.NONE) + + val defaultConstraints = VideoConstraints(180) + + val sources = listOf(s1, s2, s3, s4, s5, s6) + val zeroEffectiveConstraints = mutableMapOf( + s1 to VideoConstraints.NOTHING, + s2 to VideoConstraints.NOTHING, + s3 to VideoConstraints.NOTHING, + s4 to VideoConstraints.NOTHING, + s5 to VideoConstraints.NOTHING, + s6 to VideoConstraints.NOTHING + ) + + context("With lastN=0") { + val allocationSettings = AllocationSettings(lastN = 0, defaultConstraints = defaultConstraints) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints + } + context("With lastN=1") { + context("And no other constraints") { + val allocationSettings = AllocationSettings(lastN = 1, defaultConstraints = defaultConstraints) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + // The default defaultConstraints are 180 + put(s1, VideoConstraints(180)) + } + } + context("And different defaultConstraints") { + val allocationSettings = AllocationSettings(lastN = 1, defaultConstraints = VideoConstraints(360)) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s1, VideoConstraints(360)) + } + } + context("And all constraints 0") { + val allocationSettings = AllocationSettings( + lastN = 1, + defaultConstraints = VideoConstraints.NOTHING, + ) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints + } + context("And non-zero constraints for a source with video enabled") { + val allocationSettings = AllocationSettings( + lastN = 1, + defaultConstraints = VideoConstraints.NOTHING, + videoConstraints = mapOf("s1" to VideoConstraints(720)) + + ) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s1, VideoConstraints(720)) + } + } + context("And non-zero constraints for a source with video DISABLED") { + val allocationSettings = AllocationSettings( + lastN = 1, + defaultConstraints = VideoConstraints.NOTHING, + videoConstraints = mapOf("s4" to VideoConstraints(720)) + ) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints + } + context("When the top sources have the video DISABLED") { + // The top sources in speaker order have videoType = DISABLED + val sources = listOf(s4, s5, s6, s1, s2, s3) + + context("With default settings") { + val allocationSettings = AllocationSettings(lastN = 1, defaultConstraints = defaultConstraints) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s1, VideoConstraints(180)) + } + } + context("With default constraints 0 and non-zero constraints for a source with video DISABLED") { + val allocationSettings = AllocationSettings( + lastN = 1, + defaultConstraints = VideoConstraints.NOTHING, + videoConstraints = mapOf("s5" to VideoConstraints(180)) + ) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints + } + context("With default constraints 0 and non-zero constraints for a source with video enabled") { + val allocationSettings = AllocationSettings( + lastN = 1, + defaultConstraints = VideoConstraints.NOTHING, + videoConstraints = mapOf("s2" to VideoConstraints(180)) + ) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s2, VideoConstraints(180)) + } + } + context("With default constraints 0 and non-zero constraints for sources low on the list") { + val allocationSettings = AllocationSettings( + lastN = 1, + defaultConstraints = VideoConstraints.NOTHING, + videoConstraints = mapOf("s2" to VideoConstraints(180), "s3" to VideoConstraints(180)) + ) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s2, VideoConstraints(180)) + } + } + } + } + context("With lastN=3") { + context("And default settings") { + val allocationSettings = AllocationSettings(lastN = 3, defaultConstraints = defaultConstraints) + getEffectiveConstraints(sources, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s1, VideoConstraints(180)) + put(s2, VideoConstraints(180)) + put(s3, VideoConstraints(180)) + } + } + context("When the top sources have video DISABLED") { + // The top sources in speaker order have videoType = DISABLED + val endpoints = listOf(s4, s5, s6, s1, s2, s3) + + context("And default settings") { + val allocationSettings = AllocationSettings(lastN = 3, defaultConstraints = defaultConstraints) + getEffectiveConstraints(endpoints, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s1, VideoConstraints(180)) + put(s2, VideoConstraints(180)) + put(s3, VideoConstraints(180)) + } + } + context("And non-zero constraints for sources down the list") { + val allocationSettings = AllocationSettings( + lastN = 3, + defaultConstraints = VideoConstraints.NOTHING, + videoConstraints = mapOf( + "s6" to VideoConstraints(180), + "s2" to VideoConstraints(180), + "s3" to VideoConstraints(180) + ) + ) + getEffectiveConstraints(endpoints, allocationSettings) shouldBe zeroEffectiveConstraints.apply { + put(s2, VideoConstraints(180)) + put(s3, VideoConstraints(180)) + } + } + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMapTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMapTest.kt new file mode 100644 index 0000000..7f74579 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/ReceiverConstraintsMapTest.kt @@ -0,0 +1,84 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.allocation + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class ReceiverConstraintsMapTest : ShouldSpec({ + isolationMode = IsolationMode.InstancePerLeaf + + val constraints = ReceiverConstraintsMap() + + context("receiver constraints map") { + should("track the max height correctly") { + constraints.maxHeight shouldBe 0 + constraints.put("a", vc(1)) + constraints.maxHeight shouldBe 1 + constraints.put("b", vc(2)) + constraints.maxHeight shouldBe 2 + constraints.put("c", vc(3)) + constraints.maxHeight shouldBe 3 + + constraints.remove("c") + constraints.maxHeight shouldBe 2 + + constraints.put("a", vc(4)) + constraints.maxHeight shouldBe 4 + + constraints.put("a", vc(3)) + constraints.maxHeight shouldBe 3 + + constraints.remove("a") + constraints.maxHeight shouldBe 2 + + constraints.remove("b") + constraints.maxHeight shouldBe 0 + + constraints.put("a", vc(-1)) + constraints.maxHeight shouldBe -1 + + constraints.put("b", vc(100)) + constraints.maxHeight shouldBe -1 + + constraints.remove("a") + constraints.maxHeight shouldBe 100 + } + should("track max height correctly when existing keys are updated") { + constraints.put("b", vc(1)) + constraints.maxHeight shouldBe 1 + + constraints.put("b", vc(2)) + constraints.maxHeight shouldBe 2 + + constraints.put("b", vc(1)) + constraints.maxHeight shouldBe 1 + + constraints.put("b", vc(-1)) + constraints.maxHeight shouldBe -1 + + constraints.put("b", vc(1)) + constraints.maxHeight shouldBe 1 + + constraints.put("b", vc(2)) + constraints.maxHeight shouldBe 2 + } + } +}) + +private fun vc(maxHeight: Int) = VideoConstraints(maxHeight, 30.0) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt new file mode 100644 index 0000000..ba3562c --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocationTest.kt @@ -0,0 +1,420 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * Copyright @ 2021 - Vowel, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("NAME_SHADOWING") + +package org.jitsi.videobridge.cc.allocation + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.MediaSourceDesc +import org.jitsi.nlj.RtpEncodingDesc +import org.jitsi.nlj.VideoType +import org.jitsi.nlj.util.bps +import org.jitsi.nlj.util.kbps +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.time.FakeClock + +/** + * Test the logic for selecting the layers to be considered for a source and the "preferred" layer. + */ +class SingleSourceAllocationTest : ShouldSpec() { + private val clock = FakeClock() + private val diagnosticContext = DiagnosticContext() + + private val ld7point5 = + MockRtpLayerDesc(tid = 0, eid = 0, height = 180, frameRate = 7.5, bitrate = bitrateLd * 0.33) + private val ld15 = MockRtpLayerDesc(tid = 1, eid = 0, height = 180, frameRate = 15.0, bitrate = bitrateLd * 0.66) + private val ld30 = MockRtpLayerDesc(tid = 2, eid = 0, height = 180, frameRate = 30.0, bitrate = bitrateLd) + + private val sd7point5 = + MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = bitrateSd * 0.33) + private val sd15 = MockRtpLayerDesc(tid = 1, eid = 1, height = 360, frameRate = 15.0, bitrate = bitrateSd * 0.66) + private val sd30 = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = bitrateSd) + + private val hd7point5 = + MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = bitrateHd * 0.33) + private val hd15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = bitrateHd * 0.66) + private val hd30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = bitrateHd) + + init { + context("Camera") { + context("When all layers are active") { + val endpointId = "A" + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.CAMERA + ) + + context("Without constraints") { + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(720), + false, + diagnosticContext, + clock + ) + + // We include all resolutions up to the preferred resolution, and only high-FPS (at least + // "preferred FPS") layers for higher resolutions. + allocation.preferredLayer shouldBe sd30 + allocation.oversendLayer shouldBe null + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd30, hd30) + } + context("With constraints") { + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(360), + false, + diagnosticContext, + clock + ) + + // We include all resolutions up to the preferred resolution, and only high-FPS (at least + // "preferred FPS") layers for higher resolutions. + allocation.preferredLayer shouldBe sd30 + allocation.oversendLayer shouldBe null + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd30) + } + context("With constraints unmet by any layer") { + // Single high-res stream with 3 temporal layers. + val endpointId = "A" + val mediaSource = MediaSourceDesc( + // No simulcast. + arrayOf(RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30))), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.CAMERA + ) + + context("Non-zero constraints") { + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(360), + false, + diagnosticContext, + clock + ) + + // The receiver set 360p constraints, but we only have a 720p stream. + allocation.preferredLayer shouldBe hd30 + allocation.oversendLayer shouldBe null + allocation.layers.map { it.layer } shouldBe listOf(hd7point5, hd15, hd30) + } + context("Zero constraints") { + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(0), + false, + diagnosticContext, + clock + ) + + // The receiver set a maxHeight=0 constraint. + allocation.preferredLayer shouldBe null + allocation.oversendLayer shouldBe null + allocation.layers.map { it.layer } shouldBe emptyList() + } + } + } + context("When some layers are inactive") { + // Override layers with bitrate=0. Simulate only up to 360p/15 being active. + val sd30 = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = 0.bps) + val hd7point5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) + val hd15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = 0.bps) + val hd30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = 0.bps) + val endpointId = "A" + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.CAMERA + ) + + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(720), + false, + diagnosticContext, + clock + ) + + // We include all resolutions up to the preferred resolution, and only high-FPS (at least + // "preferred FPS") layers for higher resolutions. + allocation.preferredLayer shouldBe ld30 + allocation.oversendLayer shouldBe null + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30) + } + } + context("Screensharing") { + context("When all layers are active") { + val endpointId = "A" + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.DESKTOP + ) + + context("With no constraints") { + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock + ) + + // For screensharing the "preferred" layer is the full resolution at its cheapest frame rate: the + // resolution is prioritized over other sources, but additional frame rate is not. + allocation.preferredLayer shouldBe hd7point5 + allocation.oversendLayer shouldBe hd7point5 + allocation.layers.map { it.layer } shouldBe + listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30, hd7point5, hd15, hd30) + } + context("With 360p constraints") { + val allocation = + SingleSourceAllocation( + endpointId, + mediaSource, + VideoConstraints(360), + true, + diagnosticContext, + clock + ) + + allocation.preferredLayer shouldBe sd7point5 + allocation.oversendLayer shouldBe sd7point5 + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30) + } + } + context("High fps screensharing") { + // The sender explicitly asked for frame rate to be prioritized, so the highest layer stays + // "preferred" (while oversend still uses the cheapest layer at the full resolution). + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.DESKTOP_HIGH_FPS + ) + + val allocation = + SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock + ) + + allocation.preferredLayer shouldBe hd30 + allocation.oversendLayer shouldBe hd7point5 + allocation.layers.map { it.layer } shouldBe + listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30, hd7point5, hd15, hd30) + } + context("The high layers are inactive (send-side bwe restrictions)") { + // Override layers with bitrate=0. Simulate only up to 360p/30 being active. + val hd7point5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) + val hd15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = 0.bps) + val hd30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = 0.bps) + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.DESKTOP + ) + + val allocation = + SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock + ) + + // For screensharing the "preferred" layer is the full resolution at its cheapest frame rate: the + // resolution is prioritized over other sources, but additional frame rate is not. + allocation.preferredLayer shouldBe sd7point5 + allocation.oversendLayer shouldBe sd7point5 + allocation.layers.map { it.layer } shouldBe listOf(ld7point5, ld15, ld30, sd7point5, sd15, sd30) + } + context("The low layers are inactive (simulcast signaled but not used)") { + // Override layers with bitrate=0. Simulate simulcast being signaled but effectively disabled. + val ld7point5 = MockRtpLayerDesc(tid = 0, eid = 2, height = 720, frameRate = 7.5, bitrate = 0.bps) + val ld15 = MockRtpLayerDesc(tid = 1, eid = 2, height = 720, frameRate = 15.0, bitrate = 0.bps) + val ld30 = MockRtpLayerDesc(tid = 2, eid = 2, height = 720, frameRate = 30.0, bitrate = 0.bps) + val sd7point5 = MockRtpLayerDesc(tid = 0, eid = 1, height = 360, frameRate = 7.5, bitrate = 0.bps) + val sd15 = MockRtpLayerDesc(tid = 1, eid = 1, height = 360, frameRate = 15.0, bitrate = 0.bps) + val sd30 = MockRtpLayerDesc(tid = 2, eid = 1, height = 360, frameRate = 30.0, bitrate = 0.bps) + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(ld7point5, ld15, ld30)), + RtpEncodingDesc(1L, arrayOf(sd7point5, sd15, sd30)), + RtpEncodingDesc(1L, arrayOf(hd7point5, hd15, hd30)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.DESKTOP + ) + + context("With no constraints") { + val allocation = + SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock + ) + + // For screensharing the "preferred" layer is the full resolution at its cheapest frame rate: the + // resolution is prioritized over other sources, but additional frame rate is not. + allocation.preferredLayer shouldBe hd7point5 + allocation.oversendLayer shouldBe hd7point5 + allocation.layers.map { it.layer } shouldBe listOf(hd7point5, hd15, hd30) + } + context("With 180p constraints") { + val allocation = + SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(180), + true, + diagnosticContext, + clock + ) + + // Since no layers satisfy the resolution constraints, we consider layers from the lowest + // available resolution (which is high). The "preferred" layer is that resolution at its cheapest + // frame rate. + allocation.preferredLayer shouldBe hd7point5 + allocation.oversendLayer shouldBe hd7point5 + allocation.layers.map { it.layer } shouldBe listOf(hd7point5, hd15, hd30) + } + } + context("VP9") { + val l1 = MockRtpLayerDesc(tid = 0, eid = 0, sid = 0, height = 720, frameRate = -1.0, bitrate = 150.kbps) + val l2 = MockRtpLayerDesc(tid = 0, eid = 0, sid = 1, height = 720, frameRate = -1.0, bitrate = 370.kbps) + val l3 = MockRtpLayerDesc(tid = 0, eid = 0, sid = 2, height = 720, frameRate = -1.0, bitrate = 750.kbps) + + val mediaSource = MediaSourceDesc( + arrayOf( + RtpEncodingDesc(1L, arrayOf(l1)), + RtpEncodingDesc(1L, arrayOf(l2)), + RtpEncodingDesc(1L, arrayOf(l3)) + ), + sourceName = SOURCE_NAME, + owner = OWNER, + videoType = VideoType.DESKTOP + ) + + context("With no constraints") { + val allocation = + SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(720), + true, + diagnosticContext, + clock + ) + + // With VP9's multiple-encodings structure the layers above the cheapest improve quality (not + // frame rate), so the highest stays "preferred". Oversend still uses the cheapest. + allocation.preferredLayer shouldBe l3 + allocation.oversendLayer shouldBe l1 + allocation.layers.map { it.layer } shouldBe listOf(l1, l2, l3) + } + context("With 180p constraints") { + // Since no layers satisfy the resolution constraints, we consider layers from the lowest + // available resolution (which is high). If we are off-stage we only consider the first of + // these layers. + context("On stage") { + val allocation = SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(180), + true, + diagnosticContext, + clock + ) + + // With VP9's multiple-encodings structure the layers above the cheapest improve quality + // (not frame rate), so the highest stays "preferred". Oversend still uses the cheapest. + allocation.preferredLayer shouldBe l3 + allocation.oversendLayer shouldBe l1 + allocation.layers.map { it.layer } shouldBe listOf(l1, l2, l3) + } + context("Off stage") { + val allocation = SingleSourceAllocation( + "A", + mediaSource, + VideoConstraints(180), + false, + diagnosticContext, + clock + ) + + allocation.preferredLayer shouldBe l1 + allocation.oversendLayer shouldBe null + allocation.layers.map { it.layer } shouldBe listOf(l1) + } + } + } + } + } +} + +private const val SOURCE_NAME = "sourceName" +private const val OWNER = "owner" diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt new file mode 100644 index 0000000..4aaa4b7 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionTest.kt @@ -0,0 +1,1615 @@ +/* + * Copyright @ 2019 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.av1 + +import jakarta.xml.bind.DatatypeConverter.parseHexBinary +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.rtp.codec.av1.Av1DDPacket +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc.Companion.getIndex +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorHeaderExtension +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.rtp.rtp.header_extensions.FrameInfo +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.videobridge.cc.RtpState +import org.junit.Assert +import org.junit.Test +import java.time.Duration +import java.time.Instant +import java.util.* +import javax.xml.bind.DatatypeConverter + +class Av1DDAdaptiveSourceProjectionTest { + private val logger: Logger = LoggerImpl(javaClass.name) + + @Test + fun singlePacketProjectionTest() { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "singlePacketProjectionTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + val generator = ScalableAv1PacketGenerator(1) + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val targetIndex = getIndex(eid = 0, dt = 0) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + Assert.assertEquals(10001, packet.sequenceNumber) + Assert.assertEquals(1003000, packet.timestamp) + Assert.assertEquals(0, packet.frameNumber) + Assert.assertEquals(0, packet.frameInfo?.spatialId) + Assert.assertEquals(0, packet.frameInfo?.temporalId) + } + + private fun runInOrderTest(generator: Av1PacketGenerator, targetIndex: Int, expectAccept: (FrameInfo) -> Boolean) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + for (i in 0..99999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val frameInfo = packet.frameInfo!! + + val accepted = context.accept(packetInfo, targetIndex) + val endOfFrame = packet.isEndOfFrame + val endOfPicture = packet.isMarked // Save this before rewriteRtp + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (endOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun simpleNonScalableTest() { + val generator = NonScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + true + } + } + + @Test + fun simpleTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largerFrameTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largerFrameTemporalFilteredTest() { + val generator = TemporallyScaledPacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun hugeFrameTest() { + val generator = TemporallyScaledPacketGenerator(200) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun simpleSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun filteredSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun spatialAndTemporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun largerFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSvcTest() { + val generator = ScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun filteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun spatialAndTemporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun largerFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun largerSpatialAndTemporalFilteredKSvcTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 + } + } + + @Test + fun filteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun spatialAndTemporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 + } + } + + @Test + fun largerFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSingleEncodingSimulcastTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + private class ProjectedPacket constructor( + val packet: Av1DDPacket, + val origSeq: Int, + val extOrigSeq: Long, + val extFrameNum: Long, + ) + + /** Run an out-of-order test on a single stream, randomized order except for the first + * [initialOrderedCount] packets. */ + private fun doRunOutOfOrderTest( + generator: Av1PacketGenerator, + targetIndex: Int, + initialOrderedCount: Int, + seed: Long, + expectAccept: (FrameInfo) -> Boolean + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val expectedInitialTs: Long = RtpUtils.applyTimestampDelta(initialState.maxTimestamp, 3000) + val expectedTsOffset: Long = RtpUtils.getTimestampDiff(expectedInitialTs, generator.ts) + val reorderSize = 64 + val buffer = ArrayList(reorderSize) + for (i in 0 until reorderSize) { + buffer.add(generator.nextPacket()) + } + val random = Random(seed) + var orderedCount = initialOrderedCount - 1 + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + var latestSeq = buffer[0].packetAs().sequenceNumber + val projectedPackets = TreeMap() + val origSeqIdxTracker = RtpSequenceIndexTracker() + val newSeqIdxTracker = RtpSequenceIndexTracker() + val frameNumsDropped = HashSet() + val frameNumsIndexTracker = RtpSequenceIndexTracker() + for (i in 0..99999) { + val packetInfo = buffer[0] + val packet = packetInfo.packetAs() + val origSeq = packet.sequenceNumber + val origTs = packet.timestamp + if (latestSeq isOlderThan origSeq) { + latestSeq = origSeq + } + val frameInfo = packet.frameInfo!! + + val accepted = context.accept(packetInfo, targetIndex) + val oldestValidSeq: Int = + RtpUtils.applySequenceNumberDelta( + latestSeq, + -((Av1DDFrameMap.FRAME_MAP_SIZE - 1) * generator.packetsPerFrame) + ) + if (origSeq isOlderThan oldestValidSeq && !accepted) { + /* This is fine; packets that are too old get ignored. */ + /* Note we don't want assertFalse(accepted) here because slightly-too-old packets + * that are part of an existing accepted frame will be accepted. + */ + val extFrameNum = frameNumsIndexTracker.update(packet.frameNumber) + frameNumsDropped.add(extFrameNum) + } else if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + + context.rewriteRtp(packetInfo) + Assert.assertEquals(RtpUtils.applyTimestampDelta(origTs, expectedTsOffset), packet.timestamp) + val newSeq = packet.sequenceNumber + val extNewSeq = newSeqIdxTracker.update(newSeq) + val extOrigSeq = origSeqIdxTracker.update(origSeq) + Assert.assertFalse(projectedPackets.containsKey(extNewSeq)) + val extFrameNum = frameNumsIndexTracker.update(packet.frameNumber) + projectedPackets[extNewSeq] = ProjectedPacket(packet, origSeq, extOrigSeq, extFrameNum) + } else { + Assert.assertFalse(accepted) + } + if (orderedCount > 0) { + buffer.removeAt(0) + buffer.add(generator.nextPacket()) + orderedCount-- + } else { + buffer[0] = generator.nextPacket() + buffer.shuffle(random) + } + } + val frameNumsSeen = HashSet() + + /* Add packets that weren't added yet, or that were dropped for being too old, to frameNumsSeen. */ + frameNumsSeen.addAll(frameNumsDropped) + buffer.forEach { + frameNumsSeen.add(frameNumsIndexTracker.update(it.packetAs().frameNumber)) + } + + val iter = projectedPackets.keys.iterator() + var prevPacket = projectedPackets[iter.next()]!! + frameNumsSeen.add(prevPacket.extFrameNum) + while (iter.hasNext()) { + val packet = projectedPackets[iter.next()] + Assert.assertTrue(packet!!.origSeq isNewerThan prevPacket.origSeq) + frameNumsSeen.add(packet.extFrameNum) + Assert.assertTrue( + RtpUtils.getSequenceNumberDelta( + prevPacket.packet.frameNumber, + packet.packet.frameNumber + ) <= 0 + ) + if (packet.packet.isStartOfFrame) { + Assert.assertTrue( + RtpUtils.getSequenceNumberDelta( + prevPacket.packet.frameNumber, + packet.packet.frameNumber + ) < 0 + ) + if (prevPacket.packet.sequenceNumber == + RtpUtils.applySequenceNumberDelta(packet.packet.sequenceNumber, -1) + ) { + Assert.assertTrue(prevPacket.packet.isEndOfFrame) + } + } else { + if (prevPacket.packet.sequenceNumber == + RtpUtils.applySequenceNumberDelta(packet.packet.sequenceNumber, -1) + ) { + Assert.assertEquals(prevPacket.packet.frameNumber, packet.packet.frameNumber) + Assert.assertEquals(prevPacket.packet.timestamp, packet.packet.timestamp) + } + } + packet.packet.frameInfo?.fdiff?.forEach { + Assert.assertTrue(frameNumsSeen.contains(packet.extFrameNum - it)) + } + prevPacket = packet + } + + /* Overall, we should not have expanded sequence numbers. */ + val firstPacket = projectedPackets.firstEntry().value + val lastPacket = projectedPackets.lastEntry().value + val origDelta = lastPacket!!.extOrigSeq - firstPacket!!.extOrigSeq + val projDelta = projectedPackets.lastKey() - projectedPackets.firstKey() + Assert.assertTrue(projDelta <= origDelta) + } + + /** Run multiple instances of out-of-order test on a single stream, with different + * random seeds. */ + private fun runOutOfOrderTest( + generator: Av1PacketGenerator, + targetIndex: Int, + initialOrderedCount: Int = 1, + expectAccept: (FrameInfo) -> Boolean + ) { + /* Seeds that have triggered problems in the past for this or VP8/VP9, plus a random one. */ + val seeds = longArrayOf( + 1576267371838L, + 1578347926155L, + 1579620018479L, + 5786714086792432950L, + 5929140296748347521L, + -8226056792707023108L, + System.currentTimeMillis() + ) + for (seed in seeds) { + try { + doRunOutOfOrderTest(generator, targetIndex, initialOrderedCount, seed, expectAccept) + } catch (e: Throwable) { + logger.error( + "Exception thrown in randomized test, seed = $seed", + e + ) + throw e + } + generator.reset() + } + } + + @Test + fun simpleOutOfOrderNonScalableTest() { + val generator = NonScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + true + } + } + + @Test + fun simpleOutOfOrderTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredOutOfOrderTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largerFrameOutOfOrderTemporalProjectionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largerFrameOutOfOrderTemporalFilteredTest() { + val generator = TemporallyScaledPacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun hugeFrameOutOfOrderTest() { + val generator = TemporallyScaledPacketGenerator(200) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun simpleSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun filteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredOutOfOrderSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun spatialAndTemporalFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + true + } + } + + @Test + fun largerFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSvcOutOfOrderTest() { + val generator = ScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun filteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun spatialAndTemporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2)) { + it.spatialId == 2 || !it.hasInterPictureDependency() + } + } + + @Test + fun largerFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2)) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2)) { + it.temporalId == 0 && (it.spatialId == 2 || !it.hasInterPictureDependency()) + } + } + + @Test + fun largerSpatialAndTemporalFilteredKSvcOutOfOrderTest() { + val generator = KeyScalableAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0)) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun simpleSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2), 3) { + it.spatialId == 2 + } + } + + @Test + fun filteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2), 3) { + it.spatialId == 0 + } + } + + @Test + fun temporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2), 3) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun spatialAndTemporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0), 3) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun largerSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2 + 2), 7) { + it.spatialId == 2 + } + } + + @Test + fun largerFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 2), 7) { + it.spatialId == 0 + } + } + + @Test + fun largerTemporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 3 * 2), 7) { + it.temporalId == 0 && it.spatialId == 2 + } + } + + @Test + fun largerSpatialAndTemporalFilteredSingleEncodingSimulcastOutOfOrderTest() { + val generator = SingleEncodingSimulcastAv1PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, dt = 0), 7) { + it.spatialId == 0 && it.temporalId == 0 + } + } + + @Test + fun slightlyDelayedKeyframeTest() { + val generator = TemporallyScaledPacketGenerator(1) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "slightlyDelayedKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + val firstPacketInfo = generator.nextPacket() + val targetIndex = getIndex(eid = 0, dt = 2) + for (i in 0..2) { + val packetInfo = generator.nextPacket() + + Assert.assertFalse(context.accept(packetInfo, targetIndex)) + } + Assert.assertTrue(context.accept(firstPacketInfo, targetIndex)) + context.rewriteRtp(firstPacketInfo) + for (i in 0..9995) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun veryDelayedKeyframeTest() { + val generator = TemporallyScaledPacketGenerator(1) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "veryDelayedKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + val firstPacketInfo = generator.nextPacket() + val targetIndex = getIndex(eid = 0, dt = 2) + for (i in 0..3) { + val packetInfo = generator.nextPacket(missedStructure = true) + Assert.assertFalse(context.accept(packetInfo, targetIndex)) + } + Assert.assertFalse(context.accept(firstPacketInfo, targetIndex)) + for (i in 0..9) { + val packetInfo = generator.nextPacket() + Assert.assertFalse(context.accept(packetInfo, targetIndex)) + } + generator.requestKeyframe() + for (i in 0..9995) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun twoStreamsNoSwitchingTest() { + val generator1 = TemporallyScaledPacketGenerator(packetsPerFrame = 3, encodingId = 1) + val generator2 = TemporallyScaledPacketGenerator(packetsPerFrame = 3, encodingId = 0) + generator2.ssrc = 0xdeadbeefL + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsNoSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext(diagnosticContext, initialState, null, logger) + val targetIndex = getIndex(eid = 1, dt = 2) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + for (i in 0..9999) { + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun twoStreamsSwitchingTest() { + val generator1 = TemporallyScaledPacketGenerator(3, encodingId = 0) + val generator2 = TemporallyScaledPacketGenerator(3, encodingId = 1) + generator2.ssrc = 0xdeadbeefL + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext(diagnosticContext, initialState, null, logger) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + var expectedTemplateOffset = 0 + var targetIndex = getIndex(eid = 0, dt = 2) + + /* Start by wanting encoding 0 */ + for (i in 0..899) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + if (i == 0) { + expectedTemplateOffset = packet1.descriptor!!.structure.templateIdOffset + } + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertTrue(context.rewriteRtcp(srPacket1)) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) + Assert.assertFalse(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + Assert.assertEquals(expectedFrameNumber, packet1.frameNumber) + Assert.assertEquals(expectedTemplateOffset, packet1.descriptor?.structure?.templateIdOffset) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (packet1.isMarked) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Switch to wanting encoding 1, but don't send a keyframe. We should stay at the first encoding. */ + targetIndex = getIndex(eid = 1, dt = 2) + for (i in 0..89) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertTrue(context.rewriteRtcp(srPacket1)) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) + Assert.assertFalse(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + Assert.assertEquals(expectedFrameNumber, packet1.frameNumber) + Assert.assertEquals(expectedTemplateOffset, packet1.descriptor?.structure?.templateIdOffset) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (packet1.isMarked) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + generator1.requestKeyframe() + generator2.requestKeyframe() + + /* After a keyframe we should accept spatial layer 1 */ + for (i in 0..8999) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + + /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ + Assert.assertEquals(i == 0, context.accept(packetInfo1, targetIndex)) + Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) + if (i == 0) { + context.rewriteRtp(packetInfo1) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + expectedTemplateOffset += packet1.descriptor!!.structure.templateCount + } + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + val packet2 = packetInfo2.packetAs() + Assert.assertTrue(context.accept(packetInfo2, targetIndex)) + val expectedTemplateId = (packet2.descriptor!!.frameDependencyTemplateId + expectedTemplateOffset) % 64 + context.rewriteRtp(packetInfo2) + Assert.assertTrue(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(packet2.ssrc, srPacket2.senderSsrc) + Assert.assertEquals(packet2.timestamp, srPacket2.senderInfo.rtpTimestamp) + if (i == 0) { + /* We leave a 1-packet gap for the layer 0 keyframe. */ + expectedSeq += 2 + /* ts will advance by an extra 3000 samples for the extra frame. */ + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + /* frame number will advance by 1 for the extra keyframe. */ + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + Assert.assertEquals(expectedSeq, packet2.sequenceNumber) + Assert.assertEquals(expectedTs, packet2.timestamp) + Assert.assertEquals(expectedFrameNumber, packet2.frameNumber) + Assert.assertEquals(expectedTemplateId, packet2.descriptor?.frameDependencyTemplateId) + if (packet2.descriptor?.newTemplateDependencyStructure != null) { + Assert.assertEquals( + expectedTemplateOffset, + packet2.descriptor?.newTemplateDependencyStructure?.templateIdOffset + ) + } + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet2.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (packet2.isMarked) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun temporalLayerSwitchingTest() { + val generator = TemporallyScaledPacketGenerator(3) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "temporalLayerSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + var targetTid = 0 + var decodableTid = 0 + var targetIndex = getIndex(0, targetTid) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + for (i in 0..9999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, targetIndex) + if (accepted) { + if (decodableTid < packet.frameInfo!!.temporalId) { + decodableTid = packet.frameInfo!!.temporalId + } + } else { + if (decodableTid > packet.frameInfo!!.temporalId - 1) { + decodableTid = packet.frameInfo!!.temporalId - 1 + } + } + if (packet.frameInfo!!.temporalId <= decodableTid) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + if (i % 97 == 0) { // Prime number so it's out of sync with packet cycles. + targetTid = (targetTid + 2) % 3 + targetIndex = getIndex(0, targetTid) + } + } + } + } + + private fun runLargeDropoutTest( + generator: Av1PacketGenerator, + targetIndex: Int, + expectAccept: (FrameInfo) -> Boolean + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + for (i in 0..999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, targetIndex) + val frameInfo = packet.frameInfo!! + val endOfPicture = packet.isMarked + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + for (gap in 64..65536 step { it * 2 }) { + for (i in 0 until gap) { + generator.nextPacket() + } + var packetInfo: PacketInfo + var packet: Av1DDPacket + var frameInfo: FrameInfo + do { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + frameInfo = packet.frameInfo!! + } while (!expectAccept(frameInfo)) + var endOfPicture = packet.isMarked + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + + /* Allow any values after a gap. */ + expectedSeq = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, 1) + expectedTs = packet.timestamp + expectedFrameNumber = packet.frameNumber + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, targetIndex) + endOfPicture = packet.isMarked + frameInfo = packet.frameInfo!! + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + } + + @Test + fun largeDropoutTest() { + val generator = TemporallyScaledPacketGenerator(1) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredDropoutTest() { + val generator = TemporallyScaledPacketGenerator(1) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largeFrameDropoutTest() { + val generator = TemporallyScaledPacketGenerator(3) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largeFrameFilteredDropoutTest() { + val generator = TemporallyScaledPacketGenerator(3) + runLargeDropoutTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + private fun runSourceSuspensionTest( + generator: Av1PacketGenerator, + targetIndex: Int, + expectAccept: (FrameInfo) -> Boolean + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + null, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedFrameNumber = 0 + + var packetInfo: PacketInfo + var packet: Av1DDPacket + var frameInfo: FrameInfo + + var lastPacketAccepted = false + var lastFrameAccepted = -1 + + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + frameInfo = packet.frameInfo!! + val accepted = context.accept(packetInfo, targetIndex) + val endOfPicture = packet.isMarked + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + lastPacketAccepted = true + lastFrameAccepted = packet.frameNumber + } else { + Assert.assertFalse(accepted) + lastPacketAccepted = false + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + for (suspended in 64..65536 step { it * 2 }) { + /* If the last frame was accepted, finish the current frame if this generator is creating multi-packet + frames. */ + if (lastPacketAccepted) { + while (generator.packetOfFrame != 0) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, targetIndex) + val endOfPicture = packet.isMarked + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + /* Turn the source off for a time. */ + for (i in 0 until suspended) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, RtpLayerDesc.SUSPENDED_INDEX) + Assert.assertFalse(accepted) + val endOfPicture = packet.isMarked + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Switch back to wanting [targetIndex], but don't send a keyframe for a while. + * Should still be dropped. */ + for (i in 0 until 30) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + + val accepted = context.accept(packetInfo, targetIndex) + val endOfPicture = packet.isMarked + Assert.assertFalse(accepted) + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Request a keyframe. Will be sent as of the next frame. */ + generator.requestKeyframe() + /* If this generator is creating multi-packet frames, finish the previous frame. */ + while (generator.packetOfFrame != 0) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, targetIndex) + val endOfPicture = packet.isMarked + Assert.assertFalse(accepted) + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(lastFrameAccepted, 1) + + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + frameInfo = packet.frameInfo!! + val accepted = context.accept(packetInfo, targetIndex) + val endOfPicture = packet.isMarked + if (expectAccept(frameInfo)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedFrameNumber, packet.frameNumber) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + lastPacketAccepted = true + lastFrameAccepted = packet.frameNumber + } else { + Assert.assertFalse(accepted) + lastPacketAccepted = false + } + if (packet.isEndOfFrame) { + expectedFrameNumber = RtpUtils.applySequenceNumberDelta(expectedFrameNumber, 1) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + } + + @Test + fun sourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun filteredSourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(1) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun largeFrameSourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 2)) { + true + } + } + + @Test + fun largeFrameFilteredSourceSuspensionTest() { + val generator = TemporallyScaledPacketGenerator(3) + runSourceSuspensionTest(generator, getIndex(eid = 0, dt = 0)) { + it.temporalId == 0 + } + } + + @Test + fun persistentStateProjectionTest() { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "singlePacketProjectionTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Av1DDAdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + Av1PersistentState(5555, 7), + logger + ) + val generator = ScalableAv1PacketGenerator(1) + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val targetIndex = getIndex(eid = 0, dt = 0) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + Assert.assertEquals(10001, packet.sequenceNumber) + Assert.assertEquals(1003000, packet.timestamp) + Assert.assertEquals(5556, packet.frameNumber) + Assert.assertEquals(0, packet.frameInfo?.spatialId) + Assert.assertEquals(0, packet.frameInfo?.temporalId) + Assert.assertEquals(7, packet.descriptor?.structure?.templateIdOffset) + } +} + +private open class Av1PacketGenerator( + val packetsPerFrame: Int, + val keyframeTemplates: Array, + val normalTemplates: Array, + // Equivalent to number of layers + val framesPerTimestamp: Int, + templateDdHex: String, + val allKeyframesGetStructure: Boolean = false, + val encodingId: Int = 0 +) { + private val logger: Logger = LoggerImpl(javaClass.name) + + var packetOfFrame = 0 + private set + private var frameOfPicture = 0 + + private var seq = 0 + var ts: Long = 0L + private set + var ssrc: Long = 0xcafebabeL + private var frameNumber = 0 + private var keyframePicture = false + private var keyframeRequested = false + private var pictureCount = 0 + private var receivedTime = baseReceivedTime + private var templateIdx = 0 + private var packetCount = 0 + private var octetCount = 0 + + private val structure: Av1TemplateDependencyStructure + + init { + val dd = parseHexBinary(templateDdHex) + structure = Av1DependencyDescriptorReader(dd, 0, dd.size).parse(null).structure + } + + fun reset() { + val useRandom = true // switch off to ease debugging + val seed = System.currentTimeMillis() + val random = Random(seed) + seq = if (useRandom) random.nextInt() % 0x10000 else 0 + ts = if (useRandom) random.nextLong() % 0x100000000L else 0 + frameNumber = 0 + packetOfFrame = 0 + frameOfPicture = 0 + keyframePicture = true + keyframeRequested = false + ssrc = 0xcafebabeL + pictureCount = 0 + receivedTime = baseReceivedTime + templateIdx = 0 + packetCount = 0 + octetCount = 0 + } + + fun nextPacket(missedStructure: Boolean = false): PacketInfo { + val startOfFrame = packetOfFrame == 0 + val endOfFrame = packetOfFrame == packetsPerFrame - 1 + val startOfPicture = startOfFrame && frameOfPicture == 0 + val endOfPicture = endOfFrame && frameOfPicture == framesPerTimestamp - 1 + + val templateId = ( + (if (keyframePicture) keyframeTemplates[templateIdx] else normalTemplates[templateIdx]) + + structure.templateIdOffset + ) % 64 + + val buffer = packetTemplate.clone() + val rtpPacket = RtpPacket(buffer, 0, buffer.size) + rtpPacket.ssrc = ssrc + rtpPacket.sequenceNumber = seq + rtpPacket.timestamp = ts + rtpPacket.isMarked = endOfPicture + + val dd = Av1DependencyDescriptorHeaderExtension( + startOfFrame = startOfFrame, + endOfFrame = endOfFrame, + frameDependencyTemplateId = templateId, + frameNumber = frameNumber, + newTemplateDependencyStructure = + if (keyframePicture && startOfFrame && (startOfPicture || allKeyframesGetStructure)) { + structure + } else { + null + }, + activeDecodeTargetsBitmask = null, + customDtis = null, + customFdiffs = null, + customChains = null, + structure = structure + ) + + val ext = rtpPacket.addHeaderExtension(AV1_DD_HEADER_EXTENSION_ID, dd.encodedLength) + dd.write(ext) + rtpPacket.encodeHeaderExtensions() + + val av1Packet = Av1DDPacket( + rtpPacket, + AV1_DD_HEADER_EXTENSION_ID, + if (missedStructure) null else structure, + logger + ) + av1Packet.encodingId = encodingId + + val info = PacketInfo(av1Packet) + info.receivedTime = receivedTime + + seq = RtpUtils.applySequenceNumberDelta(seq, 1) + packetCount++ + octetCount += av1Packet.length + + if (endOfFrame) { + packetOfFrame = 0 + if (endOfPicture) { + frameOfPicture = 0 + } else { + frameOfPicture++ + } + templateIdx++ + if (keyframeRequested) { + keyframePicture = true + templateIdx = 0 + } else if (keyframePicture) { + if (templateIdx >= keyframeTemplates.size) { + keyframePicture = false + } + } + frameNumber = RtpUtils.applySequenceNumberDelta(frameNumber, 1) + } else { + packetOfFrame++ + } + + if (endOfPicture) { + ts = RtpUtils.applyTimestampDelta(ts, 3000) + + keyframeRequested = false + if (templateIdx >= normalTemplates.size) { + templateIdx = 0 + } + pictureCount++ + receivedTime = baseReceivedTime + Duration.ofMillis(pictureCount * 100L / 3) + } + + return info + } + + fun requestKeyframe() { + if (packetOfFrame == 0) { + keyframePicture = true + templateIdx = 0 + } else { + keyframeRequested = true + } + } + + val srPacket: RtcpSrPacket + get() { + val srPacketBuilder = RtcpSrPacketBuilder() + srPacketBuilder.rtcpHeader.senderSsrc = ssrc + val siBuilder = srPacketBuilder.senderInfo + siBuilder.setNtpFromJavaTime(receivedTime.toEpochMilli()) + siBuilder.rtpTimestamp = ts + siBuilder.sendersOctetCount = packetCount.toLong() + siBuilder.sendersOctetCount = octetCount.toLong() + return srPacketBuilder.build() + } + + init { + reset() + } + + companion object { + val baseReceivedTime: Instant = Instant.ofEpochMilli(1577836800000L) // 2020-01-01 00:00:00 UTC + + const val AV1_DD_HEADER_EXTENSION_ID = 11 + + private val packetTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + "80" + // V, P, X, CC + "29" + // M, PT + "0000" + // Seq + "00000000" + // TS + "cafebabe" + // SSRC + // Header extension will be added dynamically + // Dummy payload data + "0000000000000000000000" + ) + } +} + +private class NonScalableAv1PacketGenerator( + packetsPerFrame: Int +) : + Av1PacketGenerator( + packetsPerFrame, + arrayOf(0), + arrayOf(1), + 1, + "80000180003a410180ef808680" + ) + +private class TemporallyScaledPacketGenerator( + packetsPerFrame: Int, + encodingId: Int = 0 +) : Av1PacketGenerator( + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(0), + normalTemplates = arrayOf(1, 3, 2, 4), + framesPerTimestamp = 1, + templateDdHex = "800001800214eaa860414d141020842701df010d", + encodingId = encodingId +) + +private class ScalableAv1PacketGenerator( + packetsPerFrame: Int, + encodingId: Int = 0 +) : + Av1PacketGenerator( + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(1, 6, 11), + normalTemplates = arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + framesPerTimestamp = 3, + templateDdHex = "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + + "a87753013f00b3027f016704ff02cf", + encodingId = encodingId + ) + +private class KeyScalableAv1PacketGenerator( + packetsPerFrame: Int, + encodingId: Int = 0 +) : + Av1PacketGenerator( + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(0, 5, 10), + normalTemplates = arrayOf(1, 6, 11, 3, 8, 13, 2, 7, 12, 4, 9, 14), + framesPerTimestamp = 3, + templateDdHex = "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf", + encodingId = encodingId + ) + +private class SingleEncodingSimulcastAv1PacketGenerator( + packetsPerFrame: Int, + encodingId: Int = 0 +) : + Av1PacketGenerator( + packetsPerFrame = packetsPerFrame, + keyframeTemplates = arrayOf(1, 6, 11), + normalTemplates = arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + framesPerTimestamp = 3, + templateDdHex = "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434", + allKeyframesGetStructure = true, + encodingId = encodingId + ) + +private infix fun IntRange.step(next: (Int) -> Int) = + generateSequence(first, next).takeWhile { if (first < last) it <= last else it >= last } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt new file mode 100644 index 0000000..02ec7d3 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilterTest.kt @@ -0,0 +1,896 @@ +/* + * Copyright @ 2019 - present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.av1 + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import jakarta.xml.bind.DatatypeConverter +import org.jitsi.nlj.rtp.codec.av1.Av1DDRtpLayerDesc +import org.jitsi.rtp.rtp.header_extensions.Av1DependencyDescriptorReader +import org.jitsi.rtp.rtp.header_extensions.Av1TemplateDependencyStructure +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.getClassForLogging +import java.time.Instant + +internal class Av1DDQualityFilterTest : ShouldSpec() { + init { + context("A non-scalable stream") { + should("be entirely projected") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleLayerFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 0) + + testGenerator(generator, filter, targetIndex) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + context("A temporally scalable stream") { + should("be entirely projected when TL2 is requested") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + should("project only the base temporal layer when targeted") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("project only the intermediate temporal layer when targeted") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId <= 1) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch the targeted layers, without a keyframe") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = TemporallyScaledFrameGenerator(av1FrameMaps) + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 0) + + testGenerator(generator, filter, targetIndex1, numFrames = 500) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex2) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + context("A spatially scalable stream") { + should("be entirely projected when SL2/TL2 is requested (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe true + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL2 (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2 (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId <= 1) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0 (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (L3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SVCFrameGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) (f.frameInfo!!.spatialId == 0) else true + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to spatial layer 1. For SVC, dropping down in spatial layers can happen immediately. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + testGenerator(generator, filter, targetIndex3) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId <= 1) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + } + context("A K-SVC spatially scalable stream") { + should("be able to be shaped to SL2/TL2 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency() + ) + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL2 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.spatialId == 1 || ( + f.frameInfo!!.spatialId == 0 && !f.frameInfo!!.hasInterPictureDependency() + ) + ) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0 (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.temporalId == 0 && ( + f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency() + ) + ) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 2) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (L3T3_KEY)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCFrameGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency()) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to spatial layer 1. For K-SVC, dropping down in spatial layers needs a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 2 || !f.frameInfo!!.hasInterPictureDependency()) + } else { + ( + f.frameInfo!!.spatialId == 1 || ( + f.frameInfo!!.spatialId == 0 && !f.frameInfo!!.hasInterPictureDependency() + ) + ) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 2) + } else { + (f.frameInfo!!.spatialId == 1) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + context("A K-SVC spatially scalable stream with a temporal shift") { + should("be able to be shaped to SL1/TL1 (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2 * 1 + 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency() + ) + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL1 (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL0 (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2 * 1 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.frameInfo!!.temporalId == 0 && ( + f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency() + ) + ) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (L2S2_KEY_SHIFT)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = KSVCShiftFrameGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 1) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe (f.frameInfo!!.spatialId == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 1. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 2 * 1 + 1) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency()) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 1) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch back to spatial layer 0. For K-SVC, dropping down in spatial layers needs a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 1) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 1 || !f.frameInfo!!.hasInterPictureDependency()) + } else { + (f.frameInfo!!.spatialId == 0) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 1) + } else { + (f.frameInfo!!.spatialId == 0) + } + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + context("A single-encoding simulcast stream") { + should("project all of layer 2 when when SL2/TL2 is requested (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 2) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL0/TL2 (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2 (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 1) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0 (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 2 && f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers (S3T3)") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = SingleEncodingSimulcastGenerator(av1FrameMaps) + + /* Start by sending spatial layer 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.frameInfo!!.spatialId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(0, 3 * 2 + 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 0) + } else { + (f.frameInfo!!.spatialId == 2) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to spatial layer 1. Need a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(0, 3 * 1 + 2) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.frameInfo!!.spatialId == 2) + } else { + (f.frameInfo!!.spatialId == 1) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + context("A multi-encoding simulcast stream") { + should("project all of encoding 2 when when Enc 2/TL2 is requested") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(2, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 2L || f.isKeyframe) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 0/TL2") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 0L) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 1/TL2") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(1, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 1L || (f.isKeyframe && f.ssrc == 0L)) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 2/TL0") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + val targetIndex = Av1DDRtpLayerDesc.getIndex(2, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ((f.ssrc == 2L || f.isKeyframe) && f.frameInfo!!.temporalId == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch encodings") { + val av1FrameMaps = HashMap() + + val filter = Av1DDQualityFilter(av1FrameMaps, logger) + val generator = MultiEncodingSimulcastGenerator(av1FrameMaps) + + /* Start by sending encoding 0. */ + val targetIndex1 = Av1DDRtpLayerDesc.getIndex(0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.ssrc == 0L) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + + /* Switch to encoding 2. Need a keyframe. */ + val targetIndex2 = Av1DDRtpLayerDesc.getIndex(2, 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.ssrc == 0L) + } else { + (f.ssrc == 2L || f.isKeyframe) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + + /* Switch to encoding 1. Need a keyframe. */ + val targetIndex3 = Av1DDRtpLayerDesc.getIndex(1, 2) + sawKeyframe = false + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + // We don't send discardable frames for the DT while there's a pending encoding downswitch + (f.ssrc == 2L && f.frameInfo!!.temporalId != 2) + } else { + (f.ssrc == 1L || (f.ssrc == 0L && f.isKeyframe)) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe (!sawKeyframe) + } + } + } + } + } + + private fun testGenerator( + g: FrameGenerator, + filter: Av1DDQualityFilter, + targetIndex: Int, + numFrames: Int = Int.MAX_VALUE, + evaluator: (Av1DDFrame, Av1DDQualityFilter.AcceptResult) -> Unit + ) { + var lastTs = -1L + var ms = -1L + var frames = 0 + while (g.hasNext() && frames < numFrames) { + val f = g.next() + + ms = if (f.timestamp != lastTs) { + f.timestamp / 90 + } else { + ms + 1 + } + lastTs = f.timestamp + + val result = filter.acceptFrame( + frame = f, + externalTargetIndex = targetIndex, + incomingEncoding = f.ssrc.toInt(), + receivedTime = Instant.ofEpochMilli(ms) + ) + f.isAccepted = result.accept + evaluator(f, result) + frames++ + } + } + + companion object { + val logger = LoggerImpl(getClassForLogging(this::class.java).name) + } +} + +private abstract class FrameGenerator : Iterator + +private open class DDBasedGenerator( + val av1FrameMaps: HashMap, + val keyframeInterval: Int, + val keyframeTemplates: Array, + val normalTemplates: Array, + ddHex: String +) : FrameGenerator() { + private var frameCount = 0 + private val structure: Av1TemplateDependencyStructure + + init { + val dd = DatatypeConverter.parseHexBinary(ddHex) + structure = Av1DependencyDescriptorReader(dd, 0, dd.size).parse(null).structure + } + + override fun hasNext(): Boolean = frameCount < TOTAL_FRAMES + + protected open fun isKeyframe(keyCycle: Int) = keyCycle == 0 + + override fun next(): Av1DDFrame { + val tCycle = frameCount % normalTemplates.size + val keyCycle = frameCount % keyframeInterval + + val templateId = if (keyCycle < keyframeTemplates.size) { + keyframeTemplates[tCycle] + } else { + normalTemplates[tCycle] + } + + val f = Av1DDFrame( + ssrc = 0, + timestamp = frameCount * 3000L, + earliestKnownSequenceNumber = frameCount, + latestKnownSequenceNumber = frameCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + frameInfo = structure.templateInfo[templateId], + // Will be less than 0xffff + frameNumber = frameCount, + index = frameCount.toLong(), + templateId = templateId, + structure = structure, + activeDecodeTargets = null, + isKeyframe = isKeyframe(keyCycle), + rawDependencyDescriptor = null + ) + av1FrameMaps.getOrPut(f.ssrc) { Av1DDFrameMap(Av1DDQualityFilterTest.logger) }.insertFrame(f) + frameCount++ + return f + } + + companion object { + private const val TOTAL_FRAMES = 10000 + } +} + +/** Generate a non-scalable AV1 stream, with a single keyframe at the start. */ +private class SingleLayerFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 10000, + arrayOf(0), + arrayOf(1), + "80000180003a410180ef808680" +) + +/** Generate a temporally-scaled series of AV1 frames, with a single keyframe at the start. */ +private class TemporallyScaledFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 10000, + arrayOf(0), + arrayOf(1, 3, 2, 4), + "800001800214eaa860414d141020842701df010d" +) + +/** Generate a spatially-scaled series of AV1 frames (L3T3), with full spatial dependencies and periodic keyframes. */ +private class SVCFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(1, 6, 11), + arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + + "a87753013f00b3027f016704ff02cf" +) + +/** Generate a spatially-scaled series of AV1 frames (L3T3), with keyframe spatial dependencies and periodic + * keyframes. */ +private class KSVCFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(0, 5, 10), + arrayOf(1, 6, 11, 3, 8, 13, 2, 7, 12, 4, 9, 14), + "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" +) + +/** Generate a spatially-scaled series of AV1 frames (L2T2), with keyframe spatial dependencies and periodic + * keyframes, with temporal structures shifted. */ +/* Note that as of Chrome 111, L3T3_KEY_SHIFT is not supported yet, so we're testing L2T2_KEY_SHIFT instead. */ +private class KSVCShiftFrameGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(0, 4, 1), + arrayOf(2, 6, 3, 5), + "8700ed80e3061eaa82804028280514d14134518010a091889a09409fc059c13fc0b3c0" +) + +/** Generate a single-stream temporally-scaled simulcast (S3T3) series of AV1 frames, with periodic keyframes. */ +private class SingleEncodingSimulcastGenerator(av1FrameMaps: HashMap) : DDBasedGenerator( + av1FrameMaps, + 144, + arrayOf(1, 6, 11), + arrayOf(0, 5, 10, 3, 8, 13, 2, 7, 12, 4, 9, 14), + "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434" +) { + // All frames of the initial picture get the DD structure attached + override fun isKeyframe(keyCycle: Int) = keyCycle < keyframeTemplates.size +} + +private class MultiEncodingSimulcastGenerator(val av1FrameMaps: HashMap) : FrameGenerator() { + private var frameCount = 0 + + override fun hasNext(): Boolean = frameCount < TOTAL_FRAMES + + override fun next(): Av1DDFrame { + val pictureCount = frameCount / NUM_ENCODINGS + val encoding = frameCount % NUM_ENCODINGS + val tCycle = pictureCount % normalTemplates.size + val keyCycle = pictureCount % KEYFRAME_INTERVAL + + val templateId = if (keyCycle < keyframeTemplates.size) { + keyframeTemplates[tCycle] + } else { + normalTemplates[tCycle] + } + + val keyframePicture = keyCycle == 0 + + val f = Av1DDFrame( + ssrc = encoding.toLong(), + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = pictureCount, + latestKnownSequenceNumber = pictureCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + frameInfo = structure.templateInfo[templateId], + // Will be less than 0xffff + frameNumber = pictureCount, + index = pictureCount.toLong(), + templateId = templateId, + structure = structure, + activeDecodeTargets = null, + isKeyframe = keyframePicture, + rawDependencyDescriptor = null + ) + av1FrameMaps.getOrPut(f.ssrc) { Av1DDFrameMap(Av1DDQualityFilterTest.logger) }.insertFrame(f) + frameCount++ + return f + } + + companion object { + private const val TOTAL_FRAMES = 10000 + private const val KEYFRAME_INTERVAL = 144 + private const val NUM_ENCODINGS = 3 + private val keyframeTemplates = arrayOf(0) + private val normalTemplates = arrayOf(1, 3, 2, 4) + + private val structure: Av1TemplateDependencyStructure + + init { + val dd = DatatypeConverter.parseHexBinary("800001800214eaa860414d141020842701df010d") + structure = Av1DependencyDescriptorReader(dd, 0, dd.size).parse(null).structure + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt new file mode 100644 index 0000000..28fd672 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionTest.kt @@ -0,0 +1,1849 @@ +/* + * Copyright @ 2019 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.cc.vp9 + +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.nlj.RtpLayerDesc.Companion.getEidFromIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.getIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.getSidFromIndex +import org.jitsi.nlj.RtpLayerDesc.Companion.getTidFromIndex +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyExtendedPictureIdDelta +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.applyTl0PicIdxDelta +import org.jitsi.nlj.codec.vpx.VpxUtils.Companion.getExtendedPictureIdDelta +import org.jitsi.nlj.rtp.codec.vp9.Vp9Packet +import org.jitsi.nlj.util.RtpSequenceIndexTracker +import org.jitsi.rtp.rtcp.RtcpSrPacket +import org.jitsi.rtp.rtcp.RtcpSrPacketBuilder +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.isNewerThan +import org.jitsi.rtp.util.isOlderThan +import org.jitsi.rtp.util.isOlderTimestampThan +import org.jitsi.utils.logging.DiagnosticContext +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.videobridge.cc.RtpState +import org.jitsi_modified.impl.neomedia.codec.video.vp9.DePacketizer +import org.junit.Assert +import org.junit.Test +import java.time.Duration +import java.time.Instant +import java.util.Random +import java.util.TreeMap +import javax.xml.bind.DatatypeConverter +import kotlin.collections.ArrayList + +class Vp9AdaptiveSourceProjectionTest { + private val logger: Logger = LoggerImpl(javaClass.name) + + @Test + fun singlePacketProjectionTest() { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "singlePacketProjectionTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val generator = ScalableVp9PacketGenerator(1) + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val targetIndex = getIndex(eid = 0, sid = 0, tid = 0) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + Assert.assertEquals(10001, packet.sequenceNumber) + Assert.assertEquals(1003000, packet.timestamp) + Assert.assertEquals(0, packet.pictureId) + Assert.assertEquals(0, packet.spatialLayerIndex) + Assert.assertEquals(0, packet.temporalLayerIndex) + } + + private fun runInOrderTest(generator: Vp9PacketGenerator, targetIndex: Int) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedPicId = 0 + var expectedTl0PicIdx = 0 + var maybeStartOfPicture = false + var prevEncodingId = -1 + val targetEid = getEidFromIndex(targetIndex) + val targetSid = getSidFromIndex(targetIndex) + val targetTid = getTidFromIndex(targetIndex) + for (i in 0..99999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val bumpedTsAndPic: Boolean + if (maybeStartOfPicture && packet.encodingId == 0) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + bumpedTsAndPic = true + } else { + bumpedTsAndPic = false + } + val accepted = context.accept( + packetInfo, + targetIndex + ) + val bumpedTl0PicIdx: Boolean + if (!packet.hasLayerIndices) { + expectedTl0PicIdx = -1 + bumpedTl0PicIdx = false + } else if (packet.isStartOfFrame && + packet.spatialLayerIndex == 0 && + packet.temporalLayerIndex == 0 && + packet.encodingId == 0 + ) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + bumpedTl0PicIdx = true + } else { + bumpedTl0PicIdx = false + } + /* When we switch encodings we always bump the TS, picID, and tl0picidx, + * even if the source packets had them the same. */ + if (accepted && packet.isKeyframe && prevEncodingId != -1 && prevEncodingId != packet.encodingId) { + if (!bumpedTsAndPic) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + if (!bumpedTl0PicIdx) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + } + val endOfPicture = packet.isEndOfPicture // Save this before rewriteRtp + if (packet.temporalLayerIndex <= targetTid && + ( + packet.encodingId == targetEid || + (packet.isKeyframe && packet.encodingId < targetEid) + ) && + ( + packet.spatialLayerIndex == targetSid || + (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) + ) + ) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedPicId, packet.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet.TL0PICIDX) + Assert.assertEquals( + packet.isEndOfFrame && packet.effectiveSpatialLayerIndex == targetSid, + packet.isMarked + ) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + prevEncodingId = packet.encodingId + } else { + Assert.assertFalse(accepted) + prevEncodingId = -1 + } + maybeStartOfPicture = endOfPicture + } + } + + private class ProjectedPacket constructor( + val packet: Vp9Packet, + val origSeq: Int, + val extOrigSeq: Long, + val nearOldest: Boolean + ) + + /** Run an out-of-order test on a single stream, randomized order except for the first packet. */ + private fun doRunOutOfOrderTest( + generator: Vp9PacketGenerator, + targetIndex: Int, + initialOrderedCount: Int, + seed: Long + ) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val expectedInitialTs: Long = RtpUtils.applyTimestampDelta(initialState.maxTimestamp, 3000) + val expectedTsOffset: Long = RtpUtils.getTimestampDiff(expectedInitialTs, generator.ts) + val targetSid = getSidFromIndex(targetIndex) + val targetTid = getTidFromIndex(targetIndex) + val reorderSize = 64 + val buffer = ArrayList(reorderSize) + for (i in 0 until reorderSize) { + buffer.add(generator.nextPacket()) + } + val random = Random(seed) + var orderedCount = initialOrderedCount - 1 + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var latestSeq = buffer[0]!!.packetAs().sequenceNumber + val projectedPackets = TreeMap() + val origSeqIdxTracker = RtpSequenceIndexTracker() + val newSeqIdxTracker = RtpSequenceIndexTracker() + for (i in 0..99999) { + val packetInfo = buffer[0] + val packet = packetInfo!!.packetAs() + val origSeq = packet.sequenceNumber + val origTs = packet.timestamp + val origTl0PicIdx = packet.TL0PICIDX + if (latestSeq isOlderThan origSeq) { + latestSeq = origSeq + } + val accepted = context.accept(packetInfo, targetIndex) + val oldestValidSeq: Int = + RtpUtils.applySequenceNumberDelta( + latestSeq, + -((Vp9PictureMap.PICTURE_MAP_SIZE - 1) * generator.packetsPerFrame) + ) + if (origSeq isOlderThan oldestValidSeq && !accepted) { + /* This is fine; packets that are too old get ignored. */ + /* Note we don't want assertFalse(accepted) here because slightly-too-old packets + * that are part of an existing accepted frame will be accepted. + */ + } else if (packet.temporalLayerIndex <= targetTid && ( + packet.spatialLayerIndex == targetSid || + (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) + ) + ) { + Assert.assertTrue(accepted) + + /* There's an edge condition in frame projection where a packet + of a frame can be projected, then the frame can be forgotten + for being too old, then a later packet of the frame (which is + just barely not too old) can be projected, at which point it + can potentially get assigned different sequence number/TL0PICIDX + values. + + This is an unlikely enough case in real life that it's not worth + worrying about; but the incredibly aggressive packet randomizer + used by the unit tests can trigger it, so explicitly allow it. + */ + val nearOldest: Boolean = + RtpUtils.getSequenceNumberDelta(origSeq, oldestValidSeq) < generator.packetsPerFrame + context.rewriteRtp(packetInfo) + Assert.assertEquals(RtpUtils.applyTimestampDelta(origTs, expectedTsOffset), packet.timestamp) + Assert.assertEquals(origTl0PicIdx, packet.TL0PICIDX) + val newSeq = packet.sequenceNumber + val extNewSeq = newSeqIdxTracker.update(newSeq) + val extOrigSeq = origSeqIdxTracker.update(origSeq) + Assert.assertFalse(projectedPackets.containsKey(extNewSeq)) + projectedPackets[extNewSeq] = ProjectedPacket(packet, origSeq, extOrigSeq, nearOldest) + } else { + Assert.assertFalse(accepted) + } + if (orderedCount > 0) { + buffer.removeAt(0) + buffer.add(generator.nextPacket()) + orderedCount-- + } else { + buffer[0] = generator.nextPacket() + buffer.shuffle(random) + } + } + val iter: Iterator = projectedPackets.keys.iterator() + var prevPacket = projectedPackets[iter.next()] + while (iter.hasNext()) { + val packet = projectedPackets[iter.next()] + Assert.assertTrue(packet!!.origSeq isNewerThan prevPacket!!.origSeq) + if (prevPacket.packet.timestamp isOlderTimestampThan packet.packet.timestamp) { + Assert.assertTrue(getExtendedPictureIdDelta(prevPacket.packet.pictureId, packet.packet.pictureId) < 0) + } else { + Assert.assertEquals(prevPacket.packet.timestamp, packet.packet.timestamp) + Assert.assertTrue(prevPacket.packet.pictureId == packet.packet.pictureId || packet.nearOldest) + } + prevPacket = packet + } + + /* Overall, we should not have expanded sequence numbers. */ + val firstPacket = projectedPackets.firstEntry().value + val lastPacket = projectedPackets.lastEntry().value + val origDelta = lastPacket!!.extOrigSeq - firstPacket!!.extOrigSeq + val projDelta = projectedPackets.lastKey() - projectedPackets.firstKey() + Assert.assertTrue(projDelta <= origDelta) + } + + /** Run multiple instances of out-of-order test on a single stream, with different + * random seeds. */ + private fun runOutOfOrderTest(generator: Vp9PacketGenerator, targetIndex: Int, initialOrderedCount: Int = 1) { + /* Seeds that have triggered problems in the past for this or VP8, plus a random one. */ + val seeds = longArrayOf(1576267371838L, 1578347926155L, 1579620018479L, System.currentTimeMillis()) + for (seed in seeds) { + try { + doRunOutOfOrderTest(generator, targetIndex, initialOrderedCount, seed) + } catch (e: Throwable) { + logger.error( + "Exception thrown in randomized test, seed = $seed", + e + ) + throw e + } + generator.reset() + } + } + + @Test + fun simpleNonScalableTest() { + val generator = NonScalableVp9PacketGenerator() + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun simpleProjectionTest() { + val generator = ScalableVp9PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun filteredProjectionTest() { + val generator = ScalableVp9PacketGenerator(1) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largerFrameProjectionTest() { + val generator = ScalableVp9PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun largerFrameFilteredTest() { + val generator = ScalableVp9PacketGenerator(3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun hugeFrameTest() { + val generator = ScalableVp9PacketGenerator(200) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun simpleKsvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2)) + } + + @Test + fun filteredKsvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun temporalFilteredKsvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 0)) + } + + @Test + fun spatialAndTemporalFilteredKsvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largerKsvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2)) + } + + @Test + fun largerFilteredKsvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun largerTemporalFilteredKsvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 0)) + } + + @Test + fun largerSpatialAndTemporalFilteredKsvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun simpleSvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2)) + } + + @Test + fun filteredSvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun temporalFilteredSvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 0)) + } + + @Test + fun spatialAndTemporalFilteredSvcTest() { + val generator = ScalableVp9PacketGenerator(1, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largerSvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2)) + } + + @Test + fun largerFilteredSvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun largerTemporalFilteredSvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 0)) + } + + @Test + fun largerSpatialAndTemporalFilteredSvcTest() { + val generator = ScalableVp9PacketGenerator(3, 3, false) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun simpleSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 2)) + } + + @Test + fun filteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun temporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 0)) + } + + @Test + fun spatialAndTemporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(1, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largerSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 2)) + } + + @Test + fun largerFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun largerTemporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 2, sid = 0, tid = 0)) + } + + @Test + fun largerSpatialAndTemporalFilteredSimulcastTest() { + val generator = SimulcastVp9PacketGenerator(3, 3) + runInOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun simpleOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun largerOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun filteredOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(1) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largerFilteredOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(3) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun simpleKsvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(1, 3) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2), 3) + } + + @Test + fun largerKsvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(3, 3) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2), 7) + } + + @Test + fun filteredKsvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(1, 3) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2), 3) + } + + @Test + fun largerFilteredKsvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(3, 3) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2), 7) + } + + @Test + fun simpleSvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(1, 3, false) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2), 3) + } + + @Test + fun largerSvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(3, 3, false) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 2, tid = 2), 7) + } + + @Test + fun filteredSvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(1, 3, false) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2), 3) + } + + @Test + fun largerFilteredSvcOutOfOrderTest() { + val generator = ScalableVp9PacketGenerator(3, 3, false) + runOutOfOrderTest(generator, getIndex(eid = 0, sid = 0, tid = 2), 7) + } + + @Test + fun slightlyDelayedKeyframeTest() { + val generator = ScalableVp9PacketGenerator(1) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "slightlyDelayedKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val firstPacketInfo = generator.nextPacket() + val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) + for (i in 0..2) { + val packetInfo = generator.nextPacket() + Assert.assertFalse(context.accept(packetInfo, targetIndex)) + } + Assert.assertTrue(context.accept(firstPacketInfo, targetIndex)) + context.rewriteRtp(firstPacketInfo) + for (i in 0..9995) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun veryDelayedKeyframeTest() { + val generator = ScalableVp9PacketGenerator(1) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "veryDelayedKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val firstPacketInfo = generator.nextPacket() + val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) + for (i in 0..3) { + val packetInfo = generator.nextPacket() + Assert.assertFalse(context.accept(packetInfo, targetIndex)) + } + Assert.assertFalse(context.accept(firstPacketInfo, targetIndex)) + for (i in 0..9) { + val packetInfo = generator.nextPacket() + Assert.assertFalse(context.accept(packetInfo, targetIndex)) + } + generator.requestKeyframe() + for (i in 0..9995) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun delayedPartialKeyframeTest() { + val generator = ScalableVp9PacketGenerator(3) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "delayedPartialKeyframeTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val firstPacketInfo = generator.nextPacket() + val firstPacket = firstPacketInfo.packetAs() + val targetIndex = getIndex(eid = 0, sid = 0, tid = 2) + var lowestSeq = Integer.MAX_VALUE + for (i in 0..10) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + Assert.assertTrue(packet.sequenceNumber > 10001) + lowestSeq = minOf(lowestSeq, packet.sequenceNumber) + } + + Assert.assertTrue(context.accept(firstPacketInfo, targetIndex)) + context.rewriteRtp(firstPacketInfo) + Assert.assertEquals(lowestSeq - 1, firstPacket.sequenceNumber) + + for (i in 0..9980) { + val packetInfo = generator.nextPacket() + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + } + } + + @Test + fun twoStreamsNoSwitchingTest() { + val generator1 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 1) + val generator2 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 0) + generator2.ssrc = 0xdeadbeefL + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsNoSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val targetIndex = getIndex(eid = 1, sid = 0, tid = 2) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + for (i in 0..9999) { + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + } + + @Test + fun twoStreamsSwitchingTest() { + val generator1 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 0) + val generator2 = ScalableVp9PacketGenerator(packetsPerFrame = 3, encodingId = 1) + generator2.ssrc = 0xdeadbeefL + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedPicId = 0 + var expectedTl0PicIdx = 0 + var targetIndex = getIndex(eid = 0, sid = 0, tid = 2) + + /* Start by wanting spatial layer 0 */ + for (i in 0..899) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + if (packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertTrue(context.rewriteRtcp(srPacket1)) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) + Assert.assertFalse(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + Assert.assertEquals(expectedPicId, packet1.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet1.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + + /* Switch to wanting spatial layer 1, but don't send a keyframe. We should stay at the higher layer. */ + targetIndex = getIndex(eid = 1, sid = 0, tid = 2) + for (i in 0..89) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + if (packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + Assert.assertTrue(context.accept(packetInfo1, targetIndex)) + context.rewriteRtp(packetInfo1) + Assert.assertTrue(context.rewriteRtcp(srPacket1)) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + Assert.assertFalse(context.accept(packetInfo2, targetIndex)) + Assert.assertFalse(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(expectedSeq, packet1.sequenceNumber) + Assert.assertEquals(expectedTs, packet1.timestamp) + Assert.assertEquals(expectedPicId, packet1.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet1.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet1.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + generator1.requestKeyframe() + generator2.requestKeyframe() + + /* After a keyframe we should accept spatial layer 1 */ + for (i in 0..8999) { + val srPacket1 = generator1.srPacket + val packetInfo1 = generator1.nextPacket() + val packet1 = packetInfo1.packetAs() + if (i == 0 && packet1.isStartOfFrame && packet1.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + + /* We will cut off the layer 0 keyframe after 1 packet, once we see the layer 1 keyframe. */ + Assert.assertEquals(i == 0, context.accept(packetInfo1, targetIndex)) + Assert.assertEquals(i == 0, context.rewriteRtcp(srPacket1)) + if (i == 0) { + context.rewriteRtp(packetInfo1) + Assert.assertEquals(packet1.ssrc, srPacket1.senderSsrc) + Assert.assertEquals(packet1.timestamp, srPacket1.senderInfo.rtpTimestamp) + } + val srPacket2 = generator2.srPacket + val packetInfo2 = generator2.nextPacket() + val packet2 = packetInfo2.packetAs() + if (packet2.isStartOfFrame && packet2.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + Assert.assertTrue(context.accept(packetInfo2, targetIndex)) + context.rewriteRtp(packetInfo2) + Assert.assertTrue(context.rewriteRtcp(srPacket2)) + Assert.assertEquals(packet2.ssrc, srPacket2.senderSsrc) + Assert.assertEquals(packet2.timestamp, srPacket2.senderInfo.rtpTimestamp) + if (i == 0) { + /* We leave a 2-packet gap for the layer 0 keyframe. */ + expectedSeq += 2 + /* ts will advance by an extra 3000 samples for the extra frame. */ + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + /* pid id and tl0picidx will advance by 1 for the extra keyframe. */ + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + Assert.assertEquals(expectedSeq, packet2.sequenceNumber) + Assert.assertEquals(expectedTs, packet2.timestamp) + Assert.assertEquals(expectedPicId, packet2.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet2.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet2.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + } + + @Test + fun temporalLayerSwitchingTest() { + val generator = ScalableVp9PacketGenerator(3) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "temporalLayerSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var targetTid = 0 + var decodableTid = 0 + var targetIndex = getIndex(0, 0, targetTid) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedPicId = 0 + var expectedTl0PicIdx = 0 + for (i in 0..9999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, targetIndex) + if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + if (accepted) { + if (decodableTid < packet.temporalLayerIndex) { + decodableTid = packet.temporalLayerIndex + } + } else { + if (decodableTid > packet.temporalLayerIndex - 1) { + decodableTid = packet.temporalLayerIndex - 1 + } + } + if (packet.temporalLayerIndex <= decodableTid) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedPicId, packet.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (packet.isEndOfFrame) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + if (i % 97 == 0) { // Prime number so it's out of sync with packet cycles. */ + targetTid = (targetTid + 2) % 3 + targetIndex = getIndex(0, 0, targetTid) + } + } + } + } + + @Test + fun simulcastToSvcSwitchTest() { + val simulcastGenerator = SimulcastVp9PacketGenerator(packetsPerFrame = 3, numEncodings = 3) + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = "twoStreamsSwitchingTest" + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + val simulcastTargetIndex = getIndex(1, 0, 2) + for (i in 0..999) { + val packetInfo = simulcastGenerator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, simulcastTargetIndex) + Assert.assertTrue(packet.spatialLayerIndex == 0) + if (packet.encodingId == 1 || (packet.isKeyframe && packet.encodingId < 1)) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + } else { + Assert.assertFalse(accepted) + } + } + val ksvcGenerator = + ScalableVp9PacketGenerator( + packetsPerFrame = 3, + numLayers = 3, + initialRtpState = simulcastGenerator.getRtpState() + ) + val svcTargetIndex = getIndex(0, 1, 2) + for (i in 0..9999) { + val packetInfo = ksvcGenerator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept(packetInfo, svcTargetIndex) + if (packet.encodingId == 0 && + (packet.spatialLayerIndex == 1 || (packet.isKeyframe && packet.spatialLayerIndex < 1)) + ) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + } else { + Assert.assertFalse(accepted) + } + } + } + + private fun runLargeDropoutTest(generator: Vp9PacketGenerator, targetIndex: Int) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedPicId = 0 + var expectedTl0PicIdx = 0 + val targetSid = getSidFromIndex(targetIndex) + val targetTid = getTidFromIndex(targetIndex) + for (i in 0..999) { + val packetInfo = generator.nextPacket() + val packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + val endOfPicture = packet.isEndOfPicture + if (packet.temporalLayerIndex <= targetTid && + ( + packet.spatialLayerIndex == targetSid || + (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) + ) + ) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedPicId, packet.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + for (gap in 64..65536 step { it * 2 }) { + for (i in 0 until gap) { + generator.nextPacket() + } + var packetInfo: PacketInfo + var packet: Vp9Packet + do { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + } while (packet.temporalLayerIndex > targetIndex) + Assert.assertTrue(context.accept(packetInfo, targetIndex)) + context.rewriteRtp(packetInfo) + + /* Allow any values after a gap. */ + expectedSeq = RtpUtils.applySequenceNumberDelta(packet.sequenceNumber, 1) + expectedTs = packet.timestamp + expectedPicId = packet.pictureId + expectedTl0PicIdx = packet.TL0PICIDX + if (packet.isEndOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + val endOfPicture = packet.isEndOfPicture + if (packet.temporalLayerIndex <= targetIndex) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedPicId, packet.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + } else { + Assert.assertFalse(accepted) + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + } + } + + @Test + fun largeDropoutTest() { + val generator = ScalableVp9PacketGenerator(1) + runLargeDropoutTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun filteredLargeDropoutTest() { + val generator = ScalableVp9PacketGenerator(1) + runLargeDropoutTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largeFrameDropoutTest() { + val generator = ScalableVp9PacketGenerator(3) + runLargeDropoutTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun filteredLargeFrameDropoutTest() { + val generator = ScalableVp9PacketGenerator(3) + runLargeDropoutTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + private fun runSourceSuspensionTest(generator: Vp9PacketGenerator, targetIndex: Int) { + val diagnosticContext = DiagnosticContext() + diagnosticContext["test"] = Thread.currentThread().stackTrace[2].methodName + val initialState = RtpState(1, 10000, 1000000) + val context = Vp9AdaptiveSourceProjectionContext( + diagnosticContext, + initialState, + logger + ) + var expectedSeq = 10001 + var expectedTs: Long = 1003000 + var expectedPicId = 0 + var expectedTl0PicIdx = 0 + val targetSid = getSidFromIndex(targetIndex) + val targetTid = getTidFromIndex(targetIndex) + + var packetInfo: PacketInfo + var packet: Vp9Packet + + var lastPacketAccepted = false + var lastPidAccepted = -1 + + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + val endOfPicture = packet.isEndOfPicture + if (packet.temporalLayerIndex <= targetTid && + ( + packet.spatialLayerIndex == targetSid || + (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) + ) + ) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedPicId, packet.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + lastPacketAccepted = true + lastPidAccepted = packet.pictureId + } else { + Assert.assertFalse(accepted) + lastPacketAccepted = false + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + for (suspended in 64..65536 step { it * 2 }) { + /* If the last frame was accepted, finish the current frame if this generator is creating multi-packet + frames. */ + if (lastPacketAccepted) { + while (generator.packetOfFrame != 0) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + if (packet.isEndOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + } + /* Turn the source off for a time. */ + for (i in 0 until suspended) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + RtpLayerDesc.SUSPENDED_INDEX + ) + Assert.assertFalse(accepted) + if (packet.isEndOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Switch back to wanting [targetIndex], but don't send a keyframe for a while. + * Should still be dropped. */ + for (i in 0 until 30) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + Assert.assertFalse(accepted) + if (packet.isEndOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + + /* Request a keyframe. Will be sent as of the next frame. */ + generator.requestKeyframe() + /* If this generator is creating multi-packet frames, finish the previous frame. */ + while (generator.packetOfFrame != 0) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + Assert.assertFalse(accepted) + if (packet.isEndOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + } + } + expectedPicId = applyExtendedPictureIdDelta(lastPidAccepted, 1) + + for (i in 0..999) { + packetInfo = generator.nextPacket() + packet = packetInfo.packetAs() + val accepted = context.accept( + packetInfo, + targetIndex + ) + if (packet.isStartOfFrame && packet.temporalLayerIndex == 0) { + expectedTl0PicIdx = applyTl0PicIdxDelta(expectedTl0PicIdx, 1) + } + val endOfPicture = packet.isEndOfPicture + if (packet.temporalLayerIndex <= targetTid && + ( + packet.spatialLayerIndex == targetSid || + (packet.isUpperLevelReference && packet.spatialLayerIndex < targetSid) + ) + ) { + Assert.assertTrue(accepted) + context.rewriteRtp(packetInfo) + Assert.assertEquals(expectedSeq, packet.sequenceNumber) + Assert.assertEquals(expectedTs, packet.timestamp) + Assert.assertEquals(expectedPicId, packet.pictureId) + Assert.assertEquals(expectedTl0PicIdx, packet.TL0PICIDX) + expectedSeq = RtpUtils.applySequenceNumberDelta(expectedSeq, 1) + lastPacketAccepted = true + lastPidAccepted = packet.pictureId + } else { + Assert.assertFalse(accepted) + lastPacketAccepted = false + } + if (endOfPicture) { + expectedTs = RtpUtils.applyTimestampDelta(expectedTs, 3000) + expectedPicId = applyExtendedPictureIdDelta(expectedPicId, 1) + } + } + } + } + + @Test + fun sourceSuspensionTest() { + val generator = ScalableVp9PacketGenerator(1) + runSourceSuspensionTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun filteredSourceSuspensionTest() { + val generator = ScalableVp9PacketGenerator(1) + runSourceSuspensionTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + @Test + fun largeFrameSourceSuspensionTest() { + val generator = ScalableVp9PacketGenerator(3) + runSourceSuspensionTest(generator, getIndex(eid = 0, sid = 0, tid = 2)) + } + + @Test + fun filteredLargeFrameSourceSuspensionTest() { + val generator = ScalableVp9PacketGenerator(3) + runSourceSuspensionTest(generator, getIndex(eid = 0, sid = 0, tid = 0)) + } + + private abstract class Vp9PacketGenerator { + open val packetsPerFrame: Int = 1 + abstract val ts: Long + var ssrc: Long = 0xcafebabeL + + abstract fun getRtpState(): RtpState + + abstract fun reset() + abstract fun nextPacket(): PacketInfo + abstract fun requestKeyframe() + abstract val packetOfFrame: Int + + companion object { + val baseReceivedTime = Instant.ofEpochMilli(1577836800000L) // 2020-01-01 00:00:00 UTC + } + } + + private class NonScalableVp9PacketGenerator( + val initialRtpState: RtpState? = null, + val encodingId: Int = 0 + ) : Vp9PacketGenerator() { + private var seq = 0 + private set + override var ts: Long = 0 + private set + private var picId = 0 + override var packetOfFrame = 0 + private var keyframePicture = false + private var keyframeRequested = false + private var frameCount = 0 + private var receivedTime = baseReceivedTime + + private val useRandom = initialRtpState == null // or switch off to ease debugging + + override fun getRtpState(): RtpState = RtpState(ssrc, seq, ts) + + init { + reset() + } + + override fun reset() { + val seed = System.currentTimeMillis() + val random = Random(seed) + seq = if (initialRtpState != null) { + RtpUtils.applySequenceNumberDelta(initialRtpState.maxSequenceNumber, 1) + } else if (useRandom) { + random.nextInt() % 0x10000 + } else { + 0 + } + ts = if (initialRtpState != null) { + RtpUtils.applyTimestampDelta(initialRtpState.maxTimestamp, 3000) + } else if (useRandom) { + random.nextLong() % 0x100000000L + } else { + 0 + } + picId = 0 + packetOfFrame = 0 + keyframePicture = true + keyframeRequested = false + ssrc = 0xcafebabeL + frameCount = 0 + receivedTime = baseReceivedTime + } + + override fun nextPacket(): PacketInfo { + val startOfFrame = packetOfFrame == 0 + val endOfFrame = packetOfFrame == packetsPerFrame - 1 + val buffer = vp9PacketTemplate.clone() + val rtpPacket = RtpPacket(buffer, 0, buffer.size) + rtpPacket.ssrc = ssrc + rtpPacket.sequenceNumber = seq + rtpPacket.timestamp = ts + + /* Do VP9 manipulations on buffer before constructing Vp9Packet, because + Vp9Packet computes values at construct-time. */ + DePacketizer.VP9PayloadDescriptor.setStartOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + startOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setEndOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + endOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setInterPicturePredicted( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + !keyframePicture + ) + + rtpPacket.isMarked = endOfFrame + val vp9Packet = rtpPacket.toOtherType(::Vp9Packet) + + /* Make sure our manipulations of the raw buffer were correct. */ + Assert.assertEquals(startOfFrame, vp9Packet.isStartOfFrame) + Assert.assertEquals(endOfFrame, vp9Packet.isEndOfFrame) + Assert.assertEquals(!keyframePicture, vp9Packet.isInterPicturePredicted) + Assert.assertEquals(false, vp9Packet.hasLayerIndices) + Assert.assertEquals(false, vp9Packet.isSwitchingUpPoint) + Assert.assertEquals(false, vp9Packet.usesInterLayerDependency) + Assert.assertEquals(keyframePicture, vp9Packet.isKeyframe) + + vp9Packet.encodingId = encodingId + vp9Packet.pictureId = picId + val info = PacketInfo(vp9Packet) + info.receivedTime = receivedTime + seq = RtpUtils.applySequenceNumberDelta(seq, 1) + if (endOfFrame) { + packetOfFrame = 0 + ts = RtpUtils.applyTimestampDelta(ts, 3000) + picId = applyExtendedPictureIdDelta(picId, 1) + keyframePicture = keyframeRequested + keyframeRequested = false + frameCount++ + receivedTime = baseReceivedTime + Duration.ofMillis(frameCount * 100L / 3) + } else { + packetOfFrame++ + } + return info + } + override fun requestKeyframe() { + if (packetOfFrame == 0) { + keyframePicture = true + keyframeRequested = false + } else { + keyframeRequested = true + } + } + companion object { + private val vp9PacketTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + // V, P, X, CC + "80" + + // M, PT + "60" + + // Seq + "0000" + + // TS + "00000000" + + // SSRC + "cafebabe" + + // VP9 Payload descriptor + // I=1,P=0,L=0,F=0,B=1,E=0,V=0,Z=0 + "88" + + // M=1,PID=0x653e=25918 + "e53e" + + /* TODO: Add SS if necessary. Not currently parsed by the source projection context. */ + // Dummy payload data + "000000" + ) + } + } + + private class ScalableVp9PacketGenerator( + override val packetsPerFrame: Int, + val numLayers: Int = 1, + val isKsvc: Boolean = true, + val initialRtpState: RtpState? = null, + val encodingId: Int = 0 + ) : + Vp9PacketGenerator() { + private var seq = 0 + private set + override var ts: Long = 0 + private set + private var picId = 0 + private var tl0picidx = 0 + override var packetOfFrame = 0 + private var keyframePicture = false + private var keyframeRequested = false + private var sid = 0 + private var tidCycle = 0 + private var packetCount = 0 + private var octetCount = 0 + private var frameCount = 0 + private var receivedTime = baseReceivedTime + + private val useRandom = initialRtpState == null // or switch off to ease debugging + + override fun getRtpState(): RtpState = RtpState(ssrc, seq, ts) + + init { + reset() + } + + override fun reset() { + val seed = System.currentTimeMillis() + val random = Random(seed) + seq = if (initialRtpState != null) { + RtpUtils.applySequenceNumberDelta(initialRtpState.maxSequenceNumber, 1) + } else if (useRandom) { + random.nextInt() % 0x10000 + } else { + 0 + } + ts = if (initialRtpState != null) { + RtpUtils.applyTimestampDelta(initialRtpState.maxTimestamp, 3000) + } else if (useRandom) { + random.nextLong() % 0x100000000L + } else { + 0 + } + picId = 0 + tl0picidx = 0 + packetOfFrame = 0 + keyframePicture = true + keyframeRequested = false + sid = 0 + tidCycle = 0 + ssrc = 0xcafebabeL + packetCount = 0 + octetCount = 0 + frameCount = 0 + receivedTime = baseReceivedTime + } + + override fun nextPacket(): PacketInfo { + val tid = when (tidCycle % 4) { + 0 -> 0 + + 2 -> 1 + + 1, 3 -> 2 + + else -> { + assert(false) // Math is broken + -1 + } + } + val startOfFrame = packetOfFrame == 0 + val endOfFrame = packetOfFrame == packetsPerFrame - 1 + val startOfPicture = startOfFrame && sid == 0 + val endOfPicture = endOfFrame && sid == numLayers - 1 + if (startOfPicture && tid == 0) { + tl0picidx = applyTl0PicIdxDelta(tl0picidx, 1) + } + val buffer = vp9SvcPacketTemplate.clone() + val rtpPacket = RtpPacket(buffer, 0, buffer.size) + rtpPacket.ssrc = ssrc + rtpPacket.sequenceNumber = seq + rtpPacket.timestamp = ts + + /* Do VP9 manipulations on buffer before constructing Vp9Packet, because + Vp9Packet computes values at construct-time. */ + DePacketizer.VP9PayloadDescriptor.setStartOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + startOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setEndOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + endOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setInterPicturePredicted( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + !keyframePicture + ) + DePacketizer.VP9PayloadDescriptor.setUpperLevelReference( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + if (!isKsvc) { + sid != numLayers - 1 + } else { + keyframePicture + } + ) + + Assert.assertTrue( + DePacketizer.VP9PayloadDescriptor.setLayerIndices( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + sid, + tid, + tid > 0, + sid > 0 && (!isKsvc || keyframePicture) + ) + ) + + rtpPacket.isMarked = endOfPicture + val vp9Packet = rtpPacket.toOtherType(::Vp9Packet) + + /* Make sure our manipulations of the raw buffer were correct. */ + Assert.assertEquals(startOfFrame, vp9Packet.isStartOfFrame) + Assert.assertEquals(endOfFrame, vp9Packet.isEndOfFrame) + Assert.assertEquals(endOfPicture, vp9Packet.isEndOfPicture) + Assert.assertEquals(!keyframePicture, vp9Packet.isInterPicturePredicted) + Assert.assertEquals( + if (!isKsvc) { + sid != numLayers - 1 + } else { + keyframePicture + }, + vp9Packet.isUpperLevelReference + ) + Assert.assertEquals(sid, vp9Packet.spatialLayerIndex) + Assert.assertEquals(tid, vp9Packet.temporalLayerIndex) + Assert.assertEquals(tid > 0, vp9Packet.isSwitchingUpPoint) + Assert.assertEquals(sid > 0 && (!isKsvc || keyframePicture), vp9Packet.usesInterLayerDependency) + Assert.assertEquals(keyframePicture && sid == 0, vp9Packet.isKeyframe) + + vp9Packet.encodingId = encodingId + vp9Packet.pictureId = picId + vp9Packet.TL0PICIDX = tl0picidx + val info = PacketInfo(vp9Packet) + info.receivedTime = receivedTime + seq = RtpUtils.applySequenceNumberDelta(seq, 1) + packetCount++ + octetCount += vp9Packet.length + if (endOfFrame) { + packetOfFrame = 0 + if (endOfPicture) { + sid = 0 + } else { + sid++ + } + } else { + packetOfFrame++ + } + if (endOfPicture) { + ts = RtpUtils.applyTimestampDelta(ts, 3000) + picId = applyExtendedPictureIdDelta(picId, 1) + tidCycle++ + keyframePicture = keyframeRequested + keyframeRequested = false + if (keyframePicture) { + tidCycle = 0 + } + frameCount++ + receivedTime = baseReceivedTime + Duration.ofMillis(frameCount * 100L / 3) + } + return info + } + + override fun requestKeyframe() { + if (packetOfFrame == 0) { + keyframePicture = true + keyframeRequested = false + tidCycle = 0 + } else { + keyframeRequested = true + } + } + + val srPacket: RtcpSrPacket + get() { + val srPacketBuilder = RtcpSrPacketBuilder() + srPacketBuilder.rtcpHeader.senderSsrc = ssrc + val siBuilder = srPacketBuilder.senderInfo + siBuilder.setNtpFromJavaTime(receivedTime.toEpochMilli()) + siBuilder.rtpTimestamp = ts + siBuilder.sendersOctetCount = packetCount.toLong() + siBuilder.sendersOctetCount = octetCount.toLong() + return srPacketBuilder.build() + } + + companion object { + private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + // V, P, X, CC + "80" + + // M, PT + "60" + + // Seq + "0000" + + // TS + "00000000" + + // SSRC + "cafebabe" + + // VP9 Payload descriptor + // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=0 + "a8" + + // M=1,PID=0x653e=25918 + "e53e" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0x5b=91 + "5b" + + /* TODO: Add SS if necessary. Not currently parsed by the source projection context. */ + // Dummy payload data + "000000" + ) + } + } + + private class SimulcastVp9PacketGenerator( + override val packetsPerFrame: Int, + val numEncodings: Int = 1, + val initialRtpState: RtpState? = null + ) : + Vp9PacketGenerator() { + private var seq = IntArray(numEncodings) { 0 } + override var ts: Long = 0 + private set + private var picId = 0 + private var tl0picidx = 0 + override var packetOfFrame = 0 + private var keyframePicture = false + private var keyframeRequested = false + private var enc = 0 + private var tidCycle = 0 + private var packetCount = 0 + private var octetCount = 0 + private var frameCount = 0 + private var receivedTime = baseReceivedTime + + private val ssrcs = arrayOf(ssrc, 0xdeadbeefL, 0xc001d00dL) + init { + require(numEncodings <= ssrcs.size) + } + + private val useRandom = initialRtpState == null // or switch off to ease debugging + + override fun getRtpState(): RtpState = RtpState(ssrc, seq[0], ts) + + init { + reset() + } + + override fun reset() { + val seed = System.currentTimeMillis() + val random = Random(seed) + seq[0] = if (initialRtpState != null) { + RtpUtils.applySequenceNumberDelta(initialRtpState.maxSequenceNumber, 1) + } else if (useRandom) { + random.nextInt() % 0x10000 + } else { + 0 + } + for (i in 1 until numEncodings) { + seq[i] = if (useRandom) { + random.nextInt() % 0x10000 + } else { + seq[0] + } + } + ts = if (initialRtpState != null) { + RtpUtils.applyTimestampDelta(initialRtpState.maxTimestamp, 3000) + } else if (useRandom) { + random.nextLong() % 0x100000000L + } else { + 0 + } + picId = 0 + tl0picidx = 0 + packetOfFrame = 0 + keyframePicture = true + keyframeRequested = false + enc = 0 + tidCycle = 0 + ssrc = 0xcafebabeL + packetCount = 0 + octetCount = 0 + frameCount = 0 + receivedTime = baseReceivedTime + } + + override fun nextPacket(): PacketInfo { + val tid = when (tidCycle % 4) { + 0 -> 0 + + 2 -> 1 + + 1, 3 -> 2 + + else -> { + assert(false) // Math is broken + -1 + } + } + val startOfFrame = packetOfFrame == 0 + val endOfFrame = packetOfFrame == packetsPerFrame - 1 + val startOfPicture = startOfFrame && enc == 0 + val endOfPicture = endOfFrame && enc == numEncodings - 1 + if (startOfPicture && tid == 0) { + tl0picidx = applyTl0PicIdxDelta(tl0picidx, 1) + } + val buffer = vp9SvcPacketTemplate.clone() + val rtpPacket = RtpPacket(buffer, 0, buffer.size) + rtpPacket.ssrc = ssrcs[enc] + rtpPacket.sequenceNumber = seq[enc] + rtpPacket.timestamp = ts + + /* Do VP9 manipulations on buffer before constructing Vp9Packet, because + Vp9Packet computes values at construct-time. */ + DePacketizer.VP9PayloadDescriptor.setStartOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + startOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setEndOfFrame( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + endOfFrame + ) + DePacketizer.VP9PayloadDescriptor.setInterPicturePredicted( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + !keyframePicture + ) + DePacketizer.VP9PayloadDescriptor.setUpperLevelReference( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + false + ) + + Assert.assertTrue( + DePacketizer.VP9PayloadDescriptor.setLayerIndices( + rtpPacket.buffer, + rtpPacket.payloadOffset, + rtpPacket.payloadLength, + 0, + tid, + tid > 0, + false + ) + ) + + rtpPacket.isMarked = endOfFrame + val vp9Packet = rtpPacket.toOtherType(::Vp9Packet) + + /* Make sure our manipulations of the raw buffer were correct. */ + Assert.assertEquals(startOfFrame, vp9Packet.isStartOfFrame) + Assert.assertEquals(endOfFrame, vp9Packet.isEndOfFrame) + Assert.assertEquals(endOfFrame, vp9Packet.isEndOfPicture) + Assert.assertEquals(!keyframePicture, vp9Packet.isInterPicturePredicted) + Assert.assertFalse(vp9Packet.isUpperLevelReference) + Assert.assertEquals(0, vp9Packet.spatialLayerIndex) + Assert.assertEquals(tid, vp9Packet.temporalLayerIndex) + Assert.assertEquals(tid > 0, vp9Packet.isSwitchingUpPoint) + Assert.assertFalse(vp9Packet.usesInterLayerDependency) + Assert.assertEquals(keyframePicture, vp9Packet.isKeyframe) + + vp9Packet.encodingId = enc + vp9Packet.pictureId = picId + vp9Packet.TL0PICIDX = tl0picidx + val info = PacketInfo(vp9Packet) + info.receivedTime = receivedTime + seq[enc] = RtpUtils.applySequenceNumberDelta(seq[enc], 1) + packetCount++ + octetCount += vp9Packet.length + if (endOfFrame) { + packetOfFrame = 0 + if (endOfPicture) { + enc = 0 + } else { + enc++ + } + } else { + packetOfFrame++ + } + if (endOfPicture) { + ts = RtpUtils.applyTimestampDelta(ts, 3000) + picId = applyExtendedPictureIdDelta(picId, 1) + tidCycle++ + keyframePicture = keyframeRequested + keyframeRequested = false + if (keyframePicture) { + tidCycle = 0 + } + frameCount++ + receivedTime = baseReceivedTime + Duration.ofMillis(frameCount * 100L / 3) + } + return info + } + + override fun requestKeyframe() { + if (packetOfFrame == 0) { + keyframePicture = true + keyframeRequested = false + tidCycle = 0 + } else { + keyframeRequested = true + } + } + + val srPacket: RtcpSrPacket + get() { + val srPacketBuilder = RtcpSrPacketBuilder() + srPacketBuilder.rtcpHeader.senderSsrc = ssrc + val siBuilder = srPacketBuilder.senderInfo + siBuilder.setNtpFromJavaTime(receivedTime.toEpochMilli()) + siBuilder.rtpTimestamp = ts + siBuilder.sendersOctetCount = packetCount.toLong() + siBuilder.sendersOctetCount = octetCount.toLong() + return srPacketBuilder.build() + } + + companion object { + private val vp9SvcPacketTemplate = DatatypeConverter.parseHexBinary( + // RTP Header + // V, P, X, CC + "80" + + // M, PT + "60" + + // Seq + "0000" + + // TS + "00000000" + + // SSRC + "cafebabe" + + // VP9 Payload descriptor + // I=1,P=0,L=1,F=0,B=1,E=0,V=0,Z=0 + "a8" + + // M=1,PID=0x653e=25918 + "e53e" + + // TID=0,U=0,SID=0,D=0 + "00" + + // TL0PICIDX=0x5b=91 + "5b" + + /* TODO: Add SS if necessary. Not currently parsed by the source projection context. */ + // Dummy payload data + "000000" + ) + } + } +} + +private infix fun IntRange.step(next: (Int) -> Int) = + generateSequence(first, next).takeWhile { if (first < last) it <= last else it >= last } diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt new file mode 100644 index 0000000..a6d5e89 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilterTest.kt @@ -0,0 +1,674 @@ +/* + * Copyright @ 2019 - present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.cc.vp9 + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.nlj.RtpLayerDesc +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.utils.logging2.getClassForLogging +import java.time.Instant + +internal class Vp9QualityFilterTest : ShouldSpec() { + init { + context("A non-scalable stream") { + should("be entirely projected") { + val filter = Vp9QualityFilter(logger) + val generator = SingleLayerFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 0) + + testGenerator(generator, filter, targetIndex) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + + context("A temporally scalable stream") { + should("be entirely projected when TL2 is requested") { + val filter = Vp9QualityFilter(logger) + val generator = TemporallyScaledFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + should("project only the base temporal layer when targeted") { + val filter = Vp9QualityFilter(logger) + val generator = TemporallyScaledFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.temporalLayer == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("project only the intermediate temporal layer when targeted") { + val filter = Vp9QualityFilter(logger) + val generator = TemporallyScaledFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 1) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.temporalLayer <= 1) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch the targeted layers, without a keyframe") { + val filter = Vp9QualityFilter(logger) + val generator = TemporallyScaledFrameGenerator() + val targetIndex1 = RtpLayerDesc.getIndex(0, 0, 0) + + testGenerator(generator, filter, targetIndex1, numFrames = 500) { f, result -> + result.accept shouldBe (f.temporalLayer == 0) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + val targetIndex2 = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex2) { _, result -> + result.accept shouldBe true + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + + context("A spatially scalable stream") { + should("be entirely projected when SL2/TL2 is requested") { + val filter = Vp9QualityFilter(logger) + val generator = SVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 2, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe true + result.mark shouldBe (f.spatialLayer == 2) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = SVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.spatialLayer == 0) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = SVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 1, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.spatialLayer <= 1) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0") { + val filter = Vp9QualityFilter(logger) + val generator = SVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 2, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.temporalLayer == 0) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 2) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers") { + val filter = Vp9QualityFilter(logger) + val generator = SVCFrameGenerator() + + /* Start by sending spatial layer 0. */ + val targetIndex1 = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.spatialLayer == 0) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = RtpLayerDesc.getIndex(0, 2, 2) + var sawTargetLayer = false + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.spatialLayer == 2) sawTargetLayer = true + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) (f.spatialLayer == 0) else true + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) (f.spatialLayer == 0) else (f.spatialLayer == 2) + filter.needsKeyframe shouldBe (sawTargetLayer && !sawKeyframe) + } + } + + /* Switch to spatial layer 1. For SVC, dropping down in spatial layers can happen immediately. */ + val targetIndex3 = RtpLayerDesc.getIndex(0, 1, 2) + testGenerator(generator, filter, targetIndex3) { f, result -> + result.accept shouldBe (f.spatialLayer <= 1) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 1) + filter.needsKeyframe shouldBe false + } + } + } + } + + context("A K-SVC spatially scalable stream") { + should("be able to be shaped to SL2/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = KSVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 2, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.spatialLayer == 2 || !f.isInterPicturePredicted) + result.mark shouldBe (f.spatialLayer == 2) + filter.needsKeyframe shouldBe false + } + } + should("be able to be shaped to SL0/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = KSVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.spatialLayer == 0) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 0) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL1/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = KSVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 1, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.spatialLayer == 1 || (f.spatialLayer < 1 && !f.isInterPicturePredicted)) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 1) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to SL2/TL0") { + val filter = Vp9QualityFilter(logger) + val generator = KSVCFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 2, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe ( + f.temporalLayer == 0 && + (f.spatialLayer == 2 || !f.isInterPicturePredicted) + ) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 2) + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch spatial layers") { + val filter = Vp9QualityFilter(logger) + val generator = KSVCFrameGenerator() + + /* Start by sending spatial layer 0. */ + val targetIndex1 = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.spatialLayer == 0) + if (result.accept) { + result.mark shouldBe (f.spatialLayer == 0) + filter.needsKeyframe shouldBe false + } + } + + /* Switch to spatial layer 2. Need a keyframe. */ + val targetIndex2 = RtpLayerDesc.getIndex(0, 2, 2) + var sawTargetLayer = false + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.spatialLayer == 2) sawTargetLayer = true + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.spatialLayer == 0) + } else { + (f.spatialLayer == 2 || !f.isInterPicturePredicted) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) (f.spatialLayer == 0) else (f.spatialLayer == 2) + filter.needsKeyframe shouldBe (sawTargetLayer && !sawKeyframe) + } + } + + /* Switch to spatial layer 1. For K-SVC this needs a keyframe. */ + /* Until the switch is complete we send only TL0. */ + sawTargetLayer = false + sawKeyframe = false + val targetIndex3 = RtpLayerDesc.getIndex(0, 1, 2) + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.spatialLayer == 1) sawTargetLayer = true + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + ((f.spatialLayer == 2 || !f.isInterPicturePredicted) && f.temporalLayer == 0) + } else { + (f.spatialLayer == 1 || (!f.isInterPicturePredicted && f.spatialLayer < 1)) + } + if (result.accept) { + result.mark shouldBe if (!sawKeyframe) (f.spatialLayer == 2) else (f.spatialLayer == 1) + filter.needsKeyframe shouldBe !sawKeyframe + } + } + } + } + + context("A simulcast source with multiple encodings") { + should("be able to be shaped to Enc 2/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = SimulcastFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(2, 0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 2L || f.isKeyframe) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 0/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = SimulcastFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 0L) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 1/TL2") { + val filter = Vp9QualityFilter(logger) + val generator = SimulcastFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(1, 0, 2) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.ssrc == 1L || (f.isKeyframe && f.ssrc < 1L)) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to be shaped to Enc 1/TL0") { + val filter = Vp9QualityFilter(logger) + val generator = SimulcastFrameGenerator() + val targetIndex = RtpLayerDesc.getIndex(2, 0, 0) + + testGenerator(generator, filter, targetIndex) { f, result -> + result.accept shouldBe (f.temporalLayer == 0 && (f.ssrc == 2L || f.isKeyframe)) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + } + should("be able to switch encodings") { + val filter = Vp9QualityFilter(logger) + val generator = SimulcastFrameGenerator() + + /* Start by sending spatial layer 0. */ + val targetIndex1 = RtpLayerDesc.getIndex(0, 0, 2) + + testGenerator(generator, filter, targetIndex1, numFrames = 1200) { f, result -> + result.accept shouldBe (f.ssrc == 0L) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe false + } + } + + /* Switch to encoding 2. Need a keyframe. */ + val targetIndex2 = RtpLayerDesc.getIndex(2, 0, 2) + var sawKeyframe = false + testGenerator(generator, filter, targetIndex2, numFrames = 1200) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) (f.ssrc == 0L) else (f.ssrc == 2L || f.isKeyframe) + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe !sawKeyframe + } + } + + /* Switch to encoding 1. Need a keyframe. */ + /* Until the switch is complete we send only TL0. */ + sawKeyframe = false + val targetIndex3 = RtpLayerDesc.getIndex(1, 0, 2) + testGenerator(generator, filter, targetIndex3) { f, result -> + if (f.isKeyframe) sawKeyframe = true + result.accept shouldBe if (!sawKeyframe) { + (f.temporalLayer == 0 && (f.ssrc == 2L || f.isKeyframe)) + } else { + (f.ssrc == 1L || (f.isKeyframe && f.ssrc < 1L)) + } + if (result.accept) { + result.mark shouldBe true + filter.needsKeyframe shouldBe !sawKeyframe + } + } + } + } + } + + private fun testGenerator( + g: FrameGenerator, + filter: Vp9QualityFilter, + targetIndex: Int, + numFrames: Int = Int.MAX_VALUE, + evaluator: (Vp9Frame, Vp9QualityFilter.AcceptResult) -> Unit + ) { + var lastTs = -1L + var ms = -1L + var frames = 0 + while (g.hasNext() && frames < numFrames) { + val f = g.next() + + ms = if (f.timestamp != lastTs) { + f.timestamp / 90 + } else { + ms + 1 + } + lastTs = f.timestamp + + val result = filter.acceptFrame( + frame = f, + incomingEncoding = f.ssrc.toInt(), + externalTargetIndex = targetIndex, + receivedTime = Instant.ofEpochMilli(ms) + ) + evaluator(f, result) + frames++ + } + } + + companion object { + private val logger = LoggerImpl(getClassForLogging(this::class.java).name) + } +} + +private abstract class FrameGenerator : Iterator + +/** Generate a non-scalable series of VP9 frames, with a single keyframe at the start. */ +private class SingleLayerFrameGenerator : FrameGenerator() { + private val totalPictures = 10000 + private var pictureCount = 0 + + override fun hasNext(): Boolean = pictureCount < totalPictures + + override fun next(): Vp9Frame { + val f = Vp9Frame( + ssrc = 0, + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = pictureCount, + latestKnownSequenceNumber = pictureCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + temporalLayer = 0, + spatialLayer = 0, + isUpperLevelReference = false, + isSwitchingUpPoint = false, + usesInterLayerDependency = false, + isInterPicturePredicted = (pictureCount > 0), + pictureId = pictureCount, + index = pictureCount.toLong(), + tl0PICIDX = pictureCount and 0xff, + isKeyframe = (pictureCount == 0), + numSpatialLayers = if (pictureCount == 0) 1 else -1 + ) + pictureCount++ + return f + } +} + +/** Generate a temporally-scaled series of VP9 frames, with a single keyframe at the start. */ +private class TemporallyScaledFrameGenerator : FrameGenerator() { + private val totalPictures = 10000 + private var pictureCount = 0 + private var tl0Count = -1 + + override fun hasNext(): Boolean = pictureCount < totalPictures + + override fun next(): Vp9Frame { + val tCycle = pictureCount % 4 + val tLayer = when { + tCycle == 0 -> 0 + tCycle == 2 -> 1 + else -> 2 + } + + if (tLayer == 0) tl0Count++ + + val f = Vp9Frame( + ssrc = 0, + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = pictureCount, + latestKnownSequenceNumber = pictureCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + temporalLayer = tLayer, + spatialLayer = 0, + isUpperLevelReference = false, + isSwitchingUpPoint = tLayer > 0, + usesInterLayerDependency = false, + isInterPicturePredicted = (pictureCount > 0), + pictureId = pictureCount, + index = pictureCount.toLong(), + tl0PICIDX = tl0Count and 0xff, + isKeyframe = (pictureCount == 0), + numSpatialLayers = if (pictureCount == 0) 1 else -1 + ) + pictureCount++ + return f + } +} + +/** Generate a spatially-scaled series of VP9 frames, with full spatial dependencies and periodic keyframes. */ +private class SVCFrameGenerator : FrameGenerator() { + private val totalPictures = 10000 + private var pictureCount = 0 + private var frameCount = 0 + private var tl0Count = -1 + private var sLayer = 0 + + override fun hasNext(): Boolean = pictureCount < totalPictures + + override fun next(): Vp9Frame { + val tCycle = pictureCount % 4 + val tLayer = when { + tCycle == 0 -> 0 + tCycle == 2 -> 1 + else -> 2 + } + + if (sLayer == 0 && tLayer == 0) tl0Count++ + + /* Periodic keyframes. Period needs to be a multiple of the tCycle size. */ + val keyframePicture = (pictureCount % 48) == 0 + + val f = Vp9Frame( + ssrc = 0, + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = frameCount, + latestKnownSequenceNumber = frameCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = sLayer == 2, + temporalLayer = tLayer, + spatialLayer = sLayer, + isUpperLevelReference = sLayer < 2, + isSwitchingUpPoint = tLayer > 0, + usesInterLayerDependency = sLayer > 0, + isInterPicturePredicted = !keyframePicture, + pictureId = pictureCount, + index = pictureCount.toLong(), + tl0PICIDX = tl0Count and 0xff, + isKeyframe = keyframePicture && sLayer == 0, + numSpatialLayers = if (keyframePicture && sLayer == 0) 3 else -1 + ) + frameCount++ + sLayer++ + if (sLayer == 3) { + sLayer = 0 + pictureCount++ + } + return f + } +} + +/** Generate a spatially-scaled series of VP9 frames, with K-SVC spatial dependencies and periodic keyframes. */ +private class KSVCFrameGenerator : FrameGenerator() { + private val totalPictures = 10000 + private var pictureCount = 0 + private var frameCount = 0 + private var tl0Count = -1 + private var sLayer = 0 + + override fun hasNext(): Boolean = pictureCount < totalPictures + + override fun next(): Vp9Frame { + val tCycle = pictureCount % 4 + val tLayer = when { + tCycle == 0 -> 0 + tCycle == 2 -> 1 + else -> 2 + } + + if (sLayer == 0 && tLayer == 0) tl0Count++ + + /* Periodic keyframes. Period needs to be a multiple of the tCycle size. */ + val keyframePicture = (pictureCount % 48) == 0 + + val f = Vp9Frame( + ssrc = 0, + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = frameCount, + latestKnownSequenceNumber = frameCount, + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = sLayer == 2, + temporalLayer = tLayer, + spatialLayer = sLayer, + isUpperLevelReference = keyframePicture && sLayer < 2, + isSwitchingUpPoint = tLayer > 0, + usesInterLayerDependency = keyframePicture && sLayer > 0, + isInterPicturePredicted = !keyframePicture, + pictureId = pictureCount, + index = pictureCount.toLong(), + tl0PICIDX = tl0Count and 0xff, + isKeyframe = keyframePicture && sLayer == 0, + numSpatialLayers = if (keyframePicture && sLayer == 0) 3 else -1 + ) + frameCount++ + sLayer++ + if (sLayer == 3) { + sLayer = 0 + pictureCount++ + } + return f + } +} + +/** Generate a simulcast series of VP9 frame, with periodic keyframes. */ +private class SimulcastFrameGenerator : FrameGenerator() { + private val totalPictures = 1000 + private var pictureCount = 0 + private var frameCount = 0 + private var tl0Count = -1 + private var enc = 0 + + override fun hasNext(): Boolean = pictureCount < totalPictures + + override fun next(): Vp9Frame { + val tCycle = pictureCount % 4 + val tLayer = when { + tCycle == 0 -> 0 + tCycle == 2 -> 1 + else -> 2 + } + + if (enc == 0 && tLayer == 0) tl0Count++ + + /* Periodic keyframes. Period needs to be a multiple of the tCycle size. */ + val keyframePicture = (pictureCount % 48) == 0 + + val f = Vp9Frame( + // Use the encoding ID as the SSRC to make testing easier. + ssrc = enc.toLong(), + timestamp = pictureCount * 3000L, + earliestKnownSequenceNumber = pictureCount + (enc * 10000), + latestKnownSequenceNumber = pictureCount + (enc * 10000), + seenStartOfFrame = true, + seenEndOfFrame = true, + seenMarker = true, + temporalLayer = tLayer, + spatialLayer = 0, + isUpperLevelReference = false, + isSwitchingUpPoint = tLayer > 0, + usesInterLayerDependency = false, + isInterPicturePredicted = !keyframePicture, + pictureId = pictureCount, + index = pictureCount.toLong(), + tl0PICIDX = tl0Count and 0xff, + isKeyframe = keyframePicture, + numSpatialLayers = if (keyframePicture) 1 else -1 + ) + frameCount++ + enc++ + if (enc == 3) { + enc = 0 + pictureCount++ + } + return f + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt new file mode 100644 index 0000000..3f1c0e7 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/colibri2/Colibri2UtilTest.kt @@ -0,0 +1,50 @@ +/* + * Copyright @ 2022 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.videobridge.colibri2.createConferenceAlreadyExistsError +import org.jitsi.videobridge.colibri2.createConferenceNotFoundError +import org.jitsi.xmpp.extensions.colibri2.Colibri2Error +import org.jitsi.xmpp.extensions.colibri2.ConferenceModifyIQ +import org.jitsi.xmpp.extensions.colibri2.IqProviderUtils +import org.jivesoftware.smack.util.PacketParserUtils + +class Colibri2UtilTest : ShouldSpec({ + IqProviderUtils.registerProviders() + val iq = ConferenceModifyIQ.builder("id").setMeetingId("m").build() + context("createConferenceAlreadyExistsError") { + val error = createConferenceAlreadyExistsError(iq, "i") + + val parsedIq = parseIQ(error.toXML().toString()) + val colibri2ErrorExtension = + parsedIq.error.getExtension(Colibri2Error.ELEMENT, Colibri2Error.NAMESPACE) + colibri2ErrorExtension shouldNotBe null + colibri2ErrorExtension.reason shouldBe Colibri2Error.Reason.CONFERENCE_ALREADY_EXISTS + } + + context("createConferenceNotFoundError") { + val error = createConferenceNotFoundError(iq, "i") + + val parsedIq = parseIQ(error.toXML().toString()) + val colibri2ErrorExtension = + parsedIq.error.getExtension(Colibri2Error.ELEMENT, Colibri2Error.NAMESPACE) + colibri2ErrorExtension shouldNotBe null + colibri2ErrorExtension.reason shouldBe Colibri2Error.Reason.CONFERENCE_NOT_FOUND + } +}) + +fun parseIQ(xml: String) = PacketParserUtils.parseIQ(PacketParserUtils.getParserFor(xml)) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/export/ExporterTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/export/ExporterTest.kt new file mode 100644 index 0000000..6dee86a --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/export/ExporterTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.mediajson.MediaEvent +import org.jitsi.mediajson.TranscriptionResultEvent +import org.jitsi.utils.logging2.LoggerImpl +import java.net.URI + +/** + * Drives [Exporter.handleIncomingMessage] directly (no socket) to verify how inbound mediajson `start`/`stop` + * events map to the synthetic-source sending-change callback: a start/stop with a talk timestamp fires the callback + * (start -> sending=true, stop -> sending=false); one without a timestamp (a plain stream start/stop) does not. + */ +class ExporterTest : ShouldSpec() { + private data class Change(val sourceName: String, val sending: Boolean, val timestamp: Long) + + /** A fresh [Exporter] plus the list capturing its sending-change callback invocations. */ + private fun fixture(): Pair> { + val changes = mutableListOf() + val exporter = Exporter( + URI("ws://localhost:1/"), + emptyMap(), + LoggerImpl(javaClass.name), + object : ExporterEventHandler { + override fun handleTranscriptionResult(event: TranscriptionResultEvent) {} + override fun handleMediaEvent(event: MediaEvent) {} + override fun handleSendingChange(sourceName: String, sending: Boolean, timestamp: Long) { + changes.add(Change(sourceName, sending, timestamp)) + } + override fun getAudioSourceName(ssrc: Long): String? = null + override fun getDiarize(ssrc: Long): Boolean = false + } + ) + return exporter to changes + } + + init { + context("inbound start/stop dispatch to the sending-change callback") { + should("fire sending=true for a start carrying a talk timestamp") { + val (exporter, changes) = fixture() + exporter.handleIncomingMessage( + """ + {"event":"start","sequenceNumber":1,"start":{"tag":"55555555-a0.hi", + "mediaFormat":{"encoding":"opus","sampleRate":48000,"channels":1},"timestamp":384000}} + """.trimIndent().replace("\n", "") + ) + changes shouldBe listOf(Change("55555555-a0.hi", true, 384000)) + } + should("fire sending=false for a stop carrying a talk timestamp") { + val (exporter, changes) = fixture() + exporter.handleIncomingMessage( + """{"event":"stop","sequenceNumber":2,"stop":{"tag":"55555555-a0.hi","timestamp":960000}}""" + ) + changes shouldBe listOf(Change("55555555-a0.hi", false, 960000)) + } + should("ignore a start with no timestamp (a plain stream-start announcement)") { + val (exporter, changes) = fixture() + exporter.handleIncomingMessage( + """ + {"event":"start","sequenceNumber":3,"start":{"tag":"55555555-a0.hi", + "mediaFormat":{"encoding":"opus","sampleRate":48000,"channels":1}}} + """.trimIndent().replace("\n", "") + ) + changes shouldBe emptyList() + } + should("ignore a stop with no timestamp (a plain stop)") { + val (exporter, changes) = fixture() + exporter.handleIncomingMessage( + """{"event":"stop","sequenceNumber":4,"stop":{"tag":"55555555-a0.hi"}}""" + ) + changes shouldBe emptyList() + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/export/ExporterWrapperTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/export/ExporterWrapperTest.kt new file mode 100644 index 0000000..f986f1a --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/export/ExporterWrapperTest.kt @@ -0,0 +1,334 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.jitsi.mediajson.MediaEvent +import org.jitsi.mediajson.TranscriptionResultEvent +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.videobridge.colibri2.FeatureNotImplementedException +import org.jitsi.videobridge.colibri2.IqProcessingException +import org.jitsi.xmpp.extensions.colibri2.Connect +import java.net.URI + +class ExporterWrapperTest : ShouldSpec() { + private val logger = LoggerImpl(javaClass.name) + + /** A no-op event handler; these tests exercise connect bookkeeping, not the exporter's callbacks. */ + private val eventHandler = object : ExporterEventHandler { + override fun handleTranscriptionResult(event: TranscriptionResultEvent) {} + override fun handleMediaEvent(event: MediaEvent) {} + override fun handleSendingChange(sourceName: String, sending: Boolean, timestamp: Long) {} + override fun getAudioSourceName(ssrc: Long): String? = null + override fun getDiarize(ssrc: Long): Boolean = false + } + + /** Records the mock [Exporter] created for each connect id, so tests can verify calls against them. */ + private inner class Fixture { + val exporters = mutableMapOf() + val wrapper = ExporterWrapper(logger, eventHandler) { connect -> + mockk(relaxed = true).also { exporters[connect.id] = it } + } + operator fun get(id: String): Exporter = exporters.getValue(id) + } + + private fun connect( + id: String, + url: String = "wss://example.com/$id", + type: Connect.Types = Connect.Types.TRANSCRIBER, + create: Boolean = false, + expire: Boolean = false, + headers: Map = emptyMap(), + exports: List = emptyList(), + requests: List = emptyList(), + ping: Pair? = null + ): Connect = Connect( + id = id, + url = URI(url), + protocol = Connect.Protocols.MEDIAJSON, + type = type, + create = create, + expire = expire + ).apply { + headers.forEach { (n, v) -> addHttpHeader(n, v) } + exports.forEach { addExport(it) } + requests.forEach { addRequest(it) } + ping?.let { setPing(it.first, it.second) } + } + + init { + context("Creating connects") { + should("create an exporter per connect, keyed by id") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true), connect("b", create = true))) + + f.wrapper.started shouldBe true + f.exporters.keys shouldBe setOf("a", "b") + } + should("allow multiple connects with the same url but different ids") { + val f = Fixture() + val url = "wss://example.com/shared" + f.wrapper.applyConnects( + listOf(connect("a", url = url, create = true), connect("b", url = url, create = true)) + ) + + f.exporters.keys shouldBe setOf("a", "b") + } + should("reject create for an already-existing id") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true))) + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", create = true))) + } + verify(exactly = 0) { f["a"].stop() } + } + should("reject a connect for an unknown id without create") { + val f = Fixture() + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a"))) + } + f.wrapper.started shouldBe false + } + } + + context("Delta semantics") { + should("leave unmentioned connects running") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true), connect("b", create = true))) + + // A delta that only updates 'a' must not touch 'b'. + f.wrapper.applyConnects(listOf(connect("a", exports = listOf("s1")))) + + verify(exactly = 0) { f["a"].stop() } + verify(exactly = 0) { f["b"].stop() } + f.wrapper.started shouldBe true + } + should("not stop everything on an empty delta") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true))) + + f.wrapper.applyConnects(emptyList()) + + verify(exactly = 0) { f["a"].stop() } + f.wrapper.started shouldBe true + } + should("expire only the named connect") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true), connect("b", create = true))) + + f.wrapper.applyConnects(listOf(connect("b", expire = true))) + + verify(exactly = 0) { f["a"].stop() } + verify(exactly = 1) { f["b"].stop() } + f.wrapper.started shouldBe true + } + should("ignore an expire for an unknown id") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true))) + + f.wrapper.applyConnects(listOf(connect("unknown", expire = true))) + + f.exporters.keys shouldBe setOf("a") + verify(exactly = 0) { f["a"].stop() } + } + } + + context("Same-id combinations within one delta") { + should("replace a connect via expire+create of the same id, regardless of order") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", url = "wss://example.com/a1", create = true))) + val old = f["a"] + + f.wrapper.applyConnects( + listOf( + connect("a", url = "wss://example.com/a2", create = true), + connect("a", url = "wss://example.com/a1", expire = true) + ) + ) + + verify(exactly = 1) { old.stop() } + f.wrapper.started shouldBe true + // The replacement exporter (created last for id "a") must not have been stopped. + verify(exactly = 0) { f["a"].stop() } + } + should("reject two creates for the same id") { + val f = Fixture() + shouldThrow { + f.wrapper.applyConnects( + listOf( + connect("a", url = "wss://example.com/a1", create = true), + connect("a", url = "wss://example.com/a2", create = true) + ) + ) + } + f.wrapper.started shouldBe false + f.exporters shouldBe emptyMap() + } + should("reject an update for an id expired in the same delta") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true))) + + shouldThrow { + f.wrapper.applyConnects( + listOf(connect("a", expire = true), connect("a", exports = listOf("s1"))) + ) + } + // The whole delta is rejected before anything is applied. + verify(exactly = 0) { f["a"].stop() } + f.wrapper.started shouldBe true + } + should("reject duplicate expires for the same id") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true))) + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", expire = true), connect("a", expire = true))) + } + verify(exactly = 0) { f["a"].stop() } + } + } + + context("stop()") { + should("stop all exporters and mark not started") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true), connect("b", create = true))) + + f.wrapper.stop() + + f.wrapper.started shouldBe false + verify(exactly = 1) { f["a"].stop() } + verify(exactly = 1) { f["b"].stop() } + } + should("reject connects after stop, so a racing delta can't start an unstoppable exporter") { + val f = Fixture() + f.wrapper.stop() + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", create = true))) + } + f.wrapper.started shouldBe false + f.exporters shouldBe emptyMap() + } + } + + context("Updating exports/requests in place") { + should("apply changed exports and requests to the running exporter") { + val f = Fixture() + f.wrapper.applyConnects( + listOf(connect("a", create = true, exports = listOf("s1"), requests = listOf("s1.en"))) + ) + + f.wrapper.applyConnects( + listOf(connect("a", exports = listOf("s1", "s2"), requests = listOf("s1.en", "s2.fr"))) + ) + + verify(exactly = 1) { f["a"].update(listOf("s1", "s2"), listOf("s1.en", "s2.fr")) } + verify(exactly = 0) { f["a"].stop() } + } + should("not update for an identical re-signaled connect") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true, exports = listOf("s1")))) + + f.wrapper.applyConnects(listOf(connect("a", exports = listOf("s1")))) + + verify(exactly = 0) { f["a"].update(any(), any()) } + } + should("not treat reordered source names as a change") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true, exports = listOf("s1", "s2")))) + + f.wrapper.applyConnects(listOf(connect("a", exports = listOf("s2", "s1")))) + + verify(exactly = 0) { f["a"].update(any(), any()) } + } + } + + context("Rejecting non-updatable changes") { + should("throw and leave the exporter untouched when HTTP headers change") { + val f = Fixture() + f.wrapper.applyConnects( + listOf(connect("a", create = true, headers = mapOf("Authorization" to "Bearer a"))) + ) + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", headers = mapOf("Authorization" to "Bearer b")))) + } + + verify(exactly = 0) { f["a"].stop() } + verify(exactly = 0) { f["a"].update(any(), any()) } + } + should("throw when the ping configuration changes") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true, ping = 1000 to 2000))) + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", ping = 3000 to 4000))) + } + } + should("throw when the url changes for an existing id") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", url = "wss://example.com/a1", create = true))) + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", url = "wss://example.com/a2"))) + } + } + should("throw when the type changes") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true, type = Connect.Types.TRANSCRIBER))) + + shouldThrow { + f.wrapper.applyConnects(listOf(connect("a", type = Connect.Types.TRANSLATOR))) + } + } + should("reject a video connect") { + val f = Fixture() + val videoConnect = Connect( + id = "a", + url = URI("wss://example.com/a"), + protocol = Connect.Protocols.MEDIAJSON, + type = Connect.Types.RECORDER, + video = true, + create = true + ) + + shouldThrow { + f.wrapper.applyConnects(listOf(videoConnect)) + } + } + } + + context("Debug state") { + should("tag each exporter's debug info with its connect id") { + val f = Fixture() + f.wrapper.applyConnects(listOf(connect("a", create = true), connect("b", create = true))) + // The mock exporters are relaxed, so give them real debug nodes for the tag to be added to. + every { f["a"].debugState() } returns JsonNodeFactory.instance.objectNode() + every { f["b"].debugState() } returns JsonNodeFactory.instance.objectNode() + + val tags = f.wrapper.debugState().get("exporters").map { it.get("tag").asText() }.toSet() + tags shouldBe setOf("a", "b") + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/export/MediaJsonSerializerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/export/MediaJsonSerializerTest.kt new file mode 100644 index 0000000..1da7de3 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/export/MediaJsonSerializerTest.kt @@ -0,0 +1,68 @@ +/* + * Copyright @ 2024 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.export + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.string.shouldNotContain +import org.jitsi.mediajson.Event +import org.jitsi.mediajson.StartEvent +import org.jitsi.nlj.PacketInfo +import org.jitsi.nlj.format.OpusPayloadType +import org.jitsi.nlj.rtp.AudioRtpPacket + +class MediaJsonSerializerTest : ShouldSpec() { + init { + context("The diarize flag on the start event") { + should("be set to true when the resolver returns true for the source's SSRC") { + val startEvent = capturedEvents(diarize = true).filterIsInstance().single() + startEvent.start.diarize shouldBe true + startEvent.toJson().shouldContain("diarize") + } + should("be omitted (null) when the resolver returns false") { + val startEvent = capturedEvents(diarize = false).filterIsInstance().single() + startEvent.start.diarize shouldBe null + // With NON_NULL inclusion, a null diarize must not appear on the wire, so the transcriber falls back + // to its global config. + startEvent.toJson().shouldNotContain("diarize") + } + } + } + + /** Runs a single audio packet through a serializer and returns the events it emitted. */ + private fun capturedEvents(diarize: Boolean): List { + val events = mutableListOf() + val serializer = MediaJsonSerializer( + getSourceName = { "source-1" }, + getDiarize = { diarize } + ) { events.add(it) } + serializer.encode(audioPacketInfo(ssrc = 12345L, epId = "endpoint-1")) + return events + } + + private fun audioPacketInfo(ssrc: Long, epId: String): PacketInfo { + val packet = AudioRtpPacket(ByteArray(1500), 0, 100).apply { + this.ssrc = ssrc + sequenceNumber = 1 + timestamp = 1000L + } + return PacketInfo(packet).apply { + endpointId = epId + payloadType = OpusPayloadType(111) + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/health/config/HealthConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/health/config/HealthConfigTest.kt new file mode 100644 index 0000000..6a88f87 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/health/config/HealthConfigTest.kt @@ -0,0 +1,54 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.health.config + +import io.kotest.matchers.shouldBe +import org.jitsi.ConfigTest +import org.jitsi.config.withLegacyConfig +import org.jitsi.config.withNewConfig +import java.time.Duration + +class HealthConfigTest : ConfigTest() { + init { + context("health interval") { + context("when legacy config and new config define a value") { + withLegacyConfig(legacyConfigWithHealthInterval) { + withNewConfig(newConfigWithHealthInterval) { + should("use the value from legacy config") { + HealthConfig.config.interval shouldBe Duration.ofSeconds(1) + } + } + } + } + context("when only new config defines a value") { + withNewConfig(newConfigWithHealthInterval) { + should("use the value from the new config") { + HealthConfig.config.interval shouldBe Duration.ofSeconds(5) + } + } + } + } + } +} + +private val legacyConfigWithHealthInterval = """ + org.jitsi.videobridge.health.INTERVAL=1000 +""".trimIndent() + +private val newConfigWithHealthInterval = """ + videobridge.health.interval = 5 seconds +""".trimIndent() diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/ConferenceSizeLastNLimitsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/ConferenceSizeLastNLimitsTest.kt new file mode 100644 index 0000000..b82bfed --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/ConferenceSizeLastNLimitsTest.kt @@ -0,0 +1,64 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.load_management + +import io.kotest.matchers.shouldBe +import org.jitsi.ConfigTest +import org.jitsi.config.withNewConfig + +class ConferenceSizeLastNLimitsTest : ConfigTest() { + init { + context("With limits set") { + withNewConfig( + """ + videobridge.load-management.conference-last-n-limits { + // Intentionally out of order + 40 = 10, + 20 = 20, + 30 = 15, + 50 = 5 + } + """.trimIndent() + ) { + ConferenceSizeLastNLimits().apply { + getLastNLimit(19) shouldBe -1 + getLastNLimit(20) shouldBe 20 + getLastNLimit(21) shouldBe 20 + + getLastNLimit(29) shouldBe 20 + getLastNLimit(30) shouldBe 15 + getLastNLimit(31) shouldBe 15 + + getLastNLimit(40) shouldBe 10 + + getLastNLimit(50) shouldBe 5 + getLastNLimit(1000) shouldBe 5 + } + } + } + context("With no limits set") { + withNewConfig( + """ + videobridge.load-management.conference-last-n-limits { } + """.trimIndent() + ) { + ConferenceSizeLastNLimits().apply { + getLastNLimit(111) shouldBe -1 + } + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/JvbLoadManagerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/JvbLoadManagerTest.kt new file mode 100644 index 0000000..2b7a986 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/JvbLoadManagerTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import io.kotest.matchers.shouldBe +import io.mockk.Called +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.jitsi.ConfigTest +import org.jitsi.config.withNewConfig +import org.jitsi.utils.mins +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock + +class JvbLoadManagerTest : ConfigTest() { + init { + context("JvbLoadManagerTest") { + withNewConfig("videobridge.load-management.reducer-enabled=true") { + val reducer = mockk(relaxed = true) { + every { impactTime() } returns 10.secs + } + val clock = FakeClock() + + val loadManager = JvbLoadManager( + MockLoadMeasurement(10.0), + MockLoadMeasurement(7.0), + reducer, + clock + ) + + context("a load update") { + context("with a load which isn't overloaded") { + loadManager.loadUpdate(MockLoadMeasurement(9.0)) + should("not trigger any action") { + verify { reducer wasNot Called } + } + } + + context("with a load which is overloaded") { + loadManager.loadUpdate(MockLoadMeasurement(10.0)) + should("trigger a call to reduce load") { + verify(exactly = 1) { reducer.reduceLoad() } + } + context("followed by another load update") { + context("with a load which is overloaded") { + val newLoad = MockLoadMeasurement(10.0) + context("before the impact time has elapsed") { + loadManager.loadUpdate(newLoad) + should("not trigger another call to reduce load") { + verify(exactly = 1) { reducer.reduceLoad() } + } + } + context("after the impact time has elapsed") { + clock.elapse(1.mins) + loadManager.loadUpdate(newLoad) + should("trigger another call to reduce load") { + verify(exactly = 2) { reducer.reduceLoad() } + } + } + } + context("with a load which isn't overloaded, but isn't under the recovery threshold") { + clock.elapse(1.mins) + loadManager.loadUpdate(MockLoadMeasurement(9.0)) + should("not trigger any new calls") { + verify(exactly = 1) { reducer.reduceLoad() } + verify(exactly = 0) { reducer.recover() } + } + } + context("with a load below the recovery threshold") { + val newLoad = MockLoadMeasurement(2.0) + context("before the impact time has elapsed") { + loadManager.loadUpdate(newLoad) + should("not trigger any new calls") { + verify(exactly = 1) { reducer.reduceLoad() } + verify(exactly = 0) { reducer.recover() } + } + } + context("after the impact time has elapsed") { + clock.elapse(1.mins) + loadManager.loadUpdate(newLoad) + should("trigger a call to recover") { + verify(exactly = 1) { reducer.reduceLoad() } + verify(exactly = 1) { reducer.recover() } + } + } + } + } + } + } + context("the stress level") { + should("be 0 if no measurement has been received") { + loadManager.getCurrentStressLevel() shouldBe 0.0 + } + should("update with every load measurement") { + loadManager.loadUpdate(MockLoadMeasurement(1.0)) + loadManager.getCurrentStressLevel() shouldBe .1 + loadManager.loadUpdate(MockLoadMeasurement(3.0)) + loadManager.getCurrentStressLevel() shouldBe .3 + loadManager.loadUpdate(MockLoadMeasurement(11.0)) + loadManager.getCurrentStressLevel() shouldBe 1.1 + } + } + } + } + } +} + +class MockLoadMeasurement(var loadMeasurement: Double) : JvbLoadMeasurement { + override fun getLoad(): Double = loadMeasurement + + override fun div(other: JvbLoadMeasurement): Double { + other as MockLoadMeasurement + return loadMeasurement / other.loadMeasurement + } + + override fun toString(): String = "Mock load measurement of $loadMeasurement" +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt new file mode 100644 index 0000000..1ff0a25 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/LastNReducerTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import io.mockk.spyk +import io.mockk.verify +import org.jitsi.config.setNewConfig +import org.jitsi.config.withNewConfig +import org.jitsi.videobridge.AbstractEndpoint +import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.JvbLastN +import java.util.function.Supplier + +class LastNReducerTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.InstancePerLeaf + + private val jvbLastN = spyk() + + init { + context("when a bridge has no conferences") { + val reducer = LastNReducer(Supplier { emptyList() }, jvbLastN) + context("running the reducer") { + reducer.reduceLoad() + should("not set a last-n value") { + verify(exactly = 0) { jvbLastN setProperty "jvbLastN" value any() } + } + } + } + context("when a bridge has conferences") { + val conf1 = createMockConference(4, 8, 12) + val conf2 = createMockConference(2, 4, 10) + + withNewConfig( + """ + videobridge.load-management.load-reducers.last-n.reduction-scale = .5 + videobridge.load-management.load-reducers.last-n.recover-scale = 2 + """.trimIndent() + ) { + + val reducer = LastNReducer(Supplier { listOf(conf1, conf2) }, jvbLastN) + context("running the reducer") { + reducer.reduceLoad() + should("set the right last-n value") { + // The highest forwarded count was 12, and the reduction factor was .5, so + // it should be set to 6 + verify(exactly = 1) { jvbLastN setProperty "jvbLastN" value 6 } + } + } + context("and no jvb last-n has been set") { + context("running recovery") { + reducer.recover() shouldBe false + should("not alter the last-n value") { + verify(exactly = 0) { jvbLastN setProperty "jvbLastN" value any() } + } + } + } + context("and a jvb last-n has been set") { + jvbLastN.jvbLastN = 4 + context("running recovery") { + reducer.recover() shouldBe true + should("increase the jvb last-n value") { + verify(exactly = 1) { jvbLastN setProperty "jvbLastN" value 8 } + } + } + } + } + } + } +} + +/** + * We want to create and re-use a LastNReducer, but that doesn't play nicely with nested 'context' blocks. This + * function run a block (and returns the result) with the given config (suffixes only) applied. + */ +inline fun withLastNConfig(config: String, block: () -> T): T { + val prefixedConfig = config.trimIndent() + .split("\n") + .filter { it.isNotEmpty() } + .joinToString("\n") { "${JvbLoadReducer.CONFIG_BASE}.last-n.$it" } + setNewConfig(prefixedConfig, true, "LastNReducerTest") + return block().also { + setNewConfig("", true) + } +} + +/** + * Create a mock [Conference] which has endpoints with the given number of forwarded video streams + */ +private fun createMockConference(vararg epNumForwardedVideo: Int): Conference { + val eps = epNumForwardedVideo.map { + mockk { every { numForwardedSources() } returns it } + }.toList() + return mockk { + every { endpoints } returns eps + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurementTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurementTest.kt new file mode 100644 index 0000000..5995265 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/PacketRateMeasurementTest.kt @@ -0,0 +1,38 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.load_management + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class PacketRateMeasurementTest : ShouldSpec({ + context("division") { + context("between two PacketRateMeasurements") { + should("yield the correct result") { + PacketRateMeasurement(10) / PacketRateMeasurement(2) shouldBe 5.0 + } + } + context("between a PacketRateMeasurement and another measurement type") { + should("throw an exception") { + shouldThrow { + PacketRateMeasurement(10) / MockLoadMeasurement(1.1) + } + } + } + } +}) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/StealDetectionTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/StealDetectionTest.kt new file mode 100644 index 0000000..00de2a7 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/load_management/StealDetectionTest.kt @@ -0,0 +1,56 @@ +package org.jitsi.videobridge.load_management + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.engine.spec.tempfile +import io.kotest.matchers.doubles.plusOrMinus +import io.kotest.matchers.shouldBe + +class StealDetectionTest : ShouldSpec() { + override fun isolationMode() = IsolationMode.SingleInstance + + val file = tempfile().apply { + writeText( + """ + cpu0 1370589 7125 506757 214097961 48978 0 43412 63777 0 0 + cpu1 1366281 6967 490213 214122004 49125 0 31097 72697 0 0 + cpu2 1368810 6801 490900 214122156 49174 0 26168 73365 0 0 + cpu3 1381486 6142 517097 214032354 51546 0 21246 71640 0 0 + cpu 100 200 300 400 500 600 700 800 900 + """.trimIndent() + ) + } + + private val stealDetection = StealDetection(file) + + init { + context("first read") { + stealDetection.update().getLoad() shouldBe 0.0 + } + context("calculate steal percentage correctly") { + file.writeText("cpu 200 300 400 500 600 700 800 900 1000") + val result = stealDetection.update() + + result.getLoad() shouldBe (100 / 900.0).plusOrMinus(0.000001) + } + context("invalid or empty file format") { + file.writeText("cpu abcd efgh ijkl mnop qrst uvwx yz") + stealDetection.update().getLoad() shouldBe 0.0 + + file.writeText("abcd efgh ijkl mnop qrst uvwx yz") + stealDetection.update().getLoad() shouldBe 0.0 + + file.writeText("") + stealDetection.update().getLoad() shouldBe 0.0 + } + context("missing steal field") { + file.writeText("cpu 300 400 500 600 700 800 900") + stealDetection.update().getLoad() shouldBe 0.0 + } + context("zero total delta") { + file.writeText("cpu 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000") + stealDetection.update() + stealDetection.update().getLoad() shouldBe 0.0 + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt new file mode 100644 index 0000000..41d3eaf --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/message/BridgeChannelMessageTest.kt @@ -0,0 +1,740 @@ +/* + * Copyright @ 2020-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.message + +import com.fasterxml.jackson.core.JsonParseException +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.databind.node.TextNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.nulls.shouldNotBeNull +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.string.shouldNotInclude +import io.kotest.matchers.types.shouldBeInstanceOf +import org.jitsi.nlj.VideoType +import org.jitsi.videobridge.cc.allocation.VideoConstraints +import org.jitsi.videobridge.message.BridgeChannelMessage.Companion.parse + +@Suppress("BlockingMethodInNonBlockingContext") +class BridgeChannelMessageTest : ShouldSpec() { + init { + context("serializing") { + should("encode the type as colibriClass") { + // Any message will do, this one is just simple + val message = ClientHelloMessage() + + val parsed = jacksonObjectMapper().readTree(message.toJson()) + parsed.shouldBeInstanceOf() + val parsedColibriClass = parsed["colibriClass"] + parsedColibriClass.shouldBeInstanceOf() + parsedColibriClass.asText() shouldBe ClientHelloMessage.TYPE + } + } + context("parsing an invalid message") { + shouldThrow { + parse("{invalid json") + } + + shouldThrow { + parse("") + } + + shouldThrow { + parse("{}") + } + + shouldThrow { + parse("""{"colibriClass": "invalid-colibri-class" }""") + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "colibriClass": "duplicate" + } + """.trimIndent() + ) + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "to": "a", + "to": "b" + } + """.trimIndent() + ) + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "from": "a", + "from": "b" + } + """.trimIndent() + ) + } + shouldThrow { + parse( + """ + { + "colibriClass": "EndpointStats", + "non-defined-prop": "a", + "non-defined-prop": "b" + } + """.trimIndent() + ) + } + + context("when some of the message-specific fields are missing/invalid") { + shouldThrow { + // Missing dominantSpeakerEndpoint field + parse("""{"colibriClass": "DominantSpeakerEndpointChangeEvent" }""") + } + shouldThrow { + // dominantSpeakerEndpoint has the wrong type + parse("""{"colibriClass": "DominantSpeakerEndpointChangeEvent", "dominantSpeakerEndpoint": [5] }""") + } + } + } + context("serializing and parsing EndpointMessage") { + val endpointsMessage = EndpointMessage("to_value") + endpointsMessage.otherFields["other_field1"] = "other_value1" + endpointsMessage.put("other_field2", 97) + endpointsMessage.put("other_null", null) + + val json = endpointsMessage.toJson() + // Make sure we don't mistakenly serialize the "broadcast" flag. + json shouldNotInclude "broadcast" + // Make sure we don't mistakenly serialize the "type". + json shouldNotInclude """ + "type": + """.trimIndent() + val parsed = parse(json) + + parsed.shouldBeInstanceOf() + parsed.from shouldBe null + parsed.to shouldBe "to_value" + parsed.otherFields["other_field1"] shouldBe "other_value1" + parsed.otherFields["other_field2"] shouldBe 97 + parsed.otherFields["other_null"] shouldBe null + parsed.otherFields.containsKey("other_null") shouldBe true + parsed.otherFields.containsKey("nonexistent") shouldBe false + + endpointsMessage.from = "new" + (parse(endpointsMessage.toJson()) as EndpointMessage).from shouldBe "new" + + context("parsing") { + val parsed2 = parse(ENDPOINT_MESSAGE) + parsed2 as EndpointMessage + parsed2.from shouldBe null + parsed2.to shouldBe "to_value" + parsed2.otherFields["other_field1"] shouldBe "other_value1" + parsed2.otherFields["other_field2"] shouldBe 97 + parsed2.otherFields.containsKey("other_null") shouldBe true + parsed2.otherFields.containsKey("nonexistent") shouldBe false + } + + // EndpointMessage deserializes via a constructor creator ("to"). jackson-databind 2.18.x has a + // regression that drops @JsonAnySetter fields for creator-based types, which stripped msgPayload + // from forwarded EndpointMessages. This guards the round-trip (parse -> set from -> serialize). + should("preserve any-fields when forwarding (msgPayload is not stripped)") { + val payload = + """{"colibriClass":"EndpointMessage","msgPayload":{"type":"face-box","faceBox":{"left":33,"right":64,"width":31}},"to":""}""" + val parsed3 = parse(payload) as EndpointMessage + parsed3.to shouldBe "" + withClue("otherFields=${parsed3.otherFields}") { + parsed3.otherFields.containsKey("msgPayload") shouldBe true + } + parsed3.from = "3b213301" + withClue("json=${parsed3.toJson()}") { + parsed3.toJson() shouldContain "msgPayload" + } + } + } + + context("serializing and parsing DominantSpeakerMessage") { + val previousSpeakers = listOf("p1", "p2") + val original = DominantSpeakerMessage("d", previousSpeakers) + + val parsed = parse(original.toJson()) + + parsed.shouldBeInstanceOf() + parsed.dominantSpeakerEndpoint shouldBe "d" + parsed.previousSpeakers shouldBe listOf("p1", "p2") + } + + context("serializing and parsing SyntheticSourceSendingChangeEvent") { + val message = SyntheticSourceSendingChangeEvent("55555555-a0.hi", true, 384000) + + context("the wire format (matches lib-jitsi-meet BridgeChannel validation)") { + val parsed = jacksonObjectMapper().readTree(message.toJson()) + parsed.shouldBeInstanceOf() + parsed.get("colibriClass").asText() shouldBe "SyntheticSourceSendingChangeEvent" + // Flat fields (not nested under "changes"): the client reads obj.sourceName/sending/timestamp. + parsed.get("sourceName").asText() shouldBe "55555555-a0.hi" + // A real JSON boolean (the client checks typeof === 'boolean'). + parsed.get("sending").isBoolean shouldBe true + parsed.get("sending").asBoolean() shouldBe true + // A natural JSON number in the uint32 RTP range (the client checks Number.isInteger and <= 0xFFFFFFFF). + parsed.get("timestamp").isNumber shouldBe true + parsed.get("timestamp").asLong() shouldBe 384000 + } + context("round-trip") { + val parsed = parse(message.toJson()) + parsed.shouldBeInstanceOf() + parsed.sourceName shouldBe "55555555-a0.hi" + parsed.sending shouldBe true + parsed.timestamp shouldBe 384000 + } + } + + context("serializing and parsing ServerHello") { + context("without a version") { + val parsed = parse(ServerHelloMessage().toJson()) + parsed.shouldBeInstanceOf() + parsed.version shouldBe null + } + context("with a version") { + val message = ServerHelloMessage("v") + + val parsed = parse(message.toJson()) + parsed.shouldBeInstanceOf() + parsed.version shouldBe "v" + } + } + + context("serializing and parsing ClientHello") { + + val parsed = parse(ClientHelloMessage().toJson()) + parsed.shouldBeInstanceOf() + } + + context("serializing and parsing EndpointConnectionStatusMessage") { + + val parsed = parse(EndpointConnectionStatusMessage("abcdabcd", true).toJson()) + parsed.shouldBeInstanceOf() + + parsed.endpoint shouldBe "abcdabcd" + parsed.active shouldBe "true" + } + + context("serializing and parsing ForwardedSourcesMessage") { + val forwardedSources = setOf("s1", "s2", "s3") + + val message = ForwardedSourcesMessage(forwardedSources) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + + parsed.forwardedSources shouldContainExactly forwardedSources + } + + context("serializing and parsing VideoConstraints") { + val videoConstraints: VideoConstraints = jacksonObjectMapper().readValue(VIDEO_CONSTRAINTS) + videoConstraints.maxHeight shouldBe 1080 + videoConstraints.maxFrameRate shouldBe 15.0 + } + + context("serializing and parsing SenderSourceConstraintsMessage") { + val senderSourceConstraintsMessage = SenderSourceConstraintsMessage("s1", 1080) + val parsed = parse(senderSourceConstraintsMessage.toJson()) + + parsed.shouldBeInstanceOf() + + parsed.sourceName shouldBe "s1" + parsed.maxHeight shouldBe 1080 + } + + context("serializing and parsing AddReceiver") { + context("with source names") { + val message = AddReceiverMessage("bridge1", "s1", VideoConstraints(360)) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.bridgeId shouldBe "bridge1" + parsed.sourceName shouldBe "s1" + parsed.videoConstraints shouldBe VideoConstraints(360) + } + context("without source names") { + val message = AddReceiverMessage("bridge1", "source1", VideoConstraints(360)) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.bridgeId shouldBe "bridge1" + parsed.sourceName shouldBe "source1" + parsed.videoConstraints shouldBe VideoConstraints(360) + } + } + + context("serializing and parsing RemoveReceiver") { + val message = RemoveReceiverMessage("bridge1", "abcdabcd") + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.bridgeId shouldBe "bridge1" + parsed.endpointId shouldBe "abcdabcd" + } + + context("serializing and parsing VideoType") { + val videoTypeMessage = VideoTypeMessage(VideoType.DESKTOP) + videoTypeMessage.videoType shouldBe VideoType.DESKTOP + parse(videoTypeMessage.toJson()).apply { + shouldBeInstanceOf() + videoType shouldBe VideoType.DESKTOP + } + + listOf("none", "NONE", "None", "nOnE").forEach { + val jsonString = """ + { + "colibriClass" : "VideoTypeMessage", + "videoType" : "$it" + } + """ + parse(jsonString).apply { + shouldBeInstanceOf() + videoType shouldBe VideoType.NONE + } + } + val jsonString = """ + { + "colibriClass" : "VideoTypeMessage", + "videoType" : "desktop_high_fps" + } + """ + parse(jsonString).apply { + shouldBeInstanceOf() + videoType shouldBe VideoType.DESKTOP_HIGH_FPS + } + } + + context("serializing and parsing SourceVideoType") { + val testSourceName = "source1234" + val srcVideoTypeMessage = SourceVideoTypeMessage(VideoType.DESKTOP, testSourceName) + srcVideoTypeMessage.videoType shouldBe VideoType.DESKTOP + parse(srcVideoTypeMessage.toJson()).apply { + shouldBeInstanceOf() + videoType shouldBe VideoType.DESKTOP + sourceName shouldBe testSourceName + } + + listOf("none", "NONE", "None", "nOnE").forEach { + val jsonString = """ + { + "colibriClass" : "SourceVideoTypeMessage", + "sourceName": "$testSourceName", + "videoType" : "$it" + } + """ + parse(jsonString).apply { + shouldBeInstanceOf() + videoType shouldBe VideoType.NONE + sourceName shouldBe testSourceName + } + } + val jsonString = """ + { + "colibriClass" : "SourceVideoTypeMessage", + "sourceName" : "source1234", + "videoType" : "desktop_high_fps" + } + """ + parse(jsonString).apply { + shouldBeInstanceOf() + videoType shouldBe VideoType.DESKTOP_HIGH_FPS + sourceName shouldBe testSourceName + } + } + + context("serializing and parsing VideoSourceMap") { + val source1 = "source1234" + val owner1 = "endpoint1" + val ssrc1 = 12345L + val rtxSsrc1 = 45678L + + val source2 = "source5678" + val owner2 = "endpoint2" + val ssrc2 = 87654L + val rtxSsrc2 = 98765L + + val videoSourcesMapMessage = VideoSourcesMap( + listOf( + VideoSourceMapping(source1, owner1, ssrc1, rtxSsrc1, VideoType.CAMERA), + VideoSourceMapping(source2, owner2, ssrc2, rtxSsrc2, VideoType.DESKTOP) + ) + ) + + parse(videoSourcesMapMessage.toJson()).apply { + shouldBeInstanceOf() + mappedSources.size shouldBe 2 + mappedSources shouldContainExactly + listOf( + VideoSourceMapping(source1, owner1, ssrc1, rtxSsrc1, VideoType.CAMERA), + VideoSourceMapping(source2, owner2, ssrc2, rtxSsrc2, VideoType.DESKTOP) + ) + } + + val jsonString = """ + {"colibriClass":"VideoSourcesMap", + "mappedSources":[{"source":"source1234","owner":"endpoint1","ssrc":12345,"rtx":45678,"videoType":"CAMERA"}, + {"source":"source5678","owner":"endpoint2","ssrc":87654,"rtx":98765,"videoType":"DESKTOP"} + ] + } + """.trimIndent() + + parse(jsonString).apply { + shouldBeInstanceOf() + mappedSources.size shouldBe 2 + mappedSources shouldContainExactly + listOf( + VideoSourceMapping(source1, owner1, ssrc1, rtxSsrc1, VideoType.CAMERA), + VideoSourceMapping(source2, owner2, ssrc2, rtxSsrc2, VideoType.DESKTOP) + ) + } + } + + context("serializing and parsing AudioSourceMap") { + val source1 = "source1234-a" + val owner1 = "endpoint1" + val ssrc1 = 23456L + + val source2 = "source5678-a" + val owner2 = "endpoint2" + val ssrc2 = 98765L + + val audioSourcesMapMessage = AudioSourcesMap( + listOf( + AudioSourceMapping(source1, owner1, ssrc1), + AudioSourceMapping(source2, owner2, ssrc2) + ) + ) + + parse(audioSourcesMapMessage.toJson()).apply { + shouldBeInstanceOf() + mappedSources.size shouldBe 2 + mappedSources shouldContainExactly + listOf( + AudioSourceMapping(source1, owner1, ssrc1), + AudioSourceMapping(source2, owner2, ssrc2) + ) + } + + val jsonString = """ + {"colibriClass":"AudioSourcesMap", + "mappedSources":[{"source":"source1234-a","owner":"endpoint1","ssrc":23456}, + {"source":"source5678-a","owner":"endpoint2","ssrc":98765} + ] + } + """.trimIndent() + + parse(jsonString).apply { + shouldBeInstanceOf() + mappedSources.size shouldBe 2 + mappedSources shouldContainExactly + listOf( + AudioSourceMapping(source1, owner1, ssrc1), + AudioSourceMapping(source2, owner2, ssrc2) + ) + } + } + + context("Parsing ReceiverVideoConstraints") { + context("With all fields present") { + val parsed = parse(RECEIVER_VIDEO_CONSTRAINTS) + + parsed.shouldBeInstanceOf() + parsed.lastN shouldBe 3 + parsed.defaultConstraints shouldBe VideoConstraints(0) + val constraints = parsed.constraints + constraints.shouldNotBeNull() + constraints.size shouldBe 3 + constraints["epOnStage"] shouldBe VideoConstraints(720) + constraints["epThumbnail1"] shouldBe VideoConstraints(180) + constraints["epThumbnail2"] shouldBe VideoConstraints(180, 30.0) + } + + context("With fields missing") { + val parsed = parse(RECEIVER_VIDEO_CONSTRAINTS_EMPTY) + parsed.shouldBeInstanceOf() + parsed.lastN shouldBe null + parsed.defaultConstraints shouldBe null + parsed.constraints shouldBe null + } + } + + context("serializing and parsing ReceiverAudioSubscriptionMessage") { + context("all subscription") { + val message = ReceiverAudioSubscriptionMessage(all = true) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe true + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + + context("none subscription") { + val message = ReceiverAudioSubscriptionMessage(all = false) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe false + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + + context("subscription with include list") { + val includeList = listOf("endpoint1", "endpoint2", "endpoint3") + val message = ReceiverAudioSubscriptionMessage(all = false, include = includeList) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe false + parsed.include shouldContainExactly includeList + parsed.exclude shouldBe emptyList() + } + + context("subscription with exclude list") { + val excludeList = listOf("endpoint4", "endpoint5") + val message = ReceiverAudioSubscriptionMessage(all = true, exclude = excludeList) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe true + parsed.exclude shouldContainExactly excludeList + parsed.include shouldBe emptyList() + } + + context("subscription with both all and include (synthetic sources)") { + val includeList = listOf("syntheticSource") + val message = ReceiverAudioSubscriptionMessage(all = true, include = includeList) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe true + parsed.include shouldContainExactly includeList + parsed.exclude shouldBe emptyList() + } + + context("subscription with empty lists") { + val message = ReceiverAudioSubscriptionMessage(all = false) + val parsed = parse(message.toJson()) + + parsed.shouldBeInstanceOf() + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + + context("parsing from JSON with include and exclude") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription", + "all": true, + "include": ["endpoint1", "endpoint2"], + "exclude": ["endpoint3"] + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe true + parsed.include shouldContainExactly listOf("endpoint1", "endpoint2") + parsed.exclude shouldContainExactly listOf("endpoint3") + } + + context("parsing from JSON with missing fields uses defaults") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription" + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe false + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + + context("parsing legacy mode=All syntax") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription", + "mode": "All" + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe true + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + + context("parsing legacy mode=None syntax") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription", + "mode": "None" + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe false + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + + context("parsing legacy mode=Include syntax") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription", + "mode": "Include", + "list": ["source1", "source2"] + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe false + parsed.include shouldContainExactly listOf("source1", "source2") + parsed.exclude shouldBe emptyList() + } + + context("parsing legacy mode=Exclude syntax") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription", + "mode": "Exclude", + "list": ["source3"] + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe true + parsed.include shouldBe emptyList() + parsed.exclude shouldContainExactly listOf("source3") + } + + context("parsing legacy mode=Include syntax with no list") { + val jsonString = """ + { + "colibriClass": "ReceiverAudioSubscription", + "mode": "Include" + } + """ + val parsed = parse(jsonString) + + parsed.shouldBeInstanceOf() + parsed.all shouldBe false + parsed.include shouldBe emptyList() + parsed.exclude shouldBe emptyList() + } + } + + xcontext("Serializing performance") { + val times = 1_000_000 + + val objectMapper = ObjectMapper() + fun toJsonJackson(m: DominantSpeakerMessage): String = objectMapper.writeValueAsString(m) + fun toJsonJsonSimple(m: DominantSpeakerMessage) = jacksonObjectMapper().createObjectNode().apply { + put("dominantSpeakerEndpoint", m.dominantSpeakerEndpoint) + }.toString() + + fun toJsonStringConcat(m: DominantSpeakerMessage) = + "{\"colibriClass\":\"DominantSpeakerEndpointChangeEvent\",\"dominantSpeakerEndpoint\":\"" + + m.dominantSpeakerEndpoint + "\"}" + + fun toJsonStringTemplate(m: DominantSpeakerMessage) = + "{\"colibriClass\":\"${DominantSpeakerMessage.TYPE}\"," + + "\"dominantSpeakerEndpoint\":\"${m.dominantSpeakerEndpoint}\"}" + + fun toJsonRawStringTemplate(m: DominantSpeakerMessage) = """ + {"colibriClass":"${DominantSpeakerMessage.TYPE}", + "dominantSpeakerEndpoint":"${m.dominantSpeakerEndpoint}"} + """ + + fun runTest(f: (DominantSpeakerMessage) -> String): Long { + val start = System.currentTimeMillis() + for (i in 0..times) { + f(DominantSpeakerMessage(i.toString())) + } + val end = System.currentTimeMillis() + + return end - start + } + + System.err.println("Times=$times") + System.err.println("Jackson: ${runTest { toJsonJackson(it) }}") + System.err.println("Json-simple: ${runTest { toJsonJsonSimple(it) }}") + System.err.println("String concat: ${runTest { toJsonStringConcat(it) }}") + System.err.println("String template: ${runTest { toJsonStringTemplate(it) }}") + System.err.println("Raw string template: ${runTest { toJsonRawStringTemplate(it) }}") + System.err.println("Raw string template (trim): ${runTest { toJsonRawStringTemplate(it).trimMargin() }}") + } + } + + companion object { + const val ENDPOINT_MESSAGE = """ + { + "colibriClass": "EndpointMessage", + "to": "to_value", + "other_field1": "other_value1", + "other_field2": 97, + "other_null": null + } + """ + + const val VIDEO_CONSTRAINTS = """ + { + "maxHeight": 1080, + "maxFrameRate": 15.0 + } + """ + + const val RECEIVER_VIDEO_CONSTRAINTS_EMPTY = """ + { + "colibriClass": "ReceiverVideoConstraints" + } + """ + const val RECEIVER_VIDEO_CONSTRAINTS = """ + { + "colibriClass": "ReceiverVideoConstraints", + "lastN": 3, + "defaultConstraints": { "maxHeight": 0 }, + "constraints": { + "epOnStage": { "maxHeight": 720 }, + "epThumbnail1": { "maxHeight": 180 }, + "epThumbnail2": { "maxHeight": 180, "maxFrameRate": 30 } + } + } + """ + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/relay/RelayConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/relay/RelayConfigTest.kt new file mode 100644 index 0000000..f66599c --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/relay/RelayConfigTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright @ 2020 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.relay + +import io.kotest.matchers.shouldBe +import org.jitsi.ConfigTest +import org.jitsi.config.withNewConfig + +internal class RelayConfigTest : ConfigTest() { + init { + context("enabled") { + should("Be false by default") { + RelayConfig.config.enabled shouldBe false + } + context("When only the old name is present") { + withNewConfig("videobridge.octo.enabled=true") { + RelayConfig.config.enabled shouldBe true + } + } + context("When only the new name is present") { + withNewConfig("videobridge.relay.enabled=true") { + RelayConfig.config.enabled shouldBe true + } + } + context("When both names are present") { + withNewConfig("videobridge.octo.enabled=true\nvideobridge.relay.enabled=false") { + RelayConfig.config.enabled shouldBe true + } + withNewConfig("videobridge.octo.enabled=false\nvideobridge.relay.enabled=true") { + RelayConfig.config.enabled shouldBe false + } + } + } + context("relay-id") { + withNewConfig("videobridge.octo.relay-id=abc") { + RelayConfig.config.relayId shouldBe "abc" + } + withNewConfig("videobridge.relay.relay-id=abc") { + RelayConfig.config.relayId shouldBe "abc" + } + withNewConfig("videobridge.octo.relay-id=xyz\nvidebridge.relay.relay-id=abc") { + RelayConfig.config.relayId shouldBe "xyz" + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/rest/RestConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/RestConfigTest.kt new file mode 100644 index 0000000..2552bb1 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/RestConfigTest.kt @@ -0,0 +1,126 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest + +import io.kotest.matchers.shouldBe +import org.jitsi.ConfigTest +import org.jitsi.config.withLegacyConfig +import org.jitsi.config.withNewConfig +import org.jitsi.videobridge.rest.RestConfig.Companion.config + +class RestConfigTest : ConfigTest() { + init { + context("Default configuration") { + context("At least one REST API") { + config.isEnabled() shouldBe true + } + context("Colibri") { + config.isEnabled(RestApis.COLIBRI) shouldBe false + } + context("Debug") { + config.isEnabled(RestApis.DEBUG) shouldBe true + } + context("Health") { + config.isEnabled(RestApis.HEALTH) shouldBe true + } + context("Shutdown") { + config.isEnabled(RestApis.SHUTDOWN) shouldBe false + } + context("Drain") { + config.isEnabled(RestApis.DRAIN) shouldBe true + } + context("Version") { + config.isEnabled(RestApis.VERSION) shouldBe true + } + context("Prometheus") { + config.isEnabled(RestApis.PROMETHEUS) shouldBe true + } + } + context("Enable/disable with new config") { + context("Colibri") { + withNewConfig("videobridge.apis.rest.enabled=true") { + config.isEnabled(RestApis.COLIBRI) shouldBe true + } + } + context("Debug") { + withNewConfig("videobridge.rest.debug.enabled=false") { + config.isEnabled(RestApis.DEBUG) shouldBe false + } + } + context("Health") { + withNewConfig("videobridge.rest.health.enabled=false") { + config.isEnabled(RestApis.HEALTH) shouldBe false + } + } + context("Shutdown with colibri") { + withNewConfig( + """ + videobridge.apis.rest.enabled=true + videobridge.rest.shutdown.enabled=true + """.trimMargin() + ) { + config.isEnabled(RestApis.SHUTDOWN) shouldBe true + } + } + context("Shutdown without colibri") { + withNewConfig( + """ + videobridge.apis.rest.enabled=false + videobridge.rest.shutdown.enabled=true + """.trimMargin() + ) { + config.isEnabled(RestApis.SHUTDOWN) shouldBe false + } + } + context("Version") { + withNewConfig("videobridge.rest.version.enabled=false") { + config.isEnabled(RestApis.VERSION) shouldBe false + } + } + context("Prometheus") { + withNewConfig("videobridge.rest.prometheus.enabled=false") { + config.isEnabled(RestApis.PROMETHEUS) shouldBe false + } + } + } + context("Enable/disable with the old way") { + // The old way to enable REST is via command line argument, which is now simulated with a new config + context("When colibri is enabled") { + withNewConfig("videobridge.apis.rest.enabled=true") { + config.isEnabled(RestApis.COLIBRI) shouldBe true + + context("When shutdown is enabled") { + withLegacyConfig("org.jitsi.videobridge.ENABLE_REST_SHUTDOWN=true") { + config.isEnabled(RestApis.SHUTDOWN) shouldBe true + } + } + + withLegacyConfig("org.jitsi.videobridge.ENABLE_REST_COLIBRI=false") { + config.isEnabled(RestApis.COLIBRI) shouldBe false + } + } + } + + context("When colibri is disabled") { + config.isEnabled(RestApis.SHUTDOWN) shouldBe false + withLegacyConfig("org.jitsi.videobridge.ENABLE_REST_SHUTDOWN=true") { + config.isEnabled(RestApis.SHUTDOWN) shouldBe false + } + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClientTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClientTest.kt new file mode 100644 index 0000000..3e8b20d --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/mucclient/MucClientTest.kt @@ -0,0 +1,128 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.mucclient + +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import io.mockk.slot +import jakarta.ws.rs.client.Entity +import jakarta.ws.rs.core.Application +import org.eclipse.jetty.http.HttpStatus +import org.glassfish.jersey.server.ResourceConfig +import org.glassfish.jersey.test.JerseyTest +import org.glassfish.jersey.test.TestProperties +import org.jitsi.videobridge.rest.MockBinder +import org.jitsi.videobridge.xmpp.XmppConnection +import org.junit.Test + +class MucClientTest : JerseyTest() { + private lateinit var xmppConnection: XmppConnection + private val baseUrl = "/colibri/muc-client" + + override fun configure(): Application { + xmppConnection = mockk() + + enable(TestProperties.LOG_TRAFFIC) + enable(TestProperties.DUMP_ENTITY) + return object : ResourceConfig() { + init { + register(MockBinder(xmppConnection, XmppConnection::class.java)) + register(MucClient::class.java) + } + } + } + + @Test + fun testAddMuc() { + val jsonConfigSlot = slot() + + every { xmppConnection.addMucClient(capture(jsonConfigSlot)) } returns true + + val json = jacksonObjectMapper().createObjectNode().apply { + put("id", "id") + put("hostname", "hostname") + put("username", "username") + put("password", "password") + put("muc_jids", "jid1, jid2") + put("muc_nickname", "muc_nickname") + } + + val resp = target("$baseUrl/add").request().post(Entity.json(json.toString())) + resp.status shouldBe HttpStatus.OK_200 + // Compare JSON content (ObjectNode captured from request vs ObjectNode sent) + jsonConfigSlot.captured["id"]?.asText() shouldBe "id" + } + + @Test + fun testAddMucFailure() { + val jsonConfigSlot = slot() + + every { xmppConnection.addMucClient(capture(jsonConfigSlot)) } returns false + + val json = jacksonObjectMapper().createObjectNode().apply { + put("id", "id") + put("hostname", "hostname") + put("username", "username") + put("password", "password") + put("muc_jids", "jid1, jid2") + put("muc_nickname", "muc_nickname") + } + + val resp = target("$baseUrl/add").request().post(Entity.json(json.toString())) + resp.status shouldBe HttpStatus.BAD_REQUEST_400 + } + + @Test + fun testRemoveMuc() { + val jsonConfigSlot = slot() + + every { xmppConnection.removeMucClient(capture(jsonConfigSlot)) } returns true + + val json = jacksonObjectMapper().createObjectNode().apply { + put("id", "id") + } + + val resp = target("$baseUrl/remove").request().post(Entity.json(json.toString())) + resp.status shouldBe HttpStatus.OK_200 + jsonConfigSlot.captured["id"]?.asText() shouldBe "id" + } + + @Test + fun testRemoveMucFailure() { + every { xmppConnection.removeMucClient(any()) } returns false + + val json = jacksonObjectMapper().createObjectNode().apply { + put("id", "id") + } + val resp = target("$baseUrl/remove").request().post(Entity.json(json.toString())) + resp.status shouldBe HttpStatus.BAD_REQUEST_400 + } + + @Test + fun testGetIds() { + val fakeIds = jacksonObjectMapper().createArrayNode().apply { + add("id1") + } + every { xmppConnection.getMucClientIds() } returns fakeIds.toString() + + val resp = target("$baseUrl/list").request().get() + resp.readEntity(String::class.java) shouldBe "[\"id1\"]" + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt new file mode 100644 index 0000000..38bda89 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/colibri/stats/StatsTest.kt @@ -0,0 +1,52 @@ +/* + * Copyright @ 2020 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.colibri.stats + +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.shouldBeInstanceOf +import jakarta.ws.rs.core.Application +import jakarta.ws.rs.core.MediaType +import org.eclipse.jetty.http.HttpStatus +import org.glassfish.jersey.server.ResourceConfig +import org.glassfish.jersey.test.JerseyTest +import org.glassfish.jersey.test.TestProperties +import org.junit.Test + +class StatsTest : JerseyTest() { + private val baseUrl = "/colibri/stats" + + override fun configure(): Application { + enable(TestProperties.LOG_TRAFFIC) + enable(TestProperties.DUMP_ENTITY) + return object : ResourceConfig() { + init { + register(Stats::class.java) + } + } + } + + @Test + fun testGetStats() { + val resp = target(baseUrl).request().get() + resp.status shouldBe HttpStatus.OK_200 + resp.mediaType shouldBe MediaType.APPLICATION_JSON_TYPE + val parsed = jacksonObjectMapper().readTree(resp.readEntity(String::class.java)) + parsed.shouldBeInstanceOf() + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/debug/DebugTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/debug/DebugTest.kt new file mode 100644 index 0000000..acf612a --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/rest/root/debug/DebugTest.kt @@ -0,0 +1,255 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.rest.root.debug + +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkClass +import jakarta.ws.rs.Path +import jakarta.ws.rs.client.Entity +import jakarta.ws.rs.core.Application +import org.eclipse.jetty.http.HttpStatus +import org.glassfish.hk2.utilities.binding.AbstractBinder +import org.glassfish.jersey.client.ClientConfig +import org.glassfish.jersey.logging.LoggingFeature +import org.glassfish.jersey.server.ResourceConfig +import org.glassfish.jersey.test.JerseyTest +import org.glassfish.jersey.test.TestProperties +import org.jitsi.health.HealthCheckService +import org.jitsi.videobridge.Conference +import org.jitsi.videobridge.Endpoint +import org.jitsi.videobridge.Videobridge +import org.jitsi.videobridge.relay.Relay +import org.jitsi.videobridge.rest.RestApis +import org.jitsi.videobridge.rest.annotations.EnabledByConfig +import org.junit.Test +import org.reflections.Reflections +import org.reflections.scanners.SubTypesScanner +import org.reflections.scanners.TypeAnnotationsScanner +import org.reflections.util.ClasspathHelper +import org.reflections.util.ConfigurationBuilder +import org.reflections.util.FilterBuilder +import java.util.logging.Level +import java.util.logging.Logger + +class DebugTest : JerseyTest() { + private val endpoint: Endpoint = mockk(relaxed = true) + private val relay: Relay = mockk(relaxed = true) + private val conference: Conference = mockk { + every { getEndpoint("bar") } returns endpoint + every { getLocalEndpoint("bar") } returns endpoint + every { getRelay("baz") } returns relay + } + private val videobridge: Videobridge = mockk { + every { getConference("foo") } returns conference + } + + override fun configure(): Application { + enable(TestProperties.LOG_TRAFFIC) + enable(TestProperties.DUMP_ENTITY) + + return object : ResourceConfig() { + init { + register(object : AbstractBinder() { + override fun configure() { + bind(mockkClass(HealthCheckService::class)).to(HealthCheckService::class.java) + bind(videobridge).to(Videobridge::class.java) + } + }) + register(Debug::class.java) + } + } + } + + override fun configureClient(config: ClientConfig?) { + config?.register( + LoggingFeature( + Logger.getAnonymousLogger(), + Level.ALL, + LoggingFeature.Verbosity.HEADERS_ONLY, + 1500 + ) + ) + } + + @Test + fun testAllResourcesAreBehindConfig() { + val reflections = Reflections( + ConfigurationBuilder() + .setUrls(ClasspathHelper.forPackage("org.jitsi.videobridge.rest.root.debug")) + .filterInputsBy(FilterBuilder().includePackage("org.jitsi.videobridge.rest.root.debug")) + .setScanners(SubTypesScanner(false), TypeAnnotationsScanner()) + ) + + reflections.getTypesAnnotatedWith(Path::class.java).forEach { clazz -> + clazz.isAnnotationPresent(EnabledByConfig::class.java) shouldBe true + val anno = clazz.getAnnotation(EnabledByConfig::class.java) + anno.value shouldBe RestApis.DEBUG + } + } + + @Test + fun testEnableJvbFeature() { + val resp = target("$BASE_URL/features/jvb/${DebugFeatures.PAYLOAD_VERIFICATION.value}/true") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.OK_200 + } + + @Test + fun testDisableJvbFeature() { + val resp = target("$BASE_URL/features/jvb/${DebugFeatures.PAYLOAD_VERIFICATION.value}/false") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.OK_200 + } + + @Test + fun testEnableNonexistentJvbFeature() { + val resp = target("$BASE_URL/features/jvb/blah/true") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.NOT_FOUND_404 + } + + @Test + fun testDisableNonexistentJvbFeature() { + val resp = target("$BASE_URL/features/jvb/blah/false") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.NOT_FOUND_404 + } + + @Test + fun testEnableEndpointFeature() { + val resp = target("$BASE_URL/features/endpoint/foo/bar/${EndpointDebugFeatures.PCAP_DUMP.value}/true") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.OK_200 + } + + @Test + fun testGetEndpointFeatureState() { + every { endpoint.isFeatureEnabled(EndpointDebugFeatures.PCAP_DUMP) } returns true + val resp = target("$BASE_URL/features/endpoint/foo/bar/${EndpointDebugFeatures.PCAP_DUMP.value}") + .request() + .get() + + resp.status shouldBe HttpStatus.OK_200 + resp.readEntity(String::class.java)!!.toBoolean() shouldBe true + + every { endpoint.isFeatureEnabled(EndpointDebugFeatures.PCAP_DUMP) } returns false + val resp2 = target("$BASE_URL/features/endpoint/foo/bar/${EndpointDebugFeatures.PCAP_DUMP.value}") + .request() + .get() + + resp2.status shouldBe HttpStatus.OK_200 + resp2.readEntity(String::class.java)!!.toBoolean() shouldBe false + } + + @Test + fun testGetNonexistentEndpointFeatureState() { + val resp = target("$BASE_URL/features/endpoint/foo/bar/nonexistent") + .request() + .get() + + resp.status shouldBe HttpStatus.NOT_FOUND_404 + } + + @Test + fun testDisableEndpointFeature() { + val resp = target("$BASE_URL/features/endpoint/foo/bar/${EndpointDebugFeatures.PCAP_DUMP.value}/false") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.OK_200 + } + + @Test + fun testDisableInvalidEndpointFeature() { + val resp = target("$BASE_URL/features/endpoint/foo/bar/nonexistent/false") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.NOT_FOUND_404 + } + + @Test + fun testEnableRelayFeature() { + val resp = target("$BASE_URL/features/relay/foo/baz/${EndpointDebugFeatures.PCAP_DUMP.value}/true") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.OK_200 + } + + @Test + fun testGetRelayFeatureState() { + every { relay.isFeatureEnabled(EndpointDebugFeatures.PCAP_DUMP) } returns true + val resp = target("$BASE_URL/features/relay/foo/baz/${EndpointDebugFeatures.PCAP_DUMP.value}") + .request() + .get() + + resp.status shouldBe HttpStatus.OK_200 + resp.readEntity(String::class.java)!!.toBoolean() shouldBe true + + every { relay.isFeatureEnabled(EndpointDebugFeatures.PCAP_DUMP) } returns false + val resp2 = target("$BASE_URL/features/relay/foo/baz/${EndpointDebugFeatures.PCAP_DUMP.value}") + .request() + .get() + + resp2.status shouldBe HttpStatus.OK_200 + resp2.readEntity(String::class.java)!!.toBoolean() shouldBe false + } + + @Test + fun testGetNonexistentRelayFeatureState() { + val resp = target("$BASE_URL/features/relay/foo/baz/nonexistent") + .request() + .get() + + resp.status shouldBe HttpStatus.NOT_FOUND_404 + } + + @Test + fun testDisableRelayFeature() { + val resp = target("$BASE_URL/features/relay/foo/baz/${EndpointDebugFeatures.PCAP_DUMP.value}/false") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.OK_200 + } + + @Test + fun testDisableInvalidRelayFeature() { + val resp = target("$BASE_URL/features/relay/foo/baz/nonexistent/false") + .request() + .post(Entity.json(null)) + + resp.status shouldBe HttpStatus.NOT_FOUND_404 + } + + companion object { + private const val BASE_URL = "/debug" + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/transport/ice/IceTransportRestartTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/transport/ice/IceTransportRestartTest.kt new file mode 100644 index 0000000..642a40a --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/transport/ice/IceTransportRestartTest.kt @@ -0,0 +1,406 @@ +/* + * Copyright @ 2026 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.videobridge.transport.ice + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.ice4j.ice.Agent +import org.ice4j.ice.Component +import org.ice4j.ice.IceMediaStream +import org.ice4j.ice.IceProcessingState +import org.ice4j.ice.KeepAliveStrategy +import org.jitsi.config.withNewConfig +import org.jitsi.utils.concurrent.FakeScheduledExecutorService +import org.jitsi.utils.logging2.Logger +import org.jitsi.utils.logging2.LoggerImpl +import org.jitsi.videobridge.util.TaskPools +import org.jitsi.xmpp.extensions.jingle.IceUdpTransportPacketExtension +import java.beans.PropertyChangeEvent +import java.beans.PropertyChangeListener + +/** + * Tests the ICE restart state machine of [IceTransport]: which Agent is described, which one gets the peer's + * credentials, and which ones are freed. The ice4j [Agent]s are mocked, because real ones bind ports and can + * only change state by actually running ICE. + */ +class IceTransportRestartTest : ShouldSpec({ + isolationMode = IsolationMode.InstancePerLeaf + + // The restart timeout and the transition window are scheduled on TaskPools.SCHEDULED_POOL. + val scheduler = FakeScheduledExecutorService() + TaskPools.SCHEDULED_POOL = scheduler + afterSpec { TaskPools.resetScheduledPool() } + + val agents = FakeAgents() + fun createTransport() = IceTransport( + id = "test", + controlling = true, + useUniquePort = false, + advertisePrivateAddresses = false, + parentLogger = LoggerImpl("test"), + agentFactory = agents.factory + ) + + context("Before ICE is established") { + val transport = createTransport() + should("apply an ordinary transport update to the initial Agent") { + // The main negotiation path, which the restart routing sits in front of. + transport.startConnectivityEstablishment(remoteTransport(generation = null)) + + agents.created[0].remoteUfrag shouldBe "remote-ufrag" + agents.created[0].remotePassword shouldBe "remote-pwd" + agents.created[0].startCalls shouldBe 1 + } + should("keep the existing Agent instead of restarting") { + transport.requestIceRestart() shouldBe IceRestartResult.KEEP_EXISTING + agents.created.size shouldBe 1 + } + should("describe the existing Agent with no ice-generation") { + transport.requestIceRestart() + with(transport.describe()) { + ufrag shouldBe agents.created[0].ufrag + iceGeneration shouldBe IceUdpTransportPacketExtension.GENERATION_UNSPECIFIED + } + } + } + + context("With ICE established") { + val transport = createTransport() + val initial = agents.created[0] + initial.state = IceProcessingState.COMPLETED + + should("create a new Agent and describe it") { + transport.requestIceRestart() shouldBe IceRestartResult.STARTED + agents.created.size shouldBe 2 + with(transport.describe()) { + ufrag shouldBe agents.created[1].ufrag + password shouldBe agents.created[1].password + iceGeneration shouldBe 1 + } + } + should("keep sending on the established Agent until the new one connects") { + transport.requestIceRestart() + transport.send(ByteArray(10), 0, 10) + verify(exactly = 1) { initial.component.send(any(), any(), any()) } + verify(exactly = 0) { agents.created[1].component.send(any(), any(), any()) } + } + should("not start connectivity checks before the peer's credentials arrive") { + transport.requestIceRestart() + agents.created[1].startCalls shouldBe 0 + } + + context("A repeated request") { + should("re-describe the pending Agent while it has not started checks") { + transport.requestIceRestart() shouldBe IceRestartResult.STARTED + transport.requestIceRestart() shouldBe IceRestartResult.STARTED + + agents.created.size shouldBe 2 + agents.created[1].freed shouldBe false + transport.describe().iceGeneration shouldBe 1 + } + should("supersede a restart whose checks have started") { + transport.requestIceRestart() + transport.startConnectivityEstablishment(remoteTransport(generation = 1)) + agents.created[1].startCalls shouldBe 1 + + transport.requestIceRestart() shouldBe IceRestartResult.STARTED + agents.created.size shouldBe 3 + transport.describe().iceGeneration shouldBe 2 + awaitTrue { agents.created[1].freed } + } + } + + context("The peer's credentials") { + should("be applied when they carry the pending generation") { + transport.requestIceRestart() + transport.startConnectivityEstablishment(remoteTransport(generation = 1)) + + agents.created[1].remoteUfrag shouldBe "remote-ufrag" + agents.created[1].remotePassword shouldBe "remote-pwd" + agents.created[1].startCalls shouldBe 1 + } + should("be ignored when they carry another generation") { + transport.requestIceRestart() + transport.startConnectivityEstablishment(remoteTransport(generation = 7)) + + agents.created[1].remoteUfrag shouldBe null + agents.created[1].startCalls shouldBe 0 + } + should("not be taken from an untagged transport update") { + // The peer stamps the generation on the restart answer and on nothing else, so an untagged + // update is an ordinary one and carries the peer's old credentials. + transport.requestIceRestart() + transport.startConnectivityEstablishment(remoteTransport(generation = null)) + + agents.created[1].remoteUfrag shouldBe null + agents.created[1].startCalls shouldBe 0 + } + should("not start the checks twice") { + transport.requestIceRestart() + transport.startConnectivityEstablishment(remoteTransport(generation = 1)) + transport.startConnectivityEstablishment(remoteTransport(generation = 1)) + + agents.created[1].startCalls shouldBe 1 + } + should("not be replaced once the checks are running") { + // ice4j reads the remote credentials off the stream every time it signs a check or validates + // one, so a repeated update must not touch them. + transport.requestIceRestart() + transport.startConnectivityEstablishment(remoteTransport(generation = 1)) + transport.startConnectivityEstablishment( + remoteTransport(generation = 1, remoteUfrag = "other-ufrag", remotePassword = "other-pwd") + ) + + agents.created[1].remoteUfrag shouldBe "remote-ufrag" + agents.created[1].remotePassword shouldBe "remote-pwd" + } + } + + context("When the new Agent connects") { + should("cut over to it") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + + transport.send(ByteArray(10), 0, 10) + verify(exactly = 1) { agents.created[1].component.send(any(), any(), any()) } + } + should("describe it without an ice-generation") { + // The generation marks a transport to restart against. The established one is not that. + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + + with(transport.describe()) { + ufrag shouldBe agents.created[1].ufrag + iceGeneration shouldBe IceUdpTransportPacketExtension.GENERATION_UNSPECIFIED + } + } + should("free the old Agent only after the transition window") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + + initial.freed shouldBe false + } + } + + context("When the new Agent fails") { + should("keep the established one") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.FAILED) + + transport.hasFailed() shouldBe false + transport.describe().ufrag shouldBe initial.ufrag + awaitTrue { agents.created[1].freed } + } + should("not trigger the transport's own failure handling") { + var failed = false + transport.eventHandler = object : IceTransport.EventHandler { + override fun connected() {} + override fun failed() { + failed = true + } + override fun consentUpdated(time: java.time.Instant) {} + override fun writeable() {} + } + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.FAILED) + + failed shouldBe false + } + } + + context("The scheduled tasks") { + should("free the old Agent when the transition window elapses") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + initial.freed shouldBe false + + // Advances the fake clock to the task's deadline and runs it. + scheduler.runOne() + + awaitTrue { initial.freed } + } + should("abandon the restart when the timeout elapses") { + transport.requestIceRestart() + + scheduler.runOne() + + awaitTrue { agents.created[1].freed } + transport.hasFailed() shouldBe false + transport.describe().ufrag shouldBe initial.ufrag + } + should("not abandon a restart that has already cut over") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + + // Runs the transition window task. The timeout task was cancelled by the cutover, so it is + // dropped rather than run, and does not free the Agent we just cut over to. + scheduler.runOne() + + awaitTrue { initial.freed } + agents.created[1].freed shouldBe false + transport.describe().ufrag shouldBe agents.created[1].ufrag + } + should("be cancelled by stop()") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + transport.stop() + + scheduler.numPendingJobs() shouldBe 0 + } + } + + context("stop()") { + should("free the current and the pending Agent") { + transport.requestIceRestart() + transport.stop() + + initial.freed shouldBe true + agents.created[1].freed shouldBe true + } + should("free an Agent that is still inside its transition window") { + transport.requestIceRestart() + agents.created[1].fireState(IceProcessingState.COMPLETED) + transport.stop() + + initial.freed shouldBe true + agents.created[1].freed shouldBe true + } + } + + context("When the new Agent can not be created") { + should("report that a restart is unavailable") { + // A resource problem, so the endpoint is told to fall back to a full re-invite. + agents.failNext = true + transport.requestIceRestart() shouldBe IceRestartResult.UNAVAILABLE + transport.describe().ufrag shouldBe initial.ufrag + } + should("leave a later restart working") { + agents.failNext = true + transport.requestIceRestart() + transport.requestIceRestart() shouldBe IceRestartResult.STARTED + transport.describe().iceGeneration shouldBe 1 + } + } + } + + context("With ICE restarts disabled") { + should("report that a restart is unavailable") { + withNewConfig("videobridge.ice.restart.enabled = false") { + val transport = createTransport() + agents.created[0].state = IceProcessingState.COMPLETED + transport.requestIceRestart() shouldBe IceRestartResult.UNAVAILABLE + agents.created.size shouldBe 1 + } + } + } + + context("With a non-positive restart timeout") { + should("report that a restart is unavailable") { + withNewConfig("videobridge.ice.restart.timeout = 0 seconds") { + val transport = createTransport() + agents.created[0].state = IceProcessingState.COMPLETED + transport.requestIceRestart() shouldBe IceRestartResult.UNAVAILABLE + agents.created.size shouldBe 1 + } + } + } +}) + +private fun IceTransport.describe() = IceUdpTransportPacketExtension().also { describe(it) } + +private fun remoteTransport( + generation: Int?, + remoteUfrag: String = "remote-ufrag", + remotePassword: String = "remote-pwd" +) = IceUdpTransportPacketExtension().apply { + ufrag = remoteUfrag + password = remotePassword + generation?.let { iceGeneration = it } +} + +/** Waits for something another thread does (freeing an Agent happens on the IO pool). */ +private fun awaitTrue(timeoutMs: Long = 5000, condition: () -> Boolean) { + val deadline = System.currentTimeMillis() + timeoutMs + while (!condition() && System.currentTimeMillis() < deadline) { + Thread.sleep(10) + } + condition() shouldBe true +} + +private class FakeAgents { + val created = mutableListOf() + + /** Whether the next call to [factory] fails, the way a failure to bind a port would. */ + var failNext = false + + val factory: (Logger) -> Agent = { + if (failNext) { + failNext = false + throw java.io.IOException("Failed to bind") + } + FakeAgent(created.size).also { created.add(it) }.agent + } +} + +private class FakeAgent(index: Int) { + val ufrag = "ufrag-$index" + val password = "password-$index" + + var state: IceProcessingState = IceProcessingState.WAITING + + /** Written on the IO pool (that is where Agents are freed) and read from the test thread. */ + @Volatile + var freed = false + + var startCalls = 0 + var remoteUfrag: String? = null + var remotePassword: String? = null + + private val stateChangeListeners = mutableListOf() + + val component: Component = mockk(relaxed = true) + + val stream: IceMediaStream = mockk(relaxed = true) { + every { remoteUfrag = any() } answers { this@FakeAgent.remoteUfrag = firstArg() } + every { remoteUfrag } answers { this@FakeAgent.remoteUfrag } + every { remotePassword = any() } answers { this@FakeAgent.remotePassword = firstArg() } + every { remotePassword } answers { this@FakeAgent.remotePassword } + } + + val agent: Agent = mockk(relaxed = true) { + every { localUfrag } returns ufrag + every { localPassword } returns password + every { state } answers { this@FakeAgent.state } + every { createMediaStream(any()) } returns stream + every { + createComponent(any(), any(), any()) + } returns component + every { addStateChangeListener(any()) } answers { stateChangeListeners.add(firstArg()) } + every { removeStateChangeListener(any()) } answers { stateChangeListeners.remove(firstArg()) } + every { startConnectivityEstablishment() } answers { startCalls++ } + every { free() } answers { freed = true } + } + + fun fireState(newState: IceProcessingState, oldState: IceProcessingState = IceProcessingState.RUNNING) { + state = newState + val event = PropertyChangeEvent(agent, IceProcessingState::class.java.name, oldState, newState) + stateChangeListeners.toList().forEach { it.propertyChange(event) } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/util/BooleanStateTimeTrackerTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/util/BooleanStateTimeTrackerTest.kt new file mode 100644 index 0000000..1150f67 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/util/BooleanStateTimeTrackerTest.kt @@ -0,0 +1,63 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.util + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.utils.secs +import org.jitsi.utils.time.FakeClock +import java.time.Duration + +class BooleanStateTimeTrackerTest : ShouldSpec() { + private val clock = FakeClock() + private val oversendingTracker = BooleanStateTimeTracker(clock = clock) + + init { + context("BooleanStateTimeTracker") { + should("start with a time of 0") { + oversendingTracker.totalTimeOn() shouldBe Duration.ofSeconds(0) + oversendingTracker.totalTimeOff() shouldBe Duration.ofSeconds(0) + } + context("when the state changes") { + oversendingTracker.on() + should("increment the time in the 'on' state") { + clock.elapse(5.secs) + oversendingTracker.totalTimeOn() shouldBe 5.secs + oversendingTracker.totalTimeOff() shouldBe 0.secs + clock.elapse(5.secs) + oversendingTracker.totalTimeOn() shouldBe 10.secs + oversendingTracker.totalTimeOff() shouldBe 0.secs + } + context("and then changes again") { + clock.elapse(5.secs) + oversendingTracker.off() + should("have the correct time") { + oversendingTracker.totalTimeOn() shouldBe 15.secs + oversendingTracker.totalTimeOff() shouldBe 0.secs + } + context("and time elapses") { + clock.elapse(5.secs) + should("not increase the time") { + oversendingTracker.totalTimeOn() shouldBe 15.secs + oversendingTracker.totalTimeOff() shouldBe 5.secs + } + } + } + } + } + } +} diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/version/JvbVersionServiceTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/version/JvbVersionServiceTest.kt new file mode 100644 index 0000000..8f7be34 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/version/JvbVersionServiceTest.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.version + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class JvbVersionServiceTest : ShouldSpec({ + context("parsing a version") { + context("from a valid version string") { + should("parse correctly") { + with(JvbVersionService().parseVersionString("2.1-296-g817c1a45")) { + this.applicationName shouldBe "JVB" + this.versionMajor shouldBe 2 + versionMinor shouldBe 1 + nightlyBuildID shouldBe "296-g817c1a45" + } + } + } + context("from null") { + should("use the defaults") { + with(JvbVersionService().parseVersionString(null)) { + this.applicationName shouldBe "JVB" + this.versionMajor shouldBe 2 + versionMinor shouldBe 1 + nightlyBuildID shouldBe null + } + } + } + } +}) diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfigTest.kt new file mode 100644 index 0000000..edf6888 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/websocket/config/WebsocketServiceConfigTest.kt @@ -0,0 +1,128 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.websocket.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import org.jitsi.ConfigTest +import org.jitsi.config.withNewConfig +import org.jitsi.metaconfig.ConfigException +import org.jitsi.videobridge.websocket.config.WebsocketServiceConfig.Companion.config + +class WebsocketServiceConfigTest : ConfigTest() { + init { + context("when websockets are disabled") { + withNewConfig("videobridge.websockets.enabled = false") { + context("accessing useTls should throw") { + shouldThrow { + config.useTls + } + } + } + } + context("when websockets are enabled") { + context("accessing domain and relay-domain") { + withNewConfig( + """ + videobridge.websockets.enabled = true + videobridge.websockets.domain = "new_domain" + videobridge.websockets.relay-domain = "relay_domain" + """.trimIndent() + ) { + config.domains shouldBe listOf("new_domain") + config.relayDomains shouldBe listOf("relay_domain") + } + withNewConfig( + """ + videobridge.websockets.enabled = true + videobridge.websockets.domain = "new_domain" + videobridge.websockets.relay-domain = "relay_domain" + """.trimIndent() + ) { + config.domains shouldBe listOf("new_domain") + config.relayDomains shouldBe listOf("relay_domain") + } + withNewConfig( + """ + videobridge.websockets.enabled = true + videobridge.websockets.domains = ["new_domain"] + """.trimIndent() + ) { + config.domains shouldBe listOf("new_domain") + config.relayDomains shouldBe listOf("new_domain") + } + withNewConfig( + """ + videobridge.websockets.enabled = true + videobridge.websockets.domains = ["new_domain"] + videobridge.websockets.relay-domain = "relay_domain" + """.trimIndent() + ) { + config.domains shouldBe listOf("new_domain") + config.relayDomains shouldBe listOf("relay_domain") + } + withNewConfig( + """ + videobridge.websockets.enabled = true + videobridge.websockets.domains = ["new_domain"] + videobridge.websockets.relay-domains = ["relay_domain1", "relay_domain2"] + """.trimIndent() + ) { + config.domains shouldBe listOf("new_domain") + config.relayDomains shouldBe listOf("relay_domain1", "relay_domain2") + } + withNewConfig( + """ + videobridge.websockets.enabled = true + videobridge.websockets.domain = "new_domain1" + videobridge.websockets.domains = ["new_domain2"] + """.trimIndent() + ) { + config.domains shouldBe listOf("new_domain1", "new_domain2") + config.relayDomains shouldBe listOf("new_domain1", "new_domain2") + } + } + context("accessing useTls") { + context("when no value has been set") { + withNewConfig(newConfigWebsocketsEnabled) { + should("return null") { + config.useTls shouldBe null + } + } + } + context("when a value has been set") { + withNewConfig(newConfigWebsocketsEnableduseTls) { + should("get the right value") { + config.useTls shouldBe true + } + } + } + } + } + } +} +private val newConfigWebsocketsEnabled = """ + videobridge.websockets.enabled = true +""".trimIndent() + +private val newConfigWebsocketsEnabledDomain = newConfigWebsocketsEnabled + "\n" + """ + videobridge.websockets.domain = "new_domain" +""".trimIndent() + +private val newConfigWebsocketsEnableduseTls = newConfigWebsocketsEnabled + "\n" + """ + videobridge.websockets.tls = true +""".trimIndent() diff --git a/jvb/src/test/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfigTest.kt b/jvb/src/test/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfigTest.kt new file mode 100644 index 0000000..a6a4099 --- /dev/null +++ b/jvb/src/test/kotlin/org/jitsi/videobridge/xmpp/config/XmppClientConnectionConfigTest.kt @@ -0,0 +1,196 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.videobridge.xmpp.config + +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import org.jitsi.ConfigTest +import org.jitsi.config.withLegacyConfig +import org.jitsi.config.withNewConfig +import org.jivesoftware.smack.ConnectionConfiguration + +internal class XmppClientConnectionConfigTest : ConfigTest() { + init { + context("Connection configs") { + context("when defined in the legacy config") { + withLegacyConfig(legacyConfigSingleXmppConnection) { + should("parse things correctly") { + val configs = XmppClientConnectionConfig.Companion.config.clientConfigs + configs shouldHaveSize 1 + with(configs[0]) { + this.id shouldBe "shard" + this.mucNickname shouldBe "test_nick" + this.disableCertificateVerification shouldBe true + this.domain shouldBe "auth.some.domain.net" + this.hostname shouldBe "localhost" + this.username shouldBe "jvb" + this.password shouldBe "s3cr3t" + this.mucJids shouldContainExactly listOf("JvbBrewery@internal.some.domain.net") + this.iqHandlerMode shouldBe "sync" + this.securityMode shouldBe null + } + } + } + } + context("when multiple are defined in legacy and one is incomplete") { + withLegacyConfig(legacyConfigOneCompleteConnectionOneIncomplete) { + should("only parse the complete one") { + val configs = XmppClientConnectionConfig.Companion.config.clientConfigs + configs shouldHaveSize 1 + with(configs[0]) { + this.id shouldBe "shard" + this.mucNickname shouldBe "test_nick" + this.disableCertificateVerification shouldBe true + this.domain shouldBe "auth.some.domain.net" + this.hostname shouldBe "localhost" + this.username shouldBe "jvb" + this.password shouldBe "s3cr3t" + this.mucJids shouldContainExactly listOf("JvbBrewery@internal.some.domain.net") + this.iqHandlerMode shouldBe "sync" + this.securityMode shouldBe null + } + } + } + } + context("when defined in new config") { + withNewConfig(newConfigSingleXmppConnection) { + should("parse things correctly") { + val configs = XmppClientConnectionConfig.Companion.config.clientConfigs + configs shouldHaveSize 1 + with(configs[0]) { + this.id shouldBe "shard" + this.mucNickname shouldBe "test_nick" + this.disableCertificateVerification shouldBe true + this.domain shouldBe "auth.some.domain.net" + this.hostname shouldBe "localhost" + this.username shouldBe "jvb" + this.password shouldBe "s3cr3t" + this.mucJids shouldContainExactly listOf("JvbBrewery@internal.some.domain.net") + this.iqHandlerMode shouldBe "sync" + this.securityMode shouldBe ConnectionConfiguration.SecurityMode.required + } + } + } + } + context("when defined in new config with some incomplete") { + withNewConfig(newConfigOneCompleteConnectionOneIncompleteOneBroken) { + should("parse things correctly") { + val configs = XmppClientConnectionConfig.Companion.config.clientConfigs + configs shouldHaveSize 1 + with(configs[0]) { + this.id shouldBe "shard" + this.mucNickname shouldBe "test_nick" + this.disableCertificateVerification shouldBe true + this.domain shouldBe "auth.some.domain.net" + this.hostname shouldBe "localhost" + this.username shouldBe "jvb" + this.password shouldBe "s3cr3t" + this.mucJids shouldContainExactly listOf("JvbBrewery@internal.some.domain.net") + this.iqHandlerMode shouldBe "sync" + this.securityMode shouldBe null + } + } + } + } + } + } +} + +private val legacyConfigSingleXmppConnection = """ + org.jitsi.videobridge.xmpp.user.shard.DOMAIN=auth.some.domain.net + org.jitsi.videobridge.xmpp.user.shard.MUC_JIDS=JvbBrewery@internal.some.domain.net + org.jitsi.videobridge.xmpp.user.shard.MUC=JvbBrewery@internal.some.domain.net + org.jitsi.videobridge.xmpp.user.shard.DISABLE_CERTIFICATE_VERIFICATION=true + org.jitsi.videobridge.xmpp.user.shard.HOSTNAME=localhost + org.jitsi.videobridge.xmpp.user.shard.USERNAME=jvb + org.jitsi.videobridge.xmpp.user.shard.PASSWORD=s3cr3t + org.jitsi.videobridge.xmpp.user.shard.MUC_NICKNAME=test_nick + org.jitsi.videobridge.xmpp.user.shard.IQ_HANDLER_MODE=sync +""".trimIndent() + +private val legacyConfigOneCompleteConnectionOneIncomplete = """ + org.jitsi.videobridge.xmpp.user.shard.DOMAIN=auth.some.domain.net + org.jitsi.videobridge.xmpp.user.shard.MUC_JIDS=JvbBrewery@internal.some.domain.net + org.jitsi.videobridge.xmpp.user.shard.MUC=JvbBrewery@internal.some.domain.net + org.jitsi.videobridge.xmpp.user.shard.DISABLE_CERTIFICATE_VERIFICATION=true + org.jitsi.videobridge.xmpp.user.shard.HOSTNAME=localhost + org.jitsi.videobridge.xmpp.user.shard.USERNAME=jvb + org.jitsi.videobridge.xmpp.user.shard.PASSWORD=s3cr3t + org.jitsi.videobridge.xmpp.user.shard.MUC_NICKNAME=test_nick + org.jitsi.videobridge.xmpp.user.shard.IQ_HANDLER_MODE=sync + + org.jitsi.videobridge.xmpp.user.incomplete.USERNAME=jvb +""".trimIndent() + +private val newConfigSingleXmppConnection = """ + videobridge { + apis { + xmpp-client { + configs { + shard { + DOMAIN=auth.some.domain.net + MUC_JIDS="JvbBrewery@internal.some.domain.net" + MUC="JvbBrewery@internal.some.domain.net" + DISABLE_CERTIFICATE_VERIFICATION=true + HOSTNAME="localhost" + USERNAME="jvb" + PASSWORD="s3cr3t" + MUC_NICKNAME="test_nick" + IQ_HANDLER_MODE="sync" + SECURITY_MODE="required" + } + } + } + } + } +""".trimIndent() + +private val newConfigOneCompleteConnectionOneIncompleteOneBroken = """ + videobridge { + apis { + xmpp-client { + configs { + shard { + DOMAIN=auth.some.domain.net + MUC_JIDS="JvbBrewery@internal.some.domain.net" + MUC="JvbBrewery@internal.some.domain.net" + DISABLE_CERTIFICATE_VERIFICATION=true + HOSTNAME="localhost" + USERNAME="jvb" + PASSWORD="s3cr3t" + MUC_NICKNAME="test_nick" + IQ_HANDLER_MODE="sync" + } + incomplete { + DOMAIN="incomplete" + } + broken { + DOMAIN=auth.some.domain.net + MUC_JIDS="JvbBrewery@internal.some.domain.net" + MUC="JvbBrewery@internal.some.domain.net" + HOSTNAME="localhost" + USERNAME="jvb" + PASSWORD="s3cr3t" + MUC_NICKNAME="test_nick" + SECURITY_MODE="somethingBusted" + } + } + } + } + } +""".trimIndent() diff --git a/jvb/src/test/resources/bwe-events.csv b/jvb/src/test/resources/bwe-events.csv new file mode 100644 index 0000000..cb46d6a --- /dev/null +++ b/jvb/src/test/resources/bwe-events.csv @@ -0,0 +1,428 @@ +ts,bwe,bps_a_7,bps_a_30,bps_b_7,bps_b_30,bps_c_7,bps_c_30,bps_d_7,bps_d_30,bps_e_7,bps_e_30,bps_f_7,bps_f_30 +19:52:49.234,33400,93560,157626,86717,160533,54922,135183,93405,156291,92150,155582,92397,152019 +19:52:50.269,37072,92424,158069,92482,170340,49992,118440,93456,156872,93771,156555,83091,138152 +19:52:51.339,41038,89581,152131,111864,198422,49864,126290,93466,157344,94106,157277,88003,146593 +19:52:52.383,45321,91459,157171,122813,209837,54955,136704,94334,159270,94176,158290,94046,156028 +19:52:53.415,49947,91168,155157,127704,215099,63165,143687,88387,161069,93944,157354,95699,165833 +19:52:54.543,50712,89250,152442,130715,216905,68946,144826,81810,158487,95270,158840,93013,173408 +19:52:54.884,52060,88904,149248,135253,224499,70054,142272,81698,161460,95845,161872,91970,176120 +19:52:55.143,53108,86563,145961,136530,226154,76822,149792,78264,157152,96230,160739,90291,174609 +19:52:55.432,54302,87821,146475,137738,229412,77482,152202,75755,156521,93909,159467,89395,172953 +19:52:55.671,55305,84346,143016,136254,227902,79701,148931,75594,156456,95536,160990,87496,170958 +19:52:55.932,56427,82619,139702,130486,215403,74872,142731,73536,155942,97888,164171,78989,163234 +19:52:56.203,57616,80682,134773,124610,208309,79061,141467,72560,157206,98806,166342,75802,159292 +19:52:56.457,58753,77779,131977,124560,206707,77256,140720,71106,157223,98771,166526,74187,157521 +19:52:56.678,59765,76400,129203,122611,204377,78874,140930,69173,155307,97512,164838,71822,155920 +19:52:56.909,60729,71770,122114,119448,200376,79178,139013,64422,150590,95650,161108,68314,155490 +19:52:56.965,60837,71770,122114,122109,203037,79178,139013,64994,152234,96669,162818,68314,155490 +19:52:57.136,61904,71253,119024,122854,204561,80069,136999,63237,149775,95982,160987,66448,153976 +19:52:57.384,63096,66994,113613,122795,207073,87389,146707,62450,150551,96714,162472,64146,148340 +19:52:57.628,64292,63654,105923,118806,200777,80222,138737,61314,138751,92645,159963,54474,141343 +19:52:57.836,65334,61242,101818,117354,198994,84554,146815,62198,139745,91082,157274,57486,142499 +19:52:58.079,66567,60707,102105,118400,199170,87576,150619,62803,140128,92867,158363,56682,142863 +19:52:58.281,67609,63360,102262,117990,198491,89518,153294,63994,140909,93384,160952,54221,139765 +19:52:58.529,68911,58874,93861,117021,199048,90430,154963,63472,138626,94290,159834,54781,136331 +19:52:58.732,69990,55491,90333,119517,201143,93859,157625,64286,137643,94005,158691,53581,135933 +19:52:58.977,71327,53643,86371,121987,204221,94602,160840,66539,140160,95184,161222,54882,138708 +19:52:59.180,72450,51725,82410,121054,202920,97291,163808,66955,140363,96080,162346,55693,136915 +19:52:59.402,73698,47144,77846,119981,201112,101371,169390,66682,138514,94019,159464,55274,137647 +19:52:59.606,74864,43090,71319,120891,202472,99694,165641,67941,138763,94002,159832,55158,136500 +19:52:59.847,76265,42867,68379,120266,203840,96374,164857,68766,139761,93846,160102,55158,137496 +19:53:00.070,77579,38933,62781,123226,207282,95149,161813,70178,139039,94096,159622,54997,135808 +19:53:00.291,78915,36307,67176,126269,210650,97147,161790,72102,141211,94218,160703,54299,134221 +19:53:00.559,80553,38523,68427,123040,205648,98851,164131,74010,140381,94238,159936,54048,131530 +19:53:00.804,81853,36536,64387,120434,202532,99238,163600,76566,140638,92138,156215,52134,130643 +19:53:00.847,82092,36536,64387,122597,204695,99238,163600,76566,148128,92138,157093,52134,133726 +19:53:01.028,83366,36536,62658,120917,202819,98453,168008,76048,143878,88758,150708,51622,131504 +19:53:01.094,83519,36536,62658,123018,206282,98453,168008,76371,145563,89810,152749,53957,133839 +19:53:01.273,84785,33245,68453,122235,204371,101306,167816,75517,142266,89374,151113,53728,129405 +19:53:01.404,85102,31797,65210,120907,201918,97998,164508,75392,141974,89250,150840,53354,129784 +19:53:01.494,86228,31797,65210,122416,204443,102182,171158,76181,144563,90579,154227,53354,129784 +19:53:01.668,86561,30678,62649,121278,203920,101203,163320,76600,143880,91710,154672,52264,128726 +19:53:01.694,87910,30678,62649,121278,203920,101203,163320,76600,144968,91710,155625,52264,128726 +19:53:01.919,89438,30536,61074,124018,207805,100270,163568,77174,144540,92509,155375,55022,125038 +19:53:02.253,91201,32613,61279,125877,210143,95096,160944,78098,147442,94232,158766,53651,120971 +19:53:02.295,91752,32613,61279,125877,210143,95096,160944,78098,148776,94232,159627,53651,120971 +19:53:02.537,93793,32613,61279,124283,207446,95370,158959,73597,142549,94330,157784,57982,129692 +19:53:02.741,95277,29120,71126,125936,208942,94301,160653,72648,142762,93520,156872,56229,125819 +19:53:02.959,96896,29120,71126,124386,207544,89034,149072,72840,145006,93302,157406,59381,124309 +19:53:03.207,97593,25584,64154,122984,205266,84131,137765,71926,145648,92109,154965,56040,120613 +19:53:03.297,98740,31200,69770,126378,209660,84131,141422,72786,147474,93416,157949,56040,120613 +19:53:03.514,98740,28763,67333,119987,201251,86090,140711,75752,152904,91283,152027,56538,120053 +19:53:03.614,101866,30637,69207,118197,197978,81632,139678,75578,153527,92408,155400,60430,123596 +19:53:03.915,103899,30637,67219,120451,200539,83854,137200,73222,152788,93162,156672,57360,115826 +19:53:04.045,104277,28067,64649,127128,208704,80328,133674,72283,151232,92142,155561,58802,119788 +19:53:04.165,105648,28067,64649,128090,210397,80328,132646,75608,155514,94358,158488,58802,117368 +19:53:04.217,106319,28067,64649,127194,208746,75470,127788,75968,154950,92894,156067,60739,120545 +19:53:04.367,107639,28067,76406,127131,210641,75120,125272,75674,154648,94074,157012,62602,121247 +19:53:04.689,109675,28067,72739,128379,211429,84813,139357,75643,153760,94382,159105,63584,120477 +19:53:04.792,111015,38555,83227,126981,209509,84813,136821,73618,150645,92853,155802,66658,124575 +19:53:04.966,111564,36686,90515,127371,208769,85410,141136,72757,149279,93389,156674,64982,122100 +19:53:05.022,113211,36686,90515,124718,205078,81490,137216,73182,149588,93434,157583,68758,123796 +19:53:05.082,113556,36686,90515,126942,208140,83224,138950,73182,149588,93434,157583,68758,123796 +19:53:05.256,115619,45787,90753,119698,198533,79603,140208,75386,150058,93342,156513,76038,133760 +19:53:05.470,117250,45787,90753,120766,200004,92216,147752,73750,150203,93370,156631,76746,138013 +19:53:05.712,119449,39933,92717,119493,200440,87638,145710,70714,148320,93800,157438,75950,139195 +19:53:05.825,119500,43562,96346,121645,200789,86341,144413,66202,143988,92749,156723,74118,138681 +19:53:05.926,120896,50819,103603,121914,201341,86341,149101,66469,138479,95563,159441,77784,145141 +19:53:05.984,121489,50819,103603,123675,204680,93619,156379,67930,139940,95563,159441,79803,147160 +19:53:06.034,121737,50819,103603,121346,203008,91634,152050,67626,139272,92662,155739,79803,144960 +19:53:06.237,123640,58370,111272,125642,208388,91634,153423,67573,141272,93499,157430,80541,150509 +19:53:06.292,124685,62146,116922,125642,209148,98435,160224,68349,142048,94413,158344,80541,150509 +19:53:06.438,125869,62146,122544,124506,206981,99173,164477,67013,140648,93246,157067,79006,149118 +19:53:06.492,126638,62146,122544,126066,210351,99173,164477,67013,140648,93246,157067,81778,151890 +19:53:06.689,127630,70264,130662,133706,218445,91730,153396,66392,140509,92592,156064,81778,149404 +19:53:06.739,129108,70264,130662,130099,213688,91730,158164,65803,138257,91370,153308,80190,151832 +19:53:06.893,130060,87432,148928,129173,211235,98773,165207,65622,140220,92427,155325,81626,153268 +19:53:06.940,131150,89848,152442,127832,207758,94421,156706,65802,141308,93944,156779,81626,157428 +19:53:07.193,132476,86285,152599,131504,207453,96266,159112,65810,139472,93853,156299,82776,161434 +19:53:07.243,134202,91978,159749,128906,204792,98112,160958,64310,136897,91293,152906,81883,159008 +19:53:07.395,135660,91978,164125,126050,202106,92997,155781,67016,141138,93571,156093,85226,168527 +19:53:07.446,136315,91978,164125,123862,200361,96992,159776,65539,138779,91037,152751,83528,163827 +19:53:07.597,137769,94606,168278,125514,203247,93349,155679,71766,148656,95462,159396,83528,164186 +19:53:07.696,138450,95899,163854,125245,202091,93210,155540,72242,148690,94610,159548,79413,159269 +19:53:07.846,140437,97083,157664,125067,201661,93210,156725,71734,146974,94902,159254,80013,160115 +19:53:07.897,141117,98826,159728,126808,205517,93210,156725,72382,147622,94902,160150,80013,160115 +19:53:08.049,142642,102698,165580,125470,207678,96462,159977,71773,145482,95605,160271,77464,160330 +19:53:08.201,143350,108482,179252,126530,209794,106150,175112,71810,143069,94744,159042,79371,163315 +19:53:08.251,145595,112243,183230,128718,211982,106150,175112,72493,144458,95894,161012,79371,163315 +19:53:08.453,147876,108902,184577,127878,210844,108874,178820,67474,134455,95837,160979,78619,166206 +19:53:08.656,149791,116312,198976,128886,212008,102594,170828,68878,136084,95418,160023,75984,161418 +19:53:08.705,150204,119117,203008,128445,208594,102594,174615,69549,135512,93765,157573,75984,161418 +19:53:08.913,153204,128067,220203,127160,208493,98834,170855,71726,137803,95211,158883,78005,166413 +19:53:09.114,155053,136782,235912,122496,203267,98834,175197,78659,147942,97253,162354,74758,163508 +19:53:09.216,155592,143158,245025,123506,205413,103997,178032,75008,142806,93522,158020,72803,161553 +19:53:09.316,158029,143854,236648,125094,206400,103997,184069,73493,141629,95136,158733,68664,158122 +19:53:09.519,160517,149675,245235,119998,203156,102883,177126,73890,142218,94728,158293,71083,161169 +19:53:09.770,163220,155902,256537,126144,209330,93368,172149,74995,144021,94776,159288,70483,161990 +19:53:09.921,163647,146816,246768,125301,206371,94406,173825,73954,142768,94808,158782,68998,164059 +19:53:09.972,166210,147818,248672,127533,209627,94406,173825,74845,144402,96342,161156,68998,164059 +19:53:10.173,168457,154606,253427,125973,207543,96429,175848,76040,146542,97211,162656,68248,164398 +19:53:10.224,168814,146053,245475,126683,207000,96429,172495,71560,140944,93611,158141,61304,157454 +19:53:10.391,171617,147669,247767,127710,210601,94565,170631,73610,145456,97502,163648,60021,156667 +19:53:10.634,174358,151723,246109,125811,203654,82922,164365,75600,143400,94909,159223,59342,148670 +19:53:10.790,174883,153746,250157,124685,204426,86312,162770,76725,145127,95541,159869,60869,150197 +19:53:10.837,177278,154862,251881,125493,204387,83637,164690,77155,144456,94210,158316,60869,148879 +19:53:10.938,177650,145992,243838,123450,201834,83637,160002,77379,144718,94557,158787,55643,140102 +19:53:11.039,180433,147149,246243,124283,204240,76926,153291,75875,143976,94888,159965,55397,139856 +19:53:11.290,183319,151179,242528,122398,199768,80070,156995,79958,148230,97451,163568,60968,148726 +19:53:11.390,183952,140966,229116,124392,203341,71568,152222,79202,147159,96360,161808,60466,144114 +19:53:11.493,186346,143170,228359,125234,206295,71568,148707,80502,147742,94877,160205,60466,144677 +19:53:11.643,186834,136901,224309,123080,200134,71314,153008,81987,148779,96187,161320,58702,142913 +19:53:11.743,189307,139368,227488,119379,198203,71314,153008,84611,150209,97339,162696,65851,147385 +19:53:11.794,189874,140195,229053,122126,200950,74018,155712,84611,151165,98410,164501,69427,151390 +19:53:11.843,190478,130659,220305,121032,200066,74018,150944,84674,149173,96168,161646,67680,148304 +19:53:11.946,192862,125293,213379,121325,203602,66974,143900,87085,152191,97474,164365,60992,138843 +19:53:12.053,193462,123890,213495,124605,206935,66974,148971,86643,150376,95763,160830,60701,137650 +19:53:12.154,195868,125784,216034,117421,200218,69979,150000,88709,153240,96976,163712,60701,133451 +19:53:12.307,196614,118314,201071,119066,202853,71645,149016,87190,148678,96109,160296,59147,128745 +19:53:12.357,198985,119478,202235,120589,204376,77006,154377,88256,149744,96109,161053,59147,128745 +19:53:12.412,199668,120286,203593,121637,205712,77006,155212,87533,149941,95739,161416,57043,123361 +19:53:12.563,202254,122333,208021,121147,205934,73011,151217,85146,145229,95213,159571,59669,128818 +19:53:12.663,202781,120467,205123,124478,207443,73421,151250,85466,145701,95234,159796,59669,127002 +19:53:12.764,205452,121651,207761,124504,207549,68774,146603,85982,146483,94933,159639,63152,131592 +19:53:12.815,206047,122453,208314,123101,206579,69523,143403,86520,146922,93848,158514,63152,129093 +19:53:12.969,209291,121939,209590,124040,208618,69523,147619,88109,149567,96715,161281,61242,128700 +19:53:13.070,209307,122205,209679,125072,207926,66243,144339,88390,150257,94981,159386,64421,130925 +19:53:13.170,212537,118078,205851,124144,206886,61400,142891,89014,151924,95786,160205,64872,131376 +19:53:13.321,212586,113653,194861,125349,205114,63104,141573,88421,151040,94571,157761,62965,128133 +19:53:13.373,215904,113653,195623,126963,206728,63104,141573,89568,152187,95632,159541,62965,128133 +19:53:13.424,215933,111101,190987,125051,205723,63104,138478,89768,152115,92998,157150,64682,126207 +19:53:13.573,219299,108816,187208,127107,208109,66066,142668,91226,154639,95496,159717,67091,130625 +19:53:13.778,222786,103363,178024,126278,210044,69795,142686,92045,156637,96050,160308,69102,130715 +19:53:13.982,226293,97658,167540,125365,207779,71894,144785,91926,156600,96595,160865,70374,131320 +19:53:14.184,229838,95402,164218,125464,208720,73195,144769,87765,153168,96013,160672,73619,133600 +19:53:14.387,233456,90637,153240,125102,208708,70635,142302,88277,151525,96973,161819,71563,132157 +19:53:14.639,237843,84419,146801,125016,207898,77570,149287,90013,153914,94195,159091,74226,135248 +19:53:14.791,238009,83691,148030,127064,208738,81334,150443,90454,155172,96224,160786,76942,137404 +19:53:14.841,241609,82126,143921,123981,206314,76432,148370,90539,154987,93299,157942,75354,135816 +19:53:14.993,241756,81688,145928,126562,210660,79875,154840,91976,157213,96477,160877,78966,135875 +19:53:15.052,245396,81338,144676,126707,209781,83853,154461,91064,155910,94848,158997,78966,137374 +19:53:15.195,245562,76514,136799,127854,212036,80096,150704,92810,158741,96019,160518,88458,146831 +19:53:15.246,249225,76206,137233,126802,212714,80096,152688,93221,158555,95123,159733,87254,145152 +19:53:15.402,249390,75950,138177,126816,211442,83542,154513,92827,157269,93438,157156,86408,137189 +19:53:15.447,253132,76632,139581,128362,215426,83542,154513,93832,159219,94578,159324,86408,140136 +19:53:15.655,257364,74702,139449,130424,215330,81237,145992,94235,159691,94923,159227,86269,143034 +19:53:15.903,261938,72861,137347,129467,214833,82294,142454,93325,159074,94968,158779,94808,150944 +19:53:16.006,262096,72590,138542,127762,214024,79906,140066,93672,158378,93579,157392,93035,150968 +19:53:16.055,262343,73376,139328,129450,215712,79906,142730,94782,160108,94451,158264,95795,153728 +19:53:16.158,266028,72146,139296,129299,215128,83949,146795,93846,159332,93915,157685,95795,150715 +19:53:16.215,266207,71162,138336,126206,211622,83949,145200,92736,156390,91653,154552,95795,153009 +19:53:16.263,267562,71590,139972,127976,215267,83949,145200,94118,159537,94808,158552,95795,153009 +19:53:16.365,270163,71579,139851,127917,214314,85675,146926,94371,159299,94632,158405,90166,144230 +19:53:16.425,270341,69363,137910,127822,212588,85675,146857,94531,158990,93498,157335,88779,145561 +19:53:16.475,271816,70046,140062,128824,214904,85675,146857,95851,161221,94803,159480,92974,149756 +19:53:16.577,274383,70507,139221,130490,213962,90190,154516,95005,161378,96010,159895,96002,150279 +19:53:16.678,276308,69573,140200,129656,214134,86902,151228,95784,160909,94272,158434,94994,149271 +19:53:16.778,278953,69734,139987,128202,214255,91629,151400,93674,158866,95067,158288,97091,153553 +19:53:16.931,280615,68034,139464,127210,212908,88925,151856,93027,157331,94510,157164,88154,146312 +19:53:16.992,283893,68826,140256,129614,216660,93522,156453,94528,159728,95960,159410,91482,149640 +19:53:17.041,284064,67374,139155,127230,211377,93522,156453,94395,159518,94266,157623,91174,150755 +19:53:17.092,284330,67952,140578,129266,213413,93522,159333,95333,160456,94266,158333,91174,150755 +19:53:17.192,288310,67733,140397,129293,214311,93522,152572,94549,160261,95357,158578,93536,153117 +19:53:17.244,288504,66706,139620,127562,212575,90517,149567,94589,158994,94040,157362,93536,155504 +19:53:17.295,289698,67645,141824,130462,217128,95818,154868,95712,160117,94040,158216,95325,157293 +19:53:17.396,292776,67350,142160,129986,216925,95765,156155,93851,159096,94858,158066,96195,160571 +19:53:17.452,292968,66072,140331,128866,213962,90403,149958,93688,158418,93270,156580,96195,160571 +19:53:17.502,294282,65661,139667,129106,214748,91024,152251,96126,159897,92366,154785,99950,162884 +19:53:17.604,297334,64544,140008,127795,213155,90614,155185,94635,156945,92552,155789,97325,161648 +19:53:17.702,299007,65621,141059,127981,214018,95994,158020,95312,157827,92560,154389,101110,163219 +19:53:17.853,302269,64954,143312,128720,213181,100626,164341,97118,160064,93280,155554,99469,167050 +19:53:17.953,304064,64459,142280,126526,210638,101395,160894,96182,158960,93154,155272,99469,165533 +19:53:18.005,304815,63021,141235,125245,207482,97899,157398,93723,155731,91730,153149,99906,165970 +19:53:18.055,307789,63800,143037,126235,208472,97899,157398,94400,157142,92683,154827,99906,165970 +19:53:18.105,308076,63360,141285,122885,205021,97899,155701,94630,157040,92562,154056,99906,166776 +19:53:18.206,309590,62267,141429,120914,201538,97899,156277,94957,156579,91424,154062,100906,170130 +19:53:18.305,313874,62611,141360,119094,198198,90990,149368,93613,155722,93370,155421,103706,170831 +19:53:18.457,315604,61766,142041,119707,199841,92568,152926,95126,157291,94050,157383,100680,170109 +19:53:18.513,318825,61080,140995,117254,195915,88002,148360,93933,155296,94003,156733,103550,170969 +19:53:18.615,320672,59629,140848,118426,195709,88002,148674,94168,155610,93296,156608,99830,167313 +19:53:18.716,323928,60253,141187,116142,192620,89280,149952,93021,154840,93837,156263,99077,166560 +19:53:18.867,325721,59606,142880,117011,194457,89280,150874,93950,155715,93638,157105,99077,168778 +19:53:19.121,330849,56002,139946,113685,191058,90885,151282,93565,154203,93918,155750,102485,170437 +19:53:19.323,336032,53086,134040,115565,192066,94629,154389,92594,154418,94362,156333,102853,170242 +19:53:19.575,342612,54566,136019,117354,195213,92763,151675,93886,154656,95779,158633,99776,167224 +19:53:19.779,348033,54650,136368,114040,193101,90664,151829,94418,156916,96019,159526,101179,169230 +19:53:19.981,352779,54739,136222,114907,192397,90477,148813,94192,156917,96267,159565,96413,163759 +19:53:20.135,353485,53730,135077,111538,187698,86693,144040,94155,156041,94424,157125,99115,163461 +19:53:20.182,358317,54402,135749,112880,189040,86693,144040,94155,156974,95661,159226,99115,166169 +19:53:20.334,358995,53613,134880,107746,183397,83246,139284,100187,168289,94925,157648,92213,161451 +19:53:20.400,363915,54219,135486,107746,184914,87992,144030,100187,168440,96349,159979,92213,161451 +19:53:20.582,364647,54078,134736,107408,182451,87992,147222,98413,164955,96706,160240,98488,167075 +19:53:20.630,371021,54237,134002,105547,180534,93333,152563,98195,164318,93784,157267,98488,164038 +19:53:20.781,371559,54254,134987,107362,182141,97986,160314,97611,164222,96136,159549,100589,166442 +19:53:20.880,376876,54269,134939,105155,181163,97986,162938,98738,164109,94922,159207,98184,166114 +19:53:20.981,378806,54666,135056,104190,179936,98050,163002,97010,162415,96453,160000,93416,161523 +19:53:21.081,382001,54958,135416,103683,179785,98050,166119,97720,162883,94862,158870,96344,162654 +19:53:21.186,384740,54765,135384,104946,179898,94006,159411,96898,161864,96243,160477,93584,162430 +19:53:21.287,387982,54286,135176,104314,179546,98894,161089,97301,163104,96550,161809,97336,165035 +19:53:21.336,388715,54496,134178,102544,177216,94024,156219,94210,159060,94856,158446,95829,162381 +19:53:21.387,390858,54917,135327,104186,178858,94024,158989,95419,161043,96421,160925,99269,167587 +19:53:21.495,394028,54965,135719,102336,178790,97202,158506,96779,161833,95416,160768,97762,163362 +19:53:21.595,394819,55048,135170,100758,177230,97202,161532,95438,159955,96365,160755,93566,160099 +19:53:21.644,397165,55045,134733,100358,175657,92686,153872,95373,159895,95168,159762,94600,163555 +19:53:21.756,401703,54054,133360,100162,176095,96902,159648,95323,159686,95357,159928,94600,163555 +19:53:21.822,402284,54018,133346,100019,176213,92176,156483,95363,158670,94181,158682,94101,159808 +19:53:21.922,405223,53328,132806,99456,175640,96557,160946,93661,157685,94936,158531,94101,161461 +19:53:21.973,408026,53942,133420,101101,177285,96557,160946,94592,159744,96203,160713,97507,164867 +19:53:22.024,410110,53789,132960,96936,173086,93491,157880,94206,157678,93328,157733,94179,163944 +19:53:22.156,412047,53128,132045,98173,174400,96555,160251,94594,158724,94968,159587,97331,165189 +19:53:22.212,416519,52792,129560,97419,173489,100699,164395,94195,157352,94018,158528,94389,162247 +19:53:22.316,419696,50946,127394,96294,169921,93630,160387,93544,157968,95477,159309,94282,161778 +19:53:22.420,423127,51845,127437,95654,169968,94210,159626,95078,158824,94794,159845,95798,161681 +19:53:22.519,426468,51675,127805,92928,166912,93589,160147,90221,154733,96253,160301,93562,160741 +19:53:22.680,429938,50973,126056,94189,169243,97829,163971,93126,159291,97579,169577,96280,162070 +19:53:22.776,434839,49328,124430,94029,169143,98774,164916,92630,158582,99750,172600,92494,160716 +19:53:22.828,435054,49816,123066,94216,167867,91600,156053,92000,157278,97851,170653,96490,164708 +19:53:22.929,438641,50046,123707,94925,170485,91600,158947,91930,158805,97528,170728,96104,161080 +19:53:22.980,441668,50835,125265,96429,171989,96134,163481,93122,160960,100032,174387,96104,161080 +19:53:23.031,443505,50128,124088,95541,172088,95365,162712,93192,159563,99477,174309,92488,160158 +19:53:23.134,443919,49899,123694,95091,172424,99736,170066,92498,159810,98690,172900,94331,160694 +19:53:23.187,446011,49899,124897,96584,175435,99736,170066,93490,161564,99046,174065,96176,164557 +19:53:23.239,450557,49125,122098,97280,175637,99736,170402,93624,161146,97339,172209,92499,157217 +19:53:23.343,454272,47726,119571,97923,177030,102973,173639,93979,160619,96994,170972,93238,160342 +19:53:23.441,457827,48283,120829,98096,177117,102973,172107,94245,161323,94126,168816,93238,160342 +19:53:23.553,461585,47822,117808,98699,180005,105154,174288,93701,160794,94050,167496,99504,166048 +19:53:23.697,465322,47093,115951,100562,181781,105154,173445,95168,161974,92544,166266,96634,163589 +19:53:23.759,465333,45328,113029,100827,182393,107299,175590,94246,161198,91739,165445,97619,164574 +19:53:23.797,470626,46218,113919,100827,183397,107299,178177,94246,162020,91739,166360,97619,166733 +19:53:23.862,470858,45512,111149,98778,180469,103960,174838,94054,161198,90827,165742,97619,164515 +19:53:23.951,472757,44230,110025,98717,180610,104195,172361,93941,161248,90936,165003,100658,169575 +19:53:24.003,478001,43710,107552,96638,176428,102362,168700,92851,159459,88214,162300,97394,163855 +19:53:24.058,479985,45306,109893,96638,177889,102362,171210,94067,161321,88643,163853,100466,166927 +19:53:24.151,480433,44320,109973,97426,179090,106142,174990,93493,160011,88595,163704,97029,163490 +19:53:24.211,482692,43579,107166,97062,176969,102312,171160,94157,160594,87290,162367,97029,164042 +19:53:24.260,487602,44294,107881,98157,179341,102312,173789,94157,161194,87290,163706,97029,164042 +19:53:24.361,489491,42917,106911,97482,178104,105954,174957,93837,161951,87211,162373,95821,163901 +19:53:24.416,491614,43635,105667,94208,174349,102136,170029,93930,160044,84770,159856,92752,158760 +19:53:24.466,495453,44864,108053,96333,176474,102136,172566,93930,160698,84770,161016,95736,161744 +19:53:24.567,497274,43346,107980,96026,177418,99462,168782,93645,160587,84878,161729,95736,164824 +19:53:24.621,499512,43093,105496,95946,174268,99462,167457,93312,159522,83094,159168,98430,165201 +19:53:24.717,503548,42374,103619,96736,175432,100253,164506,92085,158255,82109,158083,95445,162344 +19:53:24.769,503560,43802,106170,98592,178163,100253,167226,93621,159791,82915,159873,95445,162344 +19:53:24.823,509276,43165,104960,96090,175474,100253,167226,93586,159517,81293,157267,99014,165913 +19:53:24.868,509527,43165,107205,97992,177376,100253,167226,95062,161883,82080,159014,99014,165913 +19:53:24.980,511578,43862,108379,97126,176772,98213,165186,93606,161497,81688,159634,99611,167240 +19:53:25.028,517241,44424,108710,97285,176042,101291,166225,93608,159720,79862,158587,99611,167240 +19:53:25.079,519384,44424,110386,97285,177741,101291,166225,95451,162710,80803,159528,99611,169350 +19:53:25.191,519868,45872,112539,97827,178037,101291,166225,95093,163432,80328,159558,105845,175584 +19:53:25.229,521254,45200,111387,98189,175775,97654,162588,94626,161146,78990,157356,105845,174067 +19:53:25.282,527610,46491,112678,98189,177104,97654,166147,95826,163292,79787,159269,105845,174067 +19:53:25.389,529650,48010,116340,100410,179005,102133,167459,89296,150723,79030,159009,102098,167074 +19:53:25.442,531502,47403,115137,98869,176898,97387,162713,88355,149473,76986,157120,102098,168639 +19:53:25.496,536089,48246,117272,100626,178655,97387,165358,89618,151671,77851,157985,102098,168639 +19:53:25.592,538056,49008,118256,99341,178325,97387,165358,89971,153502,77250,157804,108733,172978 +19:53:25.642,540328,48026,117000,99243,176739,92046,156825,89589,152576,75712,155912,105477,170355 +19:53:25.699,540473,48026,117000,99243,179553,96840,161619,91246,154233,76520,156720,105477,170355 +19:53:25.748,544832,48635,116763,100370,179629,93739,155421,91416,154555,76125,156581,105477,170355 +19:53:25.797,544845,50269,121083,100370,181725,93739,158846,92693,157053,77032,158976,105477,170355 +19:53:25.854,548835,49992,119936,100870,183457,92187,157294,90640,154571,74554,156824,102125,165043 +19:53:25.899,551289,49992,121421,102330,184917,92187,157294,91733,155664,75182,157452,102125,165043 +19:53:26.001,553504,49846,121116,107131,190080,91957,154440,90960,154235,72880,154443,99634,158488 +19:53:26.059,557391,50834,124008,108789,193904,91957,154440,90960,155062,72880,155707,99634,159524 +19:53:26.109,561935,49070,122619,109960,193622,91957,155331,91085,154898,72211,155117,96706,156596 +19:53:26.213,562457,49632,122768,112362,200039,96600,159974,92469,156200,71366,154387,97211,156121 +19:53:26.259,563954,51176,124312,115814,203491,96600,159974,92469,157015,71366,155033,97211,157678 +19:53:26.309,570819,50709,124195,114240,201776,91712,155086,92336,155928,69539,152081,94173,154640 +19:53:26.421,573022,50384,122070,117936,203949,91712,155069,93216,156976,69395,152160,90733,152144 +19:53:26.460,575022,50384,122070,117936,205179,91712,155069,93216,157859,69395,153321,90733,152144 +19:53:26.511,579976,49661,119861,118858,204663,95334,157179,91957,155927,69027,152873,92034,153445 +19:53:26.623,582100,48098,116076,119006,206350,95334,157326,93376,157515,68262,152446,92034,153628 +19:53:26.700,584271,48098,118296,121040,208384,95334,157326,94704,159595,69011,154093,92034,153628 +19:53:26.731,584711,47336,116800,117693,204659,91118,153110,92405,156595,67746,152064,87973,147144 +19:53:26.783,589419,49150,121014,119437,206403,91118,153110,93904,158947,68598,153942,91085,152419 +19:53:26.839,589433,47776,117605,117746,204317,94323,153193,93770,159165,66778,153119,87699,149033 +19:53:26.894,593742,48507,118336,119138,207280,94323,155011,95272,160667,67467,153808,87699,149033 +19:53:26.941,596392,47859,117627,119589,206323,89942,151027,94358,159526,67046,152686,87699,149033 +19:53:27.051,598784,48498,119476,119461,204259,89942,151027,93355,158435,65502,151003,84293,142912 +19:53:27.104,602982,48301,119317,119458,204066,85347,146432,93261,158515,64830,149440,84293,140507 +19:53:27.153,607890,49846,120862,119458,205456,85347,146432,94494,159748,65718,151620,89750,147630 +19:53:27.257,608454,49408,122115,120019,205675,91955,150853,93448,158910,64085,151898,86598,146144 +19:53:27.315,613863,50429,123551,117982,202688,87811,145433,93315,157638,62213,149475,89384,146904 +19:53:27.362,617485,51742,124864,120147,206675,87811,149005,94285,158608,62814,150076,89384,146904 +19:53:27.467,622024,50981,125039,120384,207046,83696,144890,94139,158825,61840,149446,87648,144734 +19:53:27.525,623816,50120,122410,118872,205336,83696,142075,93091,157056,60115,147134,84613,140357 +19:53:27.589,627374,50120,122410,120597,208103,91648,150027,94069,159195,60846,149123,86158,141902 +19:53:27.624,629668,50120,121101,120939,205662,87408,144323,94592,159003,60198,147339,86158,140560 +19:53:27.737,632488,50120,121101,121101,206138,87408,143915,93446,157588,56802,136740,83440,140043 +19:53:27.792,637573,50120,122154,123566,210659,89869,146376,94576,159824,57678,138472,88965,145568 +19:53:27.886,637588,50629,121493,124722,211504,85338,141845,95622,160777,57498,138580,90499,147065 +19:53:27.945,644804,50202,121157,121251,208198,85338,144578,94328,158086,57392,137229,87835,142176 +19:53:28.013,645103,49574,122184,122030,209134,80803,140043,94146,157648,55453,134515,84514,136160 +19:53:28.065,647687,49574,122184,124914,212018,90464,149704,95502,159899,55453,135992,84514,136160 +19:53:28.126,652221,49800,121694,125245,212269,90464,146722,92840,156424,54037,134338,84514,138496 +19:53:28.185,655864,50902,124552,125245,213615,90464,149384,94104,158341,54792,135755,88509,142491 +19:53:28.246,657521,51501,123947,126082,211519,86093,145013,94422,158596,55030,136188,84821,136786 +19:53:28.285,658130,51501,123947,126082,212901,88522,147442,95667,159841,55734,136892,84821,138674 +19:53:28.336,663972,51501,125747,125786,211343,88522,144832,94822,158504,54624,135091,81282,135135 +19:53:28.394,667679,53134,127380,128006,215531,88522,144832,94822,159385,55323,136789,84216,139421 +19:53:28.457,667884,51966,128833,124934,211848,84526,144740,94666,159439,55526,137468,84216,138387 +19:53:28.495,672786,51966,128833,126787,213701,84526,144740,95734,160507,56235,138177,84216,138387 +19:53:28.546,674721,50906,126104,125221,211357,87739,145080,94614,158920,55854,136656,82438,134865 +19:53:28.606,678564,51901,127099,124878,210513,87739,145080,94698,158976,55637,136549,82611,137233 +19:53:28.671,681041,51901,127099,126018,211653,87739,148165,96123,161193,56315,138437,85624,140246 +19:53:28.786,684087,51901,129240,125149,209250,92822,154372,94994,160188,55629,139370,85624,137981 +19:53:28.826,689579,51011,128350,124170,207268,89008,147971,94867,159238,55979,137691,82123,132321 +19:53:28.940,689595,51837,128837,123650,206488,85107,144070,95317,159483,55962,137605,85171,138024 +19:53:28.999,697388,53462,130462,124414,207252,88293,147256,95317,160464,55962,138525,85171,140800 +19:53:29.058,697711,53462,130462,121371,203203,89886,148849,95510,160710,55792,138594,82133,135741 +19:53:29.154,705399,51867,128121,121522,202735,89886,146339,95235,159853,55779,138593,83493,139839 +19:53:29.266,711123,51867,128542,120499,200808,87990,148516,96874,161949,56776,139635,83493,137643 +19:53:29.315,711780,52758,129433,118309,197152,89941,147839,94146,159453,55325,136991,81088,135238 +19:53:29.365,718090,52758,129433,119762,199672,91563,153400,95587,161779,56122,138676,85032,141534 +19:53:29.419,722093,52741,130634,119779,198195,92990,154827,95680,159982,55970,137596,89165,144600 +19:53:29.479,722315,52741,132507,119779,198969,92990,154827,96774,162044,56698,139373,89165,144600 +19:53:29.544,727609,51512,127699,118704,198221,92990,152289,95701,160285,55800,138686,88083,144477 +19:53:29.601,729699,51512,127699,119427,197989,92990,155681,95834,159869,55410,136901,87291,142144 +19:53:29.643,733849,51512,127699,119427,198637,92990,155681,95834,160948,56232,138656,87291,142144 +19:53:29.694,736524,52490,128677,120253,199463,98386,161077,96850,161964,56827,139251,87291,142144 +19:53:29.809,739814,50474,126199,118994,198567,94683,154654,94192,158624,55915,137398,85038,140145 +19:53:29.854,745745,51843,127568,121296,200869,94683,157373,94192,159549,55915,138445,87672,142779 +19:53:29.957,745763,51334,127091,121102,199539,99011,161701,95474,160618,55917,139011,91746,147536 +19:53:30.020,754179,50253,124885,120646,199512,94395,157085,95688,160051,55936,137325,88261,147259 +19:53:30.076,754528,52131,126763,120646,200788,94395,157085,97005,162176,56597,138727,91474,150472 +19:53:30.175,762831,51600,128949,121520,200874,94395,161438,95408,161034,55662,138412,93082,149970 +19:53:30.274,769013,50539,129573,120394,201050,100838,167881,94778,159650,56326,138340,88432,148622 +19:53:30.325,769722,50498,126284,118613,198861,100838,164323,93578,158412,54730,136834,87768,146766 +19:53:30.381,776537,51790,131902,119795,200043,100838,167636,94768,160373,55354,138080,87768,150306 +19:53:30.448,780860,48984,129021,120480,201392,96360,163158,93502,158155,54690,135852,87768,148741 +19:53:30.513,781100,52747,128761,118723,198638,101280,165434,93574,158139,54485,134539,92326,153299 +19:53:30.560,786818,52747,128761,119800,200765,101280,165434,94752,159317,55184,135238,92326,155006 +19:53:30.623,789075,51206,127220,119454,200120,101280,168424,93410,156932,54294,134387,81603,143649 +19:53:30.666,793557,51206,127220,119454,201003,105621,172765,94310,158545,54901,136168,81603,143649 +19:53:30.716,796446,51206,125795,119901,198635,100827,169718,92653,156229,54093,133859,83304,145350 +19:53:30.829,799999,50382,124614,116778,196216,100827,166293,93061,155981,53794,132845,83304,147459 +19:53:30.891,806405,50382,124614,119002,198440,100827,166293,94005,157850,54603,134403,87654,151809 +19:53:30.979,806424,49309,123349,117581,194933,105190,170656,93846,159072,54672,135818,87654,153177 +19:53:31.042,815513,47866,119738,113605,189600,100909,168159,92494,156972,53925,134557,87253,152776 +19:53:31.104,815890,46878,114993,105965,181143,100909,165936,92637,157205,53878,134241,90371,154859 +19:53:31.225,824857,45938,114592,105654,179228,100602,165629,94352,158102,54602,133103,94074,161082 +19:53:31.295,831534,48117,116771,106942,181616,100602,165629,95656,160021,55331,134825,94074,163530 +19:53:31.343,832300,46573,115227,103490,177556,100602,165629,94048,158437,54402,134415,96386,162728 +19:53:31.410,839660,48419,118517,102504,175280,100602,166408,93152,156608,53717,133027,96386,160018 +19:53:31.467,844329,49866,122533,103658,177295,105952,171758,94430,158806,54448,134688,96386,160018 +19:53:31.563,850763,51344,125814,102574,177081,99152,167834,94461,159800,54512,135190,95085,161139 +19:53:31.624,853201,51344,125814,99547,173277,99152,167834,93227,157782,54021,132055,95085,158170 +19:53:31.673,858042,51344,125814,100490,175706,99152,167834,94494,159049,54797,134223,99088,162173 +19:53:31.735,861162,51344,123594,100219,173993,103518,170539,94547,159157,54702,134390,99088,164963 +19:53:31.881,864999,50235,123621,100288,175421,100314,170762,94675,159147,55197,134816,95976,159688 +19:53:31.941,871917,49504,122890,100669,173071,101787,170417,94110,158350,54333,133210,99584,163296 +19:53:31.983,871938,54533,127919,100669,174797,104736,173366,94110,159443,54333,134296,99584,164451 +19:53:32.092,881754,54354,129120,102554,176919,104736,172800,94984,159693,55094,134532,103054,169076 +19:53:32.131,882161,53237,127146,101037,175315,104736,172800,94797,159191,54173,133959,103054,169076 +19:53:32.241,891846,52664,126573,99526,173692,102568,170632,93346,157180,53962,132879,99414,165180 +19:53:32.303,899057,51643,124571,99605,175106,102568,170632,92811,157496,53387,131883,99725,165491 +19:53:32.362,899884,51643,124571,102950,178451,102568,173968,94069,159546,54158,132654,99725,165491 +19:53:32.423,907833,51821,128515,102699,178037,107602,173645,92976,156848,54437,131826,96710,165164 +19:53:32.481,912875,55493,133965,102699,179598,107602,173645,94226,158794,55366,133939,96710,165164 +19:53:32.600,919824,54922,131247,103915,179608,99650,167748,93211,156737,54445,132770,98174,165024 +19:53:32.662,922457,54922,134764,103915,181749,103774,171872,93211,157509,54445,133715,98174,165024 +19:53:32.720,927685,56653,136495,104560,183454,103774,170816,93048,156502,53560,131426,98174,162822 +19:53:32.792,931055,56653,136495,106096,184990,103774,173923,94438,158670,54342,133396,98174,165403 +19:53:32.943,942670,59682,138519,102717,181387,105501,175650,94322,157612,53776,132202,94128,161117 +19:53:32.988,942693,61179,142965,102717,182074,105501,175650,94322,158543,53776,133293,94128,161117 +19:53:33.151,953734,60781,142487,102026,179034,98986,166932,94288,157691,54018,131474,102355,169531 +19:53:33.232,953734,59678,144915,102141,179551,100558,165841,94434,158848,53994,132845,98360,165536 +19:53:33.248,964194,59678,144915,102141,179551,100558,165841,94434,158848,53994,132845,98360,165536 +19:53:33.356,971982,58504,143741,101784,179517,98130,166224,94632,158208,54034,131773,98360,165840 +19:53:33.407,972875,58096,141533,100754,178426,98130,166224,93410,156532,53883,131715,95426,160202 +19:53:33.459,981460,58096,144309,102960,180632,98130,166224,94432,158510,53883,132416,95426,160202 +19:53:33.514,986905,59461,143053,101107,180688,96387,160577,93363,157614,53363,131747,98669,163445 +19:53:33.665,997254,58466,144117,104235,184013,95358,159548,94747,159165,54830,134124,94008,160037 +19:53:33.756,1002900,58466,144117,104997,185455,95358,160065,94755,158355,54280,133883,90995,157024 +19:53:33.818,1006539,59634,142284,106286,187331,90275,151613,94803,158289,54261,132386,94051,160080 +19:53:33.966,1019084,62389,144303,108454,192028,94894,156232,94789,159205,54302,133328,92531,159376 +19:53:34.024,1019108,60763,142677,107690,191628,91709,153047,94696,158131,54211,132317,92531,156600 +19:53:34.156,1031033,60763,142677,111696,197182,90115,154448,94517,157917,54058,133407,96896,163346 +19:53:34.285,1042330,60763,142160,112330,199130,90115,150374,95960,160395,54880,133718,100059,166299 +19:53:34.385,1050741,59157,140554,112870,200387,91690,153647,94507,158957,54267,133184,100059,166299 +19:53:34.433,1051705,57784,136766,114226,202434,86819,144837,94549,158114,54342,132371,96115,160003 +19:53:34.606,1066857,57784,134893,111502,200662,88850,148911,94387,158165,54314,132522,92307,158929 +19:53:34.695,1078034,60859,137968,112958,203820,94686,154747,95578,161248,55098,134173,96486,163108 +19:53:34.757,1084132,59882,136991,114899,205113,89291,149352,94677,159135,54366,133792,96486,163108 +19:53:34.845,1088062,59882,136794,113954,202727,89291,152584,93411,157909,54328,133952,96486,163443 +19:53:35.053,1101637,60906,135423,114032,204966,90949,153751,93517,157928,54253,133167,92125,156711 +19:53:35.160,1114516,59027,132461,115136,206187,90949,149397,94541,158943,54181,132221,87304,150285 +19:53:35.309,1126716,59536,125134,114155,205153,89774,151001,93469,156664,53368,131286,87696,151085 +19:53:35.428,1135800,58682,120229,116261,205936,94939,154243,94206,158153,54022,132913,87696,147546 +19:53:35.465,1136841,58682,121215,116261,207223,96661,155965,94206,158153,54022,132913,92205,152055 +19:53:35.667,1153206,55960,117013,116405,206271,91741,151639,93984,157995,54864,134053,92238,153750 +19:53:35.695,1165277,55960,117013,118374,208240,95818,155716,93984,158800,54864,135094,92238,153750 +19:53:35.855,1176107,55960,114683,117251,207453,91477,154247,93717,158093,54051,133577,94958,158931 +19:53:36.075,1190768,57704,118149,118272,206595,92374,158704,95851,159761,55613,135069,90608,153754 +19:53:36.185,1200463,60526,122555,117186,206210,92374,155134,94672,158352,54818,134181,91190,156433 +19:53:36.280,1204677,60526,119884,119646,207368,94395,157155,95858,159693,55342,135088,88730,151453 +19:53:36.329,1217853,58347,117705,116046,204860,94395,159493,92978,156245,53926,132291,90477,150752 +19:53:36.430,1219552,62984,123742,116640,204339,96224,157790,94445,158789,54864,134344,87451,149597 +19:53:36.528,1228788,60246,118435,116779,204797,92702,154268,93269,156751,53941,132696,91626,153772 +19:53:36.680,1243052,60246,118385,118459,207096,96056,157754,95186,159760,54694,134790,91626,153944 +19:53:36.743,1259499,65934,125827,120010,207160,93366,155064,95198,159033,54619,134456,91808,154483 +19:53:36.885,1263888,67126,127019,122110,210856,98618,163645,96275,160865,55210,135260,95918,158593 +19:53:37.014,1271196,63488,124715,117933,206207,94195,159289,94886,158468,54626,133583,92310,154347 +19:53:37.086,1287029,63488,124715,120595,208869,94195,159289,96496,161067,55509,135407,94910,156947 +19:53:37.187,1290326,64632,123251,119806,210787,98918,160940,94699,159224,54720,134525,91440,155483 +19:53:37.255,1297500,64632,120683,120094,209840,94478,158672,94781,159095,54754,134343,94203,155171 +19:53:37.294,1302051,67622,127144,120094,210779,97766,163368,95838,161062,55562,136047,94203,157950 +19:53:37.340,1314481,68146,126338,120125,207168,97766,164777,95123,158664,54677,133490,90568,154315 +19:53:37.494,1318116,66654,123515,120790,208312,96981,162122,95286,160344,54390,135526,97394,160978 +19:53:37.545,1328091,62982,114299,117634,206024,96981,163589,95242,158884,54197,133039,97394,160978 +19:53:37.695,1343496,65088,117384,118328,206789,99843,166515,94910,160176,54227,134545,94384,159821 +19:53:37.744,1359859,64384,113490,118443,204763,95718,162390,94958,158531,54274,133647,94384,159821 +19:53:38.161,1382547,66011,109912,118850,205800,103030,168096,95046,159684,54418,135218,94718,159196 +19:53:38.364,1404316,70563,112808,119493,206773,98312,163424,95683,161195,54594,135904,94286,159520 +19:53:38.566,1426318,73011,116889,118973,204664,102245,169779,95554,160456,54666,136143,95333,162603 +19:53:38.817,1454138,79230,126192,117318,203633,104928,171187,93616,157458,53606,131996,98688,165506 +19:53:39.068,1469648,86901,139207,120147,207648,103301,172155,95013,159216,54566,134068,102230,168116 +19:53:39.173,1482500,90486,146968,120360,208789,104797,170656,94123,158075,54013,132727,97866,166199 +19:53:39.270,1494151,94232,153808,119864,207040,104797,173509,95814,159803,54798,133710,97866,163818 +19:53:39.372,1505727,97162,155768,120363,208563,104797,173509,94714,159293,54400,134168,94302,157726 +19:53:39.472,1517661,98587,159363,121490,209199,101629,168643,96891,161677,55546,135565,94302,160193 +19:53:39.574,1529200,101302,163361,123781,211639,99598,169481,95706,160557,54827,134118,94533,160031 +19:53:39.675,1541423,104590,169100,126757,216117,103917,168365,96322,161079,55302,134846,94533,157663 +19:53:39.827,1553397,106002,170636,127850,216168,98080,162173,94670,157995,54389,133424,93941,154339 +19:53:39.926,1571469,107611,173389,127851,216077,97990,162083,93454,156593,53424,131370,93822,157476 +19:53:40.079,1583696,106386,173751,131214,221252,96314,163303,95955,159861,54822,134220,93822,157176 +19:53:40.128,1588220,107667,175032,128424,219338,96314,161076,93427,156350,53462,131987,93822,157176 +19:53:40.178,1602100,109003,177307,130694,221608,96314,161076,94467,158283,54157,133443,97781,161135 +19:53:40.281,1608508,110261,180431,131906,224028,96611,162413,93632,157434,54030,132969,97781,163795 +19:53:46.433,1608508,86749,145183,121547,202952,91146,154527,92576,156246,54962,133903,93070,154227 diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..4e06e8c --- /dev/null +++ b/pom.xml @@ -0,0 +1,186 @@ + + + + 4.0.0 + + org.jitsi + jvb-parent + 2.3-SNAPSHOT + pom + + + 12.0.35 + 2.4.20 + 5.9.1 + 5.14.4 + 1.14.4 + 1.0-153-gd5c0102 + 1.1-179-g52cf6ef + 1.14.11 + 1.8.0 + 3.6.1 + 3.14.0 + 3.6.0 + 3.3.1 + 3.5.3 + 4.9.4 + 4.9.4.2 + 2.21.5 + 0.8.13 + 1.83 + 3.1.11 + 0.16.0 + UTF-8 + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.6.1 + + + enforce-maven + + enforce + + + + + 3.6.3 + + + + + + + + + + + rtp + jitsi-media-transform + jvb + + + + + + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + ${project.groupId} + jicoco-config + ${jicoco.version} + + + ${project.groupId} + jicoco-health-checker + ${jicoco.version} + + + ${project.groupId} + jicoco-jetty + ${jicoco.version} + + + ${project.groupId} + jicoco-metrics + ${jicoco.version} + + + ${project.groupId} + jicoco-mucclient + ${jicoco.version} + + + ${project.groupId} + jicoco-mediajson + ${jicoco.version} + + + ${project.groupId} + jicoco-tracing + ${jicoco.version} + + + ${project.groupId} + jicoco-test-kotlin + ${jicoco.version} + + + ${project.groupId} + jitsi-utils + ${jitsi.utils.version} + + + ${project.groupId} + jitsi-metaconfig + 1.0-18-ge06bc45 + + + ${project.groupId} + ice4j + 3.2-18-g3984f7f + + + ${project.groupId} + jitsi-xmpp-extensions + 1.0-124-g024f15d + + + + + + + coverage + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + prepare-agent + + + + + report + test + + report + + + + + + + + + diff --git a/resources/analyze-timeline.pl b/resources/analyze-timeline.pl new file mode 100755 index 0000000..a31cc39 --- /dev/null +++ b/resources/analyze-timeline.pl @@ -0,0 +1,42 @@ +#!/usr/bin/perl -w + +use Statistics::Descriptive; +use strict; + +my %stats; +my @stat_names; + +while (<>) { + if (/Reference time: /) { + my $prev_time; + while (/; \(([^)]*), PT([0-9.]*)/g) { + my $name = $1; + my $time = $2; + + my $delta_time; + if (defined $prev_time) { + $delta_time = $time - $prev_time; + } + else { + $delta_time = $time; + } + $prev_time = $time; + + if ($name =~ /Toggleable pcap writer:/) { + $name =~ s/writer: .*/writer/; + } + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($delta_time * 1e3); + } + } +} + +for my $name (@stat_names) { + my $s = $stats{$name}; + printf("%s: min %.3f ms, mean %.3f ms, median %.3f ms, 90%% %.3f ms, 99%% %.3f ms, max %.3f ms\n", + $name, $s->min(), $s->mean(), $s->median(), scalar($s->percentile(90)), scalar($s->percentile(99)), $s->max()); +} diff --git a/resources/analyze-timeline2.pl b/resources/analyze-timeline2.pl new file mode 100755 index 0000000..52d1fe1 --- /dev/null +++ b/resources/analyze-timeline2.pl @@ -0,0 +1,244 @@ +#!/usr/bin/perl -w + +use Statistics::Descriptive; +use strict; + +my %stats; +my @stat_names; + +=begin comment + +This is extremely ugly, I'm driving perl in first gear because I don't know how to shift :) + +It parses the PacketInfo EventTimeline logs (see example below) and produces stats for +the following periods from the life cycle of a packet: + +1.Receiver Queue: From the moment the packet is read from the ICE transport (in an IO thread) and added the the Receiver Queue +to the moment it is removed from the Receiver Queue (in a CPU thread) and the pipeline processing starts. + +2.Receiver Pipeline: From the start of the Receiver Pipeline to the end, excluding the last Termination node. This is all +performed in the same CPU thread, and includes things like SRTP decryption/authentication, RTP parsing, updating bandwidth +estimation, video codec parsing, etc. + +3.Termination Node: This is technically the last Node in the Receiver Pipeline and executes in the same thread, but +it is conceptually different and worth looking at separately. It consists of conference-level handling of the packet: it loops +through all local endpoints and relays, checks whether they "want" the packet, and if they do then clones the packet and puts +the clone on the sender queue (see Conference#sendOut). This workload scales with the number of local endpoits and relays, +and in a large conference is the most computationally expensive. + +4.Sender Queue: From the time the Receiver Pipeline thread places the packet on the Sender Queue, to the time another CPU thread +removes it from the queue and starts executing the Sender Pipeline. + +5.Sender Pipeline: From the start of the Sender Pipeline to the end. This executes in a CPU thread and includes stripping unknown (for +this specific receiving endpoint) RTP header extensions, saving a copy of the packet in the NACK cache, adding/updating the transport-cc +header extension, and SRTP encryption. + +This script looks only at RTP packets (ignoring RTCP and retransmissions) for simplicity. + +6.SRTP Queue: From the time a SRTP packet is placed on the queue by the Sender Pipeline thread to the time an IO thread removes it +from the queue and starts sending it over the ICE transport. + +7.Send over ICE transport: The time it took to execute IceTransport.send, which sends the packet through the ice4j socket representation. + +8.Total: From the start of 1. to the end of 7. + + + +Example log line: +JVB 2024-06-11 17:57:44.989 FINER: [1531] [confId=6149c0c339432f97 conf_name=loadtest0@conference.xxx meeting_id=ec7261c3 epId=c88aca35 stats_id=Otto-emV] Endpoint.doSendSrtp#544: Reference time: 2024-06-11T17:57:44.988517466Z; (Entered RTP receiver incoming queue, PT0.00000192S); (Exited RTP receiver incoming queue, PT0.000324242S); (Entered node PacketStreamStats, PT0.000324602S); (Exited node PacketStreamStats, PT0.000325042S); (Entered node SRTP/SRTCP demuxer, PT0.000325162S); (Exited node SRTP/SRTCP demuxer, PT0.000325722S); (Entered node RTP Parser, PT0.000325962S); (Exited node RTP Parser, PT0.000326762S); (Entered node Audio level reader (pre-srtp), PT0.000326922S); (Exited node Audio level reader (pre-srtp), PT0.000327242S); (Entered node Video mute node, PT0.000327362S); (Exited node Video mute node, PT0.000327642S); (Entered node SRTP Decrypt Node, PT0.000327762S); (Exited node SRTP Decrypt Node, PT0.000332442S); (Entered node TCC generator, PT0.000332562S); (Exited node TCC generator, PT0.000333562S); (Entered node Remote Bandwidth Estimator, PT0.000344962S); (Exited node Remote Bandwidth Estimator, PT0.000345202S); (Entered node Audio level reader (post-srtp), PT0.000345322S); (Exited node Audio level reader (post-srtp), PT0.000345442S); (Entered node Toggleable pcap writer: 21913831-rx, PT0.000345562S); (Exited node Toggleable pcap writer: 21913831-rx, PT0.000345762S); (Entered node Incoming statistics tracker, PT0.000345882S); (Exited node Incoming statistics tracker, PT0.000351242S); (Entered node Padding termination, PT0.000351362S); (Exited node Padding termination, PT0.000351522S); (Entered node Media Type demuxer, PT0.000351642S); (Exited node Media Type demuxer, PT0.000351962S); (Entered node RTX handler, PT0.000352082S); (Exited node RTX handler, PT0.000352322S); (Entered node Duplicate termination, PT0.000352722S); (Exited node Duplicate termination, PT0.000355442S); (Entered node Retransmission requester, PT0.000355882S); (Exited node Retransmission requester, PT0.000357922S); (Entered node Padding-only discarder, PT0.000358642S); (Exited node Padding-only discarder, PT0.000359842S); (Entered node Video parser, PT0.000360242S); (Exited node Video parser, PT0.000363082S); (Entered node Video quality layer lookup, PT0.000363362S); (Exited node Video quality layer lookup, PT0.000364362S); (Entered node Video bitrate calculator, PT0.000364682S); (Exited node Video bitrate calculator, PT0.000369122S); (Entered node Input pipeline termination node, PT0.000369362S); (Entered node receiver chain handler, PT0.000370122S); (Entered RTP sender incoming queue, PT0.000488083S); (Exited RTP sender incoming queue, PT0.000551283S); (Entered node Pre-processor, PT0.000551723S); (Exited node Pre-processor, PT0.000555843S); (Entered node RedHandler, PT0.000556083S); (Exited node RedHandler, PT0.000556563S); (Entered node Strip header extensions, PT0.000556843S); (Exited node Strip header extensions, PT0.000557523S); (Entered node Packet cache, PT0.000557923S); (Exited node Packet cache, PT0.000561083S); (Entered node Absolute send time, PT0.000561243S); (Exited node Absolute send time, PT0.000561643S); (Entered node Outgoing statistics tracker, PT0.000561923S); (Exited node Outgoing statistics tracker, PT0.000562683S); (Entered node TCC sequence number tagger, PT0.000563003S); (Exited node TCC sequence number tagger, PT0.000563843S); (Entered node Header extension encoder, PT0.000564083S); (Exited node Header extension encoder, PT0.000565163S); (Entered node Toggleable pcap writer: c88aca35-tx, PT0.000565443S); (Exited node Toggleable pcap writer: c88aca35-tx, PT0.000565763S); (Entered node SRTP Encrypt Node, PT0.000566083S); (Exited node SRTP Encrypt Node, PT0.000572883S); (Entered node PacketStreamStats, PT0.000573243S); (Exited node PacketStreamStats, PT0.000573883S); (Entered node Output pipeline termination node, PT0.000574123S); (Entered Endpoint SRTP sender outgoing queue, PT0.000574443S); (Exited node Output pipeline termination node, PT0.000582923S); (Exited Endpoint SRTP sender outgoing queue, PT0.000668003S); (Sent over the ICE transport, PT0.000693284S) + +=end comment + +=cut + +my $receiverQStartName = "Entered RTP receiver incoming queue"; +my $receiverQEndName = "Exited RTP receiver incoming queue"; +my $receiverPStartName = "Entered node SRTP/SRTCP demuxer"; +my $receiverPEndName = "Entered node Input pipeline termination node"; +my $terminationNodeStartName = "Entered node Input pipeline termination node"; +my $terminationNodeEndName = "Exited node Input pipeline termination node"; +my $senderQStartName = "Entered RTP sender incoming queue"; +my $senderQEndName = "Exited RTP sender incoming queue"; +my $senderPStartName = "Entered node RedHandler"; +my $senderPEndName = "Entered Endpoint SRTP sender outgoing queue"; +my $srtpQStartName = "Entered Endpoint SRTP sender outgoing queue"; +my $srtpQEndName = "Exited Endpoint SRTP sender outgoing queue"; +my $iceStartName = "Exited Endpoint SRTP sender outgoing queue"; +my $iceEndName = "Sent over the ICE transport"; +my $totalStartName = "Entered RTP receiver incoming queue"; +my $totalEndName = "Sent over the ICE transport"; + +while (<>) { + if (/Reference time: /) { + my $prev_time; + my $receiverQStart = -1; + my $receiverQEnd = -1; + my $receiverPStart = -1; + my $receiverPEnd = -1 ; + my $terminationNodeStart = -1 ; + my $terminationNodeEnd = -1; + my $senderQStart = -1; + my $senderQEnd = -1; + my $senderPStart = -1; + my $senderPEnd = -1 ; + my $srtpQStart = -1; + my $srtpQEnd = -1; + my $iceStart = -1; + my $iceEnd = -1; + my $totalStart = -1; + my $totalEnd = -1; + + while (/; \(([^)]*), PT([0-9.]*)/g) { + my $name = $1; + my $time = $2; + + if ($name eq $receiverQStartName) { + $receiverQStart = $time; + } + if ($name eq $receiverQEndName) { + $receiverQEnd = $time; + } + if ($name eq $receiverPStartName) { + $receiverPStart = $time; + } + if ($name eq $receiverPEndName) { + $receiverPEnd = $time; + } + if ($name eq $terminationNodeStartName) { + $terminationNodeStart = $time; + } + if ($name eq $terminationNodeEndName) { + $terminationNodeEnd = $time; + } + if ($name eq $senderQStartName) { + $senderQStart = $time; + } + if ($name eq $senderQEndName) { + $senderQEnd = $time; + } + if ($name eq $senderPStartName) { + $senderPStart = $time; + } + if ($name eq $senderPEndName) { + $senderPEnd = $time; + } + if ($name eq $srtpQStartName) { + $srtpQStart = $time; + } + if ($name eq $srtpQEndName) { + $srtpQEnd = $time; + } + if ($name eq $iceStartName) { + $iceStart = $time; + } + if ($name eq $iceEndName) { + $iceEnd = $time; + } + if ($name eq $totalStartName) { + $totalStart = $time; + } + if ($name eq $totalEndName) { + $totalEnd = $time; + } + + #my $delta_time; + #if (defined $prev_time) { + # $delta_time = $time - $prev_time; + #} + #else { + # $delta_time = $time; + #} + #$prev_time = $time; + + #if ($name =~ /Toggleable pcap writer:/) { + # $name =~ s/writer: .*/writer/; + #} + } + if ($receiverQStart > -1 && $receiverQEnd > -1) { + my $name = "1.Receiver Queue"; + my $time = $receiverQEnd - $receiverQStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($receiverPStart > -1 && $receiverPEnd > -1) { + my $name = "2.Receiver Pipeline"; + my $time = $receiverPEnd - $receiverPStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($terminationNodeStart > -1 && $terminationNodeEnd > -1) { + my $name = "3.Termination Node"; + my $time = $terminationNodeEnd - $terminationNodeStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($senderQStart > -1 && $senderQEnd > -1) { + my $name = "4.Sender Queue"; + my $time = $senderQEnd - $senderQStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($senderPStart > -1 && $senderPEnd > -1) { + my $name = "5.Sender Pipeline"; + my $time = $senderPEnd - $senderPStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($srtpQStart > -1 && $srtpQEnd > -1) { + my $name = "6.SRTP Queue"; + my $time = $srtpQEnd - $srtpQStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($iceStart > -1 && $iceEnd > -1) { + my $name = "7.Send over ICE transport"; + my $time = $iceEnd - $iceStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + if ($totalStart > -1 && $totalEnd > -1) { + my $name = "8.Total"; + my $time = $totalEnd - $totalStart; + + if (!defined ($stats{$name})) { + $stats{$name} = Statistics::Descriptive::Full->new(); + push(@stat_names, $name); + } + $stats{$name}->add_data($time * 1e3); + } + } +} + +for my $name (@stat_names) { + my $s = $stats{$name}; + printf("%s: min %.3f ms, mean %.3f ms, median %.3f ms, 90%% %.3f ms, 99%% %.3f ms, max %.3f ms\n", + $name, $s->min(), $s->mean(), $s->median(), scalar($s->percentile(90)), scalar($s->percentile(99)), $s->max()); +} diff --git a/resources/build.sh b/resources/build.sh new file mode 100755 index 0000000..2b61a5b --- /dev/null +++ b/resources/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +mvn clean verify package "$@" diff --git a/resources/build_deb_package.sh b/resources/build_deb_package.sh new file mode 100755 index 0000000..9df5588 --- /dev/null +++ b/resources/build_deb_package.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -x +set -e + +# Make sure you have the following environment variables set, example: +# export DEBFULLNAME="Jitsi Team" +# export DEBEMAIL="dev@jitsi.org" +# You need package devscripts installed (command dch) and libxml2-utils(provides xmllint), dh-systemd. +# +# There is an optional param that can be used to add an extra string to the version. +# That string can be used to mark custom version passing "hf" as param +# will result debian version 2.1-123-gabcabcab-hf-1, instead of just 2.1-123-gabcabcab-1 +VER_EXTRA_LABEL=$1 + +echo "===================================================================" +echo " Building DEB packages... " +echo "===================================================================" + +SCRIPT_FOLDER=$(dirname "$0") +cd "$SCRIPT_FOLDER/.." + +# Let's get version from maven +MVNVER=$(xmllint --xpath "/*[local-name()='project']/*[local-name()='version']/text()" pom.xml) +TAG_NAME="v${MVNVER/-SNAPSHOT/}" + +echo "Current tag name: $TAG_NAME" + +if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1 +then + git tag -a "$TAG_NAME" -m "Tagged automatically by Jenkins" + git push origin "$TAG_NAME" +else + echo "Tag: $TAG_NAME already exists." +fi + +VERSION_FULL=$(git describe --match "v[0-9\.]*" --long) +echo "Full version: ${VERSION_FULL}" + +export VERSION=${VERSION_FULL:1} +if [ -n "${VER_EXTRA_LABEL}" ] +then + VERSION+="-${VER_EXTRA_LABEL}" +fi +echo "Package version: ${VERSION}" + +REV=$(git log --pretty=format:'%h' -n 1) +dch -v "$VERSION-1" "Build from git. $REV" +dch -D unstable -r "" + +# sets the version in the pom file so it will propagate to resulting jar +mvn versions:set -DnewVersion="${VERSION}" + +# We need to make sure all dependencies are downloaded before start building +# the debian package +mvn package + +# now build the deb +dpkg-buildpackage -tc -us -uc -A + +# clean the current changes as dch had changed the change log +git checkout debian/changelog +git checkout pom.xml + +echo "Here are the resulting files in $(pwd ..)" +echo "-----" +ls -l ../{*.changes,*.deb,*.buildinfo} +echo "-----" + +# Let's try deploying +cd .. +([ ! -x deploy.sh ] || ./deploy.sh "jvb" $VERSION ) diff --git a/resources/collect-dump-logs.sh b/resources/collect-dump-logs.sh new file mode 100755 index 0000000..41ad673 --- /dev/null +++ b/resources/collect-dump-logs.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# script that creates an archive in current folder +# containing the heap and thread dump and the current log file + +JVB_HEAPDUMP_PATH="/tmp/java_*.hprof" +STAMP=`date +%Y-%m-%d-%H%M` +PID_PATH="/var/run/jitsi-videobridge.pid" +JVB_UID=`id -u jvb` + +RUNNING="" +unset PID + +# for systemd we use different pid file +if [ ! -f $PID_PATH ]; then + PID_PATH="/var/run/jitsi-videobridge/jitsi-videobridge.pid" +fi + +[ -e $PID_PATH ] && PID=$(cat $PID_PATH) +if [ ! -z $PID ]; then + ps -p $PID | grep -q java + [ $? -eq 0 ] && RUNNING="true" +fi + +#Find any crashes in /var/crash from our user in the past 20 minutes, if they exist +CRASH_FILES=$(find /var/crash -name '*.crash' -uid $JVB_UID -mmin -20 -type f) + +if [ ! -z $RUNNING ]; then + echo "Jvb at pid $PID" + THREADS_FILE="/tmp/stack-${STAMP}-${PID}.threads" + HEAP_FILE="/tmp/heap-${STAMP}-${PID}.bin" + + # If the JVB has crashed (with OOM, for example), the JVM might be + # configured to take a heap dump. This is configured with the HeapDumpPath + # VM option. The HeapDumpPath is a manageable VM option and the default + # path format is java_pidXXXX.hprof. + # + # During the time the JVM is taking a heap dump, the process will appear + # running, but the JVM won't allow us to connect to it and take a + # heap/thread dump. Briefly, we want to make sure we grab the java heap + # dump, even if the process appears to be running. + + HEAPDUMP_FILE="/tmp/java_pid${PID}.hprof" + + # Wait for the JVM to finish writing the heapdump. + while lsof ${HEAPDUMP_FILE}; do sleep 1; done + + sudo jstack ${PID} > ${THREADS_FILE} + sudo jmap -dump:live,format=b,file=${HEAP_FILE} ${PID} + + tar zcvf jvb-dumps-${STAMP}-${PID}.tgz ${THREADS_FILE} ${HEAP_FILE} ${HEAPDUMP_FILE} ${CRASH_FILES} /var/log/jitsi/jvb.log /tmp/hs_err_* + rm ${HEAP_FILE} ${THREADS_FILE} +else + ls $JVB_HEAPDUMP_PATH >/dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "JVB not running, but previous heap dump found." + tar zcvf jvb-dumps-${STAMP}-crash.tgz $JVB_HEAPDUMP_PATH ${CRASH_FILES} /var/log/jitsi/jvb.log /tmp/hs_err_* + rm ${JVB_HEAPDUMP_PATH} + else + tar zcvf jvb-dumps-${STAMP}-crash.tgz /var/log/jitsi/jvb.log ${CRASH_FILES} /tmp/hs_err_* + echo "JVB not running, previous heap dump not found. Archiving logs only." + fi +fi diff --git a/resources/graceful_shutdown.sh b/resources/graceful_shutdown.sh new file mode 100755 index 0000000..8a99ef0 --- /dev/null +++ b/resources/graceful_shutdown.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# 1. The script issues shutdown command to the bridge over REST API. +# If HTTP status code other than 200 is returned then it exits with 1. +# 2. If the code is ok then it checks if the bridge has exited. +# 3. If not then it polls bridge statistics until participant count drops to 0. +# 4. Gives some time for the bridge to shutdown. If it does not quit after that +# time then it kills the process. If the process was successfully killed 0 is +# returned and 1 otherwise. +# +# Arguments: +# "-h"("http://localhost:8080" by default) REST requests host URI part +# "-s"(disabled by default) enable silent mode - no info output +# +# NOTE: script depends on the tool jq, used to parse json +# + +# Initialize arguments +hostUrl="http://localhost:8080" +verbose=1 + +# Parse arguments +OPTIND=1 +while getopts "p:h:t:s" opt; do + case "$opt" in + h) + hostUrl=$OPTARG + ;; + s) + verbose=0 + ;; + esac +done +shift "$((OPTIND-1))" + +# Returns local participant count by calling JVB REST statistics API and extracting +# participant count from JSON stats text returned. +function getParticipantCount { + # Total number of participants minus the remote (octo) participants + curl -s "$hostUrl/colibri/stats"| jq '.participants - .octo_endpoints' +} + +# Prints info messages +function printInfo { + if [ "$verbose" == "1" ] + then + echo "$@" + fi +} + +# Prints errors +function printError { + echo "$@" 1>&2 +} + +function waitForPid { + while [ -d /proc/$1 ] ;do + echo "PID $1 still exists" + sleep 10 + done + echo "PID $1 is done" +} + +JVB_PID=$(ps aux | grep java | grep jitsi-videobridge.jar | awk '{print $2}') + +shutdownStatus=`curl -s -o /dev/null -H "Content-Type: application/json" -d '{ "graceful-shutdown": "true" }' -w "%{http_code}" "$hostUrl/colibri/shutdown"` +if [ "$shutdownStatus" == "200" ] +then + printInfo "Graceful shutdown started" + participantCount=`getParticipantCount` + while [[ $participantCount -gt 0 ]] ; do + printInfo "There are still $participantCount participants" + sleep 10 + participantCount=`getParticipantCount` + if [[ $? -gt 0 ]] ; then + printInfo "Failed to get participant count, bridge may be already shutdown, waiting on pid $JVB_PID" + waitForPid $JVB_PID + exit 0 + fi + done + + echo "Waiting for bridge pid $JVB_PID to finish shutting down" + waitForPid $JVB_PID + printInfo "Bridge shutdown OK" + exit 0 +else + printError "Invalid HTTP status for shutdown request: $shutdownStatus" + exit 1 +fi + + diff --git a/resources/jenkins/build.sh b/resources/jenkins/build.sh new file mode 100755 index 0000000..c207037 --- /dev/null +++ b/resources/jenkins/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +./resources/build.sh diff --git a/resources/link-conditioner.sh b/resources/link-conditioner.sh new file mode 100755 index 0000000..46cc6cd --- /dev/null +++ b/resources/link-conditioner.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +PLR=${PLR:-'0.1'} +DELAY=${DELAY:-100} +BW=${BW:-'1Mbit/s'} +PORT=${PORT:-10000} +EGRESS_PIPE=1 +INGRESS_PIPE=2 + +condition_egress() { + # Create an anchor and reload pf conf. + (cat /etc/pf.conf && echo "dummynet-anchor \"jitsi-videobridge\"" && echo "anchor \"jitsi-videobridge\"") | sudo pfctl -f - + + # Pipe all UDP traffic from port PORT through the EGRESS_PIPE. + echo "dummynet out quick proto udp from any port $PORT to any pipe $EGRESS_PIPE" | sudo pfctl -a jitsi-videobridge -f - + + # Configure the egress pipe. + sudo dnctl pipe $EGRESS_PIPE config bw $BW plr $PLR delay $DELAY +} + +condition_ingress() { + # Create an anchor and reload pf conf. + (cat /etc/pf.conf && echo "dummynet-anchor \"jitsi-videobridge\"" && echo "anchor \"jitsi-videobridge\"") | sudo pfctl -f - + + # Pipe all UDP traffic to port PORT through the INGRESS_PIPE. + echo "dummynet out quick proto udp from any to any port $PORT pipe $INGRESS_PIPE" | sudo pfctl -a jitsi-videobridge -f - + + # Configure the ingress pipe. + sudo dnctl pipe $INGRESS_PIPE config bw $BW plr $PLR delay $DELAY +} + +condition_flush() { + sudo dnctl -f flush + sudo pfctl -f /etc/pf.conf +} + +main() { + case "$1" in + ingress) + condition_ingress + ;; + egress) + condition_egress + ;; + flush) + condition_flush + ;; + esac +} + +main "$@" diff --git a/resources/mvn-exec.sh b/resources/mvn-exec.sh new file mode 100755 index 0000000..ba433d2 --- /dev/null +++ b/resources/mvn-exec.sh @@ -0,0 +1,11 @@ +#!/bin/sh -e + +readonly CONFIG=$1 +if [ -z ${CONFIG+x} -o ! -f $1 ]; then + echo 'Config file missing.' + exit 1 +fi + +. $CONFIG + +exec mvn compile exec:exec -Dexec.executable=java -Dexec.args="-cp %classpath ${JAVA_SYS_PROPS} org.jitsi.videobridge.Main ${JVB_OPTS}" diff --git a/resources/timeseries-cli.py b/resources/timeseries-cli.py new file mode 100755 index 0000000..60f602b --- /dev/null +++ b/resources/timeseries-cli.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python + +import argparse +import pandas as pd +import matplotlib.pyplot as plt +from pandas.plotting import register_matplotlib_converters +register_matplotlib_converters() + + +def read_json(fname): + df = pd.read_json(fname, lines=True, convert_dates=['time'], date_unit='ms') + df['conference_id'] = df['conf_name'] + df['conf_creation_time_ms'].astype(str) + return df + + +def show(args): + df = read_json(args.infile) + print('series: {}'.format(df['series'].unique())) + print('conferences: {}'.format(df['conf_name'].unique())) + print('endpoints: {}'.format(df['endpoint_id'].unique())) + + +def vp8_inspect(df): + if args.ssrc: + df = df[df['rtp.ssrc'] == args.ssrc] + if args.endpoint: + df = df[df['endpoint_id'] == args.endpoint] + if args.conference: + df = df[df['conf_name'] == args.conference] + + df['time_delta'] = df['time'] - df['time'].shift(1) + df['rtp.seq_delta'] = df['rtp.seq'] - df['rtp.seq'].shift(1) + df['vp8.pictureid_delta'] = df['vp8.pictureid'] - df['vp8.pictureid'].shift(1) + df['vp8.timestamp_delta'] = df['rtp.timestamp'] - df['rtp.timestamp'].shift(1) + print(df) + + +def vp8_verify(df): + # tl0picidx monotonically increases + # if the timestamp changes, then the pictureid changes + # if the pictureid changes, then the timestamp changes + # the timestamp delta needs to be proportional to the pictureid delta + # there can't be big gaps in the sequence numbers + # the webrtc pacer outputs packets every 10ms + pass + + +def plot_endpoint(df, series, endpoint_id, remote_endpoint_id): + df = df[df['endpoint_id'] == endpoint_id] + if remote_endpoint_id: + df = df[df['remote_endpoint_id'] == remote_endpoint_id] + + fig = plt.figure() + fig.suptitle('Endpoint: {}'.format(endpoint_id)) + ax_bitrate = fig.subplots(1, sharex=True) + ax_bitrate.set_xlabel('time') + ax_bitrate.set_ylabel('bitrate (bps)') + + if 'calculated_rate' in series: + df_rates = df['series' == 'calculated_rate'] + + # make sure we're plotting a single remote endpoint. + remote_endpoints = df_rates['remote_endpoint_id'].unique() + if len(remote_endpoints) > 1: + raise Exception('specify a --remote-endpoint {}'.format(remote_endpoints)) + + for i in range(9): + encoding_quality = str(i) + if encoding_quality in df_rates.columns: + ax_bitrate.plot(df_rates['time'], + df_rates[encoding_quality], + label=encoding_quality) + + df_series = df['series'].unique() + if 'sent_padding' in series and 'sent_padding' in df_series: + df_padding = df[df['series'] == 'sent_padding'] + ax_bitrate.plot( + df_padding['time'], + df_padding['padding_bps'] + df_padding['total_target_bps'], + label='target + padding', + marker='^', + markersize=4, + linestyle='None') + + if 'new_bwe' in series and 'new_bwe' in df_series: + df_bwe = df[df['series'] == 'new_bwe'] + ax_bitrate.plot(df_bwe['time'], + df_bwe['bitrate_bps'], + label='estimate', + drawstyle='steps-post') + + if 'did_update' in series and 'did_update' in df_series: + df_update = df[df['series'] == 'did_update'] + ax_bitrate.plot(df_update['time'], + df_update['total_target_bps'], + label='target', + drawstyle='steps-post') + ax_bitrate.plot(df_update['time'], + df_update['total_ideal_bps'], + label='ideal', + drawstyle='steps-post') + + if 'in_pkt' in series and 'in_pkt' in df_series: + df_pkt = df[df['series'] == 'in_pkt'] + + if len(df_pkt['rbe_id'].unique()) is not 1: + raise Exception('There cannot be multiple remote bitrate estimators') + + df_sz = pd.DataFrame( + {'bits': df_pkt['pkt_sz_bytes'] * 8, 'time': df_pkt['time']}) + + df_rate = df_sz.resample('1S', on='time').sum() + ax_bitrate.plot(df_rate.index, df_rate['bits'], label='sent') + + # todo include rtt and packet loss + + ax_bitrate.legend() + + +def plot(args): + df = read_json(args.infile) + + endpoint_ids = [args.endpoint_id] if args.endpoint_id else df['endpoint_id'].unique() + + for endpoint_id in endpoint_ids: + plot_endpoint(df, args.series, endpoint_id, args.remote_endpoint_id) + + plt.show() + + +def check_endpoint(df, endpoint_id): + + # if a property is violated (for example property 3): + # zgrep 7d2b7ecf data/4/series.json.gz| grep did_update | jq '. | select(.bwe_bps >= .total_ideal_bps) | select(.total_target_idx < .total_ideal_idx)' | less -S + + df_update = df[df['series'] == 'did_update'] + if not len(df_update): + return + + # property 1: check if ideal <= estimate => target == ideal + mask_1 = df_update['bwe_bps'] >= df_update['total_ideal_bps'] + mask_2 = df_update['total_target_idx'] < df_update['total_ideal_idx'] + mask_3 = df_update['total_ideal_bps'] != 0 + mask_4 = df_update['total_target_bps'] != 0 + assert not len(df_update[mask_1 & mask_2 & mask_3 & mask_4]), '{} is not sending as much as it should.'.format(endpoint_id) + + # property 2: check that the target never exceeds the ideal + mask_5 = (df_update['total_ideal_bps'] - df_update['total_target_bps']) < 0 + assert not len(df_update[mask_5]), '{} has a target bitrate that exceeds the ideal bitrate.'.format(endpoint_id) + + # property 3: make sure that we are calculating bitrate for encodings + # disabled for now as it's a legitimate case, being video muted, for example + #mask_6 = df_update['total_ideal_bps'] > 0 + #assert len(df_update[mask_6]), '{} never computed an ideal bitrate.'.format(endpoint_id) + + #mask_7 = df_update['total_target_bps'] > 0 + #assert len(df_update[mask_7]), '{} never computed a target bitrate.'.format(endpoint_id) + + # property 4: check that the target never exceeds the estimate + # disabled for now as can happen and still be correct because we never suspend the on-stage participant + # mask_8 = (df_update['bwe_bps'] - df_update['total_target_bps']) < 0 + # assert not len(df_update[mask_8]), '{} has a target bitrate that exceeds the bandwidth estimation.'.format(endpoint_id) + + +def check_conference(df, conference_id): + endpoint_ids = [args.endpoint_id] if args.endpoint_id else df['endpoint_id'].unique() + + for endpoint_id in endpoint_ids: + check_endpoint(df[df['endpoint_id'] == endpoint_id], endpoint_id) + + +def check(args): + df = read_json(args.infile) + conference_ids = [args.conference_id] if args.conference_id else df['conference_id'].unique() + + for conference_id in conference_ids: + check_conference(df[df['conference_id'] == conference_id], conference_id) + + +if "__main__" == __name__: + parser = argparse.ArgumentParser() + parser.add_argument('infile', type=argparse.FileType('r')) + subparsers = parser.add_subparsers() + + parser_show = subparsers.add_parser('show') + parser_show.set_defaults(func=show) + + parser_plot = subparsers.add_parser('plot') + parser_plot.add_argument( + '--series', nargs='+', choices=[ + 'did_update', 'new_bwe', 'sent_padding', 'calculated_rate'], + default='did_update new_bwe sent_padding in_pkt') + parser_plot.add_argument('--endpoint-id') + parser_plot.add_argument('--remote-endpoint-id') + parser_plot.set_defaults(func=plot) + + parser_check = subparsers.add_parser('check') + parser_check.add_argument('--endpoint-id') + parser_check.add_argument('--conference-id') + parser_check.set_defaults(func=check) + + args = parser.parse_args() + args.func(args) diff --git a/rtp/README.md b/rtp/README.md new file mode 100644 index 0000000..70241a3 --- /dev/null +++ b/rtp/README.md @@ -0,0 +1,21 @@ +# Jitsi RTP +Jitsi RTP contains classes for parsing and creating RTP and RTCP packets. + +# Code style +We use [ktlint](https://pinterest.github.io/ktlint/) for linting and autoformatting. It runs as part of +`mvn verify` (the build fails on style violations), and the version used is pinned by `ktlint.version` in the +root `pom.xml`, so no separate installation is needed. + +To run only the check, or to autoformat, run these in this module's directory: +``` +mvn exec:exec@ktlint-check +mvn exec:exec@ktlint-format +``` + +If you also want the `ktlint` command-line tool (for editor integration or a git hook), install a version matching +`ktlint.version`, e.g. on macOS with Homebrew: +``` +brew install ktlint +``` +It can then be run directly with `ktlint` (or `ktlint -F` to autoformat), install a pre-commit hook with +`ktlint installGitPreCommitHook`, and configure IntelliJ IDEA with `ktlint applyToIDEAProject`. diff --git a/rtp/pom.xml b/rtp/pom.xml new file mode 100644 index 0000000..1eff511 --- /dev/null +++ b/rtp/pom.xml @@ -0,0 +1,313 @@ + + + 4.0.0 + + + org.jitsi + jvb-parent + 2.3-SNAPSHOT + + + org.jitsi + rtp + + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-reflect + ${kotlin.version} + + + org.jitsi + jitsi-utils + ${jitsi.utils.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs.version} + + + + + io.kotest + kotest-runner-junit5-jvm + ${kotest.version} + test + + + io.kotest + kotest-assertions-core-jvm + ${kotest.version} + test + + + io.pkts + pkts-core + 3.0.10 + test + + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.2 + test + + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + compile + compile + + + ${project.basedir}/src/main/kotlin + + + + + test-compile + test-compile + + + ${project.basedir}/src/test/kotlin + + + + + + 17 + + + + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper-maven-plugin.version} + + + generate-sources + + add-source + + + + ${project.basedir}/src/main/kotlin + + + + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + verify + + jar + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + 17 + 17 + + + + + default-compile + none + + + + default-testCompile + none + + + java-compile + compile + compile + + + java-test-compile + test-compile + testCompile + + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + true + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.6.3 + + + com.pinterest.ktlint + ktlint-cli + ${ktlint.version} + all + + + + + ktlint-check + verify + + exec + + + java + true + + -classpath + + com.pinterest.ktlint:ktlint-cli + + com.pinterest.ktlint.Main + --relative + src/main/kotlin/**/*.kt + src/test/kotlin/**/*.kt + + + + + ktlint-format + + exec + + + java + true + + -classpath + + com.pinterest.ktlint:ktlint-cli + + com.pinterest.ktlint.Main + --format + --relative + src/main/kotlin/**/*.kt + src/test/kotlin/**/*.kt + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + ${project.basedir}/spotbugs-exclude.xml + true + + + + verify + check + + + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + + + + + jitsi-maven-repository-releases + default + Jitsi Maven Repository (Releases) + + true + + + false + + https://github.com/jitsi/jitsi-maven-repository/raw/master/releases/ + + + jitsi-maven-repository-snapshots + default + Jitsi Maven Repository (Snapshots) + + false + + + true + + https://github.com/jitsi/jitsi-maven-repository/raw/master/snapshots/ + + + + + coverage + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + prepare-agent + + + + + report + test + + report + + + + + + + + + diff --git a/rtp/spotbugs-exclude.xml b/rtp/spotbugs-exclude.xml new file mode 100644 index 0000000..bd130ea --- /dev/null +++ b/rtp/spotbugs-exclude.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/ByteArrayBufferImpl.java b/rtp/src/main/kotlin/org/jitsi/rtp/ByteArrayBufferImpl.java new file mode 100644 index 0000000..488e7c3 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/ByteArrayBufferImpl.java @@ -0,0 +1,171 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.jitsi.rtp.util.*; +import org.jitsi.utils.*; + +//TODO documentation +@SuppressFBWarnings({"PA_PUBLIC_PRIMITIVE_ATTRIBUTE"}) +public abstract class ByteArrayBufferImpl + implements ByteArrayBuffer +{ + + public byte[] buffer; + + public int offset; + + public int length; + + public ByteArrayBufferImpl(byte[] buffer, int offset, int length) + { + this.buffer = buffer; + this.offset = offset; + this.length = length; + } + + public ByteArrayBufferImpl() + { + this.buffer = new byte[0]; + this.offset = 0; + this.length = 0; + } + + public byte[] getBuffer() + { + return buffer; + } + + public int getOffset() + { + return offset; + } + + public void setOffset(int offset) + { + //TODO check bounds? + this.offset = offset; + } + + public int getLength() + { + return length; + } + + public void setLength(int length) + { + //TODO check bounds? + this.length = length; + } + + public void setOffsetLength(int offset, int length) + { + //TODO check bounds? + this.offset = offset; + this.length = length; + } + + public void readRegionToBuff(int off, int len, byte[] outBuff) + { + int startOffset = this.offset + off; + if (off < 0 || len <= 0 || startOffset + len > this.buffer.length) + return; + + if (outBuff.length < len) + return; + + System.arraycopy(this.buffer, startOffset, outBuff, 0, len); + } + + /** + * Append a byte array to the end of the packet. This may change the data + * buffer of this packet. + * + * @param data byte array to append + * @param len the number of bytes to append + */ + public void append(byte[] data, int len) { + if (data == null || len == 0) { + return; + } + + // Ensure the internal buffer is long enough to accommodate data. (The + // method grow will re-allocate the internal buffer if it's too short.) + grow(len); + // Append data. + System.arraycopy(data, 0, buffer, length + offset, len); + length += len; + } + + + /** + * Grows the internal buffer of this {@code ByteArrayBufferImpl}. + * + * This will change the data buffer of this packet but not the length of the + * valid data. Use this to grow the internal buffer to avoid buffer + * re-allocations when appending data. + * + * @param howMuch the number of bytes by which this {@code NewRawPacket} is to + * grow + */ + public void grow(int howMuch) { + if (howMuch < 0) + throw new IllegalArgumentException("howMuch: " + howMuch); + + int newLength = length + howMuch; + + if (newLength > buffer.length - offset) { + byte[] newBuffer = BufferPool.Companion.getGetArray().invoke(newLength); + + System.arraycopy(buffer, offset, newBuffer, 0, length); + offset = 0; + byte[] oldBuffer = buffer; + setBuffer(newBuffer); + BufferPool.Companion.getReturnArray().invoke(oldBuffer); + } + } + + /** + * Shrink the buffer of this packet by specified length + * + * @param len length to shrink + */ + public void shrink(int len) + { + if (len <= 0) + return; + + this.length -= len; + if (this.length < 0) + this.length = 0; + } + + /** + * @param buffer the buffer to set + */ + public void setBuffer(byte[] buffer) + { + this.buffer = buffer; + } + + @Override + public boolean isInvalid() + { + return false; + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt b/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt new file mode 100644 index 0000000..0a7dd50 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/Packet.kt @@ -0,0 +1,69 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp + +import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment +import org.jitsi.rtp.util.BufferPool + +abstract class Packet( + buffer: ByteArray, + offset: Int, + length: Int +) : ByteArrayBufferImpl(buffer, offset, length), Cloneable { + + inline fun toOtherType(otherTypeCreator: (ByteArray, Int, Int) -> OtherType): OtherType = + otherTypeCreator(buffer, offset, length) + + public abstract override fun clone(): Packet + + /** + * A string used to verify the payload of this packet. The same payload must always produce the same verification + * string. This is similar to a hashcode of the payload, but in order to provide more debugging information it also + * includes the length of the payload (and is thus a string). + * + * Different subclasses of [Packet] will have different notions of "payload", and they need to + */ + open val payloadVerification: String + get() = "len=$length hashCode=${buffer.hashCodeOfSegment(offset, offset + length)}" + + /** + * Creates a clone of this [Packet]'s buffer. Allocates a new buffer which is large enough to contain the data of + * the packet plus additional [BYTES_TO_LEAVE_AT_END_OF_PACKET] bytes at the end and optionally + * [bytesToLeaveAtStart] bytes at the start. + * + * @param bytesToLeaveAtStart How many bytes to leave at the start of the buffer. This is effectively the offset of + * the new packet. + */ + protected fun cloneBuffer(bytesToLeaveAtStart: Int): ByteArray = + BufferPool.getArray(bytesToLeaveAtStart + length + BYTES_TO_LEAVE_AT_END_OF_PACKET).apply { + System.arraycopy( + buffer, + offset, + this, + bytesToLeaveAtStart, + length + ) + } + + companion object { + /** + * How much space to leave in the end of a new packet. Having space in the buffer after the payload allows us to + * implement adding the SRT(C)P auth tag efficiently (without having to allocate a new buffer). + */ + const val BYTES_TO_LEAVE_AT_END_OF_PACKET = 20 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt new file mode 100644 index 0000000..d839c4e --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/UnparsedPacket.kt @@ -0,0 +1,38 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp + +import org.jitsi.rtp.rtp.RtpPacket + +class UnparsedPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : Packet(buffer, offset, length) { + + constructor(buffer: ByteArray) : this(buffer, 0, buffer.size) + + /** + * Note that we leave the same space at the start as for RTP packets, because an [UnparsedPacket]'s buffer + * might be used directly to create an [RtpPacket]. + */ + override fun clone(): UnparsedPacket = UnparsedPacket( + cloneBuffer(RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET), + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ) +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/Byte.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/Byte.kt new file mode 100644 index 0000000..a8c4c30 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/Byte.kt @@ -0,0 +1,101 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +/** + * Return the value of the bit at position [bitPos], where + * 0 represents the left-most bit and 7 represents the right-most + * bit of the current byte + */ +fun Byte.getBit(bitPos: Int): Int { + val mask = 0b1 shl (7 - bitPos) + return (this.toInt() and mask) ushr (7 - bitPos) +} + +/** + * Return a byte setting or unsetting the bit at [bitPos] of a byte + * according to [isSet] + * Unfortunately we can't do this as an in-place modifier because + * we can't modify the Byte via 'this' in an extension function. + */ +fun Byte.putBit(bitPos: Int, isSet: Boolean): Byte = if (isSet) { + (this.toInt() or (0b10000000 ushr bitPos)).toByte() +} else { + (this.toInt() and (0b10000000 ushr bitPos).inv()).toByte() +} + +/** + * Return a byte setting or unsetting the bit or bits specified by [mask] + * of a byte according to [isSet] + * Unfortunately we can't do this as an in-place modifier because + * we can't modify the Byte via 'this' in an extension function. + */ +fun Byte.putBitWithMask(mask: Byte, isSet: Boolean): Byte = if (isSet) { + (this.toInt() or mask.toInt()).toByte() +} else { + (this.toInt() and mask.toInt().inv()).toByte() +} + +/** + * Return a byte with the right-most [numBits] bits from [src] put into [dest] + * starting at [bitStartPos] + * [bitStartPos] is a 0 based index into the byte [dest], where the MSB + * is position 0 and the LSB is position 7. + * Given the values: + * bitStartPos = 4 + * numBits = 3 + * src = 0b101 + * dest = 0b00000000 + * + * The returned Byte will be: + * 0b00001010 + */ +fun Byte.putBits(bitStartPos: Int, numBits: Int, src: Byte): Byte { + // Start the position in src to the first bit we'll assign. + val valueBitPosition = 7 - numBits + 1 + var result = this + for (i in 0 until numBits) { + val isSet = src.getBitAsBool(valueBitPosition + i) + result = result.putBit(bitStartPos + i, isSet) + } + return result +} + +/** + * Get the bit at [bitPos] as a [Boolean]. Will return [true] + * if the bit is set (1) and [false] if it is unset (0) + */ +fun Byte.getBitAsBool(bitPos: Int): Boolean = getBit(bitPos) == 1 + +/** + * Get [numBits] bits starting at [bitStartPos], shift them + * all the way to the right and return the result as a [Byte] + * [numBits] must be <= 8 + */ +fun Byte.getBits(bitStartPos: Int, numBits: Int): Byte { + // Subtract 1 since the bit at 'bitStartPos' will be included, e.g.: + // A call getBits(4, 2) will read bits 4 and 5 + val bitEndPos = bitStartPos + numBits - 1 + var result = 0 + // shiftOffset represents how far, from the right-most + // bit position, the bit at bitStartPos should be + // at in the result + val shiftOffset = (bitEndPos - bitStartPos) + (bitStartPos..bitEndPos).forEachIndexed { index, value -> + result = result or (getBit(value) shl (shiftOffset - index)) + } + return result.toByte() +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteArrayBuffer.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteArrayBuffer.kt new file mode 100644 index 0000000..ed84bff --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteArrayBuffer.kt @@ -0,0 +1,23 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.extensions + +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.utils.ByteArrayBuffer + +@JvmOverloads +fun ByteArrayBuffer.toHex(maxBytes: Int = Int.MAX_VALUE) = buffer.toHex(offset, maxBytes.coerceAtMost(length)) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt new file mode 100644 index 0000000..3b39617 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/ByteBuffer.kt @@ -0,0 +1,212 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +import org.jitsi.rtp.util.BufferPool +import java.nio.ByteBuffer + +/** + * Return a (deep) copy of this ByteBuffer. + * The position and mark from the original will NOT + * be carried over (no mark will be set on the copy + * and its position will be 0). The mark of the original + * will not be touched; the position of the original will + * end up what it was before this call was made, BUT, it's value + * will be modified during [clone]. + */ +fun ByteBuffer.clone(): ByteBuffer { + val startPosition = this.position() + val clone = ByteBuffer.wrap(BufferPool.getArray(this.capacity())) + this.rewind() + clone.put(this) + this.position(startPosition) + clone.flip() + // TODO(brian): handle if this was a readonly buffer + return clone +} + +/** + * Move this [ByteBuffer]'s position back one Byte + */ +fun ByteBuffer.rewindOneByte() { + this.position(this.position() - 1) +} + +/** + * Put the right-most 3 bytes of [value] + * into the buffer + */ +fun ByteBuffer.put3Bytes(value: Int) { + this.put(((value and 0x00FF0000) ushr 16).toByte()) + this.put(((value and 0x0000FF00) ushr 8).toByte()) + this.put((value and 0x000000FF).toByte()) +} + +fun ByteBuffer.put3Bytes(index: Int, value: Int) { + this.put(index, ((value and 0x00FF0000) ushr 16).toByte()) + this.put(index + 1, ((value and 0x0000FF00) ushr 8).toByte()) + this.put(index + 2, (value and 0x000000FF).toByte()) +} + +/** + * Reads the next 3 bytes into the right-most + * 3 bytes of an Int + */ +fun ByteBuffer.get3Bytes(): Int { + val byte1 = (get().toInt() and 0xFF) shl 16 + val byte2 = (get().toInt() and 0xFF) shl 8 + val byte3 = get().toInt() and 0xFF + return byte1 or byte2 or byte3 +} + +fun ByteBuffer.get3Bytes(index: Int): Int { + val byte1 = (get(index).toInt() and 0xFF) shl 16 + val byte2 = (get(index + 1).toInt() and 0xFF) shl 8 + val byte3 = get(index + 2).toInt() and 0xFF + return byte1 or byte2 or byte3 +} + +/** + * Put the right-most [numBits] bits from [src] into the byte at [byteIndex] + * starting at position [destBitPos]. [destBitPos] is a 0-based index of the + * bit in the byte at [byteIndex], where 0 is the MSB and 7 is the LSB. + */ +fun ByteBuffer.putBits(byteIndex: Int, destBitPos: Int, src: Byte, numBits: Int) { + var byte = get(byteIndex) + byte = byte.putBits(destBitPos, numBits, src) + put(byteIndex, byte) +} + +fun ByteBuffer.putBitAsBoolean(byteIndex: Int, destBitPos: Int, isSet: Boolean) { + var byte = get(byteIndex) + byte = byte.putBit(destBitPos, isSet) + put(byteIndex, byte) +} + +private val HEX_CHARS = "0123456789ABCDEF".toCharArray() + +/** + * Print the entire contents of the [ByteBuffer] as hex + * digits + * TODO: should the spacing here (every 4 chunks) be right-justified? + * (meaning, if you have a buffer with a size that isn't a multiple + * of 8, the right-most chunks should be represented as a chunk + * of 8, rather than the left-most. for example: + * data: 01000A00 64006400 7B000000 00 + * should probably be printed as: + * data: 01 000A0064 0064007B 00000000 + */ +fun ByteBuffer.toHex(): String { + val result = StringBuffer() + + val prevPosition = position() + for (i in 0 until limit()) { + val octet = get(i).toInt() + val firstIndex = (octet and 0xF0).ushr(4) + val secondIndex = octet and 0x0F + result.append(HEX_CHARS[firstIndex]) + result.append(HEX_CHARS[secondIndex]) + if ((i + 1) % 16 == 0) { + result.append("\n") + } else if ((i + 1) % 4 == 0) { + result.append(" ") + } + } + position(prevPosition) + + return result.toString() +} + +/** + * Returns a newly constructed [ByteBuffer] whose position 0 will + * start at [startPosition] in the current buffer and whose limit + * and capacity will be [size] + */ +fun ByteBuffer.subBuffer(startPosition: Int, size: Int): ByteBuffer { + if (startPosition + size > limit()) { + throw Exception( + "SubBuffer goes beyond the buffer's limit " + + "(limit ${limit()}, requested end of buffer ${startPosition + size})" + ) + } + return (duplicate().position(startPosition).limit(startPosition + size) as ByteBuffer).slice() +} + +/** + * Returns a newly constructed [ByteBuffer] whose position 0 will + * start at [startPosition] in the current buffer and whose limit + * and capacity will be the amount of bytes between [startPosition] and + * the current buffer's [limit()] + */ +fun ByteBuffer.subBuffer(startPosition: Int): ByteBuffer = subBuffer(startPosition, limit() - startPosition) + +/** + * Put [buf] into this buffer starting at [index] + */ +fun ByteBuffer.put(index: Int, buf: ByteBuffer): ByteBuffer { + val currentPosition = position() + position(index) + put(buf) + position(currentPosition) + return this +} + +/** + * Shifts the data from [startPos] to [endPos] [numBytes] to the right. + * Note that this method may increase the given buffer's limit, up to + * its capacity. + * + * Note that [startPos] and [endPos] are zero-based. + */ +fun ByteBuffer.shiftDataRight(startPos: Int, endPos: Int, numBytes: Int) { + if (endPos + numBytes >= limit()) { + if (capacity() > endPos + numBytes) { + limit(endPos + numBytes + 1) + } + } + for (index in endPos downTo startPos) { + put(index + numBytes, get(index)) + } +} + +fun ByteBuffer.shiftDataLeft(startPos: Int, endPos: Int, numBytes: Int) { + for (index in startPos..endPos) { + put(index - numBytes, get(index)) + } +} + +/** + * Compare the contents of two ByteBuffers, each starting from their position 0 + */ +fun ByteBuffer.compareToFromBeginning(other: ByteBuffer): Int { + val thisRewound = this.duplicate().rewind() as ByteBuffer + val otherRewound = other.duplicate().rewind() as ByteBuffer + return thisRewound.compareTo(otherRewound) +} + +/** + * Return a new ByteBuffer that includes the contents of this one + * plus [other] + */ +operator fun ByteBuffer.plus(other: ByteBuffer): ByteBuffer { + val newBuf = ByteBuffer.wrap(BufferPool.getArray(limit() + other.limit())) + newBuf.rewind() + newBuf.put(duplicate().rewind() as ByteBuffer) + newBuf.put(other.duplicate().rewind() as ByteBuffer) + newBuf.flip() + + return newBuf +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/PacketExtensions.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/PacketExtensions.kt new file mode 100644 index 0000000..2257e45 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/PacketExtensions.kt @@ -0,0 +1,74 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +import org.jitsi.rtp.Packet +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtcp.RtcpHeader +import org.jitsi.rtp.rtp.RtpHeader + +/** + * "The process for demultiplexing a packet is as follows. The receiver + * looks at the first byte of the packet." + * + * +----------------+ + * | [0..3] -+--> forward to STUN + * | | + * | [16..19] -+--> forward to ZRTP + * | | + * | [20..63] -+--> forward to DTLS + * | | + * | [64..79] -+--> forward to TURN Channel + * | | + * | [128..191] -+--> forward to RTP/RTCP + * +----------------+ + * + * See [https://tools.ietf.org/html/rfc7983#section-7] + * + * + * RTP/RTCP are further demultiplexed based on the packet type (second byte) + * + * Note: Kotlin's ranges are implemented efficiently. + */ +private val DTLS_RANGE = 20..63 +private val RTP_RTCP_RANGE = 128..191 +val RTCP_PACKET_TYPE_RANGE = 192..223 + +fun Packet.looksLikeRtp(): Boolean { + if (length < RtpHeader.FIXED_HEADER_SIZE_BYTES) { + return false + } + + val b0 = buffer[offset].toPositiveInt() + val b1 = buffer[offset + 1].toPositiveInt() + return b0 in RTP_RTCP_RANGE && b1 !in RTCP_PACKET_TYPE_RANGE +} + +fun Packet.looksLikeRtcp(): Boolean { + if (length < RtcpHeader.SIZE_BYTES) { + return false + } + val b0 = buffer[offset].toPositiveInt() + val b1 = buffer[offset + 1].toPositiveInt() + return b0 in RTP_RTCP_RANGE && b1 in RTCP_PACKET_TYPE_RANGE +} + +fun Packet.looksLikeDtls(): Boolean { + if (length < 1) { + return false + } + return buffer[offset].toPositiveInt() in DTLS_RANGE +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt new file mode 100644 index 0000000..873b68b --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensions.kt @@ -0,0 +1,154 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.extensions.bytearray +import org.jitsi.rtp.extensions.getBitAsBool +import org.jitsi.rtp.extensions.putBit +import org.jitsi.rtp.extensions.putBits +import org.jitsi.rtp.util.BufferPool +import org.jitsi.utils.ByteArrayUtils.readInt +import org.jitsi.utils.ByteArrayUtils.readShort +import org.jitsi.utils.ByteArrayUtils.readUint24 +import org.jitsi.utils.ByteArrayUtils.writeInt +import org.jitsi.utils.ByteArrayUtils.writeShort +import org.jitsi.utils.ByteArrayUtils.writeUint24 +import java.lang.Math.abs + +/** + * Put the right-most [numBits] bits from [src] into the byte at [byteIndex] + * starting at position [destBitPos]. [destBitPos] is a 0-based index of the + * bit in the byte at [byteIndex], where 0 is the MSB and 7 is the LSB. + */ +fun ByteArray.putBits(byteIndex: Int, destBitPos: Int, src: Byte, numBits: Int) { + var byte = get(byteIndex) + byte = byte.putBits(destBitPos, numBits, src) + set(byteIndex, byte) +} + +fun ByteArray.getBitAsBool(byteOffset: Int, bitOffset: Int): Boolean = get(byteOffset).getBitAsBool(bitOffset) + +fun ByteArray.putBitAsBoolean(byteIndex: Int, destBitPos: Int, isSet: Boolean) { + var byte = get(byteIndex) + byte = byte.putBit(destBitPos, isSet) + set(byteIndex, byte) +} + +fun ByteArray.getShort(byteIndex: Int): Short = readShort(this, byteIndex) +fun ByteArray.putShort(byteIndex: Int, value: Short) = writeShort(this, byteIndex, value) +fun ByteArray.get3Bytes(byteIndex: Int): Int = readUint24(this, byteIndex) +fun ByteArray.put3Bytes(byteIndex: Int, value: Int) = writeUint24(this, byteIndex, value) +fun ByteArray.getInt(byteIndex: Int): Int = readInt(this, byteIndex) +fun ByteArray.putInt(byteIndex: Int, value: Int) = writeInt(this, byteIndex, value) + +fun byteArrayOf(vararg elements: Number): ByteArray = elements.map { it.toByte() }.toByteArray() + +/** + * Shifts the data from [startPos] to [endPos] [numBytes] to the right. + * Note that this method may increase the given buffer's limit, up to + * its capacity. + * + * Note that [startPos] and [endPos] are zero-based and numBytes + * must be positive! + */ +fun ByteArray.shiftDataRight(startPos: Int, endPos: Int, numBytes: Int) { + if (numBytes < 0) { + throw Exception("") + } + for (index in endPos downTo startPos) { + set(index + numBytes, get(index)) + } +} + +fun ByteArray.shiftDataLeft(startPos: Int, endPos: Int, numBytes: Int) { + for (index in startPos..endPos) { + set(index - numBytes, get(index)) + } +} + +/** + * Shifts the data from [startPos] to [endPos] by [delta] bytes. + * if [delta] is negative, the data will be shifted to the left, + * if [delta] is positive, the data will be shifted to the right + */ +fun ByteArray.shiftData(startPos: Int, endPos: Int, delta: Int) { + when { + delta < 0 -> shiftDataLeft(startPos, endPos, abs(delta)) + delta > 0 -> shiftDataRight(startPos, endPos, delta) + } +} + +fun ByteArray.cloneFromPool(): ByteArray { + val clone = BufferPool.getArray(size) + System.arraycopy(this, 0, clone, 0, size) + return clone +} + +operator fun ByteArray.plus(other: ByteArray): ByteArray { + val newArray = BufferPool.getArray(size + other.size) + System.arraycopy(this, 0, newArray, 0, size) + System.arraycopy(other, 0, newArray, size, other.size) + + return newArray +} + +private val HEX_CHARS = "0123456789ABCDEF".toCharArray() + +/** + * Print the contents of the [ByteArray] as hex digits. + */ +fun ByteArray.toHex( + /** Offset to start at. */ + offset: Int = 0, + /** Maximum number of elements to print. */ + length: Int = (size - offset) +): String { + val result = StringBuffer() + + for (i in offset until (offset + length).coerceAtMost(size)) { + val position = i - offset + if (position != 0) { + if (position % 16 == 0) { + result.append("\n") + } else if (position % 4 == 0) { + result.append(" ") + } + } + val byte: Int = this[i].toInt() + val firstIndex = (byte and 0xF0) shr 4 + val secondIndex = byte and 0x0F + result.append(HEX_CHARS[firstIndex]) + result.append(HEX_CHARS[secondIndex]) + } + + return result.toString() +} + +/** + * Returns the hash code of the segment of this [ByteArray] starting at 'start' and ending in 'end' (exclusive). + */ +fun ByteArray.hashCodeOfSegment(start: Int, end: Int): Int { + var result = 1 + for (i in start.coerceIn(0, size) until end.coerceIn(0, size)) { + result = 31 * result + this[i] + } + return result +} + +class ByteArrayUtils { + companion object { + val emptyByteArray = BufferPool.getArray(0) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/extensions/unsigned/Unsigned.kt b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/unsigned/Unsigned.kt new file mode 100644 index 0000000..1b20b36 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/extensions/unsigned/Unsigned.kt @@ -0,0 +1,52 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.extensions.unsigned + +import kotlin.experimental.and + +/** + * Many times fields in packets are defined as unsigned. Since + * we don't have unsigned types (and attempts at using Kotlin's + * unsigned types have not proven to be very usable), we store + * those types as larger than they actually are ([Byte] and + * [Short] held as [Int] and [Int] held as [Long]). These + * helpers are to do the conversion correctly such that + * we get a properly 'unsigned' version of the parsed field. + * No helper is needed for the reverse operation (the use + * of toInt, toShort and toByte is sufficient). + */ +fun Byte.toPositiveInt(): Int = toInt() and 0xFF +fun Short.toPositiveInt(): Int = toInt() and 0xFFFF +fun Int.toPositiveLong(): Long = toLong() and 0xFFFFFFFF + +// TODO: i think these should be able to make the above functions obsolete +fun Number.toPositiveShort(): Short = when (this) { + is Byte -> this.toShort() and 0xFF + else -> this.toShort() +} +fun Number.toPositiveInt(): Int = when (this) { + is Byte -> this.toInt() and 0xFF + is Short -> this.toInt() and 0xFFFF + else -> this.toInt() +} + +fun Number.toPositiveLong(): Long = when (this) { + is Byte -> this.toLong() and 0xFF + is Short -> this.toLong() and 0xFFFF + is Int -> this.toLong() and 0xFFFFFFFF + else -> this.toLong() +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/CompoundRtcpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/CompoundRtcpPacket.kt new file mode 100644 index 0000000..1974945 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/CompoundRtcpPacket.kt @@ -0,0 +1,114 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.rtp.util.BufferPool + +class CompoundRtcpPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + + val packets: List by lazy { parse(buffer, offset, length) } + + companion object { + + fun parse(buffer: ByteArray, offset: Int, length: Int): List { + var bytesRemaining = length + var currOffset = offset + val rtcpPackets = mutableListOf() + while (bytesRemaining >= RtcpHeader.SIZE_BYTES) { + val rtcpPacket = try { + RtcpPacket.parse(buffer, currOffset, bytesRemaining) + } catch (e: InvalidRtcpException) { + throw CompoundRtcpContainedInvalidDataException(buffer, offset, length, currOffset, e.reason) + } + rtcpPackets.add(rtcpPacket) + currOffset += rtcpPacket.length + bytesRemaining -= rtcpPacket.length + } + return rtcpPackets + } + + operator fun invoke(packets: List): CompoundRtcpPacket { + val totalLength = packets.map { it.length }.sum() + val buf = BufferPool.getArray(totalLength + BYTES_TO_LEAVE_AT_END_OF_PACKET) + + var off = 0 + packets.forEach { + System.arraycopy(it.buffer, it.offset, buf, off, it.length) + off += it.length + } + + return CompoundRtcpPacket(buf, 0, totalLength) + } + + /** + * Create one or more compound RTCP packets from [packets], with each compound packet being + * no more than [mtu] bytes in size (unless an individual packet is bigger than that, in which + * case it will be in a compound packet on its own). + */ + fun createWithMtu(packets: List, mtu: Int = 1500): List = + packets.chunkMaxSize(mtu) { it.length }.map { CompoundRtcpPacket(it) } + } + + override fun clone(): RtcpPacket = CompoundRtcpPacket(cloneBuffer(0), 0, length) +} + +/** + * Return a List of Lists, where sub-list is made up of an ordered list + * of values pulled from [this], such that the total size of each sub-list + * is not more than maxSize. (Sizes are evaluated by [evaluate]. If an + * individual element's size is more than [maxSize] it will be returned + * in a sub-list on its own.) + */ +private fun List.chunkMaxSize(maxSize: Int, evaluate: (T) -> Int): List> { + val chunks = mutableListOf>() + if (this.isEmpty()) { + return chunks + } + var currentChunk = mutableListOf(first()) + chunks.add(currentChunk) + var chunkSize = evaluate(currentChunk.first()) + // Ignore the first value which we already put in the current chunk + drop(1).forEach { + val size = evaluate(it) + if (chunkSize + size > maxSize) { + currentChunk = mutableListOf(it) + chunks.add(currentChunk) + chunkSize = size + } else { + currentChunk.add(it) + chunkSize += size + } + } + return chunks +} + +class CompoundRtcpContainedInvalidDataException( + compoundRtcpBuf: ByteArray, + compoundRtcpOffset: Int, + compoundRtcpLength: Int, + invalidDataOffset: Int, + invalidDataReason: String +) : Exception( + "Compound RTCP contained invalid data. Compound RTCP packet data is: " + + "${compoundRtcpBuf.toHex(compoundRtcpOffset, compoundRtcpLength)} Invalid data " + + "started at offset ${invalidDataOffset - compoundRtcpOffset} and failed due to '$invalidDataReason'" +) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpByePacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpByePacket.kt new file mode 100644 index 0000000..9acca4d --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpByePacket.kt @@ -0,0 +1,74 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.extensions.bytearray.getInt +import org.jitsi.rtp.extensions.unsigned.toPositiveLong + +/** + * An [RtcpByePacket] cannot be changed once it has been created, so we + * parse the fields once (lazily) and store them. + * TODO: technically it COULD be changed, since anyone can change the buffer. + * can we enforce immutability? + * + * https://tools.ietf.org/html/rfc3550#section-6.6 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| SC | PT=BYE=203 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC/CSRC | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * : ... : + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * (opt) | length | reason for leaving ... + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class RtcpByePacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + val ssrcs: List by lazy { + val ssrcStartOffset = offset + 4 + (0 until reportCount) + .map { buffer.getInt(ssrcStartOffset + it * 4) } + .map(Int::toPositiveLong) + .toList() + } + + val reason: String? by lazy { + val headerAndSsrcsLengthBytes = RtcpHeader.SIZE_BYTES + (reportCount - 1) * 4 + val hasReason = headerAndSsrcsLengthBytes < length + + if (hasReason) { + val reasonLengthOffset = offset + headerAndSsrcsLengthBytes + val reasonLength = buffer[reasonLengthOffset].toInt() + val reasonStr = String(buffer, reasonLengthOffset + 1, reasonLength) + reasonStr + } else { + null + } + } + + override fun clone(): RtcpByePacket = RtcpByePacket(cloneBuffer(0), 0, length) + + companion object { + const val PT: Int = 203 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpHeader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpHeader.kt new file mode 100644 index 0000000..8323447 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpHeader.kt @@ -0,0 +1,115 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.extensions.bytearray.getBitAsBool +import org.jitsi.rtp.extensions.bytearray.putBitAsBoolean +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.util.getBitsAsInt +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getIntAsLong +import org.jitsi.rtp.util.getShortAsInt +import org.jitsi.rtp.util.putNumberAsBits + +/** + * Models the RTCP header as defined in https://tools.ietf.org/html/rfc3550#section-6.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| RC | PT=SR=200 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of sender | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * length: 16 bits + * The length of this RTCP packet in 32-bit words minus one, + * including the header and any padding. (The offset of one makes + * zero a valid length and avoids a possible infinite loop in + * scanning a compound RTCP packet, while counting 32-bit words + * avoids a validity check for a multiple of 4.) + * + * @author Brian Baldino + * + * TODO: Use the same utility function in RtpHeader and RtcpHeader + */ +class RtcpHeader { + companion object { + const val SIZE_BYTES = 8 + const val VERSION_OFFSET = 0 + const val PADDING_OFFSET = 0 + const val REPORT_COUNT_OFFSET = 0 + const val PACKET_TYPE_OFFSET = 1 + const val LENGTH_OFFSET = 2 + const val SENDER_SSRC_OFFSET = 4 + fun getVersion(buf: ByteArray, headerStartOffset: Int): Int = + buf.getBitsAsInt(headerStartOffset + VERSION_OFFSET, 0, 2) + fun setVersion(buf: ByteArray, headerStartOffset: Int, version: Int) = + buf.putNumberAsBits(headerStartOffset + VERSION_OFFSET, 0, 2, version) + + fun hasPadding(buf: ByteArray, headerStartOffset: Int): Boolean = + buf.getBitAsBool(headerStartOffset + PADDING_OFFSET, 2) + fun setPadding(buf: ByteArray, headerStartOffset: Int, hasPadding: Boolean) = + buf.putBitAsBoolean(headerStartOffset + PADDING_OFFSET, 2, hasPadding) + + fun getReportCount(buf: ByteArray, headerStartOffset: Int): Int = + buf.getBitsAsInt(headerStartOffset + REPORT_COUNT_OFFSET, 3, 5) + fun setReportCount(buf: ByteArray, headerStartOffset: Int, reportCount: Int) = + buf.putNumberAsBits(headerStartOffset + REPORT_COUNT_OFFSET, 3, 5, reportCount) + + fun getPacketType(buf: ByteArray, headerStartOffset: Int): Int = + buf.getByteAsInt(headerStartOffset + PACKET_TYPE_OFFSET) + fun setPacketType(buf: ByteArray, headerStartOffset: Int, packetType: Int) = + buf.set(headerStartOffset + PACKET_TYPE_OFFSET, packetType.toByte()) + + // TODO document (bytes or words?) + fun getLength(buf: ByteArray, headerStartOffset: Int): Int = + buf.getShortAsInt(headerStartOffset + LENGTH_OFFSET) + fun setLength(buf: ByteArray, headerStartOffset: Int, length: Int) = + buf.putShort(headerStartOffset + LENGTH_OFFSET, length.toShort()) + + fun getSenderSsrc(buf: ByteArray, headerStartOffset: Int): Long = + buf.getIntAsLong(headerStartOffset + SENDER_SSRC_OFFSET) + fun setSenderSsrc(buf: ByteArray, headerStartOffset: Int, senderSsrc: Long) = + buf.putInt(headerStartOffset + SENDER_SSRC_OFFSET, senderSsrc.toInt()) + } +} + +data class RtcpHeaderBuilder( + var version: Int = 2, + var hasPadding: Boolean = false, + var reportCount: Int = -1, + var packetType: Int = -1, + var length: Int = -1, + var senderSsrc: Long = 0 +) { + + fun build(): ByteArray { + val buf = ByteArray(RtcpHeader.SIZE_BYTES) + writeTo(buf, 0) + return buf + } + + fun writeTo(buf: ByteArray, offset: Int) { + RtcpHeader.setVersion(buf, offset, version) + RtcpHeader.setPadding(buf, offset, hasPadding) + RtcpHeader.setReportCount(buf, offset, reportCount) + RtcpHeader.setPacketType(buf, offset, packetType) + RtcpHeader.setLength(buf, offset, length) + RtcpHeader.setSenderSsrc(buf, offset, senderSsrc) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpPacket.kt new file mode 100644 index 0000000..1868549 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpPacket.kt @@ -0,0 +1,108 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.Packet +import org.jitsi.rtp.extensions.RTCP_PACKET_TYPE_RANGE +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket + +/** + * Models the RTCP header as defined in https://tools.ietf.org/html/rfc3550#section-6.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| RC | PT=SR=200 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of sender | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * length: 16 bits + * The length of this RTCP packet in 32-bit words minus one, + * including the header and any padding. (The offset of one makes + * zero a valid length and avoids a possible infinite loop in + * scanning a compound RTCP packet, while counting 32-bit words + * avoids a validity check for a multiple of 4.) + */ +abstract class RtcpPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : Packet(buffer, offset, length) { + + var version: Int + get() = RtcpHeader.getVersion(buffer, offset) + set(value) = RtcpHeader.setVersion(buffer, offset, value) + + val hasPadding: Boolean + get() = RtcpHeader.hasPadding(buffer, offset) + + val reportCount: Int + get() = RtcpHeader.getReportCount(buffer, offset) + + val packetType: Int + get() = RtcpHeader.getPacketType(buffer, offset) + + val lengthField: Int + get() = RtcpHeader.getLength(buffer, offset) + + var senderSsrc: Long + get() = RtcpHeader.getSenderSsrc(buffer, offset) + set(value) = RtcpHeader.setSenderSsrc(buffer, offset, value) + + val packetLength: Int = (lengthField + 1) * 4 + + /** + * Effectively disable the payload verification for RTCP packets, since in practice we change them very often + */ + override val payloadVerification = "rtcp" + + abstract override fun clone(): RtcpPacket + + companion object { + fun parse(buf: ByteArray, offset: Int, bytesRemaining: Int): RtcpPacket { + val packetType = RtcpHeader.getPacketType(buf, offset) + val packetLengthBytes = (RtcpHeader.getLength(buf, offset) + 1) * 4 + if (packetLengthBytes > bytesRemaining) { + throw InvalidRtcpException(buf, offset, "length $packetLengthBytes > available $bytesRemaining") + } + return when (packetType) { + RtcpByePacket.PT -> RtcpByePacket(buf, offset, packetLengthBytes) + + RtcpRrPacket.PT -> RtcpRrPacket(buf, offset, packetLengthBytes) + + RtcpSrPacket.PT -> RtcpSrPacket(buf, offset, packetLengthBytes) + + RtcpSdesPacket.PT -> RtcpSdesPacket(buf, offset, packetLengthBytes) + + in RtcpFbPacket.PACKET_TYPES -> RtcpFbPacket.parse(buf, offset, packetLengthBytes) + + RtcpXrPacket.PT -> RtcpXrPacket(buf, offset, packetLengthBytes) + + else -> { + return when (packetType) { + in RTCP_PACKET_TYPE_RANGE -> UnsupportedRtcpPacket(buf, offset, packetLengthBytes) + else -> throw InvalidRtcpException(buf, offset, "type $packetType") + } + } + } + } + } +} + +class InvalidRtcpException(buf: ByteArray, offset: Int, val reason: String) : + Exception("Invalid RTCP packet: $reason: ${buf.toHex(offset, RtcpHeader.SIZE_BYTES)}") diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt new file mode 100644 index 0000000..85d66f8 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlock.kt @@ -0,0 +1,150 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.extensions.bytearray.put3Bytes +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.rtp.util.get3BytesAsInt +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getIntAsLong + +/** + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * | SSRC_1 (SSRC of first source) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | fraction lost | cumulative number of packets lost | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | extended highest sequence number received | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | interarrival jitter | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | last SR (LSR) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | delay since last SR (DLSR) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class RtcpReportBlock( + val ssrc: Long, + val fractionLost: Int, + val cumulativePacketsLost: Int, + val extendedHighestSeqNum: Long, + val interarrivalJitter: Long, + val lastSrTimestamp: Long, + val delaySinceLastSr: Long +) { + constructor( + ssrc: Long, + fractionLost: Int, + cumulativePacketsLost: Int, + seqNumCycles: Int, + seqNum: Int, + interarrivalJitter: Long, + lastSrTimestamp: Long, + delaySinceLastSr: Long + ) : this( + ssrc, + fractionLost, + cumulativePacketsLost, + ((seqNumCycles shl 16) + seqNum.toShort()).toPositiveLong(), + interarrivalJitter, + lastSrTimestamp, + delaySinceLastSr + ) + + val seqNumCycles: Int by lazy { + (extendedHighestSeqNum ushr 16).toPositiveInt() + } + val seqNum: Int by lazy { + extendedHighestSeqNum.toShort().toPositiveInt() + } + + fun writeTo(buf: ByteArray, offset: Int) { + setSsrc(buf, offset, ssrc) + setFractionLost(buf, offset, fractionLost) + setCumulativePacketsLost(buf, offset, cumulativePacketsLost) + setExtendedHighestSeqNum(buf, offset, extendedHighestSeqNum) + setInterarrivalJitter(buf, offset, interarrivalJitter) + setLastSrTimestamp(buf, offset, lastSrTimestamp) + setDelaySinceLastSr(buf, offset, delaySinceLastSr) + } + + companion object { + const val SIZE_BYTES = 24 + + // Offsets relative to the start of an RTCP Report Block + const val SSRC_OFFSET = 0 + const val FRACTION_LOST_OFFSET = 4 + const val CUMULATIVE_PACKETS_LOST_OFFSET = 5 + const val EXTENDED_HIGHEST_SEQ_NUM_OFFSET = 8 + const val INTERARRIVAL_JITTER_OFFSET = 12 + const val LAST_SR_TIMESTAMP_OFFSET = 16 + const val DELAY_SINCE_LAST_SR_OFFSET = 20 + fun fromBuffer(buffer: ByteArray, offset: Int): RtcpReportBlock { + val ssrc = getSsrc(buffer, offset) + val fractionLost = getFractionLost(buffer, offset) + val cumulativePacketsLost = getCumulativePacketsLost(buffer, offset) + val extendedHighestSeqNum = getExtendedHighestSeqNum(buffer, offset) + val interarrivalJitter = getInterarrivalJitter(buffer, offset) + val lastSrTimestamp = getLastSrTimestamp(buffer, offset) + val delaySinceLastSr = getDelaySinceLastSr(buffer, offset) + + return RtcpReportBlock( + ssrc, + fractionLost, + cumulativePacketsLost, + extendedHighestSeqNum, + interarrivalJitter, + lastSrTimestamp, + delaySinceLastSr + ) + } + + fun getSsrc(buffer: ByteArray, offset: Int): Long = buffer.getIntAsLong(offset + SSRC_OFFSET) + fun setSsrc(buf: ByteArray, baseOffset: Int, value: Long) = buf.putInt(baseOffset + SSRC_OFFSET, value.toInt()) + + fun getFractionLost(buffer: ByteArray, offset: Int): Int = buffer.getByteAsInt(offset + FRACTION_LOST_OFFSET) + fun setFractionLost(buf: ByteArray, baseOffset: Int, value: Int) = + buf.set(baseOffset + FRACTION_LOST_OFFSET, value.toByte()) + + fun getCumulativePacketsLost(buffer: ByteArray, offset: Int): Int = + buffer.get3BytesAsInt(offset + CUMULATIVE_PACKETS_LOST_OFFSET) + fun setCumulativePacketsLost(buffer: ByteArray, offset: Int, value: Int) = + buffer.put3Bytes(offset + CUMULATIVE_PACKETS_LOST_OFFSET, value) + + fun getExtendedHighestSeqNum(buffer: ByteArray, offset: Int): Long = + buffer.getIntAsLong(offset + EXTENDED_HIGHEST_SEQ_NUM_OFFSET) + fun setExtendedHighestSeqNum(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + EXTENDED_HIGHEST_SEQ_NUM_OFFSET, value.toInt()) + + fun getInterarrivalJitter(buffer: ByteArray, offset: Int): Long = + buffer.getIntAsLong(offset + INTERARRIVAL_JITTER_OFFSET) + fun setInterarrivalJitter(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + INTERARRIVAL_JITTER_OFFSET, value.toInt()) + + fun getLastSrTimestamp(buffer: ByteArray, offset: Int): Long = + buffer.getIntAsLong(offset + LAST_SR_TIMESTAMP_OFFSET) + fun setLastSrTimestamp(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + LAST_SR_TIMESTAMP_OFFSET, value.toInt()) + + fun getDelaySinceLastSr(buffer: ByteArray, offset: Int): Long = + buffer.getIntAsLong(offset + DELAY_SINCE_LAST_SR_OFFSET) + fun setDelaySinceLastSr(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + DELAY_SINCE_LAST_SR_OFFSET, value.toInt()) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt new file mode 100644 index 0000000..2e76ab6 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacket.kt @@ -0,0 +1,100 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils + +/** + * https://tools.ietf.org/html/rfc3550#section-6.4.2 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * header |V=2|P| RC | PT=RR=201 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * report | SSRC_1 (SSRC of first source) | + * block +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * 1 | fraction lost | cumulative number of packets lost | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | extended highest sequence number received | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | interarrival jitter | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | last SR (LSR) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | delay since last SR (DLSR) | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * report | SSRC_2 (SSRC of second source) | + * block +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * 2 : ... : + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * | profile-specific extensions | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class RtcpRrPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + val reportBlocks: List by lazy { + (0 until reportCount).map { + RtcpReportBlock.fromBuffer(buffer, offset + REPORT_BLOCKS_OFFSET + it * RtcpReportBlock.SIZE_BYTES) + }.toList() + } + + override fun clone(): RtcpRrPacket = RtcpRrPacket(cloneBuffer(0), 0, length) + + companion object { + const val PT = 201 + const val REPORT_BLOCKS_OFFSET = 8 + } +} + +data class RtcpRrPacketBuilder( + var rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + val reportBlocks: List = listOf() +) { + + init { + require(reportBlocks.size <= 31) { "Too many report blocks ${reportBlocks.size}: RR can contain at most 31" } + } + + private fun getLengthValue(): Int = RtpUtils.calculateRtcpLengthFieldValue(sizeBytes) + + private val sizeBytes: Int + get() = RtcpHeader.SIZE_BYTES + reportBlocks.size * RtcpReportBlock.SIZE_BYTES + + fun build(): RtcpRrPacket { + val buf = BufferPool.getArray(sizeBytes) + writeTo(buf, 0) + return RtcpRrPacket(buf, 0, sizeBytes) + } + + fun writeTo(buf: ByteArray, offset: Int) { + rtcpHeader.apply { + packetType = RtcpRrPacket.PT + reportCount = reportBlocks.size + length = getLengthValue() + }.writeTo(buf, offset) + reportBlocks.forEachIndexed { index, reportBlock -> + reportBlock.writeTo(buf, offset + RtcpRrPacket.REPORT_BLOCKS_OFFSET + index * RtcpReportBlock.SIZE_BYTES) + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt new file mode 100644 index 0000000..ff3c04e --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSdesPacket.kt @@ -0,0 +1,213 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.extensions.bytearray.ByteArrayUtils +import org.jitsi.rtp.extensions.bytearray.toHex +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getIntAsLong +import java.nio.charset.StandardCharsets + +/** + * https://tools.ietf.org/html/rfc3550#section-6.5 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * header |V=2|P| SC | PT=SDES=202 | length | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * chunk | SSRC/CSRC_1 | + * 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SDES items | + * | ... | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * chunk | SSRC/CSRC_2 | + * 2 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SDES items | + * | ... | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * + */ +class RtcpSdesPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + + val sdesChunks: List by lazy { + getSdesChunks(buffer, offset, length) + } + + override fun clone(): RtcpPacket = RtcpSdesPacket(cloneBuffer(0), 0, length) + + companion object { + const val PT = 202 + const val CHUNKS_OFFSET = 4 + + fun getSdesChunks(buf: ByteArray, baseOffset: Int, length: Int): List { + var currOffset = baseOffset + CHUNKS_OFFSET + val sdesChunks = mutableListOf() + while (currOffset < length) { + val sdesChunk = SdesChunk(buf, currOffset) + sdesChunks.add(sdesChunk) + currOffset += sdesChunk.sizeBytes + } + return sdesChunks + } + } +} + +// Offset should point to the start of this sdes chunk +class SdesChunk( + buffer: ByteArray, + offset: Int +) { + val sizeBytes: Int by lazy { + val dataSize = 4 + sdesItems.map(SdesItem::sizeBytes).sum() + dataSize + RtpUtils.getNumPaddingBytes(dataSize) + } + val ssrc: Long by lazy { getSsrc(buffer, offset) } + + val sdesItems: List by lazy { + getSdesItems(buffer, offset) + } + + companion object { + const val SSRC_OFFSET = 0 + const val SDES_ITEMS_OFFSET = 4 + + fun getSsrc(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + SSRC_OFFSET) + + fun getSdesItems(buf: ByteArray, baseOffset: Int): List { + var currOffset = baseOffset + SDES_ITEMS_OFFSET + val sdesItems = mutableListOf() + while (true) { + val currItem = SdesItem.parse(buf, currOffset) + if (currItem == EmptySdesItem) { + break + } + sdesItems.add(currItem) + currOffset += currItem.sizeBytes + } + return sdesItems + } + } +} + +enum class SdesItemType(val value: Int) { + EMPTY(0), + UNKNOWN(-1), + CNAME(1); + + companion object { + private val map = SdesItemType.values().associateBy(SdesItemType::value) + fun fromInt(type: Int): SdesItemType = map.getOrDefault(type, UNKNOWN) + } +} + +/** + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Type | length | data ... + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * length = the length of the data field + */ +abstract class SdesItem( + val type: SdesItemType +) { + abstract val sizeBytes: Int + + companion object { + const val SDES_ITEM_HEADER_SIZE = 2 + const val TYPE_OFFSET = 0 + const val LENGTH_OFFSET = 1 + const val DATA_OFFSET = 2 + + fun getType(buf: ByteArray, baseOffset: Int): Int = buf.getByteAsInt(baseOffset + TYPE_OFFSET) + + fun getLength(buf: ByteArray, baseOffset: Int): Int = buf.getByteAsInt(baseOffset + LENGTH_OFFSET) + + fun copyData(buf: ByteArray, baseOffset: Int, dataLength: Int): ByteArray { + if (dataLength <= 0) { + return ByteArrayUtils.emptyByteArray + } + val copy = BufferPool.getArray(dataLength) + System.arraycopy(buf, baseOffset + DATA_OFFSET, copy, 0, dataLength) + return copy + } + + /** + * [buf]'s current position should be at the beginning of + * the SDES item + */ + fun parse(buf: ByteArray, offset: Int): SdesItem { + val typeValue = getType(buf, offset) + val type = SdesItemType.fromInt(typeValue) + return when (type) { + SdesItemType.EMPTY -> EmptySdesItem + + else -> { + val length = getLength(buf, offset) + return when (type) { + SdesItemType.CNAME -> CnameSdesItem(buf, offset, length) + else -> UnknownSdesItem(typeValue, buf, offset, length) + } + } + } + } + } +} + +class UnknownSdesItem( + private val sdesTypeValue: Int, + buf: ByteArray, + offset: Int, + length: Int +) : SdesItem(SdesItemType.UNKNOWN) { + private val dataField = copyData(buf, offset, length) + override val sizeBytes: Int = SDES_ITEM_HEADER_SIZE + dataField.size + + override fun toString(): String = "Unknown SDES type($sdesTypeValue) data = ${dataField.toHex()}" +} + +object EmptySdesItem : SdesItem(SdesItemType.EMPTY) { + override val sizeBytes: Int = 1 +} + +/** + * https://tools.ietf.org/html/rfc3550#section-6.5.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | CNAME=1 | length | user and domain name ... + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class CnameSdesItem(buf: ByteArray, offset: Int, length: Int) : SdesItem(SdesItemType.CNAME) { + + private val dataField: ByteArray = copyData(buf, offset, length) + override val sizeBytes: Int = SDES_ITEM_HEADER_SIZE + dataField.size + + val cname: String by lazy { + String(dataField, StandardCharsets.US_ASCII) + } + + override fun toString(): String = "CNAME: $cname" +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt new file mode 100644 index 0000000..65bf140 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacket.kt @@ -0,0 +1,233 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.getIntAsLong + +/** + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * sender | NTP timestamp, most significant word | + * info +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | NTP timestamp, least significant word | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | RTP timestamp | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | sender's packet count | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | sender's octet count | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * + * RTCP SenderInfo block + */ +class SenderInfoParser { + companion object { + const val SIZE_BYTES = 20 + const val NTP_TS_MSW_OFFSET = 0 + const val NTP_TS_LSW_OFFSET = 4 + const val RTP_TS_OFFSET = 8 + const val SENDERS_PACKET_COUNT_OFFSET = 12 + const val SENDERS_OCTET_COUNT_OFFSET = 16 + + fun getNtpTimestampMsw(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + NTP_TS_MSW_OFFSET) + fun setNtpTimestampMsw(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + NTP_TS_MSW_OFFSET, value.toInt()) + + fun getNtpTimestampLsw(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + NTP_TS_LSW_OFFSET) + fun setNtpTimestampLsw(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + NTP_TS_LSW_OFFSET, value.toInt()) + + fun getRtpTimestamp(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + RTP_TS_OFFSET) + fun setRtpTimestamp(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + RTP_TS_OFFSET, value.toInt()) + + fun getSendersPacketCount(buf: ByteArray, baseOffset: Int): Long = + buf.getIntAsLong(baseOffset + SENDERS_PACKET_COUNT_OFFSET) + fun setSendersPacketCount(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + SENDERS_PACKET_COUNT_OFFSET, value.toInt()) + + fun getSendersOctetCount(buf: ByteArray, baseOffset: Int): Long = + buf.getIntAsLong(baseOffset + SENDERS_OCTET_COUNT_OFFSET) + fun setSendersOctetCount(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + SENDERS_OCTET_COUNT_OFFSET, value.toInt()) + } +} + +data class SenderInfoBuilder( + var ntpTimestampMsw: Long = -1, + var ntpTimestampLsw: Long = -1, + var rtpTimestamp: Long = -1, + var sendersPacketCount: Long = -1, + var sendersOctetCount: Long = -1 +) { + fun setNtpFromJavaTime(javaTime: Long) { + val wallSecs = javaTime / 1000 + val wallMs = javaTime % 1000 + ntpTimestampMsw = wallSecs + JAVA_TO_NTP_EPOCH_OFFSET_SECS + ntpTimestampLsw = wallMs * (1L shl 32) / 1000 + } + + fun writeTo(buf: ByteArray, offset: Int) { + SenderInfoParser.setNtpTimestampMsw(buf, offset, ntpTimestampMsw) + SenderInfoParser.setNtpTimestampLsw(buf, offset, ntpTimestampLsw) + SenderInfoParser.setRtpTimestamp(buf, offset, rtpTimestamp) + SenderInfoParser.setSendersPacketCount(buf, offset, sendersPacketCount) + SenderInfoParser.setSendersOctetCount(buf, offset, sendersOctetCount) + } + + companion object { + const val JAVA_TO_NTP_EPOCH_OFFSET_SECS = 2208988800L + } +} + +/** + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * header |V=2|P| RC | PT=SR=200 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of sender | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * sender | NTP timestamp, most significant word | + * info +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | NTP timestamp, least significant word | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | RTP timestamp | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | sender's packet count | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | sender's octet count | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * report | SSRC_1 (SSRC of first source) | + * block +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * 1 | fraction lost | cumulative number of packets lost | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | extended highest sequence number received | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | interarrival jitter | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | last SR (LSR) | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | delay since last SR (DLSR) | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * report | SSRC_2 (SSRC of second source) | + * block +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * 2 : ... : + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * | profile-specific extensions | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * https://tools.ietf.org/html/rfc3550#section-6.4.1 + */ +class RtcpSrPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + + val senderInfo: SenderInfo by lazy { SenderInfo() } + + val reportBlocks: List by lazy { + (0 until reportCount).map { + RtcpReportBlock.fromBuffer(buffer, offset + REPORT_BLOCKS_OFFSET + it * RtcpReportBlock.SIZE_BYTES) + }.toList() + } + + override fun clone(): RtcpSrPacket = RtcpSrPacket(cloneBuffer(0), 0, length) + + fun cloneWithoutReportBlocks(): RtcpSrPacket = RtcpSrPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + reportCount = 0, + senderSsrc = this.senderSsrc + ), + senderInfo = SenderInfoBuilder( + ntpTimestampMsw = this.senderInfo.ntpTimestampMsw, + ntpTimestampLsw = this.senderInfo.ntpTimestampLsw, + rtpTimestamp = this.senderInfo.rtpTimestamp, + sendersPacketCount = this.senderInfo.sendersPacketCount, + sendersOctetCount = this.senderInfo.sendersOctetCount + ) + ).build() + + // SenderInfo is defined differently so that we can scope these variables under a the 'senderInfo' + // member here. Is there a better way? + inner class SenderInfo { + val ntpTimestampMsw: Long = SenderInfoParser.getNtpTimestampMsw(buffer, offset + SENDER_INFO_OFFSET) + val ntpTimestampLsw: Long = SenderInfoParser.getNtpTimestampLsw(buffer, offset + SENDER_INFO_OFFSET) + var rtpTimestamp: Long + get() = SenderInfoParser.getRtpTimestamp(buffer, offset + SENDER_INFO_OFFSET) + set(value) = SenderInfoParser.setRtpTimestamp(buffer, offset + SENDER_INFO_OFFSET, value) + var sendersPacketCount: Long + get() = SenderInfoParser.getSendersPacketCount(buffer, offset + SENDER_INFO_OFFSET) + set(value) = SenderInfoParser.setSendersPacketCount(buffer, offset + SENDER_INFO_OFFSET, value) + var sendersOctetCount: Long + get() = SenderInfoParser.getSendersOctetCount(buffer, offset + SENDER_INFO_OFFSET) + set(value) = SenderInfoParser.setSendersOctetCount(buffer, offset + SENDER_INFO_OFFSET, value) + + /** + * https://tools.ietf.org/html/rfc3550#section-4 + * In some fields where a more compact representation is + * appropriate, only the middle 32 bits are used; that is, the low 16 + * bits of the integer part and the high 16 bits of the fractional part. + * The high 16 bits of the integer part must be determined + * independently. + */ + val compactedNtpTimestamp: Long by lazy { + (((ntpTimestampMsw and 0xFFFF) shl 16) or (ntpTimestampLsw ushr 16)).toPositiveLong() + } + } + + companion object { + const val PT = 200 + const val SENDER_INFO_OFFSET = RtcpHeader.SIZE_BYTES + const val REPORT_BLOCKS_OFFSET = SENDER_INFO_OFFSET + SenderInfoParser.SIZE_BYTES + } +} + +data class RtcpSrPacketBuilder( + var rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + var senderInfo: SenderInfoBuilder = SenderInfoBuilder(), + val reportBlocks: List = listOf() +) { + + init { + require(reportBlocks.size <= 31) { "Too many report blocks ${reportBlocks.size}: SR can contain at most 31" } + } + + private val sizeBytes: Int + get() = RtcpHeader.SIZE_BYTES + SenderInfoParser.SIZE_BYTES + reportBlocks.size * RtcpReportBlock.SIZE_BYTES + + fun build(): RtcpSrPacket { + val buf = BufferPool.getArray(sizeBytes) + writeTo(buf, 0) + return RtcpSrPacket(buf, 0, sizeBytes) + } + + fun writeTo(buf: ByteArray, offset: Int) { + rtcpHeader.apply { + packetType = RtcpSrPacket.PT + reportCount = reportBlocks.size + length = RtpUtils.calculateRtcpLengthFieldValue(sizeBytes) + }.writeTo(buf, offset) + senderInfo.writeTo(buf, offset + RtcpSrPacket.SENDER_INFO_OFFSET) + reportBlocks.forEachIndexed { index, reportBlock -> + reportBlock.writeTo(buf, offset + RtcpSrPacket.REPORT_BLOCKS_OFFSET + index * RtcpReportBlock.SIZE_BYTES) + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpXrPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpXrPacket.kt new file mode 100644 index 0000000..f67226c --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/RtcpXrPacket.kt @@ -0,0 +1,44 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +/** + * TODO: This is a stub implementation, so we can parse and ignore these packets. + * + * https://tools.ietf.org/html/rfc3611#section-2 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P|reserved | PT=XR=207 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * : report blocks : + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class RtcpXrPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + override fun clone(): RtcpXrPacket = RtcpXrPacket(cloneBuffer(0), 0, length) + + companion object { + const val PT: Int = 207 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt new file mode 100644 index 0000000..8a2f56e --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/UnsupportedRtcpPacket.kt @@ -0,0 +1,29 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +/** + * Represents an RTCP packet whose packet types falls in the valid range + * (according to the spec), but isn't currently implemented/supported. + */ +class UnsupportedRtcpPacket( + buf: ByteArray, + offset: Int, + packetLengthBytes: Int +) : RtcpPacket(buf, offset, packetLengthBytes) { + override fun clone(): UnsupportedRtcpPacket = UnsupportedRtcpPacket(cloneBuffer(0), 0, length) +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt new file mode 100644 index 0000000..9253dce --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/RtcpFbPacket.kt @@ -0,0 +1,98 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb + +import org.jitsi.rtp.extensions.bytearray.getInt +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.rtp.rtcp.RtcpHeader +import org.jitsi.rtp.rtcp.RtcpPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.PayloadSpecificRtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbFirPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.RtcpFbNackPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.TransportLayerRtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb.RtcpFbCcfbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket + +/** + * https://tools.ietf.org/html/rfc4585#section-6.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| FMT | PT | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of media source | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * : Feedback Control Information (FCI) : + * : : + * + * Note that an RTCP FB packet re-interprets the standard report count + * (RC) field of the RTCP header as a FMT field + */ +abstract class RtcpFbPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpPacket(buffer, offset, length) { + + var mediaSourceSsrc: Long + get() = getMediaSourceSsrc(buffer, offset) + set(value) = setMediaSourceSsrc(buffer, offset, value) + + companion object { + val PACKET_TYPES = listOf(TransportLayerRtcpFbPacket.PT, PayloadSpecificRtcpFbPacket.PT) + const val MEDIA_SOURCE_SSRC_OFFSET = RtcpHeader.SIZE_BYTES + const val HEADER_SIZE = RtcpHeader.SIZE_BYTES + 4 + const val FCI_OFFSET = HEADER_SIZE + + fun getFmt(buf: ByteArray, baseOffset: Int): Int = RtcpHeader.getReportCount(buf, baseOffset) + fun getMediaSourceSsrc(buf: ByteArray, baseOffset: Int): Long = + buf.getInt(baseOffset + MEDIA_SOURCE_SSRC_OFFSET).toPositiveLong() + fun setMediaSourceSsrc(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + MEDIA_SOURCE_SSRC_OFFSET, value.toInt()) + + fun parse(buf: ByteArray, offset: Int, length: Int): RtcpFbPacket { + val packetType = RtcpHeader.getPacketType(buf, offset) + val fmt = getFmt(buf, offset) + return when (packetType) { + TransportLayerRtcpFbPacket.PT -> { + when (fmt) { + RtcpFbNackPacket.FMT -> RtcpFbNackPacket(buf, offset, length) + RtcpFbTccPacket.FMT -> RtcpFbTccPacket(buf, offset, length) + RtcpFbCcfbPacket.FMT -> RtcpFbCcfbPacket(buf, offset, length) + else -> UnsupportedRtcpFbPacket(buf, offset, length) + } + } + + PayloadSpecificRtcpFbPacket.PT -> { + when (fmt) { + RtcpFbFirPacket.FMT -> RtcpFbFirPacket(buf, offset, length) + RtcpFbPliPacket.FMT -> RtcpFbPliPacket(buf, offset, length) + RtcpFbRembPacket.FMT -> RtcpFbRembPacket(buf, offset, length) + else -> UnsupportedRtcpFbPacket(buf, offset, length) + } + } + + else -> throw Exception("Unrecognized RTCPFB payload type: ${packetType.toString(16)}") + } + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt new file mode 100644 index 0000000..b35c2e5 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/UnsupportedRtcpFbPacket.kt @@ -0,0 +1,29 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb + +/** + * Represents an RTCP feedback packet whose packet types falls in the valid range + * (according to the spec), but isn't currently implemented/supported. + */ +class UnsupportedRtcpFbPacket( + buf: ByteArray, + offset: Int, + packetLengthBytes: Int +) : RtcpFbPacket(buf, offset, packetLengthBytes) { + override fun clone(): UnsupportedRtcpFbPacket = UnsupportedRtcpFbPacket(cloneBuffer(0), 0, length) +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/PayloadSpecificRtcpFbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/PayloadSpecificRtcpFbPacket.kt new file mode 100644 index 0000000..103e710 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/PayloadSpecificRtcpFbPacket.kt @@ -0,0 +1,30 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb + +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket + +abstract class PayloadSpecificRtcpFbPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpFbPacket(buffer, offset, length) { + + companion object { + const val PT = 206 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt new file mode 100644 index 0000000..51e97b7 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbFirPacket.kt @@ -0,0 +1,120 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb + +import org.jitsi.rtp.extensions.bytearray.getInt +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils + +/** + * + * https://tools.ietf.org/html/rfc4585#section-6.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| FMT | PT | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of media source | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Seq nr. | Reserved | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | .... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * https://tools.ietf.org/html/rfc5104#section-4.3.1.1 + * + * The SSRC field in the FIR FCI block is used to set the media sender + * SSRC, the media source SSRC field in the RTCPFB header is unused for FIR packets. + * Within the common packet header for feedback messages (as defined in + * section 6.1 of [RFC4585]), the "SSRC of packet sender" field + * indicates the source of the request, and the "SSRC of media source" + * is not used and SHALL be set to 0. The SSRCs of the media senders to + * which the FIR command applies are in the corresponding FCI entries. + * A FIR message MAY contain requests to multiple media senders, using + * one FCI entry per target media sender. + */ +// TODO: support multiple FIR blocks +class RtcpFbFirPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : PayloadSpecificRtcpFbPacket(buffer, offset, length) { + + override fun clone(): RtcpFbFirPacket = RtcpFbFirPacket(cloneBuffer(0), 0, length) + + var mediaSenderSsrc: Long + get() = getMediaSenderSsrc(buffer, offset) + set(value) = setMediaSenderSsrc(buffer, offset, value) + + var seqNum: Int + get() = getSeqNum(buffer, offset) + set(value) = setSeqNum(buffer, offset, value) + + companion object { + const val FMT = 4 + + // TODO: support multiple FCI? + const val SIZE_BYTES = HEADER_SIZE + 8 + + const val MEDIA_SENDER_SSRC_OFFSET = FCI_OFFSET + const val SEQ_NUM_OFFSET = MEDIA_SENDER_SSRC_OFFSET + 4 + + fun getMediaSenderSsrc(buf: ByteArray, baseOffset: Int): Long = + buf.getInt(baseOffset + MEDIA_SENDER_SSRC_OFFSET).toPositiveLong() + fun setMediaSenderSsrc(buf: ByteArray, baseOffset: Int, value: Long) = + buf.putInt(baseOffset + MEDIA_SENDER_SSRC_OFFSET, value.toInt()) + + fun getSeqNum(buf: ByteArray, baseOffset: Int): Int = buf.get(baseOffset + SEQ_NUM_OFFSET).toPositiveInt() + fun setSeqNum(buf: ByteArray, baseOffset: Int, value: Int) = + buf.set(baseOffset + SEQ_NUM_OFFSET, value.toByte()) + } +} + +// TODO: we need an RtcpFbHeader so that we can fill out the mediaSourceSsrc for FB packets +class RtcpFbFirPacketBuilder( + val rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + var mediaSenderSsrc: Long = -1, + var firCommandSeqNum: Int = -1 +) { + + fun build(): RtcpFbFirPacket { + val buf = BufferPool.getArray(RtcpFbFirPacket.SIZE_BYTES) + writeTo(buf, 0) + return RtcpFbFirPacket(buf, 0, RtcpFbFirPacket.SIZE_BYTES) + } + + fun writeTo(buf: ByteArray, offset: Int) { + rtcpHeader.apply { + packetType = PayloadSpecificRtcpFbPacket.PT + reportCount = RtcpFbFirPacket.FMT + length = RtpUtils.calculateRtcpLengthFieldValue(RtcpFbFirPacket.SIZE_BYTES) + } + rtcpHeader.writeTo(buf, offset) + RtcpFbPacket.setMediaSourceSsrc(buf, offset, 0) // SHALL be 0 + RtcpFbFirPacket.setMediaSenderSsrc(buf, offset, mediaSenderSsrc) + RtcpFbFirPacket.setSeqNum(buf, offset, firCommandSeqNum) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbPliPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbPliPacket.kt new file mode 100644 index 0000000..1f7a0ca --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbPliPacket.kt @@ -0,0 +1,74 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb + +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils + +/** + * https://tools.ietf.org/html/rfc4585#section-6.3.1 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| FMT | PT | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of media source | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * PLI does not require parameters. Therefore, the length field MUST be + * 2, and there MUST NOT be any Feedback Control Information. + */ +class RtcpFbPliPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : PayloadSpecificRtcpFbPacket(buffer, offset, length) { + + override fun clone(): RtcpFbPliPacket = RtcpFbPliPacket(cloneBuffer(0), 0, length) + + companion object { + const val FMT = 1 + const val SIZE_BYTES = HEADER_SIZE + } +} + +class RtcpFbPliPacketBuilder( + val rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + var mediaSourceSsrc: Long = -1 +) { + + fun build(): RtcpFbPliPacket { + val buf = BufferPool.getArray(RtcpFbPliPacket.SIZE_BYTES) + writeTo(buf, 0) + return RtcpFbPliPacket(buf, 0, RtcpFbPliPacket.SIZE_BYTES) + } + + fun writeTo(buf: ByteArray, offset: Int) { + rtcpHeader.apply { + packetType = PayloadSpecificRtcpFbPacket.PT + reportCount = RtcpFbPliPacket.FMT + length = RtpUtils.calculateRtcpLengthFieldValue(RtcpFbPliPacket.SIZE_BYTES) + } + rtcpHeader.writeTo(buf, offset) + RtcpFbPacket.setMediaSourceSsrc(buf, offset, mediaSourceSsrc) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt new file mode 100644 index 0000000..0cd8f06 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacket.kt @@ -0,0 +1,186 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb + +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket.Companion.BR_LEN +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket.Companion.NUM_SSRC_LEN +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket.Companion.REMB_LEN +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacket.Companion.getExpAndMantissa +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.getBitsAsInt +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getIntAsLong +import org.jitsi.rtp.util.getShortAsInt + +/** + * https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| FMT=15 | PT=206 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of media source | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Unique identifier 'R' 'E' 'M' 'B' | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Num SSRC | BR Exp | BR Mantissa | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC feedback | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ... | + * + * @author George Politis + * @author Boris Grozev + */ +class RtcpFbRembPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : PayloadSpecificRtcpFbPacket(buffer, offset, length) { + + val bitrate: Long = getBitrate(buffer, offset) + + val numSsrc: Int = getNumSsrc(buffer, offset) + + /** + * one or more SSRC entries which this feedback message applies to. + */ + val ssrcs: List by lazy { + (0 until numSsrc).map { + getSsrc(buffer, offset, it) + }.toList() + } + + override fun clone(): RtcpFbRembPacket = RtcpFbRembPacket(cloneBuffer(0), 0, length) + + companion object { + const val FMT = 15 + + const val REMB_OFF = FCI_OFFSET + const val REMB_LEN = 4 + const val NUM_SSRC_OFF = REMB_OFF + REMB_LEN + const val NUM_SSRC_LEN = 1 + const val BR_OFF = NUM_SSRC_OFF + NUM_SSRC_LEN + const val BR_LEN = 3 + const val SSRCS_OFF = BR_OFF + BR_LEN + + fun getBrExp(buf: ByteArray, baseOffset: Int): Int = buf.getBitsAsInt(baseOffset + BR_OFF, 0, 6) + fun getBrMantissa(buf: ByteArray, baseOffset: Int): Int = + (buf.getBitsAsInt(baseOffset + BR_OFF, 6, 2) shl 16) + buf.getShortAsInt(baseOffset + BR_OFF + 1) + fun getBitrate(buf: ByteArray, baseOffset: Int): Long { + val mantissa = getBrMantissa(buf, baseOffset) + val exp = getBrExp(buf, baseOffset) + val brBps = mantissa.toLong() shl exp + if ((brBps shr exp).toInt() != mantissa) { + // This block catches a Java long overflow (i.e. the bitrate larger than Long.MAX_VALUE). Although this + // could potentially indicate a malformed remb or a bug in our code, we chose to interpret as the remote + // party trying to signal unbounded bandwidth. + return Long.MAX_VALUE + } + + return brBps + } + fun getNumSsrc(buf: ByteArray, baseOffset: Int): Int = buf.getByteAsInt(baseOffset + NUM_SSRC_OFF) + fun getSsrc(buf: ByteArray, baseOffset: Int, ssrcIndex: Int) = + buf.getIntAsLong(baseOffset + SSRCS_OFF + ssrcIndex * 4) + + fun getExpAndMantissa(brBps: Long): Pair { + // 6 bit Exp + // 18 bit mantissa + var exp = 0 + var mantissa = if (brBps > 0) brBps else 0 + + // 0x3ffff (262143) is the max value that can be put into an 18-bit unsigned integer + while (mantissa > 0x3ffff) { + mantissa = mantissa shr 1 + ++exp + } + + return Pair(exp, mantissa.toInt()) + } + + fun setRemb(buf: ByteArray, baseOffset: Int) { + buf[baseOffset + REMB_OFF + 0] = 'R'.code.toByte() + buf[baseOffset + REMB_OFF + 1] = 'E'.code.toByte() + buf[baseOffset + REMB_OFF + 2] = 'M'.code.toByte() + buf[baseOffset + REMB_OFF + 3] = 'B'.code.toByte() + } + + fun setNumSsrc(buf: ByteArray, off: Int, value: Int) { + buf[off + NUM_SSRC_OFF] = value.toByte() + } + + fun setBrExp(buf: ByteArray, baseOffset: Int, value: Int) { + buf[baseOffset + BR_OFF] = + ((buf[baseOffset + BR_OFF].toInt() and 0x03) or (value and 0x3f shl 2)).toByte() + } + + fun setBrMantissa(buf: ByteArray, baseOffset: Int, value: Int) { + buf[baseOffset + BR_OFF] = + ((buf[baseOffset + BR_OFF].toInt() and 0xfc) or ((value shr 16) and 0x03)).toByte() + buf[baseOffset + BR_OFF + 1] = ((value shr 8) and 0xff).toByte() + buf[baseOffset + BR_OFF + 2] = (value and 0xff).toByte() + } + + fun setSsrcs(buf: ByteArray, baseOffset: Int, ssrcs: List) { + var ssrcsOff = baseOffset + SSRCS_OFF + ssrcs.forEach { + buf.putInt(ssrcsOff, it.toInt()) + ssrcsOff += 4 + } + } + } +} + +class RtcpFbRembPacketBuilder( + val rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + val ssrcs: List = emptyList(), + val brBps: Long +) { + private val sizeBytes: Int = RtcpFbPacket.HEADER_SIZE + + REMB_LEN + NUM_SSRC_LEN + BR_LEN + ssrcs.size * 4 + + fun build(): RtcpFbRembPacket { + val buf = BufferPool.getArray(sizeBytes) + writeTo(buf, 0) + return RtcpFbRembPacket(buf, 0, sizeBytes) + } + + fun writeTo(buf: ByteArray, offset: Int) { + rtcpHeader.apply { + packetType = PayloadSpecificRtcpFbPacket.PT + reportCount = RtcpFbRembPacket.FMT + length = RtpUtils.calculateRtcpLengthFieldValue(sizeBytes) + } + rtcpHeader.writeTo(buf, offset) + RtcpFbPacket.setMediaSourceSsrc(buf, offset, 0) + RtcpFbRembPacket.setRemb(buf, offset) + RtcpFbRembPacket.setNumSsrc(buf, offset, ssrcs.size) + + val expAndMantissa = getExpAndMantissa(brBps) + RtcpFbRembPacket.setBrExp(buf, offset, expAndMantissa.first) + RtcpFbRembPacket.setBrMantissa(buf, offset, expAndMantissa.second) + RtcpFbRembPacket.setSsrcs(buf, offset, ssrcs) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/RtcpFbNackPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/RtcpFbNackPacket.kt new file mode 100644 index 0000000..a38e3b9 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/RtcpFbNackPacket.kt @@ -0,0 +1,177 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb + +import org.jitsi.rtp.extensions.bytearray.getShort +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import java.util.SortedSet + +/** + * https://tools.ietf.org/html/rfc4585#section-6.2.1 + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| FMT | PT | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of media source | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | PID | BLP | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | (optional) PID | BLP | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + */ +class RtcpFbNackPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : TransportLayerRtcpFbPacket(buffer, offset, length) { + + private val numNackBlocks: Int = + (packetLength - HEADER_SIZE) / NackBlock.SIZE_BYTES + + val missingSeqNums: SortedSet by lazy { + (0 until numNackBlocks) + .map { + NackBlock.getMissingSeqNums(buffer, offset + NACK_BLOCK_OFFSET + it * NackBlock.SIZE_BYTES) + } + .flatten() + .toSortedSet() + } + + override fun clone(): RtcpFbNackPacket = RtcpFbNackPacket(cloneBuffer(0), 0, length) + + companion object { + const val FMT = 1 + const val NACK_BLOCK_OFFSET = HEADER_SIZE + } +} + +class RtcpFbNackPacketBuilder( + val rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + var mediaSourceSsrc: Long = -1, + val missingSeqNums: SortedSet = sortedSetOf() +) { + private val nackBlocks: List = + missingSeqNums.toList().chunkMaxDifference(16) + .map(List::toSortedSet) + .map(::NackBlock) + .toList() + + private val sizeBytes: Int = + RtcpFbPacket.HEADER_SIZE + nackBlocks.size * NackBlock.SIZE_BYTES + + fun build(): RtcpFbNackPacket { + val buf = BufferPool.getArray(sizeBytes) + writeTo(buf, 0) + return RtcpFbNackPacket(buf, 0, sizeBytes) + } + + fun writeTo(buf: ByteArray, offset: Int) { + rtcpHeader.apply { + packetType = TransportLayerRtcpFbPacket.PT + reportCount = RtcpFbNackPacket.FMT + length = RtpUtils.calculateRtcpLengthFieldValue(sizeBytes) + } + rtcpHeader.writeTo(buf, offset) + RtcpFbPacket.setMediaSourceSsrc(buf, offset, mediaSourceSsrc) + nackBlocks.forEachIndexed { index, nackBlock -> + nackBlock.writeTo(buf, offset + RtcpFbNackPacket.NACK_BLOCK_OFFSET + index * NackBlock.SIZE_BYTES) + } + } +} + +/** + * Return a List of Lists, where sub-list is made up of an ordered list + * of values pulled from [this], such that the difference between the + * first element and the last element is not more than [maxDifference] + */ +private fun List.chunkMaxDifference(maxDifference: Int): List> { + val chunks = mutableListOf>() + if (this.isEmpty()) { + return chunks + } + var currentChunk = mutableListOf(first()) + chunks.add(currentChunk) + // Ignore the first value which we already put in the current chunk + drop(1).forEach { + if (it - currentChunk.first() > maxDifference) { + currentChunk = mutableListOf(it) + chunks.add(currentChunk) + } else { + currentChunk.add(it) + } + } + return chunks +} + +/** + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | PID | BLP | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +private class NackBlock( + val missingSeqNums: SortedSet = sortedSetOf() +) { + + fun writeTo(buf: ByteArray, offset: Int) { + putMissingSeqNums(buf, offset, missingSeqNums) + } + + companion object { + const val SIZE_BYTES = 4 + fun getMissingSeqNums(buf: ByteArray, offset: Int): List { + val packetId = buf.getShort(offset + 0).toPositiveInt() + val blp = buf.getShort(offset + 2).toPositiveInt() + val missingSeqNums = mutableListOf(packetId) + for (shiftAmount in 0..15) { + if (((blp ushr shiftAmount) and 0x1) == 1) { + missingSeqNums.add((packetId + shiftAmount + 1) and 0xffff) + } + } + return missingSeqNums + } + + /** + * [missingSeqNums].last() - [missingSeqNums].first() MUST be <= 16 + * [offset] should point to the start of where this NackBlock will go + */ + fun putMissingSeqNums(buf: ByteArray, offset: Int, missingSeqNums: SortedSet) { + val packetId = missingSeqNums.first() + buf.putShort(offset, packetId.toShort()) + var blpField = 0 + for (bitPos in 16 downTo 1) { + if ((bitPos + packetId) in missingSeqNums) { + blpField = blpField or 1 + } + // Don't shift the last time + if (bitPos != 1) { + blpField = blpField shl 1 + } + } + buf.putShort(offset + 2, blpField.toShort()) + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/TransportLayerRtcpFbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/TransportLayerRtcpFbPacket.kt new file mode 100644 index 0000000..9620742 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/TransportLayerRtcpFbPacket.kt @@ -0,0 +1,30 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb + +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket + +abstract class TransportLayerRtcpFbPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : RtcpFbPacket(buffer, offset, length) { + + companion object { + const val PT = 205 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/ccfb/RtcpFbCcfbPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/ccfb/RtcpFbCcfbPacket.kt new file mode 100644 index 0000000..cc861bf --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/ccfb/RtcpFbCcfbPacket.kt @@ -0,0 +1,323 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:enum-entry-name-case") + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb + +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.TransportLayerRtcpFbPacket +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.getIntAsLong +import org.jitsi.rtp.util.getShortAsInt +import org.jitsi.utils.MAX_DURATION +import org.jitsi.utils.MIN_DURATION +import org.jitsi.utils.div +import org.jitsi.utils.secs +import org.jitsi.utils.toDouble +import java.time.Duration +import kotlin.math.min + +/** + * This class is a port of CongestionControlFeedback in + * congestion_control_feedback.h/congestion_control_feedback.cc in Chrome + * https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/modules/rtp_rtcp/source/rtcp_packet/congestion_control_feedback.h;rcl=eda3abcd0f44ed2e4c36586d35bdc06885abd25c + * + * Because of this, it explicitly does NOT try to conform + * to Kotlin style or idioms, instead striving to match the + * Chrome code as closely as possible in an effort to make + * future updates easier. + */ + +enum class EcnMarking(val bits: Byte) { + kNotEct(0), // Not ECN-Capable Transport + kEct1(1), // ECN-Capable Transport + kEct0(2), // Not used by L4s (or webrtc.) + kCe(3); // Congestion experienced + + companion object { + fun fromBits(bits: Byte): EcnMarking { + entries.forEach { + if (it.bits == bits) { + return it + } + } + throw IllegalArgumentException("Invalid ECNMarking value: $bits") + } + } +} + +sealed class PacketInfo( + val ssrc: Long = 0, + val sequenceNumber: Int = 0 +) { + override fun toString(): String = "ssrc=$ssrc, sequenceNumber=$sequenceNumber" +} + +class UnreceivedPacketInfo(ssrc: Long = 0, sequenceNumber: Int = 0) : PacketInfo(ssrc, sequenceNumber) { + override fun toString(): String = "Unreceived: " + super.toString() +} + +class ReceivedPacketInfo( + ssrc: Long = 0, + sequenceNumber: Int = 0, + val arrivalTimeOffset: Duration = MIN_DURATION, + val ecn: EcnMarking = EcnMarking.kNotEct +) : PacketInfo(ssrc, sequenceNumber) { + override fun toString(): String = + "Received: " + super.toString() + ", arrivalTimeOffset=$arrivalTimeOffset, ecn=$ecn" +} + +private const val kHeaderPerMediaSsrcLength = 8 +private const val kTimestampLength = 4 + +class RtcpFbCcfbPacketBuilder( + val rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + var mediaSourceSsrc: Long = -1, + // `Packets` MUST be sorted in sequence_number order per SSRC. There MUST not + // be missing sequence numbers between `Packets`. `Packets` MUST not include + // duplicate sequence numbers. + val packets: List = listOf(), + val reportTimestampCompactNtp: Long +) { + private val groupedPackets = packets.groupBy { it.ssrc } + + private fun blockLength(): Int { + var totalSize = RtcpFbPacket.HEADER_SIZE + kTimestampLength + + groupedPackets.forEach { (ssrc, packets) -> + val blockSize = packets.size * 2 + totalSize += kHeaderPerMediaSsrcLength + blockSize + RtpUtils.getNumPaddingBytes(blockSize) + } + + return totalSize + } + + fun writeTo(buf: ByteArray, offset: Int) { + val blockLength = blockLength() + assert(blockLength % 4 == 0) // Will never need padding + val packetSize = blockLength + require(buf.size - offset >= packetSize) { + "Buffer of size ${buf.size} with offset $offset too small for CCFB packet of size $packetSize" + } + rtcpHeader.apply { + packetType = TransportLayerRtcpFbPacket.PT + reportCount = RtcpFbCcfbPacket.FMT + length = RtpUtils.calculateRtcpLengthFieldValue(blockLength) + }.writeTo(buf, offset) + + RtcpFbPacket.setMediaSourceSsrc(buf, offset, mediaSourceSsrc) + var position = offset + RtcpFbPacket.HEADER_SIZE + + groupedPackets.forEach { (ssrc, packets) -> + check(packets.size <= 16384) { + "Unexpected number of reports: ${packets.size}" + } + + buf.putInt(position, ssrc.toInt()) + position += 4 + + val beginSeq = packets.first().sequenceNumber + val numReports = packets.size + + buf.putShort(position, beginSeq.toShort()) + position += 2 + buf.putShort(position, numReports.toShort()) + position += 2 + + packets.forEachIndexed { i, packet -> + val expectedSeq = RtpUtils.applySequenceNumberDelta(beginSeq, i) + check(packet.sequenceNumber == expectedSeq) { + "Sequence number for report $i of $ssrc is wrong " + + "(expected $expectedSeq == $beginSeq + $i, got ${packet.sequenceNumber})" + } + val packetInfo = if (packet is ReceivedPacketInfo) { + 0x8000 or to2BitEcn(packet.ecn) or to13BitAto(packet.arrivalTimeOffset) + } else { + 0 + } + buf.putShort(position, packetInfo.toShort()) + position += 2 + } + repeat(RtpUtils.getNumPaddingBytes(2 * numReports)) { + buf[position++] = 0x00 + } + } + + buf.putInt(position, reportTimestampCompactNtp.toInt()) + position += 4 + assert(position - offset == packetSize) + } + + fun build(): RtcpFbCcfbPacket { + val blockLength = blockLength() + val packetSize = blockLength + RtpUtils.getNumPaddingBytes(blockLength) + val buf = BufferPool.getArray(packetSize) + writeTo(buf, 0) + return RtcpFbCcfbPacket(buf, 0, packetSize) + } +} + +// Arrival time offset (ATO, 13 bits): +// The arrival time of the RTP packet at the receiver, as an offset before the +// time represented by the Report Timestamp (RTS) field of this RTCP congestion +// control feedback report. The ATO field is in units of 1/1024 seconds (this +// unit is chosen to give exact offsets from the RTS field) so, for example, an +// ATO value of 512 indicates that the corresponding RTP packet arrived exactly +// half a second before the time instant represented by the RTS field. If the +// measured value is greater than 8189/1024 seconds (the value that would be +// coded as 0x1FFD), the value 0x1FFE MUST be reported to indicate an over-range +// measurement. If the measurement is unavailable or if the arrival time of the +// RTP packet is after the time represented by the RTS field, then an ATO value +// of 0x1FFF MUST be reported for the packet. +private fun to13BitAto(arrivalTimeOffset: Duration): Int { + if (arrivalTimeOffset.isNegative) { + return 0x1FFF + } + return min((1024 * arrivalTimeOffset.toDouble()).toInt(), 0x1FFE) +} + +private fun atoToTimeDelta(receiveInfo: Int): Duration { + val ato = receiveInfo and 0x1FFF + if (ato == 0x1FFE) { + return MAX_DURATION + } + if (ato == 0x1FFF) { + return MIN_DURATION + } + return ato.secs / 1024 +} + +private fun toEcnMarking(receiveInfo: Int): EcnMarking = EcnMarking.fromBits(((receiveInfo shr 13) and 0b11).toByte()) + +private fun to2BitEcn(ecnMarking: EcnMarking): Int = ecnMarking.bits.toInt() shl 13 + +/* + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |V=2|P| FMT=11 | PT = 205 | length | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of RTCP packet sender | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of 1st RTP Stream | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | begin_seq | num_reports | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |R|ECN| Arrival time offset | ... . + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + . . + . . + . . + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | SSRC of nth RTP Stream | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | begin_seq | num_reports | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |R|ECN| Arrival time offset | ... | + . . + . . + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Report Timestamp (32 bits) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +*/ + +class RtcpFbCcfbPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : TransportLayerRtcpFbPacket(buffer, offset, length) { + /** + * Because much of time this packet is one that we built (not one + * that came in from the network) we don't care about re-parsing all + * of these fields. To avoid doing this work, we put them in this + * data class and make its initialization lazy: they'll only be parsed + * if we access them (which we do for packets that are received from + * the network but not for ones we send out). + */ + private data class CcfbData( + val packets: List, + val reportTimestampCompactNtp: Long + ) + + private val data: CcfbData by lazy(LazyThreadSafetyMode.NONE) { + val packets = mutableListOf() + + require(packetLength - FCI_OFFSET >= kTimestampLength) { + "CCFB packet is too small for timestamp" + } + val reportTimestampCompactNtp = buffer.getIntAsLong(packetLength - kTimestampLength) + + var position = offset + FCI_OFFSET + val end = offset + packetLength - kTimestampLength + + while (position + kHeaderPerMediaSsrcLength < end) { + val ssrc = buffer.getIntAsLong(position) + position += 4 + val baseSeqno = buffer.getShortAsInt(position) + position += 2 + val numReports = buffer.getShortAsInt(position) + position += 2 + + require(position + 2 * numReports <= end) { + "Reports would extend past end of CCFB packet" + } + + for (i in 0 until numReports) { + val packetInfo = buffer.getShortAsInt(position) + position += 2 + val seqNo = RtpUtils.applySequenceNumberDelta(baseSeqno, i) + val received = (packetInfo and 0x8000) != 0 + packets.add( + if (received) { + ReceivedPacketInfo( + ssrc = ssrc, + sequenceNumber = seqNo, + arrivalTimeOffset = atoToTimeDelta(packetInfo), + ecn = toEcnMarking(packetInfo) + ) + } else { + UnreceivedPacketInfo( + ssrc = ssrc, + sequenceNumber = seqNo + ) + } + ) + } + if ((numReports % 2) != 0) { + // 2 bytes padding + position += 2 + } + } + require(position == end) { + "CCFB packet size was incorrect" + } + + CcfbData(packets, reportTimestampCompactNtp) + } + + val packets: List + get() = data.packets + + val reportTimestampCompactNtp: Long + get() = data.reportTimestampCompactNtp + + override fun clone(): RtcpFbCcfbPacket = RtcpFbCcfbPacket(cloneBuffer(0), 0, length) + companion object { + const val FMT = 11 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt new file mode 100644 index 0000000..c8e1543 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/LastChunk.kt @@ -0,0 +1,230 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:function-naming") + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc + +import kotlin.math.min + +// NOTE(brian): We have to store Chunk as an Int to avoid sign issues +typealias Chunk = Int + +/** + * This class is a port of TransportFeedback::LastChunk in + * transport_feedback.h/transport_feedback.cc in Chrome + * https://cs.chromium.org/chromium/src/third_party/webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h?l=95&rcl=20393ee9b7ba622f254908646a9c31bf87349fc7 + * + * Because of this, it explicitly does NOT try to conform + * to Kotlin style or idioms, instead striving to match the + * Chrome code as closely as possible in an effort to make + * future updates easier. + */ +class LastChunk { + fun Empty(): Boolean = size_ == 0 + + fun Clear() { + size_ = 0 + all_same_ = true + has_large_delta_ = false + } + + // Return if delta sizes still can be encoded into single chunk with added + // |delta_size|. + fun CanAdd(deltaSize: DeltaSize): Boolean { + if (size_ < kMaxTwoBitCapacity) { + return true + } + if (size_ < kMaxOneBitCapacity && !has_large_delta_ && deltaSize != kLarge) { + return true + } + if (size_ < kMaxRunLengthCapacity && all_same_ && delta_sizes_[0] == deltaSize) { + return true + } + return false + } + + // Add |delta_size|, assumes |CanAdd(delta_size)|, + fun Add(deltaSize: DeltaSize) { + if (size_ < kMaxVectorCapacity) { + delta_sizes_[size_] = deltaSize + } + size_++ + all_same_ = all_same_ && deltaSize == delta_sizes_[0] + has_large_delta_ = has_large_delta_ || deltaSize == kLarge + } + + // Encode chunk as large as possible removing encoded delta sizes. + // Assume CanAdd() == false for some valid delta_size. + fun Emit(): Chunk { + if (all_same_) { + val chunk = EncodeRunLength() + Clear() + return chunk + } + if (size_ == kMaxOneBitCapacity) { + val chunk = EncodeOneBit() + Clear() + return chunk + } + val chunk = EncodeTwoBit(kMaxTwoBitCapacity) + // Remove |kMaxTwoBitCapacity| encoded delta sizes: + // Shift remaining delta sizes and recalculate all_same_ && has_large_delta_. + size_ -= kMaxTwoBitCapacity + all_same_ = true + has_large_delta_ = false + for (i in 0 until size_) { + val deltaSize = delta_sizes_[kMaxTwoBitCapacity + i] + delta_sizes_[i] = deltaSize + all_same_ = all_same_ && deltaSize == delta_sizes_[0] + has_large_delta_ = has_large_delta_ || deltaSize == kLarge + } + + return chunk + } + + // // Encode all stored delta_sizes into single chunk, pad with 0s if needed. + fun EncodeLast(): Chunk = when { + all_same_ -> EncodeRunLength() + size_ <= kMaxTwoBitCapacity -> EncodeTwoBit(size_) + else -> EncodeOneBit() + } + + // Decode up to |max_size| delta sizes from |chunk|. + fun Decode(chunk: Chunk, max_size: Int) { + if ((chunk and 0x8000) == 0) { + DecodeRunLength(chunk, max_size) + } else if ((chunk and 0x4000) == 0) { + DecodeOneBit(chunk, max_size) + } else { + DecodeTwoBit(chunk, max_size) + } + } + + // Appends content of the Lastchunk to |deltas|. + fun AppendTo(deltas: MutableList) { + if (all_same_) { + repeat(size_) { + deltas.add(delta_sizes_[0]) + } + } else { + for (i in 0 until size_) { + deltas.add(delta_sizes_[i]) + } + } + } + + // private: + + /** + * + * Run Length Status Vector Chunk + * + * 0 1 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |T| S | Run Length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * T = 0 + * S = symbol + * Run Length = Unsigned integer denoting the run length of the symbol + */ + private fun EncodeRunLength(): Chunk = ((delta_sizes_[0] shl 13) or size_) + + private fun DecodeRunLength(chunk: Chunk, max_count: Int) { + size_ = min(chunk and 0x1fff, max_count) + val delta_size = (chunk ushr 13) and 0x03 + has_large_delta_ = delta_size >= kLarge + all_same_ = true + // To make it consistent with Add function, populate delta_sizes beyound 1st. + for (i in 0 until min(size_, kMaxVectorCapacity)) { + delta_sizes_[i] = delta_size + } + } + + /** + * One Bit Status Vector Chunk + * + * 0 1 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |T|S| symbol list | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * T = 1 + * S = 0 + * Symbol list = 14 entries where 0 = not received, 1 = received 1-byte delta. + */ + private fun EncodeOneBit(): Chunk { + var chunk = 0x8000 + for (i in 0 until size_) { + chunk = (chunk or (delta_sizes_[i] shl (kMaxOneBitCapacity - 1 - i))) + } + return chunk + } + + private fun DecodeOneBit(chunk: Chunk, max_size: Int) { + size_ = min(kMaxOneBitCapacity, max_size) + has_large_delta_ = false + all_same_ = false + for (i in 0 until size_) { + delta_sizes_[i] = (chunk ushr (kMaxOneBitCapacity - 1 - i)) and 0x01 + } + } + + /** + * Two Bit Status Vector Chunk + * + * 0 1 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |T|S| symbol list | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + * T = 1 + * S = 1 + * symbol list = 7 entries of two bits each. + */ + private fun EncodeTwoBit(size: Int): Chunk { + var chunk = 0xC000 + for (i in 0 until size) { + chunk = (chunk or (delta_sizes_[i] shl (2 * (kMaxTwoBitCapacity - 1 - i)))) + } + return chunk + } + + private fun DecodeTwoBit(chunk: Chunk, max_size: Int) { + size_ = min(kMaxTwoBitCapacity, max_size) + has_large_delta_ = true + all_same_ = false + for (i in 0 until size_) { + delta_sizes_[i] = (chunk ushr (2 * (kMaxTwoBitCapacity - 1 - i))) and 0x03 + } + } + + private var size_: Int = 0 + private var all_same_: Boolean = true + private var has_large_delta_: Boolean = false + private val delta_sizes_ = Array(kMaxVectorCapacity) { 0 } + + companion object { + private const val kMaxRunLengthCapacity = 0x1FFF + private const val kMaxOneBitCapacity = 14 + private const val kMaxTwoBitCapacity = 7 + private const val kMaxVectorCapacity = kMaxOneBitCapacity + private const val kLarge: DeltaSize = 2 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt new file mode 100644 index 0000000..30a07f9 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacket.kt @@ -0,0 +1,505 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@file:Suppress("ktlint:standard:property-naming", "ktlint:standard:function-naming") + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.rtp.extensions.bytearray.put3Bytes +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.rtp.extensions.unsigned.toPositiveShort +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.rtcp.rtcpfb.RtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.TransportLayerRtcpFbPacket +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kBaseScaleFactor +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kChunkSizeBytes +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kDeltaScaleFactor +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kMaxReportedPackets +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kMaxSizeBytes +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kTimeWrapPeriod +import org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc.RtcpFbTccPacket.Companion.kTransportFeedbackHeaderSizeBytes +import org.jitsi.rtp.rtp.RtpSequenceNumber +import org.jitsi.rtp.rtp.toRtpSequenceNumber +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.get3BytesAsInt +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getShortAsInt +import org.jitsi.utils.micros +import org.jitsi.utils.roundUpTo +import org.jitsi.utils.times +import org.jitsi.utils.toEpochMicro +import org.jitsi.utils.toMicros +import java.time.Duration +import java.time.Instant + +sealed class PacketReport(val seqNum: Int) + +class UnreceivedPacketReport(seqNum: Int) : PacketReport(seqNum) + +// Size in bytes of a delta time in rtcp packet. +// Valid values are 0 (packet wasn't received), 1 or 2. +typealias DeltaSize = Int + +class ReceivedPacketReport(seqNum: Int, val deltaTicks: Short) : PacketReport(seqNum) { + val deltaDuration: Duration + get() = deltaTicks.toInt() * kDeltaScaleFactor +} + +/** + * This class is a port of TransportFeedback in + * transport_feedback.h/transport_feedback.cc in Chrome + * https://cs.chromium.org/chromium/src/third_party/webrtc/modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h?l=95&rcl=20393ee9b7ba622f254908646a9c31bf87349fc7 + * + * Because of this, it explicitly does NOT try to conform + * to Kotlin style or idioms, instead striving to match the + * Chrome code as closely as possible in an effort to make + * future updates easier. + */ +@SuppressFBWarnings( + value = ["NM_METHOD_NAMING_CONVENTION"], + justification = "This class is a port and use the original names." +) +class RtcpFbTccPacketBuilder( + val rtcpHeader: RtcpHeaderBuilder = RtcpHeaderBuilder(), + var mediaSourceSsrc: Long = -1, + val feedbackPacketSeqNum: Int = -1 +) { + var base_seq_no_: RtpSequenceNumber = RtpSequenceNumber.INVALID + private set + + // The reference time, in ticks. Chrome passes this into BuildFeedbackPacket, but we don't + // hold the times in the same way, so we'll just assign it the first time we see + // a packet in AddReceivedPacket + private var base_time_ticks_: Long = -1 + + // The amount of packets_ whose status are represented + var num_seq_no_ = 0 + private set + + // The current chunk we're 'filling out' as packets + // are received + private var last_chunk_ = LastChunk() + + // All but last encoded packet chunks. + private val encoded_chunks_ = mutableListOf() + + // The size of the entire packet, in bytes + private var size_bytes_ = kTransportFeedbackHeaderSizeBytes + private var last_timestamp_: Instant = Instant.EPOCH + private val packets_ = mutableListOf() + + fun SetBase(base_sequence: Int, ref_timestamp: Instant) { + base_seq_no_ = base_sequence.toRtpSequenceNumber() + base_time_ticks_ = (ref_timestamp.toEpochMicro() % kTimeWrapPeriod.toMicros()) / kBaseScaleFactor.toMicros() + last_timestamp_ = BaseTime() + } + + fun AddReceivedPacket(seqNum: Int, timestampIn: Instant): Boolean { + val sequence_number = seqNum.toRtpSequenceNumber() + var timestamp = timestampIn + if (last_timestamp_ > timestamp) { + timestamp += Duration.between(timestamp, last_timestamp_).roundUpTo(kTimeWrapPeriod) + } + var delta_full = Duration.between(last_timestamp_, timestamp).toMicros() % kTimeWrapPeriod.toMicros() + if (delta_full > kTimeWrapPeriod.toMicros() / 2) { + delta_full -= kTimeWrapPeriod.toMicros() + delta_full -= kDeltaScaleFactor.toMicros() / 2 + } else { + delta_full += kDeltaScaleFactor.toMicros() / 2 + } + delta_full /= kDeltaScaleFactor.toMicros() + + val delta = delta_full.toShort() + // If larger than 16bit signed, we can't represent it - need new fb packet. + if (delta.toLong() != delta_full) { + return false + } + var next_seq_no = base_seq_no_ + num_seq_no_ + if (sequence_number != next_seq_no) { + val lastSeqNo = next_seq_no - 1 + if (sequence_number <= lastSeqNo) { + return false + } + while (next_seq_no != sequence_number) { + if (!AddDeltaSize(0)) { + return false + } + next_seq_no += 1 + } + } + val delta_size = if (delta >= 0 && delta <= 0xff) 1 else 2 + if (!AddDeltaSize(delta_size)) { + return false + } + + packets_.add(ReceivedPacketReport(sequence_number.value, delta)) + last_timestamp_ += delta.toInt() * kDeltaScaleFactor + size_bytes_ += delta_size + + return true + } + + fun BaseTime(): Instant = Instant.EPOCH + base_time_ticks_ * kBaseScaleFactor + + private fun AddDeltaSize(deltaSize: DeltaSize): Boolean { + if (num_seq_no_ == kMaxReportedPackets) { + return false + } + val add_chunk_size = if (last_chunk_.Empty()) kChunkSizeBytes else 0 + + if (size_bytes_ + deltaSize + add_chunk_size > kMaxSizeBytes) { + return false + } + + if (last_chunk_.CanAdd(deltaSize)) { + size_bytes_ += add_chunk_size + last_chunk_.Add(deltaSize) + ++num_seq_no_ + return true + } + + if (size_bytes_ + deltaSize + kChunkSizeBytes > kMaxSizeBytes) { + return false + } + + encoded_chunks_.add(last_chunk_.Emit()) + size_bytes_ += kChunkSizeBytes + last_chunk_.Add(deltaSize) + ++num_seq_no_ + return true + } + + fun build(): RtcpFbTccPacket { + val packetSize = size_bytes_ + RtpUtils.getNumPaddingBytes(size_bytes_) + val buf = BufferPool.getArray(packetSize) + writeTo(buf, 0) + return RtcpFbTccPacket(buf, 0, packetSize) + } + + fun writeTo(buf: ByteArray, offset: Int) { + // NOTE: padding is held 'internally' in the TCC FCI, so we don't set + // the padding bit on the header + val paddingBytes = RtpUtils.getNumPaddingBytes(size_bytes_) + rtcpHeader.apply { + packetType = TransportLayerRtcpFbPacket.PT + reportCount = RtcpFbTccPacket.FMT + length = RtpUtils.calculateRtcpLengthFieldValue(size_bytes_ + paddingBytes) + }.writeTo(buf, offset) + + RtcpFbPacket.setMediaSourceSsrc(buf, offset, mediaSourceSsrc) + RtcpFbTccPacket.setBaseSeqNum(buf, offset, base_seq_no_.value) + RtcpFbTccPacket.setPacketStatusCount(buf, offset, num_seq_no_) + RtcpFbTccPacket.setReferenceTimeTicks(buf, offset, base_time_ticks_.toInt()) + RtcpFbTccPacket.setFeedbackPacketCount(buf, offset, feedbackPacketSeqNum) + + var currOffset = RtcpFbTccPacket.PACKET_CHUNKS_OFFSET + encoded_chunks_.forEach { + buf.putShort(currOffset, it.toShort()) + currOffset += kChunkSizeBytes + } + if (!last_chunk_.Empty()) { + val chunk = last_chunk_.EncodeLast() + buf.putShort(currOffset, chunk.toShort()) + currOffset += kChunkSizeBytes + } + packets_.forEach { + if (it is ReceivedPacketReport) { + when (it.deltaTicks) { + in 0..0xFF -> buf[currOffset++] = it.deltaTicks.toByte() + + else -> { + buf.putShort(currOffset, it.deltaTicks) + currOffset += 2 + } + } + } + } + repeat(paddingBytes) { + buf[currOffset++] = 0x00 + } + } + + fun clear() { + num_seq_no_ = 0 + size_bytes_ = kTransportFeedbackHeaderSizeBytes + } +} + +/** + * NOTE(brian): This class is a port of the rest of the logic in TransportFeedback + * not covered by RtcpFbTccPacketBuilder. Chrome uses a single class for both + * the 'builder' and the 'parser' but because of the way we define packets + * (inheriting from the buffer type and therefore always requiring a valid buffer), + * we separate builders out into their own class. Because of that, this class + * and RtcpFbTccPacketBuilder have overlap in their members. + * + * https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01#section-3.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P| FMT=15 | PT=205 | length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of packet sender | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | SSRC of media source | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | base sequence number | packet status count | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | reference time | fb pkt. count | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | packet chunk | packet chunk | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * . . + * . . + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | packet chunk | recv delta | recv delta | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * . . + * . . + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | recv delta | recv delta | zero padding | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * packet status count: 16 bits The number of packets this feedback + * contains status for, starting with the packet identified + * by the base sequence number. + * + * feedback packet count: 8 bits A counter incremented by one for each + * feedback packet sent. Used to detect feedback packet + * losses. + */ +@SuppressFBWarnings( + value = ["NM_METHOD_NAMING_CONVENTION"], + justification = "This class is a port and use the original names." +) +class RtcpFbTccPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : TransportLayerRtcpFbPacket(buffer, offset, length), Iterable { + + /** + * Because much of time this packet is one that we built (not one + * that came in from the network) we don't care about re-parsing all + * of these fields. To avoid doing this work, we put them in this + * data class and make its initialization lazy: they'll only be parsed + * if we access them (which we do for packets that are received from + * the network but not for ones we send out). + */ + private data class TccMemberData( + val base_seq_no_: Int, + var base_time_ticks_: Long, + val encoded_chunks_: MutableList, + var last_chunk_: LastChunk, + var num_seq_no_: Int, + var last_timestamp_: Instant, + val packets_: MutableList + ) + + private val data: TccMemberData by lazy(LazyThreadSafetyMode.NONE) { + val base_seq_no_ = getBaseSeqNum(buffer, offset).toRtpSequenceNumber() + val status_count = getPacketStatusCount(buffer, offset) + val encoded_chunks_ = mutableListOf() + val last_chunk_ = LastChunk() + val num_seq_no_: Int + var last_timestamp_: Instant = Instant.EPOCH + val packets_ = mutableListOf() + + val base_time_ticks_ = getReferenceTimeTicks(buffer, offset) + val delta_sizes = mutableListOf() + var index = offset + PACKET_CHUNKS_OFFSET + val end_index = offset + length + while (delta_sizes.size < status_count) { + if (index + kChunkSizeBytes > end_index) { + throw Exception("Buffer overflow while parsing packet.") + } + val chunk = buffer.getShortAsInt(index) + index += kChunkSizeBytes + encoded_chunks_.add(chunk) + last_chunk_.Decode(chunk, status_count - delta_sizes.size) + last_chunk_.AppendTo(delta_sizes) + } + // Last chunk is stored in the |last_chunk_|. + encoded_chunks_.dropLast(1) + num_seq_no_ = status_count + + var seq_no = base_seq_no_ + var recv_delta_size = 0 + for (delta_size in delta_sizes) { + recv_delta_size += delta_size + } + + // Determine if timestamps, that is, recv_delta are included in the packet. + if (end_index >= index + recv_delta_size) { + for (delta_size in delta_sizes) { + if (index + delta_size > end_index) { + throw Exception("Buffer overflow while parsing packet.") + } + when (delta_size) { + 0 -> packets_.add(UnreceivedPacketReport(seq_no.value)) + + 1 -> { + val delta = buffer[index] + packets_.add(ReceivedPacketReport(seq_no.value, delta.toPositiveShort())) + last_timestamp_ += delta.toInt() * kDeltaScaleFactor + index += delta_size + } + + 2 -> { + val delta = buffer.getShortAsInt(index) + packets_.add(ReceivedPacketReport(seq_no.value, delta.toShort())) + last_timestamp_ += delta * kDeltaScaleFactor + index += delta_size + } + + 3 -> { + throw Exception("Warning: invalid delta size for seq_no $seq_no") + } + } + seq_no += 1 + } + } else { + // The packet does not contain receive deltas + for (delta_size in delta_sizes) { + // Use delta sizes to detect if packet was received. + if (delta_size == 0) { + packets_.add(UnreceivedPacketReport(seq_no.value)) + } else { + packets_.add(ReceivedPacketReport(seq_no.value, 0)) + } + seq_no += 1 + } + } + TccMemberData( + base_seq_no_.value, + base_time_ticks_, + encoded_chunks_, + last_chunk_, + num_seq_no_, + last_timestamp_, + packets_ + ) + } + + // All but last encoded packet chunks. + private val encoded_chunks_: MutableList + get() = data.encoded_chunks_ + + // The current chunk we're 'filling out' as packets + // are received + private var last_chunk_: LastChunk + get() = data.last_chunk_ + set(value) { + data.last_chunk_ = value + } + private val base_seq_no_: Int + get() = data.base_seq_no_ + private var num_seq_no_: Int + get() = data.num_seq_no_ + set(value) { + data.num_seq_no_ = value + } + private val packets_: MutableList + get() = data.packets_ + private var last_timestamp_: Instant + get() = data.last_timestamp_ + set(value) { + data.last_timestamp_ = value + } + + // The reference time, in ticks. + private var base_time_ticks_: Long + get() = data.base_time_ticks_ + set(value) { + data.base_time_ticks_ = value + } + + val feedbackSeqNum: Int = getFeedbackPacketCount(buffer, offset) + + fun GetPacketStatusCount(): Int = num_seq_no_ + + fun BaseTime(): Instant = Instant.EPOCH + base_time_ticks_ * kBaseScaleFactor + + fun GetBaseDelta(prev_timestamp: Instant): Duration { + var delta = Duration.between(prev_timestamp, BaseTime()) + // Compensate for wrap around + if ((delta - kTimeWrapPeriod).abs() < delta.abs()) { + delta -= kTimeWrapPeriod + } else if ((delta + kTimeWrapPeriod).abs() < delta.abs()) { + delta += kTimeWrapPeriod + } + return delta + } + + override fun iterator(): Iterator = packets_.iterator() + + override fun clone(): RtcpFbTccPacket = RtcpFbTccPacket(cloneBuffer(0), 0, length) + + companion object { + const val FMT = 15 + + // Convert to multiples of 0.25ms + val kDeltaScaleFactor = 250.micros + + // Maximum number of packets_ (including missing) TransportFeedback can report. + const val kMaxReportedPackets = 0xFFFF + const val kChunkSizeBytes = 2 + + // Fit TCC packets within an MTU and allow for further encapsulation (and perhaps compound RTCP) + const val kMaxSizeBytes = 1200 + + // Header size: + // * 4 bytes Common RTCP Packet Header + // * 8 bytes Common Packet Format for RTCP Feedback Messages + // * 8 bytes FeedbackPacket header + const val kTransportFeedbackHeaderSizeBytes = 4 + 8 + 8 + + // Used to convert from microseconds to multiples of 64ms + val kBaseScaleFactor = kDeltaScaleFactor * (1 shl 8) + + // The reference time field is 24 bits and are represented as multiples of 64ms + // When the reference time field would need to wrap around + val kTimeWrapPeriod = (1 shl 24).toLong() * kBaseScaleFactor + + const val BASE_SEQ_NUM_OFFSET = RtcpFbPacket.HEADER_SIZE + const val PACKET_STATUS_COUNT_OFFSET = RtcpFbPacket.HEADER_SIZE + 2 + const val REFERENCE_TIME_OFFSET = RtcpFbPacket.HEADER_SIZE + 4 + const val FB_PACKET_COUNT_OFFSET = RtcpFbPacket.HEADER_SIZE + 7 + const val PACKET_CHUNKS_OFFSET = RtcpFbPacket.HEADER_SIZE + 8 + + // baseOffset in all of these refers to the start of the entire RTCP TCC packet + fun getBaseSeqNum(buf: ByteArray, baseOffset: Int): Int = buf.getShortAsInt(baseOffset + BASE_SEQ_NUM_OFFSET) + fun setBaseSeqNum(buf: ByteArray, baseOffset: Int, value: Int) = + buf.putShort(baseOffset + BASE_SEQ_NUM_OFFSET, value.toShort()) + + fun getPacketStatusCount(buf: ByteArray, baseOffset: Int): Int = + buf.getShortAsInt(baseOffset + PACKET_STATUS_COUNT_OFFSET) + fun setPacketStatusCount(buf: ByteArray, baseOffset: Int, value: Int) = + buf.putShort(baseOffset + PACKET_STATUS_COUNT_OFFSET, value.toShort()) + + fun getReferenceTimeTicks(buf: ByteArray, baseOffset: Int): Long = + buf.get3BytesAsInt(baseOffset + REFERENCE_TIME_OFFSET).toPositiveLong() + fun setReferenceTimeTicks(buf: ByteArray, baseOffset: Int, refTimeTicks: Int) = + buf.put3Bytes(baseOffset + REFERENCE_TIME_OFFSET, refTimeTicks) + + fun getFeedbackPacketCount(buf: ByteArray, baseOffset: Int): Int = + buf.getByteAsInt(baseOffset + FB_PACKET_COUNT_OFFSET) + fun setFeedbackPacketCount(buf: ByteArray, baseOffset: Int, value: Int) = + buf.set(baseOffset + FB_PACKET_COUNT_OFFSET, value.toByte()) + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt new file mode 100644 index 0000000..de2da74 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RedPacketParser.kt @@ -0,0 +1,289 @@ +package org.jitsi.rtp.rtp + +import org.jitsi.rtp.Packet.Companion.BYTES_TO_LEAVE_AT_END_OF_PACKET +import org.jitsi.rtp.extensions.bytearray.getBitAsBool +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtp.RtpHeader.Companion.FIXED_HEADER_SIZE_BYTES +import org.jitsi.rtp.rtp.RtpPacket.Companion.BYTES_TO_LEAVE_AT_START_OF_PACKET +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils.Companion.getTimestampDiffAsInt +import org.jitsi.rtp.util.getBitsAsInt +import kotlin.IllegalArgumentException +import kotlin.experimental.and +import kotlin.experimental.or + +/** + * Parses RED (RFC2198) packets. + */ +class RedPacketParser( + /** + * A function to create packets from redundancy blocks (used so we can create a packet with the correct class). + */ + val createPacket: (ByteArray, Int, Int) -> PacketType +) { + /** + * Destructively parses a specific [RtpPacket] as RED. The redundant RED blocks are removed from the payload, + * leaving only the primary block as payload, and the payload type is changed to the payload type of the primary + * block. The transformed packet is no longer a RED packet, so the operation must not be performed again. + * + * If [parseRedundancy] is set, the redundancy blocks are parsed as [PacketType] into newly allocated buffers. + * + * @param rtpPacket the packet to parse + * @param parseRedundancy whether to parse redundancy packets + * @return The list of parsed redundancy packets. + */ + fun decapsulate(rtpPacket: RtpPacket, parseRedundancy: Boolean): List = with(rtpPacket) { + var currentOffset = payloadOffset + + val redundancyBlockHeaders = mutableListOf() + lateinit var primaryBlockHeader: PrimaryBlockHeader + do { + if (currentOffset >= offset + length) { + throw IllegalArgumentException( + "Invalid RED packet: no last header block found within the allowed length." + ) + } + val blockHeader = BlockHeader.parse(buffer, currentOffset) + when (blockHeader) { + is PrimaryBlockHeader -> primaryBlockHeader = blockHeader + is RedundancyBlockHeader -> redundancyBlockHeaders.add(blockHeader) + } + currentOffset += blockHeader.headerLength + } while (blockHeader is RedundancyBlockHeader) + + val packets = if (parseRedundancy) mutableListOf() else null + for ((index, it) in redundancyBlockHeaders.withIndex()) { + val blockLength = it.length + if (currentOffset + blockLength > offset + length) { + throw IllegalArgumentException("Invalid RED packet: blocks extend past the packet length.") + } + + if (parseRedundancy) { + val byteArray = BufferPool.getArray( + BYTES_TO_LEAVE_AT_START_OF_PACKET + + FIXED_HEADER_SIZE_BYTES + blockLength + + BYTES_TO_LEAVE_AT_END_OF_PACKET + ) + + System.arraycopy( + buffer, + offset, + byteArray, + BYTES_TO_LEAVE_AT_START_OF_PACKET, + FIXED_HEADER_SIZE_BYTES + ) + RtpHeader.setCsrcCount(byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET, 0) + RtpHeader.setHasExtensions(byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET, false) + System.arraycopy( + buffer, + currentOffset, + byteArray, + BYTES_TO_LEAVE_AT_START_OF_PACKET + FIXED_HEADER_SIZE_BYTES, + blockLength + ) + + val redundancyPacket = + createPacket(byteArray, BYTES_TO_LEAVE_AT_START_OF_PACKET, FIXED_HEADER_SIZE_BYTES + blockLength) + .apply { + payloadType = it.pt.toInt() + timestamp -= it.timestampOffset + // RFC2198 describes how to reconstruct the timestamp and payload type, but not an RTP + // sequence number. However, we operate on the RTP level and need to reconstruct an RTP + // packet. So we make the assumption that the redundant blocks represent the packets + // directly proceeding the primary. + sequenceNumber = + (sequenceNumber - (redundancyBlockHeaders.size - index)).toRtpSequenceNumber().value + } + + packets?.add(redundancyPacket) + } + currentOffset += blockLength + } + + // We've now read past the redundancy blocks, and currentOffset points to the payload of the primary block. + // Remove RED "in-place" by shifting the header. + val headerLength = headerLength + val newOffset = currentOffset - headerLength + val newLength = length - currentOffset + offset + headerLength + System.arraycopy( + buffer, + offset, + buffer, + newOffset, + headerLength + ) + offset = newOffset + length = newLength + payloadType = primaryBlockHeader.pt.toInt() + + return packets ?: emptyList() + } +} + +sealed class BlockHeader( + /** + * The block's payload type. + */ + val pt: Byte +) { + abstract val headerLength: Int + abstract fun write(buffer: ByteArray, offset: Int): Int + + companion object { + /** + * See RFC2198. + * + * Header blocks before the last (for redundant blocks): + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |1| block PT | timestamp offset | block length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * The last header block (for the primary block): + * 0 + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |0| block PT | + * +-+-+-+-+-+-+-+-+ + * + */ + fun parse(buffer: ByteArray, offset: Int): BlockHeader { + val follow = buffer.getBitAsBool(offset, 0) + val pt = buffer.getBitsAsInt(offset, 1, 7).toByte() + + if (!follow) return PrimaryBlockHeader(pt) + + val timestampOffset = + (buffer[offset + 1].toPositiveInt() shl 6) + buffer.getBitsAsInt(offset + 2, 0, 6) + val blockLength = + (buffer.getBitsAsInt(offset + 2, 6, 2) shl 8) + buffer[offset + 3].toPositiveInt() + + return RedundancyBlockHeader(pt, timestampOffset, blockLength) + } + } +} + +class PrimaryBlockHeader(pt: Byte) : BlockHeader(pt) { + override val headerLength = 1 + override fun write(buffer: ByteArray, offset: Int): Int { + buffer[offset] = pt and 0x7f.toByte() + return 1 + } +} + +class RedundancyBlockHeader( + pt: Byte, + /** + * The (negative) offset of the timestamp relative to the RTP packet/primary block. + */ + val timestampOffset: Int, + /** + * The length of the block's payload in bytes. + */ + val length: Int +) : BlockHeader(pt) { + + init { + require(timestampOffset <= MAX_TIMESTAMP_OFFSET) { "Invalid timestampOffset: $timestampOffset" } + } + override val headerLength = 4 + + override fun write(buffer: ByteArray, offset: Int): Int { + buffer[offset] = pt or 0x80.toByte() + buffer[offset + 1] = (timestampOffset shr 6).toByte() + buffer[offset + 2] = ((timestampOffset and 0x3f) shl 2).toByte() or ((length shr 8).toByte() and 0x03.toByte()) + buffer[offset + 3] = (length and 0xff).toByte() + return 4 + } + + companion object { + /** + * The maximum value of the [timestampOffset] field (a 14-bit uint). + */ + const val MAX_TIMESTAMP_OFFSET = 0x3fff + } +} + +internal class RtpRedPacket(buffer: ByteArray, offset: Int, length: Int) : RtpPacket(buffer, offset, length) { + override fun clone(): RtpRedPacket = RtpRedPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ).also { postClone(it) } + + companion object { + val parser = RedPacketParser { b, o, l -> RtpPacket(b, o, l) } + val builder = RedPacketBuilder { b, o, l -> RtpRedPacket(b, o, l) } + } + + fun decapsulate(parseRedundancy: Boolean) = parser.decapsulate(this, parseRedundancy) +} + +class RedPacketBuilder(val createPacket: (ByteArray, Int, Int) -> PacketType) { + + /** + * Builds and returns a RED packet that contains [primary] as the primary encodings, and [redundancy] as secondary + * encodings. + * Note that this assumes the primary and secondary packets can be encoded in RED and throws as + * [IllegalArgumentException] otherwise (notably when the timestamp difference between the primary and any of the + * redundancy packets is too large). + */ + fun build(redPayloadType: Int, primary: RtpPacket, redundancy: List): PacketType { + val bytesNeeded = primary.length + 1 + redundancy.sumOf { it.payloadLength } + redundancy.size * 4 + + val buf = BufferPool.getArray(bytesNeeded + BYTES_TO_LEAVE_AT_START_OF_PACKET + BYTES_TO_LEAVE_AT_END_OF_PACKET) + + var currentOffset = BYTES_TO_LEAVE_AT_START_OF_PACKET + val primaryHeaderLength = primary.headerLength + + System.arraycopy( + primary.buffer, + primary.offset, + buf, + currentOffset, + primaryHeaderLength + ) + currentOffset += primaryHeaderLength + + var redHeaderOffset = currentOffset + + // Skip the RED headers and point to the start of RED payloads + currentOffset += 1 + 4 * redundancy.size + + redundancy.forEach { + val payloadLength = it.payloadLength + + val header = RedundancyBlockHeader( + it.payloadType.toByte(), + getTimestampDiffAsInt(primary.timestamp, it.timestamp), + payloadLength + ) + redHeaderOffset += header.write(buf, redHeaderOffset) + + System.arraycopy( + it.buffer, + it.payloadOffset, + buf, + currentOffset, + payloadLength + ) + currentOffset += payloadLength + } + + val primaryHeader = PrimaryBlockHeader(primary.payloadType.toByte()) + redHeaderOffset += primaryHeader.write(buf, redHeaderOffset) + + System.arraycopy( + primary.buffer, + primary.payloadOffset, + buf, + currentOffset, + primary.payloadLength + ) + + return createPacket(buf, BYTES_TO_LEAVE_AT_START_OF_PACKET, bytesNeeded).apply { + payloadType = redPayloadType + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt new file mode 100644 index 0000000..8f9477c --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpHeader.kt @@ -0,0 +1,164 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp + +import org.jitsi.rtp.extensions.bytearray.putInt +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtp.header_extensions.HeaderExtensionHelpers +import org.jitsi.rtp.util.getIntAsLong +import org.jitsi.rtp.util.getShortAsInt +import kotlin.experimental.and +import kotlin.experimental.or + +/** + * [RtpHeader] exists only as a set of helper methods to retrieve and set fields inside of + * a [ByteArray]. + * + * It covers the RTP header fields until where the variability starts (the CSRCs list) + * + * + * https://tools.ietf.org/html/rfc3550#section-5.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P|X| CC |M| PT | sequence number | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | timestamp | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | synchronization source (SSRC) identifier | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * | contributing source (CSRC) identifiers | + * | .... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ...extensions (if present)... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * @author Brian Baldino + */ +class RtpHeader { + companion object { + const val FIXED_HEADER_SIZE_BYTES = 12 + const val CSRCS_OFFSET = 12 + + // The size of the RTP Extension header block + const val EXT_HEADER_SIZE_BYTES = 4 + const val VERSION = 2 + + fun getVersion(buf: ByteArray, baseOffset: Int): Int = (buf[baseOffset].toInt() and 0xC0) ushr 6 + + fun setVersion(buf: ByteArray, baseOffset: Int, version: Int) { + buf[baseOffset] = ((buf[baseOffset].toInt() and 0xC0.inv()) or ((version shl 6) and 0xC0)).toByte() + } + + fun hasPadding(buf: ByteArray, baseOffset: Int): Boolean = (buf[baseOffset].toInt() and 0x20) == 0x20 + fun setPadding(buf: ByteArray, baseOffset: Int, hasPadding: Boolean) { + buf[baseOffset] = when (hasPadding) { + true -> (buf[baseOffset].toInt() or 0x20).toByte() + false -> (buf[baseOffset].toInt() and 0x20.inv()).toByte() + } + } + + fun hasExtensions(buf: ByteArray, baseOffset: Int): Boolean = (buf[baseOffset].toInt() and 0x10) == 0x10 + fun setHasExtensions(buf: ByteArray, baseOffset: Int, hasExtension: Boolean) { + buf[baseOffset] = when (hasExtension) { + true -> (buf[baseOffset].toInt() or 0x10).toByte() + false -> (buf[baseOffset].toInt() and 0x10.inv()).toByte() + } + } + + fun getCsrcCount(buf: ByteArray, baseOffset: Int): Int = buf[baseOffset].toInt() and 0x0F + fun setCsrcCount(buf: ByteArray, baseOffset: Int, csrcCount: Int) { + buf[baseOffset] = ((buf[baseOffset].toInt() and 0xF0) or ((csrcCount and 0x0F))).toByte() + } + + fun getMarker(buf: ByteArray, baseOffset: Int): Boolean = (buf[baseOffset + 1].toInt() and 0x80) == 0x80 + fun setMarker(buf: ByteArray, baseOffset: Int, isSet: Boolean) { + buf[baseOffset + 1] = when (isSet) { + true -> (buf[baseOffset + 1].toInt() or 0x80).toByte() + false -> (buf[baseOffset + 1].toInt() and 0x80.inv()).toByte() + } + } + + fun getPayloadType(buf: ByteArray, baseOffset: Int): Int = + (buf[baseOffset + 1].toInt() and 0x7F).toPositiveInt() + fun setPayloadType(buf: ByteArray, baseOffset: Int, payloadType: Int) { + buf[baseOffset + 1] = (buf[baseOffset + 1] and 0x80.toByte()) or (payloadType and 0x7F).toByte() + } + + fun getSequenceNumber(buf: ByteArray, baseOffset: Int): Int = buf.getShortAsInt(baseOffset + 2) + fun setSequenceNumber(buf: ByteArray, baseOffset: Int, sequenceNumber: Int) { + buf.putShort(baseOffset + 2, sequenceNumber.toShort()) + } + + fun getTimestamp(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + 4) + fun setTimestamp(buf: ByteArray, baseOffset: Int, timestamp: Long) { + buf.putInt(baseOffset + 4, timestamp.toInt()) + } + + fun getSsrc(buf: ByteArray, baseOffset: Int): Long = buf.getIntAsLong(baseOffset + 8) + fun setSsrc(buf: ByteArray, baseOffset: Int, ssrc: Long) { + buf.putInt(baseOffset + 8, ssrc.toInt()) + } + + fun getCsrcs(buf: ByteArray, baseOffset: Int): List { + val numCsrcs = getCsrcCount(buf, baseOffset) + return (0 until numCsrcs).map { buf.getIntAsLong(CSRCS_OFFSET + (4 * it)) }.toList() + } + + /** + * Assumes there is already proper room for the CSRCS. Also updates + * the CSRC count field. + */ + fun setCsrcs(buf: ByteArray, baseOffset: Int, csrcs: List) { + csrcs.forEachIndexed { index, csrc -> + buf.putInt(CSRCS_OFFSET + (4 * index), csrc.toInt()) + } + setCsrcCount(buf, baseOffset, csrcs.size) + } + + private fun getFixedHeaderAndCcLength(buf: ByteArray, baseOffset: Int) = + FIXED_HEADER_SIZE_BYTES + getCsrcCount(buf, baseOffset) * 4 + + /** + * The length of the entire RTP header, including any extensions, in bytes + */ + fun getTotalLength(buf: ByteArray, baseOffset: Int): Int { + val length = getFixedHeaderAndCcLength(buf, baseOffset) + + val extLength = if (hasExtensions(buf, baseOffset)) { + // Length points to where the ext header would start + val extHeaderOffset = length + HeaderExtensionHelpers.getExtensionsTotalLength(buf, baseOffset + extHeaderOffset) + } else { + 0 + } + return length + extLength + } + + /** + * The "defined by profile" header extension field. Only valid if hasExtensions is true, otherwise + * returns an invalid value (-1) + */ + fun getExtensionsProfileType(buf: ByteArray, baseOffset: Int): Int = if (hasExtensions(buf, baseOffset)) { + val extHeaderOffset = getFixedHeaderAndCcLength(buf, baseOffset) + HeaderExtensionHelpers.getExtensionsProfileType(buf, baseOffset + extHeaderOffset) + } else { + -1 + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt new file mode 100644 index 0000000..f766a6a --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpPacket.kt @@ -0,0 +1,622 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.rtp.Packet +import org.jitsi.rtp.extensions.bytearray.hashCodeOfSegment +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.rtp.header_extensions.HeaderExtensionHelpers +import org.jitsi.rtp.rtp.header_extensions.HeaderExtensionParser +import org.jitsi.rtp.rtp.header_extensions.OneByteHeaderExtensionParser +import org.jitsi.rtp.rtp.header_extensions.TwoByteHeaderExtensionParser +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.RtpUtils +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.isPadding + +/** + * + * https://tools.ietf.org/html/rfc3550#section-5.1 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |V=2|P|X| CC |M| PT | sequence number | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | timestamp | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | synchronization source (SSRC) identifier | + * +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ + * | contributing source (CSRC) identifiers | + * | .... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ...extensions (if present)... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | payload | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +@SuppressFBWarnings( + value = ["EI_EXPOSE_REP2"], + justification = "We intentionally pass a reference to our buffer when using observableWhenChanged." +) +open class RtpPacket( + buffer: ByteArray, + offset: Int, + length: Int +) : Packet(buffer, offset, length) { + + constructor(buffer: ByteArray) : this(buffer, 0, buffer.size) + + var version: Int + get() = RtpHeader.getVersion(buffer, offset) + set(value) = RtpHeader.setVersion(buffer, offset, value) + + var hasPadding: Boolean + get() = RtpHeader.hasPadding(buffer, offset) + set(value) = RtpHeader.setPadding(buffer, offset, value) + + private var hasEncodedExtensions: Boolean + get() = RtpHeader.hasExtensions(buffer, offset) + set(value) = RtpHeader.setHasExtensions(buffer, offset, value) + + val csrcCount: Int + get() = RtpHeader.getCsrcCount(buffer, offset) + + var isMarked: Boolean + get() = RtpHeader.getMarker(buffer, offset) + set(value) = RtpHeader.setMarker(buffer, offset, value) + + /* The four values below (payloadType, sequenceNumber, timestamp, and ssrc) + * are very frequently accessed in our pipeline; store their values in + * delegated properties, rather than re-reading them from the buffer every time. + */ + + private var _payloadType: Int = RtpHeader.getPayloadType(buffer, offset) + var payloadType: Int + get() = _payloadType + set(newValue) { + if (newValue != _payloadType) { + RtpHeader.setPayloadType(this.buffer, this.offset, newValue) + _payloadType = newValue + } + } + + private var _sequenceNumber: Int = RtpHeader.getSequenceNumber(buffer, offset) + var sequenceNumber: Int + get() = _sequenceNumber + set(newValue) { + if (newValue != _sequenceNumber) { + RtpHeader.setSequenceNumber(this.buffer, this.offset, newValue) + _sequenceNumber = newValue + } + } + + private var _timestamp: Long = RtpHeader.getTimestamp(buffer, offset) + var timestamp: Long + get() = _timestamp + set(newValue) { + if (newValue != _timestamp) { + RtpHeader.setTimestamp(this.buffer, this.offset, newValue) + _timestamp = newValue + } + } + + private var _ssrc: Long = RtpHeader.getSsrc(buffer, offset) + var ssrc: Long + get() = _ssrc + set(newValue) { + if (newValue != _ssrc) { + RtpHeader.setSsrc(this.buffer, this.offset, newValue) + _ssrc = newValue + } + } + + val csrcs: List + get() = RtpHeader.getCsrcs(buffer, offset) + + val extensionsProfileType: Int + get() = RtpHeader.getExtensionsProfileType(buffer, offset) + + /** + * The length of the entire RTP header, including any extensions, in bytes + */ + var headerLength: Int = RtpHeader.getTotalLength(buffer, offset) + protected set + init { + if (headerLength > length) { + throw IllegalArgumentException("RTP packet header length $headerLength > length $length") + } + } + + val payloadLength: Int + get() = length - headerLength + + val payloadOffset: Int + get() = offset + headerLength + + var paddingSize: Int + get() { + if (!hasPadding) { + return 0 + } + // The last octet of the padding contains a count of how many + // padding octets should be ignored, including itself. + // It's an 8-bit unsigned number. + return buffer.getByteAsInt(offset + length - 1) + } + set(value) { + if (value > 0) { + hasPadding = true + buffer[offset + length - 1] = value.toByte() + } else { + hasPadding = false + } + } + + /** + * The parser to use for header extensions, depending on the type of header extensions in this packet. + */ + private val headerExtensionParser + get() = HeaderExtensionHelpers.getHeaderExtensionParser(extensionsProfileType) + + @field:Suppress("ktlint:standard:backing-property-naming") + private val _encodedHeaderExtensions: EncodedHeaderExtensions = EncodedHeaderExtensions() + private val encodedHeaderExtensions: EncodedHeaderExtensions + get() { + _encodedHeaderExtensions.reset() + return _encodedHeaderExtensions + } + + /** + * For [RtpPacket] the payload is everything after the RTP Header. + */ + override val payloadVerification: String + get() = "type=RtpPacket len=$payloadLength " + + "hashCode=${buffer.hashCodeOfSegment(payloadOffset, payloadOffset + payloadLength)}" + + private var pendingHeaderExtensions: MutableList? = null + + private fun getEncodedHeaderExtension(extensionId: Int): HeaderExtension? { + if (!hasEncodedExtensions) return null + + encodedHeaderExtensions.forEach { ext -> + if (ext.id == extensionId) { + return ext + } + } + return null + } + + var hasExtensions: Boolean + get() = pendingHeaderExtensions?.isNotEmpty() ?: hasEncodedExtensions + set(value) { + val p = pendingHeaderExtensions + if (p != null) { + if (value && p.isEmpty()) { + throw java.lang.IllegalStateException( + "Cannot set hasExtensions to true with empty pending extensions" + ) + } + if (!value) { + p.clear() + } + } else { + hasEncodedExtensions = value + } + } + + fun getHeaderExtension(extensionId: Int): HeaderExtension? { + val activeHeaderExtensions = pendingHeaderExtensions?.iterator() ?: encodedHeaderExtensions + + activeHeaderExtensions.forEach { ext -> + if (ext.id == extensionId) { + return ext + } + } + return null + } + + private fun createPendingHeaderExtensions(removeIf: ((HeaderExtension) -> Boolean)?) { + if (pendingHeaderExtensions != null) { + return + } + check(!hasEncodedExtensions || headerExtensionParser != null) { + "Cannot modify header extensions for header extension type ${Integer.toHexString(extensionsProfileType)}" + } + pendingHeaderExtensions = ArrayList().also { l -> + encodedHeaderExtensions.forEach { + if (removeIf == null || !removeIf(it)) { + l.add(PendingHeaderExtension(it)) + } + } + } + } + + /** + * Removes the header extension (or all header extensions) with the given ID. + */ + fun removeHeaderExtension(id: Int) { + pendingHeaderExtensions?.removeIf { h -> h.id == id } + ?: createPendingHeaderExtensions { h -> h.id == id } + } + + /** + * Removes all header extensions except those with ID values in [retain] + */ + fun removeHeaderExtensionsExcept(retain: Set) { + pendingHeaderExtensions?.removeIf { h -> !retain.contains(h.id) } + ?: createPendingHeaderExtensions { h -> !retain.contains(h.id) } + } + + /** + * Adds an RTP header extension with ID [id] and data length [extDataLength] to this + * packet. The contents of the extension are not set to anything, and the + * caller of this method is responsible for filling them in via the + * [HeaderExtension] reference returned. + * + * This method MUST NOT be called while iterating over the extensions using + * {@link #getHeaderExtensions()}, or while manipulating the state of this + * {@link NewRawPacket}. + * + */ + fun addHeaderExtension(id: Int, extDataLength: Int): HeaderExtension { + require(id in 1..255) { + "Invalid header extension ID $id" + } + require(extDataLength in 0..255) { + "Invalid header extension length $extDataLength" + } + + val newHeader = PendingHeaderExtension(id, extDataLength) + + if (pendingHeaderExtensions == null) { + createPendingHeaderExtensions(null) + } + + pendingHeaderExtensions!!.add(newHeader) + + return newHeader + } + + private fun pickParserForEncodedHeaders( + pendingHeaderExtensions: List + ): HeaderExtensionParser { + pendingHeaderExtensions.forEach { + if (it.id >= 15 || it.dataLengthBytes == 0 || it.dataLengthBytes > 16) { + return TwoByteHeaderExtensionParser + } + } + return OneByteHeaderExtensionParser + } + + fun encodeHeaderExtensions() { + val pendingHeaderExtensions = this.pendingHeaderExtensions ?: return + + val newParser = pickParserForEncodedHeaders(pendingHeaderExtensions) + + // The byte[] of an RtpPacket has the following structure: + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // | A: unused | B: hdr + ext | C: payload | D: unused | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + // And the regions have the following sizes: + // A: this.offset + // B: this.getHeaderLength() + // C: this.getPayloadLength() + // D: this.buffer.length - this.length - this.offset + + // If the newly-encoded header extensions block results in a header + // longer than the current one, + // we will try to extend the packet so that it uses A and/or D if + // possible, in order to avoid allocating new memory. + + // We get this early, before we modify the buffer. + val currHeaderLength = headerLength + val currPayloadLength = payloadLength + val baseHeaderLength = RtpHeader.FIXED_HEADER_SIZE_BYTES + csrcCount * 4 + + val newExtHeaderLength = if (pendingHeaderExtensions.isEmpty()) { + 0 + } else { + val rawHeaderLength = RtpHeader.EXT_HEADER_SIZE_BYTES + + pendingHeaderExtensions.sumOf { h -> newParser.extHeaderSizeBytes + h.dataLengthBytes } + rawHeaderLength + RtpUtils.getNumPaddingBytes(rawHeaderLength) + } + + val newHeaderLength = baseHeaderLength + newExtHeaderLength + val newPacketLength = newHeaderLength + currPayloadLength + + val newPayloadOffset: Int + val newBuffer = if (buffer.size >= (newPacketLength + BYTES_TO_LEAVE_AT_END_OF_PACKET)) { + // We don't need a new buffer + if ((offset + currHeaderLength) >= (newPacketLength - currPayloadLength)) { + // Region A (see above) is enough to accommodate the new + // packet, keep the payload where it is. + newPayloadOffset = payloadOffset + } else { + // We have to use region D, so move the payload all the way to the right + newPayloadOffset = buffer.size - currPayloadLength - BYTES_TO_LEAVE_AT_END_OF_PACKET + System.arraycopy(buffer, payloadOffset, buffer, newPayloadOffset, currPayloadLength) + } + buffer + } else { + // We need a new buffer. We will place the payload almost to the end + // (leaving room for an SRTP tag) + BufferPool.getArray(newPacketLength + BYTES_TO_LEAVE_AT_END_OF_PACKET).apply { + newPayloadOffset = size - currPayloadLength - BYTES_TO_LEAVE_AT_END_OF_PACKET + System.arraycopy(buffer, payloadOffset, this, newPayloadOffset, currPayloadLength) + } + } + val newOffset = newPayloadOffset - newHeaderLength + + if (buffer !== newBuffer || offset != newOffset) { + // Copy the base header into place. + System.arraycopy(buffer, offset, newBuffer, newOffset, baseHeaderLength) + } + + if (pendingHeaderExtensions.isNotEmpty()) { + var off = newOffset + baseHeaderLength + // Write the header extension + newBuffer.putShort(off, newParser.headerExtensionLabel.toShort()) + newBuffer.putShort(off + 2, ((newExtHeaderLength - RtpHeader.EXT_HEADER_SIZE_BYTES) / 4).toShort()) + off += 4 + // Write pending header extension elements + pendingHeaderExtensions.forEach { h -> + val len = h.writeToBuffer(newBuffer, off, newParser) + off += len + } + // Write padding + while (off < newOffset + newHeaderLength) { + newBuffer[off] = 0 + off++ + } + } + + val oldBuffer = buffer + buffer = newBuffer + // Reference comparison to see if we got a new buffer. If so, return the old one to the pool + if (oldBuffer !== newBuffer) { + BufferPool.returnArray(oldBuffer) + } + offset = newOffset + length = newPacketLength + headerLength = newHeaderLength + + // ... and set the extension bit. + hasEncodedExtensions = pendingHeaderExtensions.isNotEmpty() + + // Clear pending extensions. + this.pendingHeaderExtensions = null + } + + override fun clone(): RtpPacket = RtpPacket( + cloneBuffer(BYTES_TO_LEAVE_AT_START_OF_PACKET), + BYTES_TO_LEAVE_AT_START_OF_PACKET, + length + ).also { postClone(it) } + + /** Extra operations that need to be done after [clone]. All subclasses overriding [clone] + * must call this method on the newly-created clone. */ + protected fun postClone(clone: RtpPacket) { + pendingHeaderExtensions?.let { clone.pendingHeaderExtensions = ArrayList(it) } + } + + override fun toString(): String = buildString { + append("${this@RtpPacket::class.java.simpleName}: ") + append("PT=$payloadType") + append(", Ssrc=$ssrc") + append(", SeqNum=$sequenceNumber") + append(", M=$isMarked") + append(", X=$hasEncodedExtensions") + append(", Ts=$timestamp") + } + + /** + * Represents an RTP header extension. + */ + interface HeaderExtension { + val buffer: ByteArray + val dataOffset: Int + + var id: Int + + val dataLengthBytes: Int + + val totalLengthBytes: Int + + fun clone(): HeaderExtension = StandaloneHeaderExtension(this) + } + + @SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") + class StandaloneHeaderExtension(ext: HeaderExtension) : HeaderExtension { + override val buffer: ByteArray = ByteArray(ext.dataLengthBytes).also { + System.arraycopy(ext.buffer, ext.dataOffset, it, 0, ext.dataLengthBytes) + } + override val dataOffset = 0 + override var id = ext.id + override val dataLengthBytes: Int + get() = buffer.size + override val totalLengthBytes: Int + get() = buffer.size + } + + @SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") + inner class EncodedHeaderExtension : HeaderExtension { + private var currExtOffset: Int = 0 + private var currExtLength: Int = 0 + + override val dataLengthBytes: Int + get() = headerExtensionParser!!.getDataLengthBytes(buffer, currExtOffset) + + override val buffer: ByteArray + get() = this@RtpPacket.buffer + + override var id: Int + get() { + if (currExtLength <= 0) { + return -1 + } + return headerExtensionParser!!.getId(buffer, currExtOffset) + } + set(newId) { + if (currExtLength < headerExtensionParser!!.minimumExtSizeBytes) { + throw IllegalStateException("Can't set ID: Header extension too short") + } + headerExtensionParser!!.writeIdAndLength(newId, dataLengthBytes, buffer, currExtOffset) + } + + override val dataOffset: Int + get() = headerExtensionParser!!.extHeaderSizeBytes + currExtOffset + + override val totalLengthBytes: Int + get() = headerExtensionParser!!.extHeaderSizeBytes + dataLengthBytes + + fun setOffsetLength(nextHeaderExtOffset: Int, nextHeaderExtLength: Int) { + currExtOffset = nextHeaderExtOffset + currExtLength = nextHeaderExtLength + } + } + + @SuppressFBWarnings( + value = ["EI_EXPOSE_REP", "CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE"], + justification = "We intentionally expose the internal buffer." + ) + inner class PendingHeaderExtension(override var id: Int, override val dataLengthBytes: Int) : HeaderExtension { + override val buffer = ByteArray(dataLengthBytes) + override val dataOffset = 0 + + override val totalLengthBytes: Int + get() = dataLengthBytes + headerExtensionParser!!.extHeaderSizeBytes + + constructor(other: HeaderExtension) : this(other.id, other.dataLengthBytes) { + System.arraycopy(other.buffer, other.dataOffset, buffer, 0, dataLengthBytes) + } + + fun writeToBuffer(buffer: ByteArray, offset: Int, parser: HeaderExtensionParser): Int { + parser.writeIdAndLength(id, dataLengthBytes, buffer, offset) + System.arraycopy( + this.buffer, + dataOffset, + buffer, + offset + parser.extHeaderSizeBytes, + dataLengthBytes + ) + return parser.extHeaderSizeBytes + dataLengthBytes + } + } + + inner class EncodedHeaderExtensions : Iterator { + /** + * The offset of the next extension + */ + private var nextOffset = 0 + + /** + * The remaining length of the extensions headers. + */ + private var remainingLength = 0 + + private val currHeaderExtension = EncodedHeaderExtension() + + override fun hasNext(): Boolean { + if (headerExtensionParser == null) { + return false + } + // Consume any padding + while (remainingLength > 0 && buffer[nextOffset].isPadding()) { + nextOffset++ + remainingLength-- + } + if (remainingLength <= 0 || nextOffset < 0) { + return false + } + return getNextExtLength() > 0 + } + + override fun next(): HeaderExtension { + val nextExtLen = getNextExtLength() + if (nextExtLen <= 0) { + throw Exception("Invalid extension length. Did hasNext() return true?") + } + currHeaderExtension.setOffsetLength(nextOffset, nextExtLen) + nextOffset += nextExtLen + remainingLength -= nextExtLen + + return currHeaderExtension + } + + /** + * Return the entire length (including the header), in bytes, of the 'next' RTP extension + * according to [nextOffset]. If [remainingLength] is less than the minimum size for + * an extension, or the parsed length is larger than [remainingLength], return -1 + */ + private fun getNextExtLength(): Int { + headerExtensionParser?.let { parser -> + if (remainingLength < parser.minimumExtSizeBytes) { + return -1 + } + val extLen = parser.getEntireLengthBytes(buffer, nextOffset) + return if (extLen > remainingLength) -1 else extLen + } ?: run { return -1 } + } + + /** + * Resets this iterator back to the beginning of the extensions + */ + internal fun reset() { + val extLength = + // This treats unknown header extension types as no extensions, which is what we want. + if (headerExtensionParser != null) { + val extensionBlockLength = HeaderExtensionHelpers.getExtensionsTotalLength( + buffer, + offset + RtpHeader.FIXED_HEADER_SIZE_BYTES + csrcCount * 4 + ) + + extensionBlockLength - HeaderExtensionHelpers.TOP_LEVEL_EXT_HEADER_SIZE_BYTES + } else { + 0 + } + + if (extLength <= 0) { + // No extensions + nextOffset = -1 + remainingLength = -1 + } else { + nextOffset = offset + + RtpHeader.FIXED_HEADER_SIZE_BYTES + + csrcCount * 4 + + RtpHeader.EXT_HEADER_SIZE_BYTES + + remainingLength = extLength + } + } + } + + companion object { + /** + * The size of the header for individual extensions. Currently we only + * support 1 byte header extensions + */ + const val HEADER_EXT_HEADER_SIZE = 1 + + /** + * How much space to leave in the beginning of new RTP packets. Having space in the beginning allows us to + * implement adding RTP header extensions efficiently (by keeping the RTP payload in place and shifting the + * header left). + */ + const val BYTES_TO_LEAVE_AT_START_OF_PACKET = 10 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt new file mode 100644 index 0000000..69f305e --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumber.kt @@ -0,0 +1,114 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp + +import org.jitsi.rtp.util.RtpUtils + +/** + * An inline class representing an RTP sequence number. The class operates just like + * an Int but takes rollover into account for all operations. + * + * This constructor assumes that the value is already coerced. It MUST NOT be used outside this class, but it can not + * be marked private. Use [Int.toRtpSequenceNumber] to create instances. + */ +@Suppress("NON_PUBLIC_PRIMARY_CONSTRUCTOR_OF_INLINE_CLASS") +@JvmInline +value class RtpSequenceNumber internal constructor(val value: Int) : Comparable { + // These are intentionally not implemented, because using them as operators leads to inconsistent results. The + // following code: + // var n1 = RtpSequenceNumber(65535) + // var n2 = RtpSequenceNumber(65535) + // ++n1 + // n2++ + // System.err.println("n1=$n1, n2=$n2") + // + // Produces this result: + // n1=RtpSequenceNumber(value=65536) n2=RtpSequenceNumber(value=0) + // + // Using "+= 1" yields the expected result. Note that if the same code above is in a "should" block, the result is + // different (as expected). So if/when this is brought back, it should be tested outside a regular "should" block! + // + // operator fun inc(): RtpSequenceNumber = plus(1) + // operator fun dec(): RtpSequenceNumber = minus(1) + + operator fun plus(num: Int): RtpSequenceNumber = (value + num).toRtpSequenceNumber() + operator fun plus(seqNum: RtpSequenceNumber): RtpSequenceNumber = (value + seqNum.value).toRtpSequenceNumber() + + operator fun minus(num: Int): RtpSequenceNumber = plus(-num) + operator fun minus(seqNum: RtpSequenceNumber): RtpSequenceNumber = plus(-seqNum.value) + + override operator fun compareTo(other: RtpSequenceNumber): Int = RtpUtils.getSequenceNumberDelta(value, other.value) + + operator fun rangeTo(other: RtpSequenceNumber) = RtpSequenceNumberProgression(this, other) + + companion object { + val INVALID = RtpSequenceNumber(-1) + } +} + +fun Int.toRtpSequenceNumber() = RtpSequenceNumber(this and 0xffff) + +// Copied mostly from IntProgression. +// NOTE(brian): technically this should probably inherit from ClosedRange, but +// the inheritance causes issues with boxing and the inline types. See +// https://youtrack.jetbrains.com/issue/KT-30716 and the bug linked there. +class RtpSequenceNumberProgression( + val start: RtpSequenceNumber, + val endInclusive: RtpSequenceNumber, + val step: Int = 1 +) : Iterable { // , ClosedRange + + override fun iterator(): Iterator = + RtpSequenceNumberProgressionIterator(start, endInclusive, step) + + companion object { + fun fromClosedRange( + rangeStart: RtpSequenceNumber, + rangeEnd: RtpSequenceNumber, + step: Int + ): RtpSequenceNumberProgression = RtpSequenceNumberProgression(rangeStart, rangeEnd, step) + } +} + +// Copied mostly from IntProgressionIterator +class RtpSequenceNumberProgressionIterator( + first: RtpSequenceNumber, + last: RtpSequenceNumber, + val step: Int +) : Iterator { + private val finalElement = last + private var hasNext: Boolean = if (step > 0) first <= last else first >= last + private var next = if (hasNext) first else finalElement + + override fun hasNext(): Boolean = hasNext + + override fun next(): RtpSequenceNumber = nextSeqNum() + + fun nextSeqNum(): RtpSequenceNumber { + val value = next + if (value == finalElement) { + if (!hasNext) throw kotlin.NoSuchElementException() + hasNext = false + } else { + next += step + } + return value + } +} + +infix fun RtpSequenceNumber.downTo(to: RtpSequenceNumber): RtpSequenceNumberProgression = + RtpSequenceNumberProgression.fromClosedRange(this, to, -1) diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt new file mode 100644 index 0000000..f66eba0 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AbsSendTimeHeaderExtension.kt @@ -0,0 +1,71 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp.header_extensions + +import org.jitsi.rtp.extensions.bytearray.put3Bytes +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.getBitsAsInt +import org.jitsi.rtp.util.getShortAsInt +import java.time.Instant + +/** + * https://webrtc.org/experiments/rtp-hdrext/abs-send-time/ + * + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ID | LEN | AbsSendTime value | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class AbsSendTimeHeaderExtension { + companion object { + const val DATA_SIZE_BYTES = 3 + + /** + * One billion. + */ + private const val B = 1_000_000_000 + + fun setTime(ext: RtpPacket.HeaderExtension, timestampNanos: Long) = + setTime(ext.buffer, ext.dataOffset, timestampNanos) + + private fun setTime(buf: ByteArray, offset: Int, timestampNanos: Long) { + val fraction = ((timestampNanos % B) * (1 shl 18) / B) + val seconds = ((timestampNanos / B) % 64) // 6 bits only + + val timestamp = ((seconds shl 18) or fraction) and 0x00FFFFFF + + buf.put3Bytes(offset, timestamp.toInt()) + } + + /** + * Gets the timestamp converted to nanoseconds. + */ + fun getTime(ext: RtpPacket.HeaderExtension): Instant = getTime(ext.buffer, ext.dataOffset) + + private fun getTime(buf: ByteArray, dataOffset: Int): Instant { + val seconds = buf.getBitsAsInt(dataOffset, 0, 6) + val fraction = ( + buf.getBitsAsInt(dataOffset, 6, 2) + + buf.getShortAsInt(dataOffset + 1) + ).toDouble() / 0x03ffff + + val instantMillis = Instant.ofEpochSecond(seconds.toLong()) + return instantMillis.plusNanos((fraction * B).toLong()) + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt new file mode 100644 index 0000000..807698d --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/AudioLevelHeaderExtension.kt @@ -0,0 +1,44 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp.header_extensions + +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.rtp.RtpPacket +import kotlin.experimental.and + +/** + * https://tools.ietf.org/html/rfc6464#section-3 + * TODO: this can be held as either 1 byte or 2 byte. (though webrtc clients appear to all use 1 byte) + * + * 0 1 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ID | len=0 |V| level | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class AudioLevelHeaderExtension { + companion object { + private const val AUDIO_LEVEL_MASK = 0x7F.toByte() + + fun getAudioLevel(ext: RtpPacket.HeaderExtension): Int = getAudioLevel(ext.buffer, ext.dataOffset) + + private fun getAudioLevel(buf: ByteArray, offset: Int): Int = (buf[offset] and AUDIO_LEVEL_MASK).toPositiveInt() + + fun getVad(ext: RtpPacket.HeaderExtension): Boolean = getVad(ext.buffer, ext.dataOffset) + private fun getVad(buf: ByteArray, offset: Int): Boolean = (buf[offset].toInt() and 0x80) != 0 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt new file mode 100644 index 0000000..ce4ea05 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtension.kt @@ -0,0 +1,936 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.BitReader +import org.jitsi.rtp.util.BitWriter + +/** + * The subset of the fields of an AV1 Dependency Descriptor that can be parsed statelessly. + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +open class Av1DependencyDescriptorStatelessSubset( + val startOfFrame: Boolean, + val endOfFrame: Boolean, + var frameDependencyTemplateId: Int, + var frameNumber: Int, + + val newTemplateDependencyStructure: Av1TemplateDependencyStructure?, +) { + open fun clone(): Av1DependencyDescriptorStatelessSubset = Av1DependencyDescriptorStatelessSubset( + startOfFrame = startOfFrame, + endOfFrame = endOfFrame, + frameDependencyTemplateId = frameDependencyTemplateId, + frameNumber = frameNumber, + newTemplateDependencyStructure = newTemplateDependencyStructure?.clone() + ) +} + +/** + * The AV1 Dependency Descriptor header extension, as defined in https://aomediacodec.github.io/av1-rtp-spec/#appendix + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class Av1DependencyDescriptorHeaderExtension( + startOfFrame: Boolean, + endOfFrame: Boolean, + frameDependencyTemplateId: Int, + frameNumber: Int, + + newTemplateDependencyStructure: Av1TemplateDependencyStructure?, + + var activeDecodeTargetsBitmask: Int?, + + val customDtis: List?, + val customFdiffs: List?, + val customChains: List?, + + val structure: Av1TemplateDependencyStructure +) : Av1DependencyDescriptorStatelessSubset( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + newTemplateDependencyStructure +) { + val frameInfo: FrameInfo by lazy { + val templateIndex = (frameDependencyTemplateId + 64 - structure.templateIdOffset) % 64 + if (templateIndex >= structure.templateCount) { + val maxTemplate = (structure.templateIdOffset + structure.templateCount - 1) % 64 + throw Av1DependencyException( + "Invalid template ID $frameDependencyTemplateId. " + + "Should be in range ${structure.templateIdOffset} .. $maxTemplate. " + + "Missed a keyframe?" + ) + } + val templateVal = structure.templateInfo[templateIndex] + + FrameInfo( + spatialId = templateVal.spatialId, + temporalId = templateVal.temporalId, + dti = customDtis ?: templateVal.dti, + fdiff = customFdiffs ?: templateVal.fdiff, + chains = customChains ?: templateVal.chains + ) + } + + val encodedLength: Int + get() = (unpaddedLengthBits + 7) / 8 + + private val unpaddedLengthBits: Int + get() { + var length = 24 + if (newTemplateDependencyStructure != null || + activeDecodeTargetsBitmask != null || + customDtis != null || + customFdiffs != null || + customChains != null + ) { + length += 5 + } + if (newTemplateDependencyStructure != null) { + length += newTemplateDependencyStructure.unpaddedLengthBits + } + if (activeDecodeTargetsBitmask != null && + ( + newTemplateDependencyStructure == null || + activeDecodeTargetsBitmask != ((1 shl newTemplateDependencyStructure.decodeTargetCount) - 1) + ) + ) { + length += structure.decodeTargetCount + } + if (customDtis != null) { + length += 2 * structure.decodeTargetCount + } + if (customFdiffs != null) { + customFdiffs.forEach { + length += 2 + it.bitsForFdiff() + } + length += 2 + } + if (customChains != null) { + length += 8 * customChains.size + } + + return length + } + + override fun clone(): Av1DependencyDescriptorHeaderExtension { + val structureCopy = structure.clone() + val newStructure = if (newTemplateDependencyStructure == null) null else structureCopy + return Av1DependencyDescriptorHeaderExtension( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + newStructure, + activeDecodeTargetsBitmask, + // These values are not mutable so it's safe to copy them by reference + customDtis, + customFdiffs, + customChains, + structureCopy + ) + } + + fun write(ext: RtpPacket.HeaderExtension) = write(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + + fun write(buffer: ByteArray, offset: Int, length: Int) { + check(length <= encodedLength) { + "Cannot write AV1 DD to buffer: buffer length $length must be at least $encodedLength" + } + val writer = BitWriter(buffer, offset, length) + + writeMandatoryDescriptorFields(writer) + + if (newTemplateDependencyStructure != null || + activeDecodeTargetsBitmask != null || + customDtis != null || + customFdiffs != null || + customChains != null + ) { + writeOptionalDescriptorFields(writer) + writePadding(writer) + } else { + check(length == 3) { + "AV1 DD without optional descriptors must be 3 bytes in length" + } + } + } + + private fun writeMandatoryDescriptorFields(writer: BitWriter) { + writer.writeBit(startOfFrame) + writer.writeBit(endOfFrame) + writer.writeBits(6, frameDependencyTemplateId) + writer.writeBits(16, frameNumber) + } + + private fun writeOptionalDescriptorFields(writer: BitWriter) { + val templateDependencyStructurePresent = newTemplateDependencyStructure != null + val activeDecodeTargetsPresent = activeDecodeTargetsBitmask != null && + ( + newTemplateDependencyStructure == null || + activeDecodeTargetsBitmask != ((1 shl newTemplateDependencyStructure.decodeTargetCount) - 1) + ) + + val customDtisFlag = customDtis != null + val customFdiffsFlag = customFdiffs != null + val customChainsFlag = customChains != null + + writer.writeBit(templateDependencyStructurePresent) + writer.writeBit(activeDecodeTargetsPresent) + writer.writeBit(customDtisFlag) + writer.writeBit(customFdiffsFlag) + writer.writeBit(customChainsFlag) + + if (templateDependencyStructurePresent) { + newTemplateDependencyStructure.write(writer) + } + + if (activeDecodeTargetsPresent) { + writeActiveDecodeTargets(writer) + } + + if (customDtisFlag) { + writeFrameDtis(writer) + } + + if (customFdiffsFlag) { + writeFrameFdiffs(writer) + } + + if (customChainsFlag) { + writeFrameChains(writer) + } + } + + private fun writeActiveDecodeTargets(writer: BitWriter) { + writer.writeBits(structure.decodeTargetCount, activeDecodeTargetsBitmask!!) + } + + private fun writeFrameDtis(writer: BitWriter) { + customDtis!!.forEach { dti -> + writer.writeBits(2, dti.dti) + } + } + + private fun writeFrameFdiffs(writer: BitWriter) { + customFdiffs!!.forEach { fdiff -> + val bits = fdiff.bitsForFdiff() + writer.writeBits(2, bits / 4) + writer.writeBits(bits, fdiff - 1) + } + writer.writeBits(2, 0) + } + + private fun writeFrameChains(writer: BitWriter) { + customChains!!.forEach { chain -> + writer.writeBits(8, chain) + } + } + + private fun writePadding(writer: BitWriter) { + writer.writeBits(writer.remainingBits, 0) + } + + override fun toString(): String { + val mapper = ObjectMapper() + return JsonNodeFactory.instance.objectNode().apply { + put("startOfFrame", startOfFrame) + put("endOfFrame", endOfFrame) + put("frameDependencyTemplateId", frameDependencyTemplateId) + put("frameNumber", frameNumber) + newTemplateDependencyStructure?.let { + put("templateStructure", it.toString()) + } + customDtis?.let { + set( + "customDTIs", + mapper.valueToTree(it) + ) + } + customFdiffs?.let { + set( + "customFdiffs", + mapper.valueToTree(it) + ) + } + customChains?.let { + set( + "customChains", + mapper.valueToTree(it) + ) + } + activeDecodeTargetsBitmask?.let { + if (newTemplateDependencyStructure == null || + it != ((1 shl newTemplateDependencyStructure.decodeTargetCount) - 1) + ) { + put("activeDecodeTargets", Integer.toBinaryString(it)) + } + } + }.toString() + } +} + +fun Int.bitsForFdiff() = when { + this <= 0x10 -> 4 + this <= 0x100 -> 8 + this <= 0x1000 -> 12 + else -> throw IllegalArgumentException("Invalid FDiff value $this") +} + +/** + * The template information about a stream described by AV1 dependency descriptors. This is carried in the + * first packet of a codec video sequence (i.e. the first packet of a keyframe), and is necessary to interpret + * dependency descriptors carried in subsequent packets of the sequence. + */ +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class Av1TemplateDependencyStructure( + var templateIdOffset: Int, + val templateInfo: List, + val decodeTargetProtectedBy: List, + val decodeTargetLayers: List, + val maxRenderResolutions: List, + val maxSpatialId: Int, + val maxTemporalId: Int +) { + val templateCount + get() = templateInfo.size + + val decodeTargetCount + get() = decodeTargetLayers.size + + val chainCount: Int = + templateInfo.first().chains.size + + init { + check(templateInfo.all { it.chains.size == chainCount }) { + "Templates have inconsistent chain sizes" + } + check(templateInfo.all { it.temporalId <= maxTemporalId }) { + "Incorrect maxTemporalId" + } + check(maxRenderResolutions.isEmpty() || maxRenderResolutions.size == maxSpatialId + 1) { + "Non-zero number of render resolutions does not match maxSpatialId" + } + check(templateInfo.all { it.spatialId <= maxSpatialId }) { + "Incorrect maxSpatialId" + } + } + + val unpaddedLengthBits: Int + get() { + var length = 6 // Template ID offset + + length += 5 // DT Count - 1 + + length += templateCount * 2 // templateLayers + length += templateCount * decodeTargetCount * 2 // TemplateDTIs + templateInfo.forEach { + length += it.fdiffCnt * 5 + 1 // TemplateFDiffs + } + // TemplateChains + length += nsBits(decodeTargetCount + 1, chainCount) + if (chainCount > 0) { + decodeTargetProtectedBy.forEach { + length += nsBits(chainCount, it) + } + length += templateCount * chainCount * 4 + } + length += 1 // ResolutionsPresent + length += maxRenderResolutions.size * 32 // RenderResolutions + + return length + } + + fun clone(): Av1TemplateDependencyStructure = Av1TemplateDependencyStructure( + templateIdOffset, + // These objects are not mutable so it's safe to copy them by reference + templateInfo, + decodeTargetProtectedBy, + decodeTargetLayers, + maxRenderResolutions, + maxSpatialId, + maxTemporalId + ) + + fun write(writer: BitWriter) { + writer.writeBits(6, templateIdOffset) + + writer.writeBits(5, decodeTargetCount - 1) + + writeTemplateLayers(writer) + writeTemplateDtis(writer) + writeTemplateFdiffs(writer) + writeTemplateChains(writer) + + writeRenderResolutions(writer) + } + + private fun writeTemplateLayers(writer: BitWriter) { + check(templateInfo[0].spatialId == 0 && templateInfo[0].temporalId == 0) { + "First template must have spatial and temporal IDs 0/0, but found " + + "${templateInfo[0].spatialId}/${templateInfo[0].temporalId}" + } + for (templateNum in 1 until templateInfo.size) { + val layerIdc = when { + templateInfo[templateNum].spatialId == templateInfo[templateNum - 1].spatialId && + templateInfo[templateNum].temporalId == templateInfo[templateNum - 1].temporalId -> + 0 + + templateInfo[templateNum].spatialId == templateInfo[templateNum - 1].spatialId && + templateInfo[templateNum].temporalId == templateInfo[templateNum - 1].temporalId + 1 -> + 1 + + templateInfo[templateNum].spatialId == templateInfo[templateNum - 1].spatialId + 1 && + templateInfo[templateNum].temporalId == 0 -> + 2 + + else -> + throw IllegalStateException( + "Template $templateNum with spatial and temporal IDs " + + "${templateInfo[templateNum].spatialId}/${templateInfo[templateNum].temporalId} " + + "cannot follow template ${templateNum - 1} with spatial and temporal IDs " + + "${templateInfo[templateNum - 1].spatialId}/${templateInfo[templateNum - 1].temporalId}." + ) + } + writer.writeBits(2, layerIdc) + } + writer.writeBits(2, 3) + } + + private fun writeTemplateDtis(writer: BitWriter) { + templateInfo.forEach { t -> + t.dti.forEach { dti -> + writer.writeBits(2, dti.dti) + } + } + } + + private fun writeTemplateFdiffs(writer: BitWriter) { + templateInfo.forEach { t -> + t.fdiff.forEach { fdiff -> + writer.writeBit(true) + writer.writeBits(4, fdiff - 1) + } + writer.writeBit(false) + } + } + + private fun writeTemplateChains(writer: BitWriter) { + writer.writeNs(decodeTargetCount + 1, chainCount) + decodeTargetProtectedBy.forEach { + writer.writeNs(chainCount, it) + } + templateInfo.forEach { t -> + t.chains.forEach { chain -> + writer.writeBits(4, chain) + } + } + } + + private fun writeRenderResolutions(writer: BitWriter) { + if (maxRenderResolutions.isEmpty()) { + writer.writeBit(false) + } else { + writer.writeBit(true) + maxRenderResolutions.forEach { r -> + writer.writeBits(16, r.width - 1) + writer.writeBits(16, r.height - 1) + } + } + } + + /** Return whether, in this structure, it's possible to switch from DT [fromDt] to DT [toDt] + * without a keyframe. + * Note this makes certain assumptions about the encoding structure. + */ + fun canSwitchWithoutKeyframe(fromDt: Int, toDt: Int): Boolean = templateInfo.any { + it.hasInterPictureDependency() && it.dti.size > fromDt && it.dti.size > toDt && + it.dti[fromDt] != DTI.NOT_PRESENT && it.dti[toDt] == DTI.SWITCH + } + + /** Given that we are sending packets for a given DT, return a decodeTargetBitmask corresponding to all DTs + * contained in that DT. + */ + fun getDtBitmaskForDt(dt: Int): Int { + var mask = (1 shl decodeTargetCount) - 1 + templateInfo.forEach { frameInfo -> + frameInfo.dti.forEachIndexed { i, dti -> + if (frameInfo.dti[dt] == DTI.NOT_PRESENT && dti != DTI.NOT_PRESENT) { + mask = mask and (1 shl i).inv() + } + } + } + return mask + } + + override fun toString(): String { + val mapper = ObjectMapper() + return JsonNodeFactory.instance.objectNode().apply { + put("templateIdOffset", templateIdOffset) + set( + "templateInfo", + mapper.valueToTree( + templateInfo.map { + it.toString() + }.toIndexedMap() + ) + ) + set( + "decodeTargetProtectedBy", + mapper.valueToTree( + decodeTargetProtectedBy.toIndexedMap() + ) + ) + set( + "decodeTargetLayers", + mapper.valueToTree( + decodeTargetLayers.map { + it.toString() + }.toIndexedMap() + ) + ) + if (maxRenderResolutions.isNotEmpty()) { + set( + "maxRenderResolutions", + mapper.valueToTree( + maxRenderResolutions.map { + it.toString() + }.toIndexedMap() + ) + ) + } + put("maxSpatialId", maxSpatialId) + put("maxTemporalId", maxTemporalId) + }.toString() + } +} + +fun nsBits(n: Int, v: Int): Int { + require(n > 0) + if (n == 1) return 0 + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + if (v < m) return w - 1 + return w +} + +class Av1DependencyDescriptorReader( + buffer: ByteArray, + offset: Int, + val length: Int, +) { + private var startOfFrame = false + private var endOfFrame = false + private var frameDependencyTemplateId = 0 + private var frameNumber = 0 + + private var customDtis: List? = null + private var customFdiffs: List? = null + private var customChains: List? = null + + private var localTemplateDependencyStructure: Av1TemplateDependencyStructure? = null + private var templateDependencyStructure: Av1TemplateDependencyStructure? = null + + private var activeDecodeTargetsBitmask: Int? = null + + private val reader = BitReader(buffer, offset, length) + + constructor(ext: RtpPacket.HeaderExtension) : + this(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + + /** Parse those parts of the dependency descriptor that can be parsed statelessly, i.e. without an external + * template dependency structure. The returned object will not be a complete representation of the + * dependency descriptor, because some fields need the external structure to be parseable. + */ + fun parseStateless(): Av1DependencyDescriptorStatelessSubset { + reset() + readMandatoryDescriptorFields() + + if (length > 3) { + val templateDependencyStructurePresent = reader.bitAsBoolean() + + /* activeDecodeTargetsPresent, customDtisFlag, customFdiffsFlag, and customChainsFlag; + * none of these fields are parseable statelessly. + */ + reader.skipBits(4) + + if (templateDependencyStructurePresent) { + localTemplateDependencyStructure = readTemplateDependencyStructure() + } + } + return Av1DependencyDescriptorStatelessSubset( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + localTemplateDependencyStructure, + ) + } + + /** Parse the dependency descriptor in the context of [dep], the currently-applicable template dependency + * structure.*/ + fun parse(dep: Av1TemplateDependencyStructure?): Av1DependencyDescriptorHeaderExtension { + reset() + readMandatoryDescriptorFields() + if (length > 3) { + readExtendedDescriptorFields(dep) + } else { + if (dep == null) { + throw Av1DependencyException("No external dependency structure specified for non-first packet") + } + templateDependencyStructure = dep + } + return Av1DependencyDescriptorHeaderExtension( + startOfFrame, + endOfFrame, + frameDependencyTemplateId, + frameNumber, + localTemplateDependencyStructure, + activeDecodeTargetsBitmask, + customDtis, + customFdiffs, + customChains, + templateDependencyStructure!! + ) + } + + private fun reset() = reader.reset() + + private fun readMandatoryDescriptorFields() { + startOfFrame = reader.bitAsBoolean() + endOfFrame = reader.bitAsBoolean() + frameDependencyTemplateId = reader.bits(6) + frameNumber = reader.bits(16) + } + + private fun readExtendedDescriptorFields(dep: Av1TemplateDependencyStructure?) { + val templateDependencyStructurePresent = reader.bitAsBoolean() + val activeDecodeTargetsPresent = reader.bitAsBoolean() + val customDtisFlag = reader.bitAsBoolean() + val customFdiffsFlag = reader.bitAsBoolean() + val customChainsFlag = reader.bitAsBoolean() + + if (templateDependencyStructurePresent) { + localTemplateDependencyStructure = readTemplateDependencyStructure() + templateDependencyStructure = localTemplateDependencyStructure + } else { + if (dep == null) { + throw Av1DependencyException("No external dependency structure specified for non-first packet") + } + templateDependencyStructure = dep + } + if (activeDecodeTargetsPresent) { + activeDecodeTargetsBitmask = reader.bits(templateDependencyStructure!!.decodeTargetCount) + } else if (templateDependencyStructurePresent) { + activeDecodeTargetsBitmask = (1 shl templateDependencyStructure!!.decodeTargetCount) - 1 + } + + if (customDtisFlag) { + customDtis = readFrameDtis() + } + if (customFdiffsFlag) { + customFdiffs = readFrameFdiffs() + } + if (customChainsFlag) { + customChains = readFrameChains() + } + } + + /* Data for template dependency structure */ + private var templateIdOffset: Int = 0 + private val templateInfo = mutableListOf() + private val decodeTargetProtectedBy = mutableListOf() + private val decodeTargetLayers = mutableListOf() + private val maxRenderResolutions = mutableListOf() + + private var dtCnt = 0 + + private fun resetDependencyStructureInfo() { + /* These fields are assembled incrementally when parsing a dependency structure; reset them + * in case we're running a parser more than once. + */ + templateCnt = 0 + templateInfo.clear() + decodeTargetProtectedBy.clear() + decodeTargetLayers.clear() + maxRenderResolutions.clear() + } + + private fun readTemplateDependencyStructure(): Av1TemplateDependencyStructure { + resetDependencyStructureInfo() + + templateIdOffset = reader.bits(6) + + val dtCntMinusOne = reader.bits(5) + dtCnt = dtCntMinusOne + 1 + + readTemplateLayers() + readTemplateDtis() + readTemplateFdiffs() + readTemplateChains() + readDecodeTargetLayers() + + val resolutionsPresent = reader.bitAsBoolean() + + if (resolutionsPresent) { + readRenderResolutions() + } + + return Av1TemplateDependencyStructure( + templateIdOffset, + templateInfo.toList(), + decodeTargetProtectedBy.toList(), + decodeTargetLayers.toList(), + maxRenderResolutions.toList(), + maxSpatialId, + maxTemporalId + ) + } + + private var templateCnt = 0 + private var maxSpatialId = 0 + private var maxTemporalId = 0 + + @SuppressFBWarnings( + value = ["SF_SWITCH_NO_DEFAULT"], + justification = "Artifact of generated Kotlin code." + ) + private fun readTemplateLayers() { + var temporalId = 0 + var spatialId = 0 + + var nextLayerIdc: Int + do { + templateInfo.add(templateCnt, TemplateFrameInfo(spatialId, temporalId)) + templateCnt++ + nextLayerIdc = reader.bits(2) + if (nextLayerIdc == 1) { + temporalId++ + if (maxTemporalId < temporalId) { + maxTemporalId = temporalId + } + } else if (nextLayerIdc == 2) { + temporalId = 0 + spatialId++ + } + } while (nextLayerIdc != 3) + + check(templateInfo.size == templateCnt) + + maxSpatialId = spatialId + } + + private fun readTemplateDtis() { + for (templateIndex in 0 until templateCnt) { + for (dtIndex in 0 until dtCnt) { + templateInfo[templateIndex].dti.add(dtIndex, DTI.fromInt(reader.bits(2))) + } + } + } + + private fun readFrameDtis(): List = List(templateDependencyStructure!!.decodeTargetCount) { + DTI.fromInt(reader.bits(2)) + } + + private fun readTemplateFdiffs() { + for (templateIndex in 0 until templateCnt) { + var fdiffCnt = 0 + var fdiffFollowsFlag = reader.bitAsBoolean() + while (fdiffFollowsFlag) { + val fdiffMinusOne = reader.bits(4) + templateInfo[templateIndex].fdiff.add(fdiffCnt, fdiffMinusOne + 1) + fdiffCnt++ + fdiffFollowsFlag = reader.bitAsBoolean() + } + check(fdiffCnt == templateInfo[templateIndex].fdiffCnt) + } + } + + private fun readFrameFdiffs(): List = buildList { + var nextFdiffSize = reader.bits(2) + while (nextFdiffSize != 0) { + val fdiffMinus1 = reader.bits(4 * nextFdiffSize) + add(fdiffMinus1 + 1) + nextFdiffSize = reader.bits(2) + } + } + + private fun readTemplateChains() { + val chainCount = reader.ns(dtCnt + 1) + if (chainCount != 0) { + for (dtIndex in 0 until dtCnt) { + decodeTargetProtectedBy.add(dtIndex, reader.ns(chainCount)) + } + for (templateIndex in 0 until templateCnt) { + for (chainIndex in 0 until chainCount) { + templateInfo[templateIndex].chains.add(chainIndex, reader.bits(4)) + } + check(templateInfo[templateIndex].chains.size == chainCount) + } + } + } + + private fun readFrameChains(): List = List(templateDependencyStructure!!.chainCount) { + reader.bits(8) + } + + private fun readDecodeTargetLayers() { + for (dtIndex in 0 until dtCnt) { + var spatialId = 0 + var temporalId = 0 + for (templateIndex in 0 until templateCnt) { + if (templateInfo[templateIndex].dti[dtIndex] != DTI.NOT_PRESENT) { + if (templateInfo[templateIndex].spatialId > spatialId) { + spatialId = templateInfo[templateIndex].spatialId + } + if (templateInfo[templateIndex].temporalId > temporalId) { + temporalId = templateInfo[templateIndex].temporalId + } + } + } + decodeTargetLayers.add(dtIndex, DecodeTargetLayer(spatialId, temporalId)) + } + check(decodeTargetLayers.size == dtCnt) + } + + private fun readRenderResolutions() { + for (spatialId in 0..maxSpatialId) { + val widthMinus1 = reader.bits(16) + val heightMinus1 = reader.bits(16) + maxRenderResolutions.add(spatialId, Resolution(widthMinus1 + 1, heightMinus1 + 1)) + } + } +} + +open class FrameInfo( + val spatialId: Int, + val temporalId: Int, + open val dti: List, + open val fdiff: List, + open val chains: List +) { + val fdiffCnt + get() = fdiff.size + + override fun equals(other: Any?): Boolean { + if (other !is FrameInfo) { + return false + } + return other.spatialId == spatialId && + other.temporalId == temporalId && + other.dti == dti && + other.fdiff == fdiff && + other.chains == chains + } + + override fun hashCode(): Int { + var result = spatialId + result = 31 * result + temporalId + result = 31 * result + dti.hashCode() + result = 31 * result + fdiff.hashCode() + result = 31 * result + chains.hashCode() + return result + } + + override fun toString(): String = + "spatialId=$spatialId, temporalId=$temporalId, dti=$dti, fdiff=$fdiff, chains=$chains" + + fun toJson(): String { + val mapper = ObjectMapper() + return JsonNodeFactory.instance.objectNode().apply { + put("spatialId", spatialId) + put("temporalId", temporalId) + set("dti", mapper.valueToTree(dti.map { it.name })) + set("fdiff", mapper.valueToTree(fdiff)) + set("chains", mapper.valueToTree(chains)) + }.toString() + } + + /** Whether the frame has a dependency on a frame earlier than this "picture", the other frames of this + * temporal moment. If it doesn't, it's probably part of a keyframe, and not part of the regular structure. + * Note this makes assumptions about the scalability structure. + */ + fun hasInterPictureDependency(): Boolean = fdiff.any { it > spatialId } + + val dtisPresent: List + get() = dti.withIndex().filter { (_, dti) -> dti != DTI.NOT_PRESENT }.map { (i, _) -> i } +} + +/* The only thing this changes from its parent class is to make the lists mutable, so the parent equals() is fine. */ +@SuppressFBWarnings("EQ_DOESNT_OVERRIDE_EQUALS") +class TemplateFrameInfo( + spatialId: Int, + temporalId: Int, + override val dti: MutableList = mutableListOf(), + override val fdiff: MutableList = mutableListOf(), + override val chains: MutableList = mutableListOf() +) : FrameInfo(spatialId, temporalId, dti, fdiff, chains) + +class DecodeTargetLayer( + val spatialId: Int, + val temporalId: Int +) { + override fun toString(): String = JsonNodeFactory.instance.objectNode().apply { + put("spatialId", spatialId) + put("temporalId", temporalId) + }.toString() +} + +data class Resolution( + val width: Int, + val height: Int +) { + override fun toString(): String = JsonNodeFactory.instance.objectNode().apply { + put("width", width) + put("height", height) + }.toString() +} + +/** Decode target indication */ +enum class DTI(val dti: Int) { + NOT_PRESENT(0), + DISCARDABLE(1), + SWITCH(2), + REQUIRED(3); + + companion object { + private val map = DTI.values().associateBy(DTI::dti) + fun fromInt(type: Int) = map[type] ?: throw java.lang.IllegalArgumentException("Bad DTI $type") + } + + fun toShortString(): String = when (this) { + NOT_PRESENT -> "N" + DISCARDABLE -> "D" + SWITCH -> "S" + REQUIRED -> "R" + } +} + +fun List.toShortString(): String = joinToString(separator = "") { it.toShortString() } + +class Av1DependencyException(msg: String) : RuntimeException(msg) + +fun List.toIndexedMap(): Map = mapIndexed { i, t -> i to t }.toMap() diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt new file mode 100644 index 0000000..4cfe8a2 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/HeaderExtensionHelpers.kt @@ -0,0 +1,136 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp.header_extensions + +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.util.getShortAsInt +import kotlin.experimental.and +import kotlin.experimental.or + +/** An abstract parser for header extensions. */ +abstract class HeaderExtensionParser { + abstract val headerExtensionLabel: Int + + abstract val extHeaderSizeBytes: Int + abstract val minimumExtSizeBytes: Int + + /** Determine whether this header extension parser can parse header extensions with the given + * "defined by profile" field. + */ + abstract fun isMatchingType(profileField: Int): Boolean + + abstract fun getId(buf: ByteArray, offset: Int): Int + + /** + * Write the given ID and length to [buf] at [offset] + */ + abstract fun writeIdAndLength(id: Int, dataLength: Int, buf: ByteArray, offset: Int) + + /** + * Return the entire size, in bytes, of the extension in [buf] whose header + * starts at [offset] + */ + fun getEntireLengthBytes(buf: ByteArray, offset: Int): Int = getDataLengthBytes(buf, offset) + extHeaderSizeBytes + + /** + * Return the data size, in bytes, of the extension in [buf] whose header + * starts at [offset]. + */ + abstract fun getDataLengthBytes(buf: ByteArray, offset: Int): Int +} + +object OneByteHeaderExtensionParser : HeaderExtensionParser() { + override val headerExtensionLabel = 0xBEDE + + override val extHeaderSizeBytes = 1 + override val minimumExtSizeBytes = 2 + + override fun isMatchingType(profileField: Int): Boolean = profileField == headerExtensionLabel + + override fun getId(buf: ByteArray, offset: Int): Int = (buf[offset].toInt() ushr 4) and 0x0F + + override fun writeIdAndLength(id: Int, dataLength: Int, buf: ByteArray, offset: Int) { + require(id in 1..14) + require(dataLength in 1..16) + buf[offset] = ((id and 0x0F) shl 4).toByte() or ((dataLength - 1) and 0x0F).toByte() + } + + override fun getDataLengthBytes(buf: ByteArray, offset: Int): Int = + ((buf[offset] and 0x0F.toByte())).toPositiveInt() + 1 +} + +object TwoByteHeaderExtensionParser : HeaderExtensionParser() { + /* We don't support "value 256", in the low four bits of the "defined by profile" field. */ + override val headerExtensionLabel = 0x1000 + private const val HEADER_EXTENSION_MASK = 0xFFF0 + + override val extHeaderSizeBytes = 2 + override val minimumExtSizeBytes = 2 + + override fun isMatchingType(profileField: Int): Boolean = + (profileField and HEADER_EXTENSION_MASK) == headerExtensionLabel + + override fun getId(buf: ByteArray, offset: Int): Int = buf[offset].toInt() + + override fun writeIdAndLength(id: Int, dataLength: Int, buf: ByteArray, offset: Int) { + require(id in 1..255) + require(dataLength in 0..255) + + buf[offset] = id.toByte() + buf[offset + 1] = dataLength.toByte() + } + + override fun getDataLengthBytes(buf: ByteArray, offset: Int): Int = buf[offset + 1].toPositiveInt() +} + +private val headerExtensionParsers = arrayOf(OneByteHeaderExtensionParser, TwoByteHeaderExtensionParser) + +class HeaderExtensionHelpers { + companion object { + // The size of the header extension block header + const val TOP_LEVEL_EXT_HEADER_SIZE_BYTES = 4 + + /** + * Return the "defined by profile" header extension type field, as an integer. + * + * [offset] points to the start of the header extensions block. Should + * only be called if it's been verified that the header held in [buf] + * actually contains extensions + */ + fun getExtensionsProfileType(buf: ByteArray, offset: Int) = buf.getShortAsInt(offset) + + /** + * Return the length of the entire header extensions block, including + * the header, in bytes. + * + * [offset] points to the start of the header extensions block. Should + * only be called if it's been verified that the header held in [buf] + * actually contains extensions + */ + fun getExtensionsTotalLength(buf: ByteArray, offset: Int) = + TOP_LEVEL_EXT_HEADER_SIZE_BYTES + buf.getShortAsInt(offset + 2) * 4 + + /** + * Return a header extension parser for header extensions with the given "defined by profile" field + * value, or null if none match (i.e. either cryptex-encrypted or proprietary header extensions). + * Does the right thing for the invalid value -1 returned by [RtpHeader.getExtensionsProfileType] + * when no extensions are present. + */ + fun getHeaderExtensionParser(profileField: Int): HeaderExtensionParser? = + headerExtensionParsers.firstOrNull { it.isMatchingType(profileField) } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt new file mode 100644 index 0000000..3f0292c --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtension.kt @@ -0,0 +1,56 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import org.jitsi.rtp.rtp.RtpPacket +import java.nio.charset.StandardCharsets + +/** + * https://datatracker.ietf.org/doc/html/rfc7941#section-4.1.1 + * Note: this is only the One-Byte Format, because we don't support Two-Byte yet. + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ID | len | SDES item text value ... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class SdesHeaderExtension { + companion object { + fun getTextValue(ext: RtpPacket.HeaderExtension): String = + getTextValue(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + fun setTextValue(ext: RtpPacket.HeaderExtension, sdesValue: String) { + assert(ext.dataLengthBytes == sdesValue.length) { "buffer size doesn't match SDES value length" } + setTextValue(ext.buffer, ext.dataOffset, sdesValue) + } + + private fun getTextValue(buf: ByteArray, offset: Int, dataLength: Int): String { + /* RFC 7941 says the value in RTP is UTF-8. But we use this for MID and RID values + * which are define for SDP in RFC 5888 and RFC 4566 as ASCII only. Thus we don't + * support UTF-8 to keep things simpler. */ + return String(buf, offset, dataLength, StandardCharsets.US_ASCII) + } + + private fun setTextValue(buf: ByteArray, offset: Int, sdesValue: String) { + System.arraycopy( + sdesValue.toByteArray(StandardCharsets.US_ASCII), + 0, + buf, + offset, + sdesValue.length + ) + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt new file mode 100644 index 0000000..a271129 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/TccHeaderExtension.kt @@ -0,0 +1,45 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.util.getShortAsInt + +/** + * https://tools.ietf.org/html/draft-holmer-rmcat-transport-wide-cc-extensions-01#section-2.2 + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | ID | L=1 |transport-wide sequence number | zero padding | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ +class TccHeaderExtension { + + companion object { + const val DATA_SIZE_BYTES = 2 + + fun getSequenceNumber(ext: RtpPacket.HeaderExtension): Int = getSequenceNumber(ext.buffer, ext.dataOffset) + fun setSequenceNumber(ext: RtpPacket.HeaderExtension, tccSeqNum: Int) { + setSequenceNumber(ext.buffer, ext.dataOffset, tccSeqNum) + } + + private fun getSequenceNumber(buf: ByteArray, offset: Int): Int = buf.getShortAsInt(offset) + private fun setSequenceNumber(buf: ByteArray, offset: Int, seqNum: Int) { + buf.putShort(offset, seqNum.toShort()) + } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt new file mode 100644 index 0000000..7efbee9 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/rtp/header_extensions/VlaExtension.kt @@ -0,0 +1,129 @@ +/* + * Copyright @ 2024-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.Stream +import org.jitsi.rtp.util.BitReader + +/** + * A parser for the Video Layers Allocation RTP header extension. + * https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/video-layers-allocation00 + */ +@SuppressFBWarnings("SF_SWITCH_NO_DEFAULT", justification = "False positive") +class VlaExtension { + companion object { + fun parse(ext: RtpPacket.HeaderExtension): ParsedVla { + val empty = ext.dataLengthBytes == 1 && ext.buffer[ext.dataOffset] == 0.toByte() + if (empty) { + return emptyList() + } + + val reader = BitReader(ext.buffer, ext.dataOffset, ext.dataLengthBytes) + reader.skipBits(2) // RID + val ns = reader.bits(2) + 1 + val slBm = reader.bits(4) + val slBms = IntArray(4) { i -> if (i < ns) slBm else 0 } + if (slBm == 0) { + slBms[0] = reader.bits(4) + if (ns > 1) { + slBms[1] = reader.bits(4) + if (ns > 2) { + slBms[2] = reader.bits(4) + if (ns > 3) { + slBms[3] = reader.bits(4) + } + } + } + if (ns == 1 || ns == 3) { + reader.skipBits(4) + } + } + + val slCount = slBms.sumOf { it.countOneBits() } + val tlCountLenBytes = slCount / 4 + if (slCount % 4 != 0) 1 else 0 + + val tlCountReader = reader.clone(tlCountLenBytes) + reader.skipBits(tlCountLenBytes * 8) + + val streams = ArrayList(ns) + (0 until ns).forEach { streamIdx -> + val spatialLayers = ArrayList() + val stream = Stream(streamIdx, spatialLayers) + streams.add(stream) + + (0 until 4).forEach { slIdx -> + if ((slBms[streamIdx] and (1 shl slIdx)) != 0) { + val targetBitrates = buildList { + repeat(tlCountReader.bits(2) + 1) { + add(reader.leb128()) + } + } + spatialLayers.add( + SpatialLayer( + slIdx, + targetBitrates, + null + ) + ) + } + } + } + + (0 until ns).forEach outer@{ streamIdx -> + (0 until 4).forEach { slIdx -> + if ((slBms[streamIdx] and (1 shl slIdx)) != 0) { + if (reader.remainingBits() < 40) { + return@outer + } + val sl = streams[streamIdx].spatialLayers[slIdx] + sl.res = ResolutionAndFrameRate( + reader.bits(16) + 1, + reader.bits(16) + 1, + reader.bits(8) + ) + } + } + } + + return streams + } + } + + data class ResolutionAndFrameRate( + val width: Int, + val height: Int, + val maxFramerate: Int + ) + + data class SpatialLayer( + val id: Int, + // The target bitrates for each temporal layer in this spatial layer + val targetBitratesKbps: List, + var res: ResolutionAndFrameRate? + ) { + override fun toString(): String = "SpatialLayer(id=$id, targetBitratesKbps=$targetBitratesKbps, " + + "width=${res?.width}, height=${res?.height}, maxFramerate=${res?.maxFramerate})" + } + + data class Stream( + val id: Int, + val spatialLayers: List + ) +} + +typealias ParsedVla = List diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt new file mode 100644 index 0000000..9dcbd9b --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitReader.kt @@ -0,0 +1,135 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import kotlin.experimental.and + +/** + * Read individual bits, and unaligned sets of bits, from a [ByteArray], with an incrementing offset. + */ +/* TODO: put this in jitsi-utils? */ +class BitReader(val buf: ByteArray, private val byteOffset: Int = 0, private val byteLength: Int = buf.size) { + private var offset = byteOffset * 8 + private val byteBound = byteOffset + byteLength + + init { + check(byteOffset >= 0) { "byteOffset must be >= 0" } + check(byteBound <= buf.size) { "byteOffset + byteLength must be <= buf.size" } + } + + /** Clone with the current state (offset) and a new length in bytes. */ + fun clone(newByteLength: Int): BitReader { + check(offset % 8 == 0) { "Cannot clone BitReader with unaligned offset" } + check(offset / 8 + newByteLength <= byteBound) { + "newByteLength $newByteLength exceeds buffer length $byteLength after offset $byteOffset" + } + return BitReader(buf, offset / 8, newByteLength) + } + + fun remainingBits(): Int = byteBound * 8 - offset + + /** Read a single bit from the buffer, as a boolean, incrementing the offset. */ + fun bitAsBoolean(): Boolean { + val byteIdx = offset / 8 + val bitIdx = offset % 8 + check(byteIdx < byteBound) { + "offset $offset ($byteIdx/$bitIdx) invalid in buffer of length $byteLength after offset $byteOffset" + } + val byte = buf[byteIdx] + val mask = (1 shl (7 - bitIdx)).toByte() + offset++ + + return (byte and mask) != 0.toByte() + } + + /** Read a single bit from the buffer, as an integer, incrementing the offset. */ + fun bit() = if (bitAsBoolean()) 1 else 0 + + /** Read [n] bits from the buffer, returning them as an unsigned integer. */ + fun bits(n: Int): Int { + require(n < Int.SIZE_BITS) + + var ret = 0 + + /* TODO: optimize this */ + repeat(n) { + ret = ret shl 1 + ret = ret or bit() + } + + return ret + } + + /** Read [n] bits from the buffer, returning them as an unsigned long. */ + fun bitsLong(n: Int): Long { + require(n < Long.SIZE_BITS) + + var ret = 0L + + /* TODO: optimize this */ + repeat(n) { + ret = ret shl 1 + ret = ret or bit().toLong() + } + + return ret + } + + /** Skip forward [n] bits in the buffer. */ + fun skipBits(n: Int) { + offset += n + } + + /** Read a non-symmetric unsigned integer with max *value* [n] from the buffer. + * (Note: *not* the number of bits.) + * See https://aomediacodec.github.io/av1-rtp-spec/#a82-syntax + */ + fun ns(n: Int): Int { + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + val v = bits(w - 1) + if (v < m) { + return v + } + val extraBit = bit() + return (v shl 1) - m + extraBit + } + + /** + * Read a LEB128-encoded unsigned integer. + * https://aomediacodec.github.io/av1-spec/#leb128 + */ + fun leb128(): Long { + var value = 0L + (0..8).forEach { i -> + val hasNext = bitAsBoolean() + value = value or (bits(7).toLong() shl (i * 7)) + if (!hasNext) return value + } + return value + } + + /** Reset the reader to the beginning of the buffer */ + fun reset() { + offset = byteOffset * 8 + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt new file mode 100644 index 0000000..9601d0d --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/BitWriter.kt @@ -0,0 +1,73 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import java.util.* +import kotlin.experimental.or + +/** + * Write individual bits, and unaligned sets of bits, to a [ByteArray], with an incrementing offset. + */ +class BitWriter(val buf: ByteArray, val byteOffset: Int = 0, private val byteLength: Int = buf.size) { + private var offset = byteOffset * 8 + private val byteBound = byteOffset + byteLength + + init { + Arrays.fill(buf, byteOffset, byteBound, 0) + } + + fun writeBit(value: Boolean) { + val byteIdx = offset / 8 + val bitIdx = offset % 8 + check(byteIdx < byteBound) { + "offset $offset ($byteIdx/$bitIdx) invalid in buffer of length $byteLength after offset $byteOffset" + } + + if (value) { + buf[byteIdx] = buf[byteIdx] or (1 shl (7 - bitIdx)).toByte() + } + offset++ + } + + fun writeBits(bits: Int, value: Int) { + check(value < (1 shl bits)) { + "value $value cannot be represented in $bits bits" + } + repeat(bits) { i -> + writeBit((value and (1 shl (bits - i - 1))) != 0) + } + } + + fun writeNs(n: Int, v: Int) { + if (n == 1) return + var w = 0 + var x = n + while (x != 0) { + x = x shr 1 + w++ + } + val m = (1 shl w) - n + if (v < m) { + writeBits(w - 1, v) + } else { + writeBits(w, v + m) + } + } + + val remainingBits + get() = byteBound * 8 - offset +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/BufferPool.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/BufferPool.kt new file mode 100644 index 0000000..f24a4a3 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/BufferPool.kt @@ -0,0 +1,32 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +/** + * All [ByteArray]s that the library needs for packets will be acquired via [getArray]. + * + * When we're done with a buffer, we'll pass it to [returnArray]. These methods + * can be overridden so that a user of this library can use a pool for the buffers. + * + * @author Brian Baldino + */ +class BufferPool { + companion object { + var getArray: (Int) -> ByteArray = { size -> ByteArray(size) } + var returnArray: (ByteArray) -> Unit = { } + } +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/ByteBufferUtils.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/ByteBufferUtils.kt new file mode 100644 index 0000000..334dacf --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/ByteBufferUtils.kt @@ -0,0 +1,26 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.util + +import java.nio.ByteBuffer + +// TODO: i think these should only be used for tests. Move them? +fun byteBufferOf(vararg elements: Byte): ByteBuffer = ByteBuffer.wrap(byteArrayOf(*elements)) + +fun byteBufferOf(vararg elements: Number): ByteBuffer { + val bytes = elements.map { it.toByte() }.toByteArray() + return ByteBuffer.wrap(bytes) +} diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt new file mode 100644 index 0000000..eef031e --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/FieldParsers.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import org.jitsi.rtp.extensions.bytearray.get3Bytes +import org.jitsi.rtp.extensions.bytearray.getInt +import org.jitsi.rtp.extensions.bytearray.getShort +import org.jitsi.rtp.extensions.bytearray.putBits +import org.jitsi.rtp.extensions.getBits +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.extensions.unsigned.toPositiveLong + +// TODO: could do some kind of 'Offset' inline class? or 'field' which described the offset +// and size? + +fun ByteArray.getBitsAsInt(byteOffset: Int, bitStartPos: Int, numBits: Int): Int = + get(byteOffset).getBits(bitStartPos, numBits).toPositiveInt() + +fun ByteArray.putNumberAsBits(byteOffset: Int, bitOffset: Int, numBits: Int, value: Number) { + putBits(byteOffset, bitOffset, value.toByte(), numBits) +} +fun ByteArray.getByteAsInt(offset: Int): Int = get(offset).toPositiveInt() + +fun ByteArray.getShortAsInt(offset: Int): Int = getShort(offset).toPositiveInt() + +fun ByteArray.get3BytesAsInt(offset: Int): Int = get3Bytes(offset).toPositiveInt() + +fun ByteArray.getIntAsLong(offset: Int): Long = getInt(offset).toPositiveLong() diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt new file mode 100644 index 0000000..9b71536 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/RtpUtils.kt @@ -0,0 +1,182 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.util + +import org.jitsi.utils.TimeUtils +import java.time.Duration +import java.time.Instant + +class RtpUtils { + companion object { + /** + * [sizeBytes] MUST including padding (i.e. it should be 32-bit word aligned) + */ + fun calculateRtcpLengthFieldValue(sizeBytes: Int): Int { + if (sizeBytes % 4 != 0) { + throw Exception("Invalid RTCP size value") + } + return (sizeBytes / 4) - 1 + } + + /** + * Get the number of bytes needed to pad [dataSizeBytes] bytes to a 4-byte word boundary. + */ + fun getNumPaddingBytes(dataSizeBytes: Int): Int = when (dataSizeBytes % 4) { + 0 -> 0 + 1 -> 3 + 2 -> 2 + 3 -> 1 + else -> 0 // The above is exhaustive. + } + + /** + * Returns the delta between two RTP sequence numbers, taking into account + * rollover. This will return the 'shortest' delta between the two + * sequence numbers in the form of the number you'd add to b to get a. e.g.: + * getSequenceNumberDelta(1, 10) -> -9 (10 + -9 = 1) + * getSequenceNumberDelta(1, 65530) -> 7 (65530 + 7 = 1) + * @return the delta between two RTP sequence numbers (modulo 2^16). + */ + @JvmStatic + fun getSequenceNumberDelta(a: Int, b: Int): Int = getSequenceNumberDeltaAsShort(a, b).toInt() + + /** + * Like [getSequenceNumberDelta], but returning the delta as a [Short]. + * + * TODO: since [Short] can fully represent sequence number deltas, + * change this to be the public API? + */ + private inline fun getSequenceNumberDeltaAsShort(a: Int, b: Int): Short = + /* Coercing to short forces diff to the range -0x8000 - 0x7fff, + which is what we want. */ + (a - b).toShort() + + /** + * Apply a delta to a given sequence number and return the result (taking + * rollover into account) + * @param start the starting sequence number + * @param delta the delta to be applied + * @return the sequence number resulting from doing "start + delta" + */ + @JvmStatic + fun applySequenceNumberDelta(start: Int, delta: Int): Int = (start + delta) and 0xffff + + /** + * Apply a delta to a given RTP timestamp and return the result (taking + * rollover into account) + * @param start the starting timestamp + * @param delta the delta to be applied + * @return the timestamp result from doing "start + delta" + */ + @JvmStatic + fun applyTimestampDelta(start: Long, delta: Long): Long = (start + delta) and 0xffff_ffffL + + @JvmStatic + fun isNewerSequenceNumberThan(a: Int, b: Int): Boolean = getSequenceNumberDeltaAsShort(a, b) > 0 + + @JvmStatic + fun isOlderSequenceNumberThan(a: Int, b: Int): Boolean = getSequenceNumberDeltaAsShort(a, b) < 0 + + @JvmStatic + fun isNewerTimestampThan(a: Long, b: Long): Boolean = getTimestampDiffAsInt(a, b) > 0 + + @JvmStatic + fun isOlderTimestampThan(a: Long, b: Long): Boolean = getTimestampDiffAsInt(a, b) < 0 + + /** + * Returns the difference between two RTP timestamps. + * @return the difference between two RTP timestamps. + */ + @JvmStatic + fun getTimestampDiff(a: Long, b: Long): Long = getTimestampDiffAsInt(a, b).toLong() + + /** + * Returns the difference between two RTP timestamps as an [Int]. + */ + inline fun getTimestampDiffAsInt(a: Long, b: Long): Int = + /* Coercing to int forces diff to the range -0x8000_0000 - 0x7fff_ffff, + which is what we want. */ + (a - b).toInt() + + /** + * Returns a sequence of Ints from olderSeqNum (exclusive) to newerSeqNum (exclusive), + * taking rollover into account + */ + fun sequenceNumbersBetween(olderSeqNum: Int, newerSeqNum: Int): Sequence { + var currSeqNum = olderSeqNum + return generateSequence { + currSeqNum = (currSeqNum + 1) % 0x1_0000 + if (currSeqNum == newerSeqNum) { + null + } else { + currSeqNum + } + } + } + + /** + * Given [timestampMs] (a timestamp in milliseconds), convert it to an NTP timestamp represented + * as a pair of ints: the first one being the most significant word and the second being the least + * significant word. + */ + @JvmStatic + fun millisToNtpTimestamp(timestampMs: Long): Long = TimeUtils.toNtpTime(timestampMs) + + @JvmStatic + fun convertRtpTimestampToMs(rtpTimestamp: Int, ticksPerSecond: Int): Long = + ((rtpTimestamp / (ticksPerSecond.toDouble())) * 1000).toLong() + + @JvmStatic + fun convertRtpTimestampToInstant(rtpTimestamp: Int, ticksPerSecond: Int): Instant = + Instant.EPOCH.plus(Duration.ofSeconds(rtpTimestamp.toLong()).dividedBy(ticksPerSecond.toLong())) + } +} + +fun Int.isPadding(): Boolean = this.toByte().isPadding() +fun Byte.isPadding(): Boolean = this == 0x00.toByte() + +/** + * Returns true if the RTP sequence number represented by [this] represents a more recent RTP packet than the one + * represented by [otherSeqNum] + */ +infix fun Int.isNewerThan(otherSeqNum: Int): Boolean = RtpUtils.isNewerSequenceNumberThan(this, otherSeqNum) + +infix fun Int.isOlderThan(otherSeqNum: Int): Boolean = RtpUtils.isOlderSequenceNumberThan(this, otherSeqNum) + +infix fun Long.isNewerTimestampThan(otherTimestamp: Long): Boolean = RtpUtils.isNewerTimestampThan(this, otherTimestamp) + +infix fun Long.isOlderTimestampThan(otherTimestamp: Long): Boolean = RtpUtils.isOlderTimestampThan(this, otherTimestamp) + +/** + * Returns true if getting to [otherSeqNum] from the current sequence number involves wrapping around + */ +infix fun Int.rolledOverTo(otherSeqNum: Int): Boolean = + // If, according to [isOlderThan], [this] is older than [otherSeqNum] and + // yet [otherSeqNum] is less than [this], then we wrapped around to get from [this] to + // [otherSeqNum] + this isOlderThan otherSeqNum && otherSeqNum < this + +/** + * Returns true if [this] is sequentially after [otherSeqNum], according to the rules of RTP sequence + * numbers + */ +infix fun Int.isNextAfter(otherSeqNum: Int): Boolean = RtpUtils.getSequenceNumberDelta(this, otherSeqNum) == 1 + +/** + * Return the amount of packets between the RTP sequence number represented by [this] and the [otherSeqNum]. NOTE: + * [this] must represent an older RTP sequence number than [otherSeqNum] (TODO: validate/enforce that) + */ +infix fun Int.numPacketsTo(otherSeqNum: Int): Int = -RtpUtils.getSequenceNumberDelta(this, otherSeqNum) - 1 diff --git a/rtp/src/main/kotlin/org/jitsi/rtp/util/SrtpUtils.kt b/rtp/src/main/kotlin/org/jitsi/rtp/util/SrtpUtils.kt new file mode 100644 index 0000000..2e4ed24 --- /dev/null +++ b/rtp/src/main/kotlin/org/jitsi/rtp/util/SrtpUtils.kt @@ -0,0 +1,27 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import org.jitsi.rtp.extensions.bytearray.getInt +import org.jitsi.utils.ByteArrayBuffer + +class SrtpUtils { + companion object { + fun getSrtcpIndex(packet: ByteArrayBuffer, authTagLen: Int): Int = + packet.buffer.getInt(packet.offset + packet.length - (4 + authTagLen)) + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteArrayBufferTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteArrayBufferTest.kt new file mode 100644 index 0000000..c1c7404 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteArrayBufferTest.kt @@ -0,0 +1,81 @@ +package org.jitsi.rtp.extensions + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.extensions.bytearray.byteArrayOf + +class ByteArrayBufferTest : ShouldSpec() { + private val buffer = byteArrayOf( + 0x90, 0x6f, 0x16, 0xaf, + 0x65, 0xf3, 0xe8, 0xce, + 0x48, 0x0f, 0x22, 0x3a, + 0xbe, 0xde, 0x00, 0x01, + 0x10, 0xff, 0x00, 0x00 + ) + + init { + context("ByteArrayBuffer.toHex") { + should("format a full buffer correctly") { + val packet = UnparsedPacket(buffer, 0, buffer.size) + packet.toHex() shouldBe "906F16AF 65F3E8CE 480F223A BEDE0001\n10FF0000" + } + should("honor maxBytes") { + val packet = UnparsedPacket(buffer, 0, buffer.size) + packet.toHex(12) shouldBe "906F16AF 65F3E8CE 480F223A" + } + should("handle large maxBytes") { + val packet = UnparsedPacket(buffer, 0, buffer.size) + packet.toHex(40) shouldBe "906F16AF 65F3E8CE 480F223A BEDE0001\n10FF0000" + } + should("work correctly for offset packets") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 4) + packet.toHex() shouldBe "65F3E8CE 480F223A BEDE0001 10FF0000" + } + should("honor maxBytes for offset packets") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 4) + packet.toHex(12) shouldBe "65F3E8CE 480F223A BEDE0001" + } + should("handle large maxBytes for offset packets") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 4) + packet.toHex(40) shouldBe "65F3E8CE 480F223A BEDE0001 10FF0000" + } + should("handle maxBytes large enough that maxBytes + offset wraps Int") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 4) + packet.toHex(Int.MAX_VALUE - 2) shouldBe "65F3E8CE 480F223A BEDE0001 10FF0000" + } + should("work correctly for short packets") { + val packet = UnparsedPacket(buffer, 0, buffer.size - 4) + packet.toHex() shouldBe "906F16AF 65F3E8CE 480F223A BEDE0001" + } + should("honor maxBytes for short packets") { + val packet = UnparsedPacket(buffer, 0, buffer.size - 4) + packet.toHex(12) shouldBe "906F16AF 65F3E8CE 480F223A" + } + should("handle large maxBytes for short packets") { + val packet = UnparsedPacket(buffer, 0, buffer.size - 4) + packet.toHex(40) shouldBe "906F16AF 65F3E8CE 480F223A BEDE0001" + } + should("work correctly for short packets with offset") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 8) + packet.toHex() shouldBe "65F3E8CE 480F223A BEDE0001" + } + should("honor maxBytes for short packets with offset") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 8) + packet.toHex(8) shouldBe "65F3E8CE 480F223A" + } + should("handle large maxBytes for short packets with offset") { + val packet = UnparsedPacket(buffer, 4, buffer.size - 8) + packet.toHex(40) shouldBe "65F3E8CE 480F223A BEDE0001" + } + should("handle packets with invalidly large length values") { + val packet = UnparsedPacket(buffer, 0, buffer.size + 40) + packet.toHex() shouldBe "906F16AF 65F3E8CE 480F223A BEDE0001\n10FF0000" + } + should("honor maxBytes for packets with invalidly large length values") { + val packet = UnparsedPacket(buffer, 0, buffer.size + 40) + packet.toHex(40) shouldBe "906F16AF 65F3E8CE 480F223A BEDE0001\n10FF0000" + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt new file mode 100644 index 0000000..47abbc2 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteBufferExtensionsTest.kt @@ -0,0 +1,216 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.util.byteBufferOf +import org.jitsi.test_helpers.matchers.haveSameContentAs +import java.nio.ByteBuffer + +class ByteBufferExtensionsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + context("ByteBuffer") { + context("put3Bytes") { + val buf = ByteBuffer.allocate(4) + context("for a 3 byte value") { + // 424242 = 0x067932 + val num: Int = 424242 + buf.put3Bytes(num) + buf.rewind() + should("write the bytes correctly") { + buf.get() shouldBe 0x06.toByte() + buf.get() shouldBe 0x79.toByte() + buf.get() shouldBe 0x32.toByte() + } + } + context("for a 2 byte value") { + // 4242 = 0x1092 + val num: Int = 4242 + buf.put3Bytes(num) + buf.rewind() + should("write the bytes correctly") { + buf.get() shouldBe 0x00.toByte() + buf.get() shouldBe 0x10.toByte() + buf.get() shouldBe 0x92.toByte() + } + } + context("for a 1 byte value") { + // 42 = 0x2A + val num: Int = 42 + buf.put3Bytes(num) + buf.rewind() + should("write the bytes correctly") { + buf.get() shouldBe 0x00.toByte() + buf.get() shouldBe 0x00.toByte() + buf.get() shouldBe 0x2A.toByte() + } + } + } + context("get3Bytes") { + val buf = ByteBuffer.allocate(4) + context("for a 3 byte value") { + val num: Int = 0xb1b1b1 + buf.put3Bytes(num) + buf.rewind() + should("read them correctly") { + buf.get3Bytes() shouldBe 0xb1b1b1.toInt() + } + } + context("for a 2 byte value") { + val num: Int = 4242 + buf.put3Bytes(num) + buf.rewind() + should("read them correctly") { + buf.get3Bytes() shouldBe 4242.toInt() + } + } + context("for a 1 byte value") { + val num: Int = 42 + buf.put3Bytes(num) + buf.rewind() + should("read them correctly") { + buf.get3Bytes() shouldBe 42.toInt() + } + } + } + context("putBits") { + should("write the bits into the buffer correctly") { + val buf = ByteBuffer.allocate(4) + val src: Byte = 0b00001111 + + buf.putBits(1, 0, src, 4) + + buf.get(1) shouldBe 0b11110000.toByte() + // Nothing else should've changed + listOf(0, 2, 3).forEach { + buf.get(it) shouldBe 0b00000000.toByte() + } + } + } + context("subBuffer(startPosition, size)") { + val originalBuf = ByteBuffer.allocate(100) + context("created from a default original buffer") { + val subBuf = originalBuf.subBuffer(10, 30) + should("have a correctly set position, limit and capacity") { + subBuf.position() shouldBe 0 + subBuf.limit() shouldBe 30 + subBuf.capacity() shouldBe 30 + } + should("not affect the original buffer in any way") { + originalBuf.position() shouldBe 0 + originalBuf.limit() shouldBe 100 + originalBuf.capacity() shouldBe 100 + } + should("represent the correct spot in the original buffer") { + repeat(subBuf.limit()) { subBuf.put(it, 0x42) } + + (0..9).forEach { originalBuf.get(it) shouldBe 0x00.toByte() } + (10..39).forEach { originalBuf.get(it) shouldBe 0x42.toByte() } + (40..99).forEach { originalBuf.get(it) shouldBe 0x00.toByte() } + } + } + context("created from a buffer whose current position isn't 0") { + originalBuf.position(10) + val subBuf = originalBuf.subBuffer(5, 10) + should("represent the correct spot in the original buffer") { + repeat(subBuf.limit()) { subBuf.put(it, 0x42) } + + (0..4).forEach { originalBuf.get(it) shouldBe 0x00.toByte() } + (5..14).forEach { originalBuf.get(it) shouldBe 0x42.toByte() } + (15..99).forEach { originalBuf.get(it) shouldBe 0x00.toByte() } + } + } + } + context("subBuffer(startPosition)") { + val originalBuf = ByteBuffer.allocate(100) + context("created from a default original buffer") { + val subBuf = originalBuf.subBuffer(10) + should("have a correctly set position, limit and capacity") { + subBuf.position() shouldBe 0 + subBuf.limit() shouldBe 90 + subBuf.capacity() shouldBe 90 + } + should("not affect the original buffer in any way") { + originalBuf.position() shouldBe 0 + originalBuf.limit() shouldBe 100 + originalBuf.capacity() shouldBe 100 + } + should("represent the correct spot in the original buffer") { + repeat(subBuf.limit()) { subBuf.put(it, 0x42) } + + (0..9).forEach { originalBuf.get(it) shouldBe 0x00.toByte() } + (10..99).forEach { originalBuf.get(it) shouldBe 0x42.toByte() } + } + } + context("created from a buffer whose current position isn't 0") { + originalBuf.position(10) + val subBuf = originalBuf.subBuffer(5) + should("represent the correct spot in the original buffer") { + repeat(subBuf.limit()) { subBuf.put(it, 0x42) } + + (0..4).forEach { originalBuf.get(it) shouldBe 0x00.toByte() } + (5..99).forEach { originalBuf.get(it) shouldBe 0x42.toByte() } + } + } + } + context("shiftDataRight") { + context("moving data") { + val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x05, 0x06) + buf.shiftDataRight(1, 3, 2) + should("move things correctly") { + buf should haveSameContentAs(byteBufferOf(0x01, 0x02, 0x03, 0x02, 0x03, 0x04)) + } + } + context("moving data past the current limit but within the capacity") { + val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x00, 0x0, 0x00, 0x00) + buf.limit(4) + buf.shiftDataRight(1, 3, 2) + should("move things correctly") { + buf.limit() shouldBe 6 + buf should haveSameContentAs(byteBufferOf(0x01, 0x02, 0x03, 0x02, 0x03, 0x04)) + } + } + context("moving the data past the capacity") { + val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x00, 0x0, 0x00, 0x00) + shouldThrow { + buf.shiftDataRight(4, 6, 10) + } + } + } + context("shiftDataLeft") { + context("moving data") { + val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x05, 0x06) + buf.shiftDataLeft(2, 4, 2) + should("move things correctly") { + buf should haveSameContentAs(byteBufferOf(0x03, 0x04, 0x05, 0x04, 0x05, 0x06)) + } + } + context("moving the data past the start") { + val buf = byteBufferOf(0x01, 0x02, 0x03, 0x04, 0x00, 0x0, 0x00, 0x00) + shouldThrow { + buf.shiftDataLeft(1, 2, 2) + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteExtensionsTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteExtensionsTest.kt new file mode 100644 index 0000000..f5c0201 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/ByteExtensionsTest.kt @@ -0,0 +1,89 @@ +/* + * Copyright @ 2018 - Present, 8x8 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class ByteExtensionsTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("Byte.getBit") { + should("Correctly get the bit in each position") { + for (onePosition in 0..7) { + val b: Byte = (0b1 shl (7 - onePosition)).toByte() + for (currPosition in 0..7) { + when (currPosition) { + onePosition -> b.getBit(currPosition) shouldBe 1 + else -> b.getBit(currPosition) shouldBe 0 + } + } + } + } + } + context("Byte.putBit") { + should("Set bits to true correctly") { + val b: Byte = 0x00 + b.putBit(0, true) shouldBe 0b10000000.toByte() + b.putBit(3, true) shouldBe 0b00010000.toByte() + b.putBit(7, true) shouldBe 0b00000001.toByte() + } + should("Set bits to false correctly") { + val b: Byte = 0xFF.toByte() + b.putBit(1, false) shouldBe 0b10111111.toByte() + b.putBit(2, false) shouldBe 0b11011111.toByte() + b.putBit(4, false) shouldBe 0b11110111.toByte() + b.putBit(7, false) shouldBe 0b11111110.toByte() + } + should("Support a mix of sets/unsets") { + var b: Byte = 0x00 + (0..7).forEach { + b = b.putBit(it, true) + } + b shouldBe 0b11111111.toByte() + (0..7).forEach { + b = b.putBit(it, false) + } + b shouldBe 0b00000000.toByte() + } + } + context("Byte.putBits") { + should("correctly put bits") { + val dest: Byte = 0b00000000 + val src: Byte = 0b00000101 + + val result = dest.putBits(0, 3, src) + result shouldBe 0b10100000.toByte() + } + should("overwrite existing values correctly") { + val dest: Byte = 0b11111111.toByte() + val src: Byte = 0b00000000 + + val result = dest.putBits(0, 3, src) + result shouldBe 0b00011111.toByte() + } + should("work correctly with different offsets") { + val dest: Byte = 0b00000000 + val src: Byte = 0b11001100.toByte() + + val result = dest.putBits(2, 4, src) + result shouldBe 0b00110000.toByte() + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt new file mode 100644 index 0000000..d712ee5 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/VlaExtensionTest.kt @@ -0,0 +1,372 @@ +/* + * Copyright @ 2024-Present 8x8, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.extensions + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.rtp.RtpPacket.HeaderExtension +import org.jitsi.rtp.rtp.header_extensions.ParsedVla +import org.jitsi.rtp.rtp.header_extensions.VlaExtension +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.ResolutionAndFrameRate +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.SpatialLayer +import org.jitsi.rtp.rtp.header_extensions.VlaExtension.Stream + +@Suppress("ktlint:standard:no-multi-spaces") +class VlaExtensionTest : ShouldSpec() { + init { + context("Empty") { + parse(0x00) shouldBe emptyList() + } + context("VP8 single stream (with resolution)") { + parse( + 0b0000_0001, + 0b1000_0000.toByte(), + 0b0101_0000, + 0b0111_1000, + 0b1100_1000.toByte(), + 0b0000_0001, + 0b0000_0001, + 0b0011_1111, + 0b0000_0000, + 0b1011_0011.toByte(), + 0b0010_0001 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(80, 120, 200), + ResolutionAndFrameRate(320, 180, 33) + ) + ) + ) + ) + } + context("VP8 single stream (without resolution)") { + parse( + 0b0000_0001, + 0b1000_0000.toByte(), + 0b0101_0000, + 0b0111_1000, + 0b1100_1000.toByte(), + 0b0000_0001 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(80, 120, 200), + null + ) + ) + ) + ) + } + context("VP8 simulcast stream (with resolutions)") { + parse( + 0b0010_0001, + 0b1010_1000.toByte(), + 0b0011_1100, + 0b0101_1010, + 0b1001_0110.toByte(), + 0b0000_0001, + 0b1100_1000.toByte(), + 0b0000_0001, + 0b1010_1100.toByte(), + 0b0000_0010, + 0b1111_0100.toByte(), + 0b0000_0011, + 0b1110_1110.toByte(), + 0b0000_0011, + 0b1110_0110.toByte(), + 0b0000_0101, + 0b1101_0100.toByte(), + 0b0000_1001, + 0b0000_0001, + 0b0011_1111, + 0b0000_0000, + 0b1011_0011.toByte(), + 0b0001_1111, + 0b0000_0010, + 0b0111_1111, + 0b0000_0001, + 0b0110_0111, + 0b0001_1111, + 0b0000_0100, + 0b1111_1111.toByte(), + 0b0000_0010, + 0b1100_1111.toByte(), + 0b0001_1111 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(60, 90, 150), + ResolutionAndFrameRate(320, 180, 31) + ) + ) + ), + Stream( + 1, + listOf( + SpatialLayer( + 0, + listOf(200, 300, 500), + ResolutionAndFrameRate(640, 360, 31) + ) + ) + ), + Stream( + 2, + listOf( + SpatialLayer( + 0, + listOf(494, 742, 1236), + ResolutionAndFrameRate(1280, 720, 31) + ) + ) + ) + ) + } + context("VP8 simulcast stream (without resolutions)") { + parse( + 0b1010_0001.toByte(), + 0b1010_1000.toByte(), + 0b1001_0101.toByte(), + 0b0000_0010, + 0b1001_1111.toByte(), + 0b0000_0011, + 0b1011_0100.toByte(), + 0b0000_0101, + 0b1000_1101.toByte(), + 0b0000_1001, + 0b1101_0011.toByte(), + 0b0000_1101, + 0b1110_0000.toByte(), + 0b0001_0110, + 0b1101_0000.toByte(), + 0b0000_1111, + 0b1011_1000.toByte(), + 0b0001_0111, + 0b1000_1000.toByte(), + 0b0010_0111 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(277, 415, 692), + null + ) + ) + ), + Stream( + 1, + listOf( + SpatialLayer( + 0, + listOf(1165, 1747, 2912), + null + ) + ) + ), + Stream( + 2, + listOf( + SpatialLayer( + 0, + listOf(2000, 3000, 5000), + null + ) + ) + ) + ) + } + context("VP9 SVC (with resolutions)") { + parse( + 0b0000_0111, + 0b1010_1000.toByte(), + 0b0100_1101, + 0b0110_0100, + 0b1000_1110.toByte(), + 0b0000_0001, + 0b1101_0111.toByte(), + 0b0000_0001, + 0b1001_0111.toByte(), + 0b0000_0010, + 0b1000_1101.toByte(), + 0b0000_0011, + 0b1101_0110.toByte(), + 0b0000_0010, + 0b1011_1101.toByte(), + 0b0000_0011, + 0b1111_1001.toByte(), + 0b0000_0100, + 0b0000_0001, + 0b0011_1111, + 0b0000_0000, + 0b1011_0011.toByte(), + 0b0010_0000, + 0b0000_0010, + 0b0111_1111, + 0b0000_0001, + 0b0110_0111, + 0b0010_0000, + 0b0000_0100, + 0b1111_1111.toByte(), + 0b0000_0010, + 0b1100_1111.toByte(), + 0b0010_0000 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(77, 100, 142), + ResolutionAndFrameRate(320, 180, 32) + ), + SpatialLayer( + 1, + listOf(215, 279, 397), + ResolutionAndFrameRate(640, 360, 32) + ), + SpatialLayer( + 2, + listOf(342, 445, 633), + ResolutionAndFrameRate(1280, 720, 32) + ) + ) + ) + ) + } + context("VP9 SVC (without resolutions or TLs)") { + parse( + 0b0000_0111, + 0b0000_0000, + 0b1001_0110.toByte(), + 0b0000_0001, + 0b1111_0100.toByte(), + 0b0000_0011, + 0b1010_1010.toByte(), + 0b0000_1011 + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(150), + null + ), + SpatialLayer( + 1, + listOf(500), + null + ), + SpatialLayer( + 2, + listOf(1450), + null + ) + ) + ) + ) + } + context("Invalid VLAs") { + context("Resolution incomplete") { + // Cuts short with not enough bytes to contain the resolution. We succeed with no resolution. + parse( + 0b0000_0001, + 0b1000_0000.toByte(), // #tls + 0b0101_0000, // targetBitrate 1 + 0b0111_1000, // targetBitrate 2 + 0b1100_1000.toByte(), // targetBitrate 3 + 0b0000_0001, // targetBitrate 3 + 0b0000_0001, // width + 0b0011_1111, // width + 0b0000_0000, // height + 0b1011_0011.toByte(), // height + // 0b0010_0001 // maxFramerate + ) shouldBe listOf( + Stream( + 0, + listOf( + SpatialLayer( + 0, + listOf(80, 120, 200), + null + ) + ) + ) + ) + } + context("Invalid leb128") { + // Cuts short in the middle of one of the leb128 encoding of one of the target bitrates. + shouldThrow { + parse( + 0b0000_0001, + 0b1000_0000.toByte(), // #tls + 0b0101_0000, // targetBitrate 1 + 0b0111_1000, // targetBitrate 2 + 0b1100_1000.toByte(), // targetBitrate 3 + // 0b0000_0001, // targetBitrate 3 + ) + } + } + context("Missing target bitrates") { + // Does not contain the expected number of target bitrates + shouldThrow { + parse( + 0b0000_0001, // 1 spatial layer + 0b1000_0000.toByte(), // 3 temporal layers + 0b0101_0000, // targetBitrate 1 + 0b1100_1000.toByte(), // targetBitrate 2 + 0b0000_0001, // targetBitrate 2 + // 0b0111_1000, // targetBitrate 3 + ) + } + } + context("Missing temporal layer counts") { + // Does not contain the expected temporal layer counts + shouldThrow { + parse( + 0b0001_0000, // spatial layer bitmask in the next byte, 2 streams + 0b1111_1111.toByte(), // 4 spatial layers for each stream + 0b0000_0000.toByte(), // 1 temporal layer for each spatial layer of stream 1, more must follow + ) + } + } + } + } +} + +private fun parse(vararg bytes: Byte): ParsedVla = VlaExtension.parse(RawHeaderExtension(bytes)) + +@SuppressFBWarnings("CN_IMPLEMENTS_CLONE_BUT_NOT_CLONEABLE") +class RawHeaderExtension(override val buffer: ByteArray) : HeaderExtension { + override val dataOffset: Int = 0 + override var id: Int = 0 + override val dataLengthBytes: Int = buffer.size + override val totalLengthBytes: Int = buffer.size +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensionsKtTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensionsKtTest.kt new file mode 100644 index 0000000..85d77ac --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/bytearray/ByteArrayExtensionsKtTest.kt @@ -0,0 +1,57 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.extensions.bytearray + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.data.forAll +import io.kotest.data.row +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.test_helpers.matchers.haveSameContentAs + +class ByteArrayExtensionsKtTest : ShouldSpec() { + init { + context("ByteArray.getShort/putShort") { + should("parse the short correctly") { + forAll( + row(byteArrayOf(0x00, 0x00), 0.toShort()), + row(byteArrayOf(0xFF, 0xFF), 65535.toShort()) + ) { buf, expectedShort -> + buf.getShort(0) shouldBe expectedShort + + val array = ByteArray(2) + array.putShort(0, expectedShort) + array should haveSameContentAs(buf) + } + } + } + context("ByteArray.getInt/putInt") { + should("parse the short correctly") { + forAll( + row(byteArrayOf(0x00, 0x00, 0x00, 0x00), 0.toInt()), + row(byteArrayOf(0xFF, 0xFF, 0xFF, 0xFF), 4294967295.toInt()) + ) { buf, expectedInt -> + buf.getInt(0) shouldBe expectedInt + + val array = ByteArray(4) + array.putInt(0, expectedInt) + array should haveSameContentAs(buf) + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/extensions/unsigned/UnsignedKtTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/unsigned/UnsignedKtTest.kt new file mode 100644 index 0000000..6cf3fec --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/extensions/unsigned/UnsignedKtTest.kt @@ -0,0 +1,54 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.extensions.unsigned + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +internal class UnsignedKtTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("converting a byte to a positive int") { + should("convert correctly") { + val byteSizedInts = listOf(0, 1, 63, 127, 128, 244, 255) + byteSizedInts.forEach { byteSizedInt -> + val byte = byteSizedInt.toByte() + val convertedInt = byte.toPositiveInt() + convertedInt shouldBe byteSizedInt + } + } + } + context("converting a short to a positive int") { + val shortSizedInts = listOf(0, 1, 63, 127, 65530, 65535) + shortSizedInts.forEach { shortSizedInt -> + val short = shortSizedInt.toShort() + val convertedInt = short.toPositiveInt() + convertedInt shouldBe shortSizedInt + } + } + context("converting an int to a positive long") { + val intSizedLongs = listOf(0, 1, 63, 127, 2294967195, 4294967295) + intSizedLongs.forEach { intSizedLong -> + val int = intSizedLong.toInt() + val convertedLong = int.toPositiveLong() + convertedLong shouldBe intSizedLong + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/CompoundRtcpPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/CompoundRtcpPacketTest.kt new file mode 100644 index 0000000..b2fd92f --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/CompoundRtcpPacketTest.kt @@ -0,0 +1,78 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.Packet +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbPliPacketBuilder +import org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb.RtcpFbRembPacketBuilder + +internal class CompoundRtcpPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("Creating a compound packet from a list of packets") { + val rr = RtcpRrPacketBuilder(RtcpHeaderBuilder(), mutableListOf()).build() + val remb = RtcpFbRembPacketBuilder(RtcpHeaderBuilder(), listOf(123), 12345).build() + val pli = RtcpFbPliPacketBuilder(RtcpHeaderBuilder(), 456).build() + + val packets = listOf(rr, remb, pli) + val compoundRtcpPacket = CompoundRtcpPacket(packets) + compoundRtcpPacket.packets.size shouldBe packets.size + compoundRtcpPacket.packets.forEachIndexed { i, packet -> + packet.length shouldBe packets[i].length + packet::class shouldBe packets[i]::class + } + } + context("Parsing a compound RTCP packet which contains invalid data") { + val packet = DummyPacket(rtcpByeNoReason + invalidRtcpData) + shouldThrow { + CompoundRtcpPacket.parse(packet.buffer, packet.offset, packet.length) + } + } + + context("Creating multiple compound packets by limiting to MTU") { + val rr = RtcpRrPacketBuilder(RtcpHeaderBuilder(), mutableListOf()).build() + val compoundPackets = CompoundRtcpPacket.createWithMtu(generateSequence { rr }.take(200).toList()) + // rr without report blocks is 8 bytes, so 200 is 2000 bytes; default MTU value is 1500. + compoundPackets.size shouldBe 2 + } + } + + private val invalidRtcpData = byteArrayOf( + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + + private val rtcpByeNoReason = RtcpHeaderBuilder( + packetType = RtcpByePacket.PT, + reportCount = 1, + senderSsrc = 12345L, + length = 1 + ).build() +} + +private class DummyPacket( + buf: ByteArray +) : Packet(buf, 0, buf.size) { + override fun clone(): Packet = DummyPacket(cloneBuffer(0)) +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt new file mode 100644 index 0000000..73b5ea7 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpByePacketTest.kt @@ -0,0 +1,98 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContain +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.extensions.bytearray.byteArrayOf +import org.jitsi.test_helpers.matchers.haveSameContentAs +import java.nio.charset.StandardCharsets + +internal class RtcpByePacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val rtcpByeNoReason = RtcpHeaderBuilder( + packetType = RtcpByePacket.PT, + reportCount = 1, + senderSsrc = 12345L, + length = 1 + ).build() + + private val byeReason = "Connection terminated" + private val byeReasonData = byeReason.toByteArray(StandardCharsets.US_ASCII) + private val rtcpByeReasonData = byteArrayOf( + byeReasonData.size.toByte(), + *byeReasonData.toTypedArray() + ) + private val reasonSize = 1 + byeReasonData.size + private val padding = byteArrayOf(0x00, 0x00) + + private val rtcpByeWithReason = RtcpHeaderBuilder( + packetType = RtcpByePacket.PT, + hasPadding = true, + reportCount = 1, + senderSsrc = 12345L, + length = 1 + ((1 + reasonSize + 3) / 4) + ).build() + rtcpByeReasonData + padding + + init { + context("Creating an RtcpByePacket") { + context("from a buffer") { + context("of a packet without a reason") { + val packet = RtcpByePacket(rtcpByeNoReason, 0, rtcpByeNoReason.size) + should("parse the values correctly") { + packet.ssrcs shouldHaveSize 1 + packet.ssrcs shouldContain 12345L + packet.reason shouldBe null + } + } + context("of a packet with a reason") { + val packet = RtcpByePacket(rtcpByeWithReason, 0, rtcpByeWithReason.size) + should("parse the values correctly") { + packet.ssrcs shouldHaveSize 1 + packet.ssrcs shouldContain 12345L + packet.reason shouldBe byeReason + } + context("and then serializing it") { + context("via requesting a new buffer") { + val serializedBuf = packet.getBuffer() + should("write all values correctly") { + serializedBuf should haveSameContentAs(rtcpByeWithReason) + } + } +// context("to an existing buffer") { +// val existingBuf = ByteBuffer.allocate(8 + rtcpByeWithReason.limit()) +// existingBuf.position(8) +// packet.serializeTo(existingBuf) +// should("write the data to the proper place") { +// val subBuf = existingBuf.subBuffer(8, rtcpByeWithReason.limit()) +// subBuf should haveSameContentAs(rtcpByeWithReason) +// } +// should("leave the buffer's position after the field it just wrote") { +// existingBuf.position() shouldBe (8 + rtcpByeWithReason.limit()) +// } +// } + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpPacketTest.kt new file mode 100644 index 0000000..adc2317 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpPacketTest.kt @@ -0,0 +1,68 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.types.shouldBeInstanceOf +import org.jitsi.rtp.extensions.bytearray.byteArrayOf + +class RtcpPacketTest : ShouldSpec() { + + init { + context("Parsing") { + context("a valid but unsupported RTCP packet") { + val unsupportedRtcpData = byteArrayOf( + // V=2, PT=195, length = 2 + 0x80, 0xC3, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + should("return UnsupportedRtcpPacket") { + RtcpPacket.parse(unsupportedRtcpData, 0, unsupportedRtcpData.size) + .shouldBeInstanceOf() + } + } + context("an invalid RTCP packet") { + val invalidRtcpData = byteArrayOf( + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + should("throw an InvalidRtcpException") { + shouldThrow { + RtcpPacket.parse(invalidRtcpData, 0, invalidRtcpData.size) + } + } + } + context("a packet with invalid length") { + val invalidRtcpLengthData = byteArrayOf( + // V=2, PT=195, length = 8 + 0x80, 0xC3, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ) + should("throw an InvalidRtcpException") { + shouldThrow { + RtcpPacket.parse(invalidRtcpLengthData, 0, invalidRtcpLengthData.size) + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt new file mode 100644 index 0000000..6e91a46 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpReportBlockTest.kt @@ -0,0 +1,96 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.extensions.put3Bytes +import org.jitsi.rtp.extensions.unsigned.toPositiveLong +import org.jitsi.test_helpers.matchers.haveSameContentAs +import java.nio.ByteBuffer + +class RtcpReportBlockTest : ShouldSpec() { + private val expectedSsrc: Long = 12345 + private val expectedFractionLost: Int = 42 + private val expectedCumulativeLost: Int = 4242 + private val expectedSeqNumCycles: Int = 1 + private val expectedSeqNum: Int = 42 + private val expectedHighestSeqNum = ((expectedSeqNumCycles shl 16) + expectedSeqNum.toShort()).toPositiveLong() + private val expectedInterarrivalJitter: Long = 4242 + private val expectedLastSrTimestamp: Long = 23456 + private val expectedDelaySinceLastSr: Long = 34567 + + val reportBlockData = with(ByteBuffer.allocate(24)) { + putInt(expectedSsrc.toInt()) + put(expectedFractionLost.toByte()) + put3Bytes(expectedCumulativeLost) + + putShort(expectedSeqNumCycles.toShort()) + putShort(expectedSeqNum.toShort()) + + putInt(expectedInterarrivalJitter.toInt()) + putInt(expectedLastSrTimestamp.toInt()) + putInt(expectedDelaySinceLastSr.toInt()) + this.rewind() as ByteBuffer + } + + init { + context("creation") { + context("from a buffer") { + val reportBlock = RtcpReportBlock.fromBuffer(reportBlockData.array(), reportBlockData.arrayOffset()) + should("read the values correctly") { + reportBlock.ssrc shouldBe expectedSsrc + reportBlock.fractionLost shouldBe expectedFractionLost + reportBlock.cumulativePacketsLost shouldBe expectedCumulativeLost + reportBlock.seqNumCycles shouldBe expectedSeqNumCycles + reportBlock.seqNum shouldBe expectedSeqNum + reportBlock.interarrivalJitter shouldBe expectedInterarrivalJitter + reportBlock.lastSrTimestamp shouldBe expectedLastSrTimestamp + reportBlock.delaySinceLastSr shouldBe expectedDelaySinceLastSr + } + } + context("from values") { + val reportBlock = RtcpReportBlock( + ssrc = expectedSsrc, + fractionLost = expectedFractionLost, + cumulativePacketsLost = expectedCumulativeLost, + extendedHighestSeqNum = expectedHighestSeqNum, + interarrivalJitter = expectedInterarrivalJitter, + lastSrTimestamp = expectedLastSrTimestamp, + delaySinceLastSr = expectedDelaySinceLastSr + ) + should("set the values correctly") { + reportBlock.ssrc shouldBe expectedSsrc + reportBlock.fractionLost shouldBe expectedFractionLost + reportBlock.cumulativePacketsLost shouldBe expectedCumulativeLost + reportBlock.extendedHighestSeqNum shouldBe expectedHighestSeqNum + reportBlock.seqNumCycles shouldBe expectedSeqNumCycles + reportBlock.seqNum shouldBe expectedSeqNum + reportBlock.interarrivalJitter shouldBe expectedInterarrivalJitter + reportBlock.lastSrTimestamp shouldBe expectedLastSrTimestamp + reportBlock.delaySinceLastSr shouldBe expectedDelaySinceLastSr + } + should("write the data correctly") { + val buf = ByteArray(RtcpReportBlock.SIZE_BYTES) + reportBlock.writeTo(buf, 0) + buf should haveSameContentAs(reportBlockData.array()) + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacketTest.kt new file mode 100644 index 0000000..aba5dbf --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpRrPacketTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.haveSize +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import java.lang.IllegalArgumentException +import java.nio.ByteBuffer + +internal class RtcpRrPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + // Report blocks + val reportBlocks = mutableListOf( + RtcpReportBlock( + 12345, + 42, + 4242, + 1, + 42, + 4242, + 23456, + 34567 + ), + RtcpReportBlock( + 23456, + 43, + 4343, + 2, + 43, + 4343, + 34567, + 45678 + ) + ) + + val rrPacket = RtcpRrPacketBuilder( + reportBlocks = reportBlocks + ).build() + + val rrPacketBuffer = ByteBuffer.wrap(rrPacket.buffer, rrPacket.offset, rrPacket.length) + + // TODO: write expected buf (copy from below?) and verify serialization + + init { +// Header + 2 ReportBlocks +// val packetBuf = ByteBuffer.allocate(8 + 2 * 24) +// with (BitBuffer(packetBuf)) { +// // Header +// putBits(0x2, 2) // Version +// putBoolean(false) // Padding +// putBits(0x2, 5) // Report count +// packetBuf.put(RtcpRrPacket.PT.toByte()) // Payload type +// packetBuf.putShort(0xFFFF.toShort()) // length +// packetBuf.putInt(0xFFFFFFFF.toInt()) // sender ssrc +// +// packetBuf.putInt(ssrc1.toInt()) +// packetBuf.put(fractionLost1.toByte()); packetBuf.put3Bytes(cumulativeLost1) +// packetBuf.putShort(seqNumCycles1.toShort()); packetBuf.putShort(seqNum1.toShort()) +// packetBuf.putInt(interarrivalJitter1.toInt()) +// packetBuf.putInt(lastSrTimestamp1.toInt()) +// packetBuf.putInt(delaySinceLastSr1.toInt()) +// +// packetBuf.putInt(ssrc2.toInt()) +// packetBuf.put(fractionLost2.toByte()); packetBuf.put3Bytes(cumulativeLost2) +// packetBuf.putShort(seqNumCycles2.toShort()); packetBuf.putShort(seqNum2.toShort()) +// packetBuf.putInt(interarrivalJitter2.toInt()) +// packetBuf.putInt(lastSrTimestamp2.toInt()) +// packetBuf.putInt(delaySinceLastSr2.toInt()) +// packetBuf.rewind() as ByteBuffer +// } + context("creation") { + context("from a buffer") { + val rrPacket = + RtcpRrPacket(rrPacketBuffer.array(), rrPacketBuffer.arrayOffset(), rrPacketBuffer.limit()) + should("read all values correctly") { + rrPacket.packetType shouldBe RtcpRrPacket.PT + rrPacket.reportBlocks should haveSize(2) + } + } + } + + context("building") { + context("with too many report blocks") { + should("throw") { + shouldThrow { + RtcpRrPacketBuilder( + reportBlocks = generateSequence { reportBlocks[0] }.take(32).toList() + ).build() + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacketTest.kt new file mode 100644 index 0000000..7f4a971 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/RtcpSrPacketTest.kt @@ -0,0 +1,228 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.rtp.util.byteBufferOf +import java.lang.IllegalArgumentException +import java.nio.ByteBuffer + +internal class RtcpSrPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val expectedSenderInfo = SenderInfoBuilder( + ntpTimestampMsw = 0x7FFFFFFF, + ntpTimestampLsw = 0xFFFFFFFF, + rtpTimestamp = 0xFFFFFFFF, + sendersPacketCount = 0xFFFFFFFF, + sendersOctetCount = 0xFFFFFFFF + ) + + private val reportBlock1 = RtcpReportBlock( + ssrc = 12345, + fractionLost = 42, + cumulativePacketsLost = 4242, + seqNumCycles = 1, + seqNum = 42, + interarrivalJitter = 4242, + lastSrTimestamp = 23456, + delaySinceLastSr = 34567 + ) + private val reportBlock2 = RtcpReportBlock( + ssrc = 23456, + fractionLost = 42, + cumulativePacketsLost = 4242, + seqNumCycles = 1, + seqNum = 42, + interarrivalJitter = 4242, + lastSrTimestamp = 23456, + delaySinceLastSr = 34567 + ) + + private val srPacket = RtcpSrPacketBuilder( + RtcpHeaderBuilder( + senderSsrc = 12345 + ), + expectedSenderInfo, + mutableListOf(reportBlock1, reportBlock2) + ).build() + + val srPacketData = byteBufferOf( + // V=2,P=false,RC=0,PT=200,L=6(28 bytes) + 0x80, 0xc8, 0x00, 0x06, + // Sender SSRC: 1829790331 + 0x6d, 0x10, 0x62, 0x7b, + // Timestamp, MSW: 3761595357 + 0xe0, 0x35, 0x63, 0xdd, + // Timestamp, LSW: 17218523 + 0x01, 0x06, 0xbb, 0xdb, + // RTP timestamp: 3960153838 + 0xec, 0x0b, 0x26, 0xee, + // Sender's packet count: 72 + 0x00, 0x00, 0x00, 0x48, + // Sender's octet count: 76643 + 0x00, 0x01, 0x2b, 0x63 + ) + + val srPacketBuffer = ByteBuffer.wrap(srPacket.buffer, srPacket.offset, srPacket.length) + + init { + context("creation") { + context("from a buffer") { +// val srPacket = RtcpSrPacket(srPacketBuffer.array(), srPacketBuffer.arrayOffset(), srPacketBuffer.limit()) + val srPacket = RtcpSrPacket(srPacketData.array(), srPacketData.arrayOffset(), srPacketData.limit()) + should("read everything correctly") { + srPacket.version shouldBe 2 + srPacket.hasPadding shouldBe false + srPacket.reportCount shouldBe 0 + srPacket.reportBlocks shouldHaveSize 0 + srPacket.packetType shouldBe 200 + srPacket.lengthField shouldBe 6 + srPacket.senderSsrc shouldBe 1829790331 + srPacket.senderInfo.ntpTimestampMsw shouldBe 3761595357 + srPacket.senderInfo.ntpTimestampLsw shouldBe 17218523 + srPacket.senderInfo.compactedNtpTimestamp shouldBe 1675428102 + srPacket.senderInfo.rtpTimestamp shouldBe 3960153838 + srPacket.senderInfo.sendersPacketCount shouldBe 72 + srPacket.senderInfo.sendersOctetCount shouldBe 76643 + +// srPacket.packetType shouldBe RtcpSrPacket.PT +// srPacket.reportCount shouldBe 2 +// srPacket.lengthField shouldBe 18 +// srPacket.senderInfo.ntpTimestampMsw shouldBe expectedSenderInfo.ntpTimestampMsw +// srPacket.senderInfo.ntpTimestampLsw shouldBe expectedSenderInfo.ntpTimestampLsw +// srPacket.senderInfo.compactedNtpTimestamp shouldBe 0xFFFFFFFF.toPositiveLong() +// srPacket.senderInfo.rtpTimestamp shouldBe expectedSenderInfo.rtpTimestamp +// srPacket.senderInfo.sendersPacketCount shouldBe expectedSenderInfo.sendersPacketCount +// srPacket.senderInfo.sendersOctetCount shouldBe expectedSenderInfo.sendersOctetCount +// srPacket.reportBlocks should haveSize(2) + // TODO: verify report block parse +// srPacket.reportBlocks[0] shouldBe reportBlock1 +// srPacket.reportBlocks[1] shouldBe reportBlock2 + } + } + context("cloneWithoutReportBlocks") { + context("when there were report blocks") { + val clone = srPacket.cloneWithoutReportBlocks() + should("clone everything correctly") { + clone.version shouldBe srPacket.version + clone.hasPadding shouldBe srPacket.hasPadding + clone.reportCount shouldBe 0 + clone.packetType shouldBe srPacket.packetType + clone.length shouldBe RtcpHeader.SIZE_BYTES + SenderInfoParser.SIZE_BYTES + clone.senderSsrc shouldBe srPacket.senderSsrc + clone.buffer shouldNotBe srPacket.buffer + } + } + context("when there were no report blocks") { + val srPacketNoReportBlocks = RtcpSrPacketBuilder( + RtcpHeaderBuilder( + senderSsrc = 12345 + ), + expectedSenderInfo + ).build() + val clone = srPacketNoReportBlocks.cloneWithoutReportBlocks() + should("clone everything correctly") { + clone.version shouldBe srPacket.version + clone.hasPadding shouldBe srPacket.hasPadding + clone.reportCount shouldBe 0 + clone.packetType shouldBe srPacket.packetType + clone.length shouldBe RtcpHeader.SIZE_BYTES + SenderInfoParser.SIZE_BYTES + clone.senderSsrc shouldBe srPacket.senderSsrc + clone.buffer shouldNotBe srPacket.buffer + } + } + } +// context("from explicit values") { +// val srPacket = RtcpSrPacket( +// header = expectedHeader, +// senderInfo = expectedSenderInfo, +// reportBlocks = mutableListOf( +// reportBlock1, +// reportBlock2 +// ) +// ) +// should("set all values correctly") { +// srPacket.header shouldBe expectedHeader +// srPacket.senderInfo shouldBe expectedSenderInfo +// srPacket.reportBlocks should containAll(reportBlock1, reportBlock2) +// } +// } +// context("from an incomplete set of values") { +// val srPacket = RtcpSrPacket( +// header = RtcpHeader(reportCount = 2, senderSsrc = 12345), +// senderInfo = expectedSenderInfo, +// reportBlocks = mutableListOf( +// reportBlock1, +// reportBlock2 +// ) +// ) +// val parsedPacket = RtcpSrPacket.fromBuffer(srPacket.getBuffer()) +// should("set all values correctly") { +// RtcpHeaderTest.rtcpHeaderEquals(parsedPacket.header, expectedHeader) +// parsedPacket.senderInfo shouldBe expectedSenderInfo +// parsedPacket.reportBlocks should containAll(reportBlock1, reportBlock2) +// } +// } + } +// context("serialization") { +// val expectedBuf = ByteBuffer.allocate(1024) +// expectedBuf.put(expectedHeader.getBuffer()) +// expectedBuf.put(expectedSenderInfo.getBuffer()) +// expectedBuf.put(reportBlock1.getBuffer()) +// expectedBuf.put(reportBlock2.getBuffer()) +// expectedBuf.flip() +// val srPacket = RtcpSrPacket.fromBuffer(expectedBuf) +// +// context("via getting its buffer") { +// val actualBuf = srPacket.getBuffer() +// should("write all values correctly") { +// actualBuf should haveSameContentAs(expectedBuf) +// } +// } +// context("to an existing buffer") { +// val existingBuf = ByteBuffer.allocate(1024) +// existingBuf.position(8) +// srPacket.serializeTo(existingBuf) +// should("write the data to the proper place") { +// val subBuf = existingBuf.subBuffer(8, expectedBuf.limit()) +// subBuf should haveSameContentAs(expectedBuf) +// } +// should("leave the buffer's position after the field it just wrote") { +// existingBuf.position() shouldBe (8 + expectedBuf.limit()) +// } +// } +// } + + context("building") { + context("with too many report blocks") { + should("throw") { + shouldThrow { + RtcpSrPacketBuilder( + reportBlocks = generateSequence { reportBlock1 }.take(32).toList() + ).build() + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacketTest.kt new file mode 100644 index 0000000..298c993 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/payload_specific_fb/RtcpFbRembPacketTest.kt @@ -0,0 +1,141 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtcp.rtcpfb.payload_specific_fb + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.util.byteBufferOf + +internal class RtcpFbRembPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + val rembPacket1Buf = byteBufferOf( + 0x8f, 0xce, 0x00, 0x05, + 0xfe, 0x93, 0x23, 0x03, + 0x00, 0x00, 0x00, 0x00, + 0x52, 0x45, 0x4d, 0x42, + 0x01, 0x0b, 0xa4, 0x5a, + 0x9e, 0x72, 0x01, 0xc1 + ) + + val rembPacket2Buf = byteBufferOf( + 0x8f, 0xce, 0x00, 0x06, + 0xfe, 0x93, 0x23, 0x03, + 0x00, 0x00, 0x00, 0x00, + 0x52, 0x45, 0x4d, 0x42, + 0x02, 0x0b, 0xb4, 0x5a, + 0x9e, 0x72, 0x01, 0xc1, + 0x9e, 0x23, 0x54, 0xa3 + ) + + val rembPacket3Buf = byteBufferOf( + 0x8f, 0xce, 0x00, 0x06, + 0xfe, 0x93, 0x23, 0x03, + 0x00, 0x00, 0x00, 0x00, + 0x52, 0x45, 0x4d, 0x42, + 0x02, 0xff, 0xff, 0xff, + 0x9e, 0x72, 0x01, 0xc1, + 0x9e, 0x23, 0x54, 0xa3 + ) + + init { + context("Creating an RtcpFbRembPacket") { + context("from normal values") { + val rembPacket = RtcpFbRembPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + senderSsrc = 4567L + ), + ssrcs = listOf(1234L), + brBps = 1_000_000L + ).build() + + should("set the values correctly") { + rembPacket.senderSsrc shouldBe 4567L + rembPacket.ssrcs shouldBe listOf(1234L) + rembPacket.bitrate shouldBe 1_000_000L + } + } + context("from values which overflow the field size by a lot") { + val rembPacket = RtcpFbRembPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + senderSsrc = 4567L + ), + ssrcs = listOf(1234L), + brBps = Long.MAX_VALUE + ).build() + + should("set the values correctly") { + rembPacket.senderSsrc shouldBe 4567L + rembPacket.ssrcs shouldBe listOf(1234L) + // the difference between Long.MAX_VALUE and 9223336852482686976L is expected and stems from the + // fact that we (only) have 18 bits of resolution for the mantissa. + rembPacket.bitrate shouldBe 9223336852482686976L + } + } + context("from values which overflow the field size by a little") { + val rembPacket = RtcpFbRembPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + senderSsrc = 4567L + ), + ssrcs = listOf(1234L), + brBps = 50_000_000L + ).build() + + should("set the values correctly") { + rembPacket.senderSsrc shouldBe 4567L + rembPacket.ssrcs shouldBe listOf(1234L) + // the difference between 50_000_000L and 49999872L is expected and it stems from the fact that we + // (only) have 18 bits of resolution for the mantissa. + rembPacket.bitrate shouldBe 49999872L + } + } + context("creation") { + context("from a buffer of a simple REMB") { + val rembPacket = + RtcpFbRembPacket(rembPacket1Buf.array(), rembPacket1Buf.arrayOffset(), rembPacket1Buf.limit()) + should("read everything correctly") { + rembPacket.senderSsrc shouldBe 0xfe932303 + rembPacket.numSsrc shouldBe 1 + rembPacket.ssrcs shouldBe listOf(0x9e7201c1) + rembPacket.bitrate shouldBe 954728 + } + } + context("from a buffer of a REMB with two ssrcs") { + val rembPacket = + RtcpFbRembPacket(rembPacket2Buf.array(), rembPacket2Buf.arrayOffset(), rembPacket2Buf.limit()) + should("read everything correctly") { + rembPacket.senderSsrc shouldBe 0xfe932303 + rembPacket.numSsrc shouldBe 2 + rembPacket.ssrcs shouldBe listOf(0x9e7201c1, 0x9e2354a3) + rembPacket.bitrate shouldBe 971112 + } + } + context("from a buffer of a REMB signaling unbound bandwidth") { + val rembPacket = + RtcpFbRembPacket(rembPacket3Buf.array(), rembPacket3Buf.arrayOffset(), rembPacket3Buf.limit()) + should("read everything correctly") { + rembPacket.senderSsrc shouldBe 0xfe932303 + rembPacket.numSsrc shouldBe 2 + rembPacket.ssrcs shouldBe listOf(0x9e7201c1, 0x9e2354a3) + rembPacket.bitrate shouldBe Long.MAX_VALUE + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/RtcpFbNackPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/RtcpFbNackPacketTest.kt new file mode 100644 index 0000000..1c8d561 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/RtcpFbNackPacketTest.kt @@ -0,0 +1,143 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.util.byteBufferOf +import org.jitsi.test_helpers.matchers.haveSameContentAs + +internal class RtcpFbNackPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + +// private val samplePacketId = 0 +// private val sampleBlp = GenericNackBlp(listOf(1, 3, 5, 7, 9, 11, 13, 15)) +// private val sampleMediaSsrc = 12345L +// private val sampleHeader = +// RtcpHeader(length = 3, packetType = TransportLayerFbPacket.PT, +// reportCount = RtcpFbNackPacket.FMT) +// val sampleRtcpFbNackPacketBuf = ByteBuffer.allocate(16).apply { +// put(sampleHeader.getBuffer()) +// putInt(sampleMediaSsrc.toInt()) +// putShort(samplePacketId.toShort()) +// put(sampleBlp.getBuffer()) +// }.flip() as ByteBuffer + + val multipleNackBlocksBuf = byteBufferOf( + 0x81, 0xCD, 0x00, 0x04, + 0xA2, 0x78, 0xEB, 0x18, + 0x84, 0x50, 0x49, 0x6C, + // packet id = 44832, other nacked packets = 44838, 44848 + 0xAF, 0x20, 0x80, 0x20, + // packet id = 44850, other nacked packets = 44854, 44856 + 0xAF, 0x32, 0x00, 0x28 + ) + + val wrappingNackBlockBuf = byteBufferOf( + 0x81, 0xCD, 0x00, 0x03, + 0xA2, 0x78, 0xEB, 0x18, + 0x84, 0x50, 0x49, 0x6C, + // packet id = 65530, other nacked packets = 65534, 0, 7 + 0xFF, 0xFA, 0x10, 0x28, + ) + + init { + context("Creating an RtcpFbNackPacket") { +// context("from a buffer") { +// val nackPacket = RtcpFbNackPacket. +// should("parse the values correctly") { +// nackPacket.mediaSourceSsrc shouldBe sampleMediaSsrc +// nackPacket.missingSeqNums.shouldContainInOrder(1, 3, 5, 7, 9, 11, 13, 15) +// } +// should("leave the buffer's position after the parsed data") { +// sampleRtcpFbNackPacketBuf.position() shouldBe sampleRtcpFbNackPacketBuf.limit() +// } +// } + context("from a buffer with multiple NACK blocks") { + val nackPacket = RtcpFbNackPacket( + multipleNackBlocksBuf.array(), + multipleNackBlocksBuf.arrayOffset(), + multipleNackBlocksBuf.limit() + ) + should("parse the values correctly") { + nackPacket.missingSeqNums shouldContainExactly sortedSetOf(44832, 44838, 44848, 44850, 44854, 44856) + } + } + context("from a buffer with a wrapping NACK block") { + val nackPacket = RtcpFbNackPacket( + wrappingNackBlockBuf.array(), + wrappingNackBlockBuf.arrayOffset(), + wrappingNackBlockBuf.limit() + ) + should("parse the values correctly") { + nackPacket.missingSeqNums shouldContainExactly sortedSetOf(65530, 65534, 0, 7) + } + } + context("from values") { + val nackPacket = RtcpFbNackPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + senderSsrc = 4567L + ), + mediaSourceSsrc = 12345L, + missingSeqNums = (0..15 step 2).toSortedSet() + ).build() + should("set the values correctly") { + nackPacket.senderSsrc shouldBe 4567L + nackPacket.mediaSourceSsrc shouldBe 12345L + nackPacket.missingSeqNums.shouldContainExactly((0..15 step 2).toSortedSet()) + } +// context("and then serializing it") { +// context("by requesting a buffer") { +// val buf = nackPacket.getBuffer() +// should("serialize the data correctly") { +// buf should haveSameContentAs(sampleRtcpFbNackPacketBuf) +// } +// } +// context("to an existing buffer") { +// val existingBuf = ByteBuffer.allocate(RtcpFbNackPacket.SIZE_BYTES + 8) +// existingBuf.position(8) +// nackPacket.serializeTo(existingBuf) +// should("write the data to the correct place") { +// existingBuf.subBuffer(8) should haveSameContentAs(sampleRtcpFbNackPacketBuf) +// } +// should("leave the buffer's position after the written data") { +// existingBuf.position() shouldBe existingBuf.limit() +// } +// } +// } + } + context("from values too big to fit into a single NACK block") { + val rtcpFbNackPacket = + RtcpFbNackPacketBuilder( + rtcpHeader = RtcpHeaderBuilder(senderSsrc = 2725833496), + mediaSourceSsrc = 2219854188, + missingSeqNums = sortedSetOf(44832, 44838, 44848, 44850, 44854, 44856) + ).build() + context("and then serializing it") { + val buf = rtcpFbNackPacket.getBuffer() + should("serialize into multiple nack blocks") { + buf should haveSameContentAs(multipleNackBlocksBuf.array()) + } + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/ccfb/RtcpFbCcfbPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/ccfb/RtcpFbCcfbPacketTest.kt new file mode 100644 index 0000000..b5daa43 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/ccfb/RtcpFbCcfbPacketTest.kt @@ -0,0 +1,170 @@ +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.ccfb + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import io.kotest.matchers.collections.shouldMatchEach +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.utils.abs +import org.jitsi.utils.div +import org.jitsi.utils.isFinite +import org.jitsi.utils.isInfinite +import org.jitsi.utils.ms +import org.jitsi.utils.secs + +class RtcpFbCcfbPacketTest : ShouldSpec() { + init { + context("An empty CCFB") { + val builder = RtcpFbCcfbPacketBuilder(RtcpHeaderBuilder(), 0xdeadbeef, listOf(), 100_000) + val packet = builder.build() + packet.packetLength shouldBe + 4 + // RTCP header + 4 + // common FB header + 4 + // sender ssrc + 4 // timestamp + packet.mediaSourceSsrc shouldBe 0xdeadbeef + packet.reportTimestampCompactNtp shouldBe 100_000 + packet.packets shouldBe listOf() + } + + context("A CCFB with two ssrcs, one packet each") { + val packets = listOf( + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 1), + ReceivedPacketInfo(ssrc = 2, sequenceNumber = 1) + ) + val builder = RtcpFbCcfbPacketBuilder(RtcpHeaderBuilder(), 1, packets, 1) + val packet = builder.build() + packet.packetLength shouldBe + 4 + // RTCP header + 4 + // common FB header + 4 + // sender ssrc + 2 * ( + 8 + // per ssrc header + 1 * 2 + // packet info + 2 // padding + ) + + 4 // timestamp + packet.mediaSourceSsrc shouldBe 1 + packet.reportTimestampCompactNtp shouldBe 1 + packet.packets.shouldMatch(packets) + } + + context("A CCFB with two ssrcs, two packets each") { + val packets = listOf( + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 1), + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 2), + ReceivedPacketInfo(ssrc = 2, sequenceNumber = 1), + ReceivedPacketInfo(ssrc = 2, sequenceNumber = 2) + + ) + val builder = RtcpFbCcfbPacketBuilder(RtcpHeaderBuilder(), 1, packets, 1) + val packet = builder.build() + packet.packetLength shouldBe + 4 + // RTCP header + 4 + // common FB header + 4 + // sender ssrc + 2 * ( + 8 + // per ssrc header + 2 * 2 // packet info + ) + + 4 // timestamp + packet.mediaSourceSsrc shouldBe 1 + packet.reportTimestampCompactNtp shouldBe 1 + packet.packets.shouldMatch(packets) + } + + context("A CCFB with missing packets") { + val packets = listOf( + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 1, arrivalTimeOffset = 1.ms), + UnreceivedPacketInfo(ssrc = 1, sequenceNumber = 2), + UnreceivedPacketInfo(ssrc = 1, sequenceNumber = 3), + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 4, arrivalTimeOffset = 2.ms) + ) + val builder = RtcpFbCcfbPacketBuilder(RtcpHeaderBuilder(), 1, packets, 1) + val packet = builder.build() + packet.packetLength shouldBe + 4 + // RTCP header + 4 + // common FB header + 4 + // sender ssrc + 1 * ( + 8 + // per ssrc header + 4 * 2 // packet info + ) + + 4 // timestamp + packet.mediaSourceSsrc shouldBe 1 + packet.reportTimestampCompactNtp shouldBe 1 + packet.packets.shouldMatch(packets) + } + + context("A CCFB with ECN marks") { + val packets = listOf( + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 1, arrivalTimeOffset = 1.ms, ecn = EcnMarking.kEct1), + UnreceivedPacketInfo(ssrc = 1, sequenceNumber = 2), + UnreceivedPacketInfo(ssrc = 1, sequenceNumber = 3), + ReceivedPacketInfo(ssrc = 1, sequenceNumber = 4, arrivalTimeOffset = 2.ms, ecn = EcnMarking.kCe) + ) + val builder = RtcpFbCcfbPacketBuilder(RtcpHeaderBuilder(), 1, packets, 1) + val packet = builder.build() + packet.packetLength shouldBe + 4 + // RTCP header + 4 + // common FB header + 4 + // sender ssrc + 1 * ( + 8 + // per ssrc header + 4 * 2 // packet info + ) + + 4 // timestamp + packet.mediaSourceSsrc shouldBe 1 + packet.reportTimestampCompactNtp shouldBe 1 + packet.packets.shouldMatch(packets) + } + } +} + +class PacketInfoMatcher(private val expected: PacketInfo) : Matcher { + override fun test(value: PacketInfo): MatcherResult = when (expected) { + is ReceivedPacketInfo -> if (value !is ReceivedPacketInfo) { + MatcherResult( + false, + { "Value should be $expected but was $value" }, + { "Value should not be $expected" } + ) + } else { + // PacketInfo is equal after serializing-deserializing if members are equal + // except for arrival time offset that may differ because of conversion back and + // forth to CompactNtp. + + val arrivalTimeOffsetEqual = + ( + value.arrivalTimeOffset.isInfinite() && expected.arrivalTimeOffset.isInfinite() && + value == expected + ) || + ( + value.arrivalTimeOffset.isFinite() && expected.arrivalTimeOffset.isFinite() && + abs(value.arrivalTimeOffset - expected.arrivalTimeOffset) < 1.secs / 1024 + ) + MatcherResult( + value.ssrc == expected.ssrc && + value.sequenceNumber == expected.sequenceNumber && + arrivalTimeOffsetEqual && + value.ecn == expected.ecn, + { "Value should be $expected but was $value" }, + { "Value should not be $expected" } + ) + } + + is UnreceivedPacketInfo -> MatcherResult( + value is UnreceivedPacketInfo && + value.ssrc == expected.ssrc && + value.sequenceNumber == expected.sequenceNumber, + { "Value should be $expected but was $value" }, + { "Value should not be $expected" } + ) + } +} + +fun match(expected: PacketInfo) = PacketInfoMatcher(expected) + +fun List.shouldMatch(expected: List) = + this.shouldMatchEach(expected.map { { p -> match(it).test(p) } }) diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt new file mode 100644 index 0000000..12146ee --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtcp/rtcpfb/transport_layer_fb/tcc/RtcpFbTccPacketTest.kt @@ -0,0 +1,211 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtcp.rtcpfb.transport_layer_fb.tcc + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.maps.shouldContainKey +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.types.beInstanceOf +import org.jitsi.rtp.rtcp.RtcpHeaderBuilder +import org.jitsi.rtp.util.byteBufferOf +import org.jitsi.utils.instantOfEpochMicro +import org.jitsi.utils.micros +import java.time.Duration + +class RtcpFbTccPacketTest : ShouldSpec() { + fun Int.toTicks(): Short = (this * 4).toShort() + private val tccRleData = byteBufferOf( + // V=2,P=false,FMT=15,PT=205,L=7(32 bytes) + 0x8f, 0xcd, 0x00, 0x07, + // Sender SSRC = 839852602 + 0x32, 0x0f, 0x22, 0x3a, + // Media source SSRC = 2397376430 + 0x8e, 0xe5, 0x0f, 0xae, + // Base seq num = 0xfffa, packet status count = 9 + 0xff, 0xfa, 0x00, 0x09, + // Reference Time: 1683633 = 107752512ms, feedback packet count = 87 + 0x19, 0xb0, 0xb1, 0x57, + // Chunks + // RLE, small delta, length = 9 + 0x20, 0x09, + // Deltas (9), one byte each + 0xd8, 0x00, + 0x18, 0x14, 0x18, 0x14, + 0x18, 0x14, 0x18, + // Recv delta padding + 0x00 + ) + + /** + * These correspond to the Deltas section above. + */ + val expectedTccRlePacketInfo = mapOf( + 0xfffa to 0xd8, + 0xfffb to 0x00, + 0xfffc to 0x18, + 0xfffd to 0x14, + 0xfffe to 0x18, + 0xffff to 0x14, + 0x0000 to 0x18, + 0x0001 to 0x14, + 0x0002 to 0x18 + ) + + // This also has a negative delta + private val tccMixedChunkTypeData = byteBufferOf( + // V=2,P=false,FMT=15,PT=205,L=9(40 bytes) + 0x8f, 0xcd, 0x00, 0x09, + // Sender SSRC = 839852602 + 0x32, 0x0f, 0x22, 0x3a, + // Media source SSRC = 2397376430 + 0x8e, 0xe5, 0x0f, 0xae, + // Base seq num = 5376, packet status count = 12 + 0x15, 0x00, 0x00, 0x0c, + // Reference Time: 1684065 = 107780160ms, feedback packet count = 88 + 0x19, 0xb2, 0x61, 0x58, + // Chunks + // RLE: small delta, length = 9 + 0x20, 0x09, + // SV, 2 bit symbols: LD, SD, SD + 0xe5, 0x00, + // Deltas (12) + // 2, 0, 0, 0 + 0x08, 0x00, 0x00, 0x00, + // 22, 1, 0, 0 + 0x58, 0x04, 0x00, 0x00, + // 8, -1, 1 + 0x20, 0xff, 0xfc, 0x04, + // 0 + 0x00, + // Recv delta padding + 0x00, 0x00, 0x00 + ) + val expectedTccMixedChunkTypePacketInfo = mapOf( + 5376 to 2.toTicks(), + 5377 to 0.toTicks(), + 5378 to 0.toTicks(), + 5379 to 0.toTicks(), + 5380 to 22.toTicks(), + 5381 to 1.toTicks(), + 5382 to 0.toTicks(), + 5383 to 0.toTicks(), + 5384 to 8.toTicks(), + 5385 to (-1).toTicks(), + 5386 to 1.toTicks(), + 5387 to 0.toTicks() + ) + + private val tccSvChunkData = byteBufferOf( + // V=2,P=false,FMT=15,PT=205,length=5(24 bytes) + 0x8f, 0xcd, 0x00, 0x05, + // Sender SSRC: 839852602 + 0x32, 0x0f, 0x22, 0x3a, + // Media source SSRC: 2397376430 + 0x8e, 0xe5, 0x0f, 0xae, + // Base seq num = 6227, packet status count = 2 + 0x18, 0x53, 0x00, 0x02, + // Reference Time: 1684126 (107784064ms), feedback packet count = 162 + 0x19, 0xb2, 0x9e, 0xa2, + // Chunks + // SV chunk, 2 bit symbols: NR, SD + 0xc4, 0x00, + // Deltas (1) + // 00 + 0x00, + // Recv delta padding + 0x00 + ) + val expectedTccSvChunkPacketInfo = mapOf( + 6227 to -1, + 6228 to 107784064 + 27 + ) + + init { + context("Parsing an RtcpFbTccPacket") { + context("with RLE") { + val rtcpFbTccPacket = RtcpFbTccPacket(tccRleData.array(), tccRleData.arrayOffset(), tccRleData.limit()) + should("parse the values correctly") { + rtcpFbTccPacket.forEach { + it should beInstanceOf() + it as ReceivedPacketReport + expectedTccRlePacketInfo shouldContainKey it.seqNum + withClue("seqNum ${it.seqNum} deltaTicks") { + it.deltaTicks shouldBe expectedTccRlePacketInfo[it.seqNum] + it.deltaDuration shouldBe Duration.ofNanos(it.deltaTicks * 250 * 1000L) + } + } + } + } + context("with mixed chunk types and a negative delta") { + val rtcpFbTccPacket = RtcpFbTccPacket( + tccMixedChunkTypeData.array(), + tccMixedChunkTypeData.arrayOffset(), + tccMixedChunkTypeData.limit() + ) + should("parse the values correctly") { + rtcpFbTccPacket.forEach { + it should beInstanceOf() + it as ReceivedPacketReport + expectedTccMixedChunkTypePacketInfo shouldContainKey it.seqNum + withClue("seqNum ${it.seqNum} deltaTicks") { + it.deltaTicks shouldBe expectedTccMixedChunkTypePacketInfo[it.seqNum] + it.deltaDuration shouldBe Duration.ofNanos(it.deltaTicks * 250 * 1000L) + } + } + } + } + } + context("Creating an RtcpFbTccPacket") { + val rtcpFbTccPacketBuilder = RtcpFbTccPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + senderSsrc = 839852602 + ), + mediaSourceSsrc = 2397376430, + feedbackPacketSeqNum = 162 + ) + rtcpFbTccPacketBuilder.SetBase(6227, instantOfEpochMicro(107784064)) + rtcpFbTccPacketBuilder.AddReceivedPacket(6228, instantOfEpochMicro(107784064)) shouldBe true + } + context("Creating and parsing an RtcpFbTccPacket") { + context("with missing packets") { + val kBaseSeqNo = 1000 + val kBaseTimestamp = instantOfEpochMicro(10000L) + val rtcpFbTccPacketBuilder = RtcpFbTccPacketBuilder( + rtcpHeader = RtcpHeaderBuilder( + senderSsrc = 839852602 + ), + mediaSourceSsrc = 2397376430, + feedbackPacketSeqNum = 163 + ) + rtcpFbTccPacketBuilder.SetBase(kBaseSeqNo, kBaseTimestamp) + rtcpFbTccPacketBuilder.AddReceivedPacket(kBaseSeqNo + 0, kBaseTimestamp) + rtcpFbTccPacketBuilder.AddReceivedPacket(kBaseSeqNo + 3, kBaseTimestamp + 2000.micros) + + val coded = rtcpFbTccPacketBuilder.build() + + val packet = RtcpFbTccPacket(coded.buffer, coded.offset, coded.length) + val it = packet.iterator() + it.next() should beInstanceOf() + it.next() should beInstanceOf() + it.next() should beInstanceOf() + it.next() should beInstanceOf() + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RedRtpPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RedRtpPacketTest.kt new file mode 100644 index 0000000..268957b --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RedRtpPacketTest.kt @@ -0,0 +1,511 @@ +package org.jitsi.rtp.rtp + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.shouldBeTypeOf +import org.jitsi.rtp.Packet +import org.jitsi.rtp.extensions.bytearray.byteArrayOf +import org.jitsi.test_helpers.matchers.haveSameFixedHeader +import org.jitsi.test_helpers.matchers.haveSamePayload + +class RedRtpPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + context("Parsing block headers") { + context("Primary") { + val block = BlockHeader.parse(byteArrayOf(0x05), 0) + block.shouldBeTypeOf() + block.pt shouldBe 5.toByte() + } + context("Redundancy") { + val block = BlockHeader.parse(byteArrayOf(0x85, 0xff, 0xff, 0xff), 0) + block.shouldBeTypeOf() + block.pt shouldBe 5.toByte() + block.timestampOffset shouldBe 0x3fff + block.length shouldBe 0x3ff + } + } + context("Creating block headers") { + context("Primary") { + val buf = ByteArray(1) + val blockHeader = PrimaryBlockHeader(111) + blockHeader.write(buf, 0) + + val parsed = BlockHeader.parse(buf, 0) + parsed.shouldBeTypeOf() + parsed.pt shouldBe 111.toByte() + } + context("Redundancy") { + val buf = ByteArray(4) + val blockHeader = RedundancyBlockHeader(111, 960, 55) + blockHeader.write(buf, 0) + + val parsed = BlockHeader.parse(buf, 0) + parsed.shouldBeTypeOf() + parsed.pt shouldBe 111.toByte() + parsed.timestampOffset shouldBe 960 + parsed.length shouldBe 55 + } + context("Redundancy with large timestamp diff") { + shouldThrow { + RedundancyBlockHeader(111, RedundancyBlockHeader.MAX_TIMESTAMP_OFFSET + 1, 55) + } + } + } + context("Parsing a RED packet with a single block") { + val packet = RtpRedPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size) + packet.payloadType shouldBe 112 + packet.sequenceNumber shouldBe 0x7b2b + packet.timestamp shouldBe 0x44b2836e + packet.ssrc shouldBe 0x16493f2d + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 66 + + context("Without reading the redundancy blocks") { + packet.decapsulate(parseRedundancy = false) + packet.payloadType shouldBe 111 + packet.sequenceNumber shouldBe 0x7b2b + packet.timestamp shouldBe 0x44b2836e + packet.ssrc shouldBe 0x16493f2d + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 65 + } + context("And reading the redundancy blocks") { + val redundantPackets = packet.decapsulate(parseRedundancy = true) + redundantPackets.shouldBeEmpty() + packet.payloadType shouldBe 111 + packet.sequenceNumber shouldBe 0x7b2b + packet.timestamp shouldBe 0x44b2836e + packet.ssrc shouldBe 0x16493f2d + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 65 + } + } + context("Parsing a RED packet with multiple blocks") { + val packet = RtpRedPacket(redPacketBytes.clone(), 0, redPacketBytes.size) + packet.payloadType shouldBe 112 + packet.sequenceNumber shouldBe 0x7b2b + packet.timestamp shouldBe 0x44b2836e + packet.ssrc shouldBe 0x16493f2d + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 4 + 4 + 1 + 65 + 68 + 62 // block headers + block lengths + + context("Without reading the redundancy blocks") { + packet.decapsulate(parseRedundancy = false) + packet.payloadType shouldBe 111 + packet.sequenceNumber shouldBe 0x7b2b + packet.timestamp shouldBe 0x44b2836e + packet.ssrc shouldBe 0x16493f2d + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 62 + } + context("And reading the redundancy block") { + // Make sure we test the case when the sequence numbers wrap + packet.sequenceNumber = 1 + + val redundancyPackets = packet.decapsulate(parseRedundancy = true) + redundancyPackets.size shouldBe 2 + + redundancyPackets.forEachIndexed { i, p -> + // Offset relative to the sequence number of the primary. The first redundancy packet has + // offset=-2, the second has previous has offset=-1 + val offset = i - 2 + + p.payloadType shouldBe 111 + p.sequenceNumber shouldBe ((1 + offset) and 0xffff) + p.timestamp shouldBe (0x44b2836e + offset * 960) + p.ssrc shouldBe packet.ssrc + p.hasExtensions shouldBe false + } + + packet.payloadType shouldBe 111 + packet.sequenceNumber shouldBe 1 + packet.timestamp shouldBe 0x44b2836e + packet.ssrc shouldBe 0x16493f2d + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 62 + } + } + context("Parsing a RED packet with short blocks") { + val buffer = ByteArray( + redPacketBytesShortBlocks.size + + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET + ) + redPacketBytesShortBlocks.copyInto(buffer, RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET) + val packet = RtpRedPacket( + buffer, + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + redPacketBytesShortBlocks.size + ) + packet.payloadType shouldBe 112 + packet.sequenceNumber shouldBe 35006 + packet.timestamp shouldBe 862774129L + packet.ssrc shouldBe 624250097L + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 4 + 1 + 3 + 3 // block headers + block lengths + + context("Without reading the redundancy blocks") { + packet.decapsulate(parseRedundancy = false) + packet.payloadType shouldBe 111 + packet.sequenceNumber shouldBe 35006 + packet.timestamp shouldBe 862774129L + packet.ssrc shouldBe 624250097L + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 3 + } + context("And reading the redundancy block") { + val redundancyPackets = packet.decapsulate(parseRedundancy = true) + redundancyPackets.size shouldBe 1 + + redundancyPackets[0].payloadType shouldBe 111 + redundancyPackets[0].sequenceNumber shouldBe 35005 + redundancyPackets[0].timestamp shouldBe 862774129L - 960L + redundancyPackets[0].ssrc shouldBe 624250097L + redundancyPackets[0].hasExtensions shouldBe false + redundancyPackets[0].payloadLength shouldBe 3 + + packet.payloadType shouldBe 111 + packet.sequenceNumber shouldBe 35006 + packet.timestamp shouldBe 862774129L + packet.ssrc shouldBe 624250097L + packet.hasExtensions shouldBe true + packet.getHeaderExtension(1).shouldHaveId1AndLen1() + packet.payloadLength shouldBe 3 + } + } + context("Parsing a RED packet with zero-length blocks") { + val packet = RtpRedPacket( + redPacketBytesZeroLengthBlocks.clone(), + 0, + redPacketBytesZeroLengthBlocks.size + ) + packet.payloadType shouldBe 112 + packet.sequenceNumber shouldBe 35006 + packet.timestamp shouldBe 862774129L + packet.ssrc shouldBe 624250097L + packet.hasExtensions shouldBe false + packet.payloadLength shouldBe 4 + 1 // block headers + + context("Without reading the redundancy blocks") { + packet.decapsulate(parseRedundancy = false) + packet.payloadType shouldBe 0 + packet.sequenceNumber shouldBe 35006 + packet.timestamp shouldBe 862774129L + packet.ssrc shouldBe 624250097L + packet.hasExtensions shouldBe false + packet.payloadLength shouldBe 0 + } + context("And reading the redundancy block") { + val redundancyPackets = packet.decapsulate(parseRedundancy = true) + redundancyPackets.size shouldBe 1 + + redundancyPackets[0].payloadType shouldBe 0 + redundancyPackets[0].sequenceNumber shouldBe 35005 + redundancyPackets[0].timestamp shouldBe 862774129L + redundancyPackets[0].ssrc shouldBe 624250097L + redundancyPackets[0].hasExtensions shouldBe false + redundancyPackets[0].payloadLength shouldBe 0 + + packet.payloadType shouldBe 0 + packet.sequenceNumber shouldBe 35006 + packet.timestamp shouldBe 862774129L + packet.ssrc shouldBe 624250097L + packet.hasExtensions shouldBe false + packet.payloadLength shouldBe 0 + } + } + context("Parsing a truncated RED packet") { + context("Without buffer padding") { + val packet = RtpRedPacket( + redPacketBytesZeroLengthBlocks.clone(), + 0, + redPacketBytesZeroLengthBlocks.size - 1 + ) + + should("throw IllegalArgumentException") { + shouldThrow { + packet.decapsulate(parseRedundancy = false) + } + shouldThrow { + packet.decapsulate(parseRedundancy = true) + } + } + } + context("With buffer padding") { + val buffer = ByteArray( + redPacketBytesZeroLengthBlocks.size - 1 + + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET + + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET + ) + redPacketBytesZeroLengthBlocks.copyInto( + buffer, + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + 0, + redPacketBytesZeroLengthBlocks.size - 1 + ) + val packet = RtpRedPacket( + buffer, + RtpPacket.BYTES_TO_LEAVE_AT_START_OF_PACKET, + redPacketBytesZeroLengthBlocks.size - 1 + ) + + should("throw IllegalArgumentException") { + shouldThrow { + packet.decapsulate(parseRedundancy = false) + } + shouldThrow { + packet.decapsulate(parseRedundancy = true) + } + } + } + context("With a short buffer") { + val buffer = ByteArray(redPacketBytesZeroLengthBlocks.size - 1) + redPacketBytesZeroLengthBlocks.copyInto(buffer, 0, 0, redPacketBytesZeroLengthBlocks.size - 1) + val packet = RtpRedPacket(buffer, 0, redPacketBytesZeroLengthBlocks.size - 1) + + should("throw IllegalArgumentException") { + shouldThrow { + packet.decapsulate(parseRedundancy = false) + } + shouldThrow { + packet.decapsulate(parseRedundancy = true) + } + } + } + } + context("Creating a RED packet without redundancy") { + val rtpPacket = RtpPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size).apply { + payloadType = 111 + } + + val redPacket = RtpRedPacket.builder.build(112, rtpPacket, emptyList()) + redPacket.length shouldBe rtpPacket.length + 1 + redPacket.payloadType shouldBe 112 + + redPacket.decapsulate(parseRedundancy = false) + val reconstructed = RtpPacket(redPacket.buffer, redPacket.offset, redPacket.length) + reconstructed should haveSameFixedHeader(rtpPacket) + reconstructed should haveSamePayload(rtpPacket) + } + context("Creating a RED packet with redundancy") { + val primary = RtpPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size).apply { + payloadType = 111 + } + val redundancy1 = RtpPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size).apply { + payloadType = 111 + timestamp = primary.timestamp - 1920 + sequenceNumber = primary.sequenceNumber - 2 + } + val redundancy2 = RtpPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size).apply { + payloadType = 111 + timestamp = primary.timestamp - 960 + sequenceNumber = primary.sequenceNumber - 1 + } + + val redPacket = RtpRedPacket.builder.build(112, primary, listOf(redundancy1, redundancy2)) + redPacket.length shouldBe primary.length + 1 + redundancy1.payloadLength + 4 + redundancy2.payloadLength + 4 + redPacket.payloadType shouldBe 112 + + val reconstructedRedundancy = redPacket.decapsulate(parseRedundancy = true) + val reconstructedPrimary = RtpPacket(redPacket.buffer, redPacket.offset, redPacket.length) + + reconstructedPrimary should haveSameFixedHeader(primary) + reconstructedPrimary should haveSamePayload(primary) + + reconstructedRedundancy shouldHaveSize 2 + mapOf( + redundancy1 to reconstructedRedundancy[0], + redundancy2 to reconstructedRedundancy[1] + ).forEach { (expected, reconstructed) -> + + reconstructed.hasPadding shouldBe expected.hasPadding + reconstructed.hasExtensions shouldBe false + reconstructed.csrcCount shouldBe 0 + reconstructed.payloadType shouldBe expected.payloadType + reconstructed.sequenceNumber shouldBe expected.sequenceNumber + reconstructed.ssrc shouldBe expected.ssrc + reconstructed.timestamp shouldBe expected.timestamp + + reconstructed should haveSamePayload(expected) + } + } + context("Creating a RED packet with a large timestamp delta") { + val primary = RtpPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size).apply { + payloadType = 111 + } + val redundancy1 = RtpPacket(redPacketBytesSingleBlock.clone(), 0, redPacketBytesSingleBlock.size).apply { + payloadType = 111 + timestamp = primary.timestamp - RedundancyBlockHeader.MAX_TIMESTAMP_OFFSET - 1 + sequenceNumber = primary.sequenceNumber - 2 + } + + shouldThrow { RtpRedPacket.builder.build(112, primary, listOf(redundancy1)) } + } + } +} + +private fun RtpPacket.HeaderExtension?.shouldHaveId1AndLen1() { + this shouldNotBe null + this as RtpPacket.HeaderExtension + id shouldBe 1 + dataLengthBytes shouldBe 1 +} + +private val redPacketBytes = byteArrayOf( + // RTP Header. PT=112, seq=0x7b2b + 0x90, 0x70, 0x7b, 0x2b, + // TS + 0x44, 0xb2, 0x83, 0x6e, + // SSRC + 0x16, 0x49, 0x3f, 0x2d, + // Extension + 0xbe, 0xde, 0x00, 0x01, + // ID=1, value=0x3d + 0x10, 0x3d, 0x00, 0x00, + + // RTP Payload + // RED Header Block 1: follow=true, pt=111, ts_offset=1920, length=65 + 0xef, 0x1e, 0x00, 0x41, + // RED Header Block 2: follow=true, pt=111, ts_offset=960, length=68 + 0xef, 0x0f, 0x00, 0x44, + // RED Header Block 3: follow=false, pt=111 (remaining length=62) + 0x6f, + // RED Block 1 (length=65): + 0x68, 0x2f, 0x3b, 0x25, + 0xab, 0x1a, 0x72, 0xfb, + 0x75, 0x2d, 0xf0, 0x9b, + 0xa2, 0xa3, 0xfc, 0x20, + 0x51, 0xf7, 0x9c, 0xe8, + 0x75, 0xe9, 0x02, 0xd6, + 0xc1, 0xe4, 0xa1, 0x51, + 0x02, 0x00, 0x59, 0x55, + 0x04, 0x56, 0x5e, 0xed, + 0x31, 0x55, 0xb5, 0x04, + 0x9d, 0xf6, 0x1c, 0x40, + 0x7b, 0xb7, 0x00, 0x0c, + 0xd9, 0x7b, 0x5d, 0x13, + 0x4c, 0xeb, 0x7d, 0xf1, + 0x74, 0xf8, 0xd5, 0xb9, + 0x07, 0xda, 0x18, 0x19, + 0x92, + // RED Block 2 (length=68): + 0x68, 0x2f, 0x3b, 0x08, + 0xa5, 0x08, 0xb1, 0x93, + 0x5b, 0x65, 0x41, 0x30, + 0x55, 0xcd, 0xb7, 0xca, + 0xd2, 0x66, 0xc8, 0x53, + 0x18, 0x95, 0x9d, 0x49, + 0x81, 0x0a, 0xba, 0x67, + 0xf2, 0x42, 0xe3, 0xad, + 0x26, 0x73, 0x14, 0x52, + 0x62, 0x03, 0x3d, 0x1e, + 0xdd, 0x58, 0x44, 0x4e, + 0xc9, 0x56, 0x2f, 0x77, + 0xf9, 0x64, 0xf2, 0x6e, + 0x8c, 0x39, 0x35, 0x1a, + 0xba, 0x10, 0xe2, 0x85, + 0x1f, 0x28, 0x87, 0xb9, + 0x02, 0xa4, 0x68, 0x6b, + // RED Block 3 (length is the remainder, 62): + 0x68, 0x2f, 0x3b, 0x2f, + 0xb9, 0x5f, 0xc4, 0x65, + 0x67, 0x7f, 0x5e, 0xa0, + 0x46, 0xb0, 0x93, 0xf5, + 0xc3, 0x5c, 0x69, 0xd7, + 0x1f, 0x75, 0xe9, 0xef, + 0xd1, 0x94, 0xdc, 0x47, + 0x24, 0x13, 0xf8, 0x6e, + 0xda, 0x18, 0xf3, 0x72, + 0x5b, 0x73, 0x03, 0x8c, + 0xc6, 0x2c, 0x7a, 0xab, + 0x38, 0xad, 0x87, 0x6c, + 0x9b, 0x08, 0x85, 0x08, + 0x8f, 0xa3, 0x72, 0xf1, + 0xf4, 0x7d, 0x88, 0xc3, + 0x13, 0x84 +) + +private val redPacketBytesSingleBlock = byteArrayOf( + // RTP Header. PT=112, seq=0x7b2b + 0x90, 0x70, 0x7b, 0x2b, + // TS + 0x44, 0xb2, 0x83, 0x6e, + // SSRC + 0x16, 0x49, 0x3f, 0x2d, + // Extension + 0xbe, 0xde, 0x00, 0x01, + // ID=1, value=0x3d + 0x10, 0x3d, 0x00, 0x00, + + // RTP Payload + // RED Header Block 1: follow=false, pt=111 + 0x6f, + // RED Block 1 (length=65): + 0x68, 0x2f, 0x3b, 0x25, + 0xab, 0x1a, 0x72, 0xfb, + 0x75, 0x2d, 0xf0, 0x9b, + 0xa2, 0xa3, 0xfc, 0x20, + 0x51, 0xf7, 0x9c, 0xe8, + 0x75, 0xe9, 0x02, 0xd6, + 0xc1, 0xe4, 0xa1, 0x51, + 0x02, 0x00, 0x59, 0x55, + 0x04, 0x56, 0x5e, 0xed, + 0x31, 0x55, 0xb5, 0x04, + 0x9d, 0xf6, 0x1c, 0x40, + 0x7b, 0xb7, 0x00, 0x0c, + 0xd9, 0x7b, 0x5d, 0x13, + 0x4c, 0xeb, 0x7d, 0xf1, + 0x74, 0xf8, 0xd5, 0xb9, + 0x07, 0xda, 0x18, 0x19, + 0x92 +) + +private val redPacketBytesShortBlocks = byteArrayOf( + // RTP Header. PT=112, seq=35006 + 0x90, 0x70, 0x88, 0xBE, + // TS + 0x33, 0x6C, 0xE3, 0x71, + // SSRC + 0x25, 0x35, 0x4C, 0xF1, + // Extension + 0xBE, 0xDE, 0x00, 0x02, + // ID=5, val=0xEA4D; ID=1, val=0xFF + 0x51, 0xEA, 0x4D, 0x10, 0xFF, 0x00, 0x00, 0x00, + // RED Header block 1: follow=true, pt=111, ts_offset=960, length=3 + 0xEF, 0x0F, 0x00, 0x03, + // RED Header block 2: follow=false, pt=111 (remaining length=3) + 0x6F, + // Block 1 + 0xD8, 0xFF, 0xFE, + // Block 2 + 0xD8, 0xFF, 0xFE +) + +private val redPacketBytesZeroLengthBlocks = byteArrayOf( + // RTP Header. PT=112, seq=35006 + 0x80, 0x70, 0x88, 0xBE, + // TS + 0x33, 0x6C, 0xE3, 0x71, + // SSRC + 0x25, 0x35, 0x4C, 0xF1, + // RED Header block 1: follow=true, pt=0, ts_offset=0, length=0 + 0x80, 0x00, 0x00, 0x00, + // RED Header block 2: follow=false, pt=0 (remaining length=0) + 0x00 +) diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt new file mode 100644 index 0000000..f204812 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpHeaderTest.kt @@ -0,0 +1,203 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.collections.shouldContainInOrder +import io.kotest.matchers.shouldBe +import org.jitsi.rtp.extensions.bytearray.byteArrayOf + +// TODO: sets shouldn't change any other field +class RtpHeaderTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf +// override fun isolationMode(): IsolationMode? = IsolationMode.SingleInstance + + private val headerData = byteArrayOf( + // V=2,P=false,X=true,CC=2,M=false,PT=100,SeqNum=16535 + 0x92, 0x64, 0x40, 0x97, + // Timestamp: 3899068446 + 0xe8, 0x67, 0x10, 0x1e, + // SSRC: 2828806853 + 0xa8, 0x9c, 0x2a, 0xc5, + // CSRC 1: 123456 + 0x00, 0x01, 0xE2, 0x40, + // CSRC 2: 45678 + 0x00, 0x00, 0xB2, 0x6E, + // 1 extension + 0xbe, 0xde, 0x00, 0x01, + 0x51, 0x00, 0x02, 0x00 + ) + + init { + context("version") { + context("get") { + should("work correctly") { + RtpHeader.getVersion(headerData, 0) shouldBe 2 + } + } + context("set") { + should("work correctly") { + RtpHeader.setVersion(headerData, 0, 3) + RtpHeader.getVersion(headerData, 0) shouldBe 3 + RtpHeader.setVersion(headerData, 0, 0) + RtpHeader.getVersion(headerData, 0) shouldBe 0 + } + } + } + context("hasPadding") { + context("get") { + should("work correctly") { + RtpHeader.hasPadding(headerData, 0) shouldBe false + } + } + context("set") { + should("work correctly") { + RtpHeader.setPadding(headerData, 0, true) + RtpHeader.hasPadding(headerData, 0) shouldBe true + RtpHeader.setPadding(headerData, 0, false) + RtpHeader.hasPadding(headerData, 0) shouldBe false + } + } + } + context("hasExtensions") { + context("get") { + should("work correctly") { + RtpHeader.hasExtensions(headerData, 0) shouldBe true + } + } + context("set") { + should("work correctly") { + RtpHeader.setHasExtensions(headerData, 0, false) + RtpHeader.hasExtensions(headerData, 0) shouldBe false + RtpHeader.setHasExtensions(headerData, 0, true) + RtpHeader.hasExtensions(headerData, 0) shouldBe true + } + } + } + context("csrcCount") { + context("get") { + should("work correctly") { + RtpHeader.getCsrcCount(headerData, 0) shouldBe 2 + } + } + context("set") { + should("work correctly") { + RtpHeader.setCsrcCount(headerData, 0, 0) + RtpHeader.getCsrcCount(headerData, 0) shouldBe 0 + } + } + } + context("marker") { + context("get") { + should("work correctly") { + RtpHeader.getMarker(headerData, 0) shouldBe false + } + } + context("set") { + should("work correctly") { + RtpHeader.setMarker(headerData, 0, true) + RtpHeader.getMarker(headerData, 0) shouldBe true + RtpHeader.setMarker(headerData, 0, false) + RtpHeader.getMarker(headerData, 0) shouldBe false + } + should("not clobber payload type") { + val origPt = RtpHeader.getPayloadType(headerData, 0) + RtpHeader.setMarker(headerData, 0, true) + RtpHeader.getMarker(headerData, 0) shouldBe true + RtpHeader.getPayloadType(headerData, 0) shouldBe origPt + + RtpHeader.setMarker(headerData, 0, false) + RtpHeader.getMarker(headerData, 0) shouldBe false + RtpHeader.getPayloadType(headerData, 0) shouldBe origPt + } + } + } + context("payloadType") { + context("get") { + should("work correctly") { + RtpHeader.getPayloadType(headerData, 0) shouldBe 100 + } + } + context("set") { + should("work correctly") { + RtpHeader.setPayloadType(headerData, 0, 111) + RtpHeader.getPayloadType(headerData, 0) shouldBe 111 + RtpHeader.setPayloadType(headerData, 0, 115) + RtpHeader.getPayloadType(headerData, 0) shouldBe 115 + } + } + } + context("sequenceNumber") { + context("get") { + should("work correctly") { + RtpHeader.getSequenceNumber(headerData, 0) shouldBe 16535 + } + } + context("set") { + should("work correctly") { + RtpHeader.setSequenceNumber(headerData, 0, 4242) + RtpHeader.getSequenceNumber(headerData, 0) shouldBe 4242 + } + } + } + context("timestamp") { + context("get") { + should("work correctly") { + RtpHeader.getTimestamp(headerData, 0) shouldBe 3899068446L + } + } + context("set") { + should("work correctly") { + RtpHeader.setTimestamp(headerData, 0, 3899064242) + RtpHeader.getTimestamp(headerData, 0) shouldBe 3899064242 + } + } + } + context("ssrc") { + context("get") { + should("work correctly") { + RtpHeader.getSsrc(headerData, 0) shouldBe 2828806853 + } + } + context("set") { + should("work correctly") { + RtpHeader.setSsrc(headerData, 0, 424242) + RtpHeader.getSsrc(headerData, 0) shouldBe 424242 + } + } + } + context("csrcs") { + context("get") { + should("work correctly") { + RtpHeader.getCsrcs(headerData, 0) shouldContainInOrder (listOf(123456, 45678)) + } + } + context("set") { + should("work correctly") { + RtpHeader.setCsrcs(headerData, 0, listOf(2468, 1357)) + RtpHeader.getCsrcs(headerData, 0) shouldContainInOrder (listOf(2468, 1357)) + } + } + } + context("total length") { + should("be correct") { + RtpHeader.getTotalLength(headerData, 0) shouldBe 28 + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt new file mode 100644 index 0000000..623154f --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpPacketTest.kt @@ -0,0 +1,982 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.comparables.shouldBeGreaterThanOrEqualTo +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.comparables.shouldBeLessThanOrEqualTo +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.rtp.Packet +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.extensions.bytearray.byteArrayOf +import org.jitsi.rtp.extensions.bytearray.getShort +import org.jitsi.rtp.extensions.bytearray.putShort +import org.jitsi.rtp.extensions.unsigned.toPositiveInt +import org.jitsi.rtp.util.BufferPool +import org.jitsi.rtp.util.getByteAsInt +import org.jitsi.rtp.util.getIntAsLong +import org.jitsi.test_helpers.matchers.getPayload +import org.jitsi.test_helpers.matchers.haveSameContentAs +import org.jitsi.test_helpers.matchers.haveSameFixedHeader +import org.jitsi.test_helpers.matchers.haveSamePayload +import java.nio.charset.StandardCharsets +import java.util.Collections + +class RtpPacketTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + private val rtpHeaderWithXBit = byteArrayOf( + // V=2,P=false,X=true,CC=0,M=false,PT=111,SeqNum=5807 + 0x90, 0x6f, 0x16, 0xaf, + // Timestamp: 1710483662 + 0x65, 0xf3, 0xe8, 0xce, + // SSRC: 1208951354 + 0x48, 0x0f, 0x22, 0x3a + ) + + private val oneByteHeaderExtensions = byteArrayOf( + // BEDE, length=1 + 0xbe, + 0xde, + 0x00, + 0x01, + // ExtId=1,Length=0(1 byte),Data=FF,Padding + 0x10, + 0xff, + 0x00, + 0x00 + ) + + private val oneByteHeaderExtensionsPaddingBetween = byteArrayOf( + // BEDE, length=2 + 0xbe, 0xde, 0x00, 0x02, + // ExtId=1,Length=1(2 bytes),Data=FF,Padding + 0x11, 0xff, 0xff, 0x00, + // ExtId=2,Length=0(1 byte),Data=FF,Padding + 0x20, 0xff, 0x00, 0x00 + ) + + private val twoByteHeaderExtensions = byteArrayOf( + // 1000, length = 12 + 0x10, 0x00, 0x00, 0x0C, + 0x01, 0x00, 0x02, 0x01, + 0xff, 0x00, 0x03, 0x04, + 0xde, 0xad, 0xbe, 0xef, + 0x05, 0x22, 0x73, 0x75, + 0x70, 0x65, 0x72, 0x63, + 0x61, 0x6c, 0x69, 0x66, + 0x72, 0x61, 0x67, 0x69, + 0x6c, 0x69, 0x73, 0x74, + 0x69, 0x63, 0x65, 0x78, + 0x70, 0x69, 0x61, 0x6c, + 0x69, 0x64, 0x6f, 0x63, + 0x69, 0x6f, 0x75, 0x73 + ) + + private val cryptexHeaderExtensions = byteArrayOf( + 0xc0, + 0xde, + 0x00, + 0x01, + 0xeb, + 0x92, + 0x36, + 0x52 + ) + + private val rtpHeaderWithNoExtensions = byteArrayOf( + // V=2,P=false,X=false,CC=0,M=false,PT=111,SeqNum=5807 + 0x80, 0x6f, 0x16, 0xaf, + // Timestamp: 1710483662 + 0x65, 0xf3, 0xe8, 0xce, + // SSRC: 1208951354 + 0x48, 0x0f, 0x22, 0x3a + ) + + private val dummyRtpPayload = byteArrayOf( + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42 + ) + + private val rtpPacketWithExtensions = RtpPacket(rtpHeaderWithXBit + oneByteHeaderExtensions + dummyRtpPayload) + private val rtpPacketNoExtensions = RtpPacket(rtpHeaderWithNoExtensions + dummyRtpPayload) + private val rtpPacketWithExtensionsWithPaddingBetween = + RtpPacket(rtpHeaderWithXBit + oneByteHeaderExtensionsPaddingBetween + dummyRtpPayload) + private val rtpPacketWithTwoByteExtensions = + RtpPacket(rtpHeaderWithXBit + twoByteHeaderExtensions + dummyRtpPayload) + private val rtpPacketWithCryptexExtensions = + RtpPacket(rtpHeaderWithXBit + cryptexHeaderExtensions + dummyRtpPayload) + + init { + context("An RTP packet with header extensions") { + val rtpPacket = rtpPacketWithExtensions + should("be parsed correctly") { + rtpPacket.version shouldBe 2 + rtpPacket.hasPadding shouldBe false + rtpPacket.hasExtensions shouldBe true + rtpPacket.csrcCount shouldBe 0 + rtpPacket.isMarked shouldBe false + rtpPacket.payloadType shouldBe 111 + rtpPacket.sequenceNumber shouldBe 5807 + rtpPacket.timestamp shouldBe 1710483662L + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 1 + ext.buffer.getByteAsInt(ext.dataOffset) shouldBe 0xFF.toPositiveInt() + + rtpPacket.payloadLength shouldBe dummyRtpPayload.size + rtpPacket.getPayload() should haveSameContentAs(UnparsedPacket(dummyRtpPayload)) + rtpPacket.sanityCheck() + } + should("allow changing the ID of a header extension") { + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id = 12 + rtpPacket.getHeaderExtension(1) shouldBe null + rtpPacket.getHeaderExtension(12) shouldNotBe null + } + } + context("An RTP packet with header extensions with padding between them") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween + should("be parsed correctly") { + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + val ext1 = rtpPacket.getHeaderExtension(1) + ext1 shouldNotBe null + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 2 + ext2.dataLengthBytes shouldBe 1 + ext2.buffer.getByteAsInt(ext2.dataOffset) shouldBe 0xFF.toPositiveInt() + } + } + context("An RTP packet with two-byte header extensions") { + val rtpPacket = rtpPacketWithTwoByteExtensions + should("be parsed correctly") { + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext1 = rtpPacket.getHeaderExtension(1) + ext1 shouldNotBe null + ext1 as RtpPacket.HeaderExtension + ext1.dataLengthBytes shouldBe 0 + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 2 + ext2.dataLengthBytes shouldBe 1 + ext2.buffer.getByteAsInt(ext2.dataOffset) shouldBe 0xFF.toPositiveInt() + + val ext3 = rtpPacket.getHeaderExtension(3) + ext3 shouldNotBe null + ext3 as RtpPacket.HeaderExtension + ext3.id shouldBe 3 + ext3.dataLengthBytes shouldBe 4 + ext3.buffer.getIntAsLong(ext3.dataOffset) shouldBe 0xDEADBEEFL + + val ext4 = rtpPacket.getHeaderExtension(5) + ext4 shouldNotBe null + ext4 as RtpPacket.HeaderExtension + ext4.id shouldBe 5 + ext4.dataLengthBytes shouldBe 34 + String(ext4.buffer, ext4.dataOffset, ext4.dataLengthBytes, StandardCharsets.US_ASCII) shouldBe + "supercalifragilisticexpialidocious" + } + } + context("An RTP packet with cryptex header extensions") { + val rtpPacket = rtpPacketWithCryptexExtensions + should("be parsed correctly") { + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + rtpPacket.extensionsProfileType shouldBe 0xC0DE + + rtpPacket.getHeaderExtension(1) shouldBe null + rtpPacket.getHeaderExtension(0xe) shouldBe null + rtpPacket.getHeaderExtension(0xeb) shouldBe null + } + should("not allow header extensions to be modified") { + shouldThrow { + rtpPacket.addHeaderExtension(1, 1) + } + shouldThrow { + rtpPacket.removeHeaderExtension(0xe) + } + } + } + context("Adding a new RTP header extension") { + context("to an RTP packet with existing extensions") { + context("in a buffer that has no more room") { + val rtpPacket = rtpPacketWithExtensions.clone() + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 8 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 1 + ext.buffer.getByteAsInt(ext.dataOffset) shouldBe 0xFF.toPositiveInt() + + val ext2 = rtpPacket.getHeaderExtension(3) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 3 + ext2.dataLengthBytes shouldBe 2 + ext2.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + context("in a buffer that has room to the right") { + val buf = ByteArray(rtpPacketWithExtensions.length + 100) + System.arraycopy( + rtpPacketWithExtensions.buffer, + 0, + buf, + 0, + rtpPacketWithExtensions.length + ) + val rtpPacket = RtpPacket(buf, 0, rtpPacketWithExtensions.length) + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 8 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 1 + ext.buffer.getByteAsInt(ext.dataOffset) shouldBe 0xFF.toPositiveInt() + + val ext2 = rtpPacket.getHeaderExtension(3) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 3 + ext2.dataLengthBytes shouldBe 2 + ext2.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + context("in a buffer that has room to the left") { + val spaceOnTheLeft = 8 // this is what we often have with Octo. + val buf = ByteArray(rtpPacketWithExtensions.length + spaceOnTheLeft + 20) + System.arraycopy( + rtpPacketWithExtensions.buffer, + 0, + buf, + spaceOnTheLeft, + rtpPacketWithExtensions.length + ) + val rtpPacket = RtpPacket(buf, spaceOnTheLeft, rtpPacketWithExtensions.length) + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 8 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 1 + ext.buffer.getByteAsInt(ext.dataOffset) shouldBe 0xFF.toPositiveInt() + + val ext2 = rtpPacket.getHeaderExtension(3) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 3 + ext2.dataLengthBytes shouldBe 2 + ext2.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + } + context("to an RTP packet with no existing extensions") { + context("in a buffer that has no more room") { + val rtpPacket = rtpPacketNoExtensions.clone() + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + // The only difference in the fixed headers is the extension bit. + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(3) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 3 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + context("in a buffer that has more room to the right") { + val buf = ByteArray(rtpPacketNoExtensions.length + 100) + System.arraycopy(rtpPacketNoExtensions.buffer, 0, buf, 0, rtpPacketNoExtensions.length) + val rtpPacket = RtpPacket(buf, 0, rtpPacketNoExtensions.length) + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + // The only difference in the fixed headers is the extension bit. + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(3) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 3 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + context("in a buffer that has more room to the left") { + val spaceOnTheLeft = 8 // this is what we often have with Octo. + val buf = ByteArray(rtpPacketNoExtensions.length + spaceOnTheLeft + 20) + System.arraycopy( + rtpPacketNoExtensions.buffer, + 0, + buf, + spaceOnTheLeft, + rtpPacketNoExtensions.length + ) + val rtpPacket = RtpPacket(buf, spaceOnTheLeft, rtpPacketNoExtensions.length) + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + // The only difference in the fixed headers is the extension bit. + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(3) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 3 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + context("in tight padding-only packets") { + for (paddingLength in 0..255) { + val length = paddingLength + RtpHeader.FIXED_HEADER_SIZE_BYTES + val rtpPacket = PaddingOnlyPacket.create(length) + + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + rtpPacket.sanityCheck() + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(3) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 3 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xDEAD.toShort() + + // Payload length includes padding + rtpPacket.payloadLength shouldBe length - RtpHeader.FIXED_HEADER_SIZE_BYTES + rtpPacket.paddingSize shouldBe length - RtpHeader.FIXED_HEADER_SIZE_BYTES + + // This will have done a resize, so it should have left space at the end for the SRTP tag. + rtpPacket.offset + rtpPacket.length + + Packet.BYTES_TO_LEAVE_AT_END_OF_PACKET shouldBeLessThanOrEqualTo + rtpPacket.buffer.size + } + } + } + } + context("Adding a long RTP header extension") { + context("to an RTP packet with existing one-byte extensions") { + val rtpPacket = rtpPacketWithExtensions.clone() + val value = "supercalifragilisticexpialidocious".toByteArray(charset = StandardCharsets.US_ASCII) + val newExt = rtpPacket.addHeaderExtension(3, value.size) + System.arraycopy(value, 0, newExt.buffer, newExt.dataOffset, value.size) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 40 + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 1 + ext.buffer.getByteAsInt(ext.dataOffset) shouldBe 0xFF.toPositiveInt() + + val ext2 = rtpPacket.getHeaderExtension(3) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 3 + ext2.dataLengthBytes shouldBe 34 + String(ext2.buffer, ext2.dataOffset, ext2.dataLengthBytes, StandardCharsets.US_ASCII) shouldBe + "supercalifragilisticexpialidocious" + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + context("to an RTP packet without extensions") { + val rtpPacket = rtpPacketNoExtensions.clone() + val value = "supercalifragilisticexpialidocious".toByteArray(charset = StandardCharsets.US_ASCII) + val newExt = rtpPacket.addHeaderExtension(3, value.size) + System.arraycopy(value, 0, newExt.buffer, newExt.dataOffset, value.size) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 36 + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + + val ext2 = rtpPacket.getHeaderExtension(3) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 3 + ext2.dataLengthBytes shouldBe 34 + String(ext2.buffer, ext2.dataOffset, ext2.dataLengthBytes, StandardCharsets.US_ASCII) shouldBe + "supercalifragilisticexpialidocious" + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + context("to an RTP packet with two-byte extensions") { + val rtpPacket = rtpPacketWithTwoByteExtensions.clone() + val value = "Um-dittle-ittl-um-dittle-I".toByteArray(charset = StandardCharsets.US_ASCII) + val newExt = rtpPacket.addHeaderExtension(8, value.size) + System.arraycopy(value, 0, newExt.buffer, newExt.dataOffset, value.size) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 76 + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext2 = rtpPacket.getHeaderExtension(5) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 5 + ext2.dataLengthBytes shouldBe 34 + String(ext2.buffer, ext2.dataOffset, ext2.dataLengthBytes, StandardCharsets.US_ASCII) shouldBe + "supercalifragilisticexpialidocious" + + val ext3 = rtpPacket.getHeaderExtension(8) + ext3 shouldNotBe null + ext3 as RtpPacket.HeaderExtension + ext3.id shouldBe 8 + ext3.dataLengthBytes shouldBe 26 + String(ext3.buffer, ext3.dataOffset, ext3.dataLengthBytes, StandardCharsets.US_ASCII) shouldBe + "Um-dittle-ittl-um-dittle-I" + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + } + context("Adding an RTP header extension with a large ID value") { + context("to an RTP packet with existing one-byte extensions") { + val rtpPacket = rtpPacketWithExtensions.clone() + val newExt = rtpPacket.addHeaderExtension(99, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 8 + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 1 + ext.buffer.getByteAsInt(ext.dataOffset) shouldBe 0xFF.toPositiveInt() + + val ext2 = rtpPacket.getHeaderExtension(99) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 99 + ext2.dataLengthBytes shouldBe 2 + ext2.buffer.getShort(ext2.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + context("to an RTP packet without extensions") { + val rtpPacket = rtpPacketNoExtensions.clone() + val newExt = rtpPacket.addHeaderExtension(99, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + + val ext2 = rtpPacket.getHeaderExtension(99) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 99 + ext2.dataLengthBytes shouldBe 2 + ext2.buffer.getShort(ext2.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + context("to an RTP packet with two-byte extensions") { + val rtpPacket = rtpPacketWithTwoByteExtensions.clone() + val newExt = rtpPacket.addHeaderExtension(99, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 52 + rtpPacket.extensionsProfileType shouldBe 0x1000 + + val ext2 = rtpPacket.getHeaderExtension(5) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.id shouldBe 5 + ext2.dataLengthBytes shouldBe 34 + String(ext2.buffer, ext2.dataOffset, ext2.dataLengthBytes, StandardCharsets.US_ASCII) shouldBe + "supercalifragilisticexpialidocious" + + val ext3 = rtpPacket.getHeaderExtension(99) + ext3 shouldNotBe null + ext3 as RtpPacket.HeaderExtension + ext3.id shouldBe 99 + ext3.dataLengthBytes shouldBe 2 + ext3.buffer.getShort(ext3.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketWithExtensions) + } + } + } + context("Removing a header extension") { + context("when a header extension remains") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween.clone() + rtpPacket.removeHeaderExtension(2) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensionsWithPaddingBetween) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xFFFF.toShort() + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + + context("when no header extension remains") { + val rtpPacket = rtpPacketWithExtensions.clone() + rtpPacket.removeHeaderExtension(1) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketNoExtensions) + + rtpPacket.headerLength shouldBe RtpHeader.FIXED_HEADER_SIZE_BYTES + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + rtpPacket.hasExtensions shouldBe false + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + + context("from a two-byte header extension when the header removed is small") { + val rtpPacket = rtpPacketWithTwoByteExtensions.clone() + + rtpPacket.removeHeaderExtension(1) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + rtpPacket.extensionsProfileType shouldBe 0x1000 + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 48 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.dataLengthBytes shouldBe 1 + ext2.buffer.getByteAsInt(ext2.dataOffset) shouldBe 0xFF.toPositiveInt() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + + context("from a two-byte header extension when there is a zero-length header") { + val rtpPacket = rtpPacketWithTwoByteExtensions.clone() + + rtpPacket.removeHeaderExtension(5) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + rtpPacket.extensionsProfileType shouldBe 0x1000 + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 12 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.dataLengthBytes shouldBe 0 + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.dataLengthBytes shouldBe 1 + ext2.buffer.getByteAsInt(ext2.dataOffset) shouldBe 0xFF.toPositiveInt() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + context("from a two-byte header extension when all remaining headers can fit in a one-byte header") { + val rtpPacket = rtpPacketWithTwoByteExtensions.clone() + + rtpPacket.removeHeaderExtension(5) + rtpPacket.removeHeaderExtension(1) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensions) + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 8 + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldNotBe null + ext2 as RtpPacket.HeaderExtension + ext2.dataLengthBytes shouldBe 1 + ext2.buffer.getByteAsInt(ext2.dataOffset) shouldBe 0xFF.toPositiveInt() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + } + context("Removing all but a given header extension") { + context("when a header extension remains") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween.clone() + rtpPacket.removeHeaderExtensionsExcept(Collections.singleton(1)) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensionsWithPaddingBetween) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xFFFF.toShort() + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldBe null + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + + context("when no header extension remains due to no match") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween.clone() + rtpPacket.removeHeaderExtensionsExcept(Collections.singleton(3)) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketNoExtensions) + + rtpPacket.headerLength shouldBe RtpHeader.FIXED_HEADER_SIZE_BYTES + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + rtpPacket.hasExtensions shouldBe false + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + + context("when no header extension remains due to empty exception set") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween.clone() + rtpPacket.removeHeaderExtensionsExcept(Collections.emptySet()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketNoExtensions) + + rtpPacket.headerLength shouldBe RtpHeader.FIXED_HEADER_SIZE_BYTES + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + rtpPacket.hasExtensions shouldBe false + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + } + + context("Removing and then adding header extensions") { + context("when an original header extension remains") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween.clone() + rtpPacket.removeHeaderExtensionsExcept(Collections.singleton(1)) + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensionsWithPaddingBetween) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 8 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + ext as RtpPacket.HeaderExtension + ext.id shouldBe 1 + ext.dataLengthBytes shouldBe 2 + ext.buffer.getShort(ext.dataOffset) shouldBe 0xFFFF.toShort() + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldBe null + + val ext3 = rtpPacket.getHeaderExtension(3) + ext3 shouldNotBe null + ext3 as RtpPacket.HeaderExtension + ext3.id shouldBe 3 + ext3.dataLengthBytes shouldBe 2 + ext3.buffer.getShort(ext3.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + + context("when all original header extensions are removed") { + val rtpPacket = rtpPacketWithExtensionsWithPaddingBetween.clone() + rtpPacket.removeHeaderExtensionsExcept(Collections.emptySet()) + val newExt = rtpPacket.addHeaderExtension(3, 2) + newExt.buffer.putShort(newExt.dataOffset, 0xDEAD.toShort()) + rtpPacket.encodeHeaderExtensions() + + should("update the packet correctly") { + rtpPacket.sanityCheck() + rtpPacket should haveSameFixedHeader(rtpPacketWithExtensionsWithPaddingBetween) + + rtpPacket.headerLength shouldBe + RtpHeader.FIXED_HEADER_SIZE_BYTES + RtpHeader.EXT_HEADER_SIZE_BYTES + 4 + rtpPacket.extensionsProfileType shouldBe 0xBEDE + + val ext = rtpPacket.getHeaderExtension(1) + ext shouldBe null + + val ext2 = rtpPacket.getHeaderExtension(2) + ext2 shouldBe null + + val ext3 = rtpPacket.getHeaderExtension(3) + ext3 shouldNotBe null + ext3 as RtpPacket.HeaderExtension + ext3.id shouldBe 3 + ext3.dataLengthBytes shouldBe 2 + ext3.buffer.getShort(ext3.dataOffset) shouldBe 0xDEAD.toShort() + + rtpPacket should haveSamePayload(rtpPacketNoExtensions) + } + } + } + context("Changing the type of the header extension in the underlying buffer") { + // This is a simplified simulation of what happens when packets are cryptex-encrypted. + val rtpPacket = rtpPacketWithExtensions.clone() + val ext = rtpPacket.getHeaderExtension(1) + ext shouldNotBe null + + rtpPacket.buffer.putShort(rtpPacket.offset + RtpHeader.FIXED_HEADER_SIZE_BYTES, 0xC0DE.toShort()) + + should("cause header extensions to not be readable") { + val ext2 = rtpPacket.getHeaderExtension(1) + ext2 shouldBe null + } + } + } +} + +// This imitates PaddingVideoPacket in jmt +class PaddingOnlyPacket private constructor( + buffer: ByteArray, + offset: Int, + length: Int +) : RtpPacket(buffer, offset, length) { + + override fun clone(): PaddingOnlyPacket = throw NotImplementedError("clone() not supported for padding packets.") + + companion object { + /** + * Creating a PaddingVideoPacket by directly grabbing a buffer in its + * ctor is problematic because we cannot clear the buffer we retrieve + * before calling the parent class' constructor. Because the buffer + * may contain invalid data, any attempts to parse it by parent class(es) + * could fail, so we use a helper here instead + */ + fun create(length: Int): PaddingOnlyPacket { + val buf = BufferPool.getArray(length) + // It's possible we the buffer we pulled from the pool already has + // data in it, and we won't be overwriting it with anything so clear + // out the data + buf.fill(0, 0, length) + + return PaddingOnlyPacket(buf, 0, length).apply { + // Recalculate the header length now that we've zero'd everything out + // and set the fields + version = RtpHeader.VERSION + headerLength = RtpHeader.getTotalLength(buffer, offset) + paddingSize = payloadLength + } + } + } +} + +fun RtpPacket.sanityCheck() { + offset shouldBeGreaterThanOrEqualTo 0 + length shouldBeGreaterThanOrEqualTo 0 + offset + length shouldBeLessThanOrEqualTo buffer.size + headerLength shouldBeLessThanOrEqualTo length + headerLength shouldBe RtpHeader.getTotalLength(buffer, offset) + payloadLength shouldBeLessThan length + headerLength + payloadLength shouldBe length + paddingSize shouldBeLessThanOrEqualTo payloadLength +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumberTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumberTest.kt new file mode 100644 index 0000000..7f37726 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/RtpSequenceNumberTest.kt @@ -0,0 +1,90 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.rtp + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.sequences.shouldContainInOrder +import io.kotest.matchers.shouldBe + +class RtpSequenceNumberTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("rollover") { + var num = 65535.toRtpSequenceNumber() + context("via add-and-assign") { + should("work correctly") { + num += 1 + num shouldBe 0.toRtpSequenceNumber() + } + } + context("via plus") { + should("work correctly") { + (num + 1) shouldBe 0.toRtpSequenceNumber() + } + } + } + context("reverse rollover") { + var num = 0.toRtpSequenceNumber() + context("via subtract-and-assign") { + should("work correctly") { + num -= 1 + num shouldBe 65535.toRtpSequenceNumber() + } + } + context("via minus") { + should("work correctly") { + (num - 1) shouldBe 65535.toRtpSequenceNumber() + } + } + } + context("comparison") { + should("work correctly") { + (1.toRtpSequenceNumber() < 2.toRtpSequenceNumber()) shouldBe true + (1.toRtpSequenceNumber() <= 2.toRtpSequenceNumber()) shouldBe true + (1.toRtpSequenceNumber() <= 1.toRtpSequenceNumber()) shouldBe true + (1.toRtpSequenceNumber() == 1.toRtpSequenceNumber()) shouldBe true + (1.toRtpSequenceNumber() < 0.toRtpSequenceNumber()) shouldBe false + (65534.toRtpSequenceNumber() < 0.toRtpSequenceNumber()) shouldBe true + (32767.toRtpSequenceNumber() < 0.toRtpSequenceNumber()) shouldBe false + (32768.toRtpSequenceNumber() < 0.toRtpSequenceNumber()) shouldBe true + } + } + context("rangeTo") { + should("work correctly") { + (65533.toRtpSequenceNumber()..2.toRtpSequenceNumber()).asSequence().shouldContainInOrder( + 65533.toRtpSequenceNumber(), + 65534.toRtpSequenceNumber(), + 65535.toRtpSequenceNumber(), + 0.toRtpSequenceNumber(), + 1.toRtpSequenceNumber(), + 2.toRtpSequenceNumber() + ) + + (2.toRtpSequenceNumber() downTo 65533.toRtpSequenceNumber()).asSequence().shouldContainInOrder( + 2.toRtpSequenceNumber(), + 1.toRtpSequenceNumber(), + 0.toRtpSequenceNumber(), + 65535.toRtpSequenceNumber(), + 65534.toRtpSequenceNumber(), + 65533.toRtpSequenceNumber() + ) + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt new file mode 100644 index 0000000..686b673 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/Av1DependencyDescriptorHeaderExtensionTest.kt @@ -0,0 +1,605 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.assertions.withClue +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import jakarta.xml.bind.DatatypeConverter.parseHexBinary + +class Av1DependencyDescriptorHeaderExtensionTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + /* Headers generated by Chrome 110 sending AV1 in its default configuration - L1T1 */ + val descL1T1 = parseHexBinary("80000180003a410180ef808680") + + val shortDesc = parseHexBinary("400001") + + val descL1T3 = parseHexBinary("800001800214eaa860414d141020842701df010d") + + /* Header generated by Chrome 112 sending AV1 with L3T3 set. */ + val descL3T3 = parseHexBinary( + "d0013481e81485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a800060000" + + "40001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230ec" + + "a87753013f00b3027f016704ff02cf" + ) + + val midDescScalable = parseHexBinary("d10146401c") + + val midDescScalable2 = parseHexBinary("c203ce581d30100000") + val longForMid2 = parseHexBinary( + "8003ca80081485214eaaaaa8000600004000100002aa80a8000600004000100002a000a8000600004" + + "00016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" + ) + + val descS3T3 = parseHexBinary( + "c1000180081485214ea000a8000600004000100002a000a8000600004000100002a000a8000600004" + + "0001d954926caa493655248c55fe5d00032a190cc38e58803b2a1954c10e10843b2a1dd4c01dc010803bc0218077c0434" + ) + + val descL3T3Key = parseHexBinary( + "8f008581e81485214eaaaaa8000600004000100002aa80a8000600004000100002a000a80006000040" + + "0016d549241b5524906d54923157e001974ca864330e222396eca8655304224390eca87753013f00b3027f016704ff02cf" + ) + + /* As of Chrome version 111, it doesn't support L3T3_KEY_SHIFT, but it does support L2T2_KEY_SHIFT, so test that. */ + val descL2T2KeyShift = parseHexBinary( + "8700ed80e3061eaa82804028280514d14134518010a091889a09409fc059c13fc0b3c0" + ) + + /* The header Chrome 126 generates for VP8 keyframes when AV1 DD is enabled for VP8. It has no chains. */ + val descNoChains = parseHexBinary( + "8000138002044eaaaf2860414d34538a0940413fc0b3c0" + ) + + init { + context("AV1 Dependency Descriptors") { + context("a descriptor with a single-layer dependency structure") { + val ld1r = Av1DependencyDescriptorReader(descL1T1, 0, descL1T1.size) + val ld1 = ld1r.parse(null) + should("be parsed properly") { + ld1.startOfFrame shouldBe true + ld1.endOfFrame shouldBe false + ld1.frameNumber shouldBe 0x0001 + + val structure = ld1.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 1 + structure.maxTemporalId shouldBe 0 + structure.maxSpatialId shouldBe 0 + } + should("be parseable statelessly") { + val ld1s = ld1r.parseStateless() + ld1s.startOfFrame shouldBe true + ld1s.endOfFrame shouldBe false + ld1s.frameNumber shouldBe 0x0001 + + val structure = ld1s.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 1 + } + should("calculate correct frame info") { + val ld1i = ld1.frameInfo + ld1i.spatialId shouldBe 0 + ld1i.temporalId shouldBe 0 + } + should("Calculate its own length properly") { + ld1.encodedLength shouldBe descL1T1.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(ld1.encodedLength) + ld1.write(buf, 0, buf.size) + buf shouldBe descL1T1 + } + } + context("a descriptor with a scalable dependency structure") { + val ldsr = Av1DependencyDescriptorReader(descL3T3, 0, descL3T3.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe true + lds.frameNumber shouldBe 0x0134 + lds.activeDecodeTargetsBitmask shouldBe 0x1ff + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 9 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 2 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..2) { + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + for (toS in 0..2) { + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch down spatial layers, or to other temporal + * layers within the same spatial layer, without a keyframe; but switching up + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS >= toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b000000001 + structure.getDtBitmaskForDt(1) shouldBe 0b000000011 + structure.getDtBitmaskForDt(2) shouldBe 0b000000111 + structure.getDtBitmaskForDt(3) shouldBe 0b000001001 + structure.getDtBitmaskForDt(4) shouldBe 0b000011011 + structure.getDtBitmaskForDt(5) shouldBe 0b000111111 + structure.getDtBitmaskForDt(6) shouldBe 0b001001001 + structure.getDtBitmaskForDt(7) shouldBe 0b011011011 + structure.getDtBitmaskForDt(8) shouldBe 0b111111111 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL3T3.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL3T3 + } + } + context("a descriptor with a K-SVC dependency structure") { + val ldsr = Av1DependencyDescriptorReader(descL3T3Key, 0, descL3T3Key.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x0085 + lds.activeDecodeTargetsBitmask shouldBe 0x1ff + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 9 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 2 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..2) { + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + for (toS in 0..2) { + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch to other temporal + * layers within the same spatial layer, without a keyframe; but switching + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS == toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b000000001 + structure.getDtBitmaskForDt(1) shouldBe 0b000000011 + structure.getDtBitmaskForDt(2) shouldBe 0b000000111 + structure.getDtBitmaskForDt(3) shouldBe 0b000001000 + structure.getDtBitmaskForDt(4) shouldBe 0b000011000 + structure.getDtBitmaskForDt(5) shouldBe 0b000111000 + structure.getDtBitmaskForDt(6) shouldBe 0b001000000 + structure.getDtBitmaskForDt(7) shouldBe 0b011000000 + structure.getDtBitmaskForDt(8) shouldBe 0b111000000 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL3T3Key.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL3T3Key + } + } + context("a descriptor with a K-SVC dependency structure with key shift") { + val ldsr = Av1DependencyDescriptorReader(descL2T2KeyShift, 0, descL2T2KeyShift.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x00ed + lds.activeDecodeTargetsBitmask shouldBe 0x0f + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 4 + structure.maxTemporalId shouldBe 1 + structure.maxSpatialId shouldBe 1 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..1) { + for (fromT in 0..1) { + val fromDT = 2 * fromS + fromT + for (toS in 0..1) { + for (toT in 0..1) { + val toDT = 2 * toS + toT + /* With this structure you can switch to other temporal + * layers within the same spatial layer, without a keyframe; but switching + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS == toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b0001 + structure.getDtBitmaskForDt(1) shouldBe 0b0011 + structure.getDtBitmaskForDt(2) shouldBe 0b0100 + structure.getDtBitmaskForDt(3) shouldBe 0b1100 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL2T2KeyShift.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL2T2KeyShift + } + } + context("a descriptor following the dependency structure, specifying decode targets") { + val ldsr = Av1DependencyDescriptorReader(descL3T3, 0, descL3T3.size) + val lds = ldsr.parse(null) + val mdsr = Av1DependencyDescriptorReader(midDescScalable, 0, midDescScalable.size) + val mds = mdsr.parse(lds.newTemplateDependencyStructure) + + should("be parsed properly") { + mds.startOfFrame shouldBe true + mds.endOfFrame shouldBe true + mds.frameNumber shouldBe 0x0146 + + mds.newTemplateDependencyStructure shouldBe null + mds.activeDecodeTargetsBitmask shouldBe 0x7 + } + should("calculate correct frame info") { + val mdsi = mds.frameInfo + mdsi.spatialId shouldBe 0 + mdsi.temporalId shouldBe 1 + } + should("Calculate its own length properly") { + mds.encodedLength shouldBe midDescScalable.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(mds.encodedLength) + mds.write(buf, 0, buf.size) + buf shouldBe midDescScalable + } + } + context("another such descriptor") { + val ldsr = Av1DependencyDescriptorReader(longForMid2, 0, longForMid2.size) + val lds = ldsr.parse(null) + val mdsr = Av1DependencyDescriptorReader(midDescScalable2, 0, midDescScalable2.size) + val mds = mdsr.parse(lds.newTemplateDependencyStructure) + + should("be parsed properly") { + mds.startOfFrame shouldBe true + mds.endOfFrame shouldBe true + mds.frameNumber shouldBe 0x03ce + + mds.newTemplateDependencyStructure shouldBe null + mds.activeDecodeTargetsBitmask shouldBe 0x7 + } + should("calculate correct frame info") { + val mdsi = mds.frameInfo + mdsi.spatialId shouldBe 0 + mdsi.temporalId shouldBe 1 + } + should("Calculate its own length properly") { + mds.encodedLength shouldBe midDescScalable2.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(mds.encodedLength) + mds.write(buf, 0, buf.size) + buf shouldBe midDescScalable2 + } + } + context("a descriptor without a dependency structure") { + val mdsr = Av1DependencyDescriptorReader(midDescScalable, 0, midDescScalable.size) + should("be parseable as the stateless subset") { + val mds = mdsr.parseStateless() + + mds.startOfFrame shouldBe true + mds.endOfFrame shouldBe true + mds.frameNumber shouldBe 0x0146 + + mds.newTemplateDependencyStructure shouldBe null + } + should("fail to parse if the dependency structure is not present") { + shouldThrow { + mdsr.parse(null) + } + } + } + context("a descriptor without extended fields") { + val ld1r = Av1DependencyDescriptorReader(descL1T1, 0, descL1T1.size) + val ld1 = ld1r.parse(null) + val sd1r = Av1DependencyDescriptorReader(shortDesc, 0, shortDesc.size) + val sd1 = sd1r.parse(ld1.newTemplateDependencyStructure) + + should("be parsed properly") { + sd1.startOfFrame shouldBe false + sd1.endOfFrame shouldBe true + sd1.frameNumber shouldBe 0x0001 + + sd1.newTemplateDependencyStructure shouldBe null + sd1.activeDecodeTargetsBitmask shouldBe null + } + should("calculate correct frame info") { + val sd1i = sd1.frameInfo + sd1i.spatialId shouldBe 0 + sd1i.temporalId shouldBe 0 + } + should("Calculate its own length properly") { + sd1.encodedLength shouldBe shortDesc.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(sd1.encodedLength) + sd1.write(buf, 0, buf.size) + buf shouldBe shortDesc + } + } + context("A descriptor with a Temporal-only scalability structure ") { + val ldsr = Av1DependencyDescriptorReader(descL1T3, 0, descL1T3.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x0001 + lds.activeDecodeTargetsBitmask shouldBe 0x7 + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 3 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 0 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + val fromS = 0 + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + val toS = 0 + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch down spatial layers, or to other temporal + * layers within the same spatial layer, without a keyframe; but switching up + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b001 + structure.getDtBitmaskForDt(1) shouldBe 0b011 + structure.getDtBitmaskForDt(2) shouldBe 0b111 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descL1T3.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descL1T3 + } + } + context("a descriptor with a simulcast dependency structure") { + val ldsr = Av1DependencyDescriptorReader(descS3T3, 0, descS3T3.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe true + lds.frameNumber shouldBe 0x0001 + lds.activeDecodeTargetsBitmask shouldBe 0x1ff + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 9 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 2 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + for (fromS in 0..2) { + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + for (toS in 0..2) { + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch to other temporal + * layers within the same spatial layer, without a keyframe; but switching + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + if (fromS == toS) { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } else { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe false + } + } + } + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b000000001 + structure.getDtBitmaskForDt(1) shouldBe 0b000000011 + structure.getDtBitmaskForDt(2) shouldBe 0b000000111 + structure.getDtBitmaskForDt(3) shouldBe 0b000001000 + structure.getDtBitmaskForDt(4) shouldBe 0b000011000 + structure.getDtBitmaskForDt(5) shouldBe 0b000111000 + structure.getDtBitmaskForDt(6) shouldBe 0b001000000 + structure.getDtBitmaskForDt(7) shouldBe 0b011000000 + structure.getDtBitmaskForDt(8) shouldBe 0b111000000 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descS3T3.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descS3T3 + } + } + context("A descriptor with no chains") { + val ldsr = Av1DependencyDescriptorReader(descNoChains, 0, descNoChains.size) + val lds = ldsr.parse(null) + should("be parsed properly") { + lds.startOfFrame shouldBe true + lds.endOfFrame shouldBe false + lds.frameNumber shouldBe 0x0013 + lds.activeDecodeTargetsBitmask shouldBe 0x7 + + val structure = lds.newTemplateDependencyStructure + structure shouldNotBe null + structure!!.decodeTargetCount shouldBe 3 + structure.maxTemporalId shouldBe 2 + structure.maxSpatialId shouldBe 0 + } + should("calculate correct frame info") { + val ldsi = lds.frameInfo + ldsi.spatialId shouldBe 0 + ldsi.temporalId shouldBe 0 + } + should("calculate correctly whether layer switching needs keyframes") { + val structure = lds.newTemplateDependencyStructure!! + val fromS = 0 + for (fromT in 0..2) { + val fromDT = 3 * fromS + fromT + val toS = 0 + for (toT in 0..2) { + val toDT = 3 * toS + toT + /* With this structure you can switch down spatial layers, or to other temporal + * layers within the same spatial layer, without a keyframe; but switching up + * spatial layers needs a keyframe. + */ + withClue("from DT $fromDT to DT $toDT") { + structure.canSwitchWithoutKeyframe( + fromDt = fromDT, + toDt = toDT + ) shouldBe true + } + } + } + } + should("calculate DTI bitmasks corresponding to a given DT") { + val structure = lds.newTemplateDependencyStructure!! + structure.getDtBitmaskForDt(0) shouldBe 0b001 + structure.getDtBitmaskForDt(1) shouldBe 0b011 + structure.getDtBitmaskForDt(2) shouldBe 0b111 + } + should("Calculate its own length properly") { + lds.encodedLength shouldBe descNoChains.size + } + should("Be re-encoded to the same bytes") { + val buf = ByteArray(lds.encodedLength) + lds.write(buf, 0, buf.size) + buf shouldBe descNoChains + } + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt new file mode 100644 index 0000000..2f89038 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/DumpAv1DependencyDescriptor.kt @@ -0,0 +1,36 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import jakarta.xml.bind.DatatypeConverter.parseHexBinary + +fun main(args: Array) { + var structure: Av1TemplateDependencyStructure? = null + var line: String? + while (readLine().also { line = it } != null) { + try { + val descBinary = parseHexBinary(line) + val reader = Av1DependencyDescriptorReader(descBinary, 0, descBinary.size) + val desc = reader.parse(structure) + desc.newTemplateDependencyStructure?.let { structure = it } + println(desc.toString()) + val frameInfo = desc.frameInfo + println(frameInfo) + } catch (e: Exception) { + println(e.message) + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt new file mode 100644 index 0000000..4803c09 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/rtp/header_extensions/SdesHeaderExtensionTest.kt @@ -0,0 +1,154 @@ +/* + * Copyright @ 2021 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.rtp.rtp.header_extensions + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNotBe +import org.jitsi.rtp.extensions.bytearray.byteArrayOf +import org.jitsi.rtp.rtp.RtpPacket + +class SdesHeaderExtensionTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + private val rtpHeaderWithEmptyExtension = byteArrayOf( + // V=2,P=false,X=true,CC=0,M=false,PT=111,SeqNum=5807 + 0x90, 0x6f, 0x16, 0xaf, + // Timestamp: 1710483662 + 0x65, 0xf3, 0xe8, 0xce, + // SSRC: 1208951354 + 0x48, 0x0f, 0x22, 0x3a, + ) + + private val dummyRtpPayload = byteArrayOf( + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42, + 0x42, 0x42, 0x42, 0x42 + ) + + private val rtpHeaderSimpleSdesExtension = byteArrayOf( + // BEDE, length=1 + 0xbe, + 0xde, + 0x00, + 0x01, + // ExtId=1,Length=0(1 byte),Data='1',Padding + 0x10, + 0x31, + 0x00, + 0x00 + ) + + private val rtpHeaderEmojiSdesExtension = byteArrayOf( + // BEDE, length=2 + 0xbe, 0xde, 0x00, 0x02, + // ExtId=5,Length=3(4 byte),Data='Poop emoji',Padding + 0x53, 0xf0, 0x9f, 0x92, + 0xa9, 0x00, 0x00, 0x00 + ) + + private val rtpHeaderMaxSdesExtension = byteArrayOf( + // BEDE, length=5 + 0xbe, 0xde, 0x00, 0x05, + // ExtId=15,Length=15(16 byte),Data='abcdefghijklmnop',Padding + 0xff, 0x61, 0x62, 0x63, + 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, + 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x00, 0x00, 0x00 + ) + + private val simpleSdesHeaderExtension = RtpPacket( + rtpHeaderWithEmptyExtension + + rtpHeaderSimpleSdesExtension + dummyRtpPayload + ) + private val maxLengthSdesHeaderExtension = RtpPacket( + rtpHeaderWithEmptyExtension + + rtpHeaderMaxSdesExtension + dummyRtpPayload + ) + + private val emojiSdesHeaderExtension = RtpPacket( + rtpHeaderWithEmptyExtension + + rtpHeaderEmojiSdesExtension + dummyRtpPayload + ) + + init { + context("RTP with simple SDES extension value 1") { + should("simple SDES should be parsed correctly") { + val rtpPacket = simpleSdesHeaderExtension + rtpPacket shouldNotBe null + val sdesExt = rtpPacket.getHeaderExtension(1) + sdesExt shouldNotBe null + sdesExt as RtpPacket.HeaderExtension + sdesExt.id shouldBe 1 + sdesExt.dataLengthBytes shouldBe 1 + val payload = SdesHeaderExtension.getTextValue(sdesExt) + payload shouldBe "1" + } + should("allow changing the simple SDES value") { + val rtpPacket = simpleSdesHeaderExtension + rtpPacket shouldNotBe null + val sdesExt = rtpPacket.getHeaderExtension(1) + sdesExt shouldNotBe null + sdesExt as RtpPacket.HeaderExtension + SdesHeaderExtension.setTextValue(sdesExt, "2") + val payload = SdesHeaderExtension.getTextValue(sdesExt) + payload shouldBe "2" + } + } + context("RTP with max len SDES extension value abcdefghijklmop") { + should("max length SDES should be parsed correctly") { + val rtpPacket = maxLengthSdesHeaderExtension + rtpPacket shouldNotBe null + val sdesExt = rtpPacket.getHeaderExtension(15) + sdesExt shouldNotBe null + sdesExt as RtpPacket.HeaderExtension + sdesExt.id shouldBe 15 + sdesExt.dataLengthBytes shouldBe 16 + val payload = SdesHeaderExtension.getTextValue(sdesExt) + payload shouldBe "abcdefghijklmnop" + } + should("allow changing the max length SDES value") { + val rtpPacket = maxLengthSdesHeaderExtension + rtpPacket shouldNotBe null + val sdesExt = rtpPacket.getHeaderExtension(15) + sdesExt shouldNotBe null + sdesExt as RtpPacket.HeaderExtension + SdesHeaderExtension.setTextValue(sdesExt, "ZYX23acj00yxzABC") + val payload = SdesHeaderExtension.getTextValue(sdesExt) + payload shouldBe "ZYX23acj00yxzABC" + } + } + context("RTP with emjoi SDES extension") { + should("emoji SDES should not be parsed") { + val rtpPacket = emojiSdesHeaderExtension + rtpPacket shouldNotBe null + val sdesExt = rtpPacket.getHeaderExtension(5) + sdesExt shouldNotBe null + sdesExt as RtpPacket.HeaderExtension + sdesExt.id shouldBe 5 + sdesExt.dataLengthBytes shouldBe 4 + val payload = SdesHeaderExtension.getTextValue(sdesExt) + /* The payload here is UTF-8, but we only parse ASCII. + * Parsing doesn't fail, but the string contains garbage. */ + payload shouldNotBe null + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt new file mode 100644 index 0000000..2be8e51 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/util/BitReaderTest.kt @@ -0,0 +1,132 @@ +/* + * Copyright @ 2024 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +class BitReaderTest : ShouldSpec() { + override fun isolationMode(): IsolationMode = IsolationMode.InstancePerLeaf + + init { + val bitReader = BitReader( + byteArrayOf( + 0b0101_1010.toByte(), + 0b0000_1111.toByte(), + 0b1010_1010.toByte(), + 0b1010_1010.toByte(), + 0b0111_1111.toByte() + ) + ) + val bits = 40 + context("Reading single bits as boolean") { + bitReader.bitAsBoolean() shouldBe false + bitReader.bitAsBoolean() shouldBe true + bitReader.bitAsBoolean() shouldBe false + bitReader.bitAsBoolean() shouldBe true + bitReader.bitAsBoolean() shouldBe true + bitReader.remainingBits() shouldBe bits - 5 + } + + context("Reading single bits as Integer") { + bitReader.bit() shouldBe 0 + bitReader.bit() shouldBe 1 + bitReader.bit() shouldBe 0 + bitReader.bit() shouldBe 1 + bitReader.bit() shouldBe 1 + bitReader.remainingBits() shouldBe bits - 5 + } + + context("Reading multiple bits as Integer") { + bitReader.bits(4) shouldBe 0b0101 + bitReader.bits(4) shouldBe 0b1010 + bitReader.bits(6) shouldBe 0b000011 + bitReader.bits(2) shouldBe 0b11 + bitReader.remainingBits() shouldBe bits - 16 + } + + context("Reading multiple bits as Long") { + bitReader.bitsLong(4) shouldBe 0b0101L + bitReader.bitsLong(4) shouldBe 0b1010L + bitReader.bits(6) shouldBe 0b000011L + bitReader.bits(2) shouldBe 0b11L + bitReader.remainingBits() shouldBe bits - 16 + } + + context("skip bits correctly") { + bitReader.skipBits(4) + bitReader.remainingBits() shouldBe bits - 4 + bitReader.bit() shouldBe 1 + bitReader.remainingBits() shouldBe bits - 5 + } + + context("Reading LEB128-encoded unsigned integers") { + bitReader.leb128() shouldBe 0b0101_1010 + bitReader.remainingBits() shouldBe bits - 8 + bitReader.leb128() shouldBe 0b00001111 + bitReader.remainingBits() shouldBe bits - 16 + bitReader.leb128() shouldBe 0b111_1111__010_1010__010_1010 + bitReader.remainingBits() shouldBe 0 + } + + context("Cloning") { + bitReader.skipBits(8) + val clone = bitReader.clone(2) + bitReader.remainingBits() shouldBe bits - 8 + clone.remainingBits() shouldBe 16 + + bitReader.bits(8) shouldBe 0b0000_1111 + bitReader.remainingBits() shouldBe bits - 16 + clone.remainingBits() shouldBe 16 + + clone.bits(8) shouldBe 0b0000_1111 + bitReader.remainingBits() shouldBe bits - 16 + clone.remainingBits() shouldBe 8 + + clone.skipBits(8) + bitReader.remainingBits() shouldBe bits - 16 + clone.remainingBits() shouldBe 0 + + bitReader.bits(8) shouldBe 0b1010_1010 + shouldThrow { + clone.bits(8) + } + } + + context("Throwing exception when reading past the bounds") { + bitReader.skipBits(bits) + bitReader.remainingBits() shouldBe 0 + shouldThrow { + bitReader.bitAsBoolean() + } + shouldThrow { + bitReader.bit() + } + shouldThrow { + bitReader.bits(4) + } + shouldThrow { + bitReader.leb128() + } + shouldThrow { + BitReader(byteArrayOf(0b1111_0000.toByte())).leb128() + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/util/ByteBufferUtilsKtTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/util/ByteBufferUtilsKtTest.kt new file mode 100644 index 0000000..10c4d22 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/util/ByteBufferUtilsKtTest.kt @@ -0,0 +1,37 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import io.kotest.core.spec.IsolationMode +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.matchers.shouldBe + +internal class ByteBufferUtilsKtTest : ShouldSpec() { + override fun isolationMode(): IsolationMode? = IsolationMode.InstancePerLeaf + + init { + context("byteBufferOf(Any)") { + should("work correctly with values that can be converted to byte") { + val buf = byteBufferOf(0x00, 0x01, 0xFF, 0xFE) + buf.get(0) shouldBe 0x00.toByte() + buf.get(1) shouldBe 0x01.toByte() + buf.get(2) shouldBe 0xFF.toByte() + buf.get(3) shouldBe 0xFE.toByte() + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/rtp/util/RtpUtilsTest.kt b/rtp/src/test/kotlin/org/jitsi/rtp/util/RtpUtilsTest.kt new file mode 100644 index 0000000..5ad2d41 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/rtp/util/RtpUtilsTest.kt @@ -0,0 +1,120 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.rtp.util + +import io.kotest.core.spec.style.ShouldSpec +import io.kotest.data.forAll +import io.kotest.data.row +import io.kotest.matchers.collections.shouldContainInOrder +import io.kotest.matchers.shouldBe + +class RtpUtilsTest : ShouldSpec() { + + init { + context("getSequenceNumberDelta") { + should("work correctly") { + forAll( + row(1, 10, -9), + row(10, 1, 9), + row(1, 65530, 7), + row(65530, 1, -7), + row(1234, 1234, 0) + ) { a, b, expected -> + RtpUtils.getSequenceNumberDelta(a, b) shouldBe expected + } + } + } + context("applySequenceNumberDelta") { + should("work correctly") { + forAll( + row(10, -9, 1), + row(1, 9, 10), + row(65530, 7, 1), + row(1, -7, 65530), + row(1234, 0, 1234) + ) { start, delta, expected -> + RtpUtils.applySequenceNumberDelta(start, delta) shouldBe expected + } + } + } + context("isNewerThan") { + should("work correctly") { + forAll( + row(2, 1, true), + row(2, 65530, true), + row(2, 4, false) + ) { a, b, expected -> + a isNewerThan b shouldBe expected + } + } + } + context("applyTimestampDelta") { + should("work correctly") { + forAll( + row(10L, -9L, 1L), + row(1L, 9L, 10L), + row(0xffff_fff0L, 0x11L, 1L), + row(1L, -0x11L, 0xffff_fff0L), + row(1234L, 0L, 1234L) + ) { start, delta, expected -> + RtpUtils.applyTimestampDelta(start, delta) shouldBe expected + } + } + } + context("isOlderThan") { + should("work correctly") { + forAll( + row(2, 1, false), + row(2, 65530, false), + row(2, 4, true) + ) { a, b, expected -> + a isOlderThan b shouldBe expected + } + } + } + context("rolledOverTo") { + should("return true when a rollover has taken place") { + 65535 rolledOverTo 1 shouldBe true + 65000 rolledOverTo 200 shouldBe true + } + should("return false when a rollover has not occurred") { + 0 rolledOverTo 65535 shouldBe false + } + } + context("isNextAfter") { + should("return true for sequential packets") { + 2 isNextAfter 1 shouldBe true + 0 isNextAfter 65535 shouldBe true + } + should("return false for non-sequential packets") { + 2 isNextAfter 0 shouldBe false + 1 isNextAfter 2 shouldBe false + 1 isNextAfter 65535 shouldBe false + } + } + context("sequenceNumbersBetween") { + should("contain all sequence numbers between 2 values") { + RtpUtils.sequenceNumbersBetween(1, 10).toList() shouldContainInOrder + listOf(2, 3, 4, 5, 6, 7, 8, 9) + } + should("contain all sequence numbers between 2 values, even with rollover") { + RtpUtils.sequenceNumbersBetween(65530, 5).toList() shouldContainInOrder + listOf(65531, 65532, 65533, 65535, 0, 1, 2, 3, 4) + } + } + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteArray.kt b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteArray.kt new file mode 100644 index 0000000..66a16d1 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteArray.kt @@ -0,0 +1,42 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.test_helpers.matchers + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.extensions.bytearray.toHex + +fun haveSameContentAs(expected: ByteArray) = object : Matcher { + override fun test(value: ByteArray): MatcherResult { + var matches = true + if (expected.size != value.size) { + matches = false + } else { + for (i in (0 until expected.size)) { + if (value.get(i) != expected.get(i)) { + matches = false + break + } + } + } + return MatcherResult( + matches, + { "\n${value.toHex()}\nwas supposed to be:\n${expected.toHex()}" }, + { "\n${value.toHex()}\nshould not have equaled \n${expected.toHex()}" } + ) + } +} diff --git a/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteArrayBuffer.kt b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteArrayBuffer.kt new file mode 100644 index 0000000..88a50f7 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteArrayBuffer.kt @@ -0,0 +1,43 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.test_helpers.matchers + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.extensions.toHex +import org.jitsi.utils.ByteArrayBuffer + +fun ByteArrayBuffer.hasSameContentAs(other: ByteArrayBuffer): Boolean { + if (other.length != length) { + return false + } else { + for (i in (0 until length)) { + if (buffer[offset + i] != other.buffer[other.offset + i]) { + return false + } + } + } + + return true +} + +fun haveSameContentAs(expected: ByteArrayBuffer) = object : Matcher { + override fun test(value: ByteArrayBuffer): MatcherResult = MatcherResult( + value.hasSameContentAs(expected), + { "\n${value.toHex()}\nwas supposed to be:\n${expected.toHex()}" }, + { "\n${value.toHex()}\nshould not have equaled \n${expected.toHex()}" } + ) +} diff --git a/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteBuffer.kt b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteBuffer.kt new file mode 100644 index 0000000..99a9b26 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/ByteBuffer.kt @@ -0,0 +1,31 @@ +/* + * Copyright @ 2018 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jitsi.test_helpers.matchers + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.extensions.compareToFromBeginning +import org.jitsi.rtp.extensions.toHex +import java.nio.ByteBuffer + +fun haveSameContentAs(expected: ByteBuffer) = object : Matcher { + override fun test(value: ByteBuffer): MatcherResult = MatcherResult( + value.compareToFromBeginning(expected) == 0, + { "Buffer\n${value.toHex()}\nwas supposed to be:\n${expected.toHex()}" }, + { "Buffer\n${value.toHex()}\nshould not have equaled buffer\n${expected.toHex()}" } + ) +} diff --git a/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt new file mode 100644 index 0000000..155e621 --- /dev/null +++ b/rtp/src/test/kotlin/org/jitsi/test_helpers/matchers/RtpPacket.kt @@ -0,0 +1,56 @@ +/* + * Copyright @ 2019 - present 8x8, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jitsi.test_helpers.matchers + +import io.kotest.matchers.Matcher +import io.kotest.matchers.MatcherResult +import org.jitsi.rtp.UnparsedPacket +import org.jitsi.rtp.extensions.toHex +import org.jitsi.rtp.rtp.RtpHeader +import org.jitsi.rtp.rtp.RtpPacket +import org.jitsi.utils.ByteArrayBuffer + +fun RtpPacket.getPayload(): ByteArrayBuffer = UnparsedPacket(buffer, payloadOffset, payloadLength) + +fun RtpPacket.getFixedHeaderAsBAB(): ByteArrayBuffer = UnparsedPacket(buffer, offset, RtpHeader.FIXED_HEADER_SIZE_BYTES) + +fun haveSamePayload(expected: RtpPacket) = object : Matcher { + override fun test(value: RtpPacket): MatcherResult { + val valuePayload = value.getPayload() + val expectedPayload = expected.getPayload() + + return MatcherResult( + valuePayload.hasSameContentAs(expectedPayload), + { "\n${valuePayload.toHex()}\nwas supposed to be:\n${expectedPayload.toHex()}" }, + { + "\n${valuePayload.toHex()}\nshould not have equaled \n${expectedPayload.toHex()}" + } + ) + } +} + +fun haveSameFixedHeader(expected: RtpPacket) = object : Matcher { + override fun test(value: RtpPacket): MatcherResult { + val valueHeader = value.getFixedHeaderAsBAB() + val expectedHeader = expected.getFixedHeaderAsBAB() + + return MatcherResult( + valueHeader.hasSameContentAs(expectedHeader), + { "\n${valueHeader.toHex()}\nwas supposed to be:\n${expectedHeader.toHex()}" }, + { "\n${valueHeader.toHex()}\nshould not have equaled \n${expectedHeader.toHex()}" } + ) + } +}