snapshot of https://github.com/jitsi/jitsi-videobridge.git @ fcbf741c646bc237d2ed85a1e86910532532b831 (2026-09-15)
This commit is contained in:
42
resources/analyze-timeline.pl
Executable file
42
resources/analyze-timeline.pl
Executable file
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use Statistics::Descriptive;
|
||||
use strict;
|
||||
|
||||
my %stats;
|
||||
my @stat_names;
|
||||
|
||||
while (<>) {
|
||||
if (/Reference time: /) {
|
||||
my $prev_time;
|
||||
while (/; \(([^)]*), PT([0-9.]*)/g) {
|
||||
my $name = $1;
|
||||
my $time = $2;
|
||||
|
||||
my $delta_time;
|
||||
if (defined $prev_time) {
|
||||
$delta_time = $time - $prev_time;
|
||||
}
|
||||
else {
|
||||
$delta_time = $time;
|
||||
}
|
||||
$prev_time = $time;
|
||||
|
||||
if ($name =~ /Toggleable pcap writer:/) {
|
||||
$name =~ s/writer: .*/writer/;
|
||||
}
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($delta_time * 1e3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for my $name (@stat_names) {
|
||||
my $s = $stats{$name};
|
||||
printf("%s: min %.3f ms, mean %.3f ms, median %.3f ms, 90%% %.3f ms, 99%% %.3f ms, max %.3f ms\n",
|
||||
$name, $s->min(), $s->mean(), $s->median(), scalar($s->percentile(90)), scalar($s->percentile(99)), $s->max());
|
||||
}
|
||||
244
resources/analyze-timeline2.pl
Executable file
244
resources/analyze-timeline2.pl
Executable file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/perl -w
|
||||
|
||||
use Statistics::Descriptive;
|
||||
use strict;
|
||||
|
||||
my %stats;
|
||||
my @stat_names;
|
||||
|
||||
=begin comment
|
||||
|
||||
This is extremely ugly, I'm driving perl in first gear because I don't know how to shift :)
|
||||
|
||||
It parses the PacketInfo EventTimeline logs (see example below) and produces stats for
|
||||
the following periods from the life cycle of a packet:
|
||||
|
||||
1.Receiver Queue: From the moment the packet is read from the ICE transport (in an IO thread) and added the the Receiver Queue
|
||||
to the moment it is removed from the Receiver Queue (in a CPU thread) and the pipeline processing starts.
|
||||
|
||||
2.Receiver Pipeline: From the start of the Receiver Pipeline to the end, excluding the last Termination node. This is all
|
||||
performed in the same CPU thread, and includes things like SRTP decryption/authentication, RTP parsing, updating bandwidth
|
||||
estimation, video codec parsing, etc.
|
||||
|
||||
3.Termination Node: This is technically the last Node in the Receiver Pipeline and executes in the same thread, but
|
||||
it is conceptually different and worth looking at separately. It consists of conference-level handling of the packet: it loops
|
||||
through all local endpoints and relays, checks whether they "want" the packet, and if they do then clones the packet and puts
|
||||
the clone on the sender queue (see Conference#sendOut). This workload scales with the number of local endpoits and relays,
|
||||
and in a large conference is the most computationally expensive.
|
||||
|
||||
4.Sender Queue: From the time the Receiver Pipeline thread places the packet on the Sender Queue, to the time another CPU thread
|
||||
removes it from the queue and starts executing the Sender Pipeline.
|
||||
|
||||
5.Sender Pipeline: From the start of the Sender Pipeline to the end. This executes in a CPU thread and includes stripping unknown (for
|
||||
this specific receiving endpoint) RTP header extensions, saving a copy of the packet in the NACK cache, adding/updating the transport-cc
|
||||
header extension, and SRTP encryption.
|
||||
|
||||
This script looks only at RTP packets (ignoring RTCP and retransmissions) for simplicity.
|
||||
|
||||
6.SRTP Queue: From the time a SRTP packet is placed on the queue by the Sender Pipeline thread to the time an IO thread removes it
|
||||
from the queue and starts sending it over the ICE transport.
|
||||
|
||||
7.Send over ICE transport: The time it took to execute IceTransport.send, which sends the packet through the ice4j socket representation.
|
||||
|
||||
8.Total: From the start of 1. to the end of 7.
|
||||
|
||||
|
||||
|
||||
Example log line:
|
||||
JVB 2024-06-11 17:57:44.989 FINER: [1531] [confId=6149c0c339432f97 conf_name=loadtest0@conference.xxx meeting_id=ec7261c3 epId=c88aca35 stats_id=Otto-emV] Endpoint.doSendSrtp#544: Reference time: 2024-06-11T17:57:44.988517466Z; (Entered RTP receiver incoming queue, PT0.00000192S); (Exited RTP receiver incoming queue, PT0.000324242S); (Entered node PacketStreamStats, PT0.000324602S); (Exited node PacketStreamStats, PT0.000325042S); (Entered node SRTP/SRTCP demuxer, PT0.000325162S); (Exited node SRTP/SRTCP demuxer, PT0.000325722S); (Entered node RTP Parser, PT0.000325962S); (Exited node RTP Parser, PT0.000326762S); (Entered node Audio level reader (pre-srtp), PT0.000326922S); (Exited node Audio level reader (pre-srtp), PT0.000327242S); (Entered node Video mute node, PT0.000327362S); (Exited node Video mute node, PT0.000327642S); (Entered node SRTP Decrypt Node, PT0.000327762S); (Exited node SRTP Decrypt Node, PT0.000332442S); (Entered node TCC generator, PT0.000332562S); (Exited node TCC generator, PT0.000333562S); (Entered node Remote Bandwidth Estimator, PT0.000344962S); (Exited node Remote Bandwidth Estimator, PT0.000345202S); (Entered node Audio level reader (post-srtp), PT0.000345322S); (Exited node Audio level reader (post-srtp), PT0.000345442S); (Entered node Toggleable pcap writer: 21913831-rx, PT0.000345562S); (Exited node Toggleable pcap writer: 21913831-rx, PT0.000345762S); (Entered node Incoming statistics tracker, PT0.000345882S); (Exited node Incoming statistics tracker, PT0.000351242S); (Entered node Padding termination, PT0.000351362S); (Exited node Padding termination, PT0.000351522S); (Entered node Media Type demuxer, PT0.000351642S); (Exited node Media Type demuxer, PT0.000351962S); (Entered node RTX handler, PT0.000352082S); (Exited node RTX handler, PT0.000352322S); (Entered node Duplicate termination, PT0.000352722S); (Exited node Duplicate termination, PT0.000355442S); (Entered node Retransmission requester, PT0.000355882S); (Exited node Retransmission requester, PT0.000357922S); (Entered node Padding-only discarder, PT0.000358642S); (Exited node Padding-only discarder, PT0.000359842S); (Entered node Video parser, PT0.000360242S); (Exited node Video parser, PT0.000363082S); (Entered node Video quality layer lookup, PT0.000363362S); (Exited node Video quality layer lookup, PT0.000364362S); (Entered node Video bitrate calculator, PT0.000364682S); (Exited node Video bitrate calculator, PT0.000369122S); (Entered node Input pipeline termination node, PT0.000369362S); (Entered node receiver chain handler, PT0.000370122S); (Entered RTP sender incoming queue, PT0.000488083S); (Exited RTP sender incoming queue, PT0.000551283S); (Entered node Pre-processor, PT0.000551723S); (Exited node Pre-processor, PT0.000555843S); (Entered node RedHandler, PT0.000556083S); (Exited node RedHandler, PT0.000556563S); (Entered node Strip header extensions, PT0.000556843S); (Exited node Strip header extensions, PT0.000557523S); (Entered node Packet cache, PT0.000557923S); (Exited node Packet cache, PT0.000561083S); (Entered node Absolute send time, PT0.000561243S); (Exited node Absolute send time, PT0.000561643S); (Entered node Outgoing statistics tracker, PT0.000561923S); (Exited node Outgoing statistics tracker, PT0.000562683S); (Entered node TCC sequence number tagger, PT0.000563003S); (Exited node TCC sequence number tagger, PT0.000563843S); (Entered node Header extension encoder, PT0.000564083S); (Exited node Header extension encoder, PT0.000565163S); (Entered node Toggleable pcap writer: c88aca35-tx, PT0.000565443S); (Exited node Toggleable pcap writer: c88aca35-tx, PT0.000565763S); (Entered node SRTP Encrypt Node, PT0.000566083S); (Exited node SRTP Encrypt Node, PT0.000572883S); (Entered node PacketStreamStats, PT0.000573243S); (Exited node PacketStreamStats, PT0.000573883S); (Entered node Output pipeline termination node, PT0.000574123S); (Entered Endpoint SRTP sender outgoing queue, PT0.000574443S); (Exited node Output pipeline termination node, PT0.000582923S); (Exited Endpoint SRTP sender outgoing queue, PT0.000668003S); (Sent over the ICE transport, PT0.000693284S)
|
||||
|
||||
=end comment
|
||||
|
||||
=cut
|
||||
|
||||
my $receiverQStartName = "Entered RTP receiver incoming queue";
|
||||
my $receiverQEndName = "Exited RTP receiver incoming queue";
|
||||
my $receiverPStartName = "Entered node SRTP/SRTCP demuxer";
|
||||
my $receiverPEndName = "Entered node Input pipeline termination node";
|
||||
my $terminationNodeStartName = "Entered node Input pipeline termination node";
|
||||
my $terminationNodeEndName = "Exited node Input pipeline termination node";
|
||||
my $senderQStartName = "Entered RTP sender incoming queue";
|
||||
my $senderQEndName = "Exited RTP sender incoming queue";
|
||||
my $senderPStartName = "Entered node RedHandler";
|
||||
my $senderPEndName = "Entered Endpoint SRTP sender outgoing queue";
|
||||
my $srtpQStartName = "Entered Endpoint SRTP sender outgoing queue";
|
||||
my $srtpQEndName = "Exited Endpoint SRTP sender outgoing queue";
|
||||
my $iceStartName = "Exited Endpoint SRTP sender outgoing queue";
|
||||
my $iceEndName = "Sent over the ICE transport";
|
||||
my $totalStartName = "Entered RTP receiver incoming queue";
|
||||
my $totalEndName = "Sent over the ICE transport";
|
||||
|
||||
while (<>) {
|
||||
if (/Reference time: /) {
|
||||
my $prev_time;
|
||||
my $receiverQStart = -1;
|
||||
my $receiverQEnd = -1;
|
||||
my $receiverPStart = -1;
|
||||
my $receiverPEnd = -1 ;
|
||||
my $terminationNodeStart = -1 ;
|
||||
my $terminationNodeEnd = -1;
|
||||
my $senderQStart = -1;
|
||||
my $senderQEnd = -1;
|
||||
my $senderPStart = -1;
|
||||
my $senderPEnd = -1 ;
|
||||
my $srtpQStart = -1;
|
||||
my $srtpQEnd = -1;
|
||||
my $iceStart = -1;
|
||||
my $iceEnd = -1;
|
||||
my $totalStart = -1;
|
||||
my $totalEnd = -1;
|
||||
|
||||
while (/; \(([^)]*), PT([0-9.]*)/g) {
|
||||
my $name = $1;
|
||||
my $time = $2;
|
||||
|
||||
if ($name eq $receiverQStartName) {
|
||||
$receiverQStart = $time;
|
||||
}
|
||||
if ($name eq $receiverQEndName) {
|
||||
$receiverQEnd = $time;
|
||||
}
|
||||
if ($name eq $receiverPStartName) {
|
||||
$receiverPStart = $time;
|
||||
}
|
||||
if ($name eq $receiverPEndName) {
|
||||
$receiverPEnd = $time;
|
||||
}
|
||||
if ($name eq $terminationNodeStartName) {
|
||||
$terminationNodeStart = $time;
|
||||
}
|
||||
if ($name eq $terminationNodeEndName) {
|
||||
$terminationNodeEnd = $time;
|
||||
}
|
||||
if ($name eq $senderQStartName) {
|
||||
$senderQStart = $time;
|
||||
}
|
||||
if ($name eq $senderQEndName) {
|
||||
$senderQEnd = $time;
|
||||
}
|
||||
if ($name eq $senderPStartName) {
|
||||
$senderPStart = $time;
|
||||
}
|
||||
if ($name eq $senderPEndName) {
|
||||
$senderPEnd = $time;
|
||||
}
|
||||
if ($name eq $srtpQStartName) {
|
||||
$srtpQStart = $time;
|
||||
}
|
||||
if ($name eq $srtpQEndName) {
|
||||
$srtpQEnd = $time;
|
||||
}
|
||||
if ($name eq $iceStartName) {
|
||||
$iceStart = $time;
|
||||
}
|
||||
if ($name eq $iceEndName) {
|
||||
$iceEnd = $time;
|
||||
}
|
||||
if ($name eq $totalStartName) {
|
||||
$totalStart = $time;
|
||||
}
|
||||
if ($name eq $totalEndName) {
|
||||
$totalEnd = $time;
|
||||
}
|
||||
|
||||
#my $delta_time;
|
||||
#if (defined $prev_time) {
|
||||
# $delta_time = $time - $prev_time;
|
||||
#}
|
||||
#else {
|
||||
# $delta_time = $time;
|
||||
#}
|
||||
#$prev_time = $time;
|
||||
|
||||
#if ($name =~ /Toggleable pcap writer:/) {
|
||||
# $name =~ s/writer: .*/writer/;
|
||||
#}
|
||||
}
|
||||
if ($receiverQStart > -1 && $receiverQEnd > -1) {
|
||||
my $name = "1.Receiver Queue";
|
||||
my $time = $receiverQEnd - $receiverQStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($receiverPStart > -1 && $receiverPEnd > -1) {
|
||||
my $name = "2.Receiver Pipeline";
|
||||
my $time = $receiverPEnd - $receiverPStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($terminationNodeStart > -1 && $terminationNodeEnd > -1) {
|
||||
my $name = "3.Termination Node";
|
||||
my $time = $terminationNodeEnd - $terminationNodeStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($senderQStart > -1 && $senderQEnd > -1) {
|
||||
my $name = "4.Sender Queue";
|
||||
my $time = $senderQEnd - $senderQStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($senderPStart > -1 && $senderPEnd > -1) {
|
||||
my $name = "5.Sender Pipeline";
|
||||
my $time = $senderPEnd - $senderPStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($srtpQStart > -1 && $srtpQEnd > -1) {
|
||||
my $name = "6.SRTP Queue";
|
||||
my $time = $srtpQEnd - $srtpQStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($iceStart > -1 && $iceEnd > -1) {
|
||||
my $name = "7.Send over ICE transport";
|
||||
my $time = $iceEnd - $iceStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
if ($totalStart > -1 && $totalEnd > -1) {
|
||||
my $name = "8.Total";
|
||||
my $time = $totalEnd - $totalStart;
|
||||
|
||||
if (!defined ($stats{$name})) {
|
||||
$stats{$name} = Statistics::Descriptive::Full->new();
|
||||
push(@stat_names, $name);
|
||||
}
|
||||
$stats{$name}->add_data($time * 1e3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for my $name (@stat_names) {
|
||||
my $s = $stats{$name};
|
||||
printf("%s: min %.3f ms, mean %.3f ms, median %.3f ms, 90%% %.3f ms, 99%% %.3f ms, max %.3f ms\n",
|
||||
$name, $s->min(), $s->mean(), $s->median(), scalar($s->percentile(90)), scalar($s->percentile(99)), $s->max());
|
||||
}
|
||||
5
resources/build.sh
Executable file
5
resources/build.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
mvn clean verify package "$@"
|
||||
71
resources/build_deb_package.sh
Executable file
71
resources/build_deb_package.sh
Executable file
@@ -0,0 +1,71 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
set -e
|
||||
|
||||
# Make sure you have the following environment variables set, example:
|
||||
# export DEBFULLNAME="Jitsi Team"
|
||||
# export DEBEMAIL="dev@jitsi.org"
|
||||
# You need package devscripts installed (command dch) and libxml2-utils(provides xmllint), dh-systemd.
|
||||
#
|
||||
# There is an optional param that can be used to add an extra string to the version.
|
||||
# That string can be used to mark custom version passing "hf" as param
|
||||
# will result debian version 2.1-123-gabcabcab-hf-1, instead of just 2.1-123-gabcabcab-1
|
||||
VER_EXTRA_LABEL=$1
|
||||
|
||||
echo "==================================================================="
|
||||
echo " Building DEB packages... "
|
||||
echo "==================================================================="
|
||||
|
||||
SCRIPT_FOLDER=$(dirname "$0")
|
||||
cd "$SCRIPT_FOLDER/.."
|
||||
|
||||
# Let's get version from maven
|
||||
MVNVER=$(xmllint --xpath "/*[local-name()='project']/*[local-name()='version']/text()" pom.xml)
|
||||
TAG_NAME="v${MVNVER/-SNAPSHOT/}"
|
||||
|
||||
echo "Current tag name: $TAG_NAME"
|
||||
|
||||
if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1
|
||||
then
|
||||
git tag -a "$TAG_NAME" -m "Tagged automatically by Jenkins"
|
||||
git push origin "$TAG_NAME"
|
||||
else
|
||||
echo "Tag: $TAG_NAME already exists."
|
||||
fi
|
||||
|
||||
VERSION_FULL=$(git describe --match "v[0-9\.]*" --long)
|
||||
echo "Full version: ${VERSION_FULL}"
|
||||
|
||||
export VERSION=${VERSION_FULL:1}
|
||||
if [ -n "${VER_EXTRA_LABEL}" ]
|
||||
then
|
||||
VERSION+="-${VER_EXTRA_LABEL}"
|
||||
fi
|
||||
echo "Package version: ${VERSION}"
|
||||
|
||||
REV=$(git log --pretty=format:'%h' -n 1)
|
||||
dch -v "$VERSION-1" "Build from git. $REV"
|
||||
dch -D unstable -r ""
|
||||
|
||||
# sets the version in the pom file so it will propagate to resulting jar
|
||||
mvn versions:set -DnewVersion="${VERSION}"
|
||||
|
||||
# We need to make sure all dependencies are downloaded before start building
|
||||
# the debian package
|
||||
mvn package
|
||||
|
||||
# now build the deb
|
||||
dpkg-buildpackage -tc -us -uc -A
|
||||
|
||||
# clean the current changes as dch had changed the change log
|
||||
git checkout debian/changelog
|
||||
git checkout pom.xml
|
||||
|
||||
echo "Here are the resulting files in $(pwd ..)"
|
||||
echo "-----"
|
||||
ls -l ../{*.changes,*.deb,*.buildinfo}
|
||||
echo "-----"
|
||||
|
||||
# Let's try deploying
|
||||
cd ..
|
||||
([ ! -x deploy.sh ] || ./deploy.sh "jvb" $VERSION )
|
||||
63
resources/collect-dump-logs.sh
Executable file
63
resources/collect-dump-logs.sh
Executable file
@@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
|
||||
# script that creates an archive in current folder
|
||||
# containing the heap and thread dump and the current log file
|
||||
|
||||
JVB_HEAPDUMP_PATH="/tmp/java_*.hprof"
|
||||
STAMP=`date +%Y-%m-%d-%H%M`
|
||||
PID_PATH="/var/run/jitsi-videobridge.pid"
|
||||
JVB_UID=`id -u jvb`
|
||||
|
||||
RUNNING=""
|
||||
unset PID
|
||||
|
||||
# for systemd we use different pid file
|
||||
if [ ! -f $PID_PATH ]; then
|
||||
PID_PATH="/var/run/jitsi-videobridge/jitsi-videobridge.pid"
|
||||
fi
|
||||
|
||||
[ -e $PID_PATH ] && PID=$(cat $PID_PATH)
|
||||
if [ ! -z $PID ]; then
|
||||
ps -p $PID | grep -q java
|
||||
[ $? -eq 0 ] && RUNNING="true"
|
||||
fi
|
||||
|
||||
#Find any crashes in /var/crash from our user in the past 20 minutes, if they exist
|
||||
CRASH_FILES=$(find /var/crash -name '*.crash' -uid $JVB_UID -mmin -20 -type f)
|
||||
|
||||
if [ ! -z $RUNNING ]; then
|
||||
echo "Jvb at pid $PID"
|
||||
THREADS_FILE="/tmp/stack-${STAMP}-${PID}.threads"
|
||||
HEAP_FILE="/tmp/heap-${STAMP}-${PID}.bin"
|
||||
|
||||
# If the JVB has crashed (with OOM, for example), the JVM might be
|
||||
# configured to take a heap dump. This is configured with the HeapDumpPath
|
||||
# VM option. The HeapDumpPath is a manageable VM option and the default
|
||||
# path format is java_pidXXXX.hprof.
|
||||
#
|
||||
# During the time the JVM is taking a heap dump, the process will appear
|
||||
# running, but the JVM won't allow us to connect to it and take a
|
||||
# heap/thread dump. Briefly, we want to make sure we grab the java heap
|
||||
# dump, even if the process appears to be running.
|
||||
|
||||
HEAPDUMP_FILE="/tmp/java_pid${PID}.hprof"
|
||||
|
||||
# Wait for the JVM to finish writing the heapdump.
|
||||
while lsof ${HEAPDUMP_FILE}; do sleep 1; done
|
||||
|
||||
sudo jstack ${PID} > ${THREADS_FILE}
|
||||
sudo jmap -dump:live,format=b,file=${HEAP_FILE} ${PID}
|
||||
|
||||
tar zcvf jvb-dumps-${STAMP}-${PID}.tgz ${THREADS_FILE} ${HEAP_FILE} ${HEAPDUMP_FILE} ${CRASH_FILES} /var/log/jitsi/jvb.log /tmp/hs_err_*
|
||||
rm ${HEAP_FILE} ${THREADS_FILE}
|
||||
else
|
||||
ls $JVB_HEAPDUMP_PATH >/dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "JVB not running, but previous heap dump found."
|
||||
tar zcvf jvb-dumps-${STAMP}-crash.tgz $JVB_HEAPDUMP_PATH ${CRASH_FILES} /var/log/jitsi/jvb.log /tmp/hs_err_*
|
||||
rm ${JVB_HEAPDUMP_PATH}
|
||||
else
|
||||
tar zcvf jvb-dumps-${STAMP}-crash.tgz /var/log/jitsi/jvb.log ${CRASH_FILES} /tmp/hs_err_*
|
||||
echo "JVB not running, previous heap dump not found. Archiving logs only."
|
||||
fi
|
||||
fi
|
||||
91
resources/graceful_shutdown.sh
Executable file
91
resources/graceful_shutdown.sh
Executable file
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# 1. The script issues shutdown command to the bridge over REST API.
|
||||
# If HTTP status code other than 200 is returned then it exits with 1.
|
||||
# 2. If the code is ok then it checks if the bridge has exited.
|
||||
# 3. If not then it polls bridge statistics until participant count drops to 0.
|
||||
# 4. Gives some time for the bridge to shutdown. If it does not quit after that
|
||||
# time then it kills the process. If the process was successfully killed 0 is
|
||||
# returned and 1 otherwise.
|
||||
#
|
||||
# Arguments:
|
||||
# "-h"("http://localhost:8080" by default) REST requests host URI part
|
||||
# "-s"(disabled by default) enable silent mode - no info output
|
||||
#
|
||||
# NOTE: script depends on the tool jq, used to parse json
|
||||
#
|
||||
|
||||
# Initialize arguments
|
||||
hostUrl="http://localhost:8080"
|
||||
verbose=1
|
||||
|
||||
# Parse arguments
|
||||
OPTIND=1
|
||||
while getopts "p:h:t:s" opt; do
|
||||
case "$opt" in
|
||||
h)
|
||||
hostUrl=$OPTARG
|
||||
;;
|
||||
s)
|
||||
verbose=0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift "$((OPTIND-1))"
|
||||
|
||||
# Returns local participant count by calling JVB REST statistics API and extracting
|
||||
# participant count from JSON stats text returned.
|
||||
function getParticipantCount {
|
||||
# Total number of participants minus the remote (octo) participants
|
||||
curl -s "$hostUrl/colibri/stats"| jq '.participants - .octo_endpoints'
|
||||
}
|
||||
|
||||
# Prints info messages
|
||||
function printInfo {
|
||||
if [ "$verbose" == "1" ]
|
||||
then
|
||||
echo "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# Prints errors
|
||||
function printError {
|
||||
echo "$@" 1>&2
|
||||
}
|
||||
|
||||
function waitForPid {
|
||||
while [ -d /proc/$1 ] ;do
|
||||
echo "PID $1 still exists"
|
||||
sleep 10
|
||||
done
|
||||
echo "PID $1 is done"
|
||||
}
|
||||
|
||||
JVB_PID=$(ps aux | grep java | grep jitsi-videobridge.jar | awk '{print $2}')
|
||||
|
||||
shutdownStatus=`curl -s -o /dev/null -H "Content-Type: application/json" -d '{ "graceful-shutdown": "true" }' -w "%{http_code}" "$hostUrl/colibri/shutdown"`
|
||||
if [ "$shutdownStatus" == "200" ]
|
||||
then
|
||||
printInfo "Graceful shutdown started"
|
||||
participantCount=`getParticipantCount`
|
||||
while [[ $participantCount -gt 0 ]] ; do
|
||||
printInfo "There are still $participantCount participants"
|
||||
sleep 10
|
||||
participantCount=`getParticipantCount`
|
||||
if [[ $? -gt 0 ]] ; then
|
||||
printInfo "Failed to get participant count, bridge may be already shutdown, waiting on pid $JVB_PID"
|
||||
waitForPid $JVB_PID
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Waiting for bridge pid $JVB_PID to finish shutting down"
|
||||
waitForPid $JVB_PID
|
||||
printInfo "Bridge shutdown OK"
|
||||
exit 0
|
||||
else
|
||||
printError "Invalid HTTP status for shutdown request: $shutdownStatus"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
5
resources/jenkins/build.sh
Executable file
5
resources/jenkins/build.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
./resources/build.sh
|
||||
51
resources/link-conditioner.sh
Executable file
51
resources/link-conditioner.sh
Executable file
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
|
||||
PLR=${PLR:-'0.1'}
|
||||
DELAY=${DELAY:-100}
|
||||
BW=${BW:-'1Mbit/s'}
|
||||
PORT=${PORT:-10000}
|
||||
EGRESS_PIPE=1
|
||||
INGRESS_PIPE=2
|
||||
|
||||
condition_egress() {
|
||||
# Create an anchor and reload pf conf.
|
||||
(cat /etc/pf.conf && echo "dummynet-anchor \"jitsi-videobridge\"" && echo "anchor \"jitsi-videobridge\"") | sudo pfctl -f -
|
||||
|
||||
# Pipe all UDP traffic from port PORT through the EGRESS_PIPE.
|
||||
echo "dummynet out quick proto udp from any port $PORT to any pipe $EGRESS_PIPE" | sudo pfctl -a jitsi-videobridge -f -
|
||||
|
||||
# Configure the egress pipe.
|
||||
sudo dnctl pipe $EGRESS_PIPE config bw $BW plr $PLR delay $DELAY
|
||||
}
|
||||
|
||||
condition_ingress() {
|
||||
# Create an anchor and reload pf conf.
|
||||
(cat /etc/pf.conf && echo "dummynet-anchor \"jitsi-videobridge\"" && echo "anchor \"jitsi-videobridge\"") | sudo pfctl -f -
|
||||
|
||||
# Pipe all UDP traffic to port PORT through the INGRESS_PIPE.
|
||||
echo "dummynet out quick proto udp from any to any port $PORT pipe $INGRESS_PIPE" | sudo pfctl -a jitsi-videobridge -f -
|
||||
|
||||
# Configure the ingress pipe.
|
||||
sudo dnctl pipe $INGRESS_PIPE config bw $BW plr $PLR delay $DELAY
|
||||
}
|
||||
|
||||
condition_flush() {
|
||||
sudo dnctl -f flush
|
||||
sudo pfctl -f /etc/pf.conf
|
||||
}
|
||||
|
||||
main() {
|
||||
case "$1" in
|
||||
ingress)
|
||||
condition_ingress
|
||||
;;
|
||||
egress)
|
||||
condition_egress
|
||||
;;
|
||||
flush)
|
||||
condition_flush
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
11
resources/mvn-exec.sh
Executable file
11
resources/mvn-exec.sh
Executable file
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh -e
|
||||
|
||||
readonly CONFIG=$1
|
||||
if [ -z ${CONFIG+x} -o ! -f $1 ]; then
|
||||
echo 'Config file missing.'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. $CONFIG
|
||||
|
||||
exec mvn compile exec:exec -Dexec.executable=java -Dexec.args="-cp %classpath ${JAVA_SYS_PROPS} org.jitsi.videobridge.Main ${JVB_OPTS}"
|
||||
203
resources/timeseries-cli.py
Executable file
203
resources/timeseries-cli.py
Executable file
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import argparse
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from pandas.plotting import register_matplotlib_converters
|
||||
register_matplotlib_converters()
|
||||
|
||||
|
||||
def read_json(fname):
|
||||
df = pd.read_json(fname, lines=True, convert_dates=['time'], date_unit='ms')
|
||||
df['conference_id'] = df['conf_name'] + df['conf_creation_time_ms'].astype(str)
|
||||
return df
|
||||
|
||||
|
||||
def show(args):
|
||||
df = read_json(args.infile)
|
||||
print('series: {}'.format(df['series'].unique()))
|
||||
print('conferences: {}'.format(df['conf_name'].unique()))
|
||||
print('endpoints: {}'.format(df['endpoint_id'].unique()))
|
||||
|
||||
|
||||
def vp8_inspect(df):
|
||||
if args.ssrc:
|
||||
df = df[df['rtp.ssrc'] == args.ssrc]
|
||||
if args.endpoint:
|
||||
df = df[df['endpoint_id'] == args.endpoint]
|
||||
if args.conference:
|
||||
df = df[df['conf_name'] == args.conference]
|
||||
|
||||
df['time_delta'] = df['time'] - df['time'].shift(1)
|
||||
df['rtp.seq_delta'] = df['rtp.seq'] - df['rtp.seq'].shift(1)
|
||||
df['vp8.pictureid_delta'] = df['vp8.pictureid'] - df['vp8.pictureid'].shift(1)
|
||||
df['vp8.timestamp_delta'] = df['rtp.timestamp'] - df['rtp.timestamp'].shift(1)
|
||||
print(df)
|
||||
|
||||
|
||||
def vp8_verify(df):
|
||||
# tl0picidx monotonically increases
|
||||
# if the timestamp changes, then the pictureid changes
|
||||
# if the pictureid changes, then the timestamp changes
|
||||
# the timestamp delta needs to be proportional to the pictureid delta
|
||||
# there can't be big gaps in the sequence numbers
|
||||
# the webrtc pacer outputs packets every 10ms
|
||||
pass
|
||||
|
||||
|
||||
def plot_endpoint(df, series, endpoint_id, remote_endpoint_id):
|
||||
df = df[df['endpoint_id'] == endpoint_id]
|
||||
if remote_endpoint_id:
|
||||
df = df[df['remote_endpoint_id'] == remote_endpoint_id]
|
||||
|
||||
fig = plt.figure()
|
||||
fig.suptitle('Endpoint: {}'.format(endpoint_id))
|
||||
ax_bitrate = fig.subplots(1, sharex=True)
|
||||
ax_bitrate.set_xlabel('time')
|
||||
ax_bitrate.set_ylabel('bitrate (bps)')
|
||||
|
||||
if 'calculated_rate' in series:
|
||||
df_rates = df['series' == 'calculated_rate']
|
||||
|
||||
# make sure we're plotting a single remote endpoint.
|
||||
remote_endpoints = df_rates['remote_endpoint_id'].unique()
|
||||
if len(remote_endpoints) > 1:
|
||||
raise Exception('specify a --remote-endpoint {}'.format(remote_endpoints))
|
||||
|
||||
for i in range(9):
|
||||
encoding_quality = str(i)
|
||||
if encoding_quality in df_rates.columns:
|
||||
ax_bitrate.plot(df_rates['time'],
|
||||
df_rates[encoding_quality],
|
||||
label=encoding_quality)
|
||||
|
||||
df_series = df['series'].unique()
|
||||
if 'sent_padding' in series and 'sent_padding' in df_series:
|
||||
df_padding = df[df['series'] == 'sent_padding']
|
||||
ax_bitrate.plot(
|
||||
df_padding['time'],
|
||||
df_padding['padding_bps'] + df_padding['total_target_bps'],
|
||||
label='target + padding',
|
||||
marker='^',
|
||||
markersize=4,
|
||||
linestyle='None')
|
||||
|
||||
if 'new_bwe' in series and 'new_bwe' in df_series:
|
||||
df_bwe = df[df['series'] == 'new_bwe']
|
||||
ax_bitrate.plot(df_bwe['time'],
|
||||
df_bwe['bitrate_bps'],
|
||||
label='estimate',
|
||||
drawstyle='steps-post')
|
||||
|
||||
if 'did_update' in series and 'did_update' in df_series:
|
||||
df_update = df[df['series'] == 'did_update']
|
||||
ax_bitrate.plot(df_update['time'],
|
||||
df_update['total_target_bps'],
|
||||
label='target',
|
||||
drawstyle='steps-post')
|
||||
ax_bitrate.plot(df_update['time'],
|
||||
df_update['total_ideal_bps'],
|
||||
label='ideal',
|
||||
drawstyle='steps-post')
|
||||
|
||||
if 'in_pkt' in series and 'in_pkt' in df_series:
|
||||
df_pkt = df[df['series'] == 'in_pkt']
|
||||
|
||||
if len(df_pkt['rbe_id'].unique()) is not 1:
|
||||
raise Exception('There cannot be multiple remote bitrate estimators')
|
||||
|
||||
df_sz = pd.DataFrame(
|
||||
{'bits': df_pkt['pkt_sz_bytes'] * 8, 'time': df_pkt['time']})
|
||||
|
||||
df_rate = df_sz.resample('1S', on='time').sum()
|
||||
ax_bitrate.plot(df_rate.index, df_rate['bits'], label='sent')
|
||||
|
||||
# todo include rtt and packet loss
|
||||
|
||||
ax_bitrate.legend()
|
||||
|
||||
|
||||
def plot(args):
|
||||
df = read_json(args.infile)
|
||||
|
||||
endpoint_ids = [args.endpoint_id] if args.endpoint_id else df['endpoint_id'].unique()
|
||||
|
||||
for endpoint_id in endpoint_ids:
|
||||
plot_endpoint(df, args.series, endpoint_id, args.remote_endpoint_id)
|
||||
|
||||
plt.show()
|
||||
|
||||
|
||||
def check_endpoint(df, endpoint_id):
|
||||
|
||||
# if a property is violated (for example property 3):
|
||||
# zgrep 7d2b7ecf data/4/series.json.gz| grep did_update | jq '. | select(.bwe_bps >= .total_ideal_bps) | select(.total_target_idx < .total_ideal_idx)' | less -S
|
||||
|
||||
df_update = df[df['series'] == 'did_update']
|
||||
if not len(df_update):
|
||||
return
|
||||
|
||||
# property 1: check if ideal <= estimate => target == ideal
|
||||
mask_1 = df_update['bwe_bps'] >= df_update['total_ideal_bps']
|
||||
mask_2 = df_update['total_target_idx'] < df_update['total_ideal_idx']
|
||||
mask_3 = df_update['total_ideal_bps'] != 0
|
||||
mask_4 = df_update['total_target_bps'] != 0
|
||||
assert not len(df_update[mask_1 & mask_2 & mask_3 & mask_4]), '{} is not sending as much as it should.'.format(endpoint_id)
|
||||
|
||||
# property 2: check that the target never exceeds the ideal
|
||||
mask_5 = (df_update['total_ideal_bps'] - df_update['total_target_bps']) < 0
|
||||
assert not len(df_update[mask_5]), '{} has a target bitrate that exceeds the ideal bitrate.'.format(endpoint_id)
|
||||
|
||||
# property 3: make sure that we are calculating bitrate for encodings
|
||||
# disabled for now as it's a legitimate case, being video muted, for example
|
||||
#mask_6 = df_update['total_ideal_bps'] > 0
|
||||
#assert len(df_update[mask_6]), '{} never computed an ideal bitrate.'.format(endpoint_id)
|
||||
|
||||
#mask_7 = df_update['total_target_bps'] > 0
|
||||
#assert len(df_update[mask_7]), '{} never computed a target bitrate.'.format(endpoint_id)
|
||||
|
||||
# property 4: check that the target never exceeds the estimate
|
||||
# disabled for now as can happen and still be correct because we never suspend the on-stage participant
|
||||
# mask_8 = (df_update['bwe_bps'] - df_update['total_target_bps']) < 0
|
||||
# assert not len(df_update[mask_8]), '{} has a target bitrate that exceeds the bandwidth estimation.'.format(endpoint_id)
|
||||
|
||||
|
||||
def check_conference(df, conference_id):
|
||||
endpoint_ids = [args.endpoint_id] if args.endpoint_id else df['endpoint_id'].unique()
|
||||
|
||||
for endpoint_id in endpoint_ids:
|
||||
check_endpoint(df[df['endpoint_id'] == endpoint_id], endpoint_id)
|
||||
|
||||
|
||||
def check(args):
|
||||
df = read_json(args.infile)
|
||||
conference_ids = [args.conference_id] if args.conference_id else df['conference_id'].unique()
|
||||
|
||||
for conference_id in conference_ids:
|
||||
check_conference(df[df['conference_id'] == conference_id], conference_id)
|
||||
|
||||
|
||||
if "__main__" == __name__:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('infile', type=argparse.FileType('r'))
|
||||
subparsers = parser.add_subparsers()
|
||||
|
||||
parser_show = subparsers.add_parser('show')
|
||||
parser_show.set_defaults(func=show)
|
||||
|
||||
parser_plot = subparsers.add_parser('plot')
|
||||
parser_plot.add_argument(
|
||||
'--series', nargs='+', choices=[
|
||||
'did_update', 'new_bwe', 'sent_padding', 'calculated_rate'],
|
||||
default='did_update new_bwe sent_padding in_pkt')
|
||||
parser_plot.add_argument('--endpoint-id')
|
||||
parser_plot.add_argument('--remote-endpoint-id')
|
||||
parser_plot.set_defaults(func=plot)
|
||||
|
||||
parser_check = subparsers.add_parser('check')
|
||||
parser_check.add_argument('--endpoint-id')
|
||||
parser_check.add_argument('--conference-id')
|
||||
parser_check.set_defaults(func=check)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
Reference in New Issue
Block a user