snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)
Some checks failed
Release to Docker / docker (push) Failing after 2s
Release Test Server to Docker / docker (push) Failing after 3s
Test / test (push) Failing after 16s

This commit is contained in:
upstream-mirror
2026-09-15 16:14:22 +05:00
commit 78704103fc
422 changed files with 154805 additions and 0 deletions

View File

@@ -0,0 +1,118 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry
import (
"context"
"go.uber.org/atomic"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/observability/roomobs"
"github.com/livekit/protocol/rpc"
"github.com/livekit/protocol/utils/guid"
"github.com/livekit/livekit-server/pkg/config"
"github.com/livekit/livekit-server/pkg/routing"
)
//counterfeiter:generate . AnalyticsService
type AnalyticsService interface {
SendStats(ctx context.Context, stats []*livekit.AnalyticsStat)
SendEvent(ctx context.Context, events *livekit.AnalyticsEvent)
SendNodeRoomStates(ctx context.Context, nodeRooms *livekit.AnalyticsNodeRooms)
RoomProjectReporter(ctx context.Context) roomobs.ProjectReporter
}
// ----------------------------
var _ AnalyticsService = &NullAnalyticService{}
type NullAnalyticService struct{}
func (n NullAnalyticService) SendStats(_ context.Context, _ []*livekit.AnalyticsStat) {}
func (n NullAnalyticService) SendEvent(_ context.Context, _ *livekit.AnalyticsEvent) {}
func (n NullAnalyticService) SendNodeRoomStates(_ context.Context, _ *livekit.AnalyticsNodeRooms) {}
func (n NullAnalyticService) RoomProjectReporter(_ctx context.Context) roomobs.ProjectReporter {
return nil
}
// ----------------------------
type analyticsService struct {
analyticsKey string
nodeID string
sequenceNumber atomic.Uint64
events rpc.AnalyticsRecorderService_IngestEventsClient
stats rpc.AnalyticsRecorderService_IngestStatsClient
nodeRooms rpc.AnalyticsRecorderService_IngestNodeRoomStatesClient
}
func NewAnalyticsService(_ *config.Config, currentNode routing.LocalNode) AnalyticsService {
return &analyticsService{
analyticsKey: "", // TODO: conf.AnalyticsKey
nodeID: string(currentNode.NodeID()),
}
}
func (a *analyticsService) SendStats(_ context.Context, stats []*livekit.AnalyticsStat) {
if a.stats == nil {
return
}
for _, stat := range stats {
stat.Id = guid.New("AS_")
stat.AnalyticsKey = a.analyticsKey
stat.Node = a.nodeID
}
if err := a.stats.Send(&livekit.AnalyticsStats{Stats: stats}); err != nil {
logger.Errorw("failed to send stats", err)
}
}
func (a *analyticsService) SendEvent(_ context.Context, event *livekit.AnalyticsEvent) {
if a.events == nil {
return
}
event.Id = guid.New("AE_")
event.NodeId = a.nodeID
event.AnalyticsKey = a.analyticsKey
if err := a.events.Send(&livekit.AnalyticsEvents{
Events: []*livekit.AnalyticsEvent{event},
}); err != nil {
logger.Errorw("failed to send event", err, "eventType", event.Type.String())
}
}
func (a *analyticsService) SendNodeRoomStates(_ context.Context, nodeRooms *livekit.AnalyticsNodeRooms) {
if a.nodeRooms == nil {
return
}
nodeRooms.NodeId = a.nodeID
nodeRooms.SequenceNumber = a.sequenceNumber.Add(1)
nodeRooms.Timestamp = timestamppb.Now()
if err := a.nodeRooms.Send(nodeRooms); err != nil {
logger.Errorw("failed to send node room states", err)
}
}
func (a *analyticsService) RoomProjectReporter(_ context.Context) roomobs.ProjectReporter {
return roomobs.NewNoopProjectReporter()
}

659
pkg/telemetry/events.go Normal file
View File

@@ -0,0 +1,659 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry
import (
"context"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/egress"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/guid"
"github.com/livekit/protocol/webhook"
)
func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent, opts ...webhook.NotifyOption) {
if t.notifier == nil {
return
}
event.CreatedAt = time.Now().Unix()
event.Id = guid.New("EV_")
if err := t.notifier.QueueNotify(ctx, event, opts...); err != nil {
logger.Warnw("failed to notify webhook", err, "event", event.Event)
}
}
func (t *telemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {
t.enqueue(func() {
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventRoomStarted,
Room: room,
})
t.SendEvent(ctx, &livekit.AnalyticsEvent{
Type: livekit.AnalyticsEventType_ROOM_CREATED,
Timestamp: &timestamppb.Timestamp{Seconds: room.CreationTime},
Room: room,
})
})
}
func (t *telemetryService) RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) {
t.enqueue(func() {
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventRoomFinished,
Room: room,
RoomEndReason: reason,
})
t.SendEvent(ctx, &livekit.AnalyticsEvent{
Type: livekit.AnalyticsEventType_ROOM_ENDED,
Timestamp: timestamppb.Now(),
RoomId: room.Sid,
Room: room,
RoomEndReason: reason,
})
})
}
func (t *telemetryService) ParticipantJoined(
ctx context.Context,
room *livekit.Room,
participant *livekit.ParticipantInfo,
clientInfo *livekit.ClientInfo,
clientMeta *livekit.AnalyticsClientMeta,
shouldSendEvent bool,
guard *ReferenceGuard,
) {
t.enqueue(func() {
_, found := t.getOrCreateWorker(
ctx,
livekit.RoomID(room.Sid),
livekit.RoomName(room.Name),
livekit.ParticipantID(participant.Sid),
livekit.ParticipantIdentity(participant.Identity),
guard,
)
if !found {
prometheus.IncrementParticipantRtcConnected(1)
prometheus.AddParticipant()
}
if shouldSendEvent {
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_JOINED, room, participant)
ev.ClientInfo = clientInfo
ev.ClientMeta = clientMeta
t.SendEvent(ctx, ev)
}
})
}
func (t *telemetryService) ParticipantActive(
ctx context.Context,
room *livekit.Room,
participant *livekit.ParticipantInfo,
clientMeta *livekit.AnalyticsClientMeta,
isMigration bool,
isWarp bool,
guard *ReferenceGuard,
) {
t.enqueue(func() {
if !isMigration {
// a participant is considered "joined" only when they become "active"
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventParticipantJoined,
Room: room,
Participant: participant,
})
}
worker, found := t.getOrCreateWorker(
ctx,
livekit.RoomID(room.Sid),
livekit.RoomName(room.Name),
livekit.ParticipantID(participant.Sid),
livekit.ParticipantIdentity(participant.Identity),
guard,
)
if !found {
prometheus.AddParticipant()
}
worker.SetConnected()
prometheus.IncrementParticipantRtcActive(1, isWarp)
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, room, participant)
ev.ClientMeta = clientMeta
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) ParticipantResumed(
ctx context.Context,
room *livekit.Room,
participant *livekit.ParticipantInfo,
nodeID livekit.NodeID,
reason livekit.ReconnectReason,
) {
t.enqueue(func() {
// create a worker if needed.
//
// Signalling channel stats collector and media channel stats collector could both call
// ParticipantJoined and ParticipantLeft.
//
// On a resume, the signalling channel collector would call `ParticipantLeft` which would close
// the corresponding participant's stats worker.
//
// So, on a successful resume, create the worker if needed.
_, found := t.getOrCreateWorker(
ctx,
livekit.RoomID(room.Sid),
livekit.RoomName(room.Name),
livekit.ParticipantID(participant.Sid),
livekit.ParticipantIdentity(participant.Identity),
nil,
)
if !found {
prometheus.AddParticipant()
}
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_RESUMED, room, participant)
ev.ClientMeta = &livekit.AnalyticsClientMeta{
Node: string(nodeID),
ReconnectReason: reason,
}
t.SendEvent(ctx, ev)
})
}
// RoomIDChanged re-keys the room's stats workers.
//
// NOTE: this shares the queue with the stats and participant events it races with, so
// ops raised before the id changed (carrying `prevRoomID`) are applied before the
// re-key and ops raised after it (carrying the new id) are applied after. Callers
// should raise this as soon as the room starts reporting the new id.
func (t *telemetryService) RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) {
t.enqueue(func() {
t.reKeyRoom(prevRoomID, livekit.RoomID(room.Sid), livekit.RoomName(room.Name))
})
}
func (t *telemetryService) ParticipantLeft(ctx context.Context,
room *livekit.Room,
participant *livekit.ParticipantInfo,
shouldSendEvent bool,
guard *ReferenceGuard,
) {
t.enqueue(func() {
isConnected := false
if worker, ok := t.getWorker(livekit.RoomID(room.Sid), livekit.ParticipantID(participant.Sid)); ok {
isConnected = worker.IsConnected()
if worker.Close(guard) {
prometheus.SubParticipant()
} else {
logger.Infow(
"stats worker active",
"room", room.Name,
"roomID", room.Sid,
"participant", participant.Identity,
"participantID", participant.Sid,
"worker", worker,
"guard", guard,
)
}
}
if shouldSendEvent {
webhookEvent := webhook.EventParticipantLeft
analyticsEvent := livekit.AnalyticsEventType_PARTICIPANT_LEFT
if !isConnected {
webhookEvent = webhook.EventParticipantConnectionAborted
analyticsEvent = livekit.AnalyticsEventType_PARTICIPANT_CONNECTION_ABORTED
}
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhookEvent,
Room: room,
Participant: participant,
})
t.SendEvent(ctx, newParticipantEvent(analyticsEvent, room, participant))
}
})
}
func (t *telemetryService) TrackPublishRequested(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
identity livekit.ParticipantIdentity,
track *livekit.TrackInfo,
shouldSendEvent bool,
) {
t.enqueue(func() {
prometheus.RecordTrackPublishAttempt(track.Type.String())
if !shouldSendEvent {
return
}
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_REQUESTED, room, participantID, track)
if ev.Participant != nil {
ev.Participant.Identity = string(identity)
}
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackPublished(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
identity livekit.ParticipantIdentity,
track *livekit.TrackInfo,
shouldSendEvent bool,
) {
t.enqueue(func() {
prometheus.AddPublishedTrack(track.Type.String())
prometheus.RecordTrackPublishSuccess(track.Type.String())
if !shouldSendEvent {
return
}
participant := &livekit.ParticipantInfo{
Sid: string(participantID),
Identity: string(identity),
}
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventTrackPublished,
Room: room,
Participant: participant,
Track: track,
})
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED, room, participantID, track)
ev.Participant = participant
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackPublishedUpdate(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
) {
t.enqueue(func() {
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, room, participantID, track))
})
}
func (t *telemetryService) TrackMaxSubscribedVideoQuality(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
mime mime.MimeType,
maxQuality livekit.VideoQuality,
) {
t.enqueue(func() {
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_MAX_SUBSCRIBED_VIDEO_QUALITY, room, participantID, track)
ev.MaxSubscribedVideoQuality = maxQuality
ev.Mime = mime.String()
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackSubscribeRequested(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
) {
t.enqueue(func() {
prometheus.RecordTrackSubscribeAttempt()
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_REQUESTED, room, participantID, track)
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackSubscribed(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
publisher *livekit.ParticipantInfo,
shouldSendEvent bool,
) {
t.enqueue(func() {
prometheus.RecordTrackSubscribeSuccess(track.Type.String())
if !shouldSendEvent {
return
}
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBED, room, participantID, track)
ev.Publisher = publisher
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackSubscribeFailed(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
trackID livekit.TrackID,
err error,
isUserError bool,
) {
t.enqueue(func() {
prometheus.RecordTrackSubscribeFailure(err, isUserError)
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_FAILED, room, participantID, &livekit.TrackInfo{
Sid: string(trackID),
})
ev.Error = err.Error()
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackUnsubscribed(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
shouldSendEvent bool,
) {
t.enqueue(func() {
prometheus.RecordTrackUnsubscribed(track.Type.String())
if shouldSendEvent {
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNSUBSCRIBED, room, participantID, track))
}
})
}
func (t *telemetryService) TrackUnpublished(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
identity livekit.ParticipantIdentity,
track *livekit.TrackInfo,
wasPublishedLocally bool,
shouldSendEvent bool,
) {
t.enqueue(func() {
if wasPublishedLocally {
prometheus.SubPublishedTrack(track.Type.String())
}
if !shouldSendEvent {
return
}
participant := &livekit.ParticipantInfo{
Sid: string(participantID),
Identity: string(identity),
}
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventTrackUnpublished,
Room: room,
Participant: participant,
Track: track,
})
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNPUBLISHED, room, participantID, track))
})
}
func (t *telemetryService) TrackMuted(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
) {
t.enqueue(func() {
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_MUTED, room, participantID, track))
})
}
func (t *telemetryService) TrackUnmuted(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
track *livekit.TrackInfo,
) {
t.enqueue(func() {
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNMUTED, room, participantID, track))
})
}
func (t *telemetryService) TrackPublishRTPStats(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
trackID livekit.TrackID,
mimeType mime.MimeType,
layer int,
stats *livekit.RTPStats,
) {
t.enqueue(func() {
ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_STATS, toMinimalRoom(room))
ev.ParticipantId = string(participantID)
ev.TrackId = string(trackID)
ev.Mime = mimeType.String()
ev.VideoLayer = int32(layer)
ev.RtpStats = stats
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) TrackSubscribeRTPStats(
ctx context.Context,
room *livekit.Room,
participantID livekit.ParticipantID,
trackID livekit.TrackID,
mimeType mime.MimeType,
stats *livekit.RTPStats,
) {
t.enqueue(func() {
ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_STATS, toMinimalRoom(room))
ev.ParticipantId = string(participantID)
ev.TrackId = string(trackID)
ev.Mime = mimeType.String()
ev.RtpStats = stats
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) {
opts := egress.GetEgressNotifyOptions(info)
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: event,
Room: toMinimalRoomProto(livekit.RoomID(info.RoomId), livekit.RoomName(info.RoomName)),
EgressInfo: info,
}, opts...)
}
func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {
t.enqueue(func() {
t.NotifyEgressEvent(ctx, webhook.EventEgressStarted, info)
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_STARTED, info))
})
}
func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {
t.enqueue(func() {
t.NotifyEgressEvent(ctx, webhook.EventEgressUpdated, info)
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info))
})
}
func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {
t.enqueue(func() {
t.NotifyEgressEvent(ctx, webhook.EventEgressEnded, info)
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_ENDED, info))
})
}
func (t *telemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {
t.enqueue(func() {
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_CREATED, info))
})
}
func (t *telemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {
t.enqueue(func() {
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_DELETED, info))
})
}
func (t *telemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {
t.enqueue(func() {
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventIngressStarted,
Room: toMinimalRoomProto("", livekit.RoomName(info.RoomName)),
IngressInfo: info,
})
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_STARTED, info))
})
}
func (t *telemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {
t.enqueue(func() {
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_UPDATED, info))
})
}
func (t *telemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {
t.enqueue(func() {
t.NotifyEvent(ctx, &livekit.WebhookEvent{
Event: webhook.EventIngressEnded,
Room: toMinimalRoomProto("", livekit.RoomName(info.RoomName)),
IngressInfo: info,
})
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_ENDED, info))
})
}
func (t *telemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {
t.enqueue(func() {
ev := &livekit.AnalyticsEvent{
Type: livekit.AnalyticsEventType_REPORT,
Timestamp: timestamppb.Now(),
Report: reportInfo,
}
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {
t.enqueue(func() {
ev := &livekit.AnalyticsEvent{
Type: livekit.AnalyticsEventType_API_CALL,
Timestamp: timestamppb.Now(),
ApiCall: apiCallInfo,
}
t.SendEvent(ctx, ev)
})
}
func (t *telemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {
t.enqueue(func() {
ev := &livekit.AnalyticsEvent{
Type: livekit.AnalyticsEventType_WEBHOOK,
Timestamp: timestamppb.Now(),
Webhook: webhookInfo,
}
t.SendEvent(ctx, ev)
})
}
func newRoomEvent(event livekit.AnalyticsEventType, room *livekit.Room) *livekit.AnalyticsEvent {
ev := &livekit.AnalyticsEvent{
Type: event,
Timestamp: timestamppb.Now(),
}
if room != nil {
ev.Room = room
ev.RoomId = room.Sid
}
return ev
}
func newParticipantEvent(event livekit.AnalyticsEventType, room *livekit.Room, participant *livekit.ParticipantInfo) *livekit.AnalyticsEvent {
ev := newRoomEvent(event, room)
if participant != nil {
ev.ParticipantId = participant.Sid
ev.Participant = participant
}
return ev
}
func newTrackEvent(event livekit.AnalyticsEventType, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) *livekit.AnalyticsEvent {
ev := newParticipantEvent(event, toMinimalRoom(room), &livekit.ParticipantInfo{
Sid: string(participantID),
})
if track != nil {
ev.TrackId = track.Sid
ev.Track = track
}
return ev
}
func newEgressEvent(event livekit.AnalyticsEventType, egress *livekit.EgressInfo) *livekit.AnalyticsEvent {
return &livekit.AnalyticsEvent{
Type: event,
Timestamp: timestamppb.Now(),
EgressId: egress.EgressId,
RoomId: egress.RoomId,
Egress: egress,
}
}
func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressInfo) *livekit.AnalyticsEvent {
return &livekit.AnalyticsEvent{
Type: event,
Timestamp: timestamppb.Now(),
IngressId: ingress.IngressId,
Ingress: ingress,
}
}
func toMinimalRoom(room *livekit.Room) *livekit.Room {
if room == nil {
return nil
}
return toMinimalRoomProto(livekit.RoomID(room.Sid), livekit.RoomName(room.Name))
}
func toMinimalRoomProto(roomID livekit.RoomID, roomName livekit.RoomName) *livekit.Room {
return &livekit.Room{
Sid: string(roomID),
Name: string(roomName),
}
}

View File

@@ -0,0 +1,260 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/telemetry"
"github.com/livekit/protocol/livekit"
)
func Test_OnParticipantJoin_EventIsSent(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := "part1"
clientInfo := &livekit.ClientInfo{
Sdk: 2,
Version: "v1",
Os: "mac",
OsVersion: "v1",
DeviceModel: "DM1",
Browser: "chrome",
BrowserVersion: "97.0.1",
}
clientMeta := &livekit.AnalyticsClientMeta{
Region: "dark-side",
Node: "moon",
ClientAddr: "127.0.0.1",
ClientConnectTime: 420,
}
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
guard := &telemetry.ReferenceGuard{}
// do
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
time.Sleep(time.Millisecond * 500)
// test
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
_, event := fixture.analytics.SendEventArgsForCall(0)
require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_JOINED, event.Type)
require.Equal(t, partSID, event.ParticipantId)
require.Equal(t, participantInfo, event.Participant)
require.Equal(t, room.Sid, event.RoomId)
require.Equal(t, room, event.Room)
require.Equal(t, clientInfo.Sdk, event.ClientInfo.Sdk)
require.Equal(t, clientInfo.Version, event.ClientInfo.Version)
require.Equal(t, clientInfo.Os, event.ClientInfo.Os)
require.Equal(t, clientInfo.OsVersion, event.ClientInfo.OsVersion)
require.Equal(t, clientInfo.DeviceModel, event.ClientInfo.DeviceModel)
require.Equal(t, clientInfo.Browser, event.ClientInfo.Browser)
require.Equal(t, clientInfo.BrowserVersion, event.ClientInfo.BrowserVersion)
require.Equal(t, clientMeta.Region, event.ClientMeta.Region)
require.Equal(t, clientMeta.Node, event.ClientMeta.Node)
require.Equal(t, clientMeta.ClientAddr, event.ClientMeta.ClientAddr)
require.Equal(t, clientMeta.ClientConnectTime, event.ClientMeta.ClientConnectTime)
}
func Test_OnParticipantLeft_EventIsSent(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := "part1"
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
guard := &telemetry.ReferenceGuard{}
// do
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false, false, guard)
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard)
time.Sleep(time.Millisecond * 500)
// test
require.Equal(t, 2, fixture.analytics.SendEventCallCount())
_, event := fixture.analytics.SendEventArgsForCall(1)
require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_LEFT, event.Type)
require.Equal(t, partSID, event.ParticipantId)
require.Equal(t, room.Sid, event.RoomId)
require.Equal(t, room, event.Room)
}
func Test_OnTrackUpdate_EventIsSent(t *testing.T) {
fixture := createFixture()
// prepare
roomID := "room1"
roomName := "RoomName"
partID := "part1"
trackID := "track1"
layer := &livekit.VideoLayer{
Quality: livekit.VideoQuality_HIGH,
Width: uint32(360),
Height: uint32(720),
Bitrate: 2048,
}
trackInfo := &livekit.TrackInfo{
Sid: trackID,
Type: livekit.TrackType_VIDEO,
Muted: false,
Simulcast: false,
DisableDtx: false,
Layers: []*livekit.VideoLayer{layer},
}
// do
fixture.sut.TrackPublishedUpdate(context.Background(), &livekit.Room{Sid: roomID, Name: roomName}, livekit.ParticipantID(partID), trackInfo)
time.Sleep(time.Millisecond * 500)
// test
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
_, event := fixture.analytics.SendEventArgsForCall(0)
require.Equal(t, livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, event.Type)
require.Equal(t, partID, event.ParticipantId)
require.Equal(t, trackID, event.Track.Sid)
require.NotNil(t, event.Track.Layers)
require.Equal(t, layer.Width, event.Track.Layers[0].Width)
require.Equal(t, layer.Height, event.Track.Layers[0].Height)
require.Equal(t, layer.Quality, event.Track.Layers[0].Quality)
}
func Test_OnParticipantActive_EventIsSent(t *testing.T) {
fixture := createFixture()
// prepare participant to change status
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := "part1"
clientInfo := &livekit.ClientInfo{
Sdk: 2,
Version: "v1",
Os: "mac",
OsVersion: "v1",
DeviceModel: "DM1",
Browser: "chrome",
BrowserVersion: "97.0.1",
}
clientMeta := &livekit.AnalyticsClientMeta{
Region: "dark-side",
Node: "moon",
ClientAddr: "127.0.0.1",
}
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
guard := &telemetry.ReferenceGuard{}
// do
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
time.Sleep(time.Millisecond * 500)
// test
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
_, event := fixture.analytics.SendEventArgsForCall(0)
// test
// do
clientMetaConnect := &livekit.AnalyticsClientMeta{
ClientConnectTime: 420,
}
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false, false, guard)
time.Sleep(time.Millisecond * 500)
require.Equal(t, 2, fixture.analytics.SendEventCallCount())
_, eventActive := fixture.analytics.SendEventArgsForCall(1)
require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, eventActive.Type)
require.Equal(t, partSID, eventActive.ParticipantId)
require.Equal(t, room.Sid, eventActive.RoomId)
require.Equal(t, room, event.Room)
require.Equal(t, clientMetaConnect.ClientConnectTime, eventActive.ClientMeta.ClientConnectTime)
}
func Test_OnTrackSubscribed_EventIsSent(t *testing.T) {
fixture := createFixture()
// prepare participant to change status
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := "part1"
publisherInfo := &livekit.ParticipantInfo{Sid: "pub1", Identity: "publisher"}
trackInfo := &livekit.TrackInfo{Sid: "tr1", Type: livekit.TrackType_VIDEO}
clientInfo := &livekit.ClientInfo{
Sdk: 2,
Version: "v1",
Os: "mac",
OsVersion: "v1",
DeviceModel: "DM1",
Browser: "chrome",
BrowserVersion: "97.0.1",
}
clientMeta := &livekit.AnalyticsClientMeta{
Region: "dark-side",
Node: "moon",
ClientAddr: "127.0.0.1",
}
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
guard := &telemetry.ReferenceGuard{}
// do
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
time.Sleep(time.Millisecond * 500)
// test
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
_, event := fixture.analytics.SendEventArgsForCall(0)
require.Equal(t, room, event.Room)
// do
fixture.sut.TrackSubscribed(context.Background(), room, livekit.ParticipantID(partSID), trackInfo, publisherInfo, true)
time.Sleep(time.Millisecond * 500)
require.Eventually(t, func() bool {
return fixture.analytics.SendEventCallCount() == 2
}, time.Second, time.Millisecond*50, "expected send event to be called twice")
_, eventTrackSubscribed := fixture.analytics.SendEventArgsForCall(1)
require.Equal(t, livekit.AnalyticsEventType_TRACK_SUBSCRIBED, eventTrackSubscribed.Type)
require.Equal(t, partSID, eventTrackSubscribed.ParticipantId)
require.Equal(t, trackInfo.Sid, eventTrackSubscribed.Track.Sid)
require.Equal(t, trackInfo.Type, eventTrackSubscribed.Track.Type)
require.Equal(t, publisherInfo.Sid, eventTrackSubscribed.Publisher.Sid)
require.Equal(t, publisherInfo.Identity, eventTrackSubscribed.Publisher.Identity)
}
func Test_OnRoomEnded_ReasonIsSent(t *testing.T) {
fixture := createFixture()
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
fixture.sut.RoomEnded(context.Background(), room, livekit.RoomEndReason_ROOM_END_API_DELETE)
fixture.flush()
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
_, event := fixture.analytics.SendEventArgsForCall(0)
require.Equal(t, livekit.AnalyticsEventType_ROOM_ENDED, event.Type)
require.Equal(t, room.Sid, event.RoomId)
require.Equal(t, livekit.RoomEndReason_ROOM_END_API_DELETE, event.RoomEndReason)
}

View File

@@ -0,0 +1,75 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"slices"
"strings"
"github.com/prometheus/client_golang/prometheus"
"github.com/livekit/protocol/livekit"
)
var (
promDataPacketStreamLabels = []string{"type", "mime_type"}
promDataPacketStreamMimeTypes = []string{"text", "image", "application", "audio", "video"}
promDataPacketStreamDestCount *prometheus.HistogramVec
promDataPacketStreamSize *prometheus.HistogramVec
)
func initDataPacketStats(nodeID string, nodeType livekit.NodeType) {
promDataPacketStreamDestCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "datapacket_stream",
Name: "dest_count",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{1, 2, 3, 4, 5, 10, 15, 25, 50},
}, promDataPacketStreamLabels)
promDataPacketStreamSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "datapacket_stream",
Name: "bytes",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{128, 512, 2048, 8192, 32768, 131072, 524288, 2097152, 8388608, 33554432},
}, promDataPacketStreamLabels)
prometheus.MustRegister(promDataPacketStreamDestCount)
prometheus.MustRegister(promDataPacketStreamSize)
}
func RecordDataPacketStream(h *livekit.DataStream_Header, destCount int) {
streamType := "unknown"
switch h.ContentHeader.(type) {
case *livekit.DataStream_Header_TextHeader:
streamType = "text"
case *livekit.DataStream_Header_ByteHeader:
streamType = "bytes"
}
mimeType := strings.ToLower(h.MimeType)
if i := strings.IndexByte(mimeType, '/'); i != -1 {
mimeType = mimeType[:i]
}
if !slices.Contains(promDataPacketStreamMimeTypes, mimeType) {
mimeType = "unknown"
}
promDataPacketStreamDestCount.WithLabelValues(streamType, mimeType).Observe(float64(destCount))
if h.TotalLength != nil {
promDataPacketStreamSize.WithLabelValues(streamType, mimeType).Observe(float64(*h.TotalLength))
}
}

View File

@@ -0,0 +1,40 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/livekit/protocol/livekit"
)
var (
refCounts *prometheus.GaugeVec
)
func initDebugStats(nodeID string, nodeType livekit.NodeType) {
refCounts = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "debug",
Name: "ref_count",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"referrer"})
prometheus.MustRegister(refCounts)
}
func AddRef(referrer string, n int) {
refCounts.WithLabelValues(referrer).Add(float64(n))
}

View File

@@ -0,0 +1,310 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/twitchtv/twirp"
"go.uber.org/atomic"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/rpc"
"github.com/livekit/protocol/utils/hwstats"
"github.com/livekit/protocol/webhook"
)
const (
livekitNamespace string = "livekit"
)
var (
initialized atomic.Bool
promMessageCounter *prometheus.CounterVec
promServiceOperationCounter *prometheus.CounterVec
promTwirpRequestStatusCounter *prometheus.CounterVec
promTwirpRequestLatency *prometheus.HistogramVec
sysPacketsStart uint32
sysDroppedPacketsStart uint32
promSysPacketGauge *prometheus.GaugeVec
cpuStats *hwstats.CPUStats
memoryStats *hwstats.MemoryStats
)
func Init(nodeID string, nodeType livekit.NodeType) error {
if initialized.Swap(true) {
return nil
}
promMessageCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "node",
Name: "messages",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
},
[]string{"type", "status", "direction"},
)
promServiceOperationCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "node",
Name: "service_operation",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
},
[]string{"type", "status", "error_type"},
)
promTwirpRequestStatusCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "node",
Name: "twirp_request_status",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
},
[]string{"service", "method", "status", "code"},
)
promTwirpRequestLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "node",
Name: "twirp_request_latency_ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000},
},
[]string{"service", "method", "status"},
)
promSysPacketGauge = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "node",
Name: "packet_total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Help: "System level packet count. Count starts at 0 when service is first started.",
},
[]string{"type"},
)
prometheus.MustRegister(promMessageCounter)
prometheus.MustRegister(promServiceOperationCounter)
prometheus.MustRegister(promTwirpRequestStatusCounter)
prometheus.MustRegister(promTwirpRequestLatency)
prometheus.MustRegister(promSysPacketGauge)
sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats()
initPacketStats(nodeID, nodeType)
initRoomStats(nodeID, nodeType)
rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
webhook.InitWebhookStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
initQualityStats(nodeID, nodeType)
initDataPacketStats(nodeID, nodeType)
initDebugStats(nodeID, nodeType)
var err error
cpuStats, err = hwstats.NewCPUStats(nil)
if err != nil {
return err
}
memoryStats, err = hwstats.NewMemoryStats()
if err != nil {
return err
}
return nil
}
func GetNodeStats(nodeStartedAt int64, prevStats []*livekit.NodeStats, rateIntervals []time.Duration) (*livekit.NodeStats, error) {
loadAvg, err := getLoadAvg()
if err != nil {
return nil, err
}
// On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin
// So, do not error out. Use the information if it is available.
memUsed, memTotal, _ := memoryStats.GetMemory()
sysPackets, sysDroppedPackets, _ := getTCStats()
promSysPacketGauge.WithLabelValues("out").Set(float64(sysPackets - sysPacketsStart))
promSysPacketGauge.WithLabelValues("dropped").Set(float64(sysDroppedPackets - sysDroppedPacketsStart))
stats := &livekit.NodeStats{
StartedAt: nodeStartedAt,
UpdatedAt: time.Now().Unix(),
NumRooms: roomCurrent.Load(),
NumClients: participantCurrent.Load(),
NumTracksIn: trackPublishedCurrent.Load(),
NumTracksOut: trackSubscribedCurrent.Load(),
NumTrackPublishAttempts: trackPublishAttempts.Load(),
NumTrackPublishSuccess: trackPublishSuccess.Load(),
NumTrackPublishCancels: trackPublishCancels.Load(),
NumTrackSubscribeAttempts: trackSubscribeAttempts.Load(),
NumTrackSubscribeSuccess: trackSubscribeSuccess.Load(),
NumTrackSubscribeCancels: trackSubscribeCancels.Load(),
BytesIn: bytesIn.Load(),
BytesOut: bytesOut.Load(),
PacketsIn: packetsIn.Load(),
PacketsOut: packetsOut.Load(),
RetransmitBytesOut: retransmitBytes.Load(),
RetransmitPacketsOut: retransmitPackets.Load(),
NackTotal: nackTotal.Load(),
ParticipantSignalConnected: participantSignalConnected.Load(),
ParticipantRtcInit: participantRTCInit.Load(),
ParticipantRtcConnected: participantRTCConnected.Load(),
ParticipantRtcCanceled: participantRTCCanceled.Load(),
ParticipantRtcActive: participantRTCActive.Load(),
ForwardLatency: forwardLatency.Load(),
ForwardJitter: forwardJitter.Load(),
NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
CpuLoad: float32(cpuStats.GetCPULoad()),
MemoryTotal: memTotal,
MemoryUsed: memUsed,
LoadAvgLast1Min: float32(loadAvg.Loadavg1),
LoadAvgLast5Min: float32(loadAvg.Loadavg5),
LoadAvgLast15Min: float32(loadAvg.Loadavg15),
SysPacketsOut: sysPackets,
SysPacketsDropped: sysDroppedPackets,
}
for _, rateInterval := range rateIntervals {
for idx := len(prevStats) - 1; idx >= 0; idx-- {
prev := prevStats[idx]
if prev == nil {
continue
}
if stats.UpdatedAt-prev.UpdatedAt >= int64(rateInterval.Seconds()) {
if rate := getNodeStatsRate(append(prevStats[idx:], stats)); rate != nil {
stats.Rates = append(stats.Rates, rate)
}
break
}
}
}
return stats, nil
}
func getNodeStatsRate(statsHistory []*livekit.NodeStats) *livekit.NodeStatsRate {
if len(statsHistory) == 0 {
return nil
}
elapsed := statsHistory[len(statsHistory)-1].UpdatedAt - statsHistory[0].UpdatedAt
if elapsed <= 0 {
return nil
}
// time weighted averages
var cpuLoad, memoryUsed, memoryTotal, memoryLoad float32
for idx := len(statsHistory) - 1; idx > 0; idx-- {
stats := statsHistory[idx]
prevStats := statsHistory[idx-1]
if stats == nil || prevStats == nil {
continue
}
spanElapsed := stats.UpdatedAt - prevStats.UpdatedAt
if spanElapsed <= 0 {
continue
}
cpuLoad += stats.CpuLoad * float32(spanElapsed)
memoryUsed += float32(stats.MemoryUsed) * float32(spanElapsed)
memoryTotal += float32(stats.MemoryTotal) * float32(spanElapsed)
if stats.MemoryTotal > 0 {
memoryLoad += float32(stats.MemoryUsed) / float32(stats.MemoryTotal) * float32(spanElapsed)
}
}
earlier := statsHistory[0]
later := statsHistory[len(statsHistory)-1]
rate := &livekit.NodeStatsRate{
StartedAt: earlier.UpdatedAt,
EndedAt: later.UpdatedAt,
Duration: elapsed,
BytesIn: perSec(earlier.BytesIn, later.BytesIn, elapsed),
BytesOut: perSec(earlier.BytesOut, later.BytesOut, elapsed),
PacketsIn: perSec(earlier.PacketsIn, later.PacketsIn, elapsed),
PacketsOut: perSec(earlier.PacketsOut, later.PacketsOut, elapsed),
RetransmitBytesOut: perSec(earlier.RetransmitBytesOut, later.RetransmitBytesOut, elapsed),
RetransmitPacketsOut: perSec(earlier.RetransmitPacketsOut, later.RetransmitPacketsOut, elapsed),
NackTotal: perSec(earlier.NackTotal, later.NackTotal, elapsed),
ParticipantSignalConnected: perSec(earlier.ParticipantSignalConnected, later.ParticipantSignalConnected, elapsed),
ParticipantSignalFailed: perSec(earlier.ParticipantSignalFailed, later.ParticipantSignalFailed, elapsed),
ParticipantSignalValidationFailed: perSec(earlier.ParticipantSignalValidationFailed, later.ParticipantSignalValidationFailed, elapsed),
ParticipantRtcInit: perSec(earlier.ParticipantRtcInit, later.ParticipantRtcInit, elapsed),
ParticipantRtcConnected: perSec(earlier.ParticipantRtcConnected, later.ParticipantRtcConnected, elapsed),
ParticipantRtcCanceled: perSec(earlier.ParticipantRtcCanceled, later.ParticipantRtcCanceled, elapsed),
ParticipantRtcActive: perSec(earlier.ParticipantRtcActive, later.ParticipantRtcActive, elapsed),
SysPacketsOut: perSec(uint64(earlier.SysPacketsOut), uint64(later.SysPacketsOut), elapsed),
SysPacketsDropped: perSec(uint64(earlier.SysPacketsDropped), uint64(later.SysPacketsDropped), elapsed),
TrackPublishAttempts: perSec(uint64(earlier.NumTrackPublishAttempts), uint64(later.NumTrackPublishAttempts), elapsed),
TrackPublishSuccess: perSec(uint64(earlier.NumTrackPublishSuccess), uint64(later.NumTrackPublishSuccess), elapsed),
TrackPublishCancels: perSec(uint64(earlier.NumTrackPublishCancels), uint64(later.NumTrackPublishCancels), elapsed),
TrackSubscribeAttempts: perSec(uint64(earlier.NumTrackSubscribeAttempts), uint64(later.NumTrackSubscribeAttempts), elapsed),
TrackSubscribeSuccess: perSec(uint64(earlier.NumTrackSubscribeSuccess), uint64(later.NumTrackSubscribeSuccess), elapsed),
TrackSubscribeCancels: perSec(uint64(earlier.NumTrackSubscribeCancels), uint64(later.NumTrackSubscribeCancels), elapsed),
CpuLoad: cpuLoad / float32(elapsed),
MemoryLoad: memoryLoad / float32(elapsed),
MemoryUsed: memoryUsed / float32(elapsed),
MemoryTotal: memoryTotal / float32(elapsed),
}
return rate
}
func perSec(prev, curr uint64, secs int64) float32 {
return float32(curr-prev) / float32(secs)
}
func RecordSignalRequestSuccess() {
promMessageCounter.WithLabelValues("signal", "success", "request").Add(1)
}
func RecordSignalRequestFailure() {
promMessageCounter.WithLabelValues("signal", "failure", "request").Add(1)
}
func RecordSignalResponseSuccess() {
promMessageCounter.WithLabelValues("signal", "success", "response").Add(1)
}
func RecordSignalResponseFailure() {
promMessageCounter.WithLabelValues("signal", "failure", "response").Add(1)
}
func RecordServiceOperationSuccess(op string) {
promServiceOperationCounter.WithLabelValues(op, "success", "").Add(1)
}
func RecordServiceOperationError(op string, error string) {
promServiceOperationCounter.WithLabelValues(op, "error", error).Add(1)
}
func RecordTwirpRequestStatus(service string, method string, statusFamily string, code twirp.ErrorCode) {
promTwirpRequestStatusCounter.WithLabelValues(service, method, statusFamily, string(code)).Add(1)
}
func RecordTwirpRequestLatency(service, method string, duration time.Duration, statusFamily string) {
promTwirpRequestLatency.WithLabelValues(service, method, statusFamily).Observe(float64(duration.Milliseconds()))
}

View File

@@ -0,0 +1,56 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
// +build linux
package prometheus
import (
"fmt"
"github.com/florianl/go-tc"
)
func getTCStats() (packets, drops uint32, err error) {
rtnl, err := tc.Open(&tc.Config{})
if err != nil {
err = fmt.Errorf("could not open rtnetlink socket: %v", err)
return
}
defer rtnl.Close()
qdiscs, err := rtnl.Qdisc().Get()
if err != nil {
err = fmt.Errorf("could not get qdiscs: %v", err)
return
}
for _, qdisc := range qdiscs {
// Newer kernels report stats via TCA_STATS2 (Stats2), while older kernels
// only populate the legacy TCA_STATS attribute (Stats). Prefer Stats2 and
// fall back to Stats so counters are collected on both.
//
// However, go-tc (through v0.4.8 and current main) mis-parses the nested TCA_STATS2
// as a flat struct, so Stats2.Packets/.Drops are garbage. The legacy
// TCA_STATS blob is a real flat struct and is parsed correctly, and modern
// kernels populate BOTH — so prefer Stats.
if qdisc.Stats != nil {
packets += qdisc.Stats.Packets
drops += qdisc.Stats.Drops
}
}
return
}

View File

@@ -0,0 +1,22 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !linux
package prometheus
func getTCStats() (packets, drops uint32, err error) {
// linux only
return
}

View File

@@ -0,0 +1,25 @@
//go:build !windows
/*
* Copyright 2023 LiveKit, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package prometheus
import "github.com/mackerelio/go-osstat/loadavg"
func getLoadAvg() (*loadavg.Stats, error) {
return loadavg.Get()
}

View File

@@ -0,0 +1,25 @@
//go:build windows
/*
* Copyright 2023 LiveKit, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package prometheus
import "github.com/mackerelio/go-osstat/loadavg"
func getLoadAvg() (*loadavg.Stats, error) {
return &loadavg.Stats{}, nil
}

View File

@@ -0,0 +1,391 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"strconv"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/atomic"
"github.com/livekit/protocol/livekit"
)
type Direction string
const (
Incoming Direction = "incoming"
Outgoing Direction = "outgoing"
)
type TransmissionType string
const (
TransmissionInitial TransmissionType = "initial"
TransmissionRetransmit TransmissionType = "retransmit"
)
var (
bytesIn atomic.Uint64
bytesOut atomic.Uint64
packetsIn atomic.Uint64
packetsOut atomic.Uint64
nackTotal atomic.Uint64
retransmitBytes atomic.Uint64
retransmitPackets atomic.Uint64
participantSignalConnected atomic.Uint64
participantRTCConnected atomic.Uint64
participantRTCInit atomic.Uint64
participantRTCCanceled atomic.Uint64
participantRTCActive atomic.Uint64
forwardLatency atomic.Uint32
forwardJitter atomic.Uint32
promPacketLabels = []string{"direction", "transmission", "country"}
promPacketTotal *prometheus.CounterVec
promPacketBytes *prometheus.CounterVec
promRTCPLabels = []string{"direction", "country"}
promStreamLabels = []string{"direction", "source", "type", "country"}
promNackTotal *prometheus.CounterVec
promPliTotal *prometheus.CounterVec
promFirTotal *prometheus.CounterVec
promPacketLossTotal *prometheus.CounterVec
promPacketLoss *prometheus.HistogramVec
promPacketOutOfOrderTotal *prometheus.CounterVec
promPacketOutOfOrder *prometheus.HistogramVec
promJitter *prometheus.HistogramVec
promRTT *prometheus.HistogramVec
promParticipantJoin *prometheus.CounterVec
promConnections *prometheus.GaugeVec
promForwardLatency prometheus.Gauge
promForwardJitter prometheus.Gauge
promForwardLatencyHist prometheus.Histogram
)
func initPacketStats(nodeID string, nodeType livekit.NodeType) {
promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promPacketLabels)
promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet",
Name: "bytes",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promPacketLabels)
promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "nack",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promRTCPLabels)
promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "pli",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promRTCPLabels)
promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "fir",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promRTCPLabels)
promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet_loss",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promStreamLabels)
promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "packet_loss",
Name: "percent",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
}, promStreamLabels)
promPacketOutOfOrderTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "packet_out_of_order",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, promStreamLabels)
promPacketOutOfOrder = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "packet_out_of_order",
Name: "percent",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
}, promStreamLabels)
promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "jitter",
Name: "us",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
}, promStreamLabels)
promRTT = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "rtt",
Name: "ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000},
}, promStreamLabels)
promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "participant_join",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"state", "warp"})
promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "connection",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind"})
promForwardLatency = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "forward",
Name: "latency",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
})
promForwardJitter = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "forward",
Name: "jitter",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
})
promForwardLatencyHist = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "forward_latency",
Name: "ns",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
// 50us, 100us, 250us, 500us, 1ms, 2ms, 3ms, 5ms, 10ms, 20ms
Buckets: []float64{
50 * 1000,
100 * 1000,
250 * 1000,
500 * 1000,
1 * 1000 * 1000,
2 * 1000 * 1000,
3 * 1000 * 1000,
5 * 1000 * 1000,
10 * 1000 * 1000,
20 * 1000 * 1000,
},
})
prometheus.MustRegister(promPacketTotal)
prometheus.MustRegister(promPacketBytes)
prometheus.MustRegister(promNackTotal)
prometheus.MustRegister(promPliTotal)
prometheus.MustRegister(promFirTotal)
prometheus.MustRegister(promPacketLossTotal)
prometheus.MustRegister(promPacketLoss)
prometheus.MustRegister(promPacketOutOfOrderTotal)
prometheus.MustRegister(promPacketOutOfOrder)
prometheus.MustRegister(promJitter)
prometheus.MustRegister(promRTT)
prometheus.MustRegister(promParticipantJoin)
prometheus.MustRegister(promConnections)
prometheus.MustRegister(promForwardLatency)
prometheus.MustRegister(promForwardJitter)
prometheus.MustRegister(promForwardLatencyHist)
}
func IncrementPackets(country string, direction Direction, count uint64, retransmit bool) {
var transmission TransmissionType
if retransmit {
transmission = TransmissionRetransmit
} else {
transmission = TransmissionInitial
}
promPacketTotal.WithLabelValues(string(direction), string(transmission), country).Add(float64(count))
if direction == Incoming {
packetsIn.Add(count)
} else {
packetsOut.Add(count)
if retransmit {
retransmitPackets.Add(count)
}
}
}
func IncrementBytes(country string, direction Direction, count uint64, retransmit bool) {
var transmission TransmissionType
if retransmit {
transmission = TransmissionRetransmit
} else {
transmission = TransmissionInitial
}
promPacketBytes.WithLabelValues(string(direction), string(transmission), country).Add(float64(count))
if direction == Incoming {
bytesIn.Add(count)
} else {
bytesOut.Add(count)
if retransmit {
retransmitBytes.Add(count)
}
}
}
func IncrementRTCP(country string, direction Direction, nack, pli, fir uint32) {
if nack > 0 {
promNackTotal.WithLabelValues(string(direction), country).Add(float64(nack))
nackTotal.Add(uint64(nack))
}
if pli > 0 {
promPliTotal.WithLabelValues(string(direction), country).Add(float64(pli))
}
if fir > 0 {
promFirTotal.WithLabelValues(string(direction), country).Add(float64(fir))
}
}
func RecordPacketLoss(
country string,
direction Direction,
trackSource livekit.TrackSource,
trackType livekit.TrackType,
lost uint32,
total uint32,
) {
if total > 0 {
promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(lost) / float64(total) * 100)
}
if lost > 0 {
promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(lost))
}
}
func RecordPacketOutOfOrder(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) {
if total > 0 {
promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(ooo) / float64(total) * 100)
}
if ooo > 0 {
promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(ooo))
}
}
func RecordJitter(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) {
if jitter > 0 {
promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(jitter))
}
}
func RecordRTT(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) {
if rtt > 0 {
promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(rtt))
}
}
func IncrementParticipantJoin(join uint32) {
if join > 0 {
participantSignalConnected.Add(uint64(join))
promParticipantJoin.WithLabelValues("signal_connected", "").Add(float64(join))
}
}
func IncrementParticipantJoinFail(fail uint32) {
if fail > 0 {
promParticipantJoin.WithLabelValues("signal_failed", "").Add(float64(fail))
}
}
func IncrementParticipantJoinValidationFail(validationFail uint32) {
if validationFail > 0 {
promParticipantJoin.WithLabelValues("signal_validation_failed", "").Add(float64(validationFail))
}
}
func IncrementParticipantJoinUpgradeFail(upgradeFail uint32) {
if upgradeFail > 0 {
promParticipantJoin.WithLabelValues("signal_upgrade_failed", "").Add(float64(upgradeFail))
}
}
func IncrementParticipantJoinWriteInitialResponseFail(writeInitialResponseFail uint32) {
if writeInitialResponseFail > 0 {
promParticipantJoin.WithLabelValues("signal_write_initial_response_failed", "").Add(float64(writeInitialResponseFail))
}
}
func IncrementParticipantRtcInit(init uint32) {
if init > 0 {
participantRTCInit.Add(uint64(init))
promParticipantJoin.WithLabelValues("rtc_init", "").Add(float64(init))
}
}
func IncrementParticipantRtcConnected(connected uint32) {
if connected > 0 {
participantRTCConnected.Add(uint64(connected))
promParticipantJoin.WithLabelValues("rtc_connected", "").Add(float64(connected))
}
}
func IncrementParticipantRtcActive(active uint32, warp bool) {
if active > 0 {
participantRTCActive.Add(uint64(active))
promParticipantJoin.WithLabelValues("rtc_active", strconv.FormatBool(warp)).Add(float64(active))
}
}
func IncrementParticipantRtcCanceled(canceled uint64, warp bool) {
if canceled > 0 {
participantRTCCanceled.Add(canceled)
promParticipantJoin.WithLabelValues("rtc_canceled", strconv.FormatBool(warp)).Add(float64(canceled))
}
}
// todo: check if need to record warp to rtcSucc/Failure
func IncrementParticipantRtcSuccess(success uint64, warp bool) {
if success > 0 {
promParticipantJoin.WithLabelValues("rtc_success", strconv.FormatBool(warp)).Add(float64(success))
}
}
func IncrementParticipantRtcFailure(failure uint64, warp bool) {
if failure > 0 {
promParticipantJoin.WithLabelValues("rtc_failure", strconv.FormatBool(warp)).Add(float64(failure))
}
}
func AddConnection(direction Direction) {
promConnections.WithLabelValues(string(direction)).Add(1)
}
func SubConnection(direction Direction) {
promConnections.WithLabelValues(string(direction)).Sub(1)
}
func RecordForwardLatencySample(forwardLatency int64) {
promForwardLatencyHist.Observe(float64(forwardLatency))
}
func RecordForwardLatency(longTermLatencyAvg uint32) {
forwardLatency.Store(longTermLatencyAvg)
promForwardLatency.Set(float64(longTermLatencyAvg))
}
func RecordForwardJitter(longTermJitterAvg uint32) {
forwardJitter.Store(longTermJitterAvg)
promForwardJitter.Set(float64(longTermJitterAvg))
}

View File

@@ -0,0 +1,51 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/livekit/protocol/livekit"
)
var (
qualityRating prometheus.Histogram
qualityScore prometheus.Histogram
)
func initQualityStats(nodeID string, nodeType livekit.NodeType) {
qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "quality",
Name: "rating",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{0, 1, 2},
})
qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "quality",
Name: "score",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5},
})
prometheus.MustRegister(qualityRating)
prometheus.MustRegister(qualityScore)
}
func RecordQuality(rating livekit.ConnectionQuality, score float32) {
qualityRating.Observe(float64(rating))
qualityScore.Observe(float64(score))
}

View File

@@ -0,0 +1,304 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package prometheus
import (
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/atomic"
"github.com/livekit/protocol/livekit"
)
var (
roomCurrent atomic.Int32
participantCurrent atomic.Int32
trackPublishedCurrent atomic.Int32
trackSubscribedCurrent atomic.Int32
trackPublishAttempts atomic.Int32
trackPublishSuccess atomic.Int32
trackPublishCancels atomic.Int32
trackSubscribeAttempts atomic.Int32
trackSubscribeSuccess atomic.Int32
trackSubscribeCancels atomic.Int32
// count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute
// success rate by subtracting this from total attempts
trackSubscribeUserError atomic.Int32
promRoomCurrent prometheus.Gauge
promRoomDuration prometheus.Histogram
promParticipantCurrent prometheus.Gauge
promTrackPublishedCurrent *prometheus.GaugeVec
promTrackSubscribedCurrent *prometheus.GaugeVec
promTrackPublishCounter *prometheus.CounterVec
promTrackSubscribeCounter *prometheus.CounterVec
promSessionJoinLatency *prometheus.HistogramVec
promSessionStartTime *prometheus.HistogramVec
promSessionDuration *prometheus.HistogramVec
promPubSubTime *prometheus.HistogramVec
promPeerConnection *prometheus.CounterVec
)
func initRoomStats(nodeID string, nodeType livekit.NodeType) {
promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "room",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
})
promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "room",
Name: "duration_seconds",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{
5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60,
},
})
promParticipantCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "participant",
Name: "total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
})
promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "published_total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind"})
promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "subscribed_total",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind"})
promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "publish_counter",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"kind", "state"})
promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "track",
Name: "subscribe_counter",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"state", "error"})
promSessionJoinLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "session",
Name: "join_latency_ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: prometheus.ExponentialBucketsRange(10, 10000, 15),
}, []string{"protocol_version"})
promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "session",
Name: "start_time_ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15),
}, []string{"protocol_version", "warp"})
promSessionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "session",
Name: "duration_ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: prometheus.ExponentialBucketsRange(100, 4*60*60*1000, 15),
}, []string{"protocol_version"})
promPubSubTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: livekitNamespace,
Subsystem: "pubsubtime",
Name: "ms",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
Buckets: []float64{100, 200, 500, 700, 1000, 5000, 10000},
}, append(promStreamLabels, "sdk", "kind", "count"))
promPeerConnection = prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: livekitNamespace,
Subsystem: "peer_connection",
Name: "state",
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
}, []string{"transport", "state"})
prometheus.MustRegister(promRoomCurrent)
prometheus.MustRegister(promRoomDuration)
prometheus.MustRegister(promParticipantCurrent)
prometheus.MustRegister(promTrackPublishedCurrent)
prometheus.MustRegister(promTrackSubscribedCurrent)
prometheus.MustRegister(promTrackPublishCounter)
prometheus.MustRegister(promTrackSubscribeCounter)
prometheus.MustRegister(promSessionJoinLatency)
prometheus.MustRegister(promSessionStartTime)
prometheus.MustRegister(promSessionDuration)
prometheus.MustRegister(promPubSubTime)
prometheus.MustRegister(promPeerConnection)
}
func RoomStarted() {
promRoomCurrent.Add(1)
roomCurrent.Inc()
}
func RoomEnded(startedAt time.Time) {
if !startedAt.IsZero() {
promRoomDuration.Observe(float64(time.Since(startedAt)) / float64(time.Second))
}
promRoomCurrent.Sub(1)
roomCurrent.Dec()
}
func AddParticipant() {
promParticipantCurrent.Add(1)
participantCurrent.Inc()
}
func SubParticipant() {
promParticipantCurrent.Sub(1)
participantCurrent.Dec()
}
func AddPublishedTrack(kind string) {
promTrackPublishedCurrent.WithLabelValues(kind).Add(1)
trackPublishedCurrent.Inc()
}
func SubPublishedTrack(kind string) {
promTrackPublishedCurrent.WithLabelValues(kind).Sub(1)
trackPublishedCurrent.Dec()
}
func RecordTrackPublishAttempt(kind string) {
trackPublishAttempts.Inc()
promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc()
}
func RecordTrackPublishSuccess(kind string) {
trackPublishSuccess.Inc()
promTrackPublishCounter.WithLabelValues(kind, "success").Inc()
}
func RecordTrackPublishCancels(kind string, numCancels int32) {
trackPublishCancels.Add(numCancels)
promTrackPublishCounter.WithLabelValues(kind, "cancel").Add(float64(numCancels))
}
func RecordPublishTime(
country string,
source livekit.TrackSource,
trackType livekit.TrackType,
d time.Duration,
sdk livekit.ClientInfo_SDK,
kind livekit.ParticipantInfo_Kind,
) {
recordPubSubStreamStartTime("publish", country, source, trackType, d, sdk, kind, 1)
}
func RecordSubscribeTime(
country string,
source livekit.TrackSource,
trackType livekit.TrackType,
d time.Duration,
sdk livekit.ClientInfo_SDK,
kind livekit.ParticipantInfo_Kind,
count int,
) {
recordPubSubStreamStartTime("subscribe", country, source, trackType, d, sdk, kind, count)
}
func RecordSubscribeStreamStartTime(
country string,
source livekit.TrackSource,
trackType livekit.TrackType,
d time.Duration,
sdk livekit.ClientInfo_SDK,
kind livekit.ParticipantInfo_Kind,
) {
recordPubSubStreamStartTime("subscribe-stream-start", country, source, trackType, d, sdk, kind, 1)
}
func recordPubSubStreamStartTime(
direction string,
country string,
source livekit.TrackSource,
trackType livekit.TrackType,
d time.Duration,
sdk livekit.ClientInfo_SDK,
kind livekit.ParticipantInfo_Kind,
count int,
) {
promPubSubTime.WithLabelValues(
direction,
source.String(),
trackType.String(),
country,
sdk.String(),
kind.String(),
strconv.Itoa(count),
).Observe(float64(d.Milliseconds()))
}
func RecordTrackSubscribeSuccess(kind string) {
// modify both current and total counters
promTrackSubscribedCurrent.WithLabelValues(kind).Add(1)
trackSubscribedCurrent.Inc()
promTrackSubscribeCounter.WithLabelValues("success", "").Inc()
trackSubscribeSuccess.Inc()
}
func RecordTrackUnsubscribed(kind string) {
// unsubscribed modifies current counter, but we leave the total values alone since they
// are used to compute rate
promTrackSubscribedCurrent.WithLabelValues(kind).Sub(1)
trackSubscribedCurrent.Dec()
}
func RecordTrackSubscribeAttempt() {
trackSubscribeAttempts.Inc()
promTrackSubscribeCounter.WithLabelValues("attempt", "").Inc()
}
func RecordTrackSubscribeFailure(err error, isUserError bool) {
promTrackSubscribeCounter.WithLabelValues("failure", err.Error()).Inc()
if isUserError {
trackSubscribeUserError.Inc()
trackSubscribeCancels.Inc()
}
}
func RecordTrackSubscribeCancels(numCancels int32) {
trackSubscribeCancels.Add(numCancels)
promTrackSubscribeCounter.WithLabelValues("cancel", "").Add(float64(numCancels))
}
func RecordSessionJoinLatency(protocolVersion int, d time.Duration) {
promSessionJoinLatency.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
}
func RecordSessionStartTime(protocolVersion int, warp bool, d time.Duration) {
promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion), strconv.FormatBool(warp)).Observe(float64(d.Milliseconds()))
}
func RecordSessionDuration(protocolVersion int, d time.Duration) {
promSessionDuration.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
}
func RecordPeerConnectionState(transport livekit.SignalTarget, state string) {
promPeerConnection.WithLabelValues(transport.String(), state).Inc()
}

114
pkg/telemetry/stats.go Normal file
View File

@@ -0,0 +1,114 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry
import (
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
"github.com/livekit/protocol/livekit"
)
type StatsKey struct {
country string
streamType livekit.StreamType
participantID livekit.ParticipantID
trackID livekit.TrackID
trackSource livekit.TrackSource
trackType livekit.TrackType
track bool
}
func StatsKeyForTrack(
country string,
streamType livekit.StreamType,
participantID livekit.ParticipantID,
trackID livekit.TrackID,
trackSource livekit.TrackSource,
trackType livekit.TrackType,
) StatsKey {
return StatsKey{
country: country,
streamType: streamType,
participantID: participantID,
trackID: trackID,
trackSource: trackSource,
trackType: trackType,
track: true,
}
}
func StatsKeyForData(
country string,
streamType livekit.StreamType,
participantID livekit.ParticipantID,
trackID livekit.TrackID,
) StatsKey {
return StatsKey{
country: country,
streamType: streamType,
participantID: participantID,
trackID: trackID,
}
}
func (t *telemetryService) TrackStats(roomID livekit.RoomID, _roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) {
t.enqueue(func() {
direction := prometheus.Incoming
if key.streamType == livekit.StreamType_DOWNSTREAM {
direction = prometheus.Outgoing
}
nacks := uint32(0)
plis := uint32(0)
firs := uint32(0)
packets := uint32(0)
bytes := uint64(0)
retransmitBytes := uint64(0)
retransmitPackets := uint32(0)
for _, stream := range stat.Streams {
nacks += stream.Nacks
plis += stream.Plis
firs += stream.Firs
packets += stream.PrimaryPackets + stream.PaddingPackets
bytes += stream.PrimaryBytes + stream.PaddingBytes
if key.streamType == livekit.StreamType_DOWNSTREAM {
retransmitPackets += stream.RetransmitPackets
retransmitBytes += stream.RetransmitBytes
} else {
// for upstream, we don't account for these separately for now
packets += stream.RetransmitPackets
bytes += stream.RetransmitBytes
}
if key.track {
prometheus.RecordPacketLoss(key.country, direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets)
prometheus.RecordPacketOutOfOrder(key.country, direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets)
prometheus.RecordRTT(key.country, direction, key.trackSource, key.trackType, stream.Rtt)
prometheus.RecordJitter(key.country, direction, key.trackSource, key.trackType, stream.Jitter)
}
}
prometheus.IncrementRTCP(key.country, direction, nacks, plis, firs)
prometheus.IncrementPackets(key.country, direction, uint64(packets), false)
prometheus.IncrementBytes(key.country, direction, bytes, false)
if retransmitPackets != 0 {
prometheus.IncrementPackets(key.country, direction, uint64(retransmitPackets), true)
}
if retransmitBytes != 0 {
prometheus.IncrementBytes(key.country, direction, retransmitBytes, true)
}
if worker, ok := t.getWorker(roomID, key.participantID); ok {
worker.OnTrackStat(key.trackID, key.streamType, stat)
}
})
}

759
pkg/telemetry/stats_test.go Normal file
View File

@@ -0,0 +1,759 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/telemetry"
"github.com/livekit/protocol/livekit"
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
"github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes"
)
func init() {
prometheus.Init("test", livekit.NodeType_SERVER)
}
type telemetryServiceFixture struct {
sut telemetry.TelemetryService
analytics *telemetryfakes.FakeAnalyticsService
}
func createFixture() *telemetryServiceFixture {
fixture := &telemetryServiceFixture{}
fixture.analytics = &telemetryfakes.FakeAnalyticsService{}
fixture.sut = telemetry.NewTelemetryService(nil, fixture.analytics)
return fixture
}
func Test_ParticipantAndRoomDataAreSentWithAnalytics(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
clientInfo := &livekit.ClientInfo{Sdk: 2}
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
// do
packet := 33
stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet)}}}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, ""), stat)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
require.Equal(t, string(partSID), stats[0].ParticipantId)
require.Equal(t, room.Sid, stats[0].RoomId)
require.Equal(t, room.Name, stats[0].RoomName)
}
func Test_OnDownstreamPackets(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
clientInfo := &livekit.ClientInfo{Sdk: 2}
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
// do
packets := []int{33, 23}
totalBytes := packets[0] + packets[1]
totalPackets := len(packets)
trackID := livekit.TrackID("trackID")
for i := range packets {
stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packets[i]), PrimaryPackets: uint32(1)}}}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat)
}
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
require.Equal(t, totalBytes, int(stats[0].Streams[0].PrimaryBytes))
require.Equal(t, totalPackets, int(stats[0].Streams[0].PrimaryPackets))
require.Equal(t, string(trackID), stats[0].TrackId)
}
func Test_OnDownstreamPackets_SeveralTracks(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
clientInfo := &livekit.ClientInfo{Sdk: 2}
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
// do
packet1 := 33
trackID1 := livekit.TrackID("trackID1")
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet1), PrimaryPackets: 1}}}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1)
packet2 := 23
trackID2 := livekit.TrackID("trackID2")
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet2), PrimaryPackets: 1}}}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 2, len(stats))
found1 := false
found2 := false
for _, sentStat := range stats {
if livekit.TrackID(sentStat.TrackId) == trackID1 {
found1 = true
require.Equal(t, packet1, int(sentStat.Streams[0].PrimaryBytes))
require.Equal(t, 1, int(sentStat.Streams[0].PrimaryPackets))
} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
found2 = true
require.Equal(t, packet2, int(sentStat.Streams[0].PrimaryBytes))
require.Equal(t, 1, int(sentStat.Streams[0].PrimaryPackets))
}
}
require.True(t, found1)
require.True(t, found2)
}
func Test_OnDownStreamStat(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 1,
PrimaryPackets: 1,
PacketsLost: 3,
Nacks: 1,
Plis: 1,
Rtt: 23,
Jitter: 3,
},
},
}
trackID := livekit.TrackID("trackID1")
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
stat2 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 2,
PrimaryPackets: 2,
PacketsLost: 4,
Nacks: 1,
Plis: 1,
Firs: 1,
Rtt: 10,
Jitter: 5,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
require.Equal(t, 2, int(stats[0].Streams[0].Nacks))
require.Equal(t, 2, int(stats[0].Streams[0].Plis))
require.Equal(t, 1, int(stats[0].Streams[0].Firs))
require.Equal(t, 23, int(stats[0].Streams[0].Rtt)) // max of RTT
require.Equal(t, 5, int(stats[0].Streams[0].Jitter)) // max of jitter
require.Equal(t, 7, int(stats[0].Streams[0].PacketsLost)) // coalesced delta packet losses
require.Equal(t, string(trackID), stats[0].TrackId)
}
func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
trackID := livekit.TrackID("trackID1")
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 1,
PrimaryPackets: 1,
PacketsLost: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent
// flush
fixture.flush()
stat2 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 2,
PrimaryPackets: 2,
PacketsLost: 4,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
// flush
fixture.flush()
// test
require.Equal(t, 2, fixture.analytics.SendStatsCallCount()) // 2 calls to fixture.sut.FlushStats()
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
require.Equal(t, 1, int(stats[0].Streams[0].PacketsLost)) // see pkts1
_, stats = fixture.analytics.SendStatsArgsForCall(1)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
require.Equal(t, 4, int(stats[0].Streams[0].PacketsLost)) // delta loss should be sent as is
}
func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
trackID1 := livekit.TrackID("trackID1")
trackID2 := livekit.TrackID("trackID2")
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 1,
PrimaryPackets: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent
stat2 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 2,
PrimaryPackets: 2,
Nacks: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2)
stat3 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 3,
PrimaryPackets: 3,
Firs: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 2, len(stats))
found1 := false
found2 := false
for _, sentStat := range stats {
if livekit.TrackID(sentStat.TrackId) == trackID1 {
found1 = true
require.Equal(t, livekit.StreamType_DOWNSTREAM, sentStat.Kind)
require.Equal(t, 1, int(sentStat.Streams[0].Nacks)) // see pkts1 above
} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
found2 = true
require.Equal(t, livekit.StreamType_DOWNSTREAM, sentStat.Kind)
require.Equal(t, 1, int(sentStat.Streams[0].Firs)) // see pkts2 above
}
}
require.True(t, found1)
require.True(t, found2)
}
func Test_OnUpstreamStat(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 1,
PrimaryPackets: 1,
PacketsLost: 3,
Nacks: 1,
Plis: 1,
Firs: 1,
Rtt: 13,
Jitter: 5,
},
},
}
trackID := livekit.TrackID("trackID")
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
stat2 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 2,
PrimaryPackets: 2,
PacketsLost: 4,
Nacks: 1,
Plis: 1,
Firs: 1,
Rtt: 33,
Jitter: 2,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat2)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
require.Equal(t, 2, int(stats[0].Streams[0].Nacks))
require.Equal(t, 2, int(stats[0].Streams[0].Plis))
require.Equal(t, 2, int(stats[0].Streams[0].Firs))
require.Equal(t, 33, int(stats[0].Streams[0].Rtt)) // max of RTT
require.Equal(t, 5, int(stats[0].Streams[0].Jitter)) // max of jitter
require.Equal(t, 7, int(stats[0].Streams[0].PacketsLost)) // coalesced delta packet losses
require.Equal(t, string(trackID), stats[0].TrackId)
}
func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
identity := livekit.ParticipantIdentity("part1Identity")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID), Identity: string(identity)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// there should be bytes reported so that stats are sent
totalBytes := 1
totalPackets := 1
trackID1 := livekit.TrackID("trackID1")
trackID2 := livekit.TrackID("trackID2")
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: uint64(totalBytes),
PrimaryPackets: uint32(totalPackets),
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat1)
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine
// do
totalBytes++
totalPackets++
stat2 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: uint64(totalBytes),
PrimaryPackets: uint32(totalPackets),
Nacks: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat2)
stat3 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: uint64(totalBytes),
PrimaryPackets: uint32(totalPackets),
Firs: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat3)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 2, len(stats))
found1 := false
found2 := false
for _, sentStat := range stats {
if livekit.TrackID(sentStat.TrackId) == trackID1 {
found1 = true
require.Equal(t, livekit.StreamType_UPSTREAM, sentStat.Kind)
require.Equal(t, 1, int(sentStat.Streams[0].Nacks)) // see pkts1 above
} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
found2 = true
require.Equal(t, livekit.StreamType_UPSTREAM, sentStat.Kind)
require.Equal(t, 1, int(sentStat.Streams[0].Firs)) // see pkts2 above
}
require.Equal(t, 3, int(sentStat.Streams[0].PrimaryBytes))
require.Equal(t, 3, int(sentStat.Streams[0].PrimaryPackets))
}
require.True(t, found1)
require.True(t, found2)
// remove 1 track - track stats were flushed above, so no more calls to SendStats
fixture.sut.TrackUnpublished(context.Background(), room, partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true, true)
// flush
fixture.flush()
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
}
func Test_AnalyticsSentWhenParticipantLeaves(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{}
partSID := "part1"
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard)
// should not be called if there are no track stats
time.Sleep(time.Millisecond * 500)
require.Equal(t, 0, fixture.analytics.SendStatsCallCount())
}
func Test_AddUpTrack(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
var totalBytes uint64 = 3
var totalPackets uint32 = 3
stat := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: totalBytes,
PrimaryPackets: totalPackets,
},
},
}
trackID := livekit.TrackID("trackID")
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
require.Equal(t, totalBytes, stats[0].Streams[0].PrimaryBytes)
require.Equal(t, totalPackets, stats[0].Streams[0].PrimaryPackets)
require.Equal(t, string(trackID), stats[0].TrackId)
}
func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
trackID := livekit.TrackID("trackID")
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 1,
PrimaryPackets: 1,
},
{
PrimaryBytes: 2,
PrimaryPackets: 2,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 1, len(stats))
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
// should be a consolidated stream
require.Equal(t, stat1.Streams[0].PrimaryBytes+stat1.Streams[1].PrimaryBytes, stats[0].Streams[0].PrimaryBytes)
require.Equal(t, stat1.Streams[0].PrimaryPackets+stat1.Streams[1].PrimaryPackets, stats[0].Streams[0].PrimaryPackets)
require.Equal(t, string(trackID), stats[0].TrackId)
}
func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
// do
// upstream bytes
stat1 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 3,
PrimaryPackets: 3,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1)
// downstream bytes
stat2 := &livekit.AnalyticsStat{
Streams: []*livekit.AnalyticsStream{
{
PrimaryBytes: 1,
PrimaryPackets: 1,
},
},
}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2)
// flush
fixture.flush()
// test
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 2, len(stats))
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[1].Kind)
}
func Test_RoomIDChangeReKeysStatsWorkers(t *testing.T) {
fixture := createFixture()
// prepare
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
trackID := livekit.TrackID("trackID")
guard := &telemetry.ReferenceGuard{}
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}}
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
// do - the room restarts and gets a new id
restartedRoom := &livekit.Room{Sid: "RestartedSid", Name: "RoomName"}
fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(room.Sid), restartedRoom)
// stats reported with the new id reach the same worker
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}}
fixture.sut.TrackStats(livekit.RoomID(restartedRoom.Sid), livekit.RoomName(restartedRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
fixture.flush()
// one worker, one flush, but two stats - each attributed to the session it was collected in
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
_, stats := fixture.analytics.SendStatsArgsForCall(0)
require.Equal(t, 2, len(stats))
byRoom := map[string]*livekit.AnalyticsStat{}
for _, stat := range stats {
require.Equal(t, string(partSID), stat.ParticipantId)
byRoom[stat.RoomId] = stat
}
require.Len(t, byRoom, 2)
require.Equal(t, uint64(33), byRoom[room.Sid].Streams[0].PrimaryBytes)
require.Equal(t, uint64(44), byRoom[restartedRoom.Sid].Streams[0].PrimaryBytes)
// the worker moved rather than being duplicated, so closing it out drains everything
fixture.sut.ParticipantLeft(context.Background(), restartedRoom, participantInfo, true, guard)
fixture.flush()
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
}
// a forwarded participant is in more than one room at a time under the same participant
// id, so re-keying one of those rooms must leave the other alone
func Test_RoomIDChangeLeavesForwardedParticipantAlone(t *testing.T) {
fixture := createFixture()
// prepare - the same participant id in a source room and a forwarding destination room
sourceRoom := &livekit.Room{Sid: "SourceSid", Name: "SourceRoom"}
destRoom := &livekit.Room{Sid: "DestSid", Name: "DestRoom"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
trackID := livekit.TrackID("trackID")
fixture.sut.ParticipantJoined(context.Background(), sourceRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
fixture.sut.ParticipantJoined(context.Background(), destRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
// do - only the destination room restarts
restartedDest := &livekit.Room{Sid: "RestartedDestSid", Name: "DestRoom"}
fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(destRoom.Sid), restartedDest)
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}}
fixture.sut.TrackStats(livekit.RoomID(sourceRoom.Sid), livekit.RoomName(sourceRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}}
fixture.sut.TrackStats(livekit.RoomID(restartedDest.Sid), livekit.RoomName(restartedDest.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
fixture.flush()
// the source room's worker is untouched, the destination room's worker moved
byRoom := map[string]*livekit.AnalyticsStat{}
for i := 0; i < fixture.analytics.SendStatsCallCount(); i++ {
_, stats := fixture.analytics.SendStatsArgsForCall(i)
for _, stat := range stats {
byRoom[stat.RoomId] = stat
}
}
require.Len(t, byRoom, 2)
require.Equal(t, uint64(33), byRoom[sourceRoom.Sid].Streams[0].PrimaryBytes)
require.Equal(t, sourceRoom.Name, byRoom[sourceRoom.Sid].RoomName)
require.Equal(t, uint64(44), byRoom[restartedDest.Sid].Streams[0].PrimaryBytes)
require.Equal(t, restartedDest.Name, byRoom[restartedDest.Sid].RoomName)
}
// a re-key should never land on a room that already has workers, but if it does only
// one worker can be keyed at (room, participant) and the superseded one must not be
// left unreachable in the flush list
func Test_RoomIDChangeParticipantCollision(t *testing.T) {
fixture := createFixture()
// prepare - the same participant id in the room being re-keyed and in its destination
prevRoom := &livekit.Room{Sid: "PrevSid", Name: "PrevRoom"}
destRoom := &livekit.Room{Sid: "DestSid", Name: "DestRoom"}
partSID := livekit.ParticipantID("part1")
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
trackID := livekit.TrackID("trackID")
fixture.sut.ParticipantJoined(context.Background(), prevRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
fixture.sut.ParticipantJoined(context.Background(), destRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}}
fixture.sut.TrackStats(livekit.RoomID(prevRoom.Sid), livekit.RoomName(prevRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
// do
fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(prevRoom.Sid), destRoom)
// the superseded worker drains what it collected under the room it was in
fixture.flush()
byRoom := map[string]*livekit.AnalyticsStat{}
for i := 0; i < fixture.analytics.SendStatsCallCount(); i++ {
_, stats := fixture.analytics.SendStatsArgsForCall(i)
for _, stat := range stats {
byRoom[stat.RoomId] = stat
}
}
require.Equal(t, uint64(33), byRoom[prevRoom.Sid].Streams[0].PrimaryBytes)
// the worker already keyed at the destination wins and keeps receiving stats
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}}
fixture.sut.TrackStats(livekit.RoomID(destRoom.Sid), livekit.RoomName(destRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
fixture.flush()
_, stats := fixture.analytics.SendStatsArgsForCall(fixture.analytics.SendStatsCallCount() - 1)
require.Equal(t, 1, len(stats))
require.Equal(t, destRoom.Sid, stats[0].RoomId)
require.Equal(t, uint64(44), stats[0].Streams[0].PrimaryBytes)
}
func (f *telemetryServiceFixture) flush() {
time.Sleep(time.Millisecond * 500)
f.sut.FlushStats()
}

141
pkg/telemetry/statsconn.go Normal file
View File

@@ -0,0 +1,141 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry
import (
"net"
"github.com/pion/turn/v5"
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
)
type Listener struct {
net.Listener
}
func NewListener(l net.Listener) *Listener {
return &Listener{Listener: l}
}
func (l *Listener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return NewConn(conn, prometheus.Incoming), nil
}
type Conn struct {
net.Conn
direction prometheus.Direction
}
func NewConn(c net.Conn, direction prometheus.Direction) *Conn {
prometheus.AddConnection(direction)
return &Conn{Conn: c, direction: direction}
}
func (c *Conn) Read(b []byte) (n int, err error) {
n, err = c.Conn.Read(b)
if n > 0 {
prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false)
prometheus.IncrementPackets("", prometheus.Incoming, 1, false)
}
return
}
func (c *Conn) Write(b []byte) (n int, err error) {
n, err = c.Conn.Write(b)
if n > 0 {
prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false)
prometheus.IncrementPackets("", prometheus.Outgoing, 1, false)
}
return
}
func (c *Conn) Close() error {
prometheus.SubConnection(c.direction)
return c.Conn.Close()
}
type PacketConn struct {
net.PacketConn
direction prometheus.Direction
}
func NewPacketConn(c net.PacketConn, direction prometheus.Direction) *PacketConn {
prometheus.AddConnection(direction)
return &PacketConn{PacketConn: c, direction: direction}
}
func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
n, addr, err = c.PacketConn.ReadFrom(p)
if n > 0 {
prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false)
prometheus.IncrementPackets("", prometheus.Incoming, 1, false)
}
return
}
func (c *PacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
n, err = c.PacketConn.WriteTo(p, addr)
if n > 0 {
prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false)
prometheus.IncrementPackets("", prometheus.Outgoing, 1, false)
}
return
}
func (c *PacketConn) Close() error {
prometheus.SubConnection(c.direction)
return c.PacketConn.Close()
}
type RelayAddressGenerator struct {
turn.RelayAddressGenerator
}
func NewRelayAddressGenerator(g turn.RelayAddressGenerator) *RelayAddressGenerator {
return &RelayAddressGenerator{RelayAddressGenerator: g}
}
func (g *RelayAddressGenerator) AllocatePacketConn(c turn.AllocateListenerConfig) (net.PacketConn, net.Addr, error) {
conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(c)
if err != nil {
return nil, addr, err
}
return NewPacketConn(conn, prometheus.Outgoing), addr, err
}
func (g *RelayAddressGenerator) AllocateConn(c turn.AllocateConnConfig) (net.Conn, error) {
conn, err := g.RelayAddressGenerator.AllocateConn(c)
if err != nil {
return nil, err
}
return NewConn(conn, prometheus.Outgoing), err
}
func (g *RelayAddressGenerator) AllocateListener(c turn.AllocateListenerConfig) (net.Listener, net.Addr, error) {
l, addr, err := g.RelayAddressGenerator.AllocateListener(c)
if err != nil {
return nil, addr, err
}
return NewListener(l), addr, err
}

View File

@@ -0,0 +1,521 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry
import (
"context"
"sync"
"time"
"unsafe"
"go.uber.org/zap/zapcore"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/livekit-server/pkg/utils"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
protoutils "github.com/livekit/protocol/utils"
)
type ReferenceGuard struct {
activated, released bool
}
func (r *ReferenceGuard) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r != nil {
e.AddUintptr("self", uintptr(unsafe.Pointer(r)))
e.AddBool("activated", r.activated)
e.AddBool("released", r.released)
}
return nil
}
// ----------------------------------------
type ReferenceCount struct {
count int
}
func (s *ReferenceCount) Activate(guard *ReferenceGuard) {
if guard != nil && !guard.activated {
guard.activated = true
s.count++
}
}
func (s *ReferenceCount) Release(guard *ReferenceGuard) bool {
if guard == nil || !guard.activated || guard.released {
return false
}
guard.released = true
s.count--
return s.count == 0
}
// Take hands over every reference held, leaving none behind.
func (s *ReferenceCount) Take() int {
count := s.count
s.count = 0
return count
}
// Absorb takes on references handed over from elsewhere.
func (s *ReferenceCount) Absorb(count int) {
s.count += count
}
func (s ReferenceCount) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddInt("count", s.count)
return nil
}
// ----------------------------------------
// statsBatch is stats collected while the worker was in one room
type statsBatch struct {
roomID livekit.RoomID
roomName livekit.RoomName
incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
}
func (b statsBatch) isEmpty() bool {
return len(b.incomingPerTrack) == 0 && len(b.outgoingPerTrack) == 0
}
// ----------------------------------------
// StatsWorker handles participant stats
type StatsWorker struct {
next *StatsWorker
ctx context.Context
t TelemetryService
participantID livekit.ParticipantID
participantIdentity livekit.ParticipantIdentity
lock sync.RWMutex
// the room a worker belongs to can change mid-session, so it is mutable state
// guarded by `lock`. it is kept in sync with the key the worker is filed under in
// telemetryService.workers, see telemetryService.reKeyRoom.
roomID livekit.RoomID
roomName livekit.RoomName
// batches sealed off by a room change, they carry the room they were collected
// in and go out on the next flush
sealed []statsBatch
isConnected bool
outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
refCount ReferenceCount
closedAt time.Time
}
func newStatsWorker(
ctx context.Context,
t TelemetryService,
roomID livekit.RoomID,
roomName livekit.RoomName,
participantID livekit.ParticipantID,
identity livekit.ParticipantIdentity,
guard *ReferenceGuard,
) *StatsWorker {
s := &StatsWorker{
ctx: ctx,
t: t,
roomID: roomID,
roomName: roomName,
participantID: participantID,
participantIdentity: identity,
outgoingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat),
incomingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat),
}
s.refCount.Activate(guard)
return s
}
func (s *StatsWorker) OnTrackStat(trackID livekit.TrackID, direction livekit.StreamType, stat *livekit.AnalyticsStat) {
s.lock.Lock()
if direction == livekit.StreamType_DOWNSTREAM {
s.outgoingPerTrack[trackID] = append(s.outgoingPerTrack[trackID], stat)
} else {
s.incomingPerTrack[trackID] = append(s.incomingPerTrack[trackID], stat)
}
s.lock.Unlock()
}
func (s *StatsWorker) ParticipantID() livekit.ParticipantID {
return s.participantID
}
func (s *StatsWorker) RoomID() livekit.RoomID {
s.lock.RLock()
defer s.lock.RUnlock()
return s.roomID
}
// SetRoom re-points the worker at a room.
//
// Stats collected so far are sealed off rather than re-stamped - a room id changes
// because the previous session ended, so what was collected under it belongs to it.
// Sealing keeps the re-key free of any sending, the sealed stats go out on the next
// flush like every other stat.
func (s *StatsWorker) SetRoom(roomID livekit.RoomID, roomName livekit.RoomName) {
s.lock.Lock()
defer s.lock.Unlock()
if s.roomID == roomID && s.roomName == roomName {
return
}
if batch := s.sealStatsLocked(); !batch.isEmpty() {
s.sealed = append(s.sealed, batch)
}
s.roomID = roomID
s.roomName = roomName
}
// sealStatsLocked hands over everything collected since the last seal, stamped
// with the room it was collected in
func (s *StatsWorker) sealStatsLocked() statsBatch {
batch := statsBatch{
roomID: s.roomID,
roomName: s.roomName,
incomingPerTrack: s.incomingPerTrack,
outgoingPerTrack: s.outgoingPerTrack,
}
s.incomingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
return batch
}
func (s *StatsWorker) SetConnected() {
if s == nil {
return
}
s.lock.Lock()
s.isConnected = true
s.lock.Unlock()
}
func (s *StatsWorker) IsConnected() bool {
s.lock.RLock()
defer s.lock.RUnlock()
return s.isConnected
}
func (s *StatsWorker) Flush(now time.Time, closeWait time.Duration) bool {
ts := timestamppb.New(now)
s.lock.Lock()
// anything sealed off by a room change goes out along with the current batch,
// each stamped with the room it was collected in
batches := append(s.sealed, s.sealStatsLocked())
s.sealed = nil
closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > closeWait
s.lock.Unlock()
numTracks := 0
for _, batch := range batches {
numTracks += len(batch.incomingPerTrack) + len(batch.outgoingPerTrack)
}
stats := make([]*livekit.AnalyticsStat, 0, numTracks)
for _, batch := range batches {
stats = s.collectStats(ts, batch, livekit.StreamType_UPSTREAM, stats)
stats = s.collectStats(ts, batch, livekit.StreamType_DOWNSTREAM, stats)
}
if len(stats) > 0 {
s.t.SendStats(s.ctx, stats)
}
return closed
}
func (s *StatsWorker) Close(guard *ReferenceGuard) bool {
s.lock.Lock()
defer s.lock.Unlock()
if !s.refCount.Release(guard) {
return false
}
ok := s.closedAt.IsZero()
if ok {
s.closedAt = time.Now()
}
return ok
}
// ForceClose closes the worker irrespective of outstanding references. Used when a
// worker can no longer be reached through the worker map, so that it drains and is
// reaped instead of lingering in the flush list forever.
//
// Its references are handed over to `successor`, the worker that can be reached in its
// place, so that whoever holds one still has a live worker to close. A ReferenceGuard
// records that it activated some worker, not which one, so leaving them behind would
// strand the successor with references it can never see released.
func (s *StatsWorker) ForceClose(successor *StatsWorker) bool {
s.lock.Lock()
if !s.closedAt.IsZero() {
s.lock.Unlock()
return false
}
s.closedAt = time.Now()
count := s.refCount.Take()
s.lock.Unlock()
if successor != nil && count != 0 {
successor.lock.Lock()
successor.refCount.Absorb(count)
successor.lock.Unlock()
}
return true
}
func (s *StatsWorker) Closed(guard *ReferenceGuard) bool {
s.lock.Lock()
defer s.lock.Unlock()
if s.closedAt.IsZero() {
s.refCount.Activate(guard)
return false
}
return true
}
func (s *StatsWorker) collectStats(
ts *timestamppb.Timestamp,
batch statsBatch,
streamType livekit.StreamType,
stats []*livekit.AnalyticsStat,
) []*livekit.AnalyticsStat {
perTrack := batch.incomingPerTrack
if streamType == livekit.StreamType_DOWNSTREAM {
perTrack = batch.outgoingPerTrack
}
for trackID, analyticsStats := range perTrack {
coalesced := coalesce(analyticsStats)
if coalesced == nil {
continue
}
coalesced.TimeStamp = ts
coalesced.TrackId = string(trackID)
coalesced.Kind = streamType
coalesced.RoomId = string(batch.roomID)
coalesced.ParticipantId = string(s.participantID)
coalesced.RoomName = string(batch.roomName)
stats = append(stats, coalesced)
}
return stats
}
func (s *StatsWorker) MarshalLogObject(e zapcore.ObjectEncoder) error {
if s == nil {
return nil
}
s.lock.RLock()
defer s.lock.RUnlock()
e.AddString("room", string(s.roomName))
e.AddString("roomID", string(s.roomID))
e.AddString("participant", string(s.participantIdentity))
e.AddString("participantID", string(s.participantID))
e.AddBool("isConnected", s.isConnected)
e.AddTime("closedAt", s.closedAt)
e.AddObject("refCount", s.refCount)
return nil
}
// -------------------------------------------------------------------------
// create a single stream and single video layer post aggregation
func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat {
if len(stats) == 0 {
return nil
}
// find aggregates across streams
startTime := time.Time{}
endTime := time.Time{}
scoreSum := float32(0.0) // used for average
minScore := float32(0.0) // min score in batched stats
var scores []float32 // used for median
maxRtt := uint32(0)
maxJitter := uint32(0)
coalescedVideoLayers := make(map[int32]*livekit.AnalyticsVideoLayer)
coalescedStream := &livekit.AnalyticsStream{}
for _, stat := range stats {
if !isValid(stat) {
logger.Warnw("telemetry skipping invalid stat", nil, "stat", stat)
continue
}
// only consider non-zero scores
if stat.Score > 0 {
if minScore == 0 {
minScore = stat.Score
} else if stat.Score < minScore {
minScore = stat.Score
}
scoreSum += stat.Score
scores = append(scores, stat.Score)
}
for _, analyticsStream := range stat.Streams {
start := analyticsStream.StartTime.AsTime()
if startTime.IsZero() || startTime.After(start) {
startTime = start
}
end := analyticsStream.EndTime.AsTime()
if endTime.IsZero() || endTime.Before(end) {
endTime = end
}
if analyticsStream.Rtt > maxRtt {
maxRtt = analyticsStream.Rtt
}
if analyticsStream.Jitter > maxJitter {
maxJitter = analyticsStream.Jitter
}
coalescedStream.PrimaryPackets += analyticsStream.PrimaryPackets
coalescedStream.PrimaryBytes += analyticsStream.PrimaryBytes
coalescedStream.RetransmitPackets += analyticsStream.RetransmitPackets
coalescedStream.RetransmitBytes += analyticsStream.RetransmitBytes
coalescedStream.PaddingPackets += analyticsStream.PaddingPackets
coalescedStream.PaddingBytes += analyticsStream.PaddingBytes
coalescedStream.PacketsLost += analyticsStream.PacketsLost
coalescedStream.PacketsOutOfOrder += analyticsStream.PacketsOutOfOrder
coalescedStream.Frames += analyticsStream.Frames
coalescedStream.Nacks += analyticsStream.Nacks
coalescedStream.Plis += analyticsStream.Plis
coalescedStream.Firs += analyticsStream.Firs
for _, videoLayer := range analyticsStream.VideoLayers {
coalescedVideoLayer := coalescedVideoLayers[videoLayer.Layer]
if coalescedVideoLayer == nil {
coalescedVideoLayer = protoutils.CloneProto(videoLayer)
coalescedVideoLayers[videoLayer.Layer] = coalescedVideoLayer
} else {
coalescedVideoLayer.Packets += videoLayer.Packets
coalescedVideoLayer.Bytes += videoLayer.Bytes
coalescedVideoLayer.Frames += videoLayer.Frames
}
}
}
}
coalescedStream.StartTime = timestamppb.New(startTime)
coalescedStream.EndTime = timestamppb.New(endTime)
coalescedStream.Rtt = maxRtt
coalescedStream.Jitter = maxJitter
// whittle it down to one video layer, just the max available layer
maxVideoLayer := int32(-1)
for _, coalescedVideoLayer := range coalescedVideoLayers {
if maxVideoLayer == -1 || maxVideoLayer < coalescedVideoLayer.Layer {
maxVideoLayer = coalescedVideoLayer.Layer
coalescedStream.VideoLayers = []*livekit.AnalyticsVideoLayer{coalescedVideoLayer}
}
}
stat := &livekit.AnalyticsStat{
MinScore: minScore,
MedianScore: utils.Median(scores),
Streams: []*livekit.AnalyticsStream{coalescedStream},
Mime: stats[len(stats)-1].Mime, // use the latest Mime
}
numScores := len(scores)
if numScores > 0 {
stat.Score = scoreSum / float32(numScores)
}
return stat
}
type CondensedStat struct {
StartTime time.Time
EndTime time.Time
Bytes uint64
Packets uint32
PacketsLost uint32
Frames uint32
}
func CondenseStat(stat *livekit.AnalyticsStat) (ps CondensedStat, ok bool) {
if ok = isValid(stat); !ok {
return
}
for _, stream := range stat.Streams {
startTime := stream.StartTime.AsTime()
endTime := stream.EndTime.AsTime()
if ps.StartTime.IsZero() || startTime.Before(ps.StartTime) {
ps.StartTime = startTime
}
if endTime.After(ps.EndTime) {
ps.EndTime = endTime
}
ps.Bytes += stream.PrimaryBytes
ps.Packets += stream.PrimaryPackets
ps.PacketsLost += stream.PacketsLost
if stream.Frames > ps.Frames {
ps.Frames = stream.Frames
}
}
return
}
func isValid(stat *livekit.AnalyticsStat) bool {
for _, analyticsStream := range stat.Streams {
if int32(analyticsStream.PrimaryPackets) < 0 ||
int64(analyticsStream.PrimaryBytes) < 0 ||
int32(analyticsStream.RetransmitPackets) < 0 ||
int64(analyticsStream.RetransmitBytes) < 0 ||
int32(analyticsStream.PaddingPackets) < 0 ||
int64(analyticsStream.PaddingBytes) < 0 ||
int32(analyticsStream.PacketsLost) < 0 ||
int32(analyticsStream.PacketsOutOfOrder) < 0 ||
int32(analyticsStream.Frames) < 0 ||
int32(analyticsStream.Nacks) < 0 ||
int32(analyticsStream.Plis) < 0 ||
int32(analyticsStream.Firs) < 0 {
return false
}
for _, videoLayer := range analyticsStream.VideoLayers {
if int32(videoLayer.Packets) < 0 ||
int64(videoLayer.Bytes) < 0 ||
int32(videoLayer.Frames) < 0 {
return false
}
}
}
return true
}

View File

@@ -0,0 +1,102 @@
package telemetry
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zapcore"
"github.com/livekit/protocol/livekit"
)
func TestStatsWorker(t *testing.T) {
t.Run("reference counted close works", func(t *testing.T) {
var g0, g1 ReferenceGuard
w := newStatsWorker(t.Context(), nil, "", "", "", "", &g0)
require.False(t, w.Closed(&g1))
require.False(t, w.Close(&g0))
require.False(t, w.Closed(&g1))
require.True(t, w.Close(&g1))
require.True(t, w.Closed(&g1))
})
// a ReferenceGuard records that it activated some worker, not which one, so a
// superseded worker has to hand its references to the one reachable in its place
t.Run("force close hands references to the successor", func(t *testing.T) {
t.Run("a guard shared by both workers", func(t *testing.T) {
// the second worker never got a reference, the guard was already activated
var g ReferenceGuard
superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &g)
survivor := newStatsWorker(t.Context(), nil, "", "", "", "", &g)
require.Equal(t, 1, superseded.refCount.count)
require.Equal(t, 0, survivor.refCount.count)
require.True(t, superseded.ForceClose(survivor))
require.Equal(t, 0, superseded.refCount.count)
require.Equal(t, 1, survivor.refCount.count)
// without the hand over this would leave the survivor at -1 and never closed
require.True(t, survivor.Close(&g))
require.True(t, survivor.Closed(&g))
})
t.Run("a guard per worker", func(t *testing.T) {
var gSuperseded, gSurvivor ReferenceGuard
superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &gSuperseded)
survivor := newStatsWorker(t.Context(), nil, "", "", "", "", &gSurvivor)
require.True(t, superseded.ForceClose(survivor))
require.Equal(t, 2, survivor.refCount.count)
// the superseded worker's owner departs, it must not close the survivor early
require.False(t, survivor.Close(&gSuperseded))
require.True(t, survivor.Close(&gSurvivor))
})
t.Run("closing an already closed worker holds on to its references", func(t *testing.T) {
var g ReferenceGuard
superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &g)
survivor := newStatsWorker(t.Context(), nil, "", "", "", "", nil)
require.True(t, superseded.ForceClose(nil))
require.False(t, superseded.ForceClose(survivor))
require.Equal(t, 0, survivor.refCount.count)
})
})
t.Run("logging a nil worker does not panic", func(t *testing.T) {
var w *StatsWorker
require.NoError(t, w.MarshalLogObject(zapcore.NewMapObjectEncoder()))
})
}
func TestGetOrCreateWorkerReleasedGuard(t *testing.T) {
// ParticipantActive overtaken by the participant's close arrives with a guard that
// ParticipantLeft already released. It must not replace the closed worker with one
// nothing can release.
ts := &telemetryService{workers: make(map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker)}
roomID, pID := livekit.RoomID("room"), livekit.ParticipantID("participant")
var g ReferenceGuard
w, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g)
require.False(t, found)
require.True(t, w.Close(&g))
t.Run("closed worker still in the map", func(t *testing.T) {
late, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g)
require.True(t, found)
require.Same(t, w, late)
require.Same(t, w, ts.workers[roomID][pID])
})
t.Run("closed worker already reaped", func(t *testing.T) {
delete(ts.workers[roomID], pID)
late, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g)
require.True(t, found)
require.Nil(t, late)
require.Empty(t, ts.workers[roomID])
late.SetConnected()
})
}

View File

@@ -0,0 +1,234 @@
// Code generated by counterfeiter. DO NOT EDIT.
package telemetryfakes
import (
"context"
"sync"
"github.com/livekit/livekit-server/pkg/telemetry"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/observability/roomobs"
)
type FakeAnalyticsService struct {
RoomProjectReporterStub func(context.Context) roomobs.ProjectReporter
roomProjectReporterMutex sync.RWMutex
roomProjectReporterArgsForCall []struct {
arg1 context.Context
}
roomProjectReporterReturns struct {
result1 roomobs.ProjectReporter
}
roomProjectReporterReturnsOnCall map[int]struct {
result1 roomobs.ProjectReporter
}
SendEventStub func(context.Context, *livekit.AnalyticsEvent)
sendEventMutex sync.RWMutex
sendEventArgsForCall []struct {
arg1 context.Context
arg2 *livekit.AnalyticsEvent
}
SendNodeRoomStatesStub func(context.Context, *livekit.AnalyticsNodeRooms)
sendNodeRoomStatesMutex sync.RWMutex
sendNodeRoomStatesArgsForCall []struct {
arg1 context.Context
arg2 *livekit.AnalyticsNodeRooms
}
SendStatsStub func(context.Context, []*livekit.AnalyticsStat)
sendStatsMutex sync.RWMutex
sendStatsArgsForCall []struct {
arg1 context.Context
arg2 []*livekit.AnalyticsStat
}
invocations map[string][][]interface{}
invocationsMutex sync.RWMutex
}
func (fake *FakeAnalyticsService) RoomProjectReporter(arg1 context.Context) roomobs.ProjectReporter {
fake.roomProjectReporterMutex.Lock()
ret, specificReturn := fake.roomProjectReporterReturnsOnCall[len(fake.roomProjectReporterArgsForCall)]
fake.roomProjectReporterArgsForCall = append(fake.roomProjectReporterArgsForCall, struct {
arg1 context.Context
}{arg1})
stub := fake.RoomProjectReporterStub
fakeReturns := fake.roomProjectReporterReturns
fake.recordInvocation("RoomProjectReporter", []interface{}{arg1})
fake.roomProjectReporterMutex.Unlock()
if stub != nil {
return stub(arg1)
}
if specificReturn {
return ret.result1
}
return fakeReturns.result1
}
func (fake *FakeAnalyticsService) RoomProjectReporterCallCount() int {
fake.roomProjectReporterMutex.RLock()
defer fake.roomProjectReporterMutex.RUnlock()
return len(fake.roomProjectReporterArgsForCall)
}
func (fake *FakeAnalyticsService) RoomProjectReporterCalls(stub func(context.Context) roomobs.ProjectReporter) {
fake.roomProjectReporterMutex.Lock()
defer fake.roomProjectReporterMutex.Unlock()
fake.RoomProjectReporterStub = stub
}
func (fake *FakeAnalyticsService) RoomProjectReporterArgsForCall(i int) context.Context {
fake.roomProjectReporterMutex.RLock()
defer fake.roomProjectReporterMutex.RUnlock()
argsForCall := fake.roomProjectReporterArgsForCall[i]
return argsForCall.arg1
}
func (fake *FakeAnalyticsService) RoomProjectReporterReturns(result1 roomobs.ProjectReporter) {
fake.roomProjectReporterMutex.Lock()
defer fake.roomProjectReporterMutex.Unlock()
fake.RoomProjectReporterStub = nil
fake.roomProjectReporterReturns = struct {
result1 roomobs.ProjectReporter
}{result1}
}
func (fake *FakeAnalyticsService) RoomProjectReporterReturnsOnCall(i int, result1 roomobs.ProjectReporter) {
fake.roomProjectReporterMutex.Lock()
defer fake.roomProjectReporterMutex.Unlock()
fake.RoomProjectReporterStub = nil
if fake.roomProjectReporterReturnsOnCall == nil {
fake.roomProjectReporterReturnsOnCall = make(map[int]struct {
result1 roomobs.ProjectReporter
})
}
fake.roomProjectReporterReturnsOnCall[i] = struct {
result1 roomobs.ProjectReporter
}{result1}
}
func (fake *FakeAnalyticsService) SendEvent(arg1 context.Context, arg2 *livekit.AnalyticsEvent) {
fake.sendEventMutex.Lock()
fake.sendEventArgsForCall = append(fake.sendEventArgsForCall, struct {
arg1 context.Context
arg2 *livekit.AnalyticsEvent
}{arg1, arg2})
stub := fake.SendEventStub
fake.recordInvocation("SendEvent", []interface{}{arg1, arg2})
fake.sendEventMutex.Unlock()
if stub != nil {
fake.SendEventStub(arg1, arg2)
}
}
func (fake *FakeAnalyticsService) SendEventCallCount() int {
fake.sendEventMutex.RLock()
defer fake.sendEventMutex.RUnlock()
return len(fake.sendEventArgsForCall)
}
func (fake *FakeAnalyticsService) SendEventCalls(stub func(context.Context, *livekit.AnalyticsEvent)) {
fake.sendEventMutex.Lock()
defer fake.sendEventMutex.Unlock()
fake.SendEventStub = stub
}
func (fake *FakeAnalyticsService) SendEventArgsForCall(i int) (context.Context, *livekit.AnalyticsEvent) {
fake.sendEventMutex.RLock()
defer fake.sendEventMutex.RUnlock()
argsForCall := fake.sendEventArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
}
func (fake *FakeAnalyticsService) SendNodeRoomStates(arg1 context.Context, arg2 *livekit.AnalyticsNodeRooms) {
fake.sendNodeRoomStatesMutex.Lock()
fake.sendNodeRoomStatesArgsForCall = append(fake.sendNodeRoomStatesArgsForCall, struct {
arg1 context.Context
arg2 *livekit.AnalyticsNodeRooms
}{arg1, arg2})
stub := fake.SendNodeRoomStatesStub
fake.recordInvocation("SendNodeRoomStates", []interface{}{arg1, arg2})
fake.sendNodeRoomStatesMutex.Unlock()
if stub != nil {
fake.SendNodeRoomStatesStub(arg1, arg2)
}
}
func (fake *FakeAnalyticsService) SendNodeRoomStatesCallCount() int {
fake.sendNodeRoomStatesMutex.RLock()
defer fake.sendNodeRoomStatesMutex.RUnlock()
return len(fake.sendNodeRoomStatesArgsForCall)
}
func (fake *FakeAnalyticsService) SendNodeRoomStatesCalls(stub func(context.Context, *livekit.AnalyticsNodeRooms)) {
fake.sendNodeRoomStatesMutex.Lock()
defer fake.sendNodeRoomStatesMutex.Unlock()
fake.SendNodeRoomStatesStub = stub
}
func (fake *FakeAnalyticsService) SendNodeRoomStatesArgsForCall(i int) (context.Context, *livekit.AnalyticsNodeRooms) {
fake.sendNodeRoomStatesMutex.RLock()
defer fake.sendNodeRoomStatesMutex.RUnlock()
argsForCall := fake.sendNodeRoomStatesArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
}
func (fake *FakeAnalyticsService) SendStats(arg1 context.Context, arg2 []*livekit.AnalyticsStat) {
var arg2Copy []*livekit.AnalyticsStat
if arg2 != nil {
arg2Copy = make([]*livekit.AnalyticsStat, len(arg2))
copy(arg2Copy, arg2)
}
fake.sendStatsMutex.Lock()
fake.sendStatsArgsForCall = append(fake.sendStatsArgsForCall, struct {
arg1 context.Context
arg2 []*livekit.AnalyticsStat
}{arg1, arg2Copy})
stub := fake.SendStatsStub
fake.recordInvocation("SendStats", []interface{}{arg1, arg2Copy})
fake.sendStatsMutex.Unlock()
if stub != nil {
fake.SendStatsStub(arg1, arg2)
}
}
func (fake *FakeAnalyticsService) SendStatsCallCount() int {
fake.sendStatsMutex.RLock()
defer fake.sendStatsMutex.RUnlock()
return len(fake.sendStatsArgsForCall)
}
func (fake *FakeAnalyticsService) SendStatsCalls(stub func(context.Context, []*livekit.AnalyticsStat)) {
fake.sendStatsMutex.Lock()
defer fake.sendStatsMutex.Unlock()
fake.SendStatsStub = stub
}
func (fake *FakeAnalyticsService) SendStatsArgsForCall(i int) (context.Context, []*livekit.AnalyticsStat) {
fake.sendStatsMutex.RLock()
defer fake.sendStatsMutex.RUnlock()
argsForCall := fake.sendStatsArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2
}
func (fake *FakeAnalyticsService) Invocations() map[string][][]interface{} {
fake.invocationsMutex.RLock()
defer fake.invocationsMutex.RUnlock()
copiedInvocations := map[string][][]interface{}{}
for key, value := range fake.invocations {
copiedInvocations[key] = value
}
return copiedInvocations
}
func (fake *FakeAnalyticsService) recordInvocation(key string, args []interface{}) {
fake.invocationsMutex.Lock()
defer fake.invocationsMutex.Unlock()
if fake.invocations == nil {
fake.invocations = map[string][][]interface{}{}
}
if fake.invocations[key] == nil {
fake.invocations[key] = [][]interface{}{}
}
fake.invocations[key] = append(fake.invocations[key], args)
}
var _ telemetry.AnalyticsService = new(FakeAnalyticsService)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,436 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package telemetry
import (
"context"
"sync"
"time"
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
"github.com/livekit/livekit-server/pkg/utils"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/webhook"
)
//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
//counterfeiter:generate . TelemetryService
type TelemetryService interface {
// TrackStats is called periodically for each track in both directions (published/subscribed)
TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat)
// events
RoomStarted(ctx context.Context, room *livekit.Room)
RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason)
// ParticipantJoined - a participant establishes signal connection to a room
ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard)
// ParticipantActive - a participant establishes media connection
ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, isWarp bool, guard *ReferenceGuard)
// ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection
ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason)
// ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before
ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard)
// RoomIDChanged - the room kept its session, but got a different id (a provisional room id
// replaced by the resolved one), re-keys the stats workers of every participant in the room
RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room)
// TrackPublishRequested - a publication attempt has been received
TrackPublishRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
// TrackPublished - a publication attempt has been successful
TrackPublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
// TrackUnpublished - a participant unpublished a track
TrackUnpublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, wasPublishedLocally bool, shouldSendEvent bool)
// TrackSubscribeRequested - a participant requested to subscribe to a track
TrackSubscribeRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
// TrackSubscribed - a participant subscribed to a track successfully
TrackSubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool)
// TrackUnsubscribed - a participant unsubscribed from a track successfully
TrackUnsubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool)
// TrackSubscribeFailed - failure to subscribe to a track
TrackSubscribeFailed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool)
// TrackMuted - the publisher has muted the Track
TrackMuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
// TrackUnmuted - the publisher has muted the Track
TrackUnmuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
// TrackPublishedUpdate - track metadata has been updated
TrackPublishedUpdate(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
// TrackMaxSubscribedVideoQuality - publisher is notified of the max quality subscribers desire
TrackMaxSubscribedVideoQuality(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality)
TrackPublishRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats)
TrackSubscribeRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats)
EgressStarted(ctx context.Context, info *livekit.EgressInfo)
EgressUpdated(ctx context.Context, info *livekit.EgressInfo)
EgressEnded(ctx context.Context, info *livekit.EgressInfo)
IngressCreated(ctx context.Context, info *livekit.IngressInfo)
IngressDeleted(ctx context.Context, info *livekit.IngressInfo)
IngressStarted(ctx context.Context, info *livekit.IngressInfo)
IngressUpdated(ctx context.Context, info *livekit.IngressInfo)
IngressEnded(ctx context.Context, info *livekit.IngressInfo)
LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms)
Report(ctx context.Context, reportInfo *livekit.ReportInfo)
APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo)
Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo)
// helpers
AnalyticsService
NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo)
FlushStats()
}
// -----------------------------
var _ TelemetryService = (*NullTelemetryService)(nil)
type NullTelemetryService struct {
NullAnalyticService
}
func (n NullTelemetryService) TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) {
}
func (n NullTelemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {}
func (n NullTelemetryService) RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) {
}
func (n NullTelemetryService) ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) {
}
func (n NullTelemetryService) ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, isWarp bool, guard *ReferenceGuard) {
}
func (n NullTelemetryService) ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) {
}
func (n NullTelemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) {
}
func (n NullTelemetryService) RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) {
}
func (n NullTelemetryService) TrackPublishRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) {
}
func (n NullTelemetryService) TrackPublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) {
}
func (n NullTelemetryService) TrackUnpublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, wasPublishedLocally bool, shouldSendEvent bool) {
}
func (n NullTelemetryService) TrackSubscribeRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
}
func (n NullTelemetryService) TrackSubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool) {
}
func (n NullTelemetryService) TrackUnsubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool) {
}
func (n NullTelemetryService) TrackSubscribeFailed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) {
}
func (n NullTelemetryService) TrackMuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
}
func (n NullTelemetryService) TrackUnmuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
}
func (n NullTelemetryService) TrackPublishedUpdate(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
}
func (n NullTelemetryService) TrackMaxSubscribedVideoQuality(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) {
}
func (n NullTelemetryService) TrackPublishRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) {
}
func (n NullTelemetryService) TrackSubscribeRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) {
}
func (n NullTelemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {}
func (n NullTelemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {}
func (n NullTelemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {}
func (n NullTelemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {}
func (n NullTelemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {}
func (n NullTelemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {}
func (n NullTelemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {}
func (n NullTelemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {}
func (n NullTelemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {}
func (n NullTelemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {}
func (n NullTelemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {}
func (n NullTelemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {}
func (n NullTelemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) {
}
func (n NullTelemetryService) FlushStats() {}
// -----------------------------
const (
workerCleanupWait = 3 * time.Minute
jobsQueueMinSize = 2048
telemetryStatsUpdateInterval = time.Second * 30
)
type telemetryService struct {
AnalyticsService
notifier webhook.QueuedNotifier
jobsQueue *utils.OpsQueue
workersMu sync.RWMutex
workers map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker
workerList *StatsWorker
flushMu sync.Mutex
}
func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService {
t := &telemetryService{
AnalyticsService: analytics,
notifier: notifier,
jobsQueue: utils.NewOpsQueue(utils.OpsQueueParams{
Name: "telemetry",
MinSize: jobsQueueMinSize,
FlushOnStop: true,
Logger: logger.GetLogger(),
}),
workers: make(map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker),
}
t.jobsQueue.Start()
go t.run()
return t
}
func (t *telemetryService) FlushStats() {
t.flushMu.Lock()
defer t.flushMu.Unlock()
t.workersMu.RLock()
worker := t.workerList
t.workersMu.RUnlock()
now := time.Now()
var prev, reap *StatsWorker
for worker != nil {
next := worker.next
if closed := worker.Flush(now, workerCleanupWait); closed {
if prev == nil {
// this worker was at the head of the list
t.workersMu.Lock()
p := &t.workerList
for *p != worker {
// new workers have been added. scan until we find the one
// immediately before this
prev = *p
p = &prev.next
}
*p = worker.next
t.workersMu.Unlock()
} else {
prev.next = worker.next
}
worker.next = reap
reap = worker
} else {
prev = worker
}
worker = next
}
if reap != nil {
t.workersMu.Lock()
for reap != nil {
roomID := reap.RoomID()
if roomWorkers := t.workers[roomID]; reap == roomWorkers[reap.participantID] {
delete(roomWorkers, reap.participantID)
if len(roomWorkers) == 0 {
delete(t.workers, roomID)
}
}
reap = reap.next
}
t.workersMu.Unlock()
}
}
func (t *telemetryService) run() {
for range time.Tick(telemetryStatsUpdateInterval) {
t.FlushStats()
}
}
func (t *telemetryService) enqueue(op func()) {
t.jobsQueue.Enqueue(op)
}
func (t *telemetryService) getWorker(roomID livekit.RoomID, participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) {
t.workersMu.RLock()
defer t.workersMu.RUnlock()
worker, ok = t.workers[roomID][participantID]
return
}
func (t *telemetryService) getOrCreateWorker(
ctx context.Context,
roomID livekit.RoomID,
roomName livekit.RoomName,
participantID livekit.ParticipantID,
participantIdentity livekit.ParticipantIdentity,
guard *ReferenceGuard,
) (*StatsWorker, bool) {
t.workersMu.Lock()
defer t.workersMu.Unlock()
if roomID == "" {
logger.Warnw(
"telemetry stats worker keyed under an empty room id", nil,
"room", roomName,
"participant", participantIdentity,
"participantID", participantID,
"guard", guard,
)
}
roomWorkers := t.workers[roomID]
worker, ok := roomWorkers[participantID]
if ok && !worker.Closed(guard) {
return worker, true
}
// only ParticipantLeft releases a guard, so a released guard is a call landing after
// the participant left, e.g. ParticipantActive overtaken by the close. Do not create
// a worker nothing can ever release. The closed worker, if not yet reaped, is returned
// as found, otherwise nil is
if guard != nil && guard.released {
return worker, true
}
existingIsConnected := false
if ok {
existingIsConnected = worker.IsConnected()
}
// a guard references at most once, so a nil or already activated guard leaves the
// new worker with no references and its owner's release drives it negative
if guard == nil || guard.activated {
logger.Infow(
"telemetry stats worker created without a reference",
"room", roomName,
"roomID", roomID,
"participant", participantIdentity,
"participantID", participantID,
"guard", guard,
"replacedClosed", ok,
"existing", worker,
)
}
worker = newStatsWorker(
ctx,
t,
roomID,
roomName,
participantID,
participantIdentity,
guard,
)
if existingIsConnected {
worker.SetConnected()
}
if roomWorkers == nil {
roomWorkers = make(map[livekit.ParticipantID]*StatsWorker)
t.workers[roomID] = roomWorkers
}
roomWorkers[participantID] = worker
worker.next = t.workerList
t.workerList = worker
return worker, false
}
// reKeyRoom files every one of a room's stats workers under `roomID` instead of
// `prevRoomID`.
//
// A room can be restarted while participants are connected and reporting stats, which
// gives it a new id. As every worker of the room moves at once, the move is a single map
// splice - the workers themselves are untouched and keep their place in the flush list.
// Each worker then seals off what it collected under `prevRoomID` so those stats stay
// attributed to the session that ended (see StatsWorker.SetRoom).
func (t *telemetryService) reKeyRoom(prevRoomID livekit.RoomID, roomID livekit.RoomID, roomName livekit.RoomName) {
if prevRoomID == roomID {
return
}
t.workersMu.Lock()
defer t.workersMu.Unlock()
roomWorkers := t.workers[prevRoomID]
if len(roomWorkers) == 0 {
delete(t.workers, prevRoomID)
return
}
delete(t.workers, prevRoomID)
existing := t.workers[roomID]
if existing == nil {
t.workers[roomID] = roomWorkers
} else {
// should not happen as a room id is only ever replaced by a freshly minted one
logger.Warnw(
"telemetry re-keying room into an existing entry", nil,
"prevRoomID", prevRoomID,
"room", roomName,
"roomID", roomID,
"numWorkers", len(roomWorkers),
"numExistingWorkers", len(existing),
)
}
for participantID, worker := range roomWorkers {
if existing != nil {
if survivor, ok := existing[participantID]; ok {
// only one worker can be keyed at (room, participant) and the one already
// filed there wins, close the superseded one so that it drains and is
// reaped instead of lingering in the flush list unreachable
forceClosed := worker.ForceClose(survivor)
logger.Infow(
"telemetry force closing superseded stats worker",
"prevRoomID", prevRoomID,
"roomID", roomID,
"participantID", participantID,
"forceClosed", forceClosed,
"superseded", worker,
"survivor", survivor,
)
if forceClosed {
prometheus.SubParticipant()
}
continue
}
existing[participantID] = worker
}
worker.SetRoom(roomID, roomName)
}
logger.Infow(
"telemetry re-keyed room",
"prevRoomID", prevRoomID,
"room", roomName,
"roomID", roomID,
"numWorkers", len(roomWorkers),
)
}
func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {
t.enqueue(func() {
t.SendNodeRoomStates(ctx, info)
})
}