snapshot of https://github.com/livekit/livekit.git @ 08c5433449c106943231037d0b892895f6e92d3b (2026-09-15)
This commit is contained in:
118
pkg/telemetry/analyticsservice.go
Normal file
118
pkg/telemetry/analyticsservice.go
Normal file
@@ -0,0 +1,118 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.uber.org/atomic"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/observability/roomobs"
|
||||
"github.com/livekit/protocol/rpc"
|
||||
"github.com/livekit/protocol/utils/guid"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/config"
|
||||
"github.com/livekit/livekit-server/pkg/routing"
|
||||
)
|
||||
|
||||
//counterfeiter:generate . AnalyticsService
|
||||
type AnalyticsService interface {
|
||||
SendStats(ctx context.Context, stats []*livekit.AnalyticsStat)
|
||||
SendEvent(ctx context.Context, events *livekit.AnalyticsEvent)
|
||||
SendNodeRoomStates(ctx context.Context, nodeRooms *livekit.AnalyticsNodeRooms)
|
||||
RoomProjectReporter(ctx context.Context) roomobs.ProjectReporter
|
||||
}
|
||||
|
||||
// ----------------------------
|
||||
|
||||
var _ AnalyticsService = &NullAnalyticService{}
|
||||
|
||||
type NullAnalyticService struct{}
|
||||
|
||||
func (n NullAnalyticService) SendStats(_ context.Context, _ []*livekit.AnalyticsStat) {}
|
||||
func (n NullAnalyticService) SendEvent(_ context.Context, _ *livekit.AnalyticsEvent) {}
|
||||
func (n NullAnalyticService) SendNodeRoomStates(_ context.Context, _ *livekit.AnalyticsNodeRooms) {}
|
||||
func (n NullAnalyticService) RoomProjectReporter(_ctx context.Context) roomobs.ProjectReporter {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ----------------------------
|
||||
|
||||
type analyticsService struct {
|
||||
analyticsKey string
|
||||
nodeID string
|
||||
sequenceNumber atomic.Uint64
|
||||
|
||||
events rpc.AnalyticsRecorderService_IngestEventsClient
|
||||
stats rpc.AnalyticsRecorderService_IngestStatsClient
|
||||
nodeRooms rpc.AnalyticsRecorderService_IngestNodeRoomStatesClient
|
||||
}
|
||||
|
||||
func NewAnalyticsService(_ *config.Config, currentNode routing.LocalNode) AnalyticsService {
|
||||
return &analyticsService{
|
||||
analyticsKey: "", // TODO: conf.AnalyticsKey
|
||||
nodeID: string(currentNode.NodeID()),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *analyticsService) SendStats(_ context.Context, stats []*livekit.AnalyticsStat) {
|
||||
if a.stats == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, stat := range stats {
|
||||
stat.Id = guid.New("AS_")
|
||||
stat.AnalyticsKey = a.analyticsKey
|
||||
stat.Node = a.nodeID
|
||||
}
|
||||
if err := a.stats.Send(&livekit.AnalyticsStats{Stats: stats}); err != nil {
|
||||
logger.Errorw("failed to send stats", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *analyticsService) SendEvent(_ context.Context, event *livekit.AnalyticsEvent) {
|
||||
if a.events == nil {
|
||||
return
|
||||
}
|
||||
|
||||
event.Id = guid.New("AE_")
|
||||
event.NodeId = a.nodeID
|
||||
event.AnalyticsKey = a.analyticsKey
|
||||
if err := a.events.Send(&livekit.AnalyticsEvents{
|
||||
Events: []*livekit.AnalyticsEvent{event},
|
||||
}); err != nil {
|
||||
logger.Errorw("failed to send event", err, "eventType", event.Type.String())
|
||||
}
|
||||
}
|
||||
|
||||
func (a *analyticsService) SendNodeRoomStates(_ context.Context, nodeRooms *livekit.AnalyticsNodeRooms) {
|
||||
if a.nodeRooms == nil {
|
||||
return
|
||||
}
|
||||
|
||||
nodeRooms.NodeId = a.nodeID
|
||||
nodeRooms.SequenceNumber = a.sequenceNumber.Add(1)
|
||||
nodeRooms.Timestamp = timestamppb.Now()
|
||||
if err := a.nodeRooms.Send(nodeRooms); err != nil {
|
||||
logger.Errorw("failed to send node room states", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *analyticsService) RoomProjectReporter(_ context.Context) roomobs.ProjectReporter {
|
||||
return roomobs.NewNoopProjectReporter()
|
||||
}
|
||||
659
pkg/telemetry/events.go
Normal file
659
pkg/telemetry/events.go
Normal file
@@ -0,0 +1,659 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
"github.com/livekit/protocol/codecs/mime"
|
||||
"github.com/livekit/protocol/egress"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/utils/guid"
|
||||
"github.com/livekit/protocol/webhook"
|
||||
)
|
||||
|
||||
func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent, opts ...webhook.NotifyOption) {
|
||||
if t.notifier == nil {
|
||||
return
|
||||
}
|
||||
|
||||
event.CreatedAt = time.Now().Unix()
|
||||
event.Id = guid.New("EV_")
|
||||
|
||||
if err := t.notifier.QueueNotify(ctx, event, opts...); err != nil {
|
||||
logger.Warnw("failed to notify webhook", err, "event", event.Event)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *telemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventRoomStarted,
|
||||
Room: room,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, &livekit.AnalyticsEvent{
|
||||
Type: livekit.AnalyticsEventType_ROOM_CREATED,
|
||||
Timestamp: ×tamppb.Timestamp{Seconds: room.CreationTime},
|
||||
Room: room,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventRoomFinished,
|
||||
Room: room,
|
||||
RoomEndReason: reason,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, &livekit.AnalyticsEvent{
|
||||
Type: livekit.AnalyticsEventType_ROOM_ENDED,
|
||||
Timestamp: timestamppb.Now(),
|
||||
RoomId: room.Sid,
|
||||
Room: room,
|
||||
RoomEndReason: reason,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) ParticipantJoined(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participant *livekit.ParticipantInfo,
|
||||
clientInfo *livekit.ClientInfo,
|
||||
clientMeta *livekit.AnalyticsClientMeta,
|
||||
shouldSendEvent bool,
|
||||
guard *ReferenceGuard,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
_, found := t.getOrCreateWorker(
|
||||
ctx,
|
||||
livekit.RoomID(room.Sid),
|
||||
livekit.RoomName(room.Name),
|
||||
livekit.ParticipantID(participant.Sid),
|
||||
livekit.ParticipantIdentity(participant.Identity),
|
||||
guard,
|
||||
)
|
||||
if !found {
|
||||
prometheus.IncrementParticipantRtcConnected(1)
|
||||
prometheus.AddParticipant()
|
||||
}
|
||||
|
||||
if shouldSendEvent {
|
||||
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_JOINED, room, participant)
|
||||
ev.ClientInfo = clientInfo
|
||||
ev.ClientMeta = clientMeta
|
||||
t.SendEvent(ctx, ev)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) ParticipantActive(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participant *livekit.ParticipantInfo,
|
||||
clientMeta *livekit.AnalyticsClientMeta,
|
||||
isMigration bool,
|
||||
isWarp bool,
|
||||
guard *ReferenceGuard,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
if !isMigration {
|
||||
// a participant is considered "joined" only when they become "active"
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventParticipantJoined,
|
||||
Room: room,
|
||||
Participant: participant,
|
||||
})
|
||||
}
|
||||
|
||||
worker, found := t.getOrCreateWorker(
|
||||
ctx,
|
||||
livekit.RoomID(room.Sid),
|
||||
livekit.RoomName(room.Name),
|
||||
livekit.ParticipantID(participant.Sid),
|
||||
livekit.ParticipantIdentity(participant.Identity),
|
||||
guard,
|
||||
)
|
||||
if !found {
|
||||
prometheus.AddParticipant()
|
||||
}
|
||||
worker.SetConnected()
|
||||
prometheus.IncrementParticipantRtcActive(1, isWarp)
|
||||
|
||||
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, room, participant)
|
||||
ev.ClientMeta = clientMeta
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) ParticipantResumed(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participant *livekit.ParticipantInfo,
|
||||
nodeID livekit.NodeID,
|
||||
reason livekit.ReconnectReason,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
// create a worker if needed.
|
||||
//
|
||||
// Signalling channel stats collector and media channel stats collector could both call
|
||||
// ParticipantJoined and ParticipantLeft.
|
||||
//
|
||||
// On a resume, the signalling channel collector would call `ParticipantLeft` which would close
|
||||
// the corresponding participant's stats worker.
|
||||
//
|
||||
// So, on a successful resume, create the worker if needed.
|
||||
_, found := t.getOrCreateWorker(
|
||||
ctx,
|
||||
livekit.RoomID(room.Sid),
|
||||
livekit.RoomName(room.Name),
|
||||
livekit.ParticipantID(participant.Sid),
|
||||
livekit.ParticipantIdentity(participant.Identity),
|
||||
nil,
|
||||
)
|
||||
if !found {
|
||||
prometheus.AddParticipant()
|
||||
}
|
||||
|
||||
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_RESUMED, room, participant)
|
||||
ev.ClientMeta = &livekit.AnalyticsClientMeta{
|
||||
Node: string(nodeID),
|
||||
ReconnectReason: reason,
|
||||
}
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
// RoomIDChanged re-keys the room's stats workers.
|
||||
//
|
||||
// NOTE: this shares the queue with the stats and participant events it races with, so
|
||||
// ops raised before the id changed (carrying `prevRoomID`) are applied before the
|
||||
// re-key and ops raised after it (carrying the new id) are applied after. Callers
|
||||
// should raise this as soon as the room starts reporting the new id.
|
||||
func (t *telemetryService) RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) {
|
||||
t.enqueue(func() {
|
||||
t.reKeyRoom(prevRoomID, livekit.RoomID(room.Sid), livekit.RoomName(room.Name))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) ParticipantLeft(ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participant *livekit.ParticipantInfo,
|
||||
shouldSendEvent bool,
|
||||
guard *ReferenceGuard,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
isConnected := false
|
||||
if worker, ok := t.getWorker(livekit.RoomID(room.Sid), livekit.ParticipantID(participant.Sid)); ok {
|
||||
isConnected = worker.IsConnected()
|
||||
if worker.Close(guard) {
|
||||
prometheus.SubParticipant()
|
||||
} else {
|
||||
logger.Infow(
|
||||
"stats worker active",
|
||||
"room", room.Name,
|
||||
"roomID", room.Sid,
|
||||
"participant", participant.Identity,
|
||||
"participantID", participant.Sid,
|
||||
"worker", worker,
|
||||
"guard", guard,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if shouldSendEvent {
|
||||
webhookEvent := webhook.EventParticipantLeft
|
||||
analyticsEvent := livekit.AnalyticsEventType_PARTICIPANT_LEFT
|
||||
if !isConnected {
|
||||
webhookEvent = webhook.EventParticipantConnectionAborted
|
||||
analyticsEvent = livekit.AnalyticsEventType_PARTICIPANT_CONNECTION_ABORTED
|
||||
}
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhookEvent,
|
||||
Room: room,
|
||||
Participant: participant,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, newParticipantEvent(analyticsEvent, room, participant))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublishRequested(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
track *livekit.TrackInfo,
|
||||
shouldSendEvent bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackPublishAttempt(track.Type.String())
|
||||
if !shouldSendEvent {
|
||||
return
|
||||
}
|
||||
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_REQUESTED, room, participantID, track)
|
||||
if ev.Participant != nil {
|
||||
ev.Participant.Identity = string(identity)
|
||||
}
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublished(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
track *livekit.TrackInfo,
|
||||
shouldSendEvent bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.AddPublishedTrack(track.Type.String())
|
||||
prometheus.RecordTrackPublishSuccess(track.Type.String())
|
||||
if !shouldSendEvent {
|
||||
return
|
||||
}
|
||||
|
||||
participant := &livekit.ParticipantInfo{
|
||||
Sid: string(participantID),
|
||||
Identity: string(identity),
|
||||
}
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventTrackPublished,
|
||||
Room: room,
|
||||
Participant: participant,
|
||||
Track: track,
|
||||
})
|
||||
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED, room, participantID, track)
|
||||
ev.Participant = participant
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublishedUpdate(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackMaxSubscribedVideoQuality(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
mime mime.MimeType,
|
||||
maxQuality livekit.VideoQuality,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_MAX_SUBSCRIBED_VIDEO_QUALITY, room, participantID, track)
|
||||
ev.MaxSubscribedVideoQuality = maxQuality
|
||||
ev.Mime = mime.String()
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackSubscribeRequested(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackSubscribeAttempt()
|
||||
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_REQUESTED, room, participantID, track)
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackSubscribed(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
publisher *livekit.ParticipantInfo,
|
||||
shouldSendEvent bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackSubscribeSuccess(track.Type.String())
|
||||
|
||||
if !shouldSendEvent {
|
||||
return
|
||||
}
|
||||
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBED, room, participantID, track)
|
||||
ev.Publisher = publisher
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackSubscribeFailed(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
err error,
|
||||
isUserError bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackSubscribeFailure(err, isUserError)
|
||||
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_FAILED, room, participantID, &livekit.TrackInfo{
|
||||
Sid: string(trackID),
|
||||
})
|
||||
ev.Error = err.Error()
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackUnsubscribed(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
shouldSendEvent bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackUnsubscribed(track.Type.String())
|
||||
|
||||
if shouldSendEvent {
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNSUBSCRIBED, room, participantID, track))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackUnpublished(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
track *livekit.TrackInfo,
|
||||
wasPublishedLocally bool,
|
||||
shouldSendEvent bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
if wasPublishedLocally {
|
||||
prometheus.SubPublishedTrack(track.Type.String())
|
||||
}
|
||||
if !shouldSendEvent {
|
||||
return
|
||||
}
|
||||
|
||||
participant := &livekit.ParticipantInfo{
|
||||
Sid: string(participantID),
|
||||
Identity: string(identity),
|
||||
}
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventTrackUnpublished,
|
||||
Room: room,
|
||||
Participant: participant,
|
||||
Track: track,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNPUBLISHED, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackMuted(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_MUTED, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackUnmuted(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNMUTED, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublishRTPStats(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
mimeType mime.MimeType,
|
||||
layer int,
|
||||
stats *livekit.RTPStats,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_STATS, toMinimalRoom(room))
|
||||
ev.ParticipantId = string(participantID)
|
||||
ev.TrackId = string(trackID)
|
||||
ev.Mime = mimeType.String()
|
||||
ev.VideoLayer = int32(layer)
|
||||
ev.RtpStats = stats
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackSubscribeRTPStats(
|
||||
ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
mimeType mime.MimeType,
|
||||
stats *livekit.RTPStats,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_STATS, toMinimalRoom(room))
|
||||
ev.ParticipantId = string(participantID)
|
||||
ev.TrackId = string(trackID)
|
||||
ev.Mime = mimeType.String()
|
||||
ev.RtpStats = stats
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) {
|
||||
opts := egress.GetEgressNotifyOptions(info)
|
||||
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: event,
|
||||
Room: toMinimalRoomProto(livekit.RoomID(info.RoomId), livekit.RoomName(info.RoomName)),
|
||||
EgressInfo: info,
|
||||
}, opts...)
|
||||
}
|
||||
|
||||
func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {
|
||||
|
||||
t.enqueue(func() {
|
||||
t.NotifyEgressEvent(ctx, webhook.EventEgressStarted, info)
|
||||
|
||||
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_STARTED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEgressEvent(ctx, webhook.EventEgressUpdated, info)
|
||||
|
||||
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEgressEvent(ctx, webhook.EventEgressEnded, info)
|
||||
|
||||
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_ENDED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_CREATED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_DELETED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventIngressStarted,
|
||||
Room: toMinimalRoomProto("", livekit.RoomName(info.RoomName)),
|
||||
IngressInfo: info,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_STARTED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_UPDATED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventIngressEnded,
|
||||
Room: toMinimalRoomProto("", livekit.RoomName(info.RoomName)),
|
||||
IngressInfo: info,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_ENDED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {
|
||||
t.enqueue(func() {
|
||||
ev := &livekit.AnalyticsEvent{
|
||||
Type: livekit.AnalyticsEventType_REPORT,
|
||||
Timestamp: timestamppb.Now(),
|
||||
Report: reportInfo,
|
||||
}
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {
|
||||
t.enqueue(func() {
|
||||
ev := &livekit.AnalyticsEvent{
|
||||
Type: livekit.AnalyticsEventType_API_CALL,
|
||||
Timestamp: timestamppb.Now(),
|
||||
ApiCall: apiCallInfo,
|
||||
}
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {
|
||||
t.enqueue(func() {
|
||||
ev := &livekit.AnalyticsEvent{
|
||||
Type: livekit.AnalyticsEventType_WEBHOOK,
|
||||
Timestamp: timestamppb.Now(),
|
||||
Webhook: webhookInfo,
|
||||
}
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
}
|
||||
|
||||
func newRoomEvent(event livekit.AnalyticsEventType, room *livekit.Room) *livekit.AnalyticsEvent {
|
||||
ev := &livekit.AnalyticsEvent{
|
||||
Type: event,
|
||||
Timestamp: timestamppb.Now(),
|
||||
}
|
||||
if room != nil {
|
||||
ev.Room = room
|
||||
ev.RoomId = room.Sid
|
||||
}
|
||||
return ev
|
||||
}
|
||||
|
||||
func newParticipantEvent(event livekit.AnalyticsEventType, room *livekit.Room, participant *livekit.ParticipantInfo) *livekit.AnalyticsEvent {
|
||||
ev := newRoomEvent(event, room)
|
||||
if participant != nil {
|
||||
ev.ParticipantId = participant.Sid
|
||||
ev.Participant = participant
|
||||
}
|
||||
return ev
|
||||
}
|
||||
|
||||
func newTrackEvent(event livekit.AnalyticsEventType, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) *livekit.AnalyticsEvent {
|
||||
ev := newParticipantEvent(event, toMinimalRoom(room), &livekit.ParticipantInfo{
|
||||
Sid: string(participantID),
|
||||
})
|
||||
if track != nil {
|
||||
ev.TrackId = track.Sid
|
||||
ev.Track = track
|
||||
}
|
||||
return ev
|
||||
}
|
||||
|
||||
func newEgressEvent(event livekit.AnalyticsEventType, egress *livekit.EgressInfo) *livekit.AnalyticsEvent {
|
||||
return &livekit.AnalyticsEvent{
|
||||
Type: event,
|
||||
Timestamp: timestamppb.Now(),
|
||||
EgressId: egress.EgressId,
|
||||
RoomId: egress.RoomId,
|
||||
Egress: egress,
|
||||
}
|
||||
}
|
||||
|
||||
func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressInfo) *livekit.AnalyticsEvent {
|
||||
return &livekit.AnalyticsEvent{
|
||||
Type: event,
|
||||
Timestamp: timestamppb.Now(),
|
||||
IngressId: ingress.IngressId,
|
||||
Ingress: ingress,
|
||||
}
|
||||
}
|
||||
|
||||
func toMinimalRoom(room *livekit.Room) *livekit.Room {
|
||||
if room == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return toMinimalRoomProto(livekit.RoomID(room.Sid), livekit.RoomName(room.Name))
|
||||
}
|
||||
|
||||
func toMinimalRoomProto(roomID livekit.RoomID, roomName livekit.RoomName) *livekit.Room {
|
||||
return &livekit.Room{
|
||||
Sid: string(roomID),
|
||||
Name: string(roomName),
|
||||
}
|
||||
}
|
||||
260
pkg/telemetry/events_test.go
Normal file
260
pkg/telemetry/events_test.go
Normal file
@@ -0,0 +1,260 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
func Test_OnParticipantJoin_EventIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := "part1"
|
||||
clientInfo := &livekit.ClientInfo{
|
||||
Sdk: 2,
|
||||
Version: "v1",
|
||||
Os: "mac",
|
||||
OsVersion: "v1",
|
||||
DeviceModel: "DM1",
|
||||
Browser: "chrome",
|
||||
BrowserVersion: "97.0.1",
|
||||
}
|
||||
clientMeta := &livekit.AnalyticsClientMeta{
|
||||
Region: "dark-side",
|
||||
Node: "moon",
|
||||
ClientAddr: "127.0.0.1",
|
||||
ClientConnectTime: 420,
|
||||
}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
|
||||
_, event := fixture.analytics.SendEventArgsForCall(0)
|
||||
require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_JOINED, event.Type)
|
||||
require.Equal(t, partSID, event.ParticipantId)
|
||||
require.Equal(t, participantInfo, event.Participant)
|
||||
require.Equal(t, room.Sid, event.RoomId)
|
||||
require.Equal(t, room, event.Room)
|
||||
|
||||
require.Equal(t, clientInfo.Sdk, event.ClientInfo.Sdk)
|
||||
require.Equal(t, clientInfo.Version, event.ClientInfo.Version)
|
||||
require.Equal(t, clientInfo.Os, event.ClientInfo.Os)
|
||||
require.Equal(t, clientInfo.OsVersion, event.ClientInfo.OsVersion)
|
||||
require.Equal(t, clientInfo.DeviceModel, event.ClientInfo.DeviceModel)
|
||||
require.Equal(t, clientInfo.Browser, event.ClientInfo.Browser)
|
||||
require.Equal(t, clientInfo.BrowserVersion, event.ClientInfo.BrowserVersion)
|
||||
|
||||
require.Equal(t, clientMeta.Region, event.ClientMeta.Region)
|
||||
require.Equal(t, clientMeta.Node, event.ClientMeta.Node)
|
||||
require.Equal(t, clientMeta.ClientAddr, event.ClientMeta.ClientAddr)
|
||||
require.Equal(t, clientMeta.ClientConnectTime, event.ClientMeta.ClientConnectTime)
|
||||
}
|
||||
|
||||
func Test_OnParticipantLeft_EventIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := "part1"
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false, false, guard)
|
||||
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
require.Equal(t, 2, fixture.analytics.SendEventCallCount())
|
||||
_, event := fixture.analytics.SendEventArgsForCall(1)
|
||||
require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_LEFT, event.Type)
|
||||
require.Equal(t, partSID, event.ParticipantId)
|
||||
require.Equal(t, room.Sid, event.RoomId)
|
||||
require.Equal(t, room, event.Room)
|
||||
}
|
||||
|
||||
func Test_OnTrackUpdate_EventIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
roomID := "room1"
|
||||
roomName := "RoomName"
|
||||
partID := "part1"
|
||||
trackID := "track1"
|
||||
layer := &livekit.VideoLayer{
|
||||
Quality: livekit.VideoQuality_HIGH,
|
||||
Width: uint32(360),
|
||||
Height: uint32(720),
|
||||
Bitrate: 2048,
|
||||
}
|
||||
|
||||
trackInfo := &livekit.TrackInfo{
|
||||
Sid: trackID,
|
||||
Type: livekit.TrackType_VIDEO,
|
||||
Muted: false,
|
||||
Simulcast: false,
|
||||
DisableDtx: false,
|
||||
Layers: []*livekit.VideoLayer{layer},
|
||||
}
|
||||
|
||||
// do
|
||||
fixture.sut.TrackPublishedUpdate(context.Background(), &livekit.Room{Sid: roomID, Name: roomName}, livekit.ParticipantID(partID), trackInfo)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
|
||||
_, event := fixture.analytics.SendEventArgsForCall(0)
|
||||
require.Equal(t, livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, event.Type)
|
||||
require.Equal(t, partID, event.ParticipantId)
|
||||
|
||||
require.Equal(t, trackID, event.Track.Sid)
|
||||
require.NotNil(t, event.Track.Layers)
|
||||
require.Equal(t, layer.Width, event.Track.Layers[0].Width)
|
||||
require.Equal(t, layer.Height, event.Track.Layers[0].Height)
|
||||
require.Equal(t, layer.Quality, event.Track.Layers[0].Quality)
|
||||
|
||||
}
|
||||
|
||||
func Test_OnParticipantActive_EventIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare participant to change status
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := "part1"
|
||||
|
||||
clientInfo := &livekit.ClientInfo{
|
||||
Sdk: 2,
|
||||
Version: "v1",
|
||||
Os: "mac",
|
||||
OsVersion: "v1",
|
||||
DeviceModel: "DM1",
|
||||
Browser: "chrome",
|
||||
BrowserVersion: "97.0.1",
|
||||
}
|
||||
clientMeta := &livekit.AnalyticsClientMeta{
|
||||
Region: "dark-side",
|
||||
Node: "moon",
|
||||
ClientAddr: "127.0.0.1",
|
||||
}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
|
||||
_, event := fixture.analytics.SendEventArgsForCall(0)
|
||||
|
||||
// test
|
||||
// do
|
||||
clientMetaConnect := &livekit.AnalyticsClientMeta{
|
||||
ClientConnectTime: 420,
|
||||
}
|
||||
|
||||
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false, false, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
require.Equal(t, 2, fixture.analytics.SendEventCallCount())
|
||||
_, eventActive := fixture.analytics.SendEventArgsForCall(1)
|
||||
require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, eventActive.Type)
|
||||
require.Equal(t, partSID, eventActive.ParticipantId)
|
||||
require.Equal(t, room.Sid, eventActive.RoomId)
|
||||
require.Equal(t, room, event.Room)
|
||||
|
||||
require.Equal(t, clientMetaConnect.ClientConnectTime, eventActive.ClientMeta.ClientConnectTime)
|
||||
}
|
||||
|
||||
func Test_OnTrackSubscribed_EventIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare participant to change status
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := "part1"
|
||||
publisherInfo := &livekit.ParticipantInfo{Sid: "pub1", Identity: "publisher"}
|
||||
trackInfo := &livekit.TrackInfo{Sid: "tr1", Type: livekit.TrackType_VIDEO}
|
||||
|
||||
clientInfo := &livekit.ClientInfo{
|
||||
Sdk: 2,
|
||||
Version: "v1",
|
||||
Os: "mac",
|
||||
OsVersion: "v1",
|
||||
DeviceModel: "DM1",
|
||||
Browser: "chrome",
|
||||
BrowserVersion: "97.0.1",
|
||||
}
|
||||
clientMeta := &livekit.AnalyticsClientMeta{
|
||||
Region: "dark-side",
|
||||
Node: "moon",
|
||||
ClientAddr: "127.0.0.1",
|
||||
}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
|
||||
_, event := fixture.analytics.SendEventArgsForCall(0)
|
||||
require.Equal(t, room, event.Room)
|
||||
|
||||
// do
|
||||
fixture.sut.TrackSubscribed(context.Background(), room, livekit.ParticipantID(partSID), trackInfo, publisherInfo, true)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
return fixture.analytics.SendEventCallCount() == 2
|
||||
}, time.Second, time.Millisecond*50, "expected send event to be called twice")
|
||||
_, eventTrackSubscribed := fixture.analytics.SendEventArgsForCall(1)
|
||||
require.Equal(t, livekit.AnalyticsEventType_TRACK_SUBSCRIBED, eventTrackSubscribed.Type)
|
||||
require.Equal(t, partSID, eventTrackSubscribed.ParticipantId)
|
||||
require.Equal(t, trackInfo.Sid, eventTrackSubscribed.Track.Sid)
|
||||
require.Equal(t, trackInfo.Type, eventTrackSubscribed.Track.Type)
|
||||
require.Equal(t, publisherInfo.Sid, eventTrackSubscribed.Publisher.Sid)
|
||||
require.Equal(t, publisherInfo.Identity, eventTrackSubscribed.Publisher.Identity)
|
||||
|
||||
}
|
||||
|
||||
func Test_OnRoomEnded_ReasonIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
fixture.sut.RoomEnded(context.Background(), room, livekit.RoomEndReason_ROOM_END_API_DELETE)
|
||||
|
||||
fixture.flush()
|
||||
|
||||
require.Equal(t, 1, fixture.analytics.SendEventCallCount())
|
||||
_, event := fixture.analytics.SendEventArgsForCall(0)
|
||||
require.Equal(t, livekit.AnalyticsEventType_ROOM_ENDED, event.Type)
|
||||
require.Equal(t, room.Sid, event.RoomId)
|
||||
require.Equal(t, livekit.RoomEndReason_ROOM_END_API_DELETE, event.RoomEndReason)
|
||||
}
|
||||
75
pkg/telemetry/prometheus/datapacket.go
Normal file
75
pkg/telemetry/prometheus/datapacket.go
Normal file
@@ -0,0 +1,75 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
promDataPacketStreamLabels = []string{"type", "mime_type"}
|
||||
promDataPacketStreamMimeTypes = []string{"text", "image", "application", "audio", "video"}
|
||||
|
||||
promDataPacketStreamDestCount *prometheus.HistogramVec
|
||||
promDataPacketStreamSize *prometheus.HistogramVec
|
||||
)
|
||||
|
||||
func initDataPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
promDataPacketStreamDestCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "datapacket_stream",
|
||||
Name: "dest_count",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{1, 2, 3, 4, 5, 10, 15, 25, 50},
|
||||
}, promDataPacketStreamLabels)
|
||||
promDataPacketStreamSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "datapacket_stream",
|
||||
Name: "bytes",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{128, 512, 2048, 8192, 32768, 131072, 524288, 2097152, 8388608, 33554432},
|
||||
}, promDataPacketStreamLabels)
|
||||
|
||||
prometheus.MustRegister(promDataPacketStreamDestCount)
|
||||
prometheus.MustRegister(promDataPacketStreamSize)
|
||||
}
|
||||
|
||||
func RecordDataPacketStream(h *livekit.DataStream_Header, destCount int) {
|
||||
streamType := "unknown"
|
||||
switch h.ContentHeader.(type) {
|
||||
case *livekit.DataStream_Header_TextHeader:
|
||||
streamType = "text"
|
||||
case *livekit.DataStream_Header_ByteHeader:
|
||||
streamType = "bytes"
|
||||
}
|
||||
|
||||
mimeType := strings.ToLower(h.MimeType)
|
||||
if i := strings.IndexByte(mimeType, '/'); i != -1 {
|
||||
mimeType = mimeType[:i]
|
||||
}
|
||||
if !slices.Contains(promDataPacketStreamMimeTypes, mimeType) {
|
||||
mimeType = "unknown"
|
||||
}
|
||||
|
||||
promDataPacketStreamDestCount.WithLabelValues(streamType, mimeType).Observe(float64(destCount))
|
||||
if h.TotalLength != nil {
|
||||
promDataPacketStreamSize.WithLabelValues(streamType, mimeType).Observe(float64(*h.TotalLength))
|
||||
}
|
||||
}
|
||||
40
pkg/telemetry/prometheus/debug.go
Normal file
40
pkg/telemetry/prometheus/debug.go
Normal file
@@ -0,0 +1,40 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
refCounts *prometheus.GaugeVec
|
||||
)
|
||||
|
||||
func initDebugStats(nodeID string, nodeType livekit.NodeType) {
|
||||
refCounts = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "debug",
|
||||
Name: "ref_count",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"referrer"})
|
||||
|
||||
prometheus.MustRegister(refCounts)
|
||||
}
|
||||
|
||||
func AddRef(referrer string, n int) {
|
||||
refCounts.WithLabelValues(referrer).Add(float64(n))
|
||||
}
|
||||
310
pkg/telemetry/prometheus/node.go
Normal file
310
pkg/telemetry/prometheus/node.go
Normal file
@@ -0,0 +1,310 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/twitchtv/twirp"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/rpc"
|
||||
"github.com/livekit/protocol/utils/hwstats"
|
||||
"github.com/livekit/protocol/webhook"
|
||||
)
|
||||
|
||||
const (
|
||||
livekitNamespace string = "livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
initialized atomic.Bool
|
||||
|
||||
promMessageCounter *prometheus.CounterVec
|
||||
promServiceOperationCounter *prometheus.CounterVec
|
||||
promTwirpRequestStatusCounter *prometheus.CounterVec
|
||||
promTwirpRequestLatency *prometheus.HistogramVec
|
||||
|
||||
sysPacketsStart uint32
|
||||
sysDroppedPacketsStart uint32
|
||||
promSysPacketGauge *prometheus.GaugeVec
|
||||
|
||||
cpuStats *hwstats.CPUStats
|
||||
memoryStats *hwstats.MemoryStats
|
||||
)
|
||||
|
||||
func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
if initialized.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
|
||||
promMessageCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "messages",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "status", "direction"},
|
||||
)
|
||||
|
||||
promServiceOperationCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "service_operation",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "status", "error_type"},
|
||||
)
|
||||
|
||||
promTwirpRequestStatusCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "twirp_request_status",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"service", "method", "status", "code"},
|
||||
)
|
||||
|
||||
promTwirpRequestLatency = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "twirp_request_latency_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000},
|
||||
},
|
||||
[]string{"service", "method", "status"},
|
||||
)
|
||||
|
||||
promSysPacketGauge = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "packet_total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Help: "System level packet count. Count starts at 0 when service is first started.",
|
||||
},
|
||||
[]string{"type"},
|
||||
)
|
||||
|
||||
prometheus.MustRegister(promMessageCounter)
|
||||
prometheus.MustRegister(promServiceOperationCounter)
|
||||
prometheus.MustRegister(promTwirpRequestStatusCounter)
|
||||
prometheus.MustRegister(promTwirpRequestLatency)
|
||||
prometheus.MustRegister(promSysPacketGauge)
|
||||
|
||||
sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats()
|
||||
|
||||
initPacketStats(nodeID, nodeType)
|
||||
initRoomStats(nodeID, nodeType)
|
||||
rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
|
||||
webhook.InitWebhookStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
|
||||
initQualityStats(nodeID, nodeType)
|
||||
initDataPacketStats(nodeID, nodeType)
|
||||
initDebugStats(nodeID, nodeType)
|
||||
|
||||
var err error
|
||||
cpuStats, err = hwstats.NewCPUStats(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
memoryStats, err = hwstats.NewMemoryStats()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetNodeStats(nodeStartedAt int64, prevStats []*livekit.NodeStats, rateIntervals []time.Duration) (*livekit.NodeStats, error) {
|
||||
loadAvg, err := getLoadAvg()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin
|
||||
// So, do not error out. Use the information if it is available.
|
||||
memUsed, memTotal, _ := memoryStats.GetMemory()
|
||||
|
||||
sysPackets, sysDroppedPackets, _ := getTCStats()
|
||||
promSysPacketGauge.WithLabelValues("out").Set(float64(sysPackets - sysPacketsStart))
|
||||
promSysPacketGauge.WithLabelValues("dropped").Set(float64(sysDroppedPackets - sysDroppedPacketsStart))
|
||||
|
||||
stats := &livekit.NodeStats{
|
||||
StartedAt: nodeStartedAt,
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
NumRooms: roomCurrent.Load(),
|
||||
NumClients: participantCurrent.Load(),
|
||||
NumTracksIn: trackPublishedCurrent.Load(),
|
||||
NumTracksOut: trackSubscribedCurrent.Load(),
|
||||
NumTrackPublishAttempts: trackPublishAttempts.Load(),
|
||||
NumTrackPublishSuccess: trackPublishSuccess.Load(),
|
||||
NumTrackPublishCancels: trackPublishCancels.Load(),
|
||||
NumTrackSubscribeAttempts: trackSubscribeAttempts.Load(),
|
||||
NumTrackSubscribeSuccess: trackSubscribeSuccess.Load(),
|
||||
NumTrackSubscribeCancels: trackSubscribeCancels.Load(),
|
||||
BytesIn: bytesIn.Load(),
|
||||
BytesOut: bytesOut.Load(),
|
||||
PacketsIn: packetsIn.Load(),
|
||||
PacketsOut: packetsOut.Load(),
|
||||
RetransmitBytesOut: retransmitBytes.Load(),
|
||||
RetransmitPacketsOut: retransmitPackets.Load(),
|
||||
NackTotal: nackTotal.Load(),
|
||||
ParticipantSignalConnected: participantSignalConnected.Load(),
|
||||
ParticipantRtcInit: participantRTCInit.Load(),
|
||||
ParticipantRtcConnected: participantRTCConnected.Load(),
|
||||
ParticipantRtcCanceled: participantRTCCanceled.Load(),
|
||||
ParticipantRtcActive: participantRTCActive.Load(),
|
||||
ForwardLatency: forwardLatency.Load(),
|
||||
ForwardJitter: forwardJitter.Load(),
|
||||
NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
|
||||
CpuLoad: float32(cpuStats.GetCPULoad()),
|
||||
MemoryTotal: memTotal,
|
||||
MemoryUsed: memUsed,
|
||||
LoadAvgLast1Min: float32(loadAvg.Loadavg1),
|
||||
LoadAvgLast5Min: float32(loadAvg.Loadavg5),
|
||||
LoadAvgLast15Min: float32(loadAvg.Loadavg15),
|
||||
SysPacketsOut: sysPackets,
|
||||
SysPacketsDropped: sysDroppedPackets,
|
||||
}
|
||||
|
||||
for _, rateInterval := range rateIntervals {
|
||||
for idx := len(prevStats) - 1; idx >= 0; idx-- {
|
||||
prev := prevStats[idx]
|
||||
if prev == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if stats.UpdatedAt-prev.UpdatedAt >= int64(rateInterval.Seconds()) {
|
||||
if rate := getNodeStatsRate(append(prevStats[idx:], stats)); rate != nil {
|
||||
stats.Rates = append(stats.Rates, rate)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func getNodeStatsRate(statsHistory []*livekit.NodeStats) *livekit.NodeStatsRate {
|
||||
if len(statsHistory) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
elapsed := statsHistory[len(statsHistory)-1].UpdatedAt - statsHistory[0].UpdatedAt
|
||||
if elapsed <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// time weighted averages
|
||||
var cpuLoad, memoryUsed, memoryTotal, memoryLoad float32
|
||||
for idx := len(statsHistory) - 1; idx > 0; idx-- {
|
||||
stats := statsHistory[idx]
|
||||
prevStats := statsHistory[idx-1]
|
||||
if stats == nil || prevStats == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
spanElapsed := stats.UpdatedAt - prevStats.UpdatedAt
|
||||
if spanElapsed <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
cpuLoad += stats.CpuLoad * float32(spanElapsed)
|
||||
memoryUsed += float32(stats.MemoryUsed) * float32(spanElapsed)
|
||||
memoryTotal += float32(stats.MemoryTotal) * float32(spanElapsed)
|
||||
if stats.MemoryTotal > 0 {
|
||||
memoryLoad += float32(stats.MemoryUsed) / float32(stats.MemoryTotal) * float32(spanElapsed)
|
||||
}
|
||||
}
|
||||
|
||||
earlier := statsHistory[0]
|
||||
later := statsHistory[len(statsHistory)-1]
|
||||
rate := &livekit.NodeStatsRate{
|
||||
StartedAt: earlier.UpdatedAt,
|
||||
EndedAt: later.UpdatedAt,
|
||||
Duration: elapsed,
|
||||
BytesIn: perSec(earlier.BytesIn, later.BytesIn, elapsed),
|
||||
BytesOut: perSec(earlier.BytesOut, later.BytesOut, elapsed),
|
||||
PacketsIn: perSec(earlier.PacketsIn, later.PacketsIn, elapsed),
|
||||
PacketsOut: perSec(earlier.PacketsOut, later.PacketsOut, elapsed),
|
||||
RetransmitBytesOut: perSec(earlier.RetransmitBytesOut, later.RetransmitBytesOut, elapsed),
|
||||
RetransmitPacketsOut: perSec(earlier.RetransmitPacketsOut, later.RetransmitPacketsOut, elapsed),
|
||||
NackTotal: perSec(earlier.NackTotal, later.NackTotal, elapsed),
|
||||
ParticipantSignalConnected: perSec(earlier.ParticipantSignalConnected, later.ParticipantSignalConnected, elapsed),
|
||||
ParticipantSignalFailed: perSec(earlier.ParticipantSignalFailed, later.ParticipantSignalFailed, elapsed),
|
||||
ParticipantSignalValidationFailed: perSec(earlier.ParticipantSignalValidationFailed, later.ParticipantSignalValidationFailed, elapsed),
|
||||
ParticipantRtcInit: perSec(earlier.ParticipantRtcInit, later.ParticipantRtcInit, elapsed),
|
||||
ParticipantRtcConnected: perSec(earlier.ParticipantRtcConnected, later.ParticipantRtcConnected, elapsed),
|
||||
ParticipantRtcCanceled: perSec(earlier.ParticipantRtcCanceled, later.ParticipantRtcCanceled, elapsed),
|
||||
ParticipantRtcActive: perSec(earlier.ParticipantRtcActive, later.ParticipantRtcActive, elapsed),
|
||||
SysPacketsOut: perSec(uint64(earlier.SysPacketsOut), uint64(later.SysPacketsOut), elapsed),
|
||||
SysPacketsDropped: perSec(uint64(earlier.SysPacketsDropped), uint64(later.SysPacketsDropped), elapsed),
|
||||
TrackPublishAttempts: perSec(uint64(earlier.NumTrackPublishAttempts), uint64(later.NumTrackPublishAttempts), elapsed),
|
||||
TrackPublishSuccess: perSec(uint64(earlier.NumTrackPublishSuccess), uint64(later.NumTrackPublishSuccess), elapsed),
|
||||
TrackPublishCancels: perSec(uint64(earlier.NumTrackPublishCancels), uint64(later.NumTrackPublishCancels), elapsed),
|
||||
TrackSubscribeAttempts: perSec(uint64(earlier.NumTrackSubscribeAttempts), uint64(later.NumTrackSubscribeAttempts), elapsed),
|
||||
TrackSubscribeSuccess: perSec(uint64(earlier.NumTrackSubscribeSuccess), uint64(later.NumTrackSubscribeSuccess), elapsed),
|
||||
TrackSubscribeCancels: perSec(uint64(earlier.NumTrackSubscribeCancels), uint64(later.NumTrackSubscribeCancels), elapsed),
|
||||
CpuLoad: cpuLoad / float32(elapsed),
|
||||
MemoryLoad: memoryLoad / float32(elapsed),
|
||||
MemoryUsed: memoryUsed / float32(elapsed),
|
||||
MemoryTotal: memoryTotal / float32(elapsed),
|
||||
}
|
||||
return rate
|
||||
}
|
||||
|
||||
func perSec(prev, curr uint64, secs int64) float32 {
|
||||
return float32(curr-prev) / float32(secs)
|
||||
}
|
||||
|
||||
func RecordSignalRequestSuccess() {
|
||||
promMessageCounter.WithLabelValues("signal", "success", "request").Add(1)
|
||||
}
|
||||
|
||||
func RecordSignalRequestFailure() {
|
||||
promMessageCounter.WithLabelValues("signal", "failure", "request").Add(1)
|
||||
}
|
||||
|
||||
func RecordSignalResponseSuccess() {
|
||||
promMessageCounter.WithLabelValues("signal", "success", "response").Add(1)
|
||||
}
|
||||
|
||||
func RecordSignalResponseFailure() {
|
||||
promMessageCounter.WithLabelValues("signal", "failure", "response").Add(1)
|
||||
}
|
||||
|
||||
func RecordServiceOperationSuccess(op string) {
|
||||
promServiceOperationCounter.WithLabelValues(op, "success", "").Add(1)
|
||||
}
|
||||
|
||||
func RecordServiceOperationError(op string, error string) {
|
||||
promServiceOperationCounter.WithLabelValues(op, "error", error).Add(1)
|
||||
}
|
||||
|
||||
func RecordTwirpRequestStatus(service string, method string, statusFamily string, code twirp.ErrorCode) {
|
||||
promTwirpRequestStatusCounter.WithLabelValues(service, method, statusFamily, string(code)).Add(1)
|
||||
}
|
||||
|
||||
func RecordTwirpRequestLatency(service, method string, duration time.Duration, statusFamily string) {
|
||||
promTwirpRequestLatency.WithLabelValues(service, method, statusFamily).Observe(float64(duration.Milliseconds()))
|
||||
}
|
||||
56
pkg/telemetry/prometheus/node_linux.go
Normal file
56
pkg/telemetry/prometheus/node_linux.go
Normal file
@@ -0,0 +1,56 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/florianl/go-tc"
|
||||
)
|
||||
|
||||
func getTCStats() (packets, drops uint32, err error) {
|
||||
rtnl, err := tc.Open(&tc.Config{})
|
||||
if err != nil {
|
||||
err = fmt.Errorf("could not open rtnetlink socket: %v", err)
|
||||
return
|
||||
}
|
||||
defer rtnl.Close()
|
||||
|
||||
qdiscs, err := rtnl.Qdisc().Get()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("could not get qdiscs: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, qdisc := range qdiscs {
|
||||
// Newer kernels report stats via TCA_STATS2 (Stats2), while older kernels
|
||||
// only populate the legacy TCA_STATS attribute (Stats). Prefer Stats2 and
|
||||
// fall back to Stats so counters are collected on both.
|
||||
//
|
||||
// However, go-tc (through v0.4.8 and current main) mis-parses the nested TCA_STATS2
|
||||
// as a flat struct, so Stats2.Packets/.Drops are garbage. The legacy
|
||||
// TCA_STATS blob is a real flat struct and is parsed correctly, and modern
|
||||
// kernels populate BOTH — so prefer Stats.
|
||||
if qdisc.Stats != nil {
|
||||
packets += qdisc.Stats.Packets
|
||||
drops += qdisc.Stats.Drops
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
22
pkg/telemetry/prometheus/node_nonlinux.go
Normal file
22
pkg/telemetry/prometheus/node_nonlinux.go
Normal file
@@ -0,0 +1,22 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build !linux
|
||||
|
||||
package prometheus
|
||||
|
||||
func getTCStats() (packets, drops uint32, err error) {
|
||||
// linux only
|
||||
return
|
||||
}
|
||||
25
pkg/telemetry/prometheus/node_nonwindows.go
Normal file
25
pkg/telemetry/prometheus/node_nonwindows.go
Normal file
@@ -0,0 +1,25 @@
|
||||
//go:build !windows
|
||||
|
||||
/*
|
||||
* Copyright 2023 LiveKit, Inc
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package prometheus
|
||||
|
||||
import "github.com/mackerelio/go-osstat/loadavg"
|
||||
|
||||
func getLoadAvg() (*loadavg.Stats, error) {
|
||||
return loadavg.Get()
|
||||
}
|
||||
25
pkg/telemetry/prometheus/node_windows.go
Normal file
25
pkg/telemetry/prometheus/node_windows.go
Normal file
@@ -0,0 +1,25 @@
|
||||
//go:build windows
|
||||
|
||||
/*
|
||||
* Copyright 2023 LiveKit, Inc
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package prometheus
|
||||
|
||||
import "github.com/mackerelio/go-osstat/loadavg"
|
||||
|
||||
func getLoadAvg() (*loadavg.Stats, error) {
|
||||
return &loadavg.Stats{}, nil
|
||||
}
|
||||
391
pkg/telemetry/prometheus/packets.go
Normal file
391
pkg/telemetry/prometheus/packets.go
Normal file
@@ -0,0 +1,391 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
type Direction string
|
||||
|
||||
const (
|
||||
Incoming Direction = "incoming"
|
||||
Outgoing Direction = "outgoing"
|
||||
)
|
||||
|
||||
type TransmissionType string
|
||||
|
||||
const (
|
||||
TransmissionInitial TransmissionType = "initial"
|
||||
TransmissionRetransmit TransmissionType = "retransmit"
|
||||
)
|
||||
|
||||
var (
|
||||
bytesIn atomic.Uint64
|
||||
bytesOut atomic.Uint64
|
||||
packetsIn atomic.Uint64
|
||||
packetsOut atomic.Uint64
|
||||
nackTotal atomic.Uint64
|
||||
retransmitBytes atomic.Uint64
|
||||
retransmitPackets atomic.Uint64
|
||||
participantSignalConnected atomic.Uint64
|
||||
participantRTCConnected atomic.Uint64
|
||||
participantRTCInit atomic.Uint64
|
||||
participantRTCCanceled atomic.Uint64
|
||||
participantRTCActive atomic.Uint64
|
||||
forwardLatency atomic.Uint32
|
||||
forwardJitter atomic.Uint32
|
||||
|
||||
promPacketLabels = []string{"direction", "transmission", "country"}
|
||||
promPacketTotal *prometheus.CounterVec
|
||||
promPacketBytes *prometheus.CounterVec
|
||||
promRTCPLabels = []string{"direction", "country"}
|
||||
promStreamLabels = []string{"direction", "source", "type", "country"}
|
||||
promNackTotal *prometheus.CounterVec
|
||||
promPliTotal *prometheus.CounterVec
|
||||
promFirTotal *prometheus.CounterVec
|
||||
promPacketLossTotal *prometheus.CounterVec
|
||||
promPacketLoss *prometheus.HistogramVec
|
||||
promPacketOutOfOrderTotal *prometheus.CounterVec
|
||||
promPacketOutOfOrder *prometheus.HistogramVec
|
||||
promJitter *prometheus.HistogramVec
|
||||
promRTT *prometheus.HistogramVec
|
||||
promParticipantJoin *prometheus.CounterVec
|
||||
promConnections *prometheus.GaugeVec
|
||||
promForwardLatency prometheus.Gauge
|
||||
promForwardJitter prometheus.Gauge
|
||||
promForwardLatencyHist prometheus.Histogram
|
||||
)
|
||||
|
||||
func initPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promPacketLabels)
|
||||
promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet",
|
||||
Name: "bytes",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promPacketLabels)
|
||||
promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "nack",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promRTCPLabels)
|
||||
promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "pli",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promRTCPLabels)
|
||||
promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "fir",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promRTCPLabels)
|
||||
promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_loss",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promStreamLabels)
|
||||
promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_loss",
|
||||
Name: "percent",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
|
||||
}, promStreamLabels)
|
||||
promPacketOutOfOrderTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_out_of_order",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promStreamLabels)
|
||||
promPacketOutOfOrder = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_out_of_order",
|
||||
Name: "percent",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
|
||||
}, promStreamLabels)
|
||||
promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "jitter",
|
||||
Name: "us",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
|
||||
Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
|
||||
}, promStreamLabels)
|
||||
promRTT = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "rtt",
|
||||
Name: "ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000},
|
||||
}, promStreamLabels)
|
||||
promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "participant_join",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"state", "warp"})
|
||||
promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "connection",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind"})
|
||||
promForwardLatency = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "forward",
|
||||
Name: "latency",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promForwardJitter = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "forward",
|
||||
Name: "jitter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promForwardLatencyHist = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "forward_latency",
|
||||
Name: "ns",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
// 50us, 100us, 250us, 500us, 1ms, 2ms, 3ms, 5ms, 10ms, 20ms
|
||||
Buckets: []float64{
|
||||
50 * 1000,
|
||||
100 * 1000,
|
||||
250 * 1000,
|
||||
500 * 1000,
|
||||
1 * 1000 * 1000,
|
||||
2 * 1000 * 1000,
|
||||
3 * 1000 * 1000,
|
||||
5 * 1000 * 1000,
|
||||
10 * 1000 * 1000,
|
||||
20 * 1000 * 1000,
|
||||
},
|
||||
})
|
||||
|
||||
prometheus.MustRegister(promPacketTotal)
|
||||
prometheus.MustRegister(promPacketBytes)
|
||||
prometheus.MustRegister(promNackTotal)
|
||||
prometheus.MustRegister(promPliTotal)
|
||||
prometheus.MustRegister(promFirTotal)
|
||||
prometheus.MustRegister(promPacketLossTotal)
|
||||
prometheus.MustRegister(promPacketLoss)
|
||||
prometheus.MustRegister(promPacketOutOfOrderTotal)
|
||||
prometheus.MustRegister(promPacketOutOfOrder)
|
||||
prometheus.MustRegister(promJitter)
|
||||
prometheus.MustRegister(promRTT)
|
||||
prometheus.MustRegister(promParticipantJoin)
|
||||
prometheus.MustRegister(promConnections)
|
||||
prometheus.MustRegister(promForwardLatency)
|
||||
prometheus.MustRegister(promForwardJitter)
|
||||
prometheus.MustRegister(promForwardLatencyHist)
|
||||
}
|
||||
|
||||
func IncrementPackets(country string, direction Direction, count uint64, retransmit bool) {
|
||||
var transmission TransmissionType
|
||||
if retransmit {
|
||||
transmission = TransmissionRetransmit
|
||||
} else {
|
||||
transmission = TransmissionInitial
|
||||
}
|
||||
promPacketTotal.WithLabelValues(string(direction), string(transmission), country).Add(float64(count))
|
||||
|
||||
if direction == Incoming {
|
||||
packetsIn.Add(count)
|
||||
} else {
|
||||
packetsOut.Add(count)
|
||||
if retransmit {
|
||||
retransmitPackets.Add(count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementBytes(country string, direction Direction, count uint64, retransmit bool) {
|
||||
var transmission TransmissionType
|
||||
if retransmit {
|
||||
transmission = TransmissionRetransmit
|
||||
} else {
|
||||
transmission = TransmissionInitial
|
||||
}
|
||||
promPacketBytes.WithLabelValues(string(direction), string(transmission), country).Add(float64(count))
|
||||
|
||||
if direction == Incoming {
|
||||
bytesIn.Add(count)
|
||||
} else {
|
||||
bytesOut.Add(count)
|
||||
if retransmit {
|
||||
retransmitBytes.Add(count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementRTCP(country string, direction Direction, nack, pli, fir uint32) {
|
||||
if nack > 0 {
|
||||
promNackTotal.WithLabelValues(string(direction), country).Add(float64(nack))
|
||||
nackTotal.Add(uint64(nack))
|
||||
}
|
||||
if pli > 0 {
|
||||
promPliTotal.WithLabelValues(string(direction), country).Add(float64(pli))
|
||||
}
|
||||
if fir > 0 {
|
||||
promFirTotal.WithLabelValues(string(direction), country).Add(float64(fir))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordPacketLoss(
|
||||
country string,
|
||||
direction Direction,
|
||||
trackSource livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
lost uint32,
|
||||
total uint32,
|
||||
) {
|
||||
if total > 0 {
|
||||
promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(lost) / float64(total) * 100)
|
||||
}
|
||||
if lost > 0 {
|
||||
promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(lost))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordPacketOutOfOrder(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) {
|
||||
if total > 0 {
|
||||
promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(ooo) / float64(total) * 100)
|
||||
}
|
||||
if ooo > 0 {
|
||||
promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(ooo))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordJitter(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) {
|
||||
if jitter > 0 {
|
||||
promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(jitter))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordRTT(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) {
|
||||
if rtt > 0 {
|
||||
promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(rtt))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoin(join uint32) {
|
||||
if join > 0 {
|
||||
participantSignalConnected.Add(uint64(join))
|
||||
promParticipantJoin.WithLabelValues("signal_connected", "").Add(float64(join))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinFail(fail uint32) {
|
||||
if fail > 0 {
|
||||
promParticipantJoin.WithLabelValues("signal_failed", "").Add(float64(fail))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinValidationFail(validationFail uint32) {
|
||||
if validationFail > 0 {
|
||||
promParticipantJoin.WithLabelValues("signal_validation_failed", "").Add(float64(validationFail))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinUpgradeFail(upgradeFail uint32) {
|
||||
if upgradeFail > 0 {
|
||||
promParticipantJoin.WithLabelValues("signal_upgrade_failed", "").Add(float64(upgradeFail))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinWriteInitialResponseFail(writeInitialResponseFail uint32) {
|
||||
if writeInitialResponseFail > 0 {
|
||||
promParticipantJoin.WithLabelValues("signal_write_initial_response_failed", "").Add(float64(writeInitialResponseFail))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcInit(init uint32) {
|
||||
if init > 0 {
|
||||
participantRTCInit.Add(uint64(init))
|
||||
promParticipantJoin.WithLabelValues("rtc_init", "").Add(float64(init))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcConnected(connected uint32) {
|
||||
if connected > 0 {
|
||||
participantRTCConnected.Add(uint64(connected))
|
||||
promParticipantJoin.WithLabelValues("rtc_connected", "").Add(float64(connected))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcActive(active uint32, warp bool) {
|
||||
if active > 0 {
|
||||
participantRTCActive.Add(uint64(active))
|
||||
promParticipantJoin.WithLabelValues("rtc_active", strconv.FormatBool(warp)).Add(float64(active))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcCanceled(canceled uint64, warp bool) {
|
||||
if canceled > 0 {
|
||||
participantRTCCanceled.Add(canceled)
|
||||
promParticipantJoin.WithLabelValues("rtc_canceled", strconv.FormatBool(warp)).Add(float64(canceled))
|
||||
}
|
||||
}
|
||||
|
||||
// todo: check if need to record warp to rtcSucc/Failure
|
||||
func IncrementParticipantRtcSuccess(success uint64, warp bool) {
|
||||
if success > 0 {
|
||||
promParticipantJoin.WithLabelValues("rtc_success", strconv.FormatBool(warp)).Add(float64(success))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcFailure(failure uint64, warp bool) {
|
||||
if failure > 0 {
|
||||
promParticipantJoin.WithLabelValues("rtc_failure", strconv.FormatBool(warp)).Add(float64(failure))
|
||||
}
|
||||
}
|
||||
|
||||
func AddConnection(direction Direction) {
|
||||
promConnections.WithLabelValues(string(direction)).Add(1)
|
||||
}
|
||||
|
||||
func SubConnection(direction Direction) {
|
||||
promConnections.WithLabelValues(string(direction)).Sub(1)
|
||||
}
|
||||
|
||||
func RecordForwardLatencySample(forwardLatency int64) {
|
||||
promForwardLatencyHist.Observe(float64(forwardLatency))
|
||||
}
|
||||
|
||||
func RecordForwardLatency(longTermLatencyAvg uint32) {
|
||||
forwardLatency.Store(longTermLatencyAvg)
|
||||
promForwardLatency.Set(float64(longTermLatencyAvg))
|
||||
}
|
||||
|
||||
func RecordForwardJitter(longTermJitterAvg uint32) {
|
||||
forwardJitter.Store(longTermJitterAvg)
|
||||
promForwardJitter.Set(float64(longTermJitterAvg))
|
||||
}
|
||||
51
pkg/telemetry/prometheus/quality.go
Normal file
51
pkg/telemetry/prometheus/quality.go
Normal file
@@ -0,0 +1,51 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
qualityRating prometheus.Histogram
|
||||
qualityScore prometheus.Histogram
|
||||
)
|
||||
|
||||
func initQualityStats(nodeID string, nodeType livekit.NodeType) {
|
||||
qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "quality",
|
||||
Name: "rating",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{0, 1, 2},
|
||||
})
|
||||
qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "quality",
|
||||
Name: "score",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5},
|
||||
})
|
||||
|
||||
prometheus.MustRegister(qualityRating)
|
||||
prometheus.MustRegister(qualityScore)
|
||||
}
|
||||
|
||||
func RecordQuality(rating livekit.ConnectionQuality, score float32) {
|
||||
qualityRating.Observe(float64(rating))
|
||||
qualityScore.Observe(float64(score))
|
||||
}
|
||||
304
pkg/telemetry/prometheus/rooms.go
Normal file
304
pkg/telemetry/prometheus/rooms.go
Normal file
@@ -0,0 +1,304 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
roomCurrent atomic.Int32
|
||||
participantCurrent atomic.Int32
|
||||
trackPublishedCurrent atomic.Int32
|
||||
trackSubscribedCurrent atomic.Int32
|
||||
trackPublishAttempts atomic.Int32
|
||||
trackPublishSuccess atomic.Int32
|
||||
trackPublishCancels atomic.Int32
|
||||
trackSubscribeAttempts atomic.Int32
|
||||
trackSubscribeSuccess atomic.Int32
|
||||
trackSubscribeCancels atomic.Int32
|
||||
// count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute
|
||||
// success rate by subtracting this from total attempts
|
||||
trackSubscribeUserError atomic.Int32
|
||||
|
||||
promRoomCurrent prometheus.Gauge
|
||||
promRoomDuration prometheus.Histogram
|
||||
promParticipantCurrent prometheus.Gauge
|
||||
promTrackPublishedCurrent *prometheus.GaugeVec
|
||||
promTrackSubscribedCurrent *prometheus.GaugeVec
|
||||
promTrackPublishCounter *prometheus.CounterVec
|
||||
promTrackSubscribeCounter *prometheus.CounterVec
|
||||
promSessionJoinLatency *prometheus.HistogramVec
|
||||
promSessionStartTime *prometheus.HistogramVec
|
||||
promSessionDuration *prometheus.HistogramVec
|
||||
promPubSubTime *prometheus.HistogramVec
|
||||
|
||||
promPeerConnection *prometheus.CounterVec
|
||||
)
|
||||
|
||||
func initRoomStats(nodeID string, nodeType livekit.NodeType) {
|
||||
promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "room",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "room",
|
||||
Name: "duration_seconds",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{
|
||||
5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60,
|
||||
},
|
||||
})
|
||||
promParticipantCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "participant",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "published_total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind"})
|
||||
promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "subscribed_total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind"})
|
||||
promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "publish_counter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind", "state"})
|
||||
promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "subscribe_counter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"state", "error"})
|
||||
promSessionJoinLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "session",
|
||||
Name: "join_latency_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: prometheus.ExponentialBucketsRange(10, 10000, 15),
|
||||
}, []string{"protocol_version"})
|
||||
promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "session",
|
||||
Name: "start_time_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15),
|
||||
}, []string{"protocol_version", "warp"})
|
||||
promSessionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "session",
|
||||
Name: "duration_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: prometheus.ExponentialBucketsRange(100, 4*60*60*1000, 15),
|
||||
}, []string{"protocol_version"})
|
||||
promPubSubTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "pubsubtime",
|
||||
Name: "ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{100, 200, 500, 700, 1000, 5000, 10000},
|
||||
}, append(promStreamLabels, "sdk", "kind", "count"))
|
||||
promPeerConnection = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "peer_connection",
|
||||
Name: "state",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"transport", "state"})
|
||||
|
||||
prometheus.MustRegister(promRoomCurrent)
|
||||
prometheus.MustRegister(promRoomDuration)
|
||||
prometheus.MustRegister(promParticipantCurrent)
|
||||
prometheus.MustRegister(promTrackPublishedCurrent)
|
||||
prometheus.MustRegister(promTrackSubscribedCurrent)
|
||||
prometheus.MustRegister(promTrackPublishCounter)
|
||||
prometheus.MustRegister(promTrackSubscribeCounter)
|
||||
prometheus.MustRegister(promSessionJoinLatency)
|
||||
prometheus.MustRegister(promSessionStartTime)
|
||||
prometheus.MustRegister(promSessionDuration)
|
||||
prometheus.MustRegister(promPubSubTime)
|
||||
prometheus.MustRegister(promPeerConnection)
|
||||
}
|
||||
|
||||
func RoomStarted() {
|
||||
promRoomCurrent.Add(1)
|
||||
roomCurrent.Inc()
|
||||
}
|
||||
|
||||
func RoomEnded(startedAt time.Time) {
|
||||
if !startedAt.IsZero() {
|
||||
promRoomDuration.Observe(float64(time.Since(startedAt)) / float64(time.Second))
|
||||
}
|
||||
promRoomCurrent.Sub(1)
|
||||
roomCurrent.Dec()
|
||||
}
|
||||
|
||||
func AddParticipant() {
|
||||
promParticipantCurrent.Add(1)
|
||||
participantCurrent.Inc()
|
||||
}
|
||||
|
||||
func SubParticipant() {
|
||||
promParticipantCurrent.Sub(1)
|
||||
participantCurrent.Dec()
|
||||
}
|
||||
|
||||
func AddPublishedTrack(kind string) {
|
||||
promTrackPublishedCurrent.WithLabelValues(kind).Add(1)
|
||||
trackPublishedCurrent.Inc()
|
||||
}
|
||||
|
||||
func SubPublishedTrack(kind string) {
|
||||
promTrackPublishedCurrent.WithLabelValues(kind).Sub(1)
|
||||
trackPublishedCurrent.Dec()
|
||||
}
|
||||
|
||||
func RecordTrackPublishAttempt(kind string) {
|
||||
trackPublishAttempts.Inc()
|
||||
promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc()
|
||||
}
|
||||
|
||||
func RecordTrackPublishSuccess(kind string) {
|
||||
trackPublishSuccess.Inc()
|
||||
promTrackPublishCounter.WithLabelValues(kind, "success").Inc()
|
||||
}
|
||||
|
||||
func RecordTrackPublishCancels(kind string, numCancels int32) {
|
||||
trackPublishCancels.Add(numCancels)
|
||||
promTrackPublishCounter.WithLabelValues(kind, "cancel").Add(float64(numCancels))
|
||||
}
|
||||
|
||||
func RecordPublishTime(
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
) {
|
||||
recordPubSubStreamStartTime("publish", country, source, trackType, d, sdk, kind, 1)
|
||||
}
|
||||
|
||||
func RecordSubscribeTime(
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
count int,
|
||||
) {
|
||||
recordPubSubStreamStartTime("subscribe", country, source, trackType, d, sdk, kind, count)
|
||||
}
|
||||
|
||||
func RecordSubscribeStreamStartTime(
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
) {
|
||||
recordPubSubStreamStartTime("subscribe-stream-start", country, source, trackType, d, sdk, kind, 1)
|
||||
}
|
||||
|
||||
func recordPubSubStreamStartTime(
|
||||
direction string,
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
count int,
|
||||
) {
|
||||
promPubSubTime.WithLabelValues(
|
||||
direction,
|
||||
source.String(),
|
||||
trackType.String(),
|
||||
country,
|
||||
sdk.String(),
|
||||
kind.String(),
|
||||
strconv.Itoa(count),
|
||||
).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeSuccess(kind string) {
|
||||
// modify both current and total counters
|
||||
promTrackSubscribedCurrent.WithLabelValues(kind).Add(1)
|
||||
trackSubscribedCurrent.Inc()
|
||||
|
||||
promTrackSubscribeCounter.WithLabelValues("success", "").Inc()
|
||||
trackSubscribeSuccess.Inc()
|
||||
}
|
||||
|
||||
func RecordTrackUnsubscribed(kind string) {
|
||||
// unsubscribed modifies current counter, but we leave the total values alone since they
|
||||
// are used to compute rate
|
||||
promTrackSubscribedCurrent.WithLabelValues(kind).Sub(1)
|
||||
trackSubscribedCurrent.Dec()
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeAttempt() {
|
||||
trackSubscribeAttempts.Inc()
|
||||
promTrackSubscribeCounter.WithLabelValues("attempt", "").Inc()
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeFailure(err error, isUserError bool) {
|
||||
promTrackSubscribeCounter.WithLabelValues("failure", err.Error()).Inc()
|
||||
|
||||
if isUserError {
|
||||
trackSubscribeUserError.Inc()
|
||||
trackSubscribeCancels.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeCancels(numCancels int32) {
|
||||
trackSubscribeCancels.Add(numCancels)
|
||||
promTrackSubscribeCounter.WithLabelValues("cancel", "").Add(float64(numCancels))
|
||||
}
|
||||
|
||||
func RecordSessionJoinLatency(protocolVersion int, d time.Duration) {
|
||||
promSessionJoinLatency.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordSessionStartTime(protocolVersion int, warp bool, d time.Duration) {
|
||||
promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion), strconv.FormatBool(warp)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordSessionDuration(protocolVersion int, d time.Duration) {
|
||||
promSessionDuration.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordPeerConnectionState(transport livekit.SignalTarget, state string) {
|
||||
promPeerConnection.WithLabelValues(transport.String(), state).Inc()
|
||||
}
|
||||
114
pkg/telemetry/stats.go
Normal file
114
pkg/telemetry/stats.go
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
type StatsKey struct {
|
||||
country string
|
||||
streamType livekit.StreamType
|
||||
participantID livekit.ParticipantID
|
||||
trackID livekit.TrackID
|
||||
trackSource livekit.TrackSource
|
||||
trackType livekit.TrackType
|
||||
track bool
|
||||
}
|
||||
|
||||
func StatsKeyForTrack(
|
||||
country string,
|
||||
streamType livekit.StreamType,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
trackSource livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
) StatsKey {
|
||||
return StatsKey{
|
||||
country: country,
|
||||
streamType: streamType,
|
||||
participantID: participantID,
|
||||
trackID: trackID,
|
||||
trackSource: trackSource,
|
||||
trackType: trackType,
|
||||
track: true,
|
||||
}
|
||||
}
|
||||
|
||||
func StatsKeyForData(
|
||||
country string,
|
||||
streamType livekit.StreamType,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
) StatsKey {
|
||||
return StatsKey{
|
||||
country: country,
|
||||
streamType: streamType,
|
||||
participantID: participantID,
|
||||
trackID: trackID,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackStats(roomID livekit.RoomID, _roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) {
|
||||
t.enqueue(func() {
|
||||
direction := prometheus.Incoming
|
||||
if key.streamType == livekit.StreamType_DOWNSTREAM {
|
||||
direction = prometheus.Outgoing
|
||||
}
|
||||
|
||||
nacks := uint32(0)
|
||||
plis := uint32(0)
|
||||
firs := uint32(0)
|
||||
packets := uint32(0)
|
||||
bytes := uint64(0)
|
||||
retransmitBytes := uint64(0)
|
||||
retransmitPackets := uint32(0)
|
||||
for _, stream := range stat.Streams {
|
||||
nacks += stream.Nacks
|
||||
plis += stream.Plis
|
||||
firs += stream.Firs
|
||||
packets += stream.PrimaryPackets + stream.PaddingPackets
|
||||
bytes += stream.PrimaryBytes + stream.PaddingBytes
|
||||
if key.streamType == livekit.StreamType_DOWNSTREAM {
|
||||
retransmitPackets += stream.RetransmitPackets
|
||||
retransmitBytes += stream.RetransmitBytes
|
||||
} else {
|
||||
// for upstream, we don't account for these separately for now
|
||||
packets += stream.RetransmitPackets
|
||||
bytes += stream.RetransmitBytes
|
||||
}
|
||||
if key.track {
|
||||
prometheus.RecordPacketLoss(key.country, direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets)
|
||||
prometheus.RecordPacketOutOfOrder(key.country, direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets)
|
||||
prometheus.RecordRTT(key.country, direction, key.trackSource, key.trackType, stream.Rtt)
|
||||
prometheus.RecordJitter(key.country, direction, key.trackSource, key.trackType, stream.Jitter)
|
||||
}
|
||||
}
|
||||
prometheus.IncrementRTCP(key.country, direction, nacks, plis, firs)
|
||||
prometheus.IncrementPackets(key.country, direction, uint64(packets), false)
|
||||
prometheus.IncrementBytes(key.country, direction, bytes, false)
|
||||
if retransmitPackets != 0 {
|
||||
prometheus.IncrementPackets(key.country, direction, uint64(retransmitPackets), true)
|
||||
}
|
||||
if retransmitBytes != 0 {
|
||||
prometheus.IncrementBytes(key.country, direction, retransmitBytes, true)
|
||||
}
|
||||
|
||||
if worker, ok := t.getWorker(roomID, key.participantID); ok {
|
||||
worker.OnTrackStat(key.trackID, key.streamType, stat)
|
||||
}
|
||||
})
|
||||
}
|
||||
759
pkg/telemetry/stats_test.go
Normal file
759
pkg/telemetry/stats_test.go
Normal file
@@ -0,0 +1,759 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes"
|
||||
)
|
||||
|
||||
func init() {
|
||||
prometheus.Init("test", livekit.NodeType_SERVER)
|
||||
}
|
||||
|
||||
type telemetryServiceFixture struct {
|
||||
sut telemetry.TelemetryService
|
||||
analytics *telemetryfakes.FakeAnalyticsService
|
||||
}
|
||||
|
||||
func createFixture() *telemetryServiceFixture {
|
||||
fixture := &telemetryServiceFixture{}
|
||||
fixture.analytics = &telemetryfakes.FakeAnalyticsService{}
|
||||
fixture.sut = telemetry.NewTelemetryService(nil, fixture.analytics)
|
||||
return fixture
|
||||
}
|
||||
|
||||
func Test_ParticipantAndRoomDataAreSentWithAnalytics(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
clientInfo := &livekit.ClientInfo{Sdk: 2}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
|
||||
|
||||
// do
|
||||
packet := 33
|
||||
stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet)}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, ""), stat)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
|
||||
require.Equal(t, string(partSID), stats[0].ParticipantId)
|
||||
require.Equal(t, room.Sid, stats[0].RoomId)
|
||||
require.Equal(t, room.Name, stats[0].RoomName)
|
||||
}
|
||||
|
||||
func Test_OnDownstreamPackets(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
clientInfo := &livekit.ClientInfo{Sdk: 2}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
|
||||
|
||||
// do
|
||||
packets := []int{33, 23}
|
||||
totalBytes := packets[0] + packets[1]
|
||||
totalPackets := len(packets)
|
||||
trackID := livekit.TrackID("trackID")
|
||||
for i := range packets {
|
||||
stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packets[i]), PrimaryPackets: uint32(1)}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat)
|
||||
}
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
|
||||
require.Equal(t, totalBytes, int(stats[0].Streams[0].PrimaryBytes))
|
||||
require.Equal(t, totalPackets, int(stats[0].Streams[0].PrimaryPackets))
|
||||
require.Equal(t, string(trackID), stats[0].TrackId)
|
||||
}
|
||||
|
||||
func Test_OnDownstreamPackets_SeveralTracks(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
clientInfo := &livekit.ClientInfo{Sdk: 2}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
|
||||
|
||||
// do
|
||||
packet1 := 33
|
||||
trackID1 := livekit.TrackID("trackID1")
|
||||
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet1), PrimaryPackets: 1}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1)
|
||||
|
||||
packet2 := 23
|
||||
trackID2 := livekit.TrackID("trackID2")
|
||||
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet2), PrimaryPackets: 1}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 2, len(stats))
|
||||
|
||||
found1 := false
|
||||
found2 := false
|
||||
for _, sentStat := range stats {
|
||||
if livekit.TrackID(sentStat.TrackId) == trackID1 {
|
||||
found1 = true
|
||||
require.Equal(t, packet1, int(sentStat.Streams[0].PrimaryBytes))
|
||||
require.Equal(t, 1, int(sentStat.Streams[0].PrimaryPackets))
|
||||
} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
|
||||
found2 = true
|
||||
require.Equal(t, packet2, int(sentStat.Streams[0].PrimaryBytes))
|
||||
require.Equal(t, 1, int(sentStat.Streams[0].PrimaryPackets))
|
||||
}
|
||||
}
|
||||
require.True(t, found1)
|
||||
require.True(t, found2)
|
||||
}
|
||||
|
||||
func Test_OnDownStreamStat(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 1,
|
||||
PrimaryPackets: 1,
|
||||
PacketsLost: 3,
|
||||
Nacks: 1,
|
||||
Plis: 1,
|
||||
Rtt: 23,
|
||||
Jitter: 3,
|
||||
},
|
||||
},
|
||||
}
|
||||
trackID := livekit.TrackID("trackID1")
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 2,
|
||||
PrimaryPackets: 2,
|
||||
PacketsLost: 4,
|
||||
Nacks: 1,
|
||||
Plis: 1,
|
||||
Firs: 1,
|
||||
Rtt: 10,
|
||||
Jitter: 5,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
|
||||
require.Equal(t, 2, int(stats[0].Streams[0].Nacks))
|
||||
require.Equal(t, 2, int(stats[0].Streams[0].Plis))
|
||||
require.Equal(t, 1, int(stats[0].Streams[0].Firs))
|
||||
require.Equal(t, 23, int(stats[0].Streams[0].Rtt)) // max of RTT
|
||||
require.Equal(t, 5, int(stats[0].Streams[0].Jitter)) // max of jitter
|
||||
require.Equal(t, 7, int(stats[0].Streams[0].PacketsLost)) // coalesced delta packet losses
|
||||
require.Equal(t, string(trackID), stats[0].TrackId)
|
||||
}
|
||||
|
||||
func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
trackID := livekit.TrackID("trackID1")
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 1,
|
||||
PrimaryPackets: 1,
|
||||
PacketsLost: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 2,
|
||||
PrimaryPackets: 2,
|
||||
PacketsLost: 4,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 2, fixture.analytics.SendStatsCallCount()) // 2 calls to fixture.sut.FlushStats()
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
|
||||
require.Equal(t, 1, int(stats[0].Streams[0].PacketsLost)) // see pkts1
|
||||
|
||||
_, stats = fixture.analytics.SendStatsArgsForCall(1)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
|
||||
require.Equal(t, 4, int(stats[0].Streams[0].PacketsLost)) // delta loss should be sent as is
|
||||
}
|
||||
|
||||
func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
trackID1 := livekit.TrackID("trackID1")
|
||||
trackID2 := livekit.TrackID("trackID2")
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 1,
|
||||
PrimaryPackets: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 2,
|
||||
PrimaryPackets: 2,
|
||||
Nacks: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2)
|
||||
|
||||
stat3 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 3,
|
||||
PrimaryPackets: 3,
|
||||
Firs: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 2, len(stats))
|
||||
|
||||
found1 := false
|
||||
found2 := false
|
||||
for _, sentStat := range stats {
|
||||
if livekit.TrackID(sentStat.TrackId) == trackID1 {
|
||||
found1 = true
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, sentStat.Kind)
|
||||
require.Equal(t, 1, int(sentStat.Streams[0].Nacks)) // see pkts1 above
|
||||
} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
|
||||
found2 = true
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, sentStat.Kind)
|
||||
require.Equal(t, 1, int(sentStat.Streams[0].Firs)) // see pkts2 above
|
||||
}
|
||||
}
|
||||
require.True(t, found1)
|
||||
require.True(t, found2)
|
||||
}
|
||||
|
||||
func Test_OnUpstreamStat(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 1,
|
||||
PrimaryPackets: 1,
|
||||
PacketsLost: 3,
|
||||
Nacks: 1,
|
||||
Plis: 1,
|
||||
Firs: 1,
|
||||
Rtt: 13,
|
||||
Jitter: 5,
|
||||
},
|
||||
},
|
||||
}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 2,
|
||||
PrimaryPackets: 2,
|
||||
PacketsLost: 4,
|
||||
Nacks: 1,
|
||||
Plis: 1,
|
||||
Firs: 1,
|
||||
Rtt: 33,
|
||||
Jitter: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
|
||||
require.Equal(t, 2, int(stats[0].Streams[0].Nacks))
|
||||
require.Equal(t, 2, int(stats[0].Streams[0].Plis))
|
||||
require.Equal(t, 2, int(stats[0].Streams[0].Firs))
|
||||
require.Equal(t, 33, int(stats[0].Streams[0].Rtt)) // max of RTT
|
||||
require.Equal(t, 5, int(stats[0].Streams[0].Jitter)) // max of jitter
|
||||
require.Equal(t, 7, int(stats[0].Streams[0].PacketsLost)) // coalesced delta packet losses
|
||||
require.Equal(t, string(trackID), stats[0].TrackId)
|
||||
}
|
||||
|
||||
func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
identity := livekit.ParticipantIdentity("part1Identity")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID), Identity: string(identity)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// there should be bytes reported so that stats are sent
|
||||
totalBytes := 1
|
||||
totalPackets := 1
|
||||
trackID1 := livekit.TrackID("trackID1")
|
||||
trackID2 := livekit.TrackID("trackID2")
|
||||
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: uint64(totalBytes),
|
||||
PrimaryPackets: uint32(totalPackets),
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine
|
||||
|
||||
// do
|
||||
totalBytes++
|
||||
totalPackets++
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: uint64(totalBytes),
|
||||
PrimaryPackets: uint32(totalPackets),
|
||||
Nacks: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat2)
|
||||
|
||||
stat3 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: uint64(totalBytes),
|
||||
PrimaryPackets: uint32(totalPackets),
|
||||
Firs: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat3)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 2, len(stats))
|
||||
|
||||
found1 := false
|
||||
found2 := false
|
||||
for _, sentStat := range stats {
|
||||
if livekit.TrackID(sentStat.TrackId) == trackID1 {
|
||||
found1 = true
|
||||
require.Equal(t, livekit.StreamType_UPSTREAM, sentStat.Kind)
|
||||
require.Equal(t, 1, int(sentStat.Streams[0].Nacks)) // see pkts1 above
|
||||
} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
|
||||
found2 = true
|
||||
require.Equal(t, livekit.StreamType_UPSTREAM, sentStat.Kind)
|
||||
require.Equal(t, 1, int(sentStat.Streams[0].Firs)) // see pkts2 above
|
||||
}
|
||||
require.Equal(t, 3, int(sentStat.Streams[0].PrimaryBytes))
|
||||
require.Equal(t, 3, int(sentStat.Streams[0].PrimaryPackets))
|
||||
}
|
||||
require.True(t, found1)
|
||||
require.True(t, found2)
|
||||
|
||||
// remove 1 track - track stats were flushed above, so no more calls to SendStats
|
||||
fixture.sut.TrackUnpublished(context.Background(), room, partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true, true)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
}
|
||||
|
||||
func Test_AnalyticsSentWhenParticipantLeaves(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
partSID := "part1"
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard)
|
||||
|
||||
// should not be called if there are no track stats
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
require.Equal(t, 0, fixture.analytics.SendStatsCallCount())
|
||||
}
|
||||
|
||||
func Test_AddUpTrack(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
var totalBytes uint64 = 3
|
||||
var totalPackets uint32 = 3
|
||||
|
||||
stat := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: totalBytes,
|
||||
PrimaryPackets: totalPackets,
|
||||
},
|
||||
},
|
||||
}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
|
||||
require.Equal(t, totalBytes, stats[0].Streams[0].PrimaryBytes)
|
||||
require.Equal(t, totalPackets, stats[0].Streams[0].PrimaryPackets)
|
||||
require.Equal(t, string(trackID), stats[0].TrackId)
|
||||
}
|
||||
|
||||
func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
trackID := livekit.TrackID("trackID")
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 1,
|
||||
PrimaryPackets: 1,
|
||||
},
|
||||
{
|
||||
PrimaryBytes: 2,
|
||||
PrimaryPackets: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
|
||||
// should be a consolidated stream
|
||||
require.Equal(t, stat1.Streams[0].PrimaryBytes+stat1.Streams[1].PrimaryBytes, stats[0].Streams[0].PrimaryBytes)
|
||||
require.Equal(t, stat1.Streams[0].PrimaryPackets+stat1.Streams[1].PrimaryPackets, stats[0].Streams[0].PrimaryPackets)
|
||||
require.Equal(t, string(trackID), stats[0].TrackId)
|
||||
}
|
||||
|
||||
func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
// upstream bytes
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 3,
|
||||
PrimaryPackets: 3,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1)
|
||||
// downstream bytes
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: 1,
|
||||
PrimaryPackets: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
// test
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 2, len(stats))
|
||||
require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
|
||||
require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[1].Kind)
|
||||
}
|
||||
|
||||
func Test_RoomIDChangeReKeysStatsWorkers(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
|
||||
|
||||
// do - the room restarts and gets a new id
|
||||
restartedRoom := &livekit.Room{Sid: "RestartedSid", Name: "RoomName"}
|
||||
fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(room.Sid), restartedRoom)
|
||||
|
||||
// stats reported with the new id reach the same worker
|
||||
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(restartedRoom.Sid), livekit.RoomName(restartedRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
fixture.flush()
|
||||
|
||||
// one worker, one flush, but two stats - each attributed to the session it was collected in
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(0)
|
||||
require.Equal(t, 2, len(stats))
|
||||
|
||||
byRoom := map[string]*livekit.AnalyticsStat{}
|
||||
for _, stat := range stats {
|
||||
require.Equal(t, string(partSID), stat.ParticipantId)
|
||||
byRoom[stat.RoomId] = stat
|
||||
}
|
||||
require.Len(t, byRoom, 2)
|
||||
require.Equal(t, uint64(33), byRoom[room.Sid].Streams[0].PrimaryBytes)
|
||||
require.Equal(t, uint64(44), byRoom[restartedRoom.Sid].Streams[0].PrimaryBytes)
|
||||
|
||||
// the worker moved rather than being duplicated, so closing it out drains everything
|
||||
fixture.sut.ParticipantLeft(context.Background(), restartedRoom, participantInfo, true, guard)
|
||||
fixture.flush()
|
||||
require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
|
||||
}
|
||||
|
||||
// a forwarded participant is in more than one room at a time under the same participant
|
||||
// id, so re-keying one of those rooms must leave the other alone
|
||||
func Test_RoomIDChangeLeavesForwardedParticipantAlone(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare - the same participant id in a source room and a forwarding destination room
|
||||
sourceRoom := &livekit.Room{Sid: "SourceSid", Name: "SourceRoom"}
|
||||
destRoom := &livekit.Room{Sid: "DestSid", Name: "DestRoom"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
fixture.sut.ParticipantJoined(context.Background(), sourceRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
|
||||
fixture.sut.ParticipantJoined(context.Background(), destRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
|
||||
|
||||
// do - only the destination room restarts
|
||||
restartedDest := &livekit.Room{Sid: "RestartedDestSid", Name: "DestRoom"}
|
||||
fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(destRoom.Sid), restartedDest)
|
||||
|
||||
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(sourceRoom.Sid), livekit.RoomName(sourceRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
|
||||
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(restartedDest.Sid), livekit.RoomName(restartedDest.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
fixture.flush()
|
||||
|
||||
// the source room's worker is untouched, the destination room's worker moved
|
||||
byRoom := map[string]*livekit.AnalyticsStat{}
|
||||
for i := 0; i < fixture.analytics.SendStatsCallCount(); i++ {
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(i)
|
||||
for _, stat := range stats {
|
||||
byRoom[stat.RoomId] = stat
|
||||
}
|
||||
}
|
||||
require.Len(t, byRoom, 2)
|
||||
require.Equal(t, uint64(33), byRoom[sourceRoom.Sid].Streams[0].PrimaryBytes)
|
||||
require.Equal(t, sourceRoom.Name, byRoom[sourceRoom.Sid].RoomName)
|
||||
require.Equal(t, uint64(44), byRoom[restartedDest.Sid].Streams[0].PrimaryBytes)
|
||||
require.Equal(t, restartedDest.Name, byRoom[restartedDest.Sid].RoomName)
|
||||
}
|
||||
|
||||
// a re-key should never land on a room that already has workers, but if it does only
|
||||
// one worker can be keyed at (room, participant) and the superseded one must not be
|
||||
// left unreachable in the flush list
|
||||
func Test_RoomIDChangeParticipantCollision(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare - the same participant id in the room being re-keyed and in its destination
|
||||
prevRoom := &livekit.Room{Sid: "PrevSid", Name: "PrevRoom"}
|
||||
destRoom := &livekit.Room{Sid: "DestSid", Name: "DestRoom"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
fixture.sut.ParticipantJoined(context.Background(), prevRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
|
||||
fixture.sut.ParticipantJoined(context.Background(), destRoom, participantInfo, nil, nil, true, &telemetry.ReferenceGuard{})
|
||||
|
||||
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 33}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(prevRoom.Sid), livekit.RoomName(prevRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
|
||||
|
||||
// do
|
||||
fixture.sut.RoomIDChanged(context.Background(), livekit.RoomID(prevRoom.Sid), destRoom)
|
||||
|
||||
// the superseded worker drains what it collected under the room it was in
|
||||
fixture.flush()
|
||||
byRoom := map[string]*livekit.AnalyticsStat{}
|
||||
for i := 0; i < fixture.analytics.SendStatsCallCount(); i++ {
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(i)
|
||||
for _, stat := range stats {
|
||||
byRoom[stat.RoomId] = stat
|
||||
}
|
||||
}
|
||||
require.Equal(t, uint64(33), byRoom[prevRoom.Sid].Streams[0].PrimaryBytes)
|
||||
|
||||
// the worker already keyed at the destination wins and keeps receiving stats
|
||||
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: 44}}}
|
||||
fixture.sut.TrackStats(livekit.RoomID(destRoom.Sid), livekit.RoomName(destRoom.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
fixture.flush()
|
||||
_, stats := fixture.analytics.SendStatsArgsForCall(fixture.analytics.SendStatsCallCount() - 1)
|
||||
require.Equal(t, 1, len(stats))
|
||||
require.Equal(t, destRoom.Sid, stats[0].RoomId)
|
||||
require.Equal(t, uint64(44), stats[0].Streams[0].PrimaryBytes)
|
||||
}
|
||||
|
||||
func (f *telemetryServiceFixture) flush() {
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
f.sut.FlushStats()
|
||||
}
|
||||
141
pkg/telemetry/statsconn.go
Normal file
141
pkg/telemetry/statsconn.go
Normal file
@@ -0,0 +1,141 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"net"
|
||||
|
||||
"github.com/pion/turn/v5"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
)
|
||||
|
||||
type Listener struct {
|
||||
net.Listener
|
||||
}
|
||||
|
||||
func NewListener(l net.Listener) *Listener {
|
||||
return &Listener{Listener: l}
|
||||
}
|
||||
|
||||
func (l *Listener) Accept() (net.Conn, error) {
|
||||
conn, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewConn(conn, prometheus.Incoming), nil
|
||||
}
|
||||
|
||||
type Conn struct {
|
||||
net.Conn
|
||||
direction prometheus.Direction
|
||||
}
|
||||
|
||||
func NewConn(c net.Conn, direction prometheus.Direction) *Conn {
|
||||
prometheus.AddConnection(direction)
|
||||
return &Conn{Conn: c, direction: direction}
|
||||
}
|
||||
|
||||
func (c *Conn) Read(b []byte) (n int, err error) {
|
||||
n, err = c.Conn.Read(b)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Incoming, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Conn) Write(b []byte) (n int, err error) {
|
||||
n, err = c.Conn.Write(b)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Outgoing, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *Conn) Close() error {
|
||||
prometheus.SubConnection(c.direction)
|
||||
return c.Conn.Close()
|
||||
}
|
||||
|
||||
type PacketConn struct {
|
||||
net.PacketConn
|
||||
direction prometheus.Direction
|
||||
}
|
||||
|
||||
func NewPacketConn(c net.PacketConn, direction prometheus.Direction) *PacketConn {
|
||||
prometheus.AddConnection(direction)
|
||||
return &PacketConn{PacketConn: c, direction: direction}
|
||||
}
|
||||
|
||||
func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
|
||||
n, addr, err = c.PacketConn.ReadFrom(p)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Incoming, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *PacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
|
||||
n, err = c.PacketConn.WriteTo(p, addr)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Outgoing, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *PacketConn) Close() error {
|
||||
prometheus.SubConnection(c.direction)
|
||||
return c.PacketConn.Close()
|
||||
}
|
||||
|
||||
type RelayAddressGenerator struct {
|
||||
turn.RelayAddressGenerator
|
||||
}
|
||||
|
||||
func NewRelayAddressGenerator(g turn.RelayAddressGenerator) *RelayAddressGenerator {
|
||||
return &RelayAddressGenerator{RelayAddressGenerator: g}
|
||||
}
|
||||
|
||||
func (g *RelayAddressGenerator) AllocatePacketConn(c turn.AllocateListenerConfig) (net.PacketConn, net.Addr, error) {
|
||||
conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(c)
|
||||
if err != nil {
|
||||
return nil, addr, err
|
||||
}
|
||||
|
||||
return NewPacketConn(conn, prometheus.Outgoing), addr, err
|
||||
}
|
||||
|
||||
func (g *RelayAddressGenerator) AllocateConn(c turn.AllocateConnConfig) (net.Conn, error) {
|
||||
conn, err := g.RelayAddressGenerator.AllocateConn(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewConn(conn, prometheus.Outgoing), err
|
||||
}
|
||||
|
||||
func (g *RelayAddressGenerator) AllocateListener(c turn.AllocateListenerConfig) (net.Listener, net.Addr, error) {
|
||||
l, addr, err := g.RelayAddressGenerator.AllocateListener(c)
|
||||
if err != nil {
|
||||
return nil, addr, err
|
||||
}
|
||||
|
||||
return NewListener(l), addr, err
|
||||
}
|
||||
521
pkg/telemetry/statsworker.go
Normal file
521
pkg/telemetry/statsworker.go
Normal file
@@ -0,0 +1,521 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"go.uber.org/zap/zapcore"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/utils"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
protoutils "github.com/livekit/protocol/utils"
|
||||
)
|
||||
|
||||
type ReferenceGuard struct {
|
||||
activated, released bool
|
||||
}
|
||||
|
||||
func (r *ReferenceGuard) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
if r != nil {
|
||||
e.AddUintptr("self", uintptr(unsafe.Pointer(r)))
|
||||
e.AddBool("activated", r.activated)
|
||||
e.AddBool("released", r.released)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
type ReferenceCount struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (s *ReferenceCount) Activate(guard *ReferenceGuard) {
|
||||
if guard != nil && !guard.activated {
|
||||
guard.activated = true
|
||||
s.count++
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ReferenceCount) Release(guard *ReferenceGuard) bool {
|
||||
if guard == nil || !guard.activated || guard.released {
|
||||
return false
|
||||
}
|
||||
guard.released = true
|
||||
s.count--
|
||||
return s.count == 0
|
||||
}
|
||||
|
||||
// Take hands over every reference held, leaving none behind.
|
||||
func (s *ReferenceCount) Take() int {
|
||||
count := s.count
|
||||
s.count = 0
|
||||
return count
|
||||
}
|
||||
|
||||
// Absorb takes on references handed over from elsewhere.
|
||||
func (s *ReferenceCount) Absorb(count int) {
|
||||
s.count += count
|
||||
}
|
||||
|
||||
func (s ReferenceCount) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
e.AddInt("count", s.count)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
// statsBatch is stats collected while the worker was in one room
|
||||
type statsBatch struct {
|
||||
roomID livekit.RoomID
|
||||
roomName livekit.RoomName
|
||||
incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
|
||||
outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
|
||||
}
|
||||
|
||||
func (b statsBatch) isEmpty() bool {
|
||||
return len(b.incomingPerTrack) == 0 && len(b.outgoingPerTrack) == 0
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
// StatsWorker handles participant stats
|
||||
type StatsWorker struct {
|
||||
next *StatsWorker
|
||||
|
||||
ctx context.Context
|
||||
t TelemetryService
|
||||
participantID livekit.ParticipantID
|
||||
participantIdentity livekit.ParticipantIdentity
|
||||
|
||||
lock sync.RWMutex
|
||||
// the room a worker belongs to can change mid-session, so it is mutable state
|
||||
// guarded by `lock`. it is kept in sync with the key the worker is filed under in
|
||||
// telemetryService.workers, see telemetryService.reKeyRoom.
|
||||
roomID livekit.RoomID
|
||||
roomName livekit.RoomName
|
||||
// batches sealed off by a room change, they carry the room they were collected
|
||||
// in and go out on the next flush
|
||||
sealed []statsBatch
|
||||
isConnected bool
|
||||
outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
|
||||
incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
|
||||
refCount ReferenceCount
|
||||
closedAt time.Time
|
||||
}
|
||||
|
||||
func newStatsWorker(
|
||||
ctx context.Context,
|
||||
t TelemetryService,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
guard *ReferenceGuard,
|
||||
) *StatsWorker {
|
||||
s := &StatsWorker{
|
||||
ctx: ctx,
|
||||
t: t,
|
||||
roomID: roomID,
|
||||
roomName: roomName,
|
||||
participantID: participantID,
|
||||
participantIdentity: identity,
|
||||
outgoingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat),
|
||||
incomingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat),
|
||||
}
|
||||
s.refCount.Activate(guard)
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *StatsWorker) OnTrackStat(trackID livekit.TrackID, direction livekit.StreamType, stat *livekit.AnalyticsStat) {
|
||||
s.lock.Lock()
|
||||
if direction == livekit.StreamType_DOWNSTREAM {
|
||||
s.outgoingPerTrack[trackID] = append(s.outgoingPerTrack[trackID], stat)
|
||||
} else {
|
||||
s.incomingPerTrack[trackID] = append(s.incomingPerTrack[trackID], stat)
|
||||
}
|
||||
s.lock.Unlock()
|
||||
}
|
||||
|
||||
func (s *StatsWorker) ParticipantID() livekit.ParticipantID {
|
||||
return s.participantID
|
||||
}
|
||||
|
||||
func (s *StatsWorker) RoomID() livekit.RoomID {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
return s.roomID
|
||||
}
|
||||
|
||||
// SetRoom re-points the worker at a room.
|
||||
//
|
||||
// Stats collected so far are sealed off rather than re-stamped - a room id changes
|
||||
// because the previous session ended, so what was collected under it belongs to it.
|
||||
// Sealing keeps the re-key free of any sending, the sealed stats go out on the next
|
||||
// flush like every other stat.
|
||||
func (s *StatsWorker) SetRoom(roomID livekit.RoomID, roomName livekit.RoomName) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if s.roomID == roomID && s.roomName == roomName {
|
||||
return
|
||||
}
|
||||
|
||||
if batch := s.sealStatsLocked(); !batch.isEmpty() {
|
||||
s.sealed = append(s.sealed, batch)
|
||||
}
|
||||
|
||||
s.roomID = roomID
|
||||
s.roomName = roomName
|
||||
}
|
||||
|
||||
// sealStatsLocked hands over everything collected since the last seal, stamped
|
||||
// with the room it was collected in
|
||||
func (s *StatsWorker) sealStatsLocked() statsBatch {
|
||||
batch := statsBatch{
|
||||
roomID: s.roomID,
|
||||
roomName: s.roomName,
|
||||
incomingPerTrack: s.incomingPerTrack,
|
||||
outgoingPerTrack: s.outgoingPerTrack,
|
||||
}
|
||||
|
||||
s.incomingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
|
||||
s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
|
||||
|
||||
return batch
|
||||
}
|
||||
|
||||
func (s *StatsWorker) SetConnected() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
|
||||
s.lock.Lock()
|
||||
s.isConnected = true
|
||||
s.lock.Unlock()
|
||||
}
|
||||
|
||||
func (s *StatsWorker) IsConnected() bool {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
return s.isConnected
|
||||
}
|
||||
|
||||
func (s *StatsWorker) Flush(now time.Time, closeWait time.Duration) bool {
|
||||
ts := timestamppb.New(now)
|
||||
|
||||
s.lock.Lock()
|
||||
// anything sealed off by a room change goes out along with the current batch,
|
||||
// each stamped with the room it was collected in
|
||||
batches := append(s.sealed, s.sealStatsLocked())
|
||||
s.sealed = nil
|
||||
|
||||
closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > closeWait
|
||||
s.lock.Unlock()
|
||||
|
||||
numTracks := 0
|
||||
for _, batch := range batches {
|
||||
numTracks += len(batch.incomingPerTrack) + len(batch.outgoingPerTrack)
|
||||
}
|
||||
|
||||
stats := make([]*livekit.AnalyticsStat, 0, numTracks)
|
||||
for _, batch := range batches {
|
||||
stats = s.collectStats(ts, batch, livekit.StreamType_UPSTREAM, stats)
|
||||
stats = s.collectStats(ts, batch, livekit.StreamType_DOWNSTREAM, stats)
|
||||
}
|
||||
if len(stats) > 0 {
|
||||
s.t.SendStats(s.ctx, stats)
|
||||
}
|
||||
|
||||
return closed
|
||||
}
|
||||
|
||||
func (s *StatsWorker) Close(guard *ReferenceGuard) bool {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if !s.refCount.Release(guard) {
|
||||
return false
|
||||
}
|
||||
|
||||
ok := s.closedAt.IsZero()
|
||||
if ok {
|
||||
s.closedAt = time.Now()
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// ForceClose closes the worker irrespective of outstanding references. Used when a
|
||||
// worker can no longer be reached through the worker map, so that it drains and is
|
||||
// reaped instead of lingering in the flush list forever.
|
||||
//
|
||||
// Its references are handed over to `successor`, the worker that can be reached in its
|
||||
// place, so that whoever holds one still has a live worker to close. A ReferenceGuard
|
||||
// records that it activated some worker, not which one, so leaving them behind would
|
||||
// strand the successor with references it can never see released.
|
||||
func (s *StatsWorker) ForceClose(successor *StatsWorker) bool {
|
||||
s.lock.Lock()
|
||||
if !s.closedAt.IsZero() {
|
||||
s.lock.Unlock()
|
||||
return false
|
||||
}
|
||||
|
||||
s.closedAt = time.Now()
|
||||
count := s.refCount.Take()
|
||||
s.lock.Unlock()
|
||||
|
||||
if successor != nil && count != 0 {
|
||||
successor.lock.Lock()
|
||||
successor.refCount.Absorb(count)
|
||||
successor.lock.Unlock()
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *StatsWorker) Closed(guard *ReferenceGuard) bool {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
if s.closedAt.IsZero() {
|
||||
s.refCount.Activate(guard)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *StatsWorker) collectStats(
|
||||
ts *timestamppb.Timestamp,
|
||||
batch statsBatch,
|
||||
streamType livekit.StreamType,
|
||||
stats []*livekit.AnalyticsStat,
|
||||
) []*livekit.AnalyticsStat {
|
||||
perTrack := batch.incomingPerTrack
|
||||
if streamType == livekit.StreamType_DOWNSTREAM {
|
||||
perTrack = batch.outgoingPerTrack
|
||||
}
|
||||
|
||||
for trackID, analyticsStats := range perTrack {
|
||||
coalesced := coalesce(analyticsStats)
|
||||
if coalesced == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
coalesced.TimeStamp = ts
|
||||
coalesced.TrackId = string(trackID)
|
||||
coalesced.Kind = streamType
|
||||
coalesced.RoomId = string(batch.roomID)
|
||||
coalesced.ParticipantId = string(s.participantID)
|
||||
coalesced.RoomName = string(batch.roomName)
|
||||
stats = append(stats, coalesced)
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
func (s *StatsWorker) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
e.AddString("room", string(s.roomName))
|
||||
e.AddString("roomID", string(s.roomID))
|
||||
e.AddString("participant", string(s.participantIdentity))
|
||||
e.AddString("participantID", string(s.participantID))
|
||||
e.AddBool("isConnected", s.isConnected)
|
||||
e.AddTime("closedAt", s.closedAt)
|
||||
e.AddObject("refCount", s.refCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// create a single stream and single video layer post aggregation
|
||||
func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat {
|
||||
if len(stats) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// find aggregates across streams
|
||||
startTime := time.Time{}
|
||||
endTime := time.Time{}
|
||||
scoreSum := float32(0.0) // used for average
|
||||
minScore := float32(0.0) // min score in batched stats
|
||||
var scores []float32 // used for median
|
||||
maxRtt := uint32(0)
|
||||
maxJitter := uint32(0)
|
||||
coalescedVideoLayers := make(map[int32]*livekit.AnalyticsVideoLayer)
|
||||
coalescedStream := &livekit.AnalyticsStream{}
|
||||
for _, stat := range stats {
|
||||
if !isValid(stat) {
|
||||
logger.Warnw("telemetry skipping invalid stat", nil, "stat", stat)
|
||||
continue
|
||||
}
|
||||
|
||||
// only consider non-zero scores
|
||||
if stat.Score > 0 {
|
||||
if minScore == 0 {
|
||||
minScore = stat.Score
|
||||
} else if stat.Score < minScore {
|
||||
minScore = stat.Score
|
||||
}
|
||||
scoreSum += stat.Score
|
||||
scores = append(scores, stat.Score)
|
||||
}
|
||||
|
||||
for _, analyticsStream := range stat.Streams {
|
||||
start := analyticsStream.StartTime.AsTime()
|
||||
if startTime.IsZero() || startTime.After(start) {
|
||||
startTime = start
|
||||
}
|
||||
|
||||
end := analyticsStream.EndTime.AsTime()
|
||||
if endTime.IsZero() || endTime.Before(end) {
|
||||
endTime = end
|
||||
}
|
||||
|
||||
if analyticsStream.Rtt > maxRtt {
|
||||
maxRtt = analyticsStream.Rtt
|
||||
}
|
||||
|
||||
if analyticsStream.Jitter > maxJitter {
|
||||
maxJitter = analyticsStream.Jitter
|
||||
}
|
||||
|
||||
coalescedStream.PrimaryPackets += analyticsStream.PrimaryPackets
|
||||
coalescedStream.PrimaryBytes += analyticsStream.PrimaryBytes
|
||||
coalescedStream.RetransmitPackets += analyticsStream.RetransmitPackets
|
||||
coalescedStream.RetransmitBytes += analyticsStream.RetransmitBytes
|
||||
coalescedStream.PaddingPackets += analyticsStream.PaddingPackets
|
||||
coalescedStream.PaddingBytes += analyticsStream.PaddingBytes
|
||||
coalescedStream.PacketsLost += analyticsStream.PacketsLost
|
||||
coalescedStream.PacketsOutOfOrder += analyticsStream.PacketsOutOfOrder
|
||||
coalescedStream.Frames += analyticsStream.Frames
|
||||
coalescedStream.Nacks += analyticsStream.Nacks
|
||||
coalescedStream.Plis += analyticsStream.Plis
|
||||
coalescedStream.Firs += analyticsStream.Firs
|
||||
|
||||
for _, videoLayer := range analyticsStream.VideoLayers {
|
||||
coalescedVideoLayer := coalescedVideoLayers[videoLayer.Layer]
|
||||
if coalescedVideoLayer == nil {
|
||||
coalescedVideoLayer = protoutils.CloneProto(videoLayer)
|
||||
coalescedVideoLayers[videoLayer.Layer] = coalescedVideoLayer
|
||||
} else {
|
||||
coalescedVideoLayer.Packets += videoLayer.Packets
|
||||
coalescedVideoLayer.Bytes += videoLayer.Bytes
|
||||
coalescedVideoLayer.Frames += videoLayer.Frames
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
coalescedStream.StartTime = timestamppb.New(startTime)
|
||||
coalescedStream.EndTime = timestamppb.New(endTime)
|
||||
coalescedStream.Rtt = maxRtt
|
||||
coalescedStream.Jitter = maxJitter
|
||||
|
||||
// whittle it down to one video layer, just the max available layer
|
||||
maxVideoLayer := int32(-1)
|
||||
for _, coalescedVideoLayer := range coalescedVideoLayers {
|
||||
if maxVideoLayer == -1 || maxVideoLayer < coalescedVideoLayer.Layer {
|
||||
maxVideoLayer = coalescedVideoLayer.Layer
|
||||
coalescedStream.VideoLayers = []*livekit.AnalyticsVideoLayer{coalescedVideoLayer}
|
||||
}
|
||||
}
|
||||
|
||||
stat := &livekit.AnalyticsStat{
|
||||
MinScore: minScore,
|
||||
MedianScore: utils.Median(scores),
|
||||
Streams: []*livekit.AnalyticsStream{coalescedStream},
|
||||
Mime: stats[len(stats)-1].Mime, // use the latest Mime
|
||||
}
|
||||
numScores := len(scores)
|
||||
if numScores > 0 {
|
||||
stat.Score = scoreSum / float32(numScores)
|
||||
}
|
||||
return stat
|
||||
}
|
||||
|
||||
type CondensedStat struct {
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Bytes uint64
|
||||
Packets uint32
|
||||
PacketsLost uint32
|
||||
Frames uint32
|
||||
}
|
||||
|
||||
func CondenseStat(stat *livekit.AnalyticsStat) (ps CondensedStat, ok bool) {
|
||||
if ok = isValid(stat); !ok {
|
||||
return
|
||||
}
|
||||
|
||||
for _, stream := range stat.Streams {
|
||||
startTime := stream.StartTime.AsTime()
|
||||
endTime := stream.EndTime.AsTime()
|
||||
if ps.StartTime.IsZero() || startTime.Before(ps.StartTime) {
|
||||
ps.StartTime = startTime
|
||||
}
|
||||
if endTime.After(ps.EndTime) {
|
||||
ps.EndTime = endTime
|
||||
}
|
||||
|
||||
ps.Bytes += stream.PrimaryBytes
|
||||
ps.Packets += stream.PrimaryPackets
|
||||
ps.PacketsLost += stream.PacketsLost
|
||||
if stream.Frames > ps.Frames {
|
||||
ps.Frames = stream.Frames
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func isValid(stat *livekit.AnalyticsStat) bool {
|
||||
for _, analyticsStream := range stat.Streams {
|
||||
if int32(analyticsStream.PrimaryPackets) < 0 ||
|
||||
int64(analyticsStream.PrimaryBytes) < 0 ||
|
||||
int32(analyticsStream.RetransmitPackets) < 0 ||
|
||||
int64(analyticsStream.RetransmitBytes) < 0 ||
|
||||
int32(analyticsStream.PaddingPackets) < 0 ||
|
||||
int64(analyticsStream.PaddingBytes) < 0 ||
|
||||
int32(analyticsStream.PacketsLost) < 0 ||
|
||||
int32(analyticsStream.PacketsOutOfOrder) < 0 ||
|
||||
int32(analyticsStream.Frames) < 0 ||
|
||||
int32(analyticsStream.Nacks) < 0 ||
|
||||
int32(analyticsStream.Plis) < 0 ||
|
||||
int32(analyticsStream.Firs) < 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, videoLayer := range analyticsStream.VideoLayers {
|
||||
if int32(videoLayer.Packets) < 0 ||
|
||||
int64(videoLayer.Bytes) < 0 ||
|
||||
int32(videoLayer.Frames) < 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
102
pkg/telemetry/statsworker_test.go
Normal file
102
pkg/telemetry/statsworker_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zapcore"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
func TestStatsWorker(t *testing.T) {
|
||||
t.Run("reference counted close works", func(t *testing.T) {
|
||||
var g0, g1 ReferenceGuard
|
||||
w := newStatsWorker(t.Context(), nil, "", "", "", "", &g0)
|
||||
require.False(t, w.Closed(&g1))
|
||||
require.False(t, w.Close(&g0))
|
||||
require.False(t, w.Closed(&g1))
|
||||
require.True(t, w.Close(&g1))
|
||||
require.True(t, w.Closed(&g1))
|
||||
})
|
||||
|
||||
// a ReferenceGuard records that it activated some worker, not which one, so a
|
||||
// superseded worker has to hand its references to the one reachable in its place
|
||||
t.Run("force close hands references to the successor", func(t *testing.T) {
|
||||
t.Run("a guard shared by both workers", func(t *testing.T) {
|
||||
// the second worker never got a reference, the guard was already activated
|
||||
var g ReferenceGuard
|
||||
superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &g)
|
||||
survivor := newStatsWorker(t.Context(), nil, "", "", "", "", &g)
|
||||
require.Equal(t, 1, superseded.refCount.count)
|
||||
require.Equal(t, 0, survivor.refCount.count)
|
||||
|
||||
require.True(t, superseded.ForceClose(survivor))
|
||||
require.Equal(t, 0, superseded.refCount.count)
|
||||
require.Equal(t, 1, survivor.refCount.count)
|
||||
|
||||
// without the hand over this would leave the survivor at -1 and never closed
|
||||
require.True(t, survivor.Close(&g))
|
||||
require.True(t, survivor.Closed(&g))
|
||||
})
|
||||
|
||||
t.Run("a guard per worker", func(t *testing.T) {
|
||||
var gSuperseded, gSurvivor ReferenceGuard
|
||||
superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &gSuperseded)
|
||||
survivor := newStatsWorker(t.Context(), nil, "", "", "", "", &gSurvivor)
|
||||
|
||||
require.True(t, superseded.ForceClose(survivor))
|
||||
require.Equal(t, 2, survivor.refCount.count)
|
||||
|
||||
// the superseded worker's owner departs, it must not close the survivor early
|
||||
require.False(t, survivor.Close(&gSuperseded))
|
||||
require.True(t, survivor.Close(&gSurvivor))
|
||||
})
|
||||
|
||||
t.Run("closing an already closed worker holds on to its references", func(t *testing.T) {
|
||||
var g ReferenceGuard
|
||||
superseded := newStatsWorker(t.Context(), nil, "", "", "", "", &g)
|
||||
survivor := newStatsWorker(t.Context(), nil, "", "", "", "", nil)
|
||||
|
||||
require.True(t, superseded.ForceClose(nil))
|
||||
require.False(t, superseded.ForceClose(survivor))
|
||||
require.Equal(t, 0, survivor.refCount.count)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("logging a nil worker does not panic", func(t *testing.T) {
|
||||
var w *StatsWorker
|
||||
require.NoError(t, w.MarshalLogObject(zapcore.NewMapObjectEncoder()))
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetOrCreateWorkerReleasedGuard(t *testing.T) {
|
||||
// ParticipantActive overtaken by the participant's close arrives with a guard that
|
||||
// ParticipantLeft already released. It must not replace the closed worker with one
|
||||
// nothing can release.
|
||||
ts := &telemetryService{workers: make(map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker)}
|
||||
roomID, pID := livekit.RoomID("room"), livekit.ParticipantID("participant")
|
||||
|
||||
var g ReferenceGuard
|
||||
w, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g)
|
||||
require.False(t, found)
|
||||
require.True(t, w.Close(&g))
|
||||
|
||||
t.Run("closed worker still in the map", func(t *testing.T) {
|
||||
late, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g)
|
||||
require.True(t, found)
|
||||
require.Same(t, w, late)
|
||||
require.Same(t, w, ts.workers[roomID][pID])
|
||||
})
|
||||
|
||||
t.Run("closed worker already reaped", func(t *testing.T) {
|
||||
delete(ts.workers[roomID], pID)
|
||||
|
||||
late, found := ts.getOrCreateWorker(context.Background(), roomID, "", pID, "", &g)
|
||||
require.True(t, found)
|
||||
require.Nil(t, late)
|
||||
require.Empty(t, ts.workers[roomID])
|
||||
late.SetConnected()
|
||||
})
|
||||
}
|
||||
234
pkg/telemetry/telemetryfakes/fake_analytics_service.go
Normal file
234
pkg/telemetry/telemetryfakes/fake_analytics_service.go
Normal file
@@ -0,0 +1,234 @@
|
||||
// Code generated by counterfeiter. DO NOT EDIT.
|
||||
package telemetryfakes
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/observability/roomobs"
|
||||
)
|
||||
|
||||
type FakeAnalyticsService struct {
|
||||
RoomProjectReporterStub func(context.Context) roomobs.ProjectReporter
|
||||
roomProjectReporterMutex sync.RWMutex
|
||||
roomProjectReporterArgsForCall []struct {
|
||||
arg1 context.Context
|
||||
}
|
||||
roomProjectReporterReturns struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}
|
||||
roomProjectReporterReturnsOnCall map[int]struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}
|
||||
SendEventStub func(context.Context, *livekit.AnalyticsEvent)
|
||||
sendEventMutex sync.RWMutex
|
||||
sendEventArgsForCall []struct {
|
||||
arg1 context.Context
|
||||
arg2 *livekit.AnalyticsEvent
|
||||
}
|
||||
SendNodeRoomStatesStub func(context.Context, *livekit.AnalyticsNodeRooms)
|
||||
sendNodeRoomStatesMutex sync.RWMutex
|
||||
sendNodeRoomStatesArgsForCall []struct {
|
||||
arg1 context.Context
|
||||
arg2 *livekit.AnalyticsNodeRooms
|
||||
}
|
||||
SendStatsStub func(context.Context, []*livekit.AnalyticsStat)
|
||||
sendStatsMutex sync.RWMutex
|
||||
sendStatsArgsForCall []struct {
|
||||
arg1 context.Context
|
||||
arg2 []*livekit.AnalyticsStat
|
||||
}
|
||||
invocations map[string][][]interface{}
|
||||
invocationsMutex sync.RWMutex
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporter(arg1 context.Context) roomobs.ProjectReporter {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
ret, specificReturn := fake.roomProjectReporterReturnsOnCall[len(fake.roomProjectReporterArgsForCall)]
|
||||
fake.roomProjectReporterArgsForCall = append(fake.roomProjectReporterArgsForCall, struct {
|
||||
arg1 context.Context
|
||||
}{arg1})
|
||||
stub := fake.RoomProjectReporterStub
|
||||
fakeReturns := fake.roomProjectReporterReturns
|
||||
fake.recordInvocation("RoomProjectReporter", []interface{}{arg1})
|
||||
fake.roomProjectReporterMutex.Unlock()
|
||||
if stub != nil {
|
||||
return stub(arg1)
|
||||
}
|
||||
if specificReturn {
|
||||
return ret.result1
|
||||
}
|
||||
return fakeReturns.result1
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterCallCount() int {
|
||||
fake.roomProjectReporterMutex.RLock()
|
||||
defer fake.roomProjectReporterMutex.RUnlock()
|
||||
return len(fake.roomProjectReporterArgsForCall)
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterCalls(stub func(context.Context) roomobs.ProjectReporter) {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
defer fake.roomProjectReporterMutex.Unlock()
|
||||
fake.RoomProjectReporterStub = stub
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterArgsForCall(i int) context.Context {
|
||||
fake.roomProjectReporterMutex.RLock()
|
||||
defer fake.roomProjectReporterMutex.RUnlock()
|
||||
argsForCall := fake.roomProjectReporterArgsForCall[i]
|
||||
return argsForCall.arg1
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterReturns(result1 roomobs.ProjectReporter) {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
defer fake.roomProjectReporterMutex.Unlock()
|
||||
fake.RoomProjectReporterStub = nil
|
||||
fake.roomProjectReporterReturns = struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}{result1}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterReturnsOnCall(i int, result1 roomobs.ProjectReporter) {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
defer fake.roomProjectReporterMutex.Unlock()
|
||||
fake.RoomProjectReporterStub = nil
|
||||
if fake.roomProjectReporterReturnsOnCall == nil {
|
||||
fake.roomProjectReporterReturnsOnCall = make(map[int]struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
})
|
||||
}
|
||||
fake.roomProjectReporterReturnsOnCall[i] = struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}{result1}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendEvent(arg1 context.Context, arg2 *livekit.AnalyticsEvent) {
|
||||
fake.sendEventMutex.Lock()
|
||||
fake.sendEventArgsForCall = append(fake.sendEventArgsForCall, struct {
|
||||
arg1 context.Context
|
||||
arg2 *livekit.AnalyticsEvent
|
||||
}{arg1, arg2})
|
||||
stub := fake.SendEventStub
|
||||
fake.recordInvocation("SendEvent", []interface{}{arg1, arg2})
|
||||
fake.sendEventMutex.Unlock()
|
||||
if stub != nil {
|
||||
fake.SendEventStub(arg1, arg2)
|
||||
}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendEventCallCount() int {
|
||||
fake.sendEventMutex.RLock()
|
||||
defer fake.sendEventMutex.RUnlock()
|
||||
return len(fake.sendEventArgsForCall)
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendEventCalls(stub func(context.Context, *livekit.AnalyticsEvent)) {
|
||||
fake.sendEventMutex.Lock()
|
||||
defer fake.sendEventMutex.Unlock()
|
||||
fake.SendEventStub = stub
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendEventArgsForCall(i int) (context.Context, *livekit.AnalyticsEvent) {
|
||||
fake.sendEventMutex.RLock()
|
||||
defer fake.sendEventMutex.RUnlock()
|
||||
argsForCall := fake.sendEventArgsForCall[i]
|
||||
return argsForCall.arg1, argsForCall.arg2
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendNodeRoomStates(arg1 context.Context, arg2 *livekit.AnalyticsNodeRooms) {
|
||||
fake.sendNodeRoomStatesMutex.Lock()
|
||||
fake.sendNodeRoomStatesArgsForCall = append(fake.sendNodeRoomStatesArgsForCall, struct {
|
||||
arg1 context.Context
|
||||
arg2 *livekit.AnalyticsNodeRooms
|
||||
}{arg1, arg2})
|
||||
stub := fake.SendNodeRoomStatesStub
|
||||
fake.recordInvocation("SendNodeRoomStates", []interface{}{arg1, arg2})
|
||||
fake.sendNodeRoomStatesMutex.Unlock()
|
||||
if stub != nil {
|
||||
fake.SendNodeRoomStatesStub(arg1, arg2)
|
||||
}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendNodeRoomStatesCallCount() int {
|
||||
fake.sendNodeRoomStatesMutex.RLock()
|
||||
defer fake.sendNodeRoomStatesMutex.RUnlock()
|
||||
return len(fake.sendNodeRoomStatesArgsForCall)
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendNodeRoomStatesCalls(stub func(context.Context, *livekit.AnalyticsNodeRooms)) {
|
||||
fake.sendNodeRoomStatesMutex.Lock()
|
||||
defer fake.sendNodeRoomStatesMutex.Unlock()
|
||||
fake.SendNodeRoomStatesStub = stub
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendNodeRoomStatesArgsForCall(i int) (context.Context, *livekit.AnalyticsNodeRooms) {
|
||||
fake.sendNodeRoomStatesMutex.RLock()
|
||||
defer fake.sendNodeRoomStatesMutex.RUnlock()
|
||||
argsForCall := fake.sendNodeRoomStatesArgsForCall[i]
|
||||
return argsForCall.arg1, argsForCall.arg2
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendStats(arg1 context.Context, arg2 []*livekit.AnalyticsStat) {
|
||||
var arg2Copy []*livekit.AnalyticsStat
|
||||
if arg2 != nil {
|
||||
arg2Copy = make([]*livekit.AnalyticsStat, len(arg2))
|
||||
copy(arg2Copy, arg2)
|
||||
}
|
||||
fake.sendStatsMutex.Lock()
|
||||
fake.sendStatsArgsForCall = append(fake.sendStatsArgsForCall, struct {
|
||||
arg1 context.Context
|
||||
arg2 []*livekit.AnalyticsStat
|
||||
}{arg1, arg2Copy})
|
||||
stub := fake.SendStatsStub
|
||||
fake.recordInvocation("SendStats", []interface{}{arg1, arg2Copy})
|
||||
fake.sendStatsMutex.Unlock()
|
||||
if stub != nil {
|
||||
fake.SendStatsStub(arg1, arg2)
|
||||
}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendStatsCallCount() int {
|
||||
fake.sendStatsMutex.RLock()
|
||||
defer fake.sendStatsMutex.RUnlock()
|
||||
return len(fake.sendStatsArgsForCall)
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendStatsCalls(stub func(context.Context, []*livekit.AnalyticsStat)) {
|
||||
fake.sendStatsMutex.Lock()
|
||||
defer fake.sendStatsMutex.Unlock()
|
||||
fake.SendStatsStub = stub
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendStatsArgsForCall(i int) (context.Context, []*livekit.AnalyticsStat) {
|
||||
fake.sendStatsMutex.RLock()
|
||||
defer fake.sendStatsMutex.RUnlock()
|
||||
argsForCall := fake.sendStatsArgsForCall[i]
|
||||
return argsForCall.arg1, argsForCall.arg2
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) Invocations() map[string][][]interface{} {
|
||||
fake.invocationsMutex.RLock()
|
||||
defer fake.invocationsMutex.RUnlock()
|
||||
copiedInvocations := map[string][][]interface{}{}
|
||||
for key, value := range fake.invocations {
|
||||
copiedInvocations[key] = value
|
||||
}
|
||||
return copiedInvocations
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) recordInvocation(key string, args []interface{}) {
|
||||
fake.invocationsMutex.Lock()
|
||||
defer fake.invocationsMutex.Unlock()
|
||||
if fake.invocations == nil {
|
||||
fake.invocations = map[string][][]interface{}{}
|
||||
}
|
||||
if fake.invocations[key] == nil {
|
||||
fake.invocations[key] = [][]interface{}{}
|
||||
}
|
||||
fake.invocations[key] = append(fake.invocations[key], args)
|
||||
}
|
||||
|
||||
var _ telemetry.AnalyticsService = new(FakeAnalyticsService)
|
||||
1721
pkg/telemetry/telemetryfakes/fake_telemetry_service.go
Normal file
1721
pkg/telemetry/telemetryfakes/fake_telemetry_service.go
Normal file
File diff suppressed because it is too large
Load Diff
436
pkg/telemetry/telemetryservice.go
Normal file
436
pkg/telemetry/telemetryservice.go
Normal file
@@ -0,0 +1,436 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
"github.com/livekit/livekit-server/pkg/utils"
|
||||
"github.com/livekit/protocol/codecs/mime"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/webhook"
|
||||
)
|
||||
|
||||
//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
|
||||
|
||||
//counterfeiter:generate . TelemetryService
|
||||
type TelemetryService interface {
|
||||
// TrackStats is called periodically for each track in both directions (published/subscribed)
|
||||
TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat)
|
||||
|
||||
// events
|
||||
RoomStarted(ctx context.Context, room *livekit.Room)
|
||||
RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason)
|
||||
|
||||
// ParticipantJoined - a participant establishes signal connection to a room
|
||||
ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard)
|
||||
// ParticipantActive - a participant establishes media connection
|
||||
ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, isWarp bool, guard *ReferenceGuard)
|
||||
// ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection
|
||||
ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason)
|
||||
// ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before
|
||||
ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard)
|
||||
// RoomIDChanged - the room kept its session, but got a different id (a provisional room id
|
||||
// replaced by the resolved one), re-keys the stats workers of every participant in the room
|
||||
RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room)
|
||||
// TrackPublishRequested - a publication attempt has been received
|
||||
TrackPublishRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
// TrackPublished - a publication attempt has been successful
|
||||
TrackPublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
// TrackUnpublished - a participant unpublished a track
|
||||
TrackUnpublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, wasPublishedLocally bool, shouldSendEvent bool)
|
||||
// TrackSubscribeRequested - a participant requested to subscribe to a track
|
||||
TrackSubscribeRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackSubscribed - a participant subscribed to a track successfully
|
||||
TrackSubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool)
|
||||
// TrackUnsubscribed - a participant unsubscribed from a track successfully
|
||||
TrackUnsubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
// TrackSubscribeFailed - failure to subscribe to a track
|
||||
TrackSubscribeFailed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool)
|
||||
// TrackMuted - the publisher has muted the Track
|
||||
TrackMuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackUnmuted - the publisher has muted the Track
|
||||
TrackUnmuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackPublishedUpdate - track metadata has been updated
|
||||
TrackPublishedUpdate(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackMaxSubscribedVideoQuality - publisher is notified of the max quality subscribers desire
|
||||
TrackMaxSubscribedVideoQuality(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality)
|
||||
TrackPublishRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats)
|
||||
TrackSubscribeRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats)
|
||||
|
||||
EgressStarted(ctx context.Context, info *livekit.EgressInfo)
|
||||
EgressUpdated(ctx context.Context, info *livekit.EgressInfo)
|
||||
EgressEnded(ctx context.Context, info *livekit.EgressInfo)
|
||||
|
||||
IngressCreated(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressDeleted(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressStarted(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressUpdated(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressEnded(ctx context.Context, info *livekit.IngressInfo)
|
||||
|
||||
LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms)
|
||||
|
||||
Report(ctx context.Context, reportInfo *livekit.ReportInfo)
|
||||
|
||||
APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo)
|
||||
|
||||
Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo)
|
||||
|
||||
// helpers
|
||||
AnalyticsService
|
||||
NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo)
|
||||
FlushStats()
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
var _ TelemetryService = (*NullTelemetryService)(nil)
|
||||
|
||||
type NullTelemetryService struct {
|
||||
NullAnalyticService
|
||||
}
|
||||
|
||||
func (n NullTelemetryService) TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) {
|
||||
}
|
||||
func (n NullTelemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {}
|
||||
func (n NullTelemetryService) RoomEnded(ctx context.Context, room *livekit.Room, reason livekit.RoomEndReason) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, isWarp bool, guard *ReferenceGuard) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) {
|
||||
}
|
||||
func (n NullTelemetryService) RoomIDChanged(ctx context.Context, prevRoomID livekit.RoomID, room *livekit.Room) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublishRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackUnpublished(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, wasPublishedLocally bool, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribeRequested(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackUnsubscribed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribeFailed(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackMuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackUnmuted(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublishedUpdate(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackMaxSubscribedVideoQuality(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublishRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribeRTPStats(ctx context.Context, room *livekit.Room, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) {
|
||||
}
|
||||
func (n NullTelemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {}
|
||||
func (n NullTelemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {}
|
||||
func (n NullTelemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {}
|
||||
func (n NullTelemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {}
|
||||
func (n NullTelemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {}
|
||||
func (n NullTelemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {}
|
||||
func (n NullTelemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {}
|
||||
func (n NullTelemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) FlushStats() {}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
const (
|
||||
workerCleanupWait = 3 * time.Minute
|
||||
jobsQueueMinSize = 2048
|
||||
|
||||
telemetryStatsUpdateInterval = time.Second * 30
|
||||
)
|
||||
|
||||
type telemetryService struct {
|
||||
AnalyticsService
|
||||
|
||||
notifier webhook.QueuedNotifier
|
||||
jobsQueue *utils.OpsQueue
|
||||
|
||||
workersMu sync.RWMutex
|
||||
workers map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker
|
||||
workerList *StatsWorker
|
||||
|
||||
flushMu sync.Mutex
|
||||
}
|
||||
|
||||
func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService {
|
||||
t := &telemetryService{
|
||||
AnalyticsService: analytics,
|
||||
notifier: notifier,
|
||||
jobsQueue: utils.NewOpsQueue(utils.OpsQueueParams{
|
||||
Name: "telemetry",
|
||||
MinSize: jobsQueueMinSize,
|
||||
FlushOnStop: true,
|
||||
Logger: logger.GetLogger(),
|
||||
}),
|
||||
workers: make(map[livekit.RoomID]map[livekit.ParticipantID]*StatsWorker),
|
||||
}
|
||||
|
||||
t.jobsQueue.Start()
|
||||
go t.run()
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *telemetryService) FlushStats() {
|
||||
t.flushMu.Lock()
|
||||
defer t.flushMu.Unlock()
|
||||
|
||||
t.workersMu.RLock()
|
||||
worker := t.workerList
|
||||
t.workersMu.RUnlock()
|
||||
|
||||
now := time.Now()
|
||||
var prev, reap *StatsWorker
|
||||
for worker != nil {
|
||||
next := worker.next
|
||||
if closed := worker.Flush(now, workerCleanupWait); closed {
|
||||
if prev == nil {
|
||||
// this worker was at the head of the list
|
||||
t.workersMu.Lock()
|
||||
p := &t.workerList
|
||||
for *p != worker {
|
||||
// new workers have been added. scan until we find the one
|
||||
// immediately before this
|
||||
prev = *p
|
||||
p = &prev.next
|
||||
}
|
||||
*p = worker.next
|
||||
t.workersMu.Unlock()
|
||||
} else {
|
||||
prev.next = worker.next
|
||||
}
|
||||
|
||||
worker.next = reap
|
||||
reap = worker
|
||||
} else {
|
||||
prev = worker
|
||||
}
|
||||
worker = next
|
||||
}
|
||||
|
||||
if reap != nil {
|
||||
t.workersMu.Lock()
|
||||
for reap != nil {
|
||||
roomID := reap.RoomID()
|
||||
if roomWorkers := t.workers[roomID]; reap == roomWorkers[reap.participantID] {
|
||||
delete(roomWorkers, reap.participantID)
|
||||
if len(roomWorkers) == 0 {
|
||||
delete(t.workers, roomID)
|
||||
}
|
||||
}
|
||||
reap = reap.next
|
||||
}
|
||||
t.workersMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (t *telemetryService) run() {
|
||||
for range time.Tick(telemetryStatsUpdateInterval) {
|
||||
t.FlushStats()
|
||||
}
|
||||
}
|
||||
|
||||
func (t *telemetryService) enqueue(op func()) {
|
||||
t.jobsQueue.Enqueue(op)
|
||||
}
|
||||
|
||||
func (t *telemetryService) getWorker(roomID livekit.RoomID, participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) {
|
||||
t.workersMu.RLock()
|
||||
defer t.workersMu.RUnlock()
|
||||
|
||||
worker, ok = t.workers[roomID][participantID]
|
||||
return
|
||||
}
|
||||
|
||||
func (t *telemetryService) getOrCreateWorker(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
participantIdentity livekit.ParticipantIdentity,
|
||||
guard *ReferenceGuard,
|
||||
) (*StatsWorker, bool) {
|
||||
t.workersMu.Lock()
|
||||
defer t.workersMu.Unlock()
|
||||
|
||||
if roomID == "" {
|
||||
logger.Warnw(
|
||||
"telemetry stats worker keyed under an empty room id", nil,
|
||||
"room", roomName,
|
||||
"participant", participantIdentity,
|
||||
"participantID", participantID,
|
||||
"guard", guard,
|
||||
)
|
||||
}
|
||||
|
||||
roomWorkers := t.workers[roomID]
|
||||
worker, ok := roomWorkers[participantID]
|
||||
if ok && !worker.Closed(guard) {
|
||||
return worker, true
|
||||
}
|
||||
|
||||
// only ParticipantLeft releases a guard, so a released guard is a call landing after
|
||||
// the participant left, e.g. ParticipantActive overtaken by the close. Do not create
|
||||
// a worker nothing can ever release. The closed worker, if not yet reaped, is returned
|
||||
// as found, otherwise nil is
|
||||
if guard != nil && guard.released {
|
||||
return worker, true
|
||||
}
|
||||
|
||||
existingIsConnected := false
|
||||
if ok {
|
||||
existingIsConnected = worker.IsConnected()
|
||||
}
|
||||
|
||||
// a guard references at most once, so a nil or already activated guard leaves the
|
||||
// new worker with no references and its owner's release drives it negative
|
||||
if guard == nil || guard.activated {
|
||||
logger.Infow(
|
||||
"telemetry stats worker created without a reference",
|
||||
"room", roomName,
|
||||
"roomID", roomID,
|
||||
"participant", participantIdentity,
|
||||
"participantID", participantID,
|
||||
"guard", guard,
|
||||
"replacedClosed", ok,
|
||||
"existing", worker,
|
||||
)
|
||||
}
|
||||
|
||||
worker = newStatsWorker(
|
||||
ctx,
|
||||
t,
|
||||
roomID,
|
||||
roomName,
|
||||
participantID,
|
||||
participantIdentity,
|
||||
guard,
|
||||
)
|
||||
if existingIsConnected {
|
||||
worker.SetConnected()
|
||||
}
|
||||
|
||||
if roomWorkers == nil {
|
||||
roomWorkers = make(map[livekit.ParticipantID]*StatsWorker)
|
||||
t.workers[roomID] = roomWorkers
|
||||
}
|
||||
roomWorkers[participantID] = worker
|
||||
|
||||
worker.next = t.workerList
|
||||
t.workerList = worker
|
||||
|
||||
return worker, false
|
||||
}
|
||||
|
||||
// reKeyRoom files every one of a room's stats workers under `roomID` instead of
|
||||
// `prevRoomID`.
|
||||
//
|
||||
// A room can be restarted while participants are connected and reporting stats, which
|
||||
// gives it a new id. As every worker of the room moves at once, the move is a single map
|
||||
// splice - the workers themselves are untouched and keep their place in the flush list.
|
||||
// Each worker then seals off what it collected under `prevRoomID` so those stats stay
|
||||
// attributed to the session that ended (see StatsWorker.SetRoom).
|
||||
func (t *telemetryService) reKeyRoom(prevRoomID livekit.RoomID, roomID livekit.RoomID, roomName livekit.RoomName) {
|
||||
if prevRoomID == roomID {
|
||||
return
|
||||
}
|
||||
|
||||
t.workersMu.Lock()
|
||||
defer t.workersMu.Unlock()
|
||||
|
||||
roomWorkers := t.workers[prevRoomID]
|
||||
if len(roomWorkers) == 0 {
|
||||
delete(t.workers, prevRoomID)
|
||||
return
|
||||
}
|
||||
delete(t.workers, prevRoomID)
|
||||
|
||||
existing := t.workers[roomID]
|
||||
if existing == nil {
|
||||
t.workers[roomID] = roomWorkers
|
||||
} else {
|
||||
// should not happen as a room id is only ever replaced by a freshly minted one
|
||||
logger.Warnw(
|
||||
"telemetry re-keying room into an existing entry", nil,
|
||||
"prevRoomID", prevRoomID,
|
||||
"room", roomName,
|
||||
"roomID", roomID,
|
||||
"numWorkers", len(roomWorkers),
|
||||
"numExistingWorkers", len(existing),
|
||||
)
|
||||
}
|
||||
|
||||
for participantID, worker := range roomWorkers {
|
||||
if existing != nil {
|
||||
if survivor, ok := existing[participantID]; ok {
|
||||
// only one worker can be keyed at (room, participant) and the one already
|
||||
// filed there wins, close the superseded one so that it drains and is
|
||||
// reaped instead of lingering in the flush list unreachable
|
||||
forceClosed := worker.ForceClose(survivor)
|
||||
logger.Infow(
|
||||
"telemetry force closing superseded stats worker",
|
||||
"prevRoomID", prevRoomID,
|
||||
"roomID", roomID,
|
||||
"participantID", participantID,
|
||||
"forceClosed", forceClosed,
|
||||
"superseded", worker,
|
||||
"survivor", survivor,
|
||||
)
|
||||
if forceClosed {
|
||||
prometheus.SubParticipant()
|
||||
}
|
||||
continue
|
||||
}
|
||||
existing[participantID] = worker
|
||||
}
|
||||
|
||||
worker.SetRoom(roomID, roomName)
|
||||
}
|
||||
|
||||
logger.Infow(
|
||||
"telemetry re-keyed room",
|
||||
"prevRoomID", prevRoomID,
|
||||
"room", roomName,
|
||||
"roomID", roomID,
|
||||
"numWorkers", len(roomWorkers),
|
||||
)
|
||||
}
|
||||
|
||||
func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {
|
||||
t.enqueue(func() {
|
||||
t.SendNodeRoomStates(ctx, info)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user