456 lines
16 KiB
TypeScript
456 lines
16 KiB
TypeScript
import { IVirtualBackgroundAdvancedConfig } from '../../base/config/configType';
|
|
import { VIRTUAL_BACKGROUND_TYPE } from '../../virtual-background/constants';
|
|
import logger from '../../virtual-background/logger';
|
|
import { IVirtualBackground } from '../../virtual-background/reducer';
|
|
|
|
import BackgroundFrameProcessor from './BackgroundFrameProcessor';
|
|
import { IDeviceCapabilities } from './DeviceTierDetector';
|
|
import WorkerSegmentationBackend from './backend/WorkerSegmentationBackend';
|
|
import Canvas2DCompositor from './compositor/Canvas2DCompositor';
|
|
import { ICompositor } from './compositor/ICompositor';
|
|
import WebGLCompositor from './compositor/WebGLCompositor';
|
|
import InsertableStreamsPipeline from './pipeline/InsertableStreamsPipeline';
|
|
import {
|
|
CLEAR_TIMEOUT,
|
|
SET_TIMEOUT,
|
|
TIMEOUT_TICK,
|
|
timerWorkerScript
|
|
} from './workers/TimerWorker';
|
|
|
|
/**
|
|
* Segmentation canvas dimensions for the V1 landscape TFLite model. Fixed by the model's
|
|
* input tensor shape — the V1 path always runs inference on a 256x144 canvas.
|
|
*/
|
|
const V1_SEG_WIDTH = 256;
|
|
const V1_SEG_HEIGHT = 144;
|
|
|
|
/**
|
|
* V2-specific constructor options — bundled so the V1 positional signature stays unchanged.
|
|
* Presence of this object on the constructor is the V1/V2 discriminator.
|
|
*/
|
|
export interface IV2EffectInit {
|
|
capabilities: IDeviceCapabilities;
|
|
enableV2: boolean;
|
|
vbConfig?: IVirtualBackgroundAdvancedConfig;
|
|
}
|
|
|
|
|
|
/**
|
|
* Virtual background stream effect.
|
|
*
|
|
* When {@code enableV2} is false, the class behaves identically to the original V1
|
|
* engine: main-thread TFLite WASM inference, Canvas 2D compositing, TimerWorker frame driver.
|
|
*
|
|
* When {@code enableV2} is true (default), the constructor delegates to the
|
|
* pipeline/backend/compositor abstraction: Worker-based inference, WebGL compositing, and
|
|
* insertable streams (with captureStream fallback). The captureStream fallback reuses the
|
|
* shared video/canvas/timer.
|
|
*/
|
|
export default class JitsiStreamBackgroundEffect {
|
|
_backend: WorkerSegmentationBackend | null = null;
|
|
_enableV2: boolean;
|
|
_inputVideoElement: HTMLVideoElement;
|
|
_maskFrameTimerWorker: Worker | null = null;
|
|
_model: any;
|
|
_options: { height: number; virtualBackground: IVirtualBackground; width: number; };
|
|
_outputCanvasCtx: CanvasRenderingContext2D | null = null;
|
|
_outputCanvasElement: HTMLCanvasElement;
|
|
_pipeline: InsertableStreamsPipeline | null = null;
|
|
_processor: BackgroundFrameProcessor | null = null;
|
|
_segmentationMask: ImageData;
|
|
_segmentationMaskCanvas: HTMLCanvasElement;
|
|
_segmentationMaskCtx: CanvasRenderingContext2D | null;
|
|
_segmentationPixelCount: number;
|
|
_stream: any;
|
|
_virtualImage: HTMLImageElement;
|
|
_virtualVideo: HTMLVideoElement;
|
|
|
|
/**
|
|
* Resolves when V2 async initialization completes. V1 resolves immediately.
|
|
*/
|
|
initPromise: Promise<void> = Promise.resolve();
|
|
|
|
/**
|
|
* Called once when V2 inference fails persistently mid-session.
|
|
*
|
|
* @param {Function} cb - Failure callback, or null to clear.
|
|
*/
|
|
set onInferenceFailure(cb: (() => void) | null) {
|
|
if (this._processor) {
|
|
this._processor.onInferenceFailure = cb;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Creates a new background effect instance.
|
|
*
|
|
* @param {Object} model - Loaded TFLite WASM module (V1) or undefined (V2).
|
|
* @param {IVirtualBackground} virtualBackground - Active virtual background Redux slice
|
|
* (background type, blur value, image source).
|
|
* @param {IV2EffectInit} [v2Init] - V2-specific setup payload. When present, activates
|
|
* the V2 pipeline.
|
|
*/
|
|
constructor(
|
|
model: any,
|
|
virtualBackground: IVirtualBackground,
|
|
v2Init?: IV2EffectInit) {
|
|
// Workaround for FF issue https://bugzilla.mozilla.org/show_bug.cgi?id=1388974
|
|
this._outputCanvasElement = document.createElement('canvas');
|
|
this._outputCanvasElement.getContext('2d');
|
|
this._inputVideoElement = document.createElement('video');
|
|
|
|
this._enableV2 = Boolean(v2Init?.enableV2);
|
|
|
|
if (this._enableV2 && v2Init) {
|
|
const { capabilities, vbConfig } = v2Init;
|
|
|
|
this._backend = new WorkerSegmentationBackend(capabilities);
|
|
|
|
let compositor: ICompositor;
|
|
const webglCanvas = document.createElement('canvas');
|
|
const webglCompositor = new WebGLCompositor(webglCanvas);
|
|
|
|
if (webglCompositor.isAvailable) {
|
|
compositor = webglCompositor;
|
|
} else {
|
|
logger.debug('[VirtualBackground] WebGL unavailable — using Canvas 2D fallback');
|
|
compositor = new Canvas2DCompositor();
|
|
}
|
|
|
|
this._processor = new BackgroundFrameProcessor({
|
|
backend: this._backend,
|
|
compositor,
|
|
vbConfig,
|
|
virtualBackground
|
|
});
|
|
|
|
const useIS = vbConfig?.useInsertableStreams !== false
|
|
&& InsertableStreamsPipeline.isSupported();
|
|
|
|
// V2 with insertable streams uses the dedicated pipeline. Without IS, _pipeline
|
|
// stays null and startEffect falls back to the shared _setupCaptureStream +
|
|
// _startTimerLoop path, still driving _processor (worker + WebGL) every tick.
|
|
if (useIS) {
|
|
this._pipeline = new InsertableStreamsPipeline();
|
|
}
|
|
|
|
logger.debug(
|
|
`[VirtualBackground] V2 effect created — backend: ${this._backend.capabilities.backend}`
|
|
+ `, pipeline: ${useIS ? 'insertable-streams' : 'capture-stream'}`
|
|
+ `, compositor: ${compositor instanceof WebGLCompositor ? 'webgl' : 'canvas-2d'}`
|
|
);
|
|
|
|
return;
|
|
}
|
|
|
|
this._options = {
|
|
height: V1_SEG_HEIGHT,
|
|
virtualBackground,
|
|
width: V1_SEG_WIDTH
|
|
};
|
|
this._model = model;
|
|
this._segmentationPixelCount = V1_SEG_WIDTH * V1_SEG_HEIGHT;
|
|
|
|
if (virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
|
|
this._virtualImage = document.createElement('img');
|
|
this._virtualImage.crossOrigin = 'anonymous';
|
|
this._virtualImage.src = virtualBackground.virtualSource ?? '';
|
|
}
|
|
|
|
// Bind event handler so it is only bound once for every instance.
|
|
this._onMaskFrameTimer = this._onMaskFrameTimer.bind(this);
|
|
}
|
|
|
|
/**
|
|
* Checks if the local track supports this effect.
|
|
*
|
|
* @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect.
|
|
* @returns {boolean} True if this effect can run on this track.
|
|
*/
|
|
isEnabled(jitsiLocalTrack: any) {
|
|
return jitsiLocalTrack.isVideoTrack() && jitsiLocalTrack.videoType === 'camera';
|
|
}
|
|
|
|
/**
|
|
* Starts the effect on the given camera stream.
|
|
*
|
|
* @param {MediaStream} stream - Stream to be used for processing.
|
|
* @returns {MediaStream} The stream with the applied effect.
|
|
*/
|
|
startEffect(stream: MediaStream) {
|
|
if (this._enableV2 && this._pipeline && this._processor) {
|
|
this.initPromise = this._processor.init();
|
|
this.initPromise.catch(() => undefined);
|
|
|
|
return this._pipeline.start(stream, this._processor);
|
|
}
|
|
|
|
const captureStream = this._setupCaptureStream(stream);
|
|
|
|
if (this._enableV2 && this._processor) {
|
|
this.initPromise = this._processor.init();
|
|
this.initPromise.catch(() => undefined);
|
|
|
|
this._startTimerLoop(async () => {
|
|
const result = await this._processor?.processFrame(this._inputVideoElement);
|
|
|
|
if (result && this._outputCanvasCtx) {
|
|
this._outputCanvasCtx.drawImage(result, 0, 0);
|
|
} else if (this._outputCanvasCtx) {
|
|
this._outputCanvasCtx.drawImage(
|
|
this._inputVideoElement, 0, 0,
|
|
this._outputCanvasElement.width, this._outputCanvasElement.height
|
|
);
|
|
}
|
|
});
|
|
} else {
|
|
this._segmentationMask = new ImageData(this._options.width, this._options.height);
|
|
this._segmentationMaskCanvas = document.createElement('canvas');
|
|
this._segmentationMaskCanvas.width = this._options.width;
|
|
this._segmentationMaskCanvas.height = this._options.height;
|
|
this._segmentationMaskCtx = this._segmentationMaskCanvas.getContext('2d');
|
|
|
|
this._startTimerLoop(() => this._renderMask());
|
|
}
|
|
|
|
return captureStream;
|
|
}
|
|
|
|
/**
|
|
* Stops the frame pipeline. Keeps the V2 processor + backend alive so the next
|
|
* startEffect (e.g. on unmute) can process frames immediately without a re-init
|
|
* window in which raw camera frames would be passed through.
|
|
*
|
|
* @returns {void}
|
|
*/
|
|
stopEffect() {
|
|
if (this._enableV2 && this._pipeline) {
|
|
this._pipeline.stop().catch(() => undefined);
|
|
} else {
|
|
this._stopTimerLoop();
|
|
this._inputVideoElement.onloadeddata = null;
|
|
this._inputVideoElement.srcObject = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Pre-initialises the V2 backend. No-op for V1 (model is loaded in the factory).
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async init(): Promise<void> {
|
|
if (this._processor) {
|
|
await this._processor.init();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Sets up the video element and output canvas for captureStream-based frame processing.
|
|
* Used by both V1 and V2 captureStream paths.
|
|
*
|
|
* @private
|
|
* @param {MediaStream} stream - Camera stream.
|
|
* @returns {MediaStream} The captureStream output.
|
|
*/
|
|
_setupCaptureStream(stream: MediaStream): MediaStream {
|
|
this._stream = stream;
|
|
const firstVideoTrack = stream.getVideoTracks()[0];
|
|
const { height, frameRate, width }
|
|
= firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints();
|
|
|
|
this._outputCanvasElement.width = parseInt(String(width), 10);
|
|
this._outputCanvasElement.height = parseInt(String(height), 10);
|
|
this._outputCanvasCtx = this._outputCanvasElement.getContext('2d');
|
|
this._inputVideoElement.width = parseInt(String(width), 10);
|
|
this._inputVideoElement.height = parseInt(String(height), 10);
|
|
this._inputVideoElement.autoplay = true;
|
|
this._inputVideoElement.srcObject = stream;
|
|
|
|
// autoplay is unreliable for out-of-DOM elements on srcObject reassignment.
|
|
this._inputVideoElement.play().catch(() => undefined);
|
|
|
|
return this._outputCanvasElement.captureStream(parseInt(String(frameRate), 10));
|
|
}
|
|
|
|
/**
|
|
* Starts a TimerWorker that calls the given callback on each tick. The next tick is
|
|
* scheduled after the callback completes (sequential). Used by both V1 and V2.
|
|
*
|
|
* @private
|
|
* @param {Function} onTick - Callback to invoke on each timer tick.
|
|
* @returns {void}
|
|
*/
|
|
_startTimerLoop(onTick: () => void | Promise<void>): void {
|
|
this._maskFrameTimerWorker = new Worker(timerWorkerScript, { name: 'VirtualBackground timer' });
|
|
this._maskFrameTimerWorker.onmessage = (response: MessageEvent) => {
|
|
if (response.data.id === TIMEOUT_TICK) {
|
|
Promise.resolve(onTick()).then(() => {
|
|
this._maskFrameTimerWorker?.postMessage({
|
|
id: SET_TIMEOUT,
|
|
timeMs: 1000 / 30
|
|
});
|
|
});
|
|
}
|
|
};
|
|
|
|
if (this._inputVideoElement.readyState >= 2) {
|
|
this._maskFrameTimerWorker.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30 });
|
|
} else {
|
|
this._inputVideoElement.onloadeddata = () => {
|
|
this._maskFrameTimerWorker?.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30 });
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stops and terminates the timer worker.
|
|
*
|
|
* @private
|
|
* @returns {void}
|
|
*/
|
|
_stopTimerLoop(): void {
|
|
if (this._maskFrameTimerWorker) {
|
|
this._maskFrameTimerWorker.postMessage({ id: CLEAR_TIMEOUT });
|
|
this._maskFrameTimerWorker.terminate();
|
|
this._maskFrameTimerWorker = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* EventHandler onmessage for the maskFrameTimerWorker WebWorker.
|
|
*
|
|
* @private
|
|
* @param {EventHandler} response - The onmessage EventHandler parameter.
|
|
* @returns {void}
|
|
*/
|
|
_onMaskFrameTimer(response: { data: { id: number; }; }) {
|
|
if (response.data.id === TIMEOUT_TICK) {
|
|
this._renderMask();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Represents the run post processing.
|
|
*
|
|
* @returns {void}
|
|
*/
|
|
runPostProcessing() {
|
|
|
|
const track = this._stream.getVideoTracks()[0];
|
|
const { height, width } = track.getSettings() ?? track.getConstraints();
|
|
const { backgroundType } = this._options.virtualBackground;
|
|
|
|
if (!this._outputCanvasCtx) {
|
|
return;
|
|
}
|
|
|
|
this._outputCanvasElement.height = height;
|
|
this._outputCanvasElement.width = width;
|
|
this._outputCanvasCtx.globalCompositeOperation = 'copy';
|
|
|
|
// Draw segmentation mask.
|
|
|
|
// Smooth out the edges.
|
|
this._outputCanvasCtx.filter = backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE ? 'blur(4px)' : 'blur(8px)';
|
|
this._outputCanvasCtx?.drawImage( // @ts-ignore
|
|
this._segmentationMaskCanvas,
|
|
0,
|
|
0,
|
|
this._options.width,
|
|
this._options.height,
|
|
0,
|
|
0,
|
|
this._inputVideoElement.width,
|
|
this._inputVideoElement.height
|
|
);
|
|
this._outputCanvasCtx.globalCompositeOperation = 'source-in';
|
|
this._outputCanvasCtx.filter = 'none';
|
|
|
|
// Draw the foreground video.
|
|
// @ts-ignore
|
|
this._outputCanvasCtx?.drawImage(this._inputVideoElement, 0, 0);
|
|
|
|
// Draw the background.
|
|
this._outputCanvasCtx.globalCompositeOperation = 'destination-over';
|
|
if (backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
|
|
this._outputCanvasCtx?.drawImage( // @ts-ignore
|
|
backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE
|
|
? this._virtualImage : this._virtualVideo,
|
|
0,
|
|
0,
|
|
this._outputCanvasElement.width,
|
|
this._outputCanvasElement.height
|
|
);
|
|
} else {
|
|
this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;
|
|
|
|
// @ts-ignore
|
|
this._outputCanvasCtx?.drawImage(this._inputVideoElement, 0, 0);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Represents the run Tensorflow Interference.
|
|
*
|
|
* @returns {void}
|
|
*/
|
|
runInference() {
|
|
this._model._runInference();
|
|
const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
|
|
|
|
for (let i = 0; i < this._segmentationPixelCount; i++) {
|
|
const person = this._model.HEAPF32[outputMemoryOffset + i];
|
|
|
|
// Sets only the alpha component of each pixel.
|
|
this._segmentationMask.data[(i * 4) + 3] = 255 * person;
|
|
|
|
}
|
|
this._segmentationMaskCtx?.putImageData(this._segmentationMask, 0, 0);
|
|
}
|
|
|
|
/**
|
|
* Loop function to render the background mask.
|
|
*
|
|
* @private
|
|
* @returns {void}
|
|
*/
|
|
_renderMask() {
|
|
this.resizeSource();
|
|
this.runInference();
|
|
this.runPostProcessing();
|
|
}
|
|
|
|
/**
|
|
* Represents the resize source process.
|
|
*
|
|
* @returns {void}
|
|
*/
|
|
resizeSource() {
|
|
this._segmentationMaskCtx?.drawImage( // @ts-ignore
|
|
this._inputVideoElement,
|
|
0,
|
|
0,
|
|
this._inputVideoElement.width,
|
|
this._inputVideoElement.height,
|
|
0,
|
|
0,
|
|
this._options.width,
|
|
this._options.height
|
|
);
|
|
|
|
const imageData = this._segmentationMaskCtx?.getImageData(
|
|
0,
|
|
0,
|
|
this._options.width,
|
|
this._options.height
|
|
);
|
|
const inputMemoryOffset = this._model._getInputMemoryOffset() / 4;
|
|
|
|
for (let i = 0; i < this._segmentationPixelCount; i++) {
|
|
this._model.HEAPF32[inputMemoryOffset + (i * 3)] = Number(imageData?.data[i * 4]) / 255;
|
|
this._model.HEAPF32[inputMemoryOffset + (i * 3) + 1] = Number(imageData?.data[(i * 4) + 1]) / 255;
|
|
this._model.HEAPF32[inputMemoryOffset + (i * 3) + 2] = Number(imageData?.data[(i * 4) + 2]) / 255;
|
|
}
|
|
}
|
|
}
|