snapshot of https://github.com/jitsi/jitsi-meet.git @ fd1749dd28f4138aa80234fc7f311b19aaffa0e7 (2026-09-15)
This commit is contained in:
upstream-mirror
@@ -0,0 +1,399 @@
|
||||
import { IVirtualBackgroundAdvancedConfig } from '../../base/config/configType';
|
||||
import { VIRTUAL_BACKGROUND_TYPE } from '../../virtual-background/constants';
|
||||
import logger from '../../virtual-background/logger';
|
||||
import { IVirtualBackground } from '../../virtual-background/reducer';
|
||||
|
||||
import { BackendType } from './DeviceTierDetector';
|
||||
import WorkerSegmentationBackend from './backend/WorkerSegmentationBackend';
|
||||
import { ICompositor } from './compositor/ICompositor';
|
||||
|
||||
/** Benchmark log interval in frames. */
|
||||
const BENCHMARK_LOG_INTERVAL = 60;
|
||||
|
||||
/** Default temporal blend ratio — 75% toward previous blended mask. */
|
||||
const DEFAULT_TEMPORAL_BLEND_RATIO = 0.75;
|
||||
|
||||
/**
|
||||
* Default smoothstep lower threshold for the LOW tier (TFLite selfie_segmentation_landscape).
|
||||
* The landscape model (MediaPipe 2021, 256x144) produces softer confidences than modern models:
|
||||
* hair strands typically sit in [0.10, 0.35], body/face in [0.60, 0.95], and true background
|
||||
* below 0.10. A floor of 0.10 keeps hair in the transition zone without admitting background.
|
||||
*/
|
||||
const DEFAULT_EDGE_LOW_TFLITE = 0.1;
|
||||
|
||||
/**
|
||||
* Default smoothstep upper threshold for the LOW tier. The landscape model rarely saturates
|
||||
* to 1.0 even on clear body pixels, so the upper edge sits at 0.50 — pixels with confidence
|
||||
* above this are fully opaque.
|
||||
*/
|
||||
const DEFAULT_EDGE_HIGH_TFLITE = 0.5;
|
||||
|
||||
/**
|
||||
* Default smoothstep lower threshold for MEDIUM/HIGH tiers (TF.js MediaPipe body-segmentation).
|
||||
* Background pixels near edges are typically below 0.25 and hair/edge pixels fall in 0.28-0.65.
|
||||
*/
|
||||
const DEFAULT_EDGE_LOW_GPU = 0.28;
|
||||
|
||||
/**
|
||||
* Default smoothstep upper threshold for MEDIUM/HIGH tiers. Pixels above this are fully opaque.
|
||||
*/
|
||||
const DEFAULT_EDGE_HIGH_GPU = 0.65;
|
||||
|
||||
/**
|
||||
* Target Gaussian feathering radius expressed in camera-output pixels. The backend returns a mask
|
||||
* at the tier's segmentation resolution. The compositor applies blur in mask-pixel space, so the
|
||||
* radius is scaled proportionally to keep the physical blur width constant.
|
||||
*/
|
||||
const TARGET_BLUR_CAMERA_PX = 20;
|
||||
|
||||
/**
|
||||
* Target feathering width for the LOW tier (overrides TARGET_BLUR_CAMERA_PX when the backend is
|
||||
* TFLite). The 256x144 mask upscales ~5x to a 1280x720 output, so a wider feathering radius
|
||||
* softens the upscale artefacts and helps hide the model's coarse hair boundary.
|
||||
*/
|
||||
const TARGET_BLUR_CAMERA_PX_TFLITE = 32;
|
||||
|
||||
/**
|
||||
* Options for constructing a BackgroundFrameProcessor.
|
||||
*/
|
||||
interface IProcessorOptions {
|
||||
backend: WorkerSegmentationBackend;
|
||||
compositor: ICompositor;
|
||||
vbConfig?: IVirtualBackgroundAdvancedConfig;
|
||||
virtualBackground: IVirtualBackground;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrates per-frame segmentation and compositing.
|
||||
*
|
||||
* Called by the InsertableStreams pipeline (or the captureStream fallback loop in
|
||||
* JitsiStreamBackgroundEffect) for every incoming camera frame. Internally it:
|
||||
* 1. Pre-scales the frame to the backend's segmentation resolution.
|
||||
* 2. Runs inference (with configurable stride/skip for CPU backends).
|
||||
* 3. Applies exponential moving average (EMA) temporal smoothing to the mask.
|
||||
* 4. Prepares the background canvas (blur or image).
|
||||
* 5. Calls the compositor to produce the final output.
|
||||
*
|
||||
* Does NOT own the frame delivery loop or the output stream — that is the pipeline's job.
|
||||
*/
|
||||
/** Number of consecutive inference failures before firing the failure callback. */
|
||||
const FAILURE_THRESHOLD = 30;
|
||||
|
||||
export default class BackgroundFrameProcessor {
|
||||
_backend: WorkerSegmentationBackend;
|
||||
_backgroundCanvas: HTMLCanvasElement;
|
||||
_backgroundCtx: CanvasRenderingContext2D | null = null;
|
||||
_benchmarkAccumTotal = 0;
|
||||
_benchmarkFrameCount = 0;
|
||||
_cachedMaskData: ImageData | null = null;
|
||||
_compositor: ICompositor;
|
||||
_consecutiveFailures = 0;
|
||||
_edgeHigh: number;
|
||||
_edgeLow: number;
|
||||
_firstMaskFrame = true;
|
||||
_frameCount = 0;
|
||||
_inferenceSkipCounter = 0;
|
||||
_isReady = false;
|
||||
_maskAccumF32: Float32Array | null = null;
|
||||
_options: IVirtualBackground;
|
||||
_temporalBlendRatio: number;
|
||||
_vbConfig?: IVirtualBackgroundAdvancedConfig;
|
||||
_virtualImage: HTMLImageElement | null = null;
|
||||
|
||||
/**
|
||||
* Called once when inference fails persistently (FAILURE_THRESHOLD consecutive failures).
|
||||
* The processor switches to permanent passthrough after firing. Set by the effect owner
|
||||
* to dispatch a UI notification and revert Redux state.
|
||||
*/
|
||||
onInferenceFailure: (() => void) | null = null;
|
||||
|
||||
/**
|
||||
* Creates a new background frame processor.
|
||||
*
|
||||
* @param {IProcessorOptions} opts - Construction options.
|
||||
*/
|
||||
constructor(opts: IProcessorOptions) {
|
||||
this._backend = opts.backend;
|
||||
this._compositor = opts.compositor;
|
||||
this._vbConfig = opts.vbConfig;
|
||||
this._options = opts.virtualBackground;
|
||||
|
||||
this._temporalBlendRatio = opts.vbConfig?.temporalBlendRatio
|
||||
?? DEFAULT_TEMPORAL_BLEND_RATIO;
|
||||
|
||||
// Tier-specific edge thresholds — only used by WebGL compositor (V2).
|
||||
// V1 Canvas 2D path ignores these (mask is used as-is for alpha compositing).
|
||||
const isTFLite = opts.backend.capabilities.backend === BackendType.TFLITE;
|
||||
const defaultEdgeLow = isTFLite ? DEFAULT_EDGE_LOW_TFLITE : DEFAULT_EDGE_LOW_GPU;
|
||||
const defaultEdgeHigh = isTFLite ? DEFAULT_EDGE_HIGH_TFLITE : DEFAULT_EDGE_HIGH_GPU;
|
||||
|
||||
this._edgeLow = opts.vbConfig?.edgeLow ?? defaultEdgeLow;
|
||||
this._edgeHigh = opts.vbConfig?.edgeHigh ?? defaultEdgeHigh;
|
||||
|
||||
// Set up virtual image if needed.
|
||||
if (opts.virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
|
||||
this._virtualImage = document.createElement('img');
|
||||
this._virtualImage.crossOrigin = 'anonymous';
|
||||
this._virtualImage.src = opts.virtualBackground.virtualSource ?? '';
|
||||
}
|
||||
|
||||
this._backgroundCanvas = document.createElement('canvas');
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialises the backend (model loading, worker spawn, etc.). Idempotent — safe to call
|
||||
* multiple times (e.g. pre-init in the factory then again in startEffect).
|
||||
*
|
||||
* @returns {Promise<void>} Resolves when ready for inference.
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
if (this._isReady) {
|
||||
return;
|
||||
}
|
||||
await this._backend.init();
|
||||
this._isReady = true;
|
||||
logger.debug('[BackgroundFrameProcessor] Backend init complete');
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes one camera frame through segmentation and compositing.
|
||||
*
|
||||
* @param {CanvasImageSource} source - Current camera frame.
|
||||
* @returns {Promise<HTMLCanvasElement | null>} Compositor output canvas, or null for passthrough.
|
||||
*/
|
||||
async processFrame(source: CanvasImageSource): Promise<HTMLCanvasElement | null> {
|
||||
if (!this._isReady) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const frameStart = performance.now();
|
||||
let maskData: ImageData | null = null;
|
||||
|
||||
// CPU backends (TFLite) skip every Nth frame and reuse the cached mask.
|
||||
// GPU tiers (MEDIUM/HIGH) run inference every frame.
|
||||
const isCpuBackend = this._backend.capabilities.backend === BackendType.TFLITE;
|
||||
const stride = isCpuBackend
|
||||
? (this._vbConfig?.inferenceStride ?? 2)
|
||||
: 1;
|
||||
const runInference = this._inferenceSkipCounter % stride === 0;
|
||||
|
||||
this._inferenceSkipCounter++;
|
||||
|
||||
if (runInference) {
|
||||
try {
|
||||
const { segWidth, segHeight } = this._backend.capabilities;
|
||||
const bitmap = await createImageBitmap(source, {
|
||||
resizeHeight: segHeight,
|
||||
resizeQuality: 'medium',
|
||||
resizeWidth: segWidth
|
||||
});
|
||||
|
||||
const fresh = await this._backend.infer(bitmap);
|
||||
|
||||
if (fresh) {
|
||||
this._cachedMaskData = this._applyMaskEMA(fresh);
|
||||
this._consecutiveFailures = 0;
|
||||
} else {
|
||||
this._consecutiveFailures++;
|
||||
this._checkFailureThreshold();
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('[BackgroundFrameProcessor] Inference error', err);
|
||||
this._consecutiveFailures++;
|
||||
this._checkFailureThreshold();
|
||||
}
|
||||
}
|
||||
maskData = this._cachedMaskData;
|
||||
|
||||
this._frameCount++;
|
||||
|
||||
// Benchmark logging.
|
||||
if (this._vbConfig?.testMode) {
|
||||
this._benchmarkAccumTotal += performance.now() - frameStart;
|
||||
this._benchmarkFrameCount++;
|
||||
if (this._benchmarkFrameCount >= BENCHMARK_LOG_INTERVAL) {
|
||||
this._logBenchmark();
|
||||
}
|
||||
}
|
||||
|
||||
if (!maskData) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Ensure compositor and background canvas match the source dimensions.
|
||||
const srcWidth = (source as HTMLVideoElement).videoWidth
|
||||
?? (source as VideoFrame).displayWidth
|
||||
?? (source as HTMLVideoElement).width;
|
||||
const srcHeight = (source as HTMLVideoElement).videoHeight
|
||||
?? (source as VideoFrame).displayHeight
|
||||
?? (source as HTMLVideoElement).height;
|
||||
|
||||
if (this._backgroundCanvas.width !== srcWidth
|
||||
|| this._backgroundCanvas.height !== srcHeight) {
|
||||
this._backgroundCanvas.width = srcWidth;
|
||||
this._backgroundCanvas.height = srcHeight;
|
||||
this._backgroundCtx = this._backgroundCanvas.getContext('2d');
|
||||
this._compositor.resize(srcWidth, srcHeight);
|
||||
}
|
||||
|
||||
// Prepare background canvas (blur or image).
|
||||
this._prepareBackground(source);
|
||||
|
||||
// Composite. The TFLite tier gets a wider feathering radius because its 256x144 mask
|
||||
// upscales by ~5x to the output and benefits from extra softening at the person boundary.
|
||||
const targetBlurCameraPx = this._backend.capabilities.backend === BackendType.TFLITE
|
||||
? TARGET_BLUR_CAMERA_PX_TFLITE
|
||||
: TARGET_BLUR_CAMERA_PX;
|
||||
const maskBlurRadius = Math.round(targetBlurCameraPx * maskData.width / srcWidth);
|
||||
|
||||
// V1 used blur(8px) for blur backgrounds, blur(4px) for image — match that behavior.
|
||||
const isBlurBg = this._options.backgroundType === VIRTUAL_BACKGROUND_TYPE.BLUR;
|
||||
const cssBlurFilter = isBlurBg ? 'blur(8px)' : 'blur(4px)';
|
||||
|
||||
this._compositor.composite(source, this._backgroundCanvas, maskData, {
|
||||
cssBlurFilter,
|
||||
edgeHigh: this._edgeHigh,
|
||||
edgeLow: this._edgeLow,
|
||||
maskBlurRadius
|
||||
});
|
||||
|
||||
return this._compositor.outputCanvas;
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases all resources held by the processor.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
dispose(): void {
|
||||
this._compositor.dispose();
|
||||
this._cachedMaskData = null;
|
||||
this._consecutiveFailures = 0;
|
||||
this._firstMaskFrame = true;
|
||||
this._maskAccumF32 = null;
|
||||
this._inferenceSkipCounter = 0;
|
||||
this._isReady = false;
|
||||
this._backgroundCtx = null;
|
||||
this.onInferenceFailure = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether consecutive inference failures have exceeded the threshold. When they do,
|
||||
* fires the {@code onInferenceFailure} callback once and switches to permanent passthrough.
|
||||
*
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_checkFailureThreshold(): void {
|
||||
if (this._consecutiveFailures >= FAILURE_THRESHOLD && this.onInferenceFailure) {
|
||||
logger.error(
|
||||
'[BackgroundFrameProcessor] Persistent inference failure'
|
||||
+ ` (${FAILURE_THRESHOLD} consecutive) — switching to passthrough`
|
||||
);
|
||||
this.onInferenceFailure();
|
||||
this.onInferenceFailure = null;
|
||||
this._isReady = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies exponential moving average (EMA) temporal smoothing to the raw segmentation mask.
|
||||
*
|
||||
* Uses a Float32 accumulator to avoid per-step quantization from Uint8ClampedArray rounding.
|
||||
* Modified in-place on the fresh ImageData returned by the backend each call.
|
||||
*
|
||||
* @private
|
||||
* @param {ImageData} maskData - Raw segmentation mask from the backend.
|
||||
* @returns {ImageData} The same ImageData with R and A channels replaced by the EMA value.
|
||||
*/
|
||||
_applyMaskEMA(maskData: ImageData): ImageData {
|
||||
const n = maskData.width * maskData.height;
|
||||
const decay = this._temporalBlendRatio;
|
||||
const newWeight = 1 - decay;
|
||||
|
||||
if (!this._maskAccumF32 || this._maskAccumF32.length !== n || this._firstMaskFrame) {
|
||||
// First frame or resolution change: seed accumulator from the raw mask.
|
||||
this._maskAccumF32 = new Float32Array(n);
|
||||
const raw = maskData.data;
|
||||
|
||||
for (let i = 0, j = 0; i < n; i++, j += 4) {
|
||||
this._maskAccumF32[i] = raw[j]; // R = confidence * 255
|
||||
}
|
||||
this._firstMaskFrame = false;
|
||||
|
||||
return maskData;
|
||||
}
|
||||
|
||||
const raw = maskData.data;
|
||||
const acc = this._maskAccumF32;
|
||||
|
||||
for (let i = 0, j = 0; i < n; i++, j += 4) {
|
||||
acc[i] = newWeight * raw[j] + decay * acc[i];
|
||||
raw[j] = acc[i]; // R — what the compositor shader reads as .r
|
||||
raw[j + 3] = acc[i]; // A — kept in sync for Canvas 2D fallback
|
||||
}
|
||||
|
||||
return maskData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Renders the background (blur or image) onto the background canvas.
|
||||
*
|
||||
* @private
|
||||
* @param {CanvasImageSource} source - Camera frame source.
|
||||
* @returns {void}
|
||||
*/
|
||||
_prepareBackground(source: CanvasImageSource): void {
|
||||
const ctx = this._backgroundCtx;
|
||||
|
||||
if (!ctx) {
|
||||
return;
|
||||
}
|
||||
ctx.filter = 'none';
|
||||
|
||||
if (this._options.backgroundType === VIRTUAL_BACKGROUND_TYPE.BLUR) {
|
||||
ctx.filter = `blur(${this._options.blurValue ?? 8}px)`;
|
||||
ctx.drawImage(source, 0, 0, this._backgroundCanvas.width, this._backgroundCanvas.height);
|
||||
} else if (this._options.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE && this._virtualImage) {
|
||||
ctx.drawImage(this._virtualImage, 0, 0,
|
||||
this._backgroundCanvas.width, this._backgroundCanvas.height);
|
||||
} else {
|
||||
ctx.clearRect(0, 0, this._backgroundCanvas.width, this._backgroundCanvas.height);
|
||||
}
|
||||
|
||||
ctx.filter = 'none';
|
||||
}
|
||||
|
||||
/**
|
||||
* Logs accumulated benchmark data every BENCHMARK_LOG_INTERVAL frames.
|
||||
*
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_logBenchmark(): void {
|
||||
const count = this._benchmarkFrameCount;
|
||||
|
||||
if (count === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const totalMs = this._benchmarkAccumTotal;
|
||||
const avgTotal = (totalMs / count).toFixed(1);
|
||||
const fps = (1000 / (totalMs / count)).toFixed(1);
|
||||
|
||||
// performance.memory is Chrome-only (non-standard).
|
||||
const perfMem = (performance as unknown as { memory?: { usedJSHeapSize: number; }; }).memory;
|
||||
const heapMB = perfMem
|
||||
? (perfMem.usedJSHeapSize / 1024 / 1024).toFixed(1)
|
||||
: 'N/A';
|
||||
|
||||
logger.debug(
|
||||
'[BackgroundFrameProcessor] BENCHMARK'
|
||||
+ ` avgTotal=${avgTotal}ms fps=${fps} heap=${heapMB}MB`
|
||||
+ ` (frames ${this._frameCount - count + 1}-${this._frameCount})`
|
||||
);
|
||||
|
||||
this._benchmarkAccumTotal = 0;
|
||||
this._benchmarkFrameCount = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { IVirtualBackgroundAdvancedConfig } from '../../base/config/configType';
|
||||
import logger from '../../virtual-background/logger';
|
||||
|
||||
export enum BackendType {
|
||||
TFLITE = 'tflite',
|
||||
WEBGL = 'webgl',
|
||||
WEBGPU = 'webgpu'
|
||||
}
|
||||
|
||||
export enum DeviceTier {
|
||||
HIGH = 'high',
|
||||
LOW = 'low',
|
||||
MEDIUM = 'medium'
|
||||
}
|
||||
|
||||
export interface IDeviceCapabilities {
|
||||
backend: BackendType;
|
||||
segHeight: number;
|
||||
segWidth: number;
|
||||
targetFps: number;
|
||||
tier: DeviceTier;
|
||||
}
|
||||
|
||||
interface ITierProfile {
|
||||
backend: BackendType;
|
||||
segHeight: number;
|
||||
segWidth: number;
|
||||
targetFps: number;
|
||||
}
|
||||
|
||||
const TIER_PROFILES: Record<DeviceTier, ITierProfile> = {
|
||||
[DeviceTier.HIGH]: {
|
||||
backend: BackendType.WEBGPU,
|
||||
segHeight: 288,
|
||||
segWidth: 512,
|
||||
targetFps: 30
|
||||
},
|
||||
[DeviceTier.LOW]: {
|
||||
backend: BackendType.TFLITE,
|
||||
segHeight: 144,
|
||||
segWidth: 256,
|
||||
targetFps: 30
|
||||
},
|
||||
[DeviceTier.MEDIUM]: {
|
||||
backend: BackendType.WEBGL,
|
||||
segHeight: 216,
|
||||
segWidth: 384,
|
||||
targetFps: 30
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Probes hardware capabilities (WebGPU, WebGL) and returns the detected tier.
|
||||
* Falls back to LOW when no GPU is available — TFLite WASM always works as the floor.
|
||||
*
|
||||
* @returns {Promise<DeviceTier>} Detected device tier.
|
||||
*/
|
||||
async function probeHardwareTier(): Promise<DeviceTier> {
|
||||
try {
|
||||
if (navigator.gpu) {
|
||||
const adapter = await navigator.gpu.requestAdapter();
|
||||
|
||||
if (adapter) {
|
||||
logger.debug('[VirtualBackground] WebGPU available');
|
||||
|
||||
return DeviceTier.HIGH;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// WebGPU unavailable.
|
||||
}
|
||||
|
||||
const testCanvas = document.createElement('canvas');
|
||||
const gl2 = testCanvas.getContext('webgl2');
|
||||
|
||||
if (gl2) {
|
||||
gl2.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
logger.debug('[VirtualBackground] WebGL2 available');
|
||||
|
||||
return DeviceTier.MEDIUM;
|
||||
}
|
||||
|
||||
const gl1 = testCanvas.getContext('webgl');
|
||||
|
||||
if (gl1) {
|
||||
gl1.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
logger.debug('[VirtualBackground] WebGL1 available');
|
||||
|
||||
return DeviceTier.MEDIUM;
|
||||
}
|
||||
|
||||
logger.debug('[VirtualBackground] No GPU — using TFLite WASM');
|
||||
|
||||
return DeviceTier.LOW;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the device capabilities for virtual background processing.
|
||||
*
|
||||
* Delegates hardware detection to {@link probeHardwareTier} (which tests WebGPU, then
|
||||
* WebGL2/WebGL1, and falls back to LOW/TFLite), then applies config overrides on top of
|
||||
* the detected tier: {@code tierOverride} swaps the entire tier profile, and
|
||||
* {@code segmentationWidth}, {@code segmentationHeight}, and {@code targetFps} override
|
||||
* individual fields.
|
||||
*
|
||||
* @param {IVirtualBackgroundAdvancedConfig} vbConfig - Virtual background advanced configuration.
|
||||
* @returns {Promise<IDeviceCapabilities>}
|
||||
*/
|
||||
export async function detectDeviceTier(
|
||||
vbConfig?: IVirtualBackgroundAdvancedConfig): Promise<IDeviceCapabilities> {
|
||||
const hardwareTier = await probeHardwareTier();
|
||||
|
||||
logger.info(`[VirtualBackground] Hardware tier: ${hardwareTier}`);
|
||||
|
||||
const profile = TIER_PROFILES[hardwareTier];
|
||||
const caps: IDeviceCapabilities = {
|
||||
...profile,
|
||||
tier: hardwareTier
|
||||
};
|
||||
|
||||
// Apply config overrides.
|
||||
if (vbConfig) {
|
||||
if (vbConfig.tierOverride && vbConfig.tierOverride !== caps.tier) {
|
||||
const overrideTier = vbConfig.tierOverride as DeviceTier;
|
||||
const overrideProfile = TIER_PROFILES[overrideTier];
|
||||
|
||||
if (overrideProfile) {
|
||||
caps.tier = overrideTier;
|
||||
caps.backend = overrideProfile.backend;
|
||||
caps.segHeight = overrideProfile.segHeight;
|
||||
caps.segWidth = overrideProfile.segWidth;
|
||||
caps.targetFps = overrideProfile.targetFps;
|
||||
logger.debug(`[VirtualBackground] Config override: tier forced to ${overrideTier}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof vbConfig.segmentationWidth === 'number') {
|
||||
caps.segWidth = vbConfig.segmentationWidth;
|
||||
logger.debug(`[VirtualBackground] Config override: segmentationWidth = ${caps.segWidth}`);
|
||||
}
|
||||
|
||||
if (typeof vbConfig.segmentationHeight === 'number') {
|
||||
caps.segHeight = vbConfig.segmentationHeight;
|
||||
logger.debug(`[VirtualBackground] Config override: segmentationHeight = ${caps.segHeight}`);
|
||||
}
|
||||
|
||||
if (typeof vbConfig.targetFps === 'number') {
|
||||
caps.targetFps = vbConfig.targetFps;
|
||||
logger.debug(`[VirtualBackground] Config override: targetFps = ${caps.targetFps}`);
|
||||
}
|
||||
}
|
||||
|
||||
return caps;
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
import { IVirtualBackgroundAdvancedConfig } from '../../base/config/configType';
|
||||
import { VIRTUAL_BACKGROUND_TYPE } from '../../virtual-background/constants';
|
||||
import logger from '../../virtual-background/logger';
|
||||
import { IVirtualBackground } from '../../virtual-background/reducer';
|
||||
|
||||
import BackgroundFrameProcessor from './BackgroundFrameProcessor';
|
||||
import { IDeviceCapabilities } from './DeviceTierDetector';
|
||||
import WorkerSegmentationBackend from './backend/WorkerSegmentationBackend';
|
||||
import Canvas2DCompositor from './compositor/Canvas2DCompositor';
|
||||
import { ICompositor } from './compositor/ICompositor';
|
||||
import WebGLCompositor from './compositor/WebGLCompositor';
|
||||
import InsertableStreamsPipeline from './pipeline/InsertableStreamsPipeline';
|
||||
import {
|
||||
CLEAR_TIMEOUT,
|
||||
SET_TIMEOUT,
|
||||
TIMEOUT_TICK,
|
||||
timerWorkerScript
|
||||
} from './workers/TimerWorker';
|
||||
|
||||
/**
|
||||
* Segmentation canvas dimensions for the V1 landscape TFLite model. Fixed by the model's
|
||||
* input tensor shape — the V1 path always runs inference on a 256x144 canvas.
|
||||
*/
|
||||
const V1_SEG_WIDTH = 256;
|
||||
const V1_SEG_HEIGHT = 144;
|
||||
|
||||
/**
|
||||
* V2-specific constructor options — bundled so the V1 positional signature stays unchanged.
|
||||
* Presence of this object on the constructor is the V1/V2 discriminator.
|
||||
*/
|
||||
export interface IV2EffectInit {
|
||||
capabilities: IDeviceCapabilities;
|
||||
enableV2: boolean;
|
||||
vbConfig?: IVirtualBackgroundAdvancedConfig;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Virtual background stream effect.
|
||||
*
|
||||
* When {@code enableV2} is false, the class behaves identically to the original V1
|
||||
* engine: main-thread TFLite WASM inference, Canvas 2D compositing, TimerWorker frame driver.
|
||||
*
|
||||
* When {@code enableV2} is true (default), the constructor delegates to the
|
||||
* pipeline/backend/compositor abstraction: Worker-based inference, WebGL compositing, and
|
||||
* insertable streams (with captureStream fallback). The captureStream fallback reuses the
|
||||
* shared video/canvas/timer.
|
||||
*/
|
||||
export default class JitsiStreamBackgroundEffect {
|
||||
_backend: WorkerSegmentationBackend | null = null;
|
||||
_enableV2: boolean;
|
||||
_inputVideoElement: HTMLVideoElement;
|
||||
_maskFrameTimerWorker: Worker | null = null;
|
||||
_model: any;
|
||||
_options: { height: number; virtualBackground: IVirtualBackground; width: number; };
|
||||
_outputCanvasCtx: CanvasRenderingContext2D | null = null;
|
||||
_outputCanvasElement: HTMLCanvasElement;
|
||||
_pipeline: InsertableStreamsPipeline | null = null;
|
||||
_processor: BackgroundFrameProcessor | null = null;
|
||||
_segmentationMask: ImageData;
|
||||
_segmentationMaskCanvas: HTMLCanvasElement;
|
||||
_segmentationMaskCtx: CanvasRenderingContext2D | null;
|
||||
_segmentationPixelCount: number;
|
||||
_stream: any;
|
||||
_virtualImage: HTMLImageElement;
|
||||
_virtualVideo: HTMLVideoElement;
|
||||
|
||||
/**
|
||||
* Resolves when V2 async initialization completes. V1 resolves immediately.
|
||||
*/
|
||||
initPromise: Promise<void> = Promise.resolve();
|
||||
|
||||
/**
|
||||
* Called once when V2 inference fails persistently mid-session.
|
||||
*
|
||||
* @param {Function} cb - Failure callback, or null to clear.
|
||||
*/
|
||||
set onInferenceFailure(cb: (() => void) | null) {
|
||||
if (this._processor) {
|
||||
this._processor.onInferenceFailure = cb;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new background effect instance.
|
||||
*
|
||||
* @param {Object} model - Loaded TFLite WASM module (V1) or undefined (V2).
|
||||
* @param {IVirtualBackground} virtualBackground - Active virtual background Redux slice
|
||||
* (background type, blur value, image source).
|
||||
* @param {IV2EffectInit} [v2Init] - V2-specific setup payload. When present, activates
|
||||
* the V2 pipeline.
|
||||
*/
|
||||
constructor(
|
||||
model: any,
|
||||
virtualBackground: IVirtualBackground,
|
||||
v2Init?: IV2EffectInit) {
|
||||
// Workaround for FF issue https://bugzilla.mozilla.org/show_bug.cgi?id=1388974
|
||||
this._outputCanvasElement = document.createElement('canvas');
|
||||
this._outputCanvasElement.getContext('2d');
|
||||
this._inputVideoElement = document.createElement('video');
|
||||
|
||||
this._enableV2 = Boolean(v2Init?.enableV2);
|
||||
|
||||
if (this._enableV2 && v2Init) {
|
||||
const { capabilities, vbConfig } = v2Init;
|
||||
|
||||
this._backend = new WorkerSegmentationBackend(capabilities);
|
||||
|
||||
let compositor: ICompositor;
|
||||
const webglCanvas = document.createElement('canvas');
|
||||
const webglCompositor = new WebGLCompositor(webglCanvas);
|
||||
|
||||
if (webglCompositor.isAvailable) {
|
||||
compositor = webglCompositor;
|
||||
} else {
|
||||
logger.debug('[VirtualBackground] WebGL unavailable — using Canvas 2D fallback');
|
||||
compositor = new Canvas2DCompositor();
|
||||
}
|
||||
|
||||
this._processor = new BackgroundFrameProcessor({
|
||||
backend: this._backend,
|
||||
compositor,
|
||||
vbConfig,
|
||||
virtualBackground
|
||||
});
|
||||
|
||||
const useIS = vbConfig?.useInsertableStreams !== false
|
||||
&& InsertableStreamsPipeline.isSupported();
|
||||
|
||||
// V2 with insertable streams uses the dedicated pipeline. Without IS, _pipeline
|
||||
// stays null and startEffect falls back to the shared _setupCaptureStream +
|
||||
// _startTimerLoop path, still driving _processor (worker + WebGL) every tick.
|
||||
if (useIS) {
|
||||
this._pipeline = new InsertableStreamsPipeline();
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`[VirtualBackground] V2 effect created — backend: ${this._backend.capabilities.backend}`
|
||||
+ `, pipeline: ${useIS ? 'insertable-streams' : 'capture-stream'}`
|
||||
+ `, compositor: ${compositor instanceof WebGLCompositor ? 'webgl' : 'canvas-2d'}`
|
||||
);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this._options = {
|
||||
height: V1_SEG_HEIGHT,
|
||||
virtualBackground,
|
||||
width: V1_SEG_WIDTH
|
||||
};
|
||||
this._model = model;
|
||||
this._segmentationPixelCount = V1_SEG_WIDTH * V1_SEG_HEIGHT;
|
||||
|
||||
if (virtualBackground.backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
|
||||
this._virtualImage = document.createElement('img');
|
||||
this._virtualImage.crossOrigin = 'anonymous';
|
||||
this._virtualImage.src = virtualBackground.virtualSource ?? '';
|
||||
}
|
||||
|
||||
// Bind event handler so it is only bound once for every instance.
|
||||
this._onMaskFrameTimer = this._onMaskFrameTimer.bind(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the local track supports this effect.
|
||||
*
|
||||
* @param {JitsiLocalTrack} jitsiLocalTrack - Track to apply effect.
|
||||
* @returns {boolean} True if this effect can run on this track.
|
||||
*/
|
||||
isEnabled(jitsiLocalTrack: any) {
|
||||
return jitsiLocalTrack.isVideoTrack() && jitsiLocalTrack.videoType === 'camera';
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the effect on the given camera stream.
|
||||
*
|
||||
* @param {MediaStream} stream - Stream to be used for processing.
|
||||
* @returns {MediaStream} The stream with the applied effect.
|
||||
*/
|
||||
startEffect(stream: MediaStream) {
|
||||
if (this._enableV2 && this._pipeline && this._processor) {
|
||||
this.initPromise = this._processor.init();
|
||||
this.initPromise.catch(() => undefined);
|
||||
|
||||
return this._pipeline.start(stream, this._processor);
|
||||
}
|
||||
|
||||
const captureStream = this._setupCaptureStream(stream);
|
||||
|
||||
if (this._enableV2 && this._processor) {
|
||||
this.initPromise = this._processor.init();
|
||||
this.initPromise.catch(() => undefined);
|
||||
|
||||
this._startTimerLoop(async () => {
|
||||
const result = await this._processor?.processFrame(this._inputVideoElement);
|
||||
|
||||
if (result && this._outputCanvasCtx) {
|
||||
this._outputCanvasCtx.drawImage(result, 0, 0);
|
||||
} else if (this._outputCanvasCtx) {
|
||||
this._outputCanvasCtx.drawImage(
|
||||
this._inputVideoElement, 0, 0,
|
||||
this._outputCanvasElement.width, this._outputCanvasElement.height
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
this._segmentationMask = new ImageData(this._options.width, this._options.height);
|
||||
this._segmentationMaskCanvas = document.createElement('canvas');
|
||||
this._segmentationMaskCanvas.width = this._options.width;
|
||||
this._segmentationMaskCanvas.height = this._options.height;
|
||||
this._segmentationMaskCtx = this._segmentationMaskCanvas.getContext('2d');
|
||||
|
||||
this._startTimerLoop(() => this._renderMask());
|
||||
}
|
||||
|
||||
return captureStream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the frame pipeline. Keeps the V2 processor + backend alive so the next
|
||||
* startEffect (e.g. on unmute) can process frames immediately without a re-init
|
||||
* window in which raw camera frames would be passed through.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
stopEffect() {
|
||||
if (this._enableV2 && this._pipeline) {
|
||||
this._pipeline.stop().catch(() => undefined);
|
||||
} else {
|
||||
this._stopTimerLoop();
|
||||
this._inputVideoElement.onloadeddata = null;
|
||||
this._inputVideoElement.srcObject = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-initialises the V2 backend. No-op for V1 (model is loaded in the factory).
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
if (this._processor) {
|
||||
await this._processor.init();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up the video element and output canvas for captureStream-based frame processing.
|
||||
* Used by both V1 and V2 captureStream paths.
|
||||
*
|
||||
* @private
|
||||
* @param {MediaStream} stream - Camera stream.
|
||||
* @returns {MediaStream} The captureStream output.
|
||||
*/
|
||||
_setupCaptureStream(stream: MediaStream): MediaStream {
|
||||
this._stream = stream;
|
||||
const firstVideoTrack = stream.getVideoTracks()[0];
|
||||
const { height, frameRate, width }
|
||||
= firstVideoTrack.getSettings ? firstVideoTrack.getSettings() : firstVideoTrack.getConstraints();
|
||||
|
||||
this._outputCanvasElement.width = parseInt(String(width), 10);
|
||||
this._outputCanvasElement.height = parseInt(String(height), 10);
|
||||
this._outputCanvasCtx = this._outputCanvasElement.getContext('2d');
|
||||
this._inputVideoElement.width = parseInt(String(width), 10);
|
||||
this._inputVideoElement.height = parseInt(String(height), 10);
|
||||
this._inputVideoElement.autoplay = true;
|
||||
this._inputVideoElement.srcObject = stream;
|
||||
|
||||
// autoplay is unreliable for out-of-DOM elements on srcObject reassignment.
|
||||
this._inputVideoElement.play().catch(() => undefined);
|
||||
|
||||
return this._outputCanvasElement.captureStream(parseInt(String(frameRate), 10));
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts a TimerWorker that calls the given callback on each tick. The next tick is
|
||||
* scheduled after the callback completes (sequential). Used by both V1 and V2.
|
||||
*
|
||||
* @private
|
||||
* @param {Function} onTick - Callback to invoke on each timer tick.
|
||||
* @returns {void}
|
||||
*/
|
||||
_startTimerLoop(onTick: () => void | Promise<void>): void {
|
||||
this._maskFrameTimerWorker = new Worker(timerWorkerScript, { name: 'VirtualBackground timer' });
|
||||
this._maskFrameTimerWorker.onmessage = (response: MessageEvent) => {
|
||||
if (response.data.id === TIMEOUT_TICK) {
|
||||
Promise.resolve(onTick()).then(() => {
|
||||
this._maskFrameTimerWorker?.postMessage({
|
||||
id: SET_TIMEOUT,
|
||||
timeMs: 1000 / 30
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
if (this._inputVideoElement.readyState >= 2) {
|
||||
this._maskFrameTimerWorker.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30 });
|
||||
} else {
|
||||
this._inputVideoElement.onloadeddata = () => {
|
||||
this._maskFrameTimerWorker?.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30 });
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops and terminates the timer worker.
|
||||
*
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_stopTimerLoop(): void {
|
||||
if (this._maskFrameTimerWorker) {
|
||||
this._maskFrameTimerWorker.postMessage({ id: CLEAR_TIMEOUT });
|
||||
this._maskFrameTimerWorker.terminate();
|
||||
this._maskFrameTimerWorker = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* EventHandler onmessage for the maskFrameTimerWorker WebWorker.
|
||||
*
|
||||
* @private
|
||||
* @param {EventHandler} response - The onmessage EventHandler parameter.
|
||||
* @returns {void}
|
||||
*/
|
||||
_onMaskFrameTimer(response: { data: { id: number; }; }) {
|
||||
if (response.data.id === TIMEOUT_TICK) {
|
||||
this._renderMask();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the run post processing.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
runPostProcessing() {
|
||||
|
||||
const track = this._stream.getVideoTracks()[0];
|
||||
const { height, width } = track.getSettings() ?? track.getConstraints();
|
||||
const { backgroundType } = this._options.virtualBackground;
|
||||
|
||||
if (!this._outputCanvasCtx) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._outputCanvasElement.height = height;
|
||||
this._outputCanvasElement.width = width;
|
||||
this._outputCanvasCtx.globalCompositeOperation = 'copy';
|
||||
|
||||
// Draw segmentation mask.
|
||||
|
||||
// Smooth out the edges.
|
||||
this._outputCanvasCtx.filter = backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE ? 'blur(4px)' : 'blur(8px)';
|
||||
this._outputCanvasCtx?.drawImage( // @ts-ignore
|
||||
this._segmentationMaskCanvas,
|
||||
0,
|
||||
0,
|
||||
this._options.width,
|
||||
this._options.height,
|
||||
0,
|
||||
0,
|
||||
this._inputVideoElement.width,
|
||||
this._inputVideoElement.height
|
||||
);
|
||||
this._outputCanvasCtx.globalCompositeOperation = 'source-in';
|
||||
this._outputCanvasCtx.filter = 'none';
|
||||
|
||||
// Draw the foreground video.
|
||||
// @ts-ignore
|
||||
this._outputCanvasCtx?.drawImage(this._inputVideoElement, 0, 0);
|
||||
|
||||
// Draw the background.
|
||||
this._outputCanvasCtx.globalCompositeOperation = 'destination-over';
|
||||
if (backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE) {
|
||||
this._outputCanvasCtx?.drawImage( // @ts-ignore
|
||||
backgroundType === VIRTUAL_BACKGROUND_TYPE.IMAGE
|
||||
? this._virtualImage : this._virtualVideo,
|
||||
0,
|
||||
0,
|
||||
this._outputCanvasElement.width,
|
||||
this._outputCanvasElement.height
|
||||
);
|
||||
} else {
|
||||
this._outputCanvasCtx.filter = `blur(${this._options.virtualBackground.blurValue}px)`;
|
||||
|
||||
// @ts-ignore
|
||||
this._outputCanvasCtx?.drawImage(this._inputVideoElement, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the run Tensorflow Interference.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
runInference() {
|
||||
this._model._runInference();
|
||||
const outputMemoryOffset = this._model._getOutputMemoryOffset() / 4;
|
||||
|
||||
for (let i = 0; i < this._segmentationPixelCount; i++) {
|
||||
const person = this._model.HEAPF32[outputMemoryOffset + i];
|
||||
|
||||
// Sets only the alpha component of each pixel.
|
||||
this._segmentationMask.data[(i * 4) + 3] = 255 * person;
|
||||
|
||||
}
|
||||
this._segmentationMaskCtx?.putImageData(this._segmentationMask, 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loop function to render the background mask.
|
||||
*
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_renderMask() {
|
||||
this.resizeSource();
|
||||
this.runInference();
|
||||
this.runPostProcessing();
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents the resize source process.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
resizeSource() {
|
||||
this._segmentationMaskCtx?.drawImage( // @ts-ignore
|
||||
this._inputVideoElement,
|
||||
0,
|
||||
0,
|
||||
this._inputVideoElement.width,
|
||||
this._inputVideoElement.height,
|
||||
0,
|
||||
0,
|
||||
this._options.width,
|
||||
this._options.height
|
||||
);
|
||||
|
||||
const imageData = this._segmentationMaskCtx?.getImageData(
|
||||
0,
|
||||
0,
|
||||
this._options.width,
|
||||
this._options.height
|
||||
);
|
||||
const inputMemoryOffset = this._model._getInputMemoryOffset() / 4;
|
||||
|
||||
for (let i = 0; i < this._segmentationPixelCount; i++) {
|
||||
this._model.HEAPF32[inputMemoryOffset + (i * 3)] = Number(imageData?.data[i * 4]) / 255;
|
||||
this._model.HEAPF32[inputMemoryOffset + (i * 3) + 1] = Number(imageData?.data[(i * 4) + 1]) / 255;
|
||||
this._model.HEAPF32[inputMemoryOffset + (i * 3) + 2] = Number(imageData?.data[(i * 4) + 2]) / 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
import { getBaseUrl } from '../../../base/util/helpers';
|
||||
import logger from '../../../virtual-background/logger';
|
||||
import { BackendType, IDeviceCapabilities } from '../DeviceTierDetector';
|
||||
|
||||
/**
|
||||
* Worker-based segmentation backend.
|
||||
*
|
||||
* Wraps the {@link VBInferenceWorker} message protocol. The worker runs TF.js body-segmentation
|
||||
* (MEDIUM/HIGH tiers via WebGL/WebGPU) or TFLite WASM (LOW tier via selfie_segmentation_landscape)
|
||||
* in a dedicated Web Worker thread.
|
||||
*
|
||||
* The worker may fall back from a GPU backend to TFLite if the GPU context is unavailable inside
|
||||
* the Worker. When this happens, the capabilities are updated to reflect the actual backend and
|
||||
* segmentation dimensions.
|
||||
*
|
||||
* EMA temporal smoothing is NOT applied here — that is the processor's responsibility. The backend
|
||||
* returns raw mask ImageData from each inference call.
|
||||
*/
|
||||
export default class WorkerSegmentationBackend {
|
||||
_capabilities: IDeviceCapabilities;
|
||||
_pendingInferHandler: ((e: MessageEvent) => void) | null = null;
|
||||
_pendingInferResolve: ((value: ImageData | null) => void) | null = null;
|
||||
_worker: Worker | null = null;
|
||||
_workerReady = false;
|
||||
|
||||
/**
|
||||
* Creates a new worker-based segmentation backend.
|
||||
*
|
||||
* @param {IDeviceCapabilities} capabilities - Detected device capabilities.
|
||||
*/
|
||||
constructor(capabilities: IDeviceCapabilities) {
|
||||
this._capabilities = { ...capabilities };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the device capabilities for this backend.
|
||||
*
|
||||
* @returns {IDeviceCapabilities} The capabilities (may be updated after worker fallback).
|
||||
*/
|
||||
get capabilities(): IDeviceCapabilities {
|
||||
return this._capabilities;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the inference worker and sends the init message.
|
||||
*
|
||||
* The worker is loaded via an importScripts blob URL pattern (same approach as
|
||||
* FaceLandmarksDetector) to load the pre-built worker bundle from /libs/ without a separate
|
||||
* module-worker instantiation.
|
||||
*
|
||||
* Resolves when the worker signals init_done. If the worker falls back to a different backend,
|
||||
* capabilities are updated to reflect the actual backend and segmentation dimensions.
|
||||
*
|
||||
* @returns {Promise<void>} Resolves when the worker is ready for inference.
|
||||
* @throws {Error} When the worker signals init_error.
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
const base = getBaseUrl().replace(/\/?$/, '/');
|
||||
const workerUrl = `${base}libs/vb-inference-worker.min.js`;
|
||||
|
||||
// JSON.stringify escapes quotes, backslashes, and control chars in workerUrl.
|
||||
const scriptSource = `importScripts(${JSON.stringify(workerUrl)});`;
|
||||
|
||||
// @ts-ignore — @types/node's BlobOptions / URL shadow DOM BlobPropertyBag / URL here.
|
||||
const blob = new Blob([ scriptSource ], { type: 'application/javascript' });
|
||||
|
||||
// @ts-ignore
|
||||
const blobUrl = window.URL.createObjectURL(blob);
|
||||
|
||||
this._worker = new Worker(blobUrl, { name: 'VB V2 Inference' });
|
||||
|
||||
// @ts-ignore
|
||||
window.URL.revokeObjectURL(blobUrl);
|
||||
|
||||
const { promise, resolve, reject } = Promise.withResolvers<void>();
|
||||
|
||||
const handler = (e: MessageEvent) => {
|
||||
if (e.data.type === 'init_done') {
|
||||
this._worker?.removeEventListener('message', handler);
|
||||
|
||||
// Worker must report which backend it actually loaded. A missing or unknown
|
||||
// value means the init_done payload is malformed and capabilities cannot be
|
||||
// trusted (the rest of the pipeline keys off _capabilities.backend) — reject
|
||||
// so the failure surfaces instead of silently running on the wrong backend.
|
||||
const reportedBackend = e.data.backend as string;
|
||||
const validBackends: string[] = Object.values(BackendType);
|
||||
|
||||
if (!reportedBackend || !validBackends.includes(reportedBackend)) {
|
||||
reject(new Error(
|
||||
'[WorkerBackend] init_done missing or invalid backend:'
|
||||
+ ` ${reportedBackend ?? '<missing>'}`
|
||||
));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
this._workerReady = true;
|
||||
|
||||
// Worker may have fallen back to TFLite if the GPU backend was unavailable.
|
||||
if (reportedBackend !== this._capabilities.backend) {
|
||||
logger.debug(
|
||||
`[WorkerBackend] Worker fell back from ${this._capabilities.backend}`
|
||||
+ ` to ${reportedBackend}`
|
||||
);
|
||||
this._capabilities = {
|
||||
...this._capabilities,
|
||||
backend: reportedBackend as BackendType,
|
||||
segHeight: e.data.segHeight ?? this._capabilities.segHeight,
|
||||
segWidth: e.data.segWidth ?? this._capabilities.segWidth
|
||||
};
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`[WorkerBackend] Inference worker ready (tier: ${this._capabilities.tier},`
|
||||
+ ` backend: ${this._capabilities.backend})`
|
||||
);
|
||||
resolve();
|
||||
} else if (e.data.type === 'init_error') {
|
||||
this._worker?.removeEventListener('message', handler);
|
||||
reject(new Error(`Inference worker init error: ${e.data.error}`));
|
||||
}
|
||||
};
|
||||
|
||||
this._worker.addEventListener('message', handler);
|
||||
|
||||
const tfliteModelPath = `${base}libs/selfie_segmentation_landscape.tflite`;
|
||||
|
||||
// Without an explicit URL the body-segmentation library fetches from tfhub.dev,
|
||||
// breaking offline/firewalled deployments. Deployed alongside TFLite via Makefile.
|
||||
const tfjsModelUrl = `${base}libs/selfie_segmentation_landscape_tfjs/model.json`;
|
||||
|
||||
logger.debug(
|
||||
`[WorkerBackend] Initialising inference worker — backend: ${this._capabilities.backend}`
|
||||
+ ` | model: ${tfliteModelPath}`
|
||||
+ ` | seg: ${this._capabilities.segWidth}x${this._capabilities.segHeight}`
|
||||
);
|
||||
|
||||
this._worker.postMessage({
|
||||
backend: this._capabilities.backend,
|
||||
segHeight: this._capabilities.segHeight,
|
||||
segWidth: this._capabilities.segWidth,
|
||||
tfjsModelUrl,
|
||||
tfliteModelPath,
|
||||
tfliteWasmBase: `${base}libs/`,
|
||||
type: 'init'
|
||||
});
|
||||
|
||||
const timeout = new Promise<never>((_, rej) => {
|
||||
setTimeout(() => rej(new Error('Inference worker init timed out (30s)')), 30_000);
|
||||
});
|
||||
|
||||
return Promise.race([ promise, timeout ]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a pre-scaled ImageBitmap to the inference worker and awaits the mask.
|
||||
*
|
||||
* The bitmap is transferred (zero-copy). The worker runs segmentation and returns the raw
|
||||
* mask Uint8ClampedArray (also transferred). The mask is wrapped in an ImageData and returned
|
||||
* directly — no EMA smoothing is applied here.
|
||||
*
|
||||
* Only one inference call can be in flight at a time. The caller (processor) is responsible
|
||||
* for enforcing sequential call discipline.
|
||||
*
|
||||
* @param {ImageBitmap} bitmap - Pre-scaled camera frame at seg resolution.
|
||||
* @returns {Promise<ImageData | null>} Raw mask, or null if inference failed.
|
||||
*/
|
||||
infer(bitmap: ImageBitmap): Promise<ImageData | null> {
|
||||
if (!this._worker || !this._workerReady) {
|
||||
bitmap.close();
|
||||
|
||||
return Promise.resolve(null);
|
||||
}
|
||||
|
||||
const { promise, resolve } = Promise.withResolvers<ImageData | null>();
|
||||
|
||||
this._pendingInferResolve = resolve;
|
||||
|
||||
const handler = (e: MessageEvent) => {
|
||||
if (e.data.type === 'mask') {
|
||||
this._worker?.removeEventListener('message', handler);
|
||||
if (this._pendingInferHandler === handler) {
|
||||
this._pendingInferHandler = null;
|
||||
}
|
||||
if (this._pendingInferResolve === resolve) {
|
||||
this._pendingInferResolve = null;
|
||||
}
|
||||
|
||||
resolve(new ImageData(
|
||||
e.data.data as Uint8ClampedArray,
|
||||
e.data.width as number,
|
||||
e.data.height as number
|
||||
));
|
||||
} else if (e.data.type === 'infer_error') {
|
||||
this._worker?.removeEventListener('message', handler);
|
||||
if (this._pendingInferHandler === handler) {
|
||||
this._pendingInferHandler = null;
|
||||
}
|
||||
if (this._pendingInferResolve === resolve) {
|
||||
this._pendingInferResolve = null;
|
||||
}
|
||||
logger.warn('[WorkerBackend] Worker inference error:', e.data.error);
|
||||
resolve(null);
|
||||
}
|
||||
};
|
||||
|
||||
this._pendingInferHandler = handler;
|
||||
this._worker.addEventListener('message', handler);
|
||||
this._worker.postMessage(
|
||||
{ bitmap, type: 'infer' },
|
||||
[ bitmap as unknown as Transferable ]
|
||||
);
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the inference worker and releases all resources.
|
||||
*
|
||||
* Aborts any pending inference call (removes the message handler and resolves with null),
|
||||
* sends a stop message to the worker, and terminates it.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (this._pendingInferHandler && this._worker) {
|
||||
this._worker.removeEventListener('message', this._pendingInferHandler);
|
||||
}
|
||||
this._pendingInferHandler = null;
|
||||
this._pendingInferResolve?.(null);
|
||||
this._pendingInferResolve = null;
|
||||
|
||||
if (this._worker) {
|
||||
this._worker.terminate();
|
||||
this._worker = null;
|
||||
}
|
||||
this._workerReady = false;
|
||||
|
||||
logger.debug('[WorkerBackend] Stopped');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import { ICompositeOptions, ICompositor } from './ICompositor';
|
||||
|
||||
/**
|
||||
* Canvas 2D three-pass compositor.
|
||||
*
|
||||
* Extracted from V1's {@code runPostProcessing} and V2's {@code _compositeFallback}.
|
||||
* Uses {@code globalCompositeOperation} to blend the segmentation mask, camera
|
||||
* foreground, and background into a single output frame on the CPU.
|
||||
*
|
||||
* Falls back gracefully when context creation fails. Callers should check
|
||||
* {@link Canvas2DCompositor#isAvailable} before use.
|
||||
*/
|
||||
export default class Canvas2DCompositor implements ICompositor {
|
||||
_canvas: HTMLCanvasElement;
|
||||
_ctx: CanvasRenderingContext2D | null = null;
|
||||
_maskCanvas: HTMLCanvasElement | null = null;
|
||||
_maskCanvasCtx: CanvasRenderingContext2D | null = null;
|
||||
|
||||
/**
|
||||
* Creates a Canvas2DCompositor with an internal output canvas and mask helper canvas.
|
||||
*/
|
||||
constructor() {
|
||||
this._canvas = document.createElement('canvas');
|
||||
this._ctx = this._canvas.getContext('2d');
|
||||
this._maskCanvas = document.createElement('canvas');
|
||||
this._maskCanvasCtx = this._maskCanvas.getContext('2d');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if a 2D rendering context is active and ready.
|
||||
*
|
||||
* @returns {boolean}
|
||||
*/
|
||||
get isAvailable(): boolean {
|
||||
return this._ctx !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the canvas element that receives the composited output each frame.
|
||||
*
|
||||
* @returns {HTMLCanvasElement}
|
||||
*/
|
||||
get outputCanvas(): HTMLCanvasElement {
|
||||
return this._canvas;
|
||||
}
|
||||
|
||||
/**
|
||||
* Composites one frame using three-pass Canvas 2D blending.
|
||||
*
|
||||
* 1. Draw the blurred segmentation mask (person silhouette).
|
||||
* 2. Clip the camera foreground to the mask via {@code source-in}.
|
||||
* 3. Draw the background behind the person via {@code destination-over}.
|
||||
*
|
||||
* The mask {@code ImageData} is painted onto a persistent helper canvas via
|
||||
* {@code putImageData} which writes raw bytes without premultiplied-alpha
|
||||
* conversion, then scaled to the output dimensions with {@code drawImage}.
|
||||
*
|
||||
* @param {CanvasImageSource} camera - Live camera frame.
|
||||
* @param {CanvasImageSource} background - Pre-rendered background (blur or image).
|
||||
* @param {ImageData} maskData - Raw segmentation mask bytes.
|
||||
* @param {ICompositeOptions} options - Edge thresholds, blur radius, and first-frame flag.
|
||||
* @returns {void}
|
||||
*/
|
||||
composite(
|
||||
camera: CanvasImageSource,
|
||||
background: CanvasImageSource,
|
||||
maskData: ImageData,
|
||||
options: ICompositeOptions): void {
|
||||
const ctx = this._ctx;
|
||||
const maskCanvas = this._maskCanvas;
|
||||
|
||||
if (!ctx || !maskCanvas) {
|
||||
return;
|
||||
}
|
||||
const { width, height } = this._canvas;
|
||||
|
||||
ctx.globalCompositeOperation = 'copy';
|
||||
|
||||
if (maskCanvas.width !== maskData.width
|
||||
|| maskCanvas.height !== maskData.height) {
|
||||
maskCanvas.width = maskData.width;
|
||||
maskCanvas.height = maskData.height;
|
||||
}
|
||||
|
||||
// putImageData writes raw bytes — no premultiplied-alpha distortion.
|
||||
this._maskCanvasCtx?.putImageData(maskData, 0, 0);
|
||||
|
||||
ctx.filter = options.cssBlurFilter;
|
||||
ctx.drawImage(maskCanvas, 0, 0, width, height);
|
||||
|
||||
// Clip foreground to mask.
|
||||
ctx.globalCompositeOperation = 'source-in';
|
||||
ctx.filter = 'none';
|
||||
ctx.drawImage(camera, 0, 0, width, height);
|
||||
|
||||
// Draw background behind the person.
|
||||
ctx.globalCompositeOperation = 'destination-over';
|
||||
ctx.drawImage(background as CanvasImageSource, 0, 0, width, height);
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases all rendering contexts and helper canvases.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
dispose(): void {
|
||||
this._ctx = null;
|
||||
this._maskCanvas = null;
|
||||
this._maskCanvasCtx = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resizes the output canvas to the given dimensions.
|
||||
*
|
||||
* @param {number} width - New width in pixels.
|
||||
* @param {number} height - New height in pixels.
|
||||
* @returns {void}
|
||||
*/
|
||||
resize(width: number, height: number): void {
|
||||
this._canvas.width = width;
|
||||
this._canvas.height = height;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* Per-frame compositing parameters passed to every {@link ICompositor#composite} call.
|
||||
*/
|
||||
export interface ICompositeOptions {
|
||||
|
||||
/** CSS blur filter for the mask in Canvas 2D fallback (e.g. 'blur(4px)'). */
|
||||
cssBlurFilter: string;
|
||||
|
||||
/** Smoothstep upper threshold. Pixels above this are fully opaque. */
|
||||
edgeHigh: number;
|
||||
|
||||
/** Smoothstep lower threshold. Pixels below this are fully transparent. */
|
||||
edgeLow: number;
|
||||
|
||||
/** Gaussian blur radius in mask-texture pixels (0 disables blur). */
|
||||
maskBlurRadius: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Abstraction over the compositing backend that blends camera, background,
|
||||
* and segmentation mask into a single output frame.
|
||||
*
|
||||
* Two implementations exist:
|
||||
* - {@code WebGLCompositor} — GPU-accelerated with temporal smoothing and edge feathering.
|
||||
* - {@code Canvas2DCompositor} — CPU fallback using three-pass Canvas 2D compositing.
|
||||
*/
|
||||
export interface ICompositor {
|
||||
|
||||
/**
|
||||
* Composites one frame: blends camera foreground, background, and mask.
|
||||
*
|
||||
* @param {CanvasImageSource} camera - Live camera frame.
|
||||
* @param {CanvasImageSource} background - Pre-rendered background (blur or image).
|
||||
* @param {ImageData} maskData - Raw segmentation mask bytes.
|
||||
* @param {ICompositeOptions} options - Edge thresholds and blur radius.
|
||||
* @returns {void}
|
||||
*/
|
||||
composite: (
|
||||
camera: CanvasImageSource,
|
||||
background: CanvasImageSource,
|
||||
maskData: ImageData,
|
||||
options: ICompositeOptions) => void;
|
||||
|
||||
/**
|
||||
* Releases all resources held by the compositor.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
dispose: () => void;
|
||||
|
||||
/**
|
||||
* Whether the compositor was successfully initialised and is ready for use.
|
||||
*/
|
||||
readonly isAvailable: boolean;
|
||||
|
||||
/**
|
||||
* The canvas element that receives the composited output each frame.
|
||||
*/
|
||||
readonly outputCanvas: HTMLCanvasElement;
|
||||
|
||||
/**
|
||||
* Resizes the output canvas to the given dimensions.
|
||||
*
|
||||
* @param {number} width - New width in pixels.
|
||||
* @param {number} height - New height in pixels.
|
||||
* @returns {void}
|
||||
*/
|
||||
resize: (width: number, height: number) => void;
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
import logger from '../../../virtual-background/logger';
|
||||
|
||||
import { ICompositeOptions, ICompositor } from './ICompositor';
|
||||
|
||||
const VERTEX_SHADER_SOURCE = `
|
||||
attribute vec2 a_position;
|
||||
attribute vec2 a_texCoord;
|
||||
varying vec2 v_texCoord;
|
||||
void main() {
|
||||
gl_Position = vec4(a_position, 0.0, 1.0);
|
||||
v_texCoord = a_texCoord;
|
||||
}
|
||||
`;
|
||||
|
||||
// GLSL ES 1.0 — compatible with both WebGL 1 and WebGL 2.
|
||||
// Temporal smoothing (EMA) is handled CPU-side by BackgroundFrameProcessor before upload,
|
||||
// so the shader only reads the current mask — no ping-pong / prev-mask sampling needed.
|
||||
const VB_FRAGMENT_SHADER_SOURCE = `
|
||||
precision mediump float;
|
||||
|
||||
uniform sampler2D u_camera;
|
||||
uniform sampler2D u_background;
|
||||
uniform sampler2D u_mask;
|
||||
uniform float u_edgeLow;
|
||||
uniform float u_edgeHigh;
|
||||
uniform vec2 u_maskTexelSize;
|
||||
uniform vec2 u_cameraTexelSize;
|
||||
|
||||
varying vec2 v_texCoord;
|
||||
|
||||
// 3x3 Gaussian blur (sigma≈1) sampled in mask-texture space.
|
||||
// When u_maskTexelSize is (0,0) all taps collapse to tc — weights sum to 1.0, no blur.
|
||||
float sampleMaskBlurred(vec2 tc) {
|
||||
vec2 t = u_maskTexelSize;
|
||||
return
|
||||
texture2D(u_mask, tc + vec2(-t.x, -t.y)).r * 0.0625 +
|
||||
texture2D(u_mask, tc + vec2( 0.0, -t.y)).r * 0.125 +
|
||||
texture2D(u_mask, tc + vec2( t.x, -t.y)).r * 0.0625 +
|
||||
texture2D(u_mask, tc + vec2(-t.x, 0.0)).r * 0.125 +
|
||||
texture2D(u_mask, tc ).r * 0.25 +
|
||||
texture2D(u_mask, tc + vec2( t.x, 0.0)).r * 0.125 +
|
||||
texture2D(u_mask, tc + vec2(-t.x, t.y)).r * 0.0625 +
|
||||
texture2D(u_mask, tc + vec2( 0.0, t.y)).r * 0.125 +
|
||||
texture2D(u_mask, tc + vec2( t.x, t.y)).r * 0.0625;
|
||||
}
|
||||
|
||||
// Luminance gradient magnitude at this pixel using a simple cross-pattern sample.
|
||||
// Returns values in roughly [0, 0.5] — strong camera edges (e.g. shirt vs chair) hit 0.2+.
|
||||
float cameraEdgeStrength(vec2 tc) {
|
||||
vec2 t = u_cameraTexelSize;
|
||||
vec3 lv = vec3(0.299, 0.587, 0.114);
|
||||
float lL = dot(texture2D(u_camera, tc - vec2(t.x, 0.0)).rgb, lv);
|
||||
float lR = dot(texture2D(u_camera, tc + vec2(t.x, 0.0)).rgb, lv);
|
||||
float lU = dot(texture2D(u_camera, tc - vec2(0.0, t.y)).rgb, lv);
|
||||
float lD = dot(texture2D(u_camera, tc + vec2(0.0, t.y)).rgb, lv);
|
||||
return length(vec2(lR - lL, lD - lU));
|
||||
}
|
||||
|
||||
void main() {
|
||||
vec4 fg = texture2D(u_camera, v_texCoord);
|
||||
vec4 bg = texture2D(u_background, v_texCoord);
|
||||
|
||||
float blurred = sampleMaskBlurred(v_texCoord);
|
||||
float raw = texture2D(u_mask, v_texCoord).r;
|
||||
|
||||
// At strong camera color edges (e.g. dark chair vs bright shirt) snap toward the raw mask
|
||||
// so the person boundary aligns with the actual image edge. At smooth regions (hair, skin)
|
||||
// keep the blurred mask for natural feathering.
|
||||
float snapT = smoothstep(0.05, 0.20, cameraEdgeStrength(v_texCoord));
|
||||
float curAlpha = mix(blurred, raw, snapT);
|
||||
float alpha = smoothstep(u_edgeLow, u_edgeHigh, curAlpha);
|
||||
|
||||
gl_FragColor = mix(bg, fg, alpha);
|
||||
}
|
||||
`;
|
||||
|
||||
// Full-screen quad: two triangles covering clip space [-1,1].
|
||||
const QUAD_VERTICES = new Float32Array([
|
||||
-1, -1, 0, 1,
|
||||
1, -1, 1, 1,
|
||||
-1, 1, 0, 0,
|
||||
1, 1, 1, 0
|
||||
]);
|
||||
|
||||
function compileShader(gl: WebGLRenderingContext, type: number, source: string): WebGLShader {
|
||||
const shader = gl.createShader(type);
|
||||
|
||||
if (!shader) {
|
||||
throw new Error('[VirtualBackground] WebGLCompositor: createShader returned null');
|
||||
}
|
||||
gl.shaderSource(shader, source);
|
||||
gl.compileShader(shader);
|
||||
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
||||
const info = gl.getShaderInfoLog(shader);
|
||||
|
||||
gl.deleteShader(shader);
|
||||
throw new Error(`[VirtualBackground] WebGLCompositor: shader compile error — ${info}`);
|
||||
}
|
||||
|
||||
return shader;
|
||||
}
|
||||
|
||||
function createProgram(gl: WebGLRenderingContext, fragmentSource: string): WebGLProgram {
|
||||
const vert = compileShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER_SOURCE);
|
||||
const frag = compileShader(gl, gl.FRAGMENT_SHADER, fragmentSource);
|
||||
const program = gl.createProgram();
|
||||
|
||||
if (!program) {
|
||||
throw new Error('[VirtualBackground] WebGLCompositor: createProgram returned null');
|
||||
}
|
||||
gl.attachShader(program, vert);
|
||||
gl.attachShader(program, frag);
|
||||
gl.linkProgram(program);
|
||||
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
||||
const info = gl.getProgramInfoLog(program);
|
||||
|
||||
gl.deleteProgram(program);
|
||||
throw new Error(`[VirtualBackground] WebGLCompositor: program link error — ${info}`);
|
||||
}
|
||||
gl.deleteShader(vert);
|
||||
gl.deleteShader(frag);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
function createTexture(gl: WebGLRenderingContext): WebGLTexture {
|
||||
const tex = gl.createTexture();
|
||||
|
||||
if (!tex) {
|
||||
throw new Error('[VirtualBackground] WebGLCompositor: createTexture returned null');
|
||||
}
|
||||
gl.bindTexture(gl.TEXTURE_2D, tex);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
||||
|
||||
return tex;
|
||||
}
|
||||
|
||||
/**
|
||||
* WebGL-based compositor that blends camera, background, and segmentation mask
|
||||
* using a fragment shader with temporal smoothing and edge feathering (smoothstep).
|
||||
*
|
||||
* Falls back gracefully when context creation fails — callers should check
|
||||
* {@link WebGLCompositor#isAvailable} before use.
|
||||
*/
|
||||
export default class WebGLCompositor implements ICompositor {
|
||||
_canvas: HTMLCanvasElement;
|
||||
_gl: WebGLRenderingContext | null = null;
|
||||
|
||||
// Bound event handlers — stored so they can be removed in dispose().
|
||||
_onContextLostBound: ((event: Event) => void) | null = null;
|
||||
_onContextRestoredBound: (() => void) | null = null;
|
||||
|
||||
_program: WebGLProgram | null = null;
|
||||
_quadBuffer: WebGLBuffer | null = null;
|
||||
_texBackground: WebGLTexture | null = null;
|
||||
_texCamera: WebGLTexture | null = null;
|
||||
_texMask: WebGLTexture | null = null;
|
||||
|
||||
// Attribute locations — resolved once after program link, reused every frame.
|
||||
// When sharing a GL context with TF.js the vertex attribute state (buffer
|
||||
// binding + vertexAttribPointer) is global and gets overwritten by TF.js
|
||||
// inference. Re-binding before drawArrays restores our state without the
|
||||
// overhead of re-querying locations each frame.
|
||||
_aPosition = -1;
|
||||
_aTexCoord = -1;
|
||||
|
||||
// Uniform locations — VB program, resolved once after program link.
|
||||
_uBackground: WebGLUniformLocation | null = null;
|
||||
_uCamera: WebGLUniformLocation | null = null;
|
||||
_uCameraTexelSize: WebGLUniformLocation | null = null;
|
||||
_uEdgeHigh: WebGLUniformLocation | null = null;
|
||||
_uEdgeLow: WebGLUniformLocation | null = null;
|
||||
_uMask: WebGLUniformLocation | null = null;
|
||||
_uMaskTexelSize: WebGLUniformLocation | null = null;
|
||||
|
||||
/**
|
||||
* Creates a WebGLCompositor targeting the given canvas.
|
||||
*
|
||||
* @param {HTMLCanvasElement} canvas - Output canvas element.
|
||||
*/
|
||||
constructor(canvas: HTMLCanvasElement) {
|
||||
this._canvas = canvas;
|
||||
this._init();
|
||||
|
||||
this._onContextLostBound = this._onContextLost.bind(this);
|
||||
this._onContextRestoredBound = this._onContextRestored.bind(this);
|
||||
canvas.addEventListener('webglcontextlost', this._onContextLostBound, false);
|
||||
canvas.addEventListener('webglcontextrestored', this._onContextRestoredBound, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if a WebGL context is active and ready.
|
||||
*
|
||||
* @returns {boolean}
|
||||
*/
|
||||
get isAvailable(): boolean {
|
||||
return this._gl !== null && !this._gl.isContextLost();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the canvas element that receives the composited output each frame.
|
||||
*
|
||||
* @returns {HTMLCanvasElement}
|
||||
*/
|
||||
get outputCanvas(): HTMLCanvasElement {
|
||||
return this._canvas;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resizes the output canvas to the given dimensions.
|
||||
*
|
||||
* @param {number} width - New width in pixels.
|
||||
* @param {number} height - New height in pixels.
|
||||
* @returns {void}
|
||||
*/
|
||||
resize(width: number, height: number): void {
|
||||
this._canvas.width = width;
|
||||
this._canvas.height = height;
|
||||
}
|
||||
|
||||
/**
|
||||
* Composites one frame via the {@link ICompositor} interface.
|
||||
* Delegates to {@link compositeFromImageData}.
|
||||
*
|
||||
* @param {CanvasImageSource} camera - Live camera frame.
|
||||
* @param {CanvasImageSource} background - Pre-rendered background.
|
||||
* @param {ImageData} maskData - Raw segmentation mask.
|
||||
* @param {ICompositeOptions} options - Edge thresholds and blur radius.
|
||||
* @returns {void}
|
||||
*/
|
||||
composite(
|
||||
camera: CanvasImageSource,
|
||||
background: CanvasImageSource,
|
||||
maskData: ImageData,
|
||||
options: ICompositeOptions): void {
|
||||
this.compositeFromImageData(
|
||||
camera as TexImageSource,
|
||||
background as HTMLCanvasElement,
|
||||
maskData,
|
||||
options.edgeLow,
|
||||
options.edgeHigh,
|
||||
options.maskBlurRadius
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Composites using raw {@code ImageData} for the segmentation mask.
|
||||
*
|
||||
* The mask is uploaded via the {@code Uint8Array} form of {@code texImage2D} rather than
|
||||
* via the {@code CanvasImageSource} form. This is critical when the mask data comes from
|
||||
* {@code Mask.toImageData()} because the {@code CanvasImageSource} upload path reads from
|
||||
* the browser's internal canvas storage which uses <em>premultiplied alpha</em>: for this
|
||||
* model the R and A channels are both equal to the segmentation confidence, so the canvas
|
||||
* stores {@code R = confidence²} instead of {@code confidence}. The raw upload path
|
||||
* bypasses this distortion entirely.
|
||||
*
|
||||
* Temporal smoothing (EMA) is applied CPU-side by {@link BackgroundFrameProcessor} before
|
||||
* upload, so the shader only reads the current mask — no ping-pong required.
|
||||
*
|
||||
* @param {TexImageSource} cameraSource - Live camera frame (HTMLVideoElement, ImageBitmap, VideoFrame, etc.).
|
||||
* @param {HTMLCanvasElement} backgroundSource - Pre-rendered background canvas.
|
||||
* @param {ImageData} maskData - Current-frame segmentation mask (raw bytes).
|
||||
* @param {number} edgeLow - Smoothstep lower threshold.
|
||||
* @param {number} edgeHigh - Smoothstep upper threshold.
|
||||
* @param {number} maskBlurRadius - Gaussian blur radius in mask-texture pixels (0 = no blur).
|
||||
* Scaled from a target camera-pixel feathering width by the processor so the physical
|
||||
* blur width is consistent across tiers (TFLite uses a wider target to hide upscale artefacts).
|
||||
* @returns {void}
|
||||
*/
|
||||
compositeFromImageData(
|
||||
cameraSource: TexImageSource,
|
||||
backgroundSource: HTMLCanvasElement,
|
||||
maskData: ImageData,
|
||||
edgeLow: number,
|
||||
edgeHigh: number,
|
||||
maskBlurRadius = 0
|
||||
): void {
|
||||
const gl = this._gl;
|
||||
|
||||
if (!gl || !this._program) {
|
||||
return;
|
||||
}
|
||||
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
||||
gl.viewport(0, 0, this._canvas.width, this._canvas.height);
|
||||
gl.useProgram(this._program);
|
||||
|
||||
// Camera texture
|
||||
gl.activeTexture(gl.TEXTURE0);
|
||||
gl.bindTexture(gl.TEXTURE_2D, this._texCamera);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, cameraSource);
|
||||
gl.uniform1i(this._uCamera, 0);
|
||||
|
||||
// Background texture
|
||||
gl.activeTexture(gl.TEXTURE1);
|
||||
gl.bindTexture(gl.TEXTURE_2D, this._texBackground);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, backgroundSource);
|
||||
gl.uniform1i(this._uBackground, 1);
|
||||
|
||||
// Current mask — raw Uint8Array upload bypasses browser premultiplied-alpha
|
||||
// conversion that would corrupt the R channel when uploading via canvas.
|
||||
gl.activeTexture(gl.TEXTURE2);
|
||||
gl.bindTexture(gl.TEXTURE_2D, this._texMask);
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, maskData.width, maskData.height,
|
||||
0, gl.RGBA, gl.UNSIGNED_BYTE, maskData.data);
|
||||
gl.uniform1i(this._uMask, 2);
|
||||
|
||||
gl.uniform1f(this._uEdgeLow, edgeLow);
|
||||
gl.uniform1f(this._uEdgeHigh, edgeHigh);
|
||||
gl.uniform2f(this._uMaskTexelSize, maskBlurRadius / maskData.width, maskBlurRadius / maskData.height);
|
||||
gl.uniform2f(this._uCameraTexelSize, 1.0 / this._canvas.width, 1.0 / this._canvas.height);
|
||||
|
||||
this._bindQuadState(gl);
|
||||
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases all WebGL resources.
|
||||
*
|
||||
* @returns {void}
|
||||
*/
|
||||
dispose(): void {
|
||||
// Remove event listeners first so the canvas no longer holds a reference
|
||||
// to this compositor via the bound handler closures.
|
||||
if (this._onContextLostBound) {
|
||||
this._canvas.removeEventListener('webglcontextlost', this._onContextLostBound);
|
||||
this._onContextLostBound = null;
|
||||
}
|
||||
if (this._onContextRestoredBound) {
|
||||
this._canvas.removeEventListener('webglcontextrestored', this._onContextRestoredBound);
|
||||
this._onContextRestoredBound = null;
|
||||
}
|
||||
|
||||
const gl = this._gl;
|
||||
|
||||
if (!gl) {
|
||||
return;
|
||||
}
|
||||
gl.deleteTexture(this._texCamera);
|
||||
gl.deleteTexture(this._texBackground);
|
||||
gl.deleteTexture(this._texMask);
|
||||
gl.deleteBuffer(this._quadBuffer);
|
||||
gl.deleteProgram(this._program);
|
||||
|
||||
this._texCamera = null;
|
||||
this._texBackground = null;
|
||||
this._texMask = null;
|
||||
this._quadBuffer = null;
|
||||
this._program = null;
|
||||
|
||||
// Explicitly release the WebGL context so it no longer counts against the browser's
|
||||
// ~16 concurrent context limit. Without this the context stays alive until GC.
|
||||
gl.getExtension('WEBGL_lose_context')?.loseContext();
|
||||
this._gl = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialises (or re-initialises) the WebGL context, shaders, and textures.
|
||||
*
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_init(): void {
|
||||
const gl = (this._canvas.getContext('webgl2')
|
||||
?? this._canvas.getContext('webgl')) as WebGLRenderingContext | null;
|
||||
|
||||
if (!gl) {
|
||||
logger.warn('[VirtualBackground] WebGLCompositor: no WebGL context available — falling back to Canvas 2D');
|
||||
|
||||
return;
|
||||
}
|
||||
this._gl = gl;
|
||||
|
||||
try {
|
||||
this._program = createProgram(gl, VB_FRAGMENT_SHADER_SOURCE);
|
||||
} catch (err) {
|
||||
logger.error('[VirtualBackground] WebGLCompositor: VB shader build failed', err);
|
||||
this._gl = null;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Vertex buffer — shared position + texCoord interleaved
|
||||
this._quadBuffer = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, this._quadBuffer);
|
||||
gl.bufferData(gl.ARRAY_BUFFER, QUAD_VERTICES, gl.STATIC_DRAW);
|
||||
|
||||
this._aPosition = gl.getAttribLocation(this._program, 'a_position');
|
||||
this._aTexCoord = gl.getAttribLocation(this._program, 'a_texCoord');
|
||||
|
||||
// Initial attribute setup — _bindQuadState() repeats this before every
|
||||
// draw call so TF.js cannot permanently corrupt our attribute state.
|
||||
this._bindQuadState(gl);
|
||||
|
||||
// Resolve VB uniform locations.
|
||||
gl.useProgram(this._program);
|
||||
this._uCamera = gl.getUniformLocation(this._program, 'u_camera');
|
||||
this._uBackground = gl.getUniformLocation(this._program, 'u_background');
|
||||
this._uCameraTexelSize = gl.getUniformLocation(this._program, 'u_cameraTexelSize');
|
||||
this._uMask = gl.getUniformLocation(this._program, 'u_mask');
|
||||
this._uMaskTexelSize = gl.getUniformLocation(this._program, 'u_maskTexelSize');
|
||||
this._uEdgeLow = gl.getUniformLocation(this._program, 'u_edgeLow');
|
||||
this._uEdgeHigh = gl.getUniformLocation(this._program, 'u_edgeHigh');
|
||||
|
||||
// Allocate textures
|
||||
this._texCamera = createTexture(gl);
|
||||
this._texBackground = createTexture(gl);
|
||||
this._texMask = createTexture(gl);
|
||||
|
||||
logger.info('[VirtualBackground] WebGLCompositor: initialised');
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles WebGL context loss.
|
||||
*
|
||||
* @param {Event} event - The webglcontextlost event.
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_onContextLost(event: Event): void {
|
||||
event.preventDefault();
|
||||
logger.warn('[VirtualBackground] WebGLCompositor: context lost');
|
||||
// The browser automatically destroys all GL objects on context loss.
|
||||
// Null the JS references so the GC can collect the wrapper objects.
|
||||
this._gl = null;
|
||||
this._program = null;
|
||||
this._quadBuffer = null;
|
||||
this._texCamera = null;
|
||||
this._texBackground = null;
|
||||
this._texMask = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles WebGL context restoration.
|
||||
*
|
||||
* @private
|
||||
* @returns {void}
|
||||
*/
|
||||
_onContextRestored(): void {
|
||||
logger.info('[VirtualBackground] WebGLCompositor: context restored — reinitialising');
|
||||
this._init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-binds the quad vertex buffer and re-specifies vertex attribute pointers.
|
||||
*
|
||||
* When the GL context is shared with TF.js, TF.js inference overwrites the
|
||||
* global vertex attribute state (bound ARRAY_BUFFER + vertexAttribPointer
|
||||
* descriptors). Calling this before every drawArrays() restores our state
|
||||
* without re-querying attribute locations.
|
||||
*
|
||||
* @private
|
||||
* @param {WebGLRenderingContext} gl - Active GL context.
|
||||
* @returns {void}
|
||||
*/
|
||||
_bindQuadState(gl: WebGLRenderingContext): void {
|
||||
const stride = 4 * Float32Array.BYTES_PER_ELEMENT;
|
||||
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, this._quadBuffer);
|
||||
gl.enableVertexAttribArray(this._aPosition);
|
||||
gl.vertexAttribPointer(this._aPosition, 2, gl.FLOAT, false, stride, 0);
|
||||
gl.enableVertexAttribArray(this._aTexCoord);
|
||||
gl.vertexAttribPointer(this._aTexCoord, 2, gl.FLOAT, false, stride, 2 * Float32Array.BYTES_PER_ELEMENT);
|
||||
}
|
||||
}
|
||||
121
react/features/stream-effects/virtual-background/index.ts
Normal file
121
react/features/stream-effects/virtual-background/index.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
/* eslint-disable lines-around-comment */
|
||||
import { showWarningNotification } from '../../notifications/actions';
|
||||
import { NOTIFICATION_TIMEOUT_TYPE } from '../../notifications/constants';
|
||||
import { timeout } from '../../virtual-background/functions';
|
||||
import logger from '../../virtual-background/logger';
|
||||
import { IVirtualBackground } from '../../virtual-background/reducer';
|
||||
|
||||
import { detectDeviceTier } from './DeviceTierDetector';
|
||||
import JitsiStreamBackgroundEffect from './JitsiStreamBackgroundEffect';
|
||||
// @ts-ignore
|
||||
import createTFLiteModule from './vendor/tflite/tflite';
|
||||
// @ts-ignore
|
||||
import createTFLiteSIMDModule from './vendor/tflite/tflite-simd';
|
||||
/* eslint-enable lines-around-comment */
|
||||
|
||||
const models = {
|
||||
modelLandscape: 'libs/selfie_segmentation_landscape.tflite'
|
||||
};
|
||||
|
||||
let modelBuffer: ArrayBuffer;
|
||||
let tflite: any;
|
||||
let wasmCheck: any;
|
||||
let isWasmDisabled = false;
|
||||
|
||||
/**
|
||||
* Creates a new instance of the virtual background stream effect.
|
||||
*
|
||||
* @param {Object} virtualBackground - The virtual object that contains the background image source and
|
||||
* the isVirtualBackground flag that indicates if virtual image is activated.
|
||||
* @param {Function} dispatch - The Redux dispatch function.
|
||||
* @returns {Promise<JitsiStreamBackgroundEffect | undefined>}
|
||||
*/
|
||||
export async function createVirtualBackgroundEffect(virtualBackground: IVirtualBackground,
|
||||
dispatch?: Function
|
||||
): Promise<JitsiStreamBackgroundEffect | undefined> {
|
||||
if (!MediaStreamTrack.prototype.getSettings && !MediaStreamTrack.prototype.getConstraints) {
|
||||
throw new Error('JitsiStreamBackgroundEffect not supported!');
|
||||
}
|
||||
const fullVbConfig = APP.store.getState()['features/base/config'].virtualBackground;
|
||||
|
||||
// V2 is the default engine. V1 is only used when enableV2 is explicitly set to false.
|
||||
// (The type of enableV2 is `boolean | undefined` per IConfig, so `undefined` picks V2.)
|
||||
const useV2Engine = fullVbConfig?.enableV2 !== false;
|
||||
|
||||
if (useV2Engine) {
|
||||
const vbConfig = fullVbConfig?.advanced;
|
||||
const capabilities = await detectDeviceTier(vbConfig);
|
||||
|
||||
logger.info(
|
||||
'[VirtualBackground] Using V2 engine —'
|
||||
+ ` tier: ${capabilities.tier}, backend: ${capabilities.backend}`
|
||||
);
|
||||
|
||||
const effect = new JitsiStreamBackgroundEffect(undefined, virtualBackground, {
|
||||
capabilities,
|
||||
enableV2: true,
|
||||
vbConfig
|
||||
});
|
||||
|
||||
await effect.init();
|
||||
|
||||
return effect;
|
||||
}
|
||||
|
||||
if (isWasmDisabled) {
|
||||
dispatch?.(showWarningNotification({
|
||||
titleKey: 'virtualBackground.backgroundEffectError'
|
||||
}, NOTIFICATION_TIMEOUT_TYPE.LONG));
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Checks if WebAssembly feature is supported or enabled by/in the browser.
|
||||
// Conditional import of wasm-check package is done to prevent
|
||||
// the browser from crashing when the user opens the app.
|
||||
|
||||
if (!tflite) {
|
||||
try {
|
||||
wasmCheck = require('wasm-check');
|
||||
const tfliteTimeout = 10000;
|
||||
|
||||
if (wasmCheck?.feature?.simd) {
|
||||
tflite = await timeout(tfliteTimeout, createTFLiteSIMDModule());
|
||||
} else {
|
||||
tflite = await timeout(tfliteTimeout, createTFLiteModule());
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.message === '408') {
|
||||
logger.error('Failed to download tflite model!');
|
||||
dispatch?.(showWarningNotification({
|
||||
titleKey: 'virtualBackground.backgroundEffectError'
|
||||
}, NOTIFICATION_TIMEOUT_TYPE.LONG));
|
||||
} else {
|
||||
isWasmDisabled = true;
|
||||
logger.error('Looks like WebAssembly is disabled or not supported on this browser', err);
|
||||
dispatch?.(showWarningNotification({
|
||||
titleKey: 'virtualBackground.webAssemblyWarning',
|
||||
descriptionKey: 'virtualBackground.webAssemblyWarningDescription'
|
||||
}, NOTIFICATION_TIMEOUT_TYPE.LONG));
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!modelBuffer) {
|
||||
const modelResponse = await fetch(models.modelLandscape);
|
||||
|
||||
if (!modelResponse.ok) {
|
||||
throw new Error('Failed to download tflite model!');
|
||||
}
|
||||
|
||||
modelBuffer = await modelResponse.arrayBuffer();
|
||||
|
||||
tflite.HEAPU8.set(new Uint8Array(modelBuffer), tflite._getModelBufferMemoryOffset());
|
||||
|
||||
tflite._loadModel(modelBuffer.byteLength);
|
||||
}
|
||||
|
||||
return new JitsiStreamBackgroundEffect(tflite, virtualBackground);
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
import logger from '../../../virtual-background/logger';
|
||||
import BackgroundFrameProcessor from '../BackgroundFrameProcessor';
|
||||
|
||||
/**
|
||||
* Frame delivery pipeline using the Insertable Streams API.
|
||||
*
|
||||
* Reads {@code VideoFrame} objects from a {@code MediaStreamTrackProcessor}, passes them through
|
||||
* the processor, and writes the results to a {@code MediaStreamTrackGenerator}. No
|
||||
* {@code requestVideoFrameCallback} or keepalive Worker is needed — the readable stream delivers
|
||||
* frames directly from the camera regardless of tab visibility.
|
||||
*/
|
||||
export default class InsertableStreamsPipeline {
|
||||
_isLoopStarted = false;
|
||||
_isRunning = false;
|
||||
_outputCanvasElement: HTMLCanvasElement;
|
||||
_processor: BackgroundFrameProcessor | null = null;
|
||||
_trackGenerator: (MediaStreamTrack & { writable: WritableStream<VideoFrame>; }) | null = null;
|
||||
_trackProcessor: { readable: ReadableStream<VideoFrame>; } | null = null;
|
||||
_trackReader: ReadableStreamDefaultReader<VideoFrame> | null = null;
|
||||
|
||||
/**
|
||||
* Creates a new insertable streams pipeline.
|
||||
*/
|
||||
constructor() {
|
||||
// Output canvas is used as the composited-frame source for creating VideoFrames.
|
||||
// Size is set in start() when the input stream dimensions are known.
|
||||
this._outputCanvasElement = document.createElement('canvas');
|
||||
this._outputCanvasElement.getContext('2d');
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when the browser supports the insertable streams API.
|
||||
*
|
||||
* @returns {boolean}
|
||||
*/
|
||||
static isSupported(): boolean {
|
||||
return 'MediaStreamTrackProcessor' in window
|
||||
&& 'MediaStreamTrackGenerator' in window;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the insertable streams pipeline on the given input stream.
|
||||
*
|
||||
* @param {MediaStream} inputStream - Camera media stream.
|
||||
* @param {BackgroundFrameProcessor} processor - Per-frame processing callback.
|
||||
* @returns {MediaStream} Processed output stream via MediaStreamTrackGenerator.
|
||||
*/
|
||||
start(inputStream: MediaStream, processor: BackgroundFrameProcessor): MediaStream {
|
||||
this._isRunning = true;
|
||||
this._processor = processor;
|
||||
|
||||
const firstVideoTrack = inputStream.getVideoTracks()[0];
|
||||
const { height, width } = firstVideoTrack.getSettings
|
||||
? firstVideoTrack.getSettings()
|
||||
: firstVideoTrack.getConstraints();
|
||||
const videoWidth = parseInt(String(width), 10);
|
||||
const videoHeight = parseInt(String(height), 10);
|
||||
|
||||
// Size the output canvas for the Canvas 2D compositor fallback.
|
||||
this._outputCanvasElement.width = videoWidth;
|
||||
this._outputCanvasElement.height = videoHeight;
|
||||
|
||||
this._trackProcessor = new window.MediaStreamTrackProcessor(
|
||||
{ track: firstVideoTrack, maxBufferSize: 2 }
|
||||
);
|
||||
this._trackGenerator = new window.MediaStreamTrackGenerator({ kind: 'video' });
|
||||
|
||||
// Patch getSettings() on the generator so JitsiLocalTrack's constructor can read
|
||||
// dimensions immediately (before frames flow). MediaStreamTrackGenerator.getSettings()
|
||||
// returns {} until the first frame is written; this causes the constraint-caching code
|
||||
// in JitsiLocalTrack's constructor to crash. We proxy the input track's settings until
|
||||
// the generator has its own, making the IS path safe to pass to the constructor directly.
|
||||
// Any browser that supports IS also supports getSettings().
|
||||
const inputSettings = firstVideoTrack.getSettings();
|
||||
const generator = this._trackGenerator;
|
||||
const originalGetSettings = generator.getSettings.bind(generator);
|
||||
|
||||
generator.getSettings = () => {
|
||||
const settings = originalGetSettings();
|
||||
|
||||
// Once the generator reports its own width, stop proxying.
|
||||
if (settings.width) {
|
||||
return settings;
|
||||
}
|
||||
|
||||
return { ...inputSettings, ...settings };
|
||||
};
|
||||
|
||||
logger.debug('[InsertableStreamsPipeline] Using insertable streams path');
|
||||
|
||||
// Start the async frame loop. The loop processes frames immediately
|
||||
// once the processor signals readiness via processFrame returning non-null.
|
||||
this._runLoop();
|
||||
|
||||
return new MediaStream([ this._trackGenerator as MediaStreamTrack ]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the pipeline and releases all I/O resources.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
this._isRunning = false;
|
||||
this._processor = null;
|
||||
|
||||
// Cancel the reader immediately so any VideoFrames buffered in the processor's
|
||||
// queue are flushed and closed, preventing "VideoFrame GC'd without close()" warnings.
|
||||
this._trackReader?.cancel().catch(() => undefined);
|
||||
this._trackReader = null;
|
||||
this._trackProcessor = null;
|
||||
this._trackGenerator = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drives the frame processing loop via the readable stream from the track processor.
|
||||
*
|
||||
* @private
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async _runLoop(): Promise<void> {
|
||||
if (this._isLoopStarted) {
|
||||
return;
|
||||
}
|
||||
|
||||
const processor = this._trackProcessor;
|
||||
const generator = this._trackGenerator;
|
||||
|
||||
if (!processor || !generator) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._isLoopStarted = true;
|
||||
|
||||
const reader = processor.readable.getReader();
|
||||
|
||||
// Expose the reader so stop() can cancel it immediately, causing the pending
|
||||
// reader.read() to reject and the loop to exit without waiting for the next frame.
|
||||
this._trackReader = reader;
|
||||
const writer = generator.writable.getWriter();
|
||||
|
||||
try {
|
||||
while (this._isRunning) {
|
||||
const { value: frame, done } = await reader.read();
|
||||
|
||||
if (done || !frame) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (!this._processor) {
|
||||
try {
|
||||
await writer.write(frame);
|
||||
} catch (err) {
|
||||
logger.error('[InsertableStreamsPipeline] Write error', err);
|
||||
}
|
||||
frame.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
let resultCanvas: HTMLCanvasElement | null = null;
|
||||
|
||||
try {
|
||||
resultCanvas = await this._processor.processFrame(frame);
|
||||
} catch (err) {
|
||||
logger.error('[InsertableStreamsPipeline] Frame processing error', err);
|
||||
}
|
||||
|
||||
if (resultCanvas) {
|
||||
// Construct the frame outside the try so the catch path doesn't have a
|
||||
// partially-initialised reference, and close it in finally so the frame is
|
||||
// always released even when writer.write rejects.
|
||||
const outFrame = new VideoFrame(
|
||||
resultCanvas, { timestamp: frame.timestamp }
|
||||
);
|
||||
|
||||
try {
|
||||
await writer.write(outFrame);
|
||||
} catch (err) {
|
||||
logger.error('[InsertableStreamsPipeline] Write error', err);
|
||||
} finally {
|
||||
outFrame.close();
|
||||
}
|
||||
} else {
|
||||
// Passthrough — write the raw frame to the generator.
|
||||
try {
|
||||
await writer.write(frame);
|
||||
} catch (err) {
|
||||
logger.error('[InsertableStreamsPipeline] Passthrough write error', err);
|
||||
}
|
||||
}
|
||||
|
||||
frame.close();
|
||||
}
|
||||
} catch (err) {
|
||||
// When stop() cancels the reader, reader.read() rejects — that is expected.
|
||||
if (this._isRunning) {
|
||||
logger.error('[InsertableStreamsPipeline] Loop error', err);
|
||||
}
|
||||
} finally {
|
||||
this._trackReader = null;
|
||||
this._isLoopStarted = false;
|
||||
|
||||
// Cancel reader to flush any VideoFrames still buffered in the processor's queue.
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch { /* ignore — may have already been cancelled by stop() */ }
|
||||
|
||||
writer.releaseLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
24
react/features/stream-effects/virtual-background/vendor/README.md
vendored
Normal file
24
react/features/stream-effects/virtual-background/vendor/README.md
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
# Virtual Background on stream effects
|
||||
|
||||
> From https://google.github.io/mediapipe/solutions/models.html#selfie-segmentation
|
||||
|
||||
#### Canvas 2D + CPU
|
||||
|
||||
This rendering pipeline is pretty much the same as for BodyPix. It relies on Canvas compositing properties to blend rendering layers according to the segmentation mask.
|
||||
|
||||
Interactions with TFLite inference tool are executed on CPU to convert from UInt8 to Float32 for the model input and to apply softmax on the model output.
|
||||
|
||||
The framerate is higher and the quality looks better than BodyPix
|
||||
|
||||
#### SIMD and non-SIMD
|
||||
|
||||
How to test on SIMD:
|
||||
1. Go to chrome://flags/
|
||||
2. Search for SIMD flag
|
||||
3. Enable WebAssembly SIMD support(Enables support for the WebAssembly SIMD proposal).
|
||||
4. Reopen Google Chrome
|
||||
|
||||
More details:
|
||||
- [WebAssembly](https://webassembly.org/)
|
||||
- [WebAssembly SIMD](https://github.com/WebAssembly/simd)
|
||||
- [TFLite](https://blog.tensorflow.org/2020/07/accelerating-tensorflow-lite-xnnpack-integration.html)
|
||||
BIN
react/features/stream-effects/virtual-background/vendor/models/selfie_segmentation_landscape.tflite
vendored
Executable file
BIN
react/features/stream-effects/virtual-background/vendor/models/selfie_segmentation_landscape.tflite
vendored
Executable file
Binary file not shown.
Binary file not shown.
File diff suppressed because one or more lines are too long
17
react/features/stream-effects/virtual-background/vendor/tflite/tflite-simd.js
vendored
Normal file
17
react/features/stream-effects/virtual-background/vendor/tflite/tflite-simd.js
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
react/features/stream-effects/virtual-background/vendor/tflite/tflite-simd.wasm
vendored
Executable file
BIN
react/features/stream-effects/virtual-background/vendor/tflite/tflite-simd.wasm
vendored
Executable file
Binary file not shown.
17
react/features/stream-effects/virtual-background/vendor/tflite/tflite.js
vendored
Normal file
17
react/features/stream-effects/virtual-background/vendor/tflite/tflite.js
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
react/features/stream-effects/virtual-background/vendor/tflite/tflite.wasm
vendored
Executable file
BIN
react/features/stream-effects/virtual-background/vendor/tflite/tflite.wasm
vendored
Executable file
Binary file not shown.
@@ -0,0 +1,67 @@
|
||||
|
||||
/**
|
||||
* SET_TIMEOUT constant is used to set interval and it is set in
|
||||
* the id property of the request.data property. TimeMs property must
|
||||
* also be set.
|
||||
*
|
||||
* ```
|
||||
* //Request.data example:
|
||||
* {
|
||||
* id: SET_TIMEOUT,
|
||||
* timeMs: 33
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export const SET_TIMEOUT = 1;
|
||||
|
||||
/**
|
||||
* CLEAR_TIMEOUT constant is used to clear the interval and it is set in
|
||||
* the id property of the request.data property.
|
||||
*
|
||||
* ```
|
||||
* {
|
||||
* id: CLEAR_TIMEOUT
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export const CLEAR_TIMEOUT = 2;
|
||||
|
||||
/**
|
||||
* TIMEOUT_TICK constant is used as response and it is set in the id property.
|
||||
*
|
||||
* ```
|
||||
* {
|
||||
* id: TIMEOUT_TICK
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export const TIMEOUT_TICK = 3;
|
||||
|
||||
/**
|
||||
* The following code is needed as string to create a URL from a Blob.
|
||||
* The URL is then passed to a WebWorker. Reason for this is to enable
|
||||
* use of setInterval that is not throttled when tab is inactive.
|
||||
*/
|
||||
const code = `
|
||||
var timer;
|
||||
|
||||
onmessage = function(request) {
|
||||
switch (request.data.id) {
|
||||
case ${SET_TIMEOUT}: {
|
||||
timer = setTimeout(() => {
|
||||
postMessage({ id: ${TIMEOUT_TICK} });
|
||||
}, request.data.timeMs);
|
||||
break;
|
||||
}
|
||||
case ${CLEAR_TIMEOUT}: {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
`;
|
||||
|
||||
// @ts-ignore
|
||||
export const timerWorkerScript = URL.createObjectURL(new Blob([ code ], { type: 'application/javascript' }));
|
||||
@@ -0,0 +1,470 @@
|
||||
/**
|
||||
* Virtual Background V2 — inference worker.
|
||||
*
|
||||
* Handles TF.js body-segmentation (MEDIUM/HIGH tiers via WebGL/WebGPU) and TFLite WASM
|
||||
* (LOW tier via selfie_segmentation_landscape) in a single dedicated Web Worker. Running
|
||||
* all inference in a Worker keeps the main thread free for UI events. For GPU tiers, the
|
||||
* Worker's OffscreenCanvas-backed WebGL context is not subject to Chrome's tab-visibility
|
||||
* GPU scheduling throttle.
|
||||
*/
|
||||
/*
|
||||
* Message protocol
|
||||
*
|
||||
* Main -> Worker:
|
||||
* { type: 'init', backend: string, segWidth: number, segHeight: number,
|
||||
* tfjsModelUrl: string, tfliteModelPath: string, tfliteWasmBase: string }
|
||||
* { type: 'infer', bitmap: ImageBitmap } (bitmap is transferred -- zero-copy)
|
||||
*
|
||||
* Worker -> Main:
|
||||
* { type: 'init_done', backend: string, segHeight: number, segWidth: number }
|
||||
* backend/segHeight/segWidth reflect the actual tier used (may differ from requested
|
||||
* backend when the Worker fell back to TFLite because the GPU backend was unavailable)
|
||||
* { type: 'init_error', error: string }
|
||||
* { type: 'mask', data: Uint8ClampedArray, width: number, height: number }
|
||||
* (data.buffer is transferred -- zero-copy)
|
||||
* { type: 'infer_error', error: string }
|
||||
*/
|
||||
|
||||
import '@tensorflow/tfjs-backend-webgl';
|
||||
import '@tensorflow/tfjs-backend-webgpu';
|
||||
import * as tf from '@tensorflow/tfjs-core';
|
||||
import type { BodySegmenter, MediaPipeSelfieSegmentationTfjsModelConfig }
|
||||
from '@tensorflow-models/body-segmentation';
|
||||
import * as bs from '@tensorflow-models/body-segmentation';
|
||||
|
||||
import { BackendType } from '../DeviceTierDetector';
|
||||
// @ts-ignore
|
||||
import createTFLiteSIMDModule from '../vendor/tflite/tflite-simd';
|
||||
|
||||
/** Maximum segmentation dimension in pixels. Guards against OOM via crafted init messages. */
|
||||
const MAX_SEG_DIMENSION = 2048;
|
||||
|
||||
/**
|
||||
* Extracts the effective origin of this Worker script.
|
||||
*
|
||||
* Workers loaded via a blob URL have self.location.href of the form
|
||||
* "blob:https://hostname/uuid". Standard URL parsing returns origin="null" for blob URLs,
|
||||
* so we strip the "blob:" prefix and parse the embedded URL to recover the real origin.
|
||||
*
|
||||
* @returns {string} The origin (e.g. "https://meet.example.com"), or empty string if unparseable.
|
||||
*/
|
||||
function getWorkerOrigin(): string {
|
||||
try {
|
||||
const { href } = self.location;
|
||||
|
||||
return new URL(href.startsWith('blob:') ? href.slice(5) : href).origin;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when {@code url} is a valid http/https URL at the same origin as this Worker.
|
||||
* Rejects data URIs, javascript: URLs, file: URLs, and cross-origin paths.
|
||||
*
|
||||
* @param {string} url - URL to validate.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function isSameOriginUrl(url: string): boolean {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
|
||||
return (parsed.protocol === 'https:' || parsed.protocol === 'http:')
|
||||
&& parsed.origin === getWorkerOrigin();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Active TF.js segmenter (MEDIUM/HIGH tiers). Null when using TFLite backend. */
|
||||
let segmenter: BodySegmenter | null = null;
|
||||
|
||||
/** Active TFLite module (LOW tier, TFLITE backend). Null when using TF.js. */
|
||||
let tfliteModule: any = null;
|
||||
|
||||
/** TFLite input HEAPF32 offset (byte offset / 4). Pre-computed after model load. */
|
||||
let tfliteInputOffset = 0;
|
||||
|
||||
/** TFLite output HEAPF32 offset (byte offset / 4). Pre-computed after model load. */
|
||||
let tfliteOutputOffset = 0;
|
||||
|
||||
/** TFLite segmentation pixel count (segWidth x segHeight). */
|
||||
let tflitePixelCount = 0;
|
||||
|
||||
/** TFLite segmentation canvas width. */
|
||||
let tfliteSegWidth = 0;
|
||||
|
||||
/** TFLite segmentation canvas height. */
|
||||
let tfliteSegHeight = 0;
|
||||
|
||||
/** OffscreenCanvas for downscaling frames before TFLite inference. */
|
||||
let tfliteReadCanvas: any = null;
|
||||
|
||||
/** 2D context for tfliteReadCanvas. */
|
||||
let tfliteReadCtx: any = null;
|
||||
|
||||
/** Active backend type. TFLITE selects TFLite WASM; WEBGL/WEBGPU selects TF.js. */
|
||||
let currentBackend: BackendType | '' = '';
|
||||
|
||||
/**
|
||||
* Type-safe postMessage wrapper for DedicatedWorkerGlobalScope.
|
||||
* TypeScript types the global 'self' as Window (lib.dom) which has a different postMessage
|
||||
* signature to DedicatedWorkerGlobalScope. Casting through 'any' bypasses the mismatch.
|
||||
*
|
||||
* @param {any} msg - Structured-cloneable message payload.
|
||||
* @param {Transferable[]} [transfer] - Optional list of transferable objects (zero-copy).
|
||||
* @returns {void}
|
||||
*/
|
||||
function workerPost(msg: any, transfer?: Transferable[]): void {
|
||||
if (transfer?.length) {
|
||||
(self as any).postMessage(msg, transfer);
|
||||
} else {
|
||||
(self as any).postMessage(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// onmessage at module scope — runs in DedicatedWorkerGlobalScope after importScripts loads this bundle.
|
||||
// TypeScript types 'onmessage' as Window.onmessage, but the runtime target is the worker global.
|
||||
(self as any).onmessage = async (e: MessageEvent) => {
|
||||
const { type } = e.data;
|
||||
|
||||
if (type === 'init') {
|
||||
await handleInit(e.data);
|
||||
} else if (type === 'infer') {
|
||||
await handleInfer(e.data.bitmap as ImageBitmap);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Initialises the worker backend based on the tier signalled by the main thread.
|
||||
*
|
||||
* Routes to the TFLite handler for LOW tier (backend 'tflite') or the TF.js handler for
|
||||
* MEDIUM/HIGH tiers (backend 'webgl' or 'webgpu'). The TFLite model path and WASM base are
|
||||
* always passed so the TF.js handler can fall back to TFLite if GPU init fails in the Worker.
|
||||
*
|
||||
* @param {Object} data - Init message payload.
|
||||
* @param {string} data.backend - 'tflite', 'webgl', or 'webgpu'.
|
||||
* @param {number} data.segHeight - Segmentation canvas height.
|
||||
* @param {number} data.segWidth - Segmentation canvas width.
|
||||
* @param {string} data.tfjsModelUrl - Absolute URL of the self-hosted TF.js graph model (model.json).
|
||||
* @param {string} data.tfliteModelPath - Absolute URL of the TFLite segmentation model.
|
||||
* @param {string} data.tfliteWasmBase - Base URL for tflite*.wasm binaries (trailing slash).
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleInit(data: {
|
||||
backend: BackendType;
|
||||
segHeight: number;
|
||||
segWidth: number;
|
||||
tfjsModelUrl: string;
|
||||
tfliteModelPath: string;
|
||||
tfliteWasmBase: string;
|
||||
}): Promise<void> {
|
||||
// Whitelist backend — rejects unknown values before they reach tf.setBackend() or TFLite.
|
||||
if (data.backend !== BackendType.TFLITE
|
||||
&& data.backend !== BackendType.WEBGL
|
||||
&& data.backend !== BackendType.WEBGPU) {
|
||||
workerPost({ error: `Unknown backend: "${data.backend}"`, type: 'init_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Bounds-check dimensions — prevents OOM from crafted init messages.
|
||||
if (!Number.isInteger(data.segWidth) || data.segWidth < 1 || data.segWidth > MAX_SEG_DIMENSION
|
||||
|| !Number.isInteger(data.segHeight) || data.segHeight < 1
|
||||
|| data.segHeight > MAX_SEG_DIMENSION) {
|
||||
workerPost({ error: 'Invalid segmentation dimensions in init message', type: 'init_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
currentBackend = data.backend;
|
||||
|
||||
if (data.backend === BackendType.TFLITE) {
|
||||
await handleInitTflite(data);
|
||||
} else {
|
||||
await handleInitTfjs(data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialises TF.js with the tier-appropriate backend and creates the body-segmentation model.
|
||||
* In a Worker, TF.js automatically uses OffscreenCanvas for its WebGL context (document is
|
||||
* unavailable). Chrome does not apply tab-visibility GPU throttling to Worker GPU contexts.
|
||||
*
|
||||
* The segmentation model is loaded from the deployment's own libs/ directory. Passing an
|
||||
* explicit modelUrl is required: without one, the body-segmentation library fetches the model
|
||||
* from tfhub.dev, which fails in offline/firewalled deployments — and when the network hangs
|
||||
* instead of erroring, the fetch outlives the main thread's init timeout so the TFLite
|
||||
* fallback below never gets a chance to run.
|
||||
*
|
||||
* If the requested GPU backend is unavailable (no GPU, OffscreenCanvas WebGL unsupported, etc.),
|
||||
* falls back to TFLite WASM using the TFLite model path supplied in the init message. This covers
|
||||
* servers and VMs where the main-thread tier detector passes GPU checks (software rasteriser) but
|
||||
* the Worker's OffscreenCanvas context creation fails.
|
||||
*
|
||||
* @param {Object} data - Init message payload.
|
||||
* @param {string} data.backend - 'webgl' (MEDIUM tier) or 'webgpu' (HIGH tier).
|
||||
* @param {number} data.segWidth - Unused here; inference at bitmap resolution.
|
||||
* @param {number} data.segHeight - Unused here; inference at bitmap resolution.
|
||||
* @param {string} data.tfjsModelUrl - Absolute URL of the self-hosted TF.js graph model (model.json).
|
||||
* @param {string} data.tfliteModelPath - TFLite model URL used if GPU fallback is triggered.
|
||||
* @param {string} data.tfliteWasmBase - TFLite WASM base URL used if GPU fallback is triggered.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleInitTfjs(data: {
|
||||
backend: string;
|
||||
segHeight: number;
|
||||
segWidth: number;
|
||||
tfjsModelUrl: string;
|
||||
tfliteModelPath: string;
|
||||
tfliteWasmBase: string;
|
||||
}): Promise<void> {
|
||||
const { backend, tfjsModelUrl, tfliteModelPath, tfliteWasmBase } = data;
|
||||
|
||||
if (!isSameOriginUrl(tfjsModelUrl)) {
|
||||
workerPost({ error: 'TF.js model URL must be same-origin', type: 'init_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const backendSet = await tf.setBackend(backend);
|
||||
|
||||
await tf.ready();
|
||||
|
||||
if (!backendSet || tf.getBackend() !== backend) {
|
||||
// GPU backend unavailable in this Worker context — fall back to TFLite.
|
||||
currentBackend = BackendType.TFLITE;
|
||||
await handleInitTflite({ segHeight: 144, segWidth: 256, tfliteModelPath, tfliteWasmBase });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
segmenter = await bs.createSegmenter(
|
||||
bs.SupportedModels.MediaPipeSelfieSegmentation,
|
||||
{
|
||||
modelType: 'landscape',
|
||||
modelUrl: tfjsModelUrl,
|
||||
runtime: 'tfjs'
|
||||
} as MediaPipeSelfieSegmentationTfjsModelConfig
|
||||
);
|
||||
|
||||
workerPost({ backend: currentBackend, segHeight: data.segHeight, segWidth: data.segWidth, type: 'init_done' });
|
||||
} catch (err) {
|
||||
// GPU init threw (e.g. OffscreenCanvas WebGL context creation failed) — fall back to TFLite.
|
||||
currentBackend = BackendType.TFLITE;
|
||||
try {
|
||||
await handleInitTflite({ segHeight: 144, segWidth: 256, tfliteModelPath, tfliteWasmBase });
|
||||
} catch (tfliteErr) {
|
||||
workerPost({ error: String(tfliteErr), type: 'init_error' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialises the TFLite WASM backend for LOW tier using the selfie_segmentation_landscape model.
|
||||
*
|
||||
* Loads the SIMD WASM runtime via createTFLiteSIMDModule with a locateFile override so the
|
||||
* .wasm binary is resolved from the deployment libs/ directory rather than from the (unusable)
|
||||
* blob: worker origin. Fetches the .tflite model, writes it to the TFLite HEAPU8 buffer, and
|
||||
* calls _loadModel(). Pre-computes HEAPF32 input/output offsets and creates an OffscreenCanvas
|
||||
* for pixel readback.
|
||||
*
|
||||
* @param {Object} data - Init message payload for the TFLITE tier.
|
||||
* @param {string} data.modelPath - Absolute URL of the TFLite segmentation model.
|
||||
* @param {number} data.segHeight - Segmentation canvas height.
|
||||
* @param {number} data.segWidth - Segmentation canvas width.
|
||||
* @param {string} data.tfliteWasmBase - Base URL for tflite*.wasm binaries (trailing slash).
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleInitTflite(data: {
|
||||
segHeight: number;
|
||||
segWidth: number;
|
||||
tfliteModelPath: string;
|
||||
tfliteWasmBase: string;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
const { segHeight, segWidth, tfliteModelPath, tfliteWasmBase } = data;
|
||||
|
||||
if (!isSameOriginUrl(tfliteModelPath) || !isSameOriginUrl(`${tfliteWasmBase}tflite.wasm`)) {
|
||||
workerPost({ error: 'TFLite model or WASM path must be same-origin', type: 'init_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// locateFile overrides WASM path resolution — required because the worker runs from a
|
||||
// blob: URL whose origin is "null", making relative path resolution fail.
|
||||
tfliteModule = await createTFLiteSIMDModule({
|
||||
locateFile: (path: string) => `${tfliteWasmBase}${path}`
|
||||
});
|
||||
|
||||
// Fetch and load model into TFLite WASM heap.
|
||||
const modelResponse = await fetch(tfliteModelPath, { credentials: 'same-origin' });
|
||||
|
||||
if (!modelResponse.ok) {
|
||||
workerPost({ error: `TFLite model fetch failed: HTTP ${modelResponse.status}`, type: 'init_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const modelBuffer = await modelResponse.arrayBuffer();
|
||||
|
||||
tfliteModule.HEAPU8.set(new Uint8Array(modelBuffer), tfliteModule._getModelBufferMemoryOffset());
|
||||
tfliteModule._loadModel(modelBuffer.byteLength);
|
||||
|
||||
// Pre-compute HEAPF32 offsets to avoid divide-by-4 on every frame.
|
||||
tfliteInputOffset = tfliteModule._getInputMemoryOffset() / 4;
|
||||
tfliteOutputOffset = tfliteModule._getOutputMemoryOffset() / 4;
|
||||
tfliteSegWidth = segWidth;
|
||||
tfliteSegHeight = segHeight;
|
||||
tflitePixelCount = segWidth * segHeight;
|
||||
|
||||
// Bounds-check the HEAPF32 input/output regions against the requested seg dimensions.
|
||||
// Catches a mismatch between the configured seg size and the model's actual tensor
|
||||
// shape before the inference loop starts silently clobbering WASM heap memory.
|
||||
const heapLen = tfliteModule.HEAPF32.length;
|
||||
const inputEnd = tfliteInputOffset + tflitePixelCount * 3; // NHWC float32, 3 channels.
|
||||
const outputEnd = tfliteOutputOffset + tflitePixelCount; // Single float per pixel.
|
||||
|
||||
if (inputEnd > heapLen || outputEnd > heapLen) {
|
||||
workerPost({
|
||||
error: `TFLite heap bounds mismatch: segWidth=${segWidth} segHeight=${segHeight}`
|
||||
+ ` requires inputEnd=${inputEnd}/outputEnd=${outputEnd} but HEAPF32 is ${heapLen}.`
|
||||
+ ' Model input shape likely differs from the configured seg resolution.',
|
||||
type: 'init_error'
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
tfliteReadCanvas = new OffscreenCanvas(segWidth, segHeight);
|
||||
tfliteReadCtx = tfliteReadCanvas.getContext('2d', { willReadFrequently: true });
|
||||
|
||||
workerPost({ backend: currentBackend, segHeight: tfliteSegHeight, segWidth: tfliteSegWidth, type: 'init_done' });
|
||||
} catch (err) {
|
||||
workerPost({ error: String(err), type: 'init_error' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatches an inference call to the TFLite or TF.js handler based on the active backend.
|
||||
*
|
||||
* @param {ImageBitmap} bitmap - Pre-scaled camera frame at the tier's segmentation resolution.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleInfer(bitmap: ImageBitmap): Promise<void> {
|
||||
if (currentBackend === BackendType.TFLITE) {
|
||||
await handleInferTflite(bitmap);
|
||||
} else {
|
||||
await handleInferTfjs(bitmap);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one TF.js inference cycle on the transferred ImageBitmap.
|
||||
*
|
||||
* The bitmap is at the tier's segmentation resolution (pre-scaled on the main thread via
|
||||
* createImageBitmap resize options). TF.js skips its own internal resize, reducing per-frame cost.
|
||||
* The raw mask Uint8ClampedArray is transferred back (zero-copy) for EMA smoothing and WebGL
|
||||
* compositing on the main thread.
|
||||
*
|
||||
* @param {ImageBitmap} bitmap - Pre-scaled camera frame at seg resolution.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleInferTfjs(bitmap: ImageBitmap): Promise<void> {
|
||||
if (!segmenter) {
|
||||
bitmap.close();
|
||||
workerPost({ error: 'Segmenter not initialised', type: 'infer_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
tf.engine().startScope();
|
||||
try {
|
||||
const segmentations = await segmenter.segmentPeople(
|
||||
bitmap as unknown as HTMLImageElement, { flipHorizontal: false }
|
||||
);
|
||||
|
||||
if (!segmentations.length) {
|
||||
workerPost({ error: 'No segmentation result', type: 'infer_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
const maskData = await segmentations[0].mask.toImageData();
|
||||
const data = maskData.data;
|
||||
|
||||
// Transfer the backing ArrayBuffer — zero-copy, data becomes detached in this worker.
|
||||
workerPost(
|
||||
{ data, height: maskData.height, type: 'mask', width: maskData.width },
|
||||
[ data.buffer as ArrayBuffer ]
|
||||
);
|
||||
} catch (err) {
|
||||
workerPost({ error: String(err), type: 'infer_error' });
|
||||
} finally {
|
||||
// Close unconditionally — called in finally so the bitmap is always released
|
||||
// even when segmentPeople() throws before the try-block close is reached.
|
||||
bitmap.close();
|
||||
tf.engine().endScope();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs one TFLite inference cycle on the transferred ImageBitmap.
|
||||
*
|
||||
* Draws the bitmap to an OffscreenCanvas, fills the TFLite HEAPF32 input buffer with NHWC
|
||||
* float32 RGB values normalised to [0,1], calls _runInference(), and reads the single-channel
|
||||
* output (the model outputs one float per pixel: person confidence in [0,1]).
|
||||
* Writes the confidence value to both R and A channels of the returned Uint8ClampedArray so
|
||||
* it is compatible with both the WebGL compositor (reads .r) and the Canvas 2D fallback (reads alpha).
|
||||
*
|
||||
* @param {ImageBitmap} bitmap - Pre-scaled camera frame at seg resolution.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleInferTflite(bitmap: ImageBitmap): Promise<void> {
|
||||
if (!tfliteModule || !tfliteReadCtx) {
|
||||
bitmap.close();
|
||||
workerPost({ error: 'TFLite not initialised', type: 'infer_error' });
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Draw pre-scaled bitmap to the read canvas and extract RGBA pixels.
|
||||
tfliteReadCtx.drawImage(bitmap, 0, 0, tfliteSegWidth, tfliteSegHeight);
|
||||
const imageData = tfliteReadCtx.getImageData(0, 0, tfliteSegWidth, tfliteSegHeight);
|
||||
|
||||
// Fill NHWC float32 input [1, H, W, 3] with RGB channels normalised to [0, 1].
|
||||
const pixelCount = tflitePixelCount;
|
||||
|
||||
for (let i = 0; i < pixelCount; i++) {
|
||||
tfliteModule.HEAPF32[tfliteInputOffset + i * 3] = imageData.data[i * 4] / 255;
|
||||
tfliteModule.HEAPF32[tfliteInputOffset + i * 3 + 1] = imageData.data[i * 4 + 1] / 255;
|
||||
tfliteModule.HEAPF32[tfliteInputOffset + i * 3 + 2] = imageData.data[i * 4 + 2] / 255;
|
||||
}
|
||||
|
||||
// Run TFLite inference (synchronous WASM call).
|
||||
tfliteModule._runInference();
|
||||
|
||||
// Read single-channel output: one float per pixel (person confidence in [0, 1]).
|
||||
// Model outputs probabilities directly — no softmax needed.
|
||||
const maskBytes = new Uint8ClampedArray(pixelCount * 4);
|
||||
|
||||
for (let i = 0; i < pixelCount; i++) {
|
||||
const v = Math.round(tfliteModule.HEAPF32[tfliteOutputOffset + i] * 255);
|
||||
|
||||
maskBytes[i * 4] = v; // R — read by WebGL compositor shader (.r channel)
|
||||
maskBytes[i * 4 + 3] = v; // A — read by Canvas 2D fallback compositing path
|
||||
}
|
||||
|
||||
workerPost(
|
||||
{ data: maskBytes, height: tfliteSegHeight, type: 'mask', width: tfliteSegWidth },
|
||||
[ maskBytes.buffer ]
|
||||
);
|
||||
} catch (err) {
|
||||
workerPost({ error: String(err), type: 'infer_error' });
|
||||
} finally {
|
||||
bitmap.close();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user