995 lines
40 KiB
TypeScript
995 lines
40 KiB
TypeScript
import AllureReporter from '@wdio/allure-reporter';
|
|
import { Buffer } from 'buffer';
|
|
import fs from 'fs';
|
|
import { glob } from 'glob';
|
|
import junitReportBuilder from 'junit-report-builder';
|
|
import { randomUUID } from 'node:crypto';
|
|
import path from 'node:path';
|
|
import process from 'node:process';
|
|
import pretty from 'pretty';
|
|
|
|
import { getTestProperties, loadTestFiles } from './helpers/TestProperties';
|
|
import { config as testsConfig } from './helpers/TestsConfig';
|
|
import WebhookProxy from './helpers/WebhookProxy';
|
|
import { getLogs, initLogger, logInfo, saveLogs } from './helpers/browserLogger';
|
|
import { registerCustomMatchers } from './helpers/matchers';
|
|
import { IContext } from './helpers/types';
|
|
import { generateRoomName } from './helpers/utils';
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
const allure = require('allure-commandline');
|
|
|
|
// This is deprecated without alternative (https://github.com/nodejs/node/issues/32483)
|
|
// we need it to be able to reuse jitsi-meet code in tests
|
|
require.extensions['.web.ts'] = require.extensions['.ts'];
|
|
|
|
const chromeArgs = [
|
|
'--allow-insecure-localhost',
|
|
'--use-fake-ui-for-media-stream',
|
|
'--use-fake-device-for-media-stream',
|
|
'--disable-plugins',
|
|
'--mute-audio',
|
|
'--disable-infobars',
|
|
'--autoplay-policy=no-user-gesture-required',
|
|
'--auto-select-desktop-capture-source=Your Entire screen',
|
|
'--no-sandbox',
|
|
'--disable-dev-shm-usage',
|
|
'--disable-setuid-sandbox',
|
|
|
|
// Avoids - "You are checking for animations on an inactive tab, animations do not run for inactive tabs"
|
|
// when executing waitForStable()
|
|
'--disable-renderer-backgrounding',
|
|
|
|
// Absolute (__dirname-based, not CWD-relative): this is passed to Chrome, which resolves it
|
|
// against wherever the launching process's CWD happens to be - the repo root when running
|
|
// `npm test` from there, but tests/ itself when running via tests/package.json's own scripts.
|
|
`--use-file-for-fake-audio-capture=${path.join(__dirname, 'resources', 'fakeAudioStream.wav')}`
|
|
];
|
|
|
|
if (process.env.RESOLVER_RULES) {
|
|
chromeArgs.push(`--host-resolver-rules=${process.env.RESOLVER_RULES}`);
|
|
}
|
|
if (process.env.ALLOW_INSECURE_CERTS === 'true') {
|
|
chromeArgs.push('--ignore-certificate-errors');
|
|
}
|
|
if (process.env.HEADLESS === 'true') {
|
|
chromeArgs.push('--headless');
|
|
chromeArgs.push('--window-size=1280,1024');
|
|
}
|
|
if (process.env.VIDEO_CAPTURE_FILE) {
|
|
chromeArgs.push(`--use-file-for-fake-video-capture=${process.env.VIDEO_CAPTURE_FILE}`);
|
|
}
|
|
|
|
const chromePreferences = {
|
|
'intl.accept_languages': 'en-US'
|
|
};
|
|
|
|
const specs = [
|
|
'specs/**/*.spec.ts'
|
|
];
|
|
|
|
/**
|
|
* Analyzes test files at config construction time to determine browser requirements
|
|
* and generate capabilities with appropriate exclusions.
|
|
*/
|
|
function generateCapabilitiesFromSpecs(): { capabilities: Record<string, any>; excludedSpecs: string[]; } {
|
|
const allSpecFiles: string[] = [];
|
|
const allBrowsers = [ 'p1', 'p2', 'p3', 'p4', 'p5', 'p6' ];
|
|
const excludedSpecs: string[] = [];
|
|
|
|
for (const pattern of specs) {
|
|
const matches = glob.sync(pattern, { cwd: path.join(__dirname) });
|
|
|
|
allSpecFiles.push(...matches.map(f => path.resolve(__dirname, f)));
|
|
}
|
|
|
|
// Load test files to populate the testProperties registry
|
|
loadTestFiles(allSpecFiles);
|
|
|
|
// Import TestProperties to access the populated registry
|
|
const { testProperties } = require('./helpers/TestProperties');
|
|
|
|
// Detect if specific spec files are targeted via --spec CLI argument (e.g. npm run test-single).
|
|
// When targeted, only create capabilities for the browsers those specs actually need.
|
|
const targetedSpecFiles: string[] = [];
|
|
|
|
for (let i = 0; i < process.argv.length; i++) {
|
|
if (process.argv[i] === '--spec') {
|
|
// Collect all consecutive non-flag arguments after --spec (wdio accepts space-separated specs).
|
|
let j = i + 1;
|
|
|
|
while (j < process.argv.length && !process.argv[j].startsWith('--')) {
|
|
process.argv[j].split(',').forEach(f => {
|
|
const normalized = f.trim().replace(/\\/g, '/');
|
|
// Try exact resolution first, then fall back to case-insensitive suffix matching
|
|
// (macOS filesystem is case-insensitive but string comparison is not).
|
|
const resolved = path.resolve(normalized);
|
|
const resolvedLower = resolved.toLowerCase();
|
|
const normalizedLower = normalized.toLowerCase();
|
|
const match = allSpecFiles.find(sf => {
|
|
const sfNormalized = sf.replace(/\\/g, '/');
|
|
|
|
return sfNormalized === resolved.replace(/\\/g, '/')
|
|
|| sfNormalized.toLowerCase() === resolvedLower.replace(/\\/g, '/')
|
|
|| sfNormalized.toLowerCase().endsWith(`/${normalizedLower}`);
|
|
});
|
|
|
|
if (match) {
|
|
targetedSpecFiles.push(match);
|
|
} else {
|
|
console.warn(`[wdio] --spec file not found in spec list: ${f}`);
|
|
}
|
|
});
|
|
j++;
|
|
}
|
|
i = j - 1;
|
|
}
|
|
}
|
|
|
|
console.log(`[wdio] process.argv: ${process.argv.join(' ')}`);
|
|
console.log(`[wdio] targeted specs: ${targetedSpecFiles.join(', ') || '(all)'}`);
|
|
|
|
const scopedFiles = targetedSpecFiles.length > 0 ? targetedSpecFiles : allSpecFiles;
|
|
const requiredBrowsers = new Set<string>();
|
|
|
|
for (const file of scopedFiles) {
|
|
const props = testProperties[file];
|
|
|
|
if (props?.useJaas && !testsConfig.jaas.enabled) {
|
|
continue;
|
|
}
|
|
if (props?.usesBrowsers) {
|
|
props.usesBrowsers.forEach((b: string) => requiredBrowsers.add(b));
|
|
} else {
|
|
requiredBrowsers.add('p1');
|
|
}
|
|
}
|
|
|
|
// Preserve the predefined browser order; fall back to all browsers if nothing was determined.
|
|
const browsers = requiredBrowsers.size > 0
|
|
? allBrowsers.filter(b => requiredBrowsers.has(b))
|
|
: allBrowsers;
|
|
|
|
// Determine which browsers need which exclusions
|
|
const browserExclusions: Record<string, Set<string>> = {};
|
|
|
|
browsers.forEach(b => {
|
|
browserExclusions[b] = new Set();
|
|
});
|
|
|
|
for (const file of allSpecFiles) {
|
|
const props = testProperties[file];
|
|
const relativeFile = path.relative(__dirname, file);
|
|
|
|
// If a test requires JaaS but JaaS is not configured, exclude it at the top level so no worker is created.
|
|
if (props?.useJaas && !testsConfig.jaas.enabled) {
|
|
excludedSpecs.push(relativeFile);
|
|
continue;
|
|
}
|
|
|
|
// If a test doesn't use a particular browser, add it to exclusions for that browser
|
|
if (props?.usesBrowsers) {
|
|
browsers.forEach(browser => {
|
|
if (!props.usesBrowsers!.includes(browser)) {
|
|
browserExclusions[browser].add(relativeFile);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
return {
|
|
capabilities: Object.fromEntries(
|
|
browsers.map(browser => [
|
|
browser,
|
|
{
|
|
capabilities: {
|
|
browserName: 'chrome',
|
|
// Only pin the custom jitsi-stable/jitsi-beta aliases when routing through
|
|
// Selenium Grid. Local chromedriver/geckodriver doesn't know these aliases.
|
|
...(process.env.GRID_HOST_URL ? {
|
|
browserVersion: browser === 'p1' && process.env.BROWSER_CHROME_BETA
|
|
? 'jitsi-beta' : 'jitsi-stable'
|
|
} : {}),
|
|
'goog:chromeOptions': {
|
|
args: chromeArgs,
|
|
prefs: chromePreferences
|
|
},
|
|
'wdio:exclude': Array.from(browserExclusions[browser] || [])
|
|
}
|
|
}
|
|
])
|
|
),
|
|
excludedSpecs
|
|
};
|
|
}
|
|
|
|
const { capabilities, excludedSpecs } = generateCapabilitiesFromSpecs();
|
|
|
|
const TEST_RESULTS_DIR = 'test-results';
|
|
|
|
const keepAlive: Array<any> = [];
|
|
|
|
// Titles of the tests that failed in this worker. Recorded by afterTest and handed to the launcher
|
|
// by the `after` hook, so onWorkerEnd can tell a clean run from a failed one even when the
|
|
// reporters never got the chance to flush their results.
|
|
const failedTestTitles: Array<string> = [];
|
|
|
|
// Tracks browser-session lifecycle events written by each worker's
|
|
// beforeSession/afterSession hooks. The launcher's onComplete reads it to
|
|
// compute the peak number of concurrent browser sessions during the run.
|
|
const CONCURRENCY_LOG_PATH = path.join(TEST_RESULTS_DIR, 'concurrency.jsonl');
|
|
|
|
/**
|
|
* Path of the sentinel a worker writes once its whole spec file has finished running.
|
|
*
|
|
* Written by the framework `after` hook (which @wdio/mocha-framework runs after every test and
|
|
* hook, and before the runner deletes the sessions) and read by the launcher's `onWorkerEnd`: a
|
|
* worker that exits non-zero *with* this file present got through its whole spec file, so whatever
|
|
* killed it happened while tearing the session(s) down.
|
|
*
|
|
* This has to be a file rather than something parsed out of the worker log. The worker's debug
|
|
* output goes to stdout, which the launcher forwards to the console; WDIO_LOG_PATH only ever
|
|
* receives launcher-level lines, so the hook traces a log-scraping check would need are absent.
|
|
*
|
|
* @param cid - The worker (capability) id.
|
|
*/
|
|
function completionSentinelPath(cid: string): string {
|
|
return path.join(TEST_RESULTS_DIR, `completed-${cid}.json`);
|
|
}
|
|
|
|
/**
|
|
* Reads a worker's completion sentinel, if it wrote one.
|
|
*
|
|
* @param cid - The worker (capability) id.
|
|
* @returns The failed test titles and mocha failure count the worker recorded, or undefined when
|
|
* it never reached its `after` hook.
|
|
*/
|
|
function readWorkerCompletion(cid: string): { failed: Array<string>; failures: number; } | undefined {
|
|
try {
|
|
const parsed = JSON.parse(fs.readFileSync(completionSentinelPath(cid), 'utf8'));
|
|
|
|
return {
|
|
failed: Array.isArray(parsed.failed) ? parsed.failed : [],
|
|
failures: typeof parsed.failures === 'number' ? parsed.failures : 0
|
|
};
|
|
} catch {
|
|
// No sentinel, or it is unreadable — the worker did not finish its spec file.
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extracts the last error line from a worker's wdio log, so a synthesised report can say what
|
|
* actually went wrong instead of only that the worker exited non-zero.
|
|
*
|
|
* @param specFile - The worker's first spec file (path or URL); used to locate its wdio log.
|
|
* @param cid - The worker (capability) id.
|
|
* @returns The error line, trimmed and truncated, or undefined if the log has none.
|
|
*/
|
|
function readWorkerError(specFile: string | undefined, cid: string): string | undefined {
|
|
// @wdio/local-runner writes the worker log to `${specBaseName}-${cid}.log`, where specBaseName
|
|
// strips only the final extension (so `displayName.spec.ts` -> `displayName.spec`), falling back
|
|
// to `wdio-${cid}.log` when no spec is associated. Mirror that exactly so the log is found.
|
|
const candidates = [];
|
|
|
|
if (specFile) {
|
|
const specBaseName = path.basename(specFile, path.extname(specFile));
|
|
|
|
candidates.push(path.join(TEST_RESULTS_DIR, `${specBaseName}-${cid}.log`));
|
|
}
|
|
candidates.push(path.join(TEST_RESULTS_DIR, `wdio-${cid}.log`));
|
|
|
|
const logPath = candidates.find(p => fs.existsSync(p));
|
|
|
|
if (!logPath) {
|
|
return undefined;
|
|
}
|
|
|
|
try {
|
|
// Worker logs can be tens of MB; only scan the tail where the teardown sequence lives.
|
|
const stats = fs.statSync(logPath);
|
|
const tailSize = Math.min(stats.size, 64 * 1024);
|
|
const fd = fs.openSync(logPath, 'r');
|
|
const buf = Buffer.alloc(tailSize);
|
|
|
|
fs.readSync(fd, buf, 0, tailSize, Math.max(0, stats.size - tailSize));
|
|
fs.closeSync(fd);
|
|
|
|
return buf.toString('utf8')
|
|
.split('\n')
|
|
.reverse()
|
|
.find(line => line.includes('ERROR'))
|
|
?.trim()
|
|
.slice(0, 500);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
export const config: WebdriverIO.MultiremoteConfig = {
|
|
|
|
runner: 'local',
|
|
|
|
specs,
|
|
exclude: excludedSpecs,
|
|
|
|
maxInstances: parseInt(process.env.MAX_INSTANCES || '1', 10), // if changing check onWorkerStart logic
|
|
|
|
baseUrl: process.env.BASE_URL || 'https://alpha.jitsi.net/torture/',
|
|
tsConfigPath: './tsconfig.json',
|
|
|
|
// Default timeout for all waitForXXX commands.
|
|
waitforTimeout: 1000,
|
|
|
|
// Default timeout in milliseconds for request
|
|
// if browser driver or grid doesn't send response
|
|
connectionRetryTimeout: 30_000,
|
|
|
|
// Default request retries count
|
|
connectionRetryCount: 3,
|
|
|
|
framework: 'mocha',
|
|
|
|
mochaOpts: {
|
|
timeout: 180_000
|
|
},
|
|
|
|
capabilities,
|
|
|
|
// Level of logging verbosity: trace | debug | info | warn | error | silent
|
|
logLevel: 'trace',
|
|
logLevels: {
|
|
webdriver: 'info'
|
|
},
|
|
|
|
// Can be used to debug chromedriver, depends on chromedriver and wdio-chromedriver-service
|
|
// services: [
|
|
// [ 'chromedriver', {
|
|
// // Pass the --verbose flag to Chromedriver
|
|
// args: [ '--verbose' ],
|
|
// // Optionally, define a file to store the logs instead of stdout
|
|
// logFileName: 'wdio-chromedriver.log',
|
|
// // Optionally, define a directory for the log file
|
|
// outputDir: 'test-results',
|
|
// } ]
|
|
// ],
|
|
|
|
// Set directory to store all logs into
|
|
outputDir: TEST_RESULTS_DIR,
|
|
|
|
reporters: [
|
|
[ 'junit', {
|
|
outputDir: TEST_RESULTS_DIR,
|
|
outputFileFormat(options) { // optional
|
|
return `results-${options.cid}.xml`;
|
|
}
|
|
} ],
|
|
[ 'allure', {
|
|
// addConsoleLogs: true,
|
|
outputDir: `${TEST_RESULTS_DIR}/allure-results`,
|
|
disableWebdriverStepsReporting: true,
|
|
disableWebdriverScreenshotsReporting: true,
|
|
useCucumberStepReporter: false
|
|
} ]
|
|
],
|
|
|
|
execArgv: [ '--stack-trace-limit=100' ],
|
|
|
|
// =====
|
|
// Hooks
|
|
// =====
|
|
/**
|
|
* Gets executed once before all workers are spawned (launcher process only).
|
|
* Reset the concurrency log so onComplete only sees events from this run.
|
|
*/
|
|
onPrepare() {
|
|
try {
|
|
if (fs.existsSync(CONCURRENCY_LOG_PATH)) {
|
|
fs.unlinkSync(CONCURRENCY_LOG_PATH);
|
|
}
|
|
} catch {
|
|
// best effort — if we can't reset, onComplete will still parse what it sees
|
|
}
|
|
|
|
// Drop completion sentinels left behind by an earlier run in the same workspace; a stale
|
|
// one would make onWorkerEnd read a genuine crash as a teardown-only crash.
|
|
try {
|
|
fs.readdirSync(TEST_RESULTS_DIR)
|
|
.filter(name => (/^completed-.*\.json$/).test(name))
|
|
.forEach(name => fs.unlinkSync(path.join(TEST_RESULTS_DIR, name)));
|
|
} catch {
|
|
// best effort — the directory may not exist yet
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Gets executed before test execution begins. At this point you can access to all global
|
|
* variables like `browser`. It is the perfect place to define custom commands.
|
|
* We have overriden this function in beforeSession to be able to pass cid as first param.
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async before(cid, _, files) {
|
|
// Register custom matchers here (not at matchers.ts module load): @wdio/runner replaces
|
|
// globalThis.expect via _setGlobal AFTER mocha has already loaded the spec files, so any
|
|
// expect.extend at module load runs against an expect that's about to be discarded.
|
|
registerCustomMatchers();
|
|
|
|
if (files.length !== 1) {
|
|
console.warn('We expect to run a single suite, but got more than one');
|
|
}
|
|
|
|
const testFilePath = files[0].replace(/^file:\/\//, '');
|
|
// __dirname-based (not CWD-relative) for the same reason as the chromeArgs entry above.
|
|
const testName = path.relative(path.join(__dirname, 'specs'), testFilePath)
|
|
.replace(/.spec.ts$/, '')
|
|
.replace(/\//g, '-');
|
|
const testProperties = await getTestProperties(testFilePath);
|
|
|
|
console.log(`Running test: ${testName} via worker: ${cid} browser instances:${multiRemoteBrowser.instances.length}`);
|
|
|
|
// Log the browsers in use, the console logs of the participants are attached to the report only when a test
|
|
// fails, so this is the only place where a passing run records what it was executed against.
|
|
console.log(`Using browsers: ${multiRemoteBrowser.instances.map((instance: string) => {
|
|
const { browserName, browserVersion } = multiRemoteBrowser.getInstance(instance).capabilities;
|
|
|
|
return `${instance}:${browserName}/${browserVersion}`;
|
|
}).join(' ')}`);
|
|
|
|
const globalAny: any = global;
|
|
|
|
globalAny.ctx = {
|
|
times: {}
|
|
} as IContext;
|
|
globalAny.ctx.testProperties = testProperties;
|
|
|
|
if (testProperties.useJaas && !testsConfig.jaas.enabled) {
|
|
globalAny.ctx.skipSuiteTests = 'JaaS is not configured';
|
|
|
|
return;
|
|
}
|
|
|
|
await Promise.all(multiRemoteBrowser.instances.map(async (instance: string) => {
|
|
const bInstance = multiRemoteBrowser.getInstance(instance);
|
|
|
|
// @ts-ignore
|
|
initLogger(bInstance, `${instance}-${cid}-${testName}`, TEST_RESULTS_DIR);
|
|
|
|
// setup keepalive
|
|
keepAlive.push(setInterval(async () => {
|
|
await bInstance.execute(() => console.log(`${new Date().toISOString()} keep-alive`));
|
|
}, 20_000));
|
|
|
|
if (bInstance.isFirefox) {
|
|
return;
|
|
}
|
|
|
|
// __dirname-based (not CWD-relative) for the same reason as the chromeArgs entry above -
|
|
// this crashed the worker outright (ENOENT) once tests started running from tests/
|
|
// itself instead of the repo root.
|
|
const rpath = await bInstance.uploadFile(path.join(__dirname, 'resources', 'iframeAPITest.html'));
|
|
|
|
// @ts-ignore
|
|
bInstance.iframePageBase = `file://${path.dirname(rpath)}`;
|
|
}));
|
|
|
|
globalAny.ctx.roomName = generateRoomName(testName);
|
|
console.log(`Using room name: ${globalAny.ctx.roomName}`);
|
|
|
|
if (testProperties.useWebhookProxy && testsConfig.webhooksProxy.enabled && !globalAny.ctx.webhooksProxy) {
|
|
const tenant = testsConfig.jaas.tenant;
|
|
|
|
if (!testProperties.useJaas) {
|
|
throw new Error('The test tries to use WebhookProxy without JaaS.');
|
|
}
|
|
if (!tenant) {
|
|
console.log(`Can not configure WebhookProxy, missing tenant in config. Skipping ${testName}.`);
|
|
globalAny.ctx.skipSuiteTests = 'WebHookProxy is required but not configured (missing tenant)';
|
|
|
|
return;
|
|
}
|
|
|
|
globalAny.ctx.webhooksProxy = new WebhookProxy(
|
|
`${testsConfig.webhooksProxy.url}?tenant=${tenant}&room=${globalAny.ctx.roomName}`,
|
|
testsConfig.webhooksProxy.sharedSecret!,
|
|
`${TEST_RESULTS_DIR}/webhooks-${cid}-${testName}.log`);
|
|
globalAny.ctx.webhooksProxy.connect();
|
|
}
|
|
|
|
if (testProperties.requireWebhookProxy && !globalAny.ctx.webhooksProxy) {
|
|
throw new Error('The test requires WebhookProxy, but it is not available.');
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Gets executed after all tests in the spec file are done, before the sessions are deleted.
|
|
*
|
|
* @param {number} result - Mocha's number of failing tests for this spec file.
|
|
* @returns {void}
|
|
*/
|
|
after(result) {
|
|
const { ctx }: any = global;
|
|
|
|
ctx?.webhooksProxy?.disconnect();
|
|
keepAlive.forEach(clearInterval);
|
|
|
|
// Tell the launcher this worker got through every test and hook. If it still exits
|
|
// non-zero, the failure was in session teardown, which onWorkerEnd reports separately.
|
|
const cid = process.env.WDIO_WORKER_ID;
|
|
|
|
if (cid) {
|
|
try {
|
|
fs.writeFileSync(completionSentinelPath(cid), JSON.stringify({
|
|
failed: failedTestTitles,
|
|
failures: typeof result === 'number' ? result : 1
|
|
}));
|
|
} catch {
|
|
// best effort — without it, a teardown crash is reported as a runner crash
|
|
}
|
|
}
|
|
},
|
|
|
|
async beforeSession(c, capabilities_, spec, cid) {
|
|
// Clear any completion sentinel left by an earlier attempt at this spec — worker ids can
|
|
// be reused when a spec file is retried — so onWorkerEnd never reads a stale completion.
|
|
try {
|
|
fs.rmSync(completionSentinelPath(cid), { force: true });
|
|
} catch {
|
|
// best effort — onPrepare already cleared sentinels from previous runs
|
|
}
|
|
|
|
// Record this worker's browser sessions opening, so the launcher's
|
|
// onComplete can compute peak concurrent sessions across the run.
|
|
try {
|
|
fs.appendFileSync(CONCURRENCY_LOG_PATH, `${JSON.stringify({
|
|
type: 'start',
|
|
t: Date.now(),
|
|
cid,
|
|
n: Object.keys(capabilities_ as Record<string, unknown>).length
|
|
})}\n`);
|
|
} catch {
|
|
// non-fatal — concurrency stats are best-effort
|
|
}
|
|
|
|
const originalBefore = c.before;
|
|
|
|
if (spec && spec.length == 1) {
|
|
const testFilePath = spec[0].replace(/^file:\/\//, '');
|
|
const testProperties = await getTestProperties(testFilePath);
|
|
|
|
if (testProperties.retry) {
|
|
c.specFileRetries = 1;
|
|
c.specFileRetriesDeferred = true;
|
|
c.specFileRetriesDelay = 1;
|
|
console.log(`Enabling retry for ${testFilePath}`);
|
|
}
|
|
} else {
|
|
console.log('No test file or multiple test files specified, will not enable retries');
|
|
}
|
|
|
|
if (!originalBefore || !Array.isArray(originalBefore) || originalBefore.length !== 1) {
|
|
console.warn('No before hook found or more than one found, skipping');
|
|
|
|
return;
|
|
}
|
|
|
|
if (originalBefore) {
|
|
c.before = [ async function(...args) {
|
|
// Call original with cid as first param, followed by original args
|
|
// @ts-ignore
|
|
return await originalBefore[0].call(c, cid, ...args);
|
|
} ];
|
|
}
|
|
},
|
|
|
|
afterSession(_c, capabilities_, _specs) {
|
|
try {
|
|
fs.appendFileSync(CONCURRENCY_LOG_PATH, `${JSON.stringify({
|
|
type: 'end',
|
|
t: Date.now(),
|
|
n: Object.keys(capabilities_ as Record<string, unknown>).length
|
|
})}\n`);
|
|
} catch {
|
|
// non-fatal — concurrency stats are best-effort
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Gets executed before the suite starts (in Mocha/Jasmine only).
|
|
*
|
|
* @param {Object} suite - Suite details.
|
|
*/
|
|
beforeSuite(suite) {
|
|
multiRemoteBrowser.instances.forEach((instance: string) => {
|
|
logInfo(multiRemoteBrowser.getInstance(instance),
|
|
`---=== Begin ${suite.file.substring(suite.file.lastIndexOf('/') + 1)} ===---`);
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Function to be executed before a test (in Mocha/Jasmine only).
|
|
*
|
|
* @param {Object} test - Test object.
|
|
* @param {Object} context - The context object.
|
|
*/
|
|
beforeTest(test, context) {
|
|
// Extract directory to use as parent suite and describe block name as suite
|
|
const dirMatch = test.file.match(/.*\/tests\/specs\/([^\/]+)\//);
|
|
const dir = dirMatch ? dirMatch[1] : false;
|
|
const fileMatch = test.file.match(/.*\/tests\/specs\/(.*)/);
|
|
const file = fileMatch ? fileMatch[1] : false;
|
|
|
|
if (ctx.testProperties.description) {
|
|
AllureReporter.addDescription(ctx.testProperties.description, 'text');
|
|
}
|
|
|
|
if (file) {
|
|
AllureReporter.addLink(`https://github.com/jitsi/jitsi-meet/blob/master/tests/specs/${file}`, 'Code');
|
|
}
|
|
|
|
// For Allure v3: set directory as parent suite and describe block as suite
|
|
if (dir && test.parent) {
|
|
AllureReporter.addParentSuite(dir);
|
|
AllureReporter.addSuite(test.parent);
|
|
}
|
|
|
|
// A failure in an earlier test skips the rest of that describe block, but not other describe blocks in the
|
|
// same file (ctx.failedSuite is scoped to the suite that failed, while ctx.skipSuiteTests is file-wide).
|
|
const failedSameSuite = Boolean(ctx.failedSuite) && ctx.failedSuite === test.parent;
|
|
const skipReason = ctx.skipSuiteTests
|
|
|| (failedSameSuite ? `A previous test in suite "${test.parent}" has failed.` : false);
|
|
|
|
if (skipReason) {
|
|
if ((typeof skipReason) === 'string') {
|
|
AllureReporter.addDescription((ctx.testProperties.description || '')
|
|
+ '\n\nSkipped because: ' + skipReason, 'text');
|
|
}
|
|
console.log(`Skipping because: ${skipReason}`);
|
|
|
|
context.skip();
|
|
|
|
return;
|
|
}
|
|
|
|
multiRemoteBrowser.instances.forEach((instance: string) => {
|
|
logInfo(multiRemoteBrowser.getInstance(instance), `---=== Start test ${test.title} ===---`);
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Function to be executed after a test (in Mocha/Jasmine only).
|
|
*
|
|
* @param {Object} test - Test object.
|
|
* @param {Object} context - Scope object the test was executed with.
|
|
* @param {Error} error - Error object in case the test fails, otherwise `undefined`.
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async afterTest(test, context, { error }) {
|
|
multiRemoteBrowser.instances.forEach((instance: string) =>
|
|
logInfo(multiRemoteBrowser.getInstance(instance), `---=== End test ${test.title} ===---`));
|
|
|
|
if (error) {
|
|
|
|
// Record the failure for the `after` hook, so the launcher still knows this spec had
|
|
// failing tests if the worker dies before the reporters flush.
|
|
failedTestTitles.push(test.title);
|
|
|
|
// Skip the remaining tests in the same describe block (but not other describe blocks in the file).
|
|
ctx.failedSuite = test.parent;
|
|
|
|
// make sure all browsers are at the main app in iframe (if used), so we collect debug info
|
|
await Promise.all(multiRemoteBrowser.instances.map(async (instance: string) => {
|
|
// @ts-ignore
|
|
await ctx[instance]?.switchToIFrame();
|
|
}));
|
|
|
|
const allProcessing: Promise<any>[] = [];
|
|
const attachments: { content: string | Buffer; filename: string; type: string; }[] = [];
|
|
|
|
multiRemoteBrowser.instances.forEach((instance: string) => {
|
|
const bInstance = multiRemoteBrowser.getInstance(instance);
|
|
|
|
allProcessing.push(bInstance.takeScreenshot().then(shot => {
|
|
attachments.push({
|
|
filename: `${instance}-screenshot`,
|
|
content: Buffer.from(shot, 'base64'),
|
|
type: 'image/png' });
|
|
}));
|
|
|
|
// @ts-ignore
|
|
allProcessing.push(bInstance.execute(() => typeof APP !== 'undefined' && APP.connection?.getLogs())
|
|
.then(logs =>
|
|
logs && attachments.push({
|
|
filename: `${instance}-debug-logs`,
|
|
content: JSON.stringify(logs, null, ' '),
|
|
type: 'text/plain' }))
|
|
.catch(e => console.error('Failed grabbing debug logs', e)));
|
|
|
|
allProcessing.push(
|
|
bInstance.execute(() => window.APP?.debugLogs?.logs?.join('\n')).then(res => {
|
|
if (res) {
|
|
saveLogs(bInstance, res);
|
|
}
|
|
|
|
attachments.push({
|
|
filename: `${instance}-console-logs`,
|
|
content: getLogs(bInstance) || '',
|
|
type: 'text/plain' });
|
|
}));
|
|
|
|
allProcessing.push(bInstance.getPageSource().then(source => {
|
|
attachments.push({
|
|
filename: `${instance}-html-source`,
|
|
content: pretty(source),
|
|
type: 'text/plain' });
|
|
}));
|
|
});
|
|
|
|
await Promise.allSettled(allProcessing);
|
|
attachments.sort(
|
|
(a, b) => {
|
|
return a.filename < b.filename ? -1 : 1;
|
|
}).forEach(
|
|
a => {
|
|
AllureReporter.addAttachment(a.filename, a.content, a.type);
|
|
}
|
|
);
|
|
|
|
console.log('Hanging up after test failure');
|
|
// let's hangup all the calls and give time for it to push any stats and logs
|
|
await Promise.all(multiRemoteBrowser.instances.map(async (instance: string) => {
|
|
// @ts-ignore
|
|
await ctx[instance]?.hangup();
|
|
}));
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Hook that gets executed after the suite has ended (in Mocha/Jasmine only).
|
|
*
|
|
* @param {Object} suite - Suite details.
|
|
* @returns {Promise<void>}
|
|
*/
|
|
afterSuite(suite) {
|
|
multiRemoteBrowser.instances.forEach((instance: string) => {
|
|
logInfo(multiRemoteBrowser.getInstance(instance),
|
|
`---=== End ${suite.file.substring(suite.file.lastIndexOf('/') + 1)} ===---`);
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Gets executed after a worker process has exited.
|
|
* Handles three crash scenarios:
|
|
* 1. Session teardown crash AFTER the spec file completed (common with Firefox on Selenium
|
|
* Grid): the worker wrote its completion sentinel, then the session DELETE either timed out
|
|
* or the worker was killed mid-teardown, so the runner exited non-zero. The tests themselves
|
|
* ran, so we report their outcome (from the sentinel) rather than a misleading "crashed",
|
|
* and record the teardown anomaly in the entry. See completionSentinelPath.
|
|
* 2. Session DELETE timeout BEFORE tests completed: JUnit reporter never flushes,
|
|
* leaving a zero-byte XML. Synthesise a failure entry.
|
|
* 3. Session INIT timeout: JUnit reporter writes a non-empty XML but with empty name/classname,
|
|
* and the allure reporter never fires (no beforeTest hook ran). Synthesise a failure entry.
|
|
*/
|
|
onWorkerEnd(cid, exitCode, workerSpecs) {
|
|
if (exitCode === 0) {
|
|
return;
|
|
}
|
|
const xmlPath = path.join(TEST_RESULTS_DIR, `results-${cid}.xml`);
|
|
|
|
const specName = workerSpecs?.[0] ? path.basename(workerSpecs[0], '.spec.ts') : 'unknown';
|
|
const dirMatch = workerSpecs?.[0]?.match(/\/tests\/specs\/([^/]+)\//);
|
|
const dir = dirMatch ? dirMatch[1] : 'unknown';
|
|
const message = `Worker exited with code ${exitCode} before results were written. Test result is unknown - tests may have passed.`;
|
|
|
|
// Check whether the XML has real test content. The JUnit reporter writes a non-empty XML
|
|
// even for session-init failures, but the testcase has empty name/classname in that case.
|
|
// If real test results were written, the allure reporter will have already produced its
|
|
// own result files, so we skip both. If the XML is missing or has no named test cases,
|
|
// we need to synthesise both.
|
|
let xmlHasNamedTests = false;
|
|
|
|
try {
|
|
const xmlContent = fs.readFileSync(xmlPath, 'utf8');
|
|
|
|
xmlHasNamedTests = xmlContent.includes('name="') && !xmlContent.includes('name=""');
|
|
} catch {
|
|
// file doesn't exist — fall through and create it
|
|
}
|
|
|
|
if (xmlHasNamedTests) {
|
|
// Real test results were written; allure reporter handled this worker normally.
|
|
return;
|
|
}
|
|
|
|
// A worker that wrote its completion sentinel ran every test and hook in the spec file, so
|
|
// it died in session teardown. Report what the tests actually did. Pass the raw spec path
|
|
// so the log file name is derived the same way @wdio/local-runner derives it (only the
|
|
// final extension is stripped).
|
|
const completion = readWorkerCompletion(cid);
|
|
|
|
if (completion) {
|
|
const failureCount = Math.max(completion.failures, completion.failed.length);
|
|
const clean = failureCount === 0;
|
|
const workerError = readWorkerError(workerSpecs?.[0], cid);
|
|
const teardownDetail = `Session teardown crashed (worker exit ${exitCode}), so the `
|
|
+ `reporters never wrote their results.${workerError ? ` Worker error: ${workerError}` : ''}`;
|
|
const teardownName = clean
|
|
? 'Session teardown crashed after tests completed'
|
|
: 'Tests failed, then session teardown crashed';
|
|
const teardownMessage = clean
|
|
? `The spec file ran to completion with no failures. ${teardownDetail}`
|
|
: `The spec file ran to completion with ${failureCount} failing test(s)`
|
|
+ `${completion.failed.length ? ` (${completion.failed.join(', ')})` : ''}. `
|
|
+ teardownDetail;
|
|
const b = junitReportBuilder.newBuilder();
|
|
const testCase = b.testSuite().name(specName).testCase()
|
|
.name(teardownName)
|
|
.className(specName)
|
|
.standardOutput(teardownMessage);
|
|
|
|
if (!clean) {
|
|
testCase.failure(teardownMessage);
|
|
}
|
|
b.writeTo(xmlPath);
|
|
|
|
// When the worker is killed mid-teardown the reporters never flush, so allure has no
|
|
// per-test results for this spec and it would otherwise vanish from the report. Write a
|
|
// single entry so the spec stays visible, carrying the outcome the worker recorded. If
|
|
// allure already captured the per-test results (e.g. a slow-DELETE timeout after a
|
|
// clean flush), this is just an extra informational entry alongside them.
|
|
const allureResult = {
|
|
uuid: randomUUID(),
|
|
name: teardownName,
|
|
status: clean ? 'passed' : 'failed',
|
|
statusDetails: { message: teardownMessage },
|
|
stage: 'finished',
|
|
steps: [],
|
|
attachments: [],
|
|
parameters: [],
|
|
labels: [
|
|
{ name: 'parentSuite', value: dir },
|
|
{ name: 'suite', value: specName }
|
|
],
|
|
links: []
|
|
};
|
|
const allurePath
|
|
= path.join(TEST_RESULTS_DIR, 'allure-results', `${allureResult.uuid}-result.json`);
|
|
|
|
fs.writeFileSync(allurePath, JSON.stringify(allureResult));
|
|
console.log(`[onWorkerEnd] Worker ${cid} (${specName}): ${teardownName} — synthesising `
|
|
+ `${clean ? 'passing' : 'failing'} JUnit + allure entry. ${teardownMessage}`);
|
|
|
|
return;
|
|
}
|
|
|
|
const b = junitReportBuilder.newBuilder();
|
|
|
|
b.testSuite().name(specName).testCase()
|
|
.name('Test runner crashed')
|
|
.className(specName)
|
|
.error(message);
|
|
b.writeTo(xmlPath);
|
|
|
|
const allureResult = {
|
|
uuid: randomUUID(),
|
|
name: 'Test runner crashed',
|
|
status: 'broken',
|
|
statusDetails: { message },
|
|
stage: 'finished',
|
|
steps: [],
|
|
attachments: [],
|
|
parameters: [],
|
|
labels: [
|
|
{ name: 'parentSuite', value: dir },
|
|
{ name: 'suite', value: specName }
|
|
],
|
|
links: []
|
|
};
|
|
const allurePath = path.join(TEST_RESULTS_DIR, 'allure-results', `${allureResult.uuid}-result.json`);
|
|
|
|
fs.writeFileSync(allurePath, JSON.stringify(allureResult));
|
|
console.log(`[onWorkerEnd] Wrote error XML and allure result for crashed worker ${cid} (spec: ${specName})`);
|
|
},
|
|
|
|
/**
|
|
* Gets executed after all workers have shut down and the process is about to exit.
|
|
* An error thrown in the `onComplete` hook will result in the test run failing.
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
onComplete() {
|
|
// Replay session start/end events written by each worker to print
|
|
// the peak concurrent browser-session count for this run.
|
|
try {
|
|
if (fs.existsSync(CONCURRENCY_LOG_PATH)) {
|
|
const events = fs.readFileSync(CONCURRENCY_LOG_PATH, 'utf8')
|
|
.split('\n')
|
|
.filter(Boolean)
|
|
.map(line => JSON.parse(line) as { n: number; t: number; type: 'start' | 'end'; });
|
|
|
|
events.sort((a, b) => a.t - b.t);
|
|
let active = 0;
|
|
let peak = 0;
|
|
let totalStarts = 0;
|
|
|
|
for (const ev of events) {
|
|
if (ev.type === 'start') {
|
|
active += ev.n;
|
|
totalStarts += ev.n;
|
|
if (active > peak) {
|
|
peak = active;
|
|
}
|
|
} else {
|
|
active -= ev.n;
|
|
}
|
|
}
|
|
console.log(`[concurrency] Peak concurrent browser sessions: ${peak}`);
|
|
console.log(`[concurrency] Total browser sessions started: ${totalStarts}`);
|
|
}
|
|
} catch (err) {
|
|
console.warn(`[concurrency] Could not compute peak concurrency: ${err}`);
|
|
}
|
|
|
|
// Clean up duplicate parentSuite labels from Allure results
|
|
const resultsDir = `${TEST_RESULTS_DIR}/allure-results`;
|
|
const resultFiles = fs.readdirSync(resultsDir).filter(f => f.endsWith('-result.json'));
|
|
|
|
resultFiles.forEach(file => {
|
|
const filePath = path.join(resultsDir, file);
|
|
const result = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
|
|
// Keep only the LAST parentSuite label (the manual one with directory name)
|
|
// Remove the automatic one from WebDriverIO (describe block name)
|
|
const parentSuiteLabels: any[] = [];
|
|
const otherLabels: any[] = [];
|
|
|
|
result.labels.forEach((label: any) => {
|
|
if (label.name === 'parentSuite') {
|
|
parentSuiteLabels.push(label);
|
|
} else {
|
|
otherLabels.push(label);
|
|
}
|
|
});
|
|
|
|
// Keep only the last parentSuite (the directory name we manually added)
|
|
if (parentSuiteLabels.length > 1) {
|
|
result.labels = [ ...otherLabels, parentSuiteLabels[parentSuiteLabels.length - 1] ];
|
|
fs.writeFileSync(filePath, JSON.stringify(result));
|
|
}
|
|
});
|
|
|
|
const generation = allure([
|
|
'generate', `${TEST_RESULTS_DIR}/allure-results`,
|
|
'--clean', '--single-file',
|
|
'--report-dir', `${TEST_RESULTS_DIR}/allure-report`
|
|
]);
|
|
|
|
return new Promise<void>((resolve, reject) => {
|
|
const generationTimeout = setTimeout(
|
|
() => reject(new Error('Could not generate Allure report: timed out after 60s')),
|
|
60_000);
|
|
|
|
// @ts-ignore
|
|
generation.on('exit', eCode => {
|
|
clearTimeout(generationTimeout);
|
|
|
|
if (eCode !== 0) {
|
|
return reject(new Error(`Could not generate Allure report: allure exited with code ${eCode}`));
|
|
}
|
|
|
|
console.log('Allure report successfully generated');
|
|
|
|
// An ugly hack to sort by test order by default in the allure report.
|
|
const content = fs.readFileSync(`${TEST_RESULTS_DIR}/allure-report/index.html`, 'utf8');
|
|
const modifiedContent = content.replace('<body>',
|
|
'<body><script>localStorage.setItem("ALLURE_REPORT_SETTINGS_SUITES", \'{"treeSorting":{"sorter":"sorter.order","ascending":true}}\')</script>'
|
|
);
|
|
|
|
fs.writeFileSync(`${TEST_RESULTS_DIR}/allure-report/index.html`, modifiedContent);
|
|
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
} as WebdriverIO.MultiremoteConfig;
|