'use strict';
const assert = require('assert');
const fixtures = require('../common/fixtures');
const fs = require('fs');
const fsPromises = fs.promises;
const path = require('path');
const events = require('events');
const os = require('os');
const { inspect } = require('util');
const { pathToFileURL } = require('url');
const { Worker } = require('worker_threads');
const { fork } = require('child_process');
const workerPath = path.join(__dirname, 'wpt/worker.js');
const wptNonTestDirs = new Set(['resources', 'support', 'tools']);
function getBrowserProperties() {
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
const release = /^\d+\.\d+\.\d+$/.test(version);
const browser = {
browser_channel: release ? 'stable' : 'experimental',
browser_version: version,
};
return browser;
}
/**
* Return one of three expected values
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
* @returns {'linux'|'mac'|'win'}
*/
function getOs() {
switch (os.type()) {
case 'Linux':
return 'linux';
case 'Darwin':
return 'mac';
case 'Windows_NT':
return 'win';
default:
throw new Error('Unsupported os.type()');
}
}
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
function sanitizeUnpairedSurrogates(str) {
return str.replace(
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
function(_, low, prefix, high) {
let output = prefix || ''; // Prefix may be undefined
const string = low || high; // Only one of these alternates can match
for (let i = 0; i < string.length; i++) {
output += codeUnitStr(string[i]);
}
return output;
});
}
function codeUnitStr(char) {
return 'U+' + char.charCodeAt(0).toString(16);
}
class ReportResult {
#startTime;
constructor(name) {
this.test = name;
this.status = 'OK';
this.subtests = [];
this.#startTime = Date.now();
}
addSubtest(name, status, message) {
const subtest = {
status,
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
name: sanitizeUnpairedSurrogates(name),
};
if (message) {
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
subtest.message = sanitizeUnpairedSurrogates(message);
}
this.subtests.push(subtest);
return subtest;
}
finish(status) {
this.status = status ?? 'OK';
this.duration = Date.now() - this.#startTime;
}
}
// Generates a report that can be uploaded to wpt.fyi.
// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
// for more details.
class WPTReport {
constructor(testPath, suffix = '') {
this.filename = `report-${testPath.replaceAll('/', '-')}${suffix}.json`;
this.filepath = path.join(__dirname, `../../out/wpt/${this.filename}`);
/** @type {Map<string, ReportResult>} */
this.results = new Map();
this.time_start = Date.now();
}
/**
* Get or create a ReportResult for a test spec.
* @param {WPTTestSpec} spec
* @returns {ReportResult}
*/
getResult(spec) {
const name = `/${spec.getTestPath()}`;
if (this.results.has(name)) {
return this.results.get(name);
}
const result = new ReportResult(name);
this.results.set(name, result);
return result;
}
/**
* @returns {void}
*/
write() {
this.time_end = Date.now();
const results = Array.from(this.results.values());
/**
* Return required and some optional properties
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
*/
this.run_info = {
product: 'node.js',
...getBrowserProperties(),
revision: process.env.WPT_REVISION || 'unknown',
os: getOs(),
};
fs.writeFileSync(this.filepath, JSON.stringify({
time_start: this.time_start,
time_end: this.time_end,
run_info: this.run_info,
results: results,
}));
}
}
// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
// TODO: get rid of this half-baked harness in favor of the one
// pulled from WPT
const harnessMock = {
test: (fn, desc) => {
try {
fn();
} catch (err) {
console.error(`In ${desc}:`);
throw err;
}
},
assert_equals: assert.strictEqual,
assert_true: (value, message) => assert.strictEqual(value, true, message),
assert_false: (value, message) => assert.strictEqual(value, false, message),
assert_throws: (code, func, desc) => {
assert.throws(func, function(err) {
return typeof err === 'object' &&
'name' in err &&
err.name.startsWith(code.name);
}, desc);
},
assert_array_equals: assert.deepStrictEqual,
assert_unreached(desc) {
assert.fail(`Reached unreachable code: ${desc}`);
},
};
class ResourceLoader {
constructor(path) {
this.path = path;
}
toRealFilePath(from, url) {
// We need to patch this to load the WebIDL parser
url = url.replace(
'/resources/WebIDLParser.js',
'/resources/webidl2/lib/webidl2.js',
);
const base = path.dirname(from);
return url.startsWith('/') ?
fixtures.path('wpt', url) :
fixtures.path('wpt', base, url);
}
/**
* Map a URL that a test would have fetched from the WPT server (an
* absolute path, or a path relative to the test file) to a file: URL
* into the fixtures directory. URLs that already have a scheme (data:,
* blob:, http:, ...) are returned unchanged.
* @param {string} from the path of the file loading this resource,
* relative to the WPT folder.
* @param {string|URL} url the url of the resource being loaded.
* @returns {string}
*/
mapServerURL(from, url) {
url = `${url}`;
if (/^[a-zA-Z][a-zA-Z0-9+.-]*:|^\/\//.test(url)) {
return url;
}
return pathToFileURL(this.toRealFilePath(from, url)).href;
}
/**
* Load a resource in test/fixtures/wpt specified with a URL
* @param {string} from the path of the file loading this resource,
* relative to the WPT folder.
* @param {string} url the url of the resource being loaded.
* @returns {string}
*/
read(from, url) {
const file = this.toRealFilePath(from, url);
return fs.readFileSync(file, 'utf8');
}
/**
* Load a resource in test/fixtures/wpt specified with a URL
* @param {string} from the path of the file loading this resource,
* relative to the WPT folder.
* @param {string} url the url of the resource being loaded.
* @returns {Promise<{
* ok: string,
* arrayBuffer: function(): Buffer,
* json: function(): object,
* text: function(): string,
* }>}
*/
async readAsFetch(from, url) {
const file = this.toRealFilePath(from, url);
const data = await fsPromises.readFile(file);
return {
ok: true,
arrayBuffer() { return data.buffer; },
bytes() { return new Uint8Array(data); },
json() { return JSON.parse(data.toString()); },
text() { return data.toString(); },
};
}
}
class StatusRule {
constructor(key, value, pattern) {
this.key = key;
this.requires = value.requires || [];
this.fail = value.fail;
this.skip = value.skip;
this.skipTests = value.skipTests;
if (pattern) {
this.pattern = this.transformPattern(pattern);
}
// TODO(joyeecheung): implement this
this.scope = value.scope;
this.comment = value.comment;
}
/**
* Transform a filename pattern into a RegExp
* @param {string} pattern
* @returns {RegExp}
*/
transformPattern(pattern) {
const result = path.normalize(pattern).replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&');
return new RegExp(result.replace('*', '.*'));
}
}
class StatusRuleSet {
constructor() {
// We use two sets of rules to speed up matching
this.exactMatch = {};
this.patternMatch = [];
}
/**
* @param {object} rules
*/
addRules(rules) {
for (const key of Object.keys(rules)) {
if (key.includes('*')) {
this.patternMatch.push(new StatusRule(key, rules[key], key));
} else {
const normalizedPath = path.normalize(key);
this.exactMatch[normalizedPath] = new StatusRule(key, rules[key]);
}
}
}
match(file) {
const result = [];
const exact = this.exactMatch[file];
if (exact) {
result.push(exact);
}
for (const item of this.patternMatch) {
if (item.pattern.test(file)) {
result.push(item);
}
}
return result;
}
}
// A specification of WPT test
class WPTTestSpec {
#content;
/**
* @param {string} mod name of the WPT module, e.g.
* 'html/webappapis/microtask-queuing'
* @param {string} filename path of the test, relative to mod, e.g.
* 'test.any.js'
* @param {StatusRule[]} rules
* @param {string} variant test file variant
* @param {'window'|'dedicatedworker'} [globalScope] generated test global
*/
constructor(mod, filename, rules, variant = '', globalScope) {
this.module = mod;
this.filename = filename;
this.variant = variant;
this.globalScope = globalScope;
this.rules = [...new Set(rules)];
this.requires = new Set();
this.failedTests = [];
this.flakyTests = [];
this.skipReasons = [];
this.skippedTests = [];
for (const item of this.rules) {
if (item.requires.length) {
for (const req of item.requires) {
this.requires.add(req);
}
}
if (Array.isArray(item.fail?.expected)) {
this.failedTests.push(...item.fail.expected);
}
if (Array.isArray(item.fail?.flaky)) {
this.failedTests.push(...item.fail.flaky);
this.flakyTests.push(...item.fail.flaky);
}
if (item.skip) {
this.skipReasons.push(item.skip);
}
if (Array.isArray(item.skipTests)) {
this.skippedTests.push(...item.skipTests);
}
}
this.failedTests = [...new Set(this.failedTests)];
this.flakyTests = [...new Set(this.flakyTests)];
this.skipReasons = [...new Set(this.skipReasons)];
}
/**
* @param {string} mod
* @param {string} filename
* @param {StatusRule[]} rules
* @param {(spec: WPTTestSpec) => StatusRule[]} [getAdditionalRules]
* @returns {WPTTestSpec[]}
*/
static from(mod, filename, rules, getAdditionalRules) {
const spec = new WPTTestSpec(mod, filename, rules);
const meta = spec.getMeta();
const variants = meta.variant || [''];
const createSpec = (variant, globalScope) => {
let result = new WPTTestSpec(mod, filename, rules, variant, globalScope);
const additionalRules = getAdditionalRules?.(result) || [];
if (additionalRules.length > 0) {
result = new WPTTestSpec(
mod,
filename,
[...new Set([...rules, ...additionalRules])],
variant,
globalScope,
);
}
return result;
};
if (!spec.isAnyTest()) {
return variants.map((variant) => createSpec(variant));
}
const requestedGlobals = meta.global ?
meta.global.split(',').map((item) => item.trim()) :
['window', 'dedicatedworker'];
const supportedGlobals = new Set();
for (const globalScope of requestedGlobals) {
if (globalScope === 'window' || globalScope === 'dedicatedworker') {
supportedGlobals.add(globalScope);
} else if (globalScope === 'worker') {
supportedGlobals.add('dedicatedworker');
}
}
return ['window', 'dedicatedworker']
.filter((globalScope) => supportedGlobals.has(globalScope))
.flatMap((globalScope) => variants.map(
(variant) => createSpec(variant, globalScope)));
}
/**
* @returns {boolean}
*/
isAnyTest() {
return /\.any\.js$/.test(this.filename);
}
/**
* @returns {boolean}
*/
isWebWorkerTest() {
return /\.worker\.js$/.test(this.filename) ||
this.globalScope === 'dedicatedworker';
}
/**
* Check if a subtest should be skipped by name.
* @param {string} name
* @returns {boolean}
*/
isSkippedTest(name) {
for (const matcher of this.skippedTests) {
if (typeof matcher === 'string') {
if (name === matcher) return true;
} else if (matcher.test(name)) {
return true;
}
}
return false;
}
getRelativePath() {
return path.join(this.module, this.filename);
}
getStatusKey() {
if (this.globalScope === 'dedicatedworker') {
return this.filename.replace(/\.any\.js$/, '.any.worker.html');
}
if (this.globalScope === 'window') {
return this.filename.replace(/\.any\.js$/, '.any.html');
}
return this.filename;
}
getTestPath() {
let testPath = path.join(this.module, this.getStatusKey());
testPath = testPath.replace(/\.js$/, '.html');
return `${testPath.split(path.sep).join('/')}${this.variant}`;
}
/**
* Whether a command line argument selects this spec. Accepts the source file
* name, which selects every global and variant generated from it, or a test
* path as printed alongside the results. Omitting its query selects all
* variants of that global.
* @param {string} arg
* @returns {boolean}
*/
isSelectedBy(arg) {
const testPath = this.getTestPath();
if (arg === testPath || arg === testPath.split('?')[0]) {
return true;
}
const [filename, variant = ''] = arg.split('?');
// Spec filenames are relative paths, so they use the platform separator,
// while the argument is written with forward slashes.
return filename.split(path.sep).join('/') ===
this.filename.split(path.sep).join('/') &&
(!variant || this.variant.substring(1) === variant);
}
getAbsolutePath() {
return fixtures.path('wpt', this.getRelativePath());
}
/**
* @returns {string}
*/
getContent() {
this.#content ||= fs.readFileSync(this.getAbsolutePath(), 'utf8');
return this.#content;
}
/**
* @returns {{ script?: string[]; variant?: string[]; [key: string]: string }} parsed META tags of a spec file
*/
getMeta() {
// Like upstream, tolerate missing whitespace around "META:".
// Refs: https://github.com/web-platform-tests/wpt/blob/master/tools/manifest/sourcefile.py
const matches = this.getContent().match(/\/\/\s*META:\s*.+/g);
if (!matches) {
return {};
}
const result = {};
for (const match of matches) {
const parts = match.match(/\/\/\s*META:\s*([^=]+?)=(.+)/);
const key = parts[1];
const value = parts[2];
if (key === 'script' || key === 'variant') {
if (result[key]) {
result[key].push(value);
} else {
result[key] = [value];
}
} else {
result[key] = value;
}
}
return result;
}
}
const kIntlRequirement = {
none: 0,
small: 1,
full: 2,
// TODO(joyeecheung): we may need to deal with --with-intl=system-icu
};
class BuildRequirement {
constructor() {
this.currentIntl = kIntlRequirement.none;
if (process.config.variables.v8_enable_i18n_support === 0) {
this.currentIntl = kIntlRequirement.none;
return;
}
// i18n enabled
if (process.config.variables.icu_small) {
this.currentIntl = kIntlRequirement.small;
} else {
this.currentIntl = kIntlRequirement.full;
}
// Not using common.hasCrypto because of the global leak checks
this.hasCrypto = Boolean(process.versions.openssl) &&
!process.env.NODE_SKIP_CRYPTO;
// Not using common.hasInspector because of the global leak checks
this.hasInspector = Boolean(process.features.inspector);
}
/**
* @param {Set} requires
* @returns {string|false} The config that the build is lacking, or false
*/
isLacking(requires) {
const current = this.currentIntl;
if (requires.has('full-icu') && current !== kIntlRequirement.full) {
return 'full-icu';
}
if (requires.has('small-icu') && current < kIntlRequirement.small) {
return 'small-icu';
}
if (requires.has('crypto') && !this.hasCrypto) {
return 'crypto';
}
if (requires.has('inspector') && !this.hasInspector) {
return 'inspector';
}
return false;
}
}
const buildRequirements = new BuildRequirement();
class StatusLoader {
/**
* @param {string} path relative path of the WPT subset
*/
constructor(path) {
this.path = path;
this.rules = new StatusRuleSet();
/** @type {WPTTestSpec[]} */
this.specs = [];
}
/**
* Grep for all .*.js file recursively in a directory.
* @param {string} dir
* @returns {any[]}
*/
grep(dir) {
let result = [];
const list = fs.readdirSync(dir);
for (const file of list) {
const filepath = path.join(dir, file);
const stat = fs.statSync(filepath);
if (stat.isDirectory()) {
if (wptNonTestDirs.has(file)) {
continue;
}
const list = this.grep(filepath);
result = result.concat(list);
} else {
if (!(/\.\w+\.js$/.test(filepath))) {
continue;
}
result.push(filepath);
}
}
return result;
}
load(source) {
const dir = path.join(__dirname, '..', 'wpt');
let result;
try {
this.statusFile = `${this.path}.json`;
const jsonFile = path.join(dir, 'status', this.statusFile);
result = JSON.parse(fs.readFileSync(jsonFile, 'utf8'));
} catch (err) {
if (err?.code !== 'ENOENT') throw err;
this.statusFile = `${this.path}.cjs`;
result = require(path.join(dir, 'status', this.statusFile));
}
this.rules.addRules(result);
const subDir = fixtures.path('wpt', this.path);
const list = source === undefined ? this.grep(subDir) : [path.join(subDir, source)];
for (const file of list) {
const relativePath = path.relative(subDir, file);
const match = this.rules.match(relativePath);
this.specs.push(...WPTTestSpec.from(
this.path,
relativePath,
match,
(spec) => [
...this.rules.match(spec.getStatusKey()),
...this.rules.match(`${spec.getStatusKey()}${spec.variant}`),
],
));
}
}
}
const kPass = 'pass';
const kFail = 'fail';
const kSkip = 'skip';
const kTimeout = 'timeout';
const kIncomplete = 'incomplete';
const kUncaught = 'uncaught';
const NODE_UNCAUGHT = 100;
const limit = (concurrency) => {
let running = 0;
const queue = [];
const execute = async ({ fn, resolve, reject }) => {
running++;
try {
resolve(await fn());
} catch (err) {
reject(err);
} finally {
running--;
if (queue.length > 0) {
execute(queue.shift());
}
}
};
return (fn) => new Promise((resolve, reject) => {
const task = { fn, resolve, reject };
if (running < concurrency) {
execute(task);
} else {
queue.push(task);
}
});
};
function isUnexpectedPass(spec, name) {
return spec.failedTests.includes(name) && !spec.flakyTests.includes(name);
}
function getUnexpectedPasses(queue, results) {
const unexpectedPasses = [];
for (const spec of queue) {
const key = spec.getStatusKey();
if (results[key]?.skip) {
continue;
}
for (const expectedToFail of spec.failedTests) {
if (isUnexpectedPass(spec, expectedToFail) &&
!results[key]?.fail?.expected?.includes(expectedToFail)) {
unexpectedPasses.push(`${key}:${expectedToFail}`);
}
}
}
return unexpectedPasses;
}
function getHarnessErrorName(harnessStatus) {
if (typeof harnessStatus.stack === 'string') {
const name = harnessStatus.stack.split('\n', 1)[0];
if (name) {
return name;
}
}
return harnessStatus.message || 'WPT test harness error';
}
/**
* @typedef {object} SpecHandlers
* @property {(message: object) => void} message Handles a message from the spec.
* @property {(failure: { name: string, message: string, stack: string })
* => boolean} failure Reports a spec that died without completing. Returns
* false when the spec had already finished and the failure was ignored.
*/
/**
* @typedef {object} SpecHandle
* @property {() => void} kill Forces the spec to stop running.
* @property {Promise<unknown>} finished Settles once the spec has stopped.
*/
/**
* Run a spec on a worker thread.
* @param {string[]} execArgv
* @param {object} workerData
* @param {SpecHandlers} handlers
* @returns {SpecHandle}
*/
function runSpecOnThread(execArgv, workerData, handlers) {
const worker = new Worker(workerPath, { execArgv, workerData });
worker.on('message', handlers.message);
worker.on('error', (err) => handlers.failure({
name: `${err}`,
message: err.message,
stack: inspect(err),
}));
return {
kill: () => worker.terminate(),
finished: events.once(worker, 'exit').catch(() => {}),
};
}
/**
* Run a spec in a child process, so that a spec crashing the process only
* takes down its own run and the runner can attribute the crash to it.
* @param {string[]} execArgv
* @param {object} workerData
* @param {SpecHandlers} handlers
* @returns {SpecHandle}
*/
function runSpecInProcess(execArgv, workerData, handlers) {
const forwardStderr = execArgv.some(
(flag) => flag === '--inspect-brk' || flag.startsWith('--inspect-brk='));
const child = fork(workerPath, {
execArgv,
// Status files may skip subtests by regular expression, which JSON
// serialization would not preserve.
serialization: 'advanced',
stdio: ['ignore', 'inherit', 'pipe', 'ipc'],
});
child.send(workerData);
let stderr = '';
child.stderr.setEncoding('utf8');
child.stderr.on('data', (chunk) => {
stderr += chunk;
if (forwardStderr) {
process.stderr.write(chunk);
}
});
child.on('message', (message) => {
// The spec reports uncaught errors itself so that they are named the same
// way as they would be on the worker thread backend.
if (message.type === 'uncaught') {
handlers.failure(message.error);
return;
}
handlers.message(message);
});
child.on('error', (err) => handlers.failure({
name: `${err}`,
message: err.message,
stack: inspect(err),
}));
// `close` rather than `exit` so that everything the process wrote to stderr
// on its way out is part of the reported failure.
child.on('close', (code, signal) => {
const name = signal ?
`Test process was killed by signal ${signal}` :
`Test process exited with code ${code}`;
if (!handlers.failure({ name, message: name, stack: stderr }) &&
stderr && !forwardStderr) {
process.stderr.write(stderr);
}
});
return {
kill: () => child.kill('SIGKILL'),
finished: events.once(child, 'close').catch(() => {}),
};
}
const backends = {
__proto__: null,
thread: runSpecOnThread,
process: runSpecInProcess,
};
class WPTRunner {
constructor(path, options = {}) {
let {
concurrency = os.availableParallelism() - 1 || 1,
backend = 'thread',
} = options;
if (!Number.isInteger(concurrency) || concurrency < 1) {
throw new TypeError('WPT concurrency must be a positive integer');
}
if (process.env.NODE_TEST_WPT !== undefined) {
this.managed = JSON.parse(process.env.NODE_TEST_WPT);
if (!this.managed || !['list', 'run'].includes(this.managed.mode) ||
(this.managed.mode === 'run' &&
(['source', 'key'].some((key) =>
typeof this.managed[key] !== 'string' || !this.managed[key]) ||
(this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) {
throw new Error('Invalid WPT runner configuration');
}
}
this.isListing = this.managed?.mode === 'list';
this.serial = options.concurrency === 1;
if (this.managed?.mode === 'run') concurrency = 1;
// RISC-V has very limited virtual address space in the currently common
// sv39 mode, in which we can only create a very limited number of wasm
// memories(27 from a fresh node repl). Limit the concurrency to avoid
// creating too many wasm memories that would fail.
if (process.arch === 'riscv64' || process.arch === 'riscv32') {
concurrency = Math.min(10, concurrency);
}
this.inspectBrk = process.env.WPT_INSPECT !== undefined;
// The override exists so that every suite can be run either way without
// editing the drivers, which is how the two backends are kept compatible.
if (this.inspectBrk) {
backend = 'process';
} else {
backend = process.env.WPT_BACKEND || backend;
}
this.runSpec = backends[backend];
if (this.runSpec === undefined) {
throw new Error(`Invalid WPT backend ${backend}, expected one of ` +
`${Object.keys(backends).join(', ')}`);
}
this.path = path;
this.resource = new ResourceLoader(path);
this.concurrency = concurrency;
// Since we need to prepare the Web Worker APIs
// in the harness that runs on all WPT workers,
// we enable the API globally. This has no practical
// effect on the non-web-worker tests, however.
this.flags = ['--experimental-web-worker'];
if (this.inspectBrk) {
this.flags.push('--inspect-brk=0');
}
this.globalThisInitScripts = [];
this.initScript = null;
this.status = new StatusLoader(path);
this.status.load(this.managed?.mode === 'run' ? this.managed.source : undefined);
this.statusFile = this.status.statusFile;
this.specs = new Set(this.status.specs);
this.results = {};
this.inProgress = new Set();
this.handles = new Map();
this.unexpectedFailures = [];
this.skippedSpecCount = 0;
this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 };
if (process.env.WPT_REPORT != null && !this.isListing) {
const suffix = this.managed ? `-${process.env.TEST_SERIAL_ID || process.pid}` : '';
this.report = new WPTReport(path, suffix);
}
}
/**
* Sets the Node.js flags passed to the worker.
* @param {string[]} flags
*/
setFlags(flags) {
this.flags = this.flags.concat(flags);
}
/**
* Sets a script to be run in the worker before executing the tests.
* @param {string} script
*/
setInitScript(script) {
this.initScript = script;
}
/**
* Set the scripts modifier for each script.
* @param {(meta: { code: string, filename: string }) => void} modifier
*/
setScriptModifier(modifier) {
this.scriptsModifier = modifier;
}
/**
* @param {WPTTestSpec} spec
* @returns {string}
*/
fullInitScript(spec) {
const url = new URL(`/${spec.getTestPath()}`, 'http://wpt');
const title = spec.getMeta().title;
let { initScript } = this;
initScript = `${initScript}\n\n//===\nglobalThis.location = new URL("${url.href}");`;
if (title) {
initScript = `${initScript}\n\n//===\nglobalThis.META_TITLE = "${title}";`;
}
if (this.globalThisInitScripts.length === null) {
return initScript;
}
if (spec.isWebWorkerTest()) {
return initScript;
}
const globalThisInitScript = this.globalThisInitScripts.join('\n\n//===\n');
if (initScript === null) {
return globalThisInitScript;
}
return `${globalThisInitScript}\n\n//===\n${initScript}`;
}
/**
* Pretend the runner is run in `name`'s environment (globalThis).
* @param {'Window'} name
* @see {@link https://github.com/nodejs/node/blob/24673ace8ae196bd1c6d4676507d6e8c94cf0b90/test/fixtures/wpt/resources/idlharness.js#L654-L671}
*/
pretendGlobalThisAs(name) {
switch (name) {
case 'Window': {
this.globalThisInitScripts.push('globalThis.Window = Object.getPrototypeOf(globalThis).constructor;');
break;
}
// TODO(XadillaX): implement `ServiceWorkerGlobalScope`,
// `DedicateWorkerGlobalScope`, etc.
//
// e.g. `ServiceWorkerGlobalScope` should implement dummy
// `addEventListener` and so on.
default: throw new Error(`Invalid globalThis type ${name}.`);
}
}
// TODO(joyeecheung): work with the upstream to port more tests in .html
// to .js.
async runJsTests() {
if (this.isListing) {
const groups = new Map();
for (const spec of this.specs) {
const key = spec.getStatusKey();
if (!groups.has(key)) groups.set(key, []);
groups.get(key).push(spec);
}
const tests = [...groups.values()].flatMap((specs) => {
// Strict expected failures are checked across all query variants.
// Keep that group together; all other variants are independent tasks.
const grouped = specs.some((spec) =>
spec.failedTests.some((name) => isUnexpectedPass(spec, name)));
return (grouped ? [specs[0]] : specs).map((spec) => {
const selector = grouped ? spec.getTestPath().split('?')[0] : spec.getTestPath();
return {
source: spec.filename.split(path.sep).join('/'),
key: spec.getStatusKey(),
...(grouped ? {} : { variant: spec.variant }),
id: selector.slice(this.path.length + 1),
selector,
};
});
});
console.log(`NODE_TEST_WPT_MANIFEST:${JSON.stringify({
version: 1,
serial: this.serial,
tests: tests.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)),
})}`);
return;
}
const queue = this.buildQueue();
if (this.managed && queue.length === 0) {
console.log('1..0 # SKIP No runnable WPT variants');
}
const run = limit(this.concurrency);
const jobs = [];
for (const spec of queue) {
const content = spec.getContent();
const meta = spec.getMeta(content);
const absolutePath = spec.getAbsolutePath();
const relativePath = spec.getRelativePath();
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
// *.worker.js tests are dedicated worker tests by definition. Each
// dedicated worker variant generated from a multi-global (*.any.js)
// test also runs inside an actual Web Worker. Refs:
// https://web-platform-tests.org/writing-tests/testharness.html#multi-global-tests
const isAnyTest = spec.isAnyTest();
const isWebWorkerTest = spec.isWebWorkerTest();
// Scripts specified with the `// META: script=` header. For tests
// that run inside a Web Worker they are imported by the worker
// instead.
const scriptsToRun = isWebWorkerTest ? [] : meta.script?.map((script) => {
const obj = {
filename: this.resource.toRealFilePath(relativePath, script),
code: this.resource.read(relativePath, script),
};
this.scriptsModifier?.(obj);
return obj;
}) ?? [];
// The actual test, including tests imported by a Web Worker.
const testScript = { code: content, filename: absolutePath };
this.scriptsModifier?.(testScript);
if (!isWebWorkerTest) scriptsToRun.push(testScript);
jobs.push(run(async () => {
this.inProgress.add(spec);
const reportResult = this.report?.getResult(spec);
const handle = this.runSpec(this.flags, {
testRelativePath: relativePath,
wptRunner: __filename,
wptPath: this.path,
initScript: this.fullInitScript(spec),
harness: {
code: fs.readFileSync(harnessPath, 'utf8'),
filename: harnessPath,
},
scriptsToRun,
// Set when the test runs inside an actual Web Worker.
webWorker: isWebWorkerTest ? {
path: absolutePath,
modifiedScript: testScript.code !== content || testScript.filename !== absolutePath ?
testScript : undefined,
isAnyTest,
initScript: this.initScript,
title: meta.title,
variant: spec.variant,
scripts: meta.script?.map(
(script) => this.resource.toRealFilePath(relativePath, script),
) ?? [],
skippedTests: spec.skippedTests,
} : undefined,
needsGc: !!meta.script?.find((script) => script === '/common/gc.js'),
skippedTests: spec.skippedTests,
}, {
message: (message) => {
switch (message.type) {
case 'result':
return this.resultCallback(spec, message.result, reportResult);
case 'skip':
return this.skipTest(spec, { name: message.name }, reportResult);
case 'completion':
return this.completionCallback(spec, message.status, reportResult);
default:
throw new Error(`Unexpected message from spec runner: ${message.type}`);
}
},
failure: (failure) => {
if (!this.inProgress.has(spec)) {
// The test is already finished. Ignore anything that happens
// after it, including the runner terminating it itself.
return false;
}
// Generate a subtest failure for visibility.
// No need to record this synthetic failure with wpt.fyi.
this.fail(spec, { status: NODE_UNCAUGHT, ...failure }, kUncaught);
// Mark the whole test as failed in wpt.fyi report.
reportResult?.finish('ERROR');
this.inProgress.delete(spec);
this.report?.write();
return true;
},
});
this.handles.set(spec, handle);
await handle.finished;
}));
}
process.on('exit', () => {
for (const spec of this.inProgress) {
// No need to record this synthetic failure with wpt.fyi.
this.fail(spec, { name: 'Incomplete' }, kIncomplete);
// Mark the whole test as failed in wpt.fyi report.
const reportResult = this.report?.getResult(spec);
reportResult?.finish('ERROR');
}
inspect.defaultOptions.depth = Infinity;
// Sorts the rules to have consistent output
console.log('');
console.log(JSON.stringify(Object.keys(this.results).sort().reduce(
(obj, key) => {
obj[key] = this.results[key];
return obj;
},
{},
), null, 2));
const failures = [];
let expectedFailures = 0;
for (const [key, item] of Object.entries(this.results)) {
if (item.fail?.unexpected) {
failures.push(key);
}
if (item.fail?.expected) {
expectedFailures++;
}
}
const unexpectedPasses = getUnexpectedPasses(queue, this.results);
// Write the report on clean exit. The report is also written
// incrementally after each spec completes (see completionCallback)
// so that results survive if the process is killed.
this.report?.write();
const p = (n, word, suffix = 's') => `${n} ${word}${n === 1 ? '' : suffix}`;
const ran = queue.length;
const skipped = this.skippedSpecCount;
const total = ran + skipped;
const passed = ran - expectedFailures - failures.length;
const { subtestCounts } = this;
console.log('');
console.log(`Files: ${ran}/${total} ran, ${passed} passed,`,
`${skipped} skipped, ${p(expectedFailures, 'expected failure')},`,
`${p(failures.length, 'unexpected failure')},`,
`${p(unexpectedPasses.length, 'unexpected pass', 'es')}`);
console.log(`Subtests: ${subtestCounts.passed} passed,`,
`${subtestCounts.skipped} skipped, ${p(subtestCounts.expectedFailures, 'expected failure')},`,
`${p(subtestCounts.failed, 'unexpected failure')},`,
`${p(subtestCounts.unexpectedPasses, 'unexpected pass', 'es')}`);
if (failures.length > 0) {
const file = path.join('test', 'wpt', 'status', this.statusFile);
throw new Error(
`Found ${failures.length} unexpected failures. ` +
`Consider updating ${file} for these files:\n${failures.join('\n')}`);
}
if (unexpectedPasses.length > 0) {
const file = path.join('test', 'wpt', 'status', this.statusFile);
throw new Error(
`Found ${unexpectedPasses.length} unexpected passes. ` +
`Consider updating ${file} for these files:\n${unexpectedPasses.join('\n')}`);
}
});
// Promises do not keep the event loop alive. Keep a referenced handle
// until every queued spec has run, including the gap between terminating
// one spec runner and receiving its exit event.
const keepAlive = setInterval(() => {}, 2 ** 31 - 1);
try {
const outcomes = await Promise.allSettled(jobs);
const errors = outcomes
.filter((outcome) => outcome.status === 'rejected')
.map((outcome) => outcome.reason);
if (errors.length === 1) {
throw errors[0];
}
if (errors.length > 1) {
throw new AggregateError(errors, 'Multiple WPT spec runners failed');
}
} finally {
clearInterval(keepAlive);
}
}
// Map WPT test status to strings
getTestStatus(status) {
switch (status) {
case 1:
return kFail;
case 2:
return kTimeout;
case 3:
return kIncomplete;
case NODE_UNCAUGHT:
return kUncaught;
default:
return kPass;
}
}
/**
* Report the status of each specific test case (there could be multiple
* in one test file).
* @param {WPTTestSpec} spec
* @param {Test} test The Test object returned by WPT harness
* @param {ReportResult} reportResult The report result object
*/
resultCallback(spec, test, reportResult) {
const status = this.getTestStatus(test.status);
if (status !== kPass) {
this.fail(spec, test, status, reportResult);
} else {
this.succeed(spec, test, status, reportResult);
}
}
/**
* Report the status of each WPT test (one per file)
* @param {WPTTestSpec} spec
* @param {object} harnessStatus - The status object returned by WPT harness.
* @param {ReportResult} reportResult The report result object
*/
completionCallback(spec, harnessStatus, reportResult) {
const status = this.getTestStatus(harnessStatus.status);
// Treat it like a test case failure
if (status === kTimeout) {
// No need to record this synthetic failure with wpt.fyi.
this.fail(spec, { name: 'WPT testharness timeout' }, kTimeout);
// Mark the whole test as TIMEOUT in wpt.fyi report.
reportResult?.finish('TIMEOUT');
} else if (status !== kPass) {
// No need to record this synthetic failure with wpt.fyi.
this.fail(spec, {
status: status,
name: getHarnessErrorName(harnessStatus),
message: harnessStatus.message,
stack: harnessStatus.stack,
}, status);
// Mark the whole test as ERROR in wpt.fyi report.
reportResult?.finish('ERROR');
} else {
reportResult?.finish();
}
this.inProgress.delete(spec);
// Write report incrementally so results survive even if the process
// is killed before the exit handler runs.
this.report?.write();
// Always force termination of the spec runner. Some tests allocate
// resources that would otherwise keep it alive.
this.handles.get(spec).kill();
}
addTestResult(spec, item) {
const key = spec.getStatusKey();
let result = this.results[key];
result ||= this.results[key] = {};
if (item.status === kSkip) {
if (item.name) {
// Subtest-level skip: { filename: { skipTests: [ ... ] } }
result.skipTests ||= [];
if (!result.skipTests.includes(item.name)) {
result.skipTests.push(item.name);
}
} else {
// File-level skip: { filename: { skip: 'reason' } }
result[kSkip] = item.reason;
}
} else {
// { filename: { fail: { expected: [ ... ],
// unexpected: [ ... ] } }}
result[item.status] ||= {};
const key = item.expected ? 'expected' : 'unexpected';
result[item.status][key] ||= [];
const hasName = result[item.status][key].includes(item.name);
if (!hasName) {
result[item.status][key].push(item.name);
}
}
}
succeed(spec, test, status, reportResult) {
const unexpectedPass = isUnexpectedPass(spec, test.name);
if (unexpectedPass) {
console.log(`[UNEXPECTED_PASS][${status.toUpperCase()}] ${spec.getTestPath()}: ${test.name}`);
this.subtestCounts.unexpectedPasses++;
} else {
console.log(`[${status.toUpperCase()}] ${spec.getTestPath()}: ${test.name}`);
this.subtestCounts.passed++;
}
reportResult?.addSubtest(test.name, 'PASS');
}
skipTest(spec, test, reportResult) {
console.log(`[SKIP] ${spec.getTestPath()}: ${test.name}`);
reportResult?.addSubtest(test.name, 'NOTRUN');
this.subtestCounts.skipped++;
this.addTestResult(spec, {
name: test.name,
status: kSkip,
});
}
fail(spec, test, status, reportResult) {
const expected = spec.failedTests.includes(test.name);
if (expected) {
console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${spec.getTestPath()}: ${test.name}`);
} else {
console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${spec.getTestPath()}: ${test.name}`);
}
if (status === kFail || status === kUncaught) {
console.log(test.message);
console.log(test.stack);
}
const command = [
process.execPath,
...process.execArgv,
require.main?.filename,
`'${spec.getTestPath()}'`,
].join(' ');
console.log(`Command: ${command}\n`);
reportResult?.addSubtest(test.name, 'FAIL', test.message);
if (expected) {
this.subtestCounts.expectedFailures++;
} else {
this.subtestCounts.failed++;
}
this.addTestResult(spec, {
name: test.name,
expected,
status: kFail,
reason: test.message || status,
});
}
skip(spec, reasons) {
const joinedReasons = reasons.join('; ');
console.log(`[SKIPPED] ${spec.getTestPath()}: ${joinedReasons}`);
this.skippedSpecCount++;
this.addTestResult(spec, {
status: kSkip,
reason: joinedReasons,
});
}
buildQueue() {
const queue = [];
this.skippedSpecCount = 0;
const key = this.managed?.mode === 'run' ? this.managed.key : undefined;
const variant = this.managed?.variant;
const matches = (spec) => spec.getStatusKey() === key &&
(variant === undefined || spec.variant === variant);
const arg = key === undefined ? process.argv[2] : undefined;
if (key !== undefined && ![...this.specs].some(matches)) {
throw new Error(`${key}${variant ?? ''} not found!`);
}
if (this.inspectBrk && !arg) {
throw new Error('WPT_INSPECT requires a WPT test path');
}
for (const spec of this.specs) {
if (key !== undefined && !matches(spec)) continue;
if (arg) {
if (spec.isSelectedBy(arg)) {
queue.push(spec);
}
continue;
}
if (spec.skipReasons.length > 0) {
this.skip(spec, spec.skipReasons);
continue;
}
const lackingSupport = buildRequirements.isLacking(spec.requires);
if (lackingSupport) {
this.skip(spec, [ `requires ${lackingSupport}` ]);
continue;
}
queue.push(spec);
}
// If the tests are run as `node test/wpt/test-something.js subset.any.js`,
// only `subset.any.js` (all variants and globals) will be run by the runner.
// If the tests are run as `node test/wpt/test-something.js 'subset.any.js?1-10'`,
// only the `?1-10` variant of `subset.any.js` will be run by the runner.
// A test path as printed with the results, e.g.
// `'dir/subset.any.worker.html?1-10'`, runs exactly that one.
if (arg && queue.length === 0) {
throw new Error(`${arg} not found!`);
}
if (this.inspectBrk && queue.length !== 1) {
const matches = queue.map((spec) => spec.getTestPath()).join('\n');
throw new Error(
`WPT_INSPECT requires exactly one generated WPT test path; ` +
`${arg} matched ${queue.length}:\n${matches}`,
);
}
if (this.inspectBrk && queue[0].isWebWorkerTest()) {
throw new Error(
`WPT_INSPECT does not support worker tests: ${queue[0].getTestPath()}`,
);
}
return queue;
}
}
module.exports = {
backends,
getHarnessErrorName,
getUnexpectedPasses,
harness: harnessMock,
isUnexpectedPass,
ResourceLoader,
WPTTestSpec,
WPTRunner,
};