feat: switch to generic flags passed to test pages

- flags are set for all test pages
- each isolated flag creates its own test page

Author: timmywil

bin/command.js

@@ -6,13 +6,18 @@ import { resolve } from "node:path";
 import { browsers } from "../flags/browsers.js";
 import { getPlan, listBrowsers, stopWorkers } from "../browserstack/api.js";
 import { buildBrowserFromString } from "../browserstack/buildBrowserFromString.js";
-import { modules } from "../flags/modules.js";
 import { run } from "../run.js";
 import readYAML from "../lib/readYAML.js";
 import { createTestServer } from "../createTestServer.js";
 
 const pkg = JSON.parse( await readFile( new URL( "../package.json", import.meta.url ) ) );
 
+function parseFlags( flags ) {
+	return Object.keys( flags ?? [] )
+		.flatMap( ( key ) => flags[ key ]
+		.map( ( value ) => `${ key }=${ value }` ) );
+}
+
 async function parseMiddleware( config, argv ) {
 	const middleware = await Promise.all( [
 		...( config.middleware ?? [] ),
@@ -38,14 +43,17 @@ yargs( process.argv.slice( 2 ) )
 				description: "Path to a YAML configuration file. " +
 					"Use this to avoid passing options via the command line."
 			} )
-			.option( "module", {
-				alias: "m",
+			.option( "flag", {
+				alias: "f",
 				type: "array",
-				choices: modules,
-				description:
-					"Run tests for a specific module. " +
-					"Pass multiple modules by repeating the option. " +
-					"Defaults to all modules."
+				description: "Add a universal flag to be added as a query parameter " +
+					"to the test URL for all test pages."
+			} )
+			.option( "isolated-flag", {
+				alias: "i",
+				type: "array",
+				description: "Add an isolated flag to be added as a query parameter " +
+					"to the test URL for each. Each isolated flag creates a new test page."
 			} )
 			.option( "browser", {
 				alias: "b",
@@ -71,12 +79,6 @@ yargs( process.argv.slice( 2 ) )
 					"Run tests in headless mode. Cannot be used with --debug or --browserstack.",
 				conflicts: [ "debug", "browserstack" ]
 			} )
-			.option( "esm", {
-				alias: "esmodules",
-				type: "boolean",
-				description: "Run tests using jQuery's source, " +
-					"which is written with ECMAScript Modules."
-			} )
 			.option( "concurrency", {
 				type: "number",
 				description:
@@ -108,11 +110,6 @@ yargs( process.argv.slice( 2 ) )
 				type: "boolean",
 				description: "Log additional information."
 			} )
-			.option( "isolate", {
-				type: "boolean",
-				description: "Run each module by itself in the test page. " +
-					"This can extend testing time."
-			} )
 			.option( "browserstack", {
 				type: "array",
 				description:
@@ -135,8 +132,18 @@ yargs( process.argv.slice( 2 ) )
 		handler: async( { configFile, ...argv } ) => {
 			console.log( "Running tests..." );
 			const config = await readYAML( configFile );
+			const flag = [
+				...parseFlags( config.flags ),
+				...( config.flag ?? [] ),
+				...( argv.flag ?? [] )
+			];
+			const isolatedFlag = [
+				...parseFlags( config.isolatedFlags ),
+				...( config.isolatedFlag ?? [] ),
+				...( argv.isolatedFlag ?? [] )
+			];
 			const middleware = await parseMiddleware( config, argv );
-			return run( { ...config, ...argv, middleware } );
+			return run( { ...config, ...argv, flag, isolatedFlag, middleware } );
 		}
 	} )
 	.command( {
@@ -170,6 +177,7 @@ yargs( process.argv.slice( 2 ) )
 			} );
 		},
 		handler: async( { configFile, quiet, ...argv } ) => {
+			console.log( "Starting server..." );
 			const config = await readYAML( configFile );
 			const middleware = await parseMiddleware( config, argv );
 
@@ -215,6 +223,7 @@ yargs( process.argv.slice( 2 ) )
 			"including any workers running from other projects.\n" +
 			"This can be used as a failsafe when there are too many stray workers.",
 		handler: () => {
+			console.log( "Stopping workers..." );
 			stopWorkers();
 		}
 	} )

lib/buildTestUrl.js

@@ -1,17 +1,28 @@
 import { generateModuleId } from "./generateHash.js";
 
-export function buildTestUrl( modules, { browserstack, esm, jsdom, port, reportId } ) {
+export function buildTestUrl( {
+	browserstack,
+	flags,
+	isolatedFlag,
+	jsdom,
+	port,
+	reportId
+} ) {
 	if ( !port ) {
 		throw new Error( "No port specified." );
 	}
 
 	const query = new URLSearchParams();
-	for ( const module of modules ) {
-		query.append( "moduleId", generateModuleId( module ) );
-	}
+	const allFlags = [ ...flags, ...( isolatedFlag ? [ isolatedFlag ] : [] ) ];
+	for ( const flag of allFlags ) {
+		const [ key, value ] = flag.split( "=" );
 
-	if ( esm ) {
-		query.append( "esmodules", "true" );
+		// Special handling for the module flag
+		if ( key === "module" ) {
+			query.append( "moduleId", generateModuleId( value ) );
+		} else {
+			query.append( key, value ?? "true" );
+		}
 	}
 
 	if ( jsdom ) {

reporter.js

@@ -15,7 +15,7 @@ function serializeForDiff( value ) {
 	return `${ value }`;
 }
 
-export function reportTest( test, reportId, { browser, headless } ) {
+export function reportTest( test, { browser, headless, id } ) {
 	if ( test.status === "passed" ) {
 
 		// Write to console without newlines
@@ -26,7 +26,7 @@ export function reportTest( test, reportId, { browser, headless } ) {
 	let message = `${ chalk.bold( `${ test.suiteName }: ${ test.name }` ) }`;
 	message += `\nTest ${ test.status } on ${ chalk.yellow(
 		getBrowserString( browser, headless )
-	) } (${ chalk.bold( reportId ) }).`;
+	) } (${ chalk.bold( id ) }).`;
 
 	// test.assertions only contains passed assertions;
 	// test.errors contains all failed asssertions
@@ -114,12 +114,14 @@ export function reportTest( test, reportId, { browser, headless } ) {
 	}
 }
 
-export function reportEnd( result, reportId, { browser, headless, modules } ) {
+export function reportEnd( result, { browser, flags, headless, id, isolatedFlag } ) {
 	const fullBrowser = getBrowserString( browser, headless );
+	const allFlags = [ ...flags, ...( isolatedFlag ? [ isolatedFlag ] : [] ) ];
+
 	console.log(
 		`\n\nTests finished in ${ prettyMs( result.runtime ) } ` +
-			`for ${ chalk.yellow( modules.join( "," ) ) } ` +
-			`in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( reportId ) })...`
+			`with flags "${ chalk.yellow( allFlags.join( "&" ) ) }" ` +
+			`in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( id ) })...`
 	);
 	console.log(
 		( result.status !== "passed" ?

run.js

@@ -6,9 +6,8 @@ import { localTunnel } from "./browserstack/local.js";
 import { reportEnd, reportTest } from "./reporter.js";
 import { createTestServer } from "./createTestServer.js";
 import { buildTestUrl } from "./lib/buildTestUrl.js";
-import { generateHash, printModuleHashes } from "./lib/generateHash.js";
+import { generateHash } from "./lib/generateHash.js";
 import { getBrowserString } from "./lib/getBrowserString.js";
-import { modules as allModules } from "./flags/modules.js";
 import { cleanupAllBrowsers, touchBrowser } from "./browsers.js";
 import {
 	addRun,
@@ -20,30 +19,24 @@ import {
 
 const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000;
 
-/**
- * Run modules in parallel in different browser instances.
- */
 export async function run( {
 	browser: browserNames = [],
 	browserstack,
 	concurrency,
 	debug,
-	esm,
+	flag: flags = [],
 	hardRetries,
 	headless,
-	isolate,
+	isolatedFlag: isolatedFlags = [],
 	middleware = [],
-	module: modules = [],
 	retries = 0,
 	runId,
 	verbose
 } ) {
+	console.log( isolatedFlags );
 	if ( !browserNames.length ) {
 		browserNames = [ "chrome" ];
 	}
-	if ( !modules.length ) {
-		modules = allModules;
-	}
 	if ( headless && debug ) {
 		throw new Error(
 			"Cannot run in headless mode and debug mode at the same time."
@@ -59,11 +52,11 @@ export async function run( {
 
 	// Convert browser names to browser objects
 	let browsers = browserNames.map( ( b ) => ( { browser: b } ) );
-	const tunnelId = generateHash(
-		`${ Date.now() }-${ modules.join( ":" ) }-${ ( browserstack || [] )
-			.concat( browserNames )
-			.join( ":" ) }`
-	);
+	const hashValue = flags
+		.concat( browserstack || [] )
+		.concat( browserNames )
+		.join( ":" );
+	const tunnelId = generateHash( `${ Date.now() }-${ hashValue }` );
 
 	// A unique identifier for this run
 	if ( !runId ) {
@@ -84,7 +77,7 @@ export async function run( {
 					const reportId = message.id;
 					const report = reports[ reportId ];
 					touchBrowser( report.browser );
-					const errors = reportTest( message.data, reportId, report );
+					const errors = reportTest( message.data, report );
 					pendingErrors[ reportId ] ??= Object.create( null );
 					if ( errors ) {
 						pendingErrors[ reportId ][ message.data.name ] = errors;
@@ -111,7 +104,6 @@ export async function run( {
 					touchBrowser( report.browser );
 					const { failed, total } = reportEnd(
 						message.data,
-						message.id,
 						reports[ reportId ]
 					);
 					report.total = total;
@@ -243,19 +235,18 @@ export async function run( {
 		tunnel = await localTunnel( tunnelId );
 		if ( verbose ) {
 			console.log( "Started BrowserStackLocal." );
-
-			printModuleHashes( modules );
 		}
 	}
 
-	function queueRun( modules, browser ) {
+	function queueRun( browser, isolatedFlag ) {
 		const fullBrowser = getBrowserString( browser, headless );
-		const reportId = generateHash( `${ modules.join( ":" ) } ${ fullBrowser }` );
-		reports[ reportId ] = { browser, headless, modules };
+		const reportId = generateHash( `${ hashValue }-${ isolatedFlag }-${ fullBrowser }` );
+		reports[ reportId ] = { browser, flags, headless, id: reportId, isolatedFlag };
 
-		const url = buildTestUrl( modules, {
+		const url = buildTestUrl( {
 			browserstack,
-			esm,
+			flags,
+			isolatedFlag,
 			jsdom: browser.browser === "jsdom",
 			port,
 			reportId
@@ -265,8 +256,9 @@ export async function run( {
 			browserstack,
 			concurrency,
 			debug,
+			flags,
 			headless,
-			modules,
+			isolatedFlag,
 			reportId,
 			runId,
 			tunnelId,
@@ -277,12 +269,12 @@ export async function run( {
 	}
 
 	for ( const browser of browsers ) {
-		if ( isolate ) {
-			for ( const module of modules ) {
-				queueRun( [ module ], browser );
-			}
+		if ( isolatedFlags.length > 0 ) {
+			isolatedFlags.forEach( ( isolatedFlag ) => {
+				queueRun( browser, isolatedFlag );
+			} );
 		} else {
-			queueRun( modules, browser );
+			queueRun( browser );
 		}
 	}
 
@@ -301,11 +293,17 @@ export async function run( {
 			for ( const report of Object.values( reports ) ) {
 				if ( !report.total ) {
 					stop = true;
+					const allFlags = [
+						...report.flags,
+						...( report.isolatedFlag ? [ report.isolatedFlag ] : [] )
+					];
 					console.error(
 						chalk.red(
-							`No tests were run for ${ report.modules.join(
-								", "
-							) } in ${ getBrowserString( report.browser ) }`
+							`No tests were run for page with flags "${
+								allFlags.join( "&" )
+							}" in ${
+								getBrowserString( report.browser )
+							} (${ report.id })`
 						)
 					);
 				}