Testgrep not working with --verbose flag

[d9r-dev]

What are you trying to achieve?

We have a config that runs multiple tests with multiple workers. Since a week or so the testgrep is not working when the tests are startet with the --verbose flag.

What do you get instead?

When the --verbose flag is set all tests in the whole project get executed and --verbose mode is also not working correctly. I don't see any steps etc. Following command runs the tests:

npx codeceptjs run-multiple smoke --verbose --plugins allure

If I use --steps it works as it should and runs only the tests greped by the smoke property. If I use --verbose it just runs all tests ever.

Details

• CodeceptJS version: 3.5.5
• NodeJS Version: 16.21.0
• Operating System: Linux 4.12 Ubuntu 18.04.6 LTS (Bionic Beaver)
• puppeteer
• Configuration file:

// load test vars
const LOAD_TEST = require('./loadtest.conf');
const WORKER = Number.parseInt(process.env.load_test_worker) || 0;

// for screencomparision
const UPDATE_BASE_IMAGE = (process.env.updateBaseImage === 'true') ? true : false;
const SHOW_BROWSER_WINDOW = (process.env.jenkins === 'true' || process.env.headless === 'true') ? false : true;

// set retries to 0 if local execution
const RETRIES = process.env.env === 'test' ? 2 : 0; //(process.env.jenkins === 'true') ? 1 : 0;

// for correct bootstrapping when using run-multiple
const IS_MULTIPLE = (process.env.multiple === 'true') ? true : false;


// i18n of tests
const COUNTRY_CODE = process.env.country_code || 'de';
const BUSINESS_PREFIX = process.env.business_prefix;

async function mergeTestData(){
	// create directory 'gen' for generated files
	require('fs').mkdir('./gen', { recursive: true }, () => {});

	const mergedTestdata = require('merge-json').merge(
		require('./testdata/_global.json'), 
		require(`./testdata/${BUSINESS_PREFIX}/${process.env.env}_${COUNTRY_CODE}.json`));
	
	// create (merged|persisted).testdata.json
	const jsonfile = require('jsonfile');

	await jsonfile.writeFile('./gen/merged.testdata.json', mergedTestdata);
}

exports.config = {
	bootstrapAll: async () => {
		console.log('parent');
		await mergeTestData();
	},
	bootstrap: async () => {
		// executed before tests
		console.log(`Chosen Test-System: ${process.env.env}`); //eg. ex2, ex3, prod
		console.log(`Chosen Store: : ${BUSINESS_PREFIX}`); //eg. ex2, ex3, prod
		
		// only run if there is no parent process as in run-multiple
		if(!IS_MULTIPLE){
			await mergeTestData();
		}

		// set jenkins to true, prevents debugging pause method from execution
		// eslint-disable-next-line no-global-assign
		(process.env.jenkins) && (pause = function () { console.log('process.env.jenkins = true => deactivates function pause().'); });
	},
	tests: `./testcases/${BUSINESS_PREFIX}/{,*/}*.tests.js`, // braced sections expands into a set -> './testcases/*.tests.js', './testcases/*/*.tests.js'
	retries: RETRIES,
	country_code: COUNTRY_CODE,
	output: './.results',
	helpers: {
		Puppeteer: {
			'chrome':{
				'args': ['--no-sandbox','--window-size=1500,900'],
				'ignoreHTTPSErrors': true,
				'defaultViewport': {
					'width': 1500,
					'height': 900,
					'isMobile': false,
					'hasTouch': false,
					'isLandscape': false
				}
			},
			restart: true,		 		// close browser after test (!) can not be changed on the fly
			show: SHOW_BROWSER_WINDOW,  // false to run headless
			waitForTimeout: 4500, // timeout for "wait for something" - commands, default 1000ms
			waitForAction: 100,    // waitingtime between commands, default 100ms,
			getPageTimeout: process.env.env === 'test' ? 100000 : 30000,
		},
		ChaiWrapper: {
			require: 'codeceptjs-chai'
		},
		ResembleHelper : {
			require: 'codeceptjs-resemblehelper',
			baseFolder: './.screenshots/base/',
			diffFolder: './.screenshots/diff/',
			prepareBaseImage: UPDATE_BASE_IMAGE // testcases will just shoot the base images, set to false if testing
		},
		// custom helpers
		VisibilityHelper: {
			require: './testobjects/utilities/visibilityhelper.js'
		},
		WindowHelper: {
			require: './helpers/windowhelper'
		},
		PuppeteerUtilityHelper: {
			require: './helpers/puppeteerutilityhelper'
		}
	},
	include: require(`./codecept.${BUSINESS_PREFIX}.definitions.json`).include,
	mocha: {
		reporterOptions: {
			'codeceptjs-cli-reporter': {
				stdout: '-',
				options: {
					verbose: false, // show all callbacks
					steps: true    // just show executed steps
				}
			},
			'mocha-junit-reporter': {
				stdout: './output/mocha-junit-reporter/console.log',
				options: {
					mochaFile: './test-results.xml'
				},
				attachments: false //add screenshot for a failed test
			}
		}
	},
	name: 'server',
	plugins: {
		retryFailedStep: {
			enabled: true
		},
		screenshotOnFail: {
			enabled: true
		},
		allure: {
			enabled: true, // enables plugin per default, without --plugins allure must be in commandline
			outputDir: './.results',
			require: '@codeceptjs/allure-legacy'
		},
		pauseOnFail: {
			// call by adding -p pauseOnFail
			// no enabled option
		},
		customLocator: {
			enabled: true,
			prefix: '$',
			attribute: 'data-id',
			showActual: true
		},
		tryTo: {
			enabled: true // optional steps, user action must return true/false
		},
		autoDelay: {
			enabled: true,
			delayBefore: 2000,
			delayAfter: 0,
			methods: ['click']
		}
	},
	timeout: 10000, // default timeout for testcase
	load_test_max_values: LOAD_TEST.MAX_VALUES,
	loadTestWorker: WORKER,
	multiple: {
		// smoke test
		smoke_cad_nav: {
			grep: '@cadNavigation',
			browsers: ['chrome']
		},
		smoke_logged_nav: {
			grep: '@loggedNavigation',
			browsers: ['chrome']
		},
		smoke_anon_nav: {
			grep: '(?=.*@navigation)^(?!.*@loggedNavigation)^(?!.*@cadNavigation)',
			browsers: ['chrome']
		},
		smoke_checkout: {
			grep: '(?=.*@checkout)^(?!.*@cart)^(?!.*@loadTest)^(?!.*@tableCheckout)^(?!.*@quantityChangeCheckout)^(?!.*@partialDeliveryPopup)',
			browsers: ['chrome']
		},
		smoke_cart: {
			grep: '@cart',
			browsers: ['chrome']
		},
		smoke_search_login_registration: {
			grep: '\@login|\@search|\@registration',
			browsers: ['chrome']
		},
		smoke_product_details: {
			grep: '@productFamily'	,
			browsers: ['chrome']
		},
		smoke_cad_search: {
			grep: '@cadSearch',
			browsers: ['chrome']
		},
		smoke_cad_redirection: {
			grep: '@cadRedirection',
			browsers: ['chrome']
		},
		smoke_saved_carts: {
			grep: '@savedCarts',
			browsers: ['chrome']
		},
		smoke_downloads: {
			grep: '@downloads',
			browsers: ['chrome']
		},
		smoke_watchlist: {
			grep: '@watchlist',
			browsers: ['chrome']
		},
		smoke_filters: {
			grep: '\@filterArticleTab|\@filterTable',
			browsers: ['chrome']
		},
		smoke_homepage: {
			grep: '@homepage',
			browsers: ['chrome']
		},
		regression: {
			chunks: 3,
			browsers: ['chrome'],
			grep: '@regression'
		},
		smoke: {
			chunks: 4,
			browsers: ['chrome'],
			grep: '@smoke'
		},
		short_smoke: {
			browsers: ['chrome'],
			grep: '@shortsmoke'
		},
		// load test
		anon_search: {
			grep: '(?=.*@search)^(?!.*@filter)',
			browsers: Array(LOAD_TEST.MAX_VALUES.SEARCH).fill('chrome'),
		},
		cad_navigation: {
			grep: '@cadNavigation',
			browsers: Array(LOAD_TEST.MAX_VALUES.NAVIGATION.CAD).fill('chrome')
		},
		logged_navigation: {
			grep: '@loggedNavigation',
			browsers: Array(LOAD_TEST.MAX_VALUES.NAVIGATION.LOGGED).fill('chrome')
		},
		anon_navigation: {
			grep: '(?=.*@navigation)^(?!.*@loggedNavigation)^(?!.*@cadNavigation)',
			browsers: Array(LOAD_TEST.MAX_VALUES.NAVIGATION.ANON).fill('chrome')
		},
		anon_cart: {
			grep: '@anonCart',
			browsers: Array(LOAD_TEST.MAX_VALUES.CART.ANON).fill('chrome')
		}
		/*		logged_cart: {
			grep: "@loggedCart",
			browsers: Array(LOAD_TEST.MAX_VALUES.CART.LOGGED).fill("chrome")
		}
		*/
	}
};

// append "dynamically" to multiple-config
for(const i of Array(LOAD_TEST.MAX_VALUES.CHECKOUT).keys()){
	const start = i + WORKER * LOAD_TEST.MAX_VALUES.CHECKOUT;

	exports.config.multiple['checkout'+start] = {
		grep: '@checkout'+start,
		browsers: ['chrome']
	};
}

package.json

 {
  "name": "xxx",
  "version": "0.1.0",
  "description": "xxx",
  "repository": "xxx",
  "scripts": {
    "lint": "eslint testcases/** testobjects/** --fix",
  },
  "keywords": [
    "codeceptjs"
  ],
  "main": "main.js",
  "devDependencies": {
    "@codeceptjs/allure-legacy": "^1.0.2",
    "@codeceptjs/ui": "^0.4.7",
    "allure-commandline": "~2.13.0",
    "codeceptjs": "^3.3.4",
    "codeceptjs-chai": "~1.1.1",
    "codeceptjs-resemblehelper": "^1.9.6",
    "eslint": "^7.2.0",
    "eslint-plugin-codeceptjs": "^1.3.0",
    "eslint-plugin-no-floating-promise": "latest",
    "jsonfile": "~6.0.1",
    "merge-json": "~0.1.0-b.3",
    "mocha-multi": "~1.1.3",
    "puppeteer": "14.0.0"
  },
  "author": "xxx",
  "license": "UNLICENSED",
  "private": true,
  "dependencies": {
    "jsdom": "^19.0.0"
  }
}

[d9r-dev]

I debugged today. Found that codecept.init() is getting called twice when --verbose is set. Second time in info.js when --verbose is set but without options. So the testgrep gets ignored. Opened a PR.

[d9r-dev]

PR #3898 fixes this issue already.