formatters: include html-formatter as a built-in option (#1432)

* add html formatter as dependency

* update run-script to use local install

* add doc

* add as first party formatter in switch

* create our new formatter

* use in formatters test

* changelog, improve doc a bit

* add pre script to make sure we're built

* add concept of an async `finished` method on formatters

* rework so it works

* test html formatter roughly via formatters feature test

* dont need this on the test path now

* whoops missed one

* what even is a computer

* end stream from within formatter finished method except if stdout

* update changelog again

* fix botched merge

Author: davidjgoss

CHANGELOG.md

@@ -11,6 +11,8 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
 
 ### Added
 
+* Add a built in `html` formatter for rich HTML reports output as a standalone page
+
 ### Changed
 
 ### Deprecated

cucumber.js

@@ -28,7 +28,7 @@ const FORMATTERS_INCLUDE = [
   '--publish-quiet',
 ]
 
-const formatters = [
+const htmlFormatter = [
   `node_modules/@cucumber/compatibility-kit/features/{${FORMATTERS_INCLUDE.join(
     ','
   )}}/*.feature`,
@@ -37,12 +37,12 @@ const formatters = [
   '--require',
   `compatibility/features/{${FORMATTERS_INCLUDE.join(',')}}/*.ts`,
   '--format',
-  'message',
+  'html:html-formatter.html',
   '--publish-quiet',
 ].join(' ')
 
 module.exports = {
   default: feature,
   cck,
-  formatters,
+  htmlFormatter,
 }

docs/cli.md

@@ -45,6 +45,7 @@ If multiple formats are specified with the same output, only the last is used.
 
 Built-in formatters
 * message - prints each [message](https://github.com/cucumber/cucumber/tree/master/cucumber-messages) in NDJSON form, which can then be consumed by other tools.
+* html - prints a rich HTML report to a standalone page
 * json - prints the feature as JSON. *Note: this formatter is deprecated and will be removed in the next major release. Where you need a structured data representation of your test run, it's best to use the `message` formatter. For legacy tools that depend on the deprecated JSON format, a standalone formatter is available (see https://github.com/cucumber/cucumber/tree/master/json-formatter).
 * progress - prints one character per scenario (default).
 * progress-bar - prints a progress bar and outputs errors/warnings along the way.

features/formatters.feature

@@ -10,6 +10,7 @@ Feature: Formatters
     When I run cucumber-js with all formatters and `--tags @a`
     Then the "message" formatter output matches the fixture "formatters/rejected-pickle.message.json"
     Then the "json" formatter output matches the fixture "formatters/rejected-pickle.json"
+    Then the html formatter output is complete
 
   Scenario: passed from Scenario
     Given a file named "features/a.feature" with:
@@ -27,6 +28,7 @@ Feature: Formatters
     When I run cucumber-js with all formatters
     Then the "message" formatter output matches the fixture "formatters/passed-scenario.message.json"
     Then the "json" formatter output matches the fixture "formatters/passed-scenario.json"
+    Then the html formatter output is complete
 
   Scenario: passed from Rule
     Given a file named "features/a.feature" with:
@@ -45,6 +47,7 @@ Feature: Formatters
     When I run cucumber-js with all formatters
     Then the "message" formatter output matches the fixture "formatters/passed-rule.message.json"
     Then the "json" formatter output matches the fixture "formatters/passed-rule.json"
+    Then the html formatter output is complete
 
   Scenario: failed
     Given a file named "features/a.feature" with:
@@ -62,6 +65,7 @@ Feature: Formatters
     When I run cucumber-js with all formatters
     Then the "message" formatter output matches the fixture "formatters/failed.message.json"
     Then the "json" formatter output matches the fixture "formatters/failed.json"
+    Then the html formatter output is complete
     And it fails
 
   Scenario: retried and passed
@@ -89,3 +93,4 @@ Feature: Formatters
     When I run cucumber-js with all formatters and `--retry 1`
     Then the "message" formatter output matches the fixture "formatters/retried.message.json"
     Then the "json" formatter output matches the fixture "formatters/retried.json"
+    Then the html formatter output is complete

features/step_definitions/cli_steps.ts

@@ -51,7 +51,7 @@ When(
       args = ''
     }
     // message is always outputted as part of run
-    const formats = ['json:json.out']
+    const formats = ['html:html.out', 'json:json.out']
     args += ' ' + formats.map((f) => `--format ${f}`).join(' ')
     const renderedArgs = Mustache.render(args, this)
     const stringArgs = stringArgv(renderedArgs)

features/step_definitions/fixture_steps.ts renamed to features/step_definitions/formatter_steps.ts

@@ -31,3 +31,10 @@ Then(
     expect(actual).to.eql(expected)
   }
 )
+
+Then('the html formatter output is complete', async function (this: World) {
+  const actualPath = path.join(this.tmpDir, `html.out`)
+  const actual = await fs.readFile(actualPath, 'utf8')
+  expect(actual).to.contain('<html lang="en">')
+  expect(actual).to.contain('</html>')
+})

package.json

@@ -167,6 +167,7 @@
     "@cucumber/create-meta": "^2.0.2",
     "@cucumber/cucumber-expressions": "^10.3.0",
     "@cucumber/gherkin": "^15.0.2",
+    "@cucumber/html-formatter": "^9.0.0",
     "@cucumber/messages": "^13.0.1",
     "@cucumber/query": "^7.0.0",
     "@cucumber/tag-expressions": "^2.0.4",
@@ -188,6 +189,7 @@
     "mz": "^2.7.0",
     "progress": "^2.0.3",
     "resolve": "^1.17.0",
+    "resolve-pkg": "^2.0.0",
     "stack-chain": "^2.0.0",
     "stacktrace-js": "^2.0.2",
     "string-argv": "^0.3.1",
@@ -261,7 +263,7 @@
     "build-release": "browserify scripts/cucumber.ts -o dist/cucumber.js -p [ tsify -p tsconfig.browser.json ] --standalone Cucumber --debug",
     "cck-test": "mocha 'compatibility/**/*_spec.ts'",
     "feature-test": "node ./bin/cucumber-js",
-    "formatters-test": "node ./bin/cucumber-js --profile formatters | npx @cucumber/[email protected] --format ndjson > html-formatter.html",
+    "html-formatter": "node ./bin/cucumber-js --profile htmlFormatter",
     "lint-autofix": "eslint --fix \"{compatibility,example,features,scripts,src,test}/**/*.ts\"",
     "lint-code": "eslint \"{compatibility,example,features,scripts,src,test}/**/*.ts\"",
     "lint-dependencies": "dependency-lint",
@@ -272,8 +274,8 @@
     "prefeature-test": "yarn run build-local",
     "prepublishOnly": "rm -rf lib && npm run build-local",
     "test-browser-example": "yarn build-release && yarn build-browser-example && git checkout -- dist/cucumber.js",
-    "test-coverage": "yarn run lint && ./scripts/test-coverage.sh && yarn run formatters-test",
-    "test": "yarn run lint && yarn run unit-test && yarn run cck-test && yarn run feature-test && yarn run formatters-test",
+    "test-coverage": "yarn run lint && ./scripts/test-coverage.sh",
+    "test": "yarn run lint && yarn run unit-test && yarn run cck-test && yarn run feature-test",
     "unit-test": "mocha 'src/**/*_spec.ts'",
     "update-dependencies": "npx npm-check-updates --upgrade"
   },

src/cli/index.ts

@@ -88,50 +88,55 @@ export default class Cli {
     formats,
     supportCodeLibrary,
   }: IInitializeFormattersRequest): Promise<() => Promise<void>> {
-    const streamsToClose: IFormatterStream[] = []
-    await bluebird.map(formats, async ({ type, outputTo }) => {
-      let stream: IFormatterStream = this.stdout
-      if (outputTo !== '') {
-        if (outputTo.match(new RegExp('^https?://')) !== null) {
-          const headers: { [key: string]: string } = {}
-          if (process.env.CUCUMBER_PUBLISH_TOKEN !== undefined) {
-            headers.Authorization = `Bearer ${process.env.CUCUMBER_PUBLISH_TOKEN}`
-          }
+    const formatters = await bluebird.map(
+      formats,
+      async ({ type, outputTo }) => {
+        let stream: IFormatterStream = this.stdout
+        if (outputTo !== '') {
+          if (outputTo.match(new RegExp('^https?://')) !== null) {
+            const headers: { [key: string]: string } = {}
+            if (process.env.CUCUMBER_PUBLISH_TOKEN !== undefined) {
+              headers.Authorization = `Bearer ${process.env.CUCUMBER_PUBLISH_TOKEN}`
+            }
 
-          stream = new HttpStream(outputTo, 'GET', headers, (content) =>
-            console.error(content)
+            stream = new HttpStream(outputTo, 'GET', headers, (content) =>
+              console.error(content)
+            )
+          } else {
+            const fd = await fs.open(path.resolve(this.cwd, outputTo), 'w')
+            stream = fs.createWriteStream(null, { fd })
+          }
+        }
+        const typeOptions = {
+          cwd: this.cwd,
+          eventBroadcaster,
+          eventDataCollector,
+          log: stream.write.bind(stream),
+          parsedArgvOptions: formatOptions,
+          stream,
+          cleanup:
+            stream === this.stdout
+              ? async () => await Promise.resolve()
+              : bluebird.promisify(stream.end.bind(stream)),
+          supportCodeLibrary,
+        }
+        if (doesNotHaveValue(formatOptions.colorsEnabled)) {
+          typeOptions.parsedArgvOptions.colorsEnabled = (stream as TtyWriteStream).isTTY
+        }
+        if (type === 'progress-bar' && !(stream as TtyWriteStream).isTTY) {
+          const outputToName = outputTo === '' ? 'stdout' : outputTo
+          console.warn(
+            `Cannot use 'progress-bar' formatter for output to '${outputToName}' as not a TTY. Switching to 'progress' formatter.`
           )
-        } else {
-          const fd = await fs.open(path.resolve(this.cwd, outputTo), 'w')
-          stream = fs.createWriteStream(null, { fd })
+          type = 'progress'
         }
-        streamsToClose.push(stream)
+        return FormatterBuilder.build(type, typeOptions)
       }
-      const typeOptions = {
-        cwd: this.cwd,
-        eventBroadcaster,
-        eventDataCollector,
-        log: stream.write.bind(stream),
-        parsedArgvOptions: formatOptions,
-        stream,
-        supportCodeLibrary,
-      }
-      if (doesNotHaveValue(formatOptions.colorsEnabled)) {
-        typeOptions.parsedArgvOptions.colorsEnabled = (stream as TtyWriteStream).isTTY
-      }
-      if (type === 'progress-bar' && !(stream as TtyWriteStream).isTTY) {
-        const outputToName = outputTo === '' ? 'stdout' : outputTo
-        console.warn(
-          `Cannot use 'progress-bar' formatter for output to '${outputToName}' as not a TTY. Switching to 'progress' formatter.`
-        )
-        type = 'progress'
-      }
-      return FormatterBuilder.build(type, typeOptions)
-    })
+    )
     return async function () {
-      await bluebird.each(streamsToClose, (stream) =>
-        bluebird.promisify(stream.end.bind(stream))()
-      )
+      await bluebird.each(formatters, async (formatter) => {
+        await formatter.finished()
+      })
     }
   }
 

src/formatter/builder.ts

@@ -12,13 +12,14 @@ import SummaryFormatter from './summary_formatter'
 import UsageFormatter from './usage_formatter'
 import UsageJsonFormatter from './usage_json_formatter'
 import { ISupportCodeLibrary } from '../support_code_library_builder/types'
-import Formatter, { IFormatterLogFn } from '.'
+import Formatter, { IFormatterCleanupFn, IFormatterLogFn } from '.'
 import { doesHaveValue, doesNotHaveValue } from '../value_checker'
 import { EventEmitter } from 'events'
 import EventDataCollector from './helpers/event_data_collector'
 import { Writable as WritableStream } from 'stream'
 import { IParsedArgvFormatOptions } from '../cli/argv_parser'
 import { SnippetInterface } from './step_definition_snippet_builder/snippet_syntax'
+import HtmlFormatter from './html_formatter'
 
 interface IGetStepDefinitionSnippetBuilderOptions {
   cwd: string
@@ -34,6 +35,7 @@ export interface IBuildOptions {
   log: IFormatterLogFn
   parsedArgvOptions: IParsedArgvFormatOptions
   stream: WritableStream
+  cleanup: IFormatterCleanupFn
   supportCodeLibrary: ISupportCodeLibrary
 }
 
@@ -63,6 +65,8 @@ const FormatterBuilder = {
         return JsonFormatter
       case 'message':
         return MessageFormatter
+      case 'html':
+        return HtmlFormatter
       case 'progress':
         return ProgressFormatter
       case 'progress-bar':

src/formatter/html_formatter.ts

@@ -0,0 +1,34 @@
+import Formatter, { IFormatterOptions } from '.'
+import { messages } from '@cucumber/messages'
+import resolvePkg from 'resolve-pkg'
+import CucumberHtmlStream from '@cucumber/html-formatter'
+import { doesHaveValue } from '../value_checker'
+import { finished } from 'stream'
+import { promisify } from 'util'
+
+export default class HtmlFormatter extends Formatter {
+  private readonly _finished: Promise<void>
+
+  constructor(options: IFormatterOptions) {
+    super(options)
+    const cucumberHtmlStream = new CucumberHtmlStream(
+      resolvePkg('@cucumber/react', { cwd: __dirname }) +
+        '/dist/src/styles/cucumber-react.css',
+      resolvePkg('@cucumber/html-formatter', { cwd: __dirname }) +
+        '/dist/main.js'
+    )
+    options.eventBroadcaster.on('envelope', (envelope: messages.Envelope) => {
+      cucumberHtmlStream.write(envelope)
+      if (doesHaveValue(envelope.testRunFinished)) {
+        cucumberHtmlStream.end()
+      }
+    })
+    cucumberHtmlStream.on('data', (chunk) => this.log(chunk))
+    this._finished = promisify(finished)(cucumberHtmlStream)
+  }
+
+  async finished(): Promise<void> {
+    await this._finished
+    await super.finished()
+  }
+}

src/formatter/index.ts

@@ -15,6 +15,7 @@ export type IFormatterStream =
   | PassThrough
   | HttpStream
 export type IFormatterLogFn = (buffer: string | Uint8Array) => void
+export type IFormatterCleanupFn = () => Promise<any>
 
 export interface IFormatterOptions {
   colorFns: IColorFns
@@ -25,6 +26,7 @@ export interface IFormatterOptions {
   parsedArgvOptions: IParsedArgvFormatOptions
   snippetBuilder: StepDefinitionSnippetBuilder
   stream: WritableStream
+  cleanup: IFormatterCleanupFn
   supportCodeLibrary: ISupportCodeLibrary
 }
 
@@ -36,6 +38,7 @@ export default class Formatter {
   protected snippetBuilder: StepDefinitionSnippetBuilder
   protected stream: WritableStream
   protected supportCodeLibrary: ISupportCodeLibrary
+  private readonly cleanup: IFormatterCleanupFn
 
   constructor(options: IFormatterOptions) {
     this.colorFns = options.colorFns
@@ -45,5 +48,10 @@ export default class Formatter {
     this.snippetBuilder = options.snippetBuilder
     this.stream = options.stream
     this.supportCodeLibrary = options.supportCodeLibrary
+    this.cleanup = options.cleanup
+  }
+
+  async finished(): Promise<void> {
+    await this.cleanup()
   }
 }

src/formatter/progress_bar_formatter_spec.ts

@@ -20,6 +20,7 @@ import ProgressBarFormatter from './progress_bar_formatter'
 import { doesHaveValue, doesNotHaveValue } from '../value_checker'
 import { PassThrough } from 'stream'
 import ProgressBar from 'progress'
+import bluebird from 'bluebird'
 
 interface ITestProgressBarFormatterOptions {
   runtimeOptions?: Partial<IRuntimeOptions>
@@ -53,13 +54,15 @@ async function testProgressBarFormatter({
   const logFn = (data: string): void => {
     output += data
   }
+  const passThrough = new PassThrough()
   const progressBarFormatter = FormatterBuilder.build('progress-bar', {
     cwd: '',
     eventBroadcaster,
     eventDataCollector: new EventDataCollector(eventBroadcaster),
     log: logFn,
     parsedArgvOptions: {},
-    stream: new PassThrough(),
+    stream: passThrough,
+    cleanup: bluebird.promisify(passThrough.end.bind(passThrough)),
     supportCodeLibrary,
   }) as ProgressBarFormatter
   let mocked = false

test/formatter_helpers.ts

@@ -11,6 +11,7 @@ import { doesNotHaveValue } from '../src/value_checker'
 import { IParsedArgvFormatOptions } from '../src/cli/argv_parser'
 import { PassThrough } from 'stream'
 import { emitSupportCodeMessages } from '../src/cli/helpers'
+import bluebird from 'bluebird'
 import IEnvelope = messages.IEnvelope
 
 const { uuid } = IdGenerator
@@ -57,13 +58,15 @@ export async function testFormatter({
   const logFn = (data: string): void => {
     output += data
   }
+  const passThrough = new PassThrough()
   FormatterBuilder.build(type, {
     cwd: '',
     eventBroadcaster,
     eventDataCollector,
     log: logFn,
     parsedArgvOptions,
-    stream: new PassThrough(),
+    stream: passThrough,
+    cleanup: bluebird.promisify(passThrough.end.bind(passThrough)),
     supportCodeLibrary,
   })
   let pickleIds: string[] = []