format: change message formatter to output NDJSON instead of binary (#1295)

* emit ndjson from message formatter

* update cli documentation

* get everything passing with ndjson

* no need to write message output as json as well

* add module def for ndjson-parse

Author: davidjgoss

docs/cli.md

@@ -44,7 +44,7 @@ This option may be used multiple times in order to output different formats to d
 If multiple formats are specified with the same output, only the last is used.
 
 Built-in formatters
-* message - prints the full [`cucumber-messages`](https://github.com/cucumber/cucumber/tree/master/cucumber-messages) object in [varint](https://developers.google.com/protocol-buffers/docs/encoding#varints)-delimited protobuf binary form, which can then be piped to other tools.
+* message - prints each [message](https://github.com/cucumber/cucumber/tree/master/cucumber-messages) in NDJSON form, which can then be consumed by other tools.
 * json - prints the feature as JSON.
 * progress - prints one character per scenario (default).
 * progress-bar - prints a progress bar and outputs errors/warnings along the way.

features/support/world.ts

@@ -8,7 +8,7 @@ import fs from 'fs'
 import path from 'path'
 import VError from 'verror'
 import _ from 'lodash'
-import protobuf from 'protobufjs'
+import ndjsonParse from 'ndjson-parse'
 import { messages } from 'cucumber-messages'
 
 interface ILastRun {
@@ -34,7 +34,7 @@ export class World {
   public globalExecutablePath: string
 
   async run(executablePath: string, inputArgs: string[]): Promise<void> {
-    const messageFilename = 'message.out'
+    const messageFilename = 'message.ndjson'
     const args = ['node', executablePath]
       .concat(inputArgs, [
         '--backtrace',
@@ -80,22 +80,11 @@ export class World {
       stdout.end()
       result = { error, stdout: await toString(stdout), stderr }
     }
-    const envelopes: messages.Envelope[] = []
+    let envelopes: messages.Envelope[] = []
     const messageOutputPath = path.join(cwd, messageFilename)
     if (fs.existsSync(messageOutputPath)) {
-      const data = fs.readFileSync(messageOutputPath)
-      const reader = protobuf.Reader.create(data)
-      while (reader.pos < reader.len) {
-        envelopes.push(messages.Envelope.decodeDelimited(reader))
-      }
-      fs.writeFileSync(
-        path.join(cwd, 'message.out.json'),
-        JSON.stringify(
-          envelopes.map(e => e.toJSON()),
-          null,
-          2
-        )
-      )
+      const data = fs.readFileSync(messageOutputPath, { encoding: 'utf-8' })
+      envelopes = ndjsonParse(data).map(messages.Envelope.fromObject)
     }
     if (this.debug) {
       console.log(result.stdout + result.stderr) // eslint-disable-line no-console

package.json

@@ -223,9 +223,9 @@
     "lolex": "^5.1.1",
     "mocha": "^6.1.3",
     "mustache": "^3.0.1",
+    "ndjson-parse": "^1.0.4",
     "nyc": "^15.0.0",
     "prettier": "^1.5.2",
-    "protobufjs": "^6.8.8",
     "regenerator-runtime": "^0.13.3",
     "serve-static": "^1.10.0",
     "sinon": "^8.0.1",

src/formatter/message_formatter.ts

@@ -1,12 +1,11 @@
 import Formatter, { IFormatterOptions } from '.'
 import { messages } from 'cucumber-messages'
-import IEnvelope = messages.IEnvelope
 
 export default class MessageFormatter extends Formatter {
   constructor(options: IFormatterOptions) {
     super(options)
-    options.eventBroadcaster.on('envelope', (envelope: IEnvelope) =>
-      this.log(messages.Envelope.encodeDelimited(envelope).finish())
+    options.eventBroadcaster.on('envelope', (envelope: messages.Envelope) =>
+      this.log(JSON.stringify(envelope.toJSON()) + '\n')
     )
   }
 }

src/types/ndjson-parse/index.d.ts

@@ -0,0 +1,3 @@
+declare module 'ndjson-parse' {
+  export default function parse(input: string): any[]
+}

yarn.lock

@@ -2829,6 +2829,11 @@ natural-compare@^1.4.0:
   resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
   integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
 
+ndjson-parse@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/ndjson-parse/-/ndjson-parse-1.0.4.tgz#65c031147ea1b5fa6f692e4fd63ab75f89dbf648"
+  integrity sha512-xwglvz2dMbxvX4NAVKnww8xEJ4kp4+CKVseQQdtkA79yI3abPqyBYqk6A6HvNci5oS0cUsSHheMEV1c+9MWlEw==
+
 next-tick@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c"