feat(launcher): reorganize launcher + add option to store test results as JSON

Author: hankduan

docs/referenceConf.js

@@ -212,6 +212,10 @@ exports.config = {
     }
   },
 
+  // If set, protractor will save the test output in json format at this path.
+  // The path is relative to the location of this config.
+  resultJsonOutputFile: null,
+
   // ---------------------------------------------------------------------------
   // ----- The test framework --------------------------------------------------
   // ---------------------------------------------------------------------------

lib/cli.js

@@ -50,6 +50,7 @@ var optimist = require('optimist').
     describe('stackTrace', 'Print stack trace on error').
     describe('params', 'Param object to be passed to the tests').
     describe('framework', 'Test framework to use: jasmine, cucumber or mocha').
+    describe('resultJsonOutputFile', 'Path to save JSON test result').
     alias('browser', 'capabilities.browserName').
     alias('name', 'capabilities.name').
     alias('platform', 'capabilities.platform').

lib/frameworks/README.md

@@ -23,5 +23,16 @@ Requirements
 
  - `runner.getConfig().onComplete` must be called when tests are finished.
 
- - The returned promise must be resolved when tests are finished and it should return a results object.
- This object must have a `failedCount` property.
+ - The returned promise must be resolved when tests are finished and it should return a results object. This object must have a `failedCount` property and optionally a `specResults` 
+ object of the following structure:
+ ```
+   specResults = [{
+     description: string,
+     assertions: [{
+       passed: boolean,
+       errorMsg: string,
+       stackTrace: string 
+     }],
+     duration: integer
+   }]
+ ```

lib/frameworks/cucumber.js

@@ -58,17 +58,65 @@ exports.run = function(runner, specs) {
   }
   global.cucumber = Cucumber.Cli(execOptions);
 
-  return runner.runTestPreparer().then(function () {
-    return q.promise(function (resolve, reject) {
-      global.cucumber.run(function (succeeded) {
+  var testResult = [];
+  var failedCount = 0; 
+  // Add a listener into cucumber so that protractor can find out which
+  // steps passed/failed
+  var addResultListener = function(formatter) {
+    var originalHandleStepResultEvent = formatter.handleStepResultEvent;
+    formatter.handleStepResultEvent = function(event, callback) {
+
+      var stepResult = event.getPayloadItem('stepResult');
+      if (stepResult.isSuccessful()) {
+        runner.emit('testPass');
+        testResult.push({
+          description: stepResult.getStep().getName(),
+          assertions: [{
+            passed: true
+          }],
+          duration: stepResult.getDuration()
+        });
+      }
+      else if (stepResult.isFailed()) {
+        runner.emit('testFail');
+        ++failedCount;
+        var failureMessage = stepResult.getFailureException();
+        testResult.push({
+          description: stepResult.getStep().getName(),
+          assertions: [{
+            passed: false,
+            errorMsg: failureMessage.message,
+            stackTrace: failureMessage.stack
+          }],
+          duration: stepResult.getDuration()
+        });
+      }
+
+      if (originalHandleStepResultEvent 
+          && typeof(originalHandleStepResultEvent) === 'function') {
+            originalHandleStepResultEvent(event, callback);
+      } else {
+        callback();
+      }
+    }
+  };
+
+  return runner.runTestPreparer().then(function() {
+    return q.promise(function(resolve, reject) {
+      var cucumberConf = Cucumber.Cli.Configuration(execOptions);
+      var runtime   = Cucumber.Runtime(cucumberConf);
+      var formatter = cucumberConf.getFormatter();
+      addResultListener(formatter);
+      runtime.attachListener(formatter);
+      runtime.start(function(succeeded) {
         try {
           if (runner.getConfig().onComplete) {
             runner.getConfig().onComplete();
           }
-          var resolvedObj = {
-            failedCount: succeeded ? 0 : 1
-          };
-          resolve(resolvedObj);
+          resolve({
+            failedCount: failedCount,
+            specResults: testResult
+          });
         } catch (err) {
           reject(err);
         }

lib/frameworks/jasmine.js

@@ -13,20 +13,38 @@ exports.run = function(runner, specs) {
   require('jasminewd');
   /* global jasmine */
 
+  var testResult = [];
+
   var RunnerReporter = function(emitter) {
     this.emitter = emitter;
   };
 
   RunnerReporter.prototype.reportRunnerStarting = function() {};
   RunnerReporter.prototype.reportRunnerResults = function() {};
   RunnerReporter.prototype.reportSuiteResults = function() {};
-  RunnerReporter.prototype.reportSpecStarting = function() {};
+  RunnerReporter.prototype.reportSpecStarting = function() {
+    this.startTime = new Date();
+  };
   RunnerReporter.prototype.reportSpecResults = function(spec) {
     if (spec.results().passed()) {
       this.emitter.emit('testPass');
     } else {
       this.emitter.emit('testFail');
     }
+
+    var entry = {
+      description: spec.results().description,
+      assertions: [],
+      duration: new Date().getTime() - this.startTime.getTime()
+    };
+    spec.results().getItems().forEach(function(item) {
+      entry.assertions.push({
+        passed: item.passed(),
+        errorMsg: item.passed() ? undefined : item.message,
+        stackTrace: item.passed() ? undefined : item.trace.stack 
+      });
+    });
+    testResult.push(entry);
   };
   RunnerReporter.prototype.log = function() {};
 
@@ -47,7 +65,10 @@ exports.run = function(runner, specs) {
           if (originalOnComplete) {
             originalOnComplete(jasmineRunner, log);
           }
-          resolve(jasmineRunner.results());
+          resolve({
+            failedCount: jasmineRunner.results().failedCount,
+            specResults: testResult
+          });
         } catch(err) {
           reject(err);
         }

lib/frameworks/mocha.js

@@ -35,19 +35,11 @@ exports.run = function(runner, specs) {
       global.it.only = global.iit = originalOnly;
 
       global.it.skip = global.xit = mochaAdapters.xit;
-    }catch(err){
+    } catch(err) {
       deferred.reject(err);
     }
   });
 
-  mocha.suite.on('pass', function() {
-    runner.emit('testPass');
-  });
-
-  mocha.suite.on('fail', function() {
-    runner.emit('testFail');
-  });
-
   mocha.loadFiles();
 
   runner.runTestPreparer().then(function() {
@@ -56,21 +48,46 @@ exports.run = function(runner, specs) {
       mocha.addFile(file);
     });
 
-    mocha.run(function(failures) {
+    var testResult = [];
+
+    var mochaRunner = mocha.run(function(failures) {
       try {
         if (runner.getConfig().onComplete) {
           runner.getConfig().onComplete();
         }
-        var resolvedObj = {
-          failedCount: failures
-        };
-
-        deferred.resolve(resolvedObj);
-      }catch(err){
+        deferred.resolve({
+          failedCount: failures,
+          specResults: testResult
+        });
+      } catch(err) {
         deferred.reject(err);
       }
     });
-  }).catch(function(reason){
+
+    mochaRunner.on('pass', function(test) {
+      runner.emit('testPass');
+      testResult.push({
+        description: test.title,
+        assertions: [{
+          passed: true
+        }],
+        duration: test.duration
+      });
+    });
+
+    mochaRunner.on('fail', function(test) {
+      runner.emit('testFail');
+      testResult.push({
+        description: test.title,
+        assertions: [{
+          passed: false,
+          errorMsg: test.err.message,
+          stackTrace: test.err.stack 
+        }],
+        duration: test.duration
+      });
+    });
+  }).catch(function(reason) {
       deferred.reject(reason);
   });