Improved statistics for test case and test suite results

Author: samitbadle

Diff for: ide/main/src/content/editor.js

@@ -845,15 +845,22 @@ Editor.prototype.showInBrowser = function (url, newWindow) {
 };
 
 Editor.prototype.playCurrentTestCase = function (next, index, total) {
+  var start = Date.now();
   var self = this;
   self.getUserLog().info("Playing test case " + (self.app.getTestCase().getTitle() || ''));
   self.app.notify("testCasePlayStart", self.app.getTestCase());
   this.selDebugger.start(function (failed) {
     self.log.debug("finished execution of test case: failed=" + failed);
     var testCase = self.suiteTreeView.getCurrentTestCase();
     if (testCase) {
-      testCase.testResult = failed ? "failed" : "passed";
-      self.getUserLog().info("Test case " + testCase.testResult);
+      testCase.testResult = {
+        summary: failed ? "failed" : "passed",
+        // remember all in milliseconds
+        start: start,
+        end: Date.now(),
+        dur: Date.now() - start
+      };
+      self.getUserLog().info("Test case " + testCase.testResult.summary);
       self.app.notify("testCasePlayDone", testCase);
     } else {
       self.getUserLog().error("current test case not found");
@@ -872,25 +879,39 @@ Editor.prototype.playTestSuite = function (startIndex) {
     startIndex = 0;
   }
   var index = startIndex - 1;
-  this.app.getTestSuite().tests.forEach(function (test) {
+  var testSuite = this.app.getTestSuite();
+  testSuite.tests.forEach(function (test) {
     if (test.testResult) {
       delete test.testResult;
     }
   });
   this.suiteTreeView.refresh();
   this.testSuiteProgress.reset();
+  var start = Date.now();
+  testSuite.testSuiteProgress = this.testSuiteProgress;
   var self = this;
   self.app.notify("testSuitePlayStart");
-  var total = this.app.getTestSuite().tests.length - startIndex;
+  var total = testSuite.tests.length - startIndex;
   (function () {
-    if (++index < self.app.getTestSuite().tests.length) {
+    if (++index < testSuite.tests.length) {
       self.suiteTreeView.scrollToRow(index);
-      self.app.showTestCaseFromSuite(self.app.getTestSuite().tests[index]);
+      self.app.showTestCaseFromSuite(testSuite.tests[index]);
       self.playCurrentTestCase(arguments.callee, index, total);
     } else {
       //Suite done
+      testSuite.suiteResult = {
+        summary: total == self.testSuiteProgress.runs && self.testSuiteProgress.failures == 0 ? 'passed' : 'failed',
+        total: total,
+        ran: self.testSuiteProgress.runs,
+        failed: self.testSuiteProgress.failures,
+        passed: self.testSuiteProgress.runs - self.testSuiteProgress.failures,
+        // remember all in milliseconds
+        start: start,
+        end: Date.now(),
+        dur: Date.now() - start
+      };
       self.getUserLog().info("Test suite completed: " + self.testSuiteProgress.runs + " played, " + (self.testSuiteProgress.failures ? self.testSuiteProgress.failures + " failed" : " all passed!"));
-      self.app.notify("testSuitePlayDone", total, self.testSuiteProgress.runs, self.testSuiteProgress.failures);
+      self.app.notify("testSuitePlayDone", testSuite.suiteResult);
     }
   })();
 };

Diff for: ide/main/src/content/selenium-runner.js

@@ -237,6 +237,7 @@ objectExtend(IDETestLoop.prototype, {
       LOG.error(result.failureMessage);
       testCase.debugContext.failed = true;
       testCase.debugContext.currentCommand().result = 'failed';
+      testCase.debugContext.currentCommand().failureMessage = result.failureMessage;
     } else if (result.passed) {
       testCase.debugContext.currentCommand().result = 'passed';
     } else {
@@ -256,6 +257,7 @@ objectExtend(IDETestLoop.prototype, {
       LOG.debug("commandError");
       testCase.debugContext.failed = true;
       testCase.debugContext.currentCommand().result = 'failed';
+      testCase.debugContext.currentCommand().failureMessage = errorMessage;
       editor.view.rowUpdated(testCase.debugContext.debugIndex);
     }
   },

Diff for: ide/main/src/content/suiteTreeView.js

@@ -195,10 +195,10 @@ objectExtend(SuiteTreeView.prototype, {
         getRowProperties: function(row, props) {
             if (this.selection.isSelected(row)) return;
             var testCase = this.getTestSuite().tests[row];
-            if (testCase.testResult) {
-                if (testCase.testResult == 'passed') {
+            if (testCase.testResult && testCase.testResult.summary) {
+                if (testCase.testResult.summary == 'passed') {
                     return XulUtils.setProperty(props, "commandPassed");
-                } else if (testCase.testResult == 'failed') {
+                } else if (testCase.testResult.summary == 'failed') {
                     return XulUtils.setProperty(props, "commandFailed");
                 }
             }

Diff for: ide/main/src/content/testCase.js

@@ -352,6 +352,7 @@ function TestCase(tempTitle) {
             this.failed = false;
 			this.started = false;
 			this.debugIndex = -1;
+      this.runTimeStamp = 0;
 		},
 
 		nextCommand: function() {
@@ -364,6 +365,7 @@ function TestCase(tempTitle) {
 			for (; this.debugIndex < testCase.commands.length; this.debugIndex++) {
 				var command = testCase.commands[this.debugIndex];
 				if (command.type == 'command') {
+          this.runTimeStamp = Date.now();
 					return command;
 				}
 			}

Diff for: ide/main/src/content/testSuite.js

@@ -16,6 +16,7 @@
 
 function TestSuite() {
     this.tests = [];
+    this.title = "Test Suite";
     this.modified = false;	//Samit: Enh: Support for change detection
 }
 
@@ -153,6 +154,19 @@ TestSuite.prototype = {
         return content;
     },
 
+  result: function() {
+    var r = {
+      title: "Test Suite",
+      result: this.suiteResult,
+      //TODO progress and baseURL and maybe real title?
+      tests: []
+    };
+    r.tests = this.tests.map(function(test) {
+      return test.result();
+    });
+    return r;
+  },
+
     generateNewTestCaseTitle: function() {
         if (this.tests.some(function(test) { return /^Untitled/.test(test.getTitle()) })) {
             var max = 1;
@@ -261,7 +275,28 @@ TestSuite.TestCase.prototype = {
     format: function() {
         return "<tr><td><a href=\"" + this.getRelativeFilePath() + "\">" +
             this.getTitle() + "</a></td></tr>\n";
+    },
+
+  result: function() {
+    var r = {
+      title: this.getTitle(),
+      result: this.testResult,
+      commands: []
+    };
+    if (this.content) {
+      r.baseURL = this.content.baseURL;
+      r.commands = this.content.commands.map(function(cmd) {
+        if (cmd.type == 'comment') {
+          return { type: cmd.type, comment: cmd.comment };
+        } else if (cmd.type == 'command') {
+          return { type: cmd.type, command: cmd.command, target: cmd.target, value: cmd.value, result: cmd.result, error: cmd.failureMessage };
+        }
+        return { type: cmd.type };
+      });
     }
-}
+    return r;
+  }
+
+};
 
 observable(TestSuite);