[e2e] Adding failure details for driver tests (flutter#2593)

* Adding failure details for driver tests

* formate fix

* addressing reviewer comments

* addressing the reviewer comments. part 2

* fix error in response_data

* addressing reviewer comments

* fixing formatting

* addressin reviewer comments. Carrying the repeated driver test code to e2e package

* chaning the failure's details type from map to list

* format changes

* add documentation to public members

* formatting

* changing namespace name

Author: Nurhan Turgut

packages/e2e/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.4.0
+
+* **Breaking change** Driver request_data call's response has changed to
+  encapsulate the failure details.
+* Details for failure cases are added: failed method name, stack trace.
+
 ## 0.3.0+1
 
 * Replace deprecated `getFlutterEngine` call on Android.

packages/e2e/README.md

@@ -43,16 +43,11 @@ Put the a file named `<package_name>_e2e_test.dart` in the app' `test_driver` di
 
 ```dart
 import 'dart:async';
-import 'dart:io';
-import 'package:flutter_driver/flutter_driver.dart';
-
-Future<void> main() async {
-  final FlutterDriver driver = await FlutterDriver.connect();
-  final String result =
-      await driver.requestData(null, timeout: const Duration(minutes: 1));
-  await driver.close();
-  exit(result == 'pass' ? 0 : 1);
-}
+
+import 'package:e2e/e2e_driver.dart' as e2e;
+
+Future<void> main() async => e2e.main();
+
 ```
 
 To run a example app test with Flutter driver:
@@ -69,7 +64,7 @@ cd example
 flutter drive --driver=test_driver/<package_name>_test.dart test/<package_name>_e2e.dart
 ```
 
-You can run tests on web on release mode.
+You can run tests on web in release or profile mode.
 
 First you need to make sure you have downloaded the driver for the browser.
 

packages/e2e/example/test_driver/example_e2e_test.dart

@@ -1,12 +1,5 @@
 import 'dart:async';
-import 'dart:io';
 
-import 'package:flutter_driver/flutter_driver.dart';
+import 'package:e2e/e2e_driver.dart' as e2e;
 
-Future<void> main() async {
-  final FlutterDriver driver = await FlutterDriver.connect();
-  final String result =
-      await driver.requestData(null, timeout: const Duration(minutes: 1));
-  await driver.close();
-  exit(result == 'pass' ? 0 : 1);
-}
+Future<void> main() async => e2e.main();

packages/e2e/example/test_driver/failure_test.dart

@@ -1,17 +1,19 @@
 import 'dart:async';
 
+import 'package:e2e/common.dart' as common;
 import 'package:flutter_driver/flutter_driver.dart';
 import 'package:test/test.dart';
 
 Future<void> main() async {
   test('fails gracefully', () async {
     final FlutterDriver driver = await FlutterDriver.connect();
-    final String result =
+    final String jsonResult =
         await driver.requestData(null, timeout: const Duration(minutes: 1));
+    common.Response response = common.Response.fromJson(jsonResult);
     await driver.close();
     expect(
-      result,
-      'fail',
+      response.allTestsPassed,
+      false,
     );
   });
 }

packages/e2e/lib/common.dart

@@ -0,0 +1,116 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+/// An object sent from e2e back to the Flutter Driver in response to
+/// `request_data` command.
+class Response {
+  final List<Failure> _failureDetails;
+
+  final bool _allTestsPassed;
+
+  /// Constructor to use for positive response.
+  Response.allTestsPassed()
+      : this._allTestsPassed = true,
+        this._failureDetails = null;
+
+  /// Constructor for failure response.
+  Response.someTestsFailed(this._failureDetails) : this._allTestsPassed = false;
+
+  /// Whether the test ran successfully or not.
+  bool get allTestsPassed => _allTestsPassed;
+
+  /// If the result are failures get the formatted details.
+  String get formattedFailureDetails =>
+      _allTestsPassed ? '' : formatFailures(_failureDetails);
+
+  /// Failure details as a list.
+  List<Failure> get failureDetails => _failureDetails;
+
+  /// Serializes this message to a JSON map.
+  String toJson() => json.encode(<String, dynamic>{
+        'result': allTestsPassed.toString(),
+        'failureDetails': _failureDetailsAsString(),
+      });
+
+  /// Deserializes the result from JSON.
+  static Response fromJson(String source) {
+    Map<String, dynamic> responseJson = json.decode(source);
+    if (responseJson['result'] == 'true') {
+      return Response.allTestsPassed();
+    } else {
+      return Response.someTestsFailed(
+          _failureDetailsFromJson(responseJson['failureDetails']));
+    }
+  }
+
+  /// Method for formating the test failures' details.
+  String formatFailures(List<Failure> failureDetails) {
+    if (failureDetails.isEmpty) {
+      return '';
+    }
+
+    StringBuffer sb = StringBuffer();
+    int failureCount = 1;
+    failureDetails.forEach((Failure f) {
+      sb.writeln('Failure in method: ${f.methodName}');
+      sb.writeln('${f.details}');
+      sb.writeln('end of failure ${failureCount.toString()}\n\n');
+      failureCount++;
+    });
+    return sb.toString();
+  }
+
+  /// Create a list of Strings from [_failureDetails].
+  List<String> _failureDetailsAsString() {
+    final List<String> list = List<String>();
+    if (_failureDetails == null || _failureDetails.isEmpty) {
+      return list;
+    }
+
+    _failureDetails.forEach((Failure f) {
+      list.add(f.toString());
+    });
+
+    return list;
+  }
+
+  /// Creates a [Failure] list using a json response.
+  static List<Failure> _failureDetailsFromJson(List<dynamic> list) {
+    final List<Failure> failureList = List<Failure>();
+    list.forEach((s) {
+      final String failure = s as String;
+      failureList.add(Failure.fromJsonString(failure));
+    });
+    return failureList;
+  }
+}
+
+/// Representing a failure includes the method name and the failure details.
+class Failure {
+  /// The name of the test method which failed.
+  final String methodName;
+
+  /// The details of the failure such as stack trace.
+  final String details;
+
+  /// Constructor requiring all fields during initialization.
+  Failure(this.methodName, this.details);
+
+  /// Serializes the object to JSON.
+  @override
+  String toString() {
+    return json.encode(<String, String>{
+      'methodName': methodName,
+      'details': details,
+    });
+  }
+
+  /// Decode a JSON string to create a Failure object.
+  static Failure fromJsonString(String jsonString) {
+    Map<String, dynamic> failure = json.decode(jsonString);
+    return Failure(failure['methodName'], failure['details']);
+  }
+}

packages/e2e/lib/e2e.dart

@@ -3,11 +3,13 @@
 // found in the LICENSE file.
 
 import 'dart:async';
+
 import 'package:flutter_test/flutter_test.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/widgets.dart';
 
+import 'common.dart';
 import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
 
 /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
@@ -36,6 +38,11 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
 
   final Completer<bool> _allTestsPassed = Completer<bool>();
 
+  /// Stores failure details.
+  ///
+  /// Failed test method's names used as key.
+  final List<Failure> _failureMethodsDetails = List<Failure>();
+
   /// Similar to [WidgetsFlutterBinding.ensureInitialized].
   ///
   /// Returns an instance of the [E2EWidgetsFlutterBinding], creating and
@@ -63,7 +70,9 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
         case 'request_data':
           final bool allTestsPassed = await _allTestsPassed.future;
           response = <String, String>{
-            'message': allTestsPassed ? 'pass' : 'fail',
+            'message': allTestsPassed
+                ? Response.allTestsPassed().toJson()
+                : Response.someTestsFailed(_failureMethodsDetails).toJson(),
           };
           break;
         case 'get_health':
@@ -94,6 +103,7 @@ class E2EWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding {
     reportTestException =
         (FlutterErrorDetails details, String testDescription) {
       _results[description] = 'failed';
+      _failureMethodsDetails.add(Failure(testDescription, details.toString()));
       if (!_allTestsPassed.isCompleted) _allTestsPassed.complete(false);
       valueBeforeTest(details, testDescription);
     };

packages/e2e/lib/e2e_driver.dart

@@ -0,0 +1,21 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:e2e/common.dart' as e2e;
+import 'package:flutter_driver/flutter_driver.dart';
+
+Future<void> main() async {
+  final FlutterDriver driver = await FlutterDriver.connect();
+  final String jsonResult =
+      await driver.requestData(null, timeout: const Duration(minutes: 1));
+  final e2e.Response response = e2e.Response.fromJson(jsonResult);
+  await driver.close();
+
+  if (response.allTestsPassed) {
+    print('All tests passed.');
+    exit(0);
+  } else {
+    print('Failure Details:\n${response.formattedFailureDetails}');
+    exit(1);
+  }
+}

packages/e2e/pubspec.yaml

@@ -1,6 +1,6 @@
 name: e2e
 description: Runs tests that use the flutter_test API as integration tests.
-version: 0.3.0+1
+version: 0.4.0
 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e
 
 environment:
@@ -10,6 +10,8 @@ environment:
 dependencies:
   flutter:
     sdk: flutter
+  flutter_driver:
+    sdk: flutter
   flutter_test:
     sdk: flutter