Parser machine logs (#118707)

* remove file reporter optional

* decouple stack trace from metric collections

* update tests; add collect metrics option

* add failed tests property

* add test for multiple stack failed

* change path on result

* create factory method

* throw exception when test file failed to generate

* remove catch of file exception

* handle when no stacktrace on file reporter

Author: Jasguerrero

dev/bots/test.dart

@@ -1802,6 +1802,7 @@ Future<void> _runDartTest(String workingDirectory, {
   final List<String> args = <String>[
     'run',
     'test',
+    '--file-reporter=json:${metricFile.path}',
     if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
     '-j$cpus',
     if (!hasColor)
@@ -1813,8 +1814,6 @@ Future<void> _runDartTest(String workingDirectory, {
     if (testPaths != null)
       for (final String testPath in testPaths)
         testPath,
-    if (collectMetrics)
-      '--file-reporter=json:${metricFile.path}',
   ];
   final Map<String, String> environment = <String, String>{
     'FLUTTER_ROOT': flutterRoot,
@@ -1840,19 +1839,24 @@ Future<void> _runDartTest(String workingDirectory, {
     removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
   );
 
+  final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
+  final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
+  info.writeAsStringSync(json.encode(test.errors));
+
   if (collectMetrics) {
     try {
       final List<String> testList = <String>[];
-      final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
+      final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
       for (final TestSpecs testSpecs in allTestSpecs.values) {
         testList.add(testSpecs.toJson());
       }
       if (testList.isNotEmpty) {
         final String testJson = json.encode(testList);
-        final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
+        final File testResults = fileSystem.file(
+            path.join(flutterRoot, 'test_results.json'));
         testResults.writeAsStringSync(testJson);
       }
-    } on fs.FileSystemException catch (e){
+    } on fs.FileSystemException catch (e) {
       print('Failed to generate metrics: $e');
     }
   }

dev/bots/test/tool_subsharding_test.dart

@@ -23,8 +23,8 @@ void main() {
       {"missing": "entry"}
       {"other": true}''';
       file.writeAsStringSync(output);
-      final Map<int, TestSpecs> result = generateMetrics(file);
-      expect(result, isEmpty);
+      final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
+      expect(result.allTestSpecs, isEmpty);
     });
 
     test('have metrics', () async {
@@ -47,7 +47,7 @@ void main() {
       {"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}
       {"success":true,"type":"done","time":4870}''';
       file.writeAsStringSync(output);
-      final Map<int, TestSpecs> result = generateMetrics(file);
+      final Map<int, TestSpecs> result = TestFileReporterResults.fromFile(file).allTestSpecs;
       expect(result, contains(0));
       expect(result, contains(1));
       expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart');
@@ -62,8 +62,37 @@ void main() {
       {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
       {"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}''';
       file.writeAsStringSync(output);
-      final Map<int, TestSpecs> result = generateMetrics(file);
-      expect(result, isEmpty);
+      final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
+      expect(result.hasFailedTests, true);
+    });
+
+    test('has failed stack traces', () async {
+      final File file = fileSystem.file('success_file');
+      const String output = '''
+      {"protocolVersion":"0.1.1","runnerVersion":"1.22.1","pid":47372,"type":"start","time":0}
+      {"suite":{"id":0,"platform":"vm","path":"test/tool_subsharding_test.dart"},"type":"suite","time":0}
+      {"test":{"id":1,"name":"loading test/tool_subsharding_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":2}
+      {"count":1,"time":11,"type":"allSuites"}
+      {"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1021}
+      {"group":{"id":2,"suiteID":0,"parentID":null,"name":"","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":null,"column":null,"url":null},"type":"group","time":1026}
+      {"group":{"id":3,"suiteID":0,"parentID":2,"name":"generateMetrics","metadata":{"skip":false,"skipReason":null},"testCount":3,"line":13,"column":3,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"group","time":1027}
+      {"test":{"id":4,"name":"generateMetrics empty metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":20,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1027}
+      {"testID":4,"error":"Expected: <true>  Actual: <false>","stackTrace":"package:test_api                      expect test/tool_subsharding_test.dart 28:7  main.<fn>.<fn> ","isFailure":true,"type":"error","time":1095}
+      {"testID":4,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1096}
+      {"test":{"id":5,"name":"generateMetrics have metrics","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":31,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1097}
+      {"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":1103}
+      {"test":{"id":6,"name":"generateMetrics missing success entry","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":60,"column":5,"url":"file:///Users/user/Documents/flutter/dev/bots/test/tool_subsharding_test.dart"},"type":"testStart","time":1103}
+      {"testID":6,"error":"Expected: <false>  Actual: <true>","stackTrace":"package:test_api                      expect test/tool_subsharding_test.dart 68:7  main.<fn>.<fn> ","isFailure":true,"type":"error","time":1107}
+      {"testID":6,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":1107}
+      {"testID":6,"error":"my error","isFailure":true,"type":"error","time":1107}
+      {"success":false,"type":"done","time":1120}''';
+      file.writeAsStringSync(output);
+      final TestFileReporterResults result = TestFileReporterResults.fromFile(file);
+      expect(result.hasFailedTests, true);
+      expect(result.errors.length == 3, true);
+      expect(result.errors[0].contains('Expected: <true>  Actual: <false>'), true);
+      expect(result.errors[1].contains('Expected: <false>  Actual: <true>'), true);
+      expect(result.errors[2].contains('my error'), true);
     });
   });
 }

dev/bots/tool_subsharding.dart

@@ -38,42 +38,65 @@ class TestSpecs {
   }
 }
 
-/// Intended to parse the output file of `dart test --file-reporter json:file_name
-Map<int, TestSpecs> generateMetrics(File metrics) {
-  final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{};
-  if (!metrics.existsSync()) {
-    return allTestSpecs;
-  }
+class TestFileReporterResults {
+  TestFileReporterResults._({
+    required this.allTestSpecs,
+    required this.hasFailedTests,
+    required this.errors,
+  });
 
-  bool success = false;
-  for(final String metric in metrics.readAsLinesSync()) {
-    final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>;
-    if (entry.containsKey('suite')) {
-      final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>;
-      allTestSpecs[suite['id'] as int] = TestSpecs(
-        path: suite['path'] as String,
-        startTime: entry['time'] as int,
-      );
-    } else if (_isMetricDone(entry, allTestSpecs)) {
-      final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
-      final int suiteID = group['suiteID'] as int;
-      final TestSpecs testSpec = allTestSpecs[suiteID]!;
-      testSpec.endTime = entry['time'] as int;
-    } else if (entry.containsKey('success') && entry['success'] == true) {
-      success = true;
+  /// Intended to parse the output file of `dart test --file-reporter json:file_name
+  factory TestFileReporterResults.fromFile(File metrics) {
+    if (!metrics.existsSync()) {
+      throw Exception('${metrics.path} does not exist');
     }
+
+    final Map<int, TestSpecs> testSpecs = <int, TestSpecs>{};
+    bool hasFailedTests = true;
+    final List<String> errors = <String>[];
+
+    for (final String metric in metrics.readAsLinesSync()) {
+      final Map<String, Object?> entry = json.decode(metric) as Map<String, Object?>;
+      if (entry.containsKey('suite')) {
+        final Map<String, Object?> suite = entry['suite']! as Map<String, Object?>;
+        addTestSpec(suite, entry['time']! as int, testSpecs);
+      } else if (isMetricDone(entry, testSpecs)) {
+        final Map<String, Object?> group = entry['group']! as Map<String, Object?>;
+        final int suiteID = group['suiteID']! as int;
+        addMetricDone(suiteID, entry['time']! as int, testSpecs);
+      } else if (entry.containsKey('error')) {
+        final String stackTrace = entry.containsKey('stackTrace') ? entry['stackTrace']! as String : '';
+        errors.add('${entry['error']}\n $stackTrace');
+      } else if (entry.containsKey('success') && entry['success'] == true) {
+        hasFailedTests = false;
+      }
+    }
+
+    return TestFileReporterResults._(allTestSpecs: testSpecs, hasFailedTests: hasFailedTests, errors: errors);
   }
 
-  if (!success) { // means that not all tests succeeded therefore no metrics are stored
-    return <int, TestSpecs>{};
+  final Map<int, TestSpecs> allTestSpecs;
+  final bool hasFailedTests;
+  final List<String> errors;
+
+
+  static void addTestSpec(Map<String, Object?> suite, int time, Map<int, TestSpecs> allTestSpecs) {
+    allTestSpecs[suite['id']! as int] = TestSpecs(
+      path: suite['path']! as String,
+      startTime: time,
+    );
   }
-  return allTestSpecs;
-}
 
-bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) {
-  if (entry.containsKey('group') && entry['type'] as String == 'group') {
-    final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
-    return allTestSpecs.containsKey(group['suiteID'] as int);
+  static void addMetricDone(int suiteID, int time, Map<int, TestSpecs> allTestSpecs) {
+    final TestSpecs testSpec = allTestSpecs[suiteID]!;
+    testSpec.endTime = time;
+  }
+
+  static bool isMetricDone(Map<String, Object?> entry, Map<int, TestSpecs> allTestSpecs) {
+    if (entry.containsKey('group') && entry['type']! as String == 'group') {
+      final Map<String, Object?> group = entry['group']! as Map<String, Object?>;
+      return allTestSpecs.containsKey(group['suiteID']! as int);
+    }
+    return false;
   }
-  return false;
 }