Test flutter run does not have unexpected engine logs (#116798)

Author: loic-sharma

.ci.yaml

@@ -3206,6 +3206,30 @@ targets:
         ["devicelab", "android", "mac"]
       task_name: microbenchmarks
 
+  - name: Mac_android run_debug_test_android
+    recipe: devicelab/devicelab_drone
+    bringup: true
+    presubmit: false
+    runIf:
+      - dev/**
+    timeout: 60
+    properties:
+      tags: >
+        ["devicelab", "android", "mac"]
+      task_name: run_debug_test_android
+
+  - name: Mac_arm64_android run_debug_test_android
+    recipe: devicelab/devicelab_drone
+    bringup: true
+    presubmit: false
+    runIf:
+      - dev/**
+    timeout: 60
+    properties:
+      tags: >
+        ["devicelab", "android", "mac", "arm64"]
+      task_name: run_debug_test_android
+
   - name: Mac_android run_release_test
     recipe: devicelab/devicelab_drone
     presubmit: false
@@ -3879,8 +3903,42 @@ targets:
       - bin/**
       - .ci.yaml
 
+  - name: Mac run_debug_test_macos
+    recipe: devicelab/devicelab_drone
+    bringup: true
+    timeout: 60
+    properties:
+      dependencies: >-
+        [
+          {"dependency": "xcode", "version": "14a5294e"},
+          {"dependency": "gems", "version": "v3.3.14"}
+        ]
+      tags: >
+        ["devicelab", "hostonly", "mac"]
+      task_name: run_debug_test_macos
+    runIf:
+      - dev/**
+      - packages/flutter_tools/**
+      - bin/**
+      - .ci.yaml
+
+  - name: Mac_arm64_ios run_debug_test_macos
+    recipe: devicelab/devicelab_drone
+    bringup: true
+    timeout: 60
+    properties:
+      tags: >
+        ["devicelab", "ios", "mac", "arm64"]
+      task_name: run_debug_test_macos
+    runIf:
+      - dev/**
+      - packages/flutter_tools/**
+      - bin/**
+      - .ci.yaml
+
   - name: Mac run_release_test_macos
     recipe: devicelab/devicelab_drone
+    presubmit: false
     timeout: 60
     properties:
       dependencies: >-
@@ -4193,9 +4251,29 @@ targets:
       - bin/**
       - .ci.yaml
 
-  - name: Windows run_release_test_windows
+  - name: Windows run_debug_test_windows
+    recipe: devicelab/devicelab_drone
     bringup: true
+    presubmit: false
+    timeout: 60
+    properties:
+      dependencies: >-
+        [
+          {"dependency": "vs_build", "version": "version:vs2019"}
+        ]
+      tags: >
+        ["devicelab", "hostonly", "windows"]
+      task_name: run_debug_test_windows
+    runIf:
+      - dev/**
+      - packages/flutter_tools/**
+      - bin/**
+      - .ci.yaml
+
+  - name: Windows run_release_test_windows
     recipe: devicelab/devicelab_drone
+    bringup: true
+    presubmit: false
     timeout: 60
     properties:
       dependencies: >-

TESTOWNERS

@@ -249,6 +249,9 @@
 /dev/devicelab/bin/tasks/plugin_lint_mac.dart @stuartmorgan @flutter/plugin
 /dev/devicelab/bin/tasks/plugin_test_ios.dart @jmagman @flutter/ios
 /dev/devicelab/bin/tasks/plugin_test.dart @stuartmorgan @flutter/plugin
+/dev/devicelab/bin/tasks/run_debug_test_android.dart @zanderso @flutter/tool
+/dev/devicelab/bin/tasks/run_debug_test_macos.dart @cbracken @flutter/tool
+/dev/devicelab/bin/tasks/run_debug_test_windows.dart @loic-sharma @flutter/tool
 /dev/devicelab/bin/tasks/run_release_test_macos.dart @cbracken @flutter/tool
 /dev/devicelab/bin/tasks/run_release_test_windows.dart @loic-sharma @flutter/tool
 /dev/devicelab/bin/tasks/technical_debt__cost.dart @HansMuller @flutter/framework

dev/devicelab/README.md

@@ -44,7 +44,7 @@ You must set the `ANDROID_SDK_ROOT` environment variable to run
 tests on Android. If you have a local build of the Flutter engine, then you have
 a copy of the Android SDK at `.../engine/src/third_party/android_tools/sdk`.
 
-You can find where your Android SDK is using `flutter doctor`.
+You can find where your Android SDK is using `flutter doctor -v`.
 
 ### Warnings
 

dev/devicelab/bin/tasks/run_debug_test_android.dart

@@ -0,0 +1,10 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/run_tests.dart';
+
+void main() {
+  task(createAndroidRunDebugTest());
+}

dev/devicelab/bin/tasks/run_debug_test_macos.dart

@@ -0,0 +1,12 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/devices.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/run_tests.dart';
+
+void main() {
+  deviceOperatingSystem = DeviceOperatingSystem.macos;
+  task(createMacOSRunDebugTest());
+}

dev/devicelab/bin/tasks/run_debug_test_windows.dart

@@ -0,0 +1,12 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/devices.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/run_tests.dart';
+
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.windows;
+  await task(createWindowsRunDebugTest());
+}

dev/devicelab/lib/tasks/run_tests.dart

@@ -11,10 +11,25 @@ import '../framework/framework.dart';
 import '../framework/task_result.dart';
 import '../framework/utils.dart';
 
+TaskFunction createAndroidRunDebugTest() {
+  return AndroidRunOutputTest(release: false);
+}
+
 TaskFunction createAndroidRunReleaseTest() {
   return AndroidRunOutputTest(release: true);
 }
 
+TaskFunction createMacOSRunDebugTest() {
+  return DesktopRunOutputTest(
+    // TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201
+    // Switch to dev/integration_tests/ui once we have CocoaPods working on M1 Macs.
+    '${flutterDirectory.path}/examples/hello_world',
+    'lib/main.dart',
+    release: false,
+    allowStderr: true,
+  );
+}
+
 TaskFunction createMacOSRunReleaseTest() {
   return DesktopRunOutputTest(
     // TODO(cbracken): https://github.com/flutter/flutter/issues/87508#issuecomment-1043753201
@@ -26,6 +41,14 @@ TaskFunction createMacOSRunReleaseTest() {
   );
 }
 
+TaskFunction createWindowsRunDebugTest() {
+  return DesktopRunOutputTest(
+    '${flutterDirectory.path}/dev/integration_tests/ui',
+    'lib/empty.dart',
+    release: false,
+  );
+}
+
 TaskFunction createWindowsRunReleaseTest() {
   return DesktopRunOutputTest(
     '${flutterDirectory.path}/dev/integration_tests/ui',
@@ -168,6 +191,10 @@ abstract class RunOutputTask {
     }
   );
 
+  static final RegExp _engineLogRegex = RegExp(
+    r'\[(VERBOSE|INFO|WARNING|ERROR|FATAL):.+\(\d+\)\]',
+  );
+
   /// The directory where the app under test is defined.
   final String testDirectory;
   /// The main entry-point file of the application, as run on the device.
@@ -232,6 +259,13 @@ abstract class RunOutputTask {
         throw 'flutter run ${release ? '--release' : ''} had unexpected output on standard error.';
       }
 
+      final List<String> engineLogs = List<String>.from(
+        stdout.where(_engineLogRegex.hasMatch),
+      );
+      if (engineLogs.isNotEmpty) {
+        throw 'flutter run had unexpected Flutter engine logs $engineLogs';
+      }
+
       return verify(stdout, stderr);
     });
   }