Test on multiple device profile emulators (#238)

* Test on multiple ephemeral profile targets

* Rename the option to --generate-emulators

* Exclude some tests

* Add issue link in recipe file

Author: HakkyuKim

.github/workflows/integration_test.yml

@@ -23,20 +23,20 @@ jobs:
         run: |
           export PATH=`pwd`/flutter-tizen/bin:$PATH
           ./tools/run_command.py test \
-            --platforms wearable-5.5 \
+            --recipe ./tools/recipe.yaml \
+            --generate-emulators \
             --run-on-changed-packages \
             --base-sha=$(git rev-parse HEAD^) \
             --exclude wearable_rotary image_picker camera webview_flutter \
               video_player permission_handler geolocator battery connectivity \
               device_info package_info sensors share wifi_info_flutter \
-              google_maps_flutter tizen_app_control
+              google_maps_flutter tizen_app_control url_launcher network_info_plus
         # The following packages are excluded from tests:
         #   wearable_rotary, image_picker: no tests.
-        #   camera: the current test target is wearable emulator 5.5, which doesn't support this package.
-        #   webview_flutter: the current test target is wearable emulator 5.5, which doesn't support this package.
+        #   camera: no supported profiles.
         #   video_player: test frequently breaks due to memory issue(https://github.com/flutter-tizen/plugins/issues/135).
         #   permission_handler: permission related test.
         #   geolocator: test requires console manipulation.
         #   battery, connectivity, device_info, package_info, sensors, share, wifi_info_flutter: deprecated.
-        #   google_maps_flutter: device limitation not yet specified.
+        #   google_maps_flutter, webview_flutter, url_launcher, network_info_plus: not supported by emulators
         #   tizen_app_control: test available after Flutter 2.5 migration.

tools/commands/integration_test.py

@@ -130,8 +130,8 @@ def run_integration_test(self, plugin_name, directory, timeout):
             ])
 
 
-class EphemeralTarget(Target):
-    """A Tizen emulator that launches/poweroffs itself during test."""
+class TemporaryEmulator(Target):
+    """A Tizen emulator that is created temporary by the tool for testing purposes."""
 
     def __init__(self, name, platform):
         super().__init__(name, platform)
@@ -289,31 +289,31 @@ def _parse_target_info(self, capability_info):
         return device_profile, tizen_version
 
 
-class EphemeralTargetManager(TargetManager):
-    """A TargetManager for EphemeralTargets."""
+class TemporaryEmulatorManager(TargetManager):
+    """A TargetManager for TemporaryEmulators."""
 
     def __init__(self, platforms):
         super().__init__()
         self.platforms = platforms
 
     def __enter__(self):
         for platform in self.platforms:
-            self._create_ephemeral_target(platform)
+            self._create_emulator(platform)
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        self._delete_ephemeral_targets()
+        self._delete_emulators()
         super().__exit__(exc_type, exc_value, traceback)
 
-    def _create_ephemeral_target(self, platform):
+    def _create_emulator(self, platform):
         device_profile, tizen_version = platform.split('-', 1)
         # Target name valid characters are [A-Za-z0-9-_].
-        target_name = f'{device_profile}-{tizen_version.replace(".", "_")}-{os.getpid()}'
-        target = EphemeralTarget(target_name, platform)
-        target.create()
-        self.targets_per_platform[platform].append(target)
+        emulator_name = f'{device_profile}-{tizen_version.replace(".", "_")}-{os.getpid()}'
+        emulator = TemporaryEmulator(emulator_name, platform)
+        emulator.create()
+        self.targets_per_platform[platform].append(emulator)
 
-    def _delete_ephemeral_targets(self):
+    def _delete_emulators(self):
         for targets in self.targets_per_platform.values():
             for target in targets:
                 target.delete()
@@ -384,23 +384,23 @@ def set_subparser(subparsers):
   c: [wearable-4.0]
 )''')
     parser.add_argument(
-        '--use-ephemeral-targets',
+        '--generate-emulators',
         default=False,
         action='store_true',
-        help='''Create and destroy ephemeral targets during test. 
-Must provide --platforms or --recipe option to specify which 
-platform targets to create.''')
+        help='''Create and destroy emulators during test. 
+Must provide either --platforms or --recipe option to specify which 
+platforms to create.''')
     parser.set_defaults(func=run_integration_test)
 
 
-def _get_target_manager(use_ephemeral_targets, platforms):
-    if use_ephemeral_targets:
-        return EphemeralTargetManager(platforms)
+def _get_target_manager(generate_emulators, platforms):
+    if generate_emulators:
+        return TemporaryEmulatorManager(platforms)
     else:
         return TargetManager()
 
 
-def _integration_test(plugin_dir, platforms, timeout, use_ephemeral_targets):
+def _integration_test(plugin_dir, platforms, timeout, generate_emulators):
     """Runs integration test in the example package for plugin_dir
 
     Currently the tools assumes that there's only one example package per plugin.
@@ -409,7 +409,7 @@ def _integration_test(plugin_dir, platforms, timeout, use_ephemeral_targets):
         plugin_dir (str): The path to a single plugin directory.
         platforms (List[str]): A list of testing platforms.
         timeout (int): Time limit in seconds before cancelling the test.
-        use_ephemeral_targets (bool): Whether to create and delete targets 
+        generate_emulators (bool): Whether to create and delete targets 
                                       for test.
 
     Returns:
@@ -450,7 +450,7 @@ def _integration_test(plugin_dir, platforms, timeout, use_ephemeral_targets):
 
     try:
         with _get_target_manager(
-                use_ephemeral_targets,
+                generate_emulators,
                 platforms,
         ) as target_manager:
             if not platforms:
@@ -521,9 +521,9 @@ def run_integration_test(args):
                 )
                 exit(1)
 
-    if args.use_ephemeral_targets and not args.platforms and not args.recipe:
+    if args.generate_emulators and not args.platforms and not args.recipe:
         print(
-            '--use-ephemeral-targets option must be used with either --platforms or --recipe option.'
+            '--generate-emulators option must be used with either --platforms or --recipe option.'
         )
         exit(1)
 
@@ -552,7 +552,7 @@ def run_integration_test(args):
                 os.path.join(packages_dir, testing_plugin),
                 platforms,
                 args.timeout,
-                args.use_ephemeral_targets,
+                args.generate_emulators,
             ))
 
     print(f'============= TEST RESULT =============')

tools/recipe.yaml

@@ -0,0 +1,27 @@
+plugins:
+  # audioplayers should also test for tv-6.0, currently the test fails on tv-6.0.
+  # https://github.com/flutter-tizen/plugins/issues/239
+  audioplayers: ["wearable-5.5"]
+  battery_plus: ["wearable-5.5"]
+  camera: []
+  connectivity_plus: ["wearable-5.5", "tv-6.0"]
+  device_info_plus: ["wearable-5.5", "tv-6.0"]
+  flutter_tts: ["wearable-5.5", "tv-6.0"]
+  geolocator: ["wearable-5.5"]
+  google_maps_flutter: []
+  image_picker: []
+  integration_test: ["wearable-5.5", "tv-6.0"]
+  messageport: ["wearable-5.5", "tv-6.0"]
+  network_info_plus: []
+  package_info_plus: ["wearable-5.5", "tv-6.0"]
+  path_provider: ["wearable-5.5", "tv-6.0"]
+  permission_handler: ["wearable-5.5"]
+  sensors_plus: ["wearable-5.5"]
+  share_plus: ["wearable-5.5"]
+  shared_preferences: ["wearable-5.5", "tv-6.0"]
+  tizen_app_control: ["wearable-5.5", "tv-6.0"]
+  url_launcher: []
+  video_player: ["wearable-5.5"]
+  wakelock: ["wearable-5.5"]
+  wearable_rotary: ["wearable-5.5"]
+  webview_flutter: []