From 06c9c7f6c629a3ccbf694c3daef0ea7c5901e2d4 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Wed, 3 Nov 2021 17:03:47 -0700
Subject: [PATCH 01/13] Reset Android when install app failed

---
 scripts/gha/test_simulator.py | 45 +++++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 274d8476d9..b99e379dbd 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -549,27 +549,25 @@ def _create_and_boot_emulator(sdk_id):
   logging.info("Wait for emulator to boot: %s", " ".join(args))
   subprocess.run(args=args, check=True)
   if FLAGS.ci: 
-    # wait extra 90 seconds to ensure emulator fully booted.
+    # wait extra 180 seconds to ensure emulator fully booted.
     time.sleep(180)
   else:
     time.sleep(45)
 
 
-def _reset_emulator_on_error(instrumented_test_result):
-  logging.info("game-loop test result: %s", instrumented_test_result)
-  if "FAILURES!!!" in instrumented_test_result:
-    logging.info("game-loop test error!!! reboot emualtor...")
-    args = ["adb", "-e", "reboot"]
-    logging.info("Reboot android emulator: %s", " ".join(args))
-    subprocess.run(args=args, check=True)
-    args = ["adb", "wait-for-device"]
-    logging.info("Wait for emulator to boot: %s", " ".join(args))
-    subprocess.run(args=args, check=True)
-    if FLAGS.ci: 
-      # wait extra 90 seconds to ensure emulator booted.
-      time.sleep(90)
-    else:
-      time.sleep(45)
+def _reset_emulator_on_error():
+  logging.info("game-loop test error!!! reboot emualtor...")
+  args = ["adb", "-e", "reboot"]
+  logging.info("Reboot android emulator: %s", " ".join(args))
+  subprocess.run(args=args, check=True)
+  args = ["adb", "wait-for-device"]
+  logging.info("Wait for emulator to boot: %s", " ".join(args))
+  subprocess.run(args=args, check=True)
+  if FLAGS.ci: 
+    # wait extra 180 seconds to ensure emulator booted.
+    time.sleep(180)
+  else:
+    time.sleep(45)
 
 
 def _get_package_name(app_path):
@@ -599,7 +597,16 @@ def _install_android_app(app_path):
   """Install integration_test app into the emulator."""
   args = ["adb", "install", app_path]
   logging.info("Install testapp: %s", " ".join(args))
-  subprocess.run(args=args, check=False)
+  result = subprocess.run(args=args, check=False)
+  if result.returncode != 0:
+    logging.info("Install testapp %s failed, Reset Emualtor now...", app_path)
+    args = ["adb", "shell", "recovery", "--wipe_data"]
+    logging.info("Erase my Emualtor: %s", " ".join(args))
+    subprocess.run(args=args, check=True)
+    _reset_emulator_on_error()
+    args = ["adb", "install", app_path]
+    logging.info("Reinstall testapp: %s", " ".join(args))
+    subprocess.run(args=args, check=True)
 
 
 def _uninstall_android_app(package_name):
@@ -629,7 +636,9 @@ def _run_instrumented_test():
     "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % _GAMELOOP_PACKAGE] 
   logging.info("Running game-loop test: %s", " ".join(args))
   result = subprocess.run(args=args, capture_output=True, text=True, check=False) 
-  _reset_emulator_on_error(result.stdout)
+  if "FAILURES!!!" in result.stdout:
+    logging.info("game-loop test result: %s", result.stdout)
+    _reset_emulator_on_error()
 
 
 def _get_android_test_log(test_package):

From 7b666d852317d2c08c07f1c23ee32f8f0fcba3a2 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Wed, 3 Nov 2021 17:17:03 -0700
Subject: [PATCH 02/13] Wait 210s for emualtor boot

---
 scripts/gha/test_simulator.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index b99e379dbd..120f7ed651 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -549,8 +549,8 @@ def _create_and_boot_emulator(sdk_id):
   logging.info("Wait for emulator to boot: %s", " ".join(args))
   subprocess.run(args=args, check=True)
   if FLAGS.ci: 
-    # wait extra 180 seconds to ensure emulator fully booted.
-    time.sleep(180)
+    # wait extra 210 seconds to ensure emulator fully booted.
+    time.sleep(210)
   else:
     time.sleep(45)
 
@@ -564,8 +564,8 @@ def _reset_emulator_on_error():
   logging.info("Wait for emulator to boot: %s", " ".join(args))
   subprocess.run(args=args, check=True)
   if FLAGS.ci: 
-    # wait extra 180 seconds to ensure emulator booted.
-    time.sleep(180)
+    # wait extra 210 seconds to ensure emulator booted.
+    time.sleep(210)
   else:
     time.sleep(45)
 

From a3c4c1ad5514a96baf320361fc1a3d6a6b589fdc Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Wed, 3 Nov 2021 20:17:37 -0700
Subject: [PATCH 03/13] build_tools_version 29.0.2

---
 .../gha/integration_testing/gameloop_android/build.gradle   | 2 +-
 scripts/gha/test_simulator.py                               | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/gha/integration_testing/gameloop_android/build.gradle b/scripts/gha/integration_testing/gameloop_android/build.gradle
index ed97082da3..fbb3465046 100644
--- a/scripts/gha/integration_testing/gameloop_android/build.gradle
+++ b/scripts/gha/integration_testing/gameloop_android/build.gradle
@@ -23,7 +23,7 @@ allprojects {
 
 ext {
     compileSdk = 28
-    buildTools = '28.0.3'
+    buildTools = '29.0.2'
     minSdk = 16
     targetSdk = 28
 }
diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 120f7ed651..311820764d 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -59,12 +59,12 @@
 Device Information is stored in TEST_DEVICES in print_matrix_configuration.py
 Example:
 sdk id "system-images;android-29;google_apis;x86":
-  --android_sdk "system-images;android-29;google_apis;x86" --build_tools_version "28.0.3"
+  --android_sdk "system-images;android-29;google_apis;x86" --build_tools_version "29.0.2"
 
 Alternatively, to set an Android device, use the one of the values below:
 [emulator_min, emulator_target, emulator_latest]
 Example:
-  --android_device "emulator_target" --build_tools_version "28.0.3"
+  --android_device "emulator_target" --build_tools_version "29.0.2"
 
 Returns:
    1: No iOS/Android integration_test apps found
@@ -129,7 +129,7 @@
     "android_sdk", "system-images;android-29;google_apis;x86",
     "See module docstring for details on how to set and get this id.")
 flags.DEFINE_string(
-    "build_tools_version", "28.0.3",
+    "build_tools_version", "29.0.2",
     "android build_tools_version")
 flags.DEFINE_string(
     "logfile_name", "simulator-test",

From dd7ceca1ee47052fe6bed0a41ec59c19b9397569 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Thu, 4 Nov 2021 15:35:51 -0700
Subject: [PATCH 04/13] retry 3 time install/uninstall android apps

---
 scripts/gha/test_simulator.py | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 311820764d..e72c6623a1 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -280,7 +280,7 @@ def main(argv):
     _create_and_boot_emulator(sdk_id)
 
     android_gameloop_project = os.path.join(current_dir, "integration_testing", "gameloop_android")
-    _install_android_gameloop_app(android_gameloop_project)
+    _install_android_gameloop_app(android_gameloop_project, _TEST_RETRY)
 
     for app_path in android_testapps:
       package_name = _get_package_name(app_path)
@@ -580,10 +580,10 @@ def _get_package_name(app_path):
 
 def _run_android_gameloop_test(package_name, app_path, gameloop_project, retry=1): 
   logging.info("Running android gameloop test: %s, %s, %s", package_name, app_path, gameloop_project)
-  _install_android_app(app_path)
+  _install_android_app(app_path, _TEST_RETRY)
   _run_instrumented_test()
   log = _get_android_test_log(package_name)
-  _uninstall_android_app(package_name)
+  _uninstall_android_app(package_name, _TEST_RETRY)
   if retry > 1:
     result = test_validation.validate_results(log, test_validation.CPP)
     if not result.complete:
@@ -593,30 +593,34 @@ def _run_android_gameloop_test(package_name, app_path, gameloop_project, retry=1
   return log
 
 
-def _install_android_app(app_path):
+def _install_android_app(app_path, retry=1):
   """Install integration_test app into the emulator."""
   args = ["adb", "install", app_path]
   logging.info("Install testapp: %s", " ".join(args))
-  result = subprocess.run(args=args, check=False)
+  check = (retry <= 1)
+  result = subprocess.run(args=args, check=check)
   if result.returncode != 0:
-    logging.info("Install testapp %s failed, Reset Emualtor now...", app_path)
+    logging.info("Install testapp %s failed, Reset Emualtor now... Remaining retry: %s", app_path, retry-1)
     args = ["adb", "shell", "recovery", "--wipe_data"]
     logging.info("Erase my Emualtor: %s", " ".join(args))
     subprocess.run(args=args, check=True)
     _reset_emulator_on_error()
-    args = ["adb", "install", app_path]
-    logging.info("Reinstall testapp: %s", " ".join(args))
-    subprocess.run(args=args, check=True)
+    _install_android_app(app_path, retry-1)
 
 
-def _uninstall_android_app(package_name):
+def _uninstall_android_app(package_name, retry=1):
   """Uninstall integration_test app from the emulator."""
   args = ["adb", "uninstall", package_name]
   logging.info("Uninstall testapp: %s", " ".join(args))
-  subprocess.run(args=args, check=False)
+  check = (retry <= 1)
+  result = subprocess.run(args=args, check=check)
+  if result.returncode != 0:
+    logging.info("Uninstall testapp %s failed, Reset Emualtor now... Remaining retry: %s", package_name, retry-1)
+    _reset_emulator_on_error()
+    _uninstall_android_app(package_name, retry-1)
 
 
-def _install_android_gameloop_app(gameloop_project):
+def _install_android_gameloop_app(gameloop_project, retry=1):
   os.chdir(gameloop_project)
   logging.info("CD to gameloop_project: %s", gameloop_project)
   _uninstall_android_app("com.google.firebase.gameloop")
@@ -625,7 +629,12 @@ def _install_android_gameloop_app(gameloop_project):
   subprocess.run(args=args, check=False)
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
-  subprocess.run(args=args, check=True)
+  check = (retry <= 1)
+  result = subprocess.run(args=args, check=check)
+  if result.returncode != 0:
+    logging.info("Install gameloop_project failed, Reset Emualtor now... Remaining retry: %s", retry-1)
+    _reset_emulator_on_error()
+    _install_android_gameloop_app(gameloop_project, retry-1)
 
 
 def _run_instrumented_test():

From 823b05671853f02624b2b96b94e60b5b49b44b67 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Thu, 4 Nov 2021 16:15:43 -0700
Subject: [PATCH 05/13] critical steps 5 minutes timeout. Testing job 90
 minutes timeout

---
 .github/workflows/integration_tests.yml |  2 +-
 scripts/gha/test_simulator.py           | 26 +++++++++++++++----------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 78f4676b9e..eaaf4441f4 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -829,7 +829,7 @@ jobs:
           npm install -g firebase-tools
           firebase emulators:start --only firestore --project demo-example &
       - name: Run Android integration tests on Emulator locally
-        timeout-minutes: 60
+        timeout-minutes: 90
         if: steps.get-device-type.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index e72c6623a1..c54604cd6f 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -92,6 +92,7 @@
 _GAMELOOP_PACKAGE = "com.google.firebase.gameloop"
 _RESULT_FILE = "Results1.json"
 _TEST_RETRY = 3
+_CMD_TIMEOUT = 300
 
 FLAGS = flags.FLAGS
 
@@ -275,7 +276,7 @@ def main(argv):
       logging.error("Please set JAVA_HOME to java 8")
       return 31
 
-    _setup_android(platform_version, build_tool_version, sdk_id)  
+    _setup_android(platform_version, build_tool_version, sdk_id, _TEST_RETRY)  
 
     _create_and_boot_emulator(sdk_id)
 
@@ -487,7 +488,8 @@ def _check_java_version():
   return "1.8" in java_version
 
 
-def _setup_android(platform_version, build_tool_version, sdk_id):
+def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
+  check = (retry <= 1)
   android_home = os.environ["ANDROID_HOME"]
   pathlist = [os.path.join(android_home, "emulator"), 
     os.path.join(android_home, "tools"), 
@@ -501,7 +503,9 @@ def _setup_android(platform_version, build_tool_version, sdk_id):
     "platforms;%s" % platform_version, 
     "build-tools;%s" % build_tool_version]
   logging.info("Install packages: %s", " ".join(args))
-  subprocess.run(args=args, check=True)
+  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
+  if result.returncode != 0:
+    _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
 
   command = "yes | sdkmanager --licenses"
   logging.info("Accept all licenses: %s", command)
@@ -509,11 +513,13 @@ def _setup_android(platform_version, build_tool_version, sdk_id):
 
   args = ["sdkmanager", sdk_id]
   logging.info("Download an emulator: %s", " ".join(args))
-  subprocess.run(args=args, check=True)
+  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
+  if result.returncode != 0:
+    _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
 
   args = ["sdkmanager", "--update"]
   logging.info("Update all installed packages: %s", " ".join(args))
-  subprocess.run(args=args, check=True)
+  subprocess.run(args=args, check=False)
 
 
 def _shutdown_emulator():
@@ -598,7 +604,7 @@ def _install_android_app(app_path, retry=1):
   args = ["adb", "install", app_path]
   logging.info("Install testapp: %s", " ".join(args))
   check = (retry <= 1)
-  result = subprocess.run(args=args, check=check)
+  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
   if result.returncode != 0:
     logging.info("Install testapp %s failed, Reset Emualtor now... Remaining retry: %s", app_path, retry-1)
     args = ["adb", "shell", "recovery", "--wipe_data"]
@@ -613,7 +619,7 @@ def _uninstall_android_app(package_name, retry=1):
   args = ["adb", "uninstall", package_name]
   logging.info("Uninstall testapp: %s", " ".join(args))
   check = (retry <= 1)
-  result = subprocess.run(args=args, check=check)
+  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
   if result.returncode != 0:
     logging.info("Uninstall testapp %s failed, Reset Emualtor now... Remaining retry: %s", package_name, retry-1)
     _reset_emulator_on_error()
@@ -622,7 +628,7 @@ def _uninstall_android_app(package_name, retry=1):
 
 def _install_android_gameloop_app(gameloop_project, retry=1):
   os.chdir(gameloop_project)
-  logging.info("CD to gameloop_project: %s", gameloop_project)
+  logging.info("cd to gameloop_project: %s", gameloop_project)
   _uninstall_android_app("com.google.firebase.gameloop")
   args = ["./gradlew", "clean"]
   logging.info("Clean game-loop cache: %s", " ".join(args))
@@ -630,7 +636,7 @@ def _install_android_gameloop_app(gameloop_project, retry=1):
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
   check = (retry <= 1)
-  result = subprocess.run(args=args, check=check)
+  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
   if result.returncode != 0:
     logging.info("Install gameloop_project failed, Reset Emualtor now... Remaining retry: %s", retry-1)
     _reset_emulator_on_error()
@@ -644,7 +650,7 @@ def _run_instrumented_test():
   args = ["adb", "shell", "am", "instrument",
     "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % _GAMELOOP_PACKAGE] 
   logging.info("Running game-loop test: %s", " ".join(args))
-  result = subprocess.run(args=args, capture_output=True, text=True, check=False) 
+  result = subprocess.run(args=args, capture_output=True, text=True, check=False, timeout=_CMD_TIMEOUT) 
   if "FAILURES!!!" in result.stdout:
     logging.info("game-loop test result: %s", result.stdout)
     _reset_emulator_on_error()

From 6092ee9a02e14bc85b060a06f166ccb3313e2b2b Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Thu, 4 Nov 2021 20:02:17 -0700
Subject: [PATCH 06/13] do not force unistall apps

---
 scripts/gha/test_simulator.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index c54604cd6f..cb9b63dc6a 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -618,9 +618,8 @@ def _uninstall_android_app(package_name, retry=1):
   """Uninstall integration_test app from the emulator."""
   args = ["adb", "uninstall", package_name]
   logging.info("Uninstall testapp: %s", " ".join(args))
-  check = (retry <= 1)
-  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
-  if result.returncode != 0:
+  result = subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+  if retry > 1 and result.returncode != 0:
     logging.info("Uninstall testapp %s failed, Reset Emualtor now... Remaining retry: %s", package_name, retry-1)
     _reset_emulator_on_error()
     _uninstall_android_app(package_name, retry-1)

From 8e514bceda6e95c47cd26221e7ae4e93a98d3a90 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Thu, 4 Nov 2021 23:43:13 -0700
Subject: [PATCH 07/13] retry catch except

---
 scripts/gha/test_simulator.py | 83 ++++++++++++++++++++++-------------
 1 file changed, 53 insertions(+), 30 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index cb9b63dc6a..75c93ddd06 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -489,7 +489,6 @@ def _check_java_version():
 
 
 def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
-  check = (retry <= 1)
   android_home = os.environ["ANDROID_HOME"]
   pathlist = [os.path.join(android_home, "emulator"), 
     os.path.join(android_home, "tools"), 
@@ -503,9 +502,14 @@ def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
     "platforms;%s" % platform_version, 
     "build-tools;%s" % build_tool_version]
   logging.info("Install packages: %s", " ".join(args))
-  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
-  if result.returncode != 0:
-    _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+    except:
+      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
+      return
+  else:
+    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
 
   command = "yes | sdkmanager --licenses"
   logging.info("Accept all licenses: %s", command)
@@ -513,9 +517,14 @@ def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
 
   args = ["sdkmanager", sdk_id]
   logging.info("Download an emulator: %s", " ".join(args))
-  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
-  if result.returncode != 0:
-    _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+    except:
+      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
+      return
+  else:
+      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
 
   args = ["sdkmanager", "--update"]
   logging.info("Update all installed packages: %s", " ".join(args))
@@ -603,26 +612,33 @@ def _install_android_app(app_path, retry=1):
   """Install integration_test app into the emulator."""
   args = ["adb", "install", app_path]
   logging.info("Install testapp: %s", " ".join(args))
-  check = (retry <= 1)
-  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
-  if result.returncode != 0:
-    logging.info("Install testapp %s failed, Reset Emualtor now... Remaining retry: %s", app_path, retry-1)
-    args = ["adb", "shell", "recovery", "--wipe_data"]
-    logging.info("Erase my Emualtor: %s", " ".join(args))
-    subprocess.run(args=args, check=True)
-    _reset_emulator_on_error()
-    _install_android_app(app_path, retry-1)
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+    except:
+      logging.info("Install testapp %s failed, Reset Emualtor now... Remaining retry: %s", app_path, retry-1)
+      args = ["adb", "shell", "recovery", "--wipe_data"]
+      logging.info("Erase my Emualtor: %s", " ".join(args))
+      subprocess.run(args=args, check=True)
+      _reset_emulator_on_error()
+      _install_android_app(app_path, retry-1)
+  else:
+    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
 
 
 def _uninstall_android_app(package_name, retry=1):
   """Uninstall integration_test app from the emulator."""
   args = ["adb", "uninstall", package_name]
   logging.info("Uninstall testapp: %s", " ".join(args))
-  result = subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
-  if retry > 1 and result.returncode != 0:
-    logging.info("Uninstall testapp %s failed, Reset Emualtor now... Remaining retry: %s", package_name, retry-1)
-    _reset_emulator_on_error()
-    _uninstall_android_app(package_name, retry-1)
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+    except:
+      logging.info("Uninstall testapp %s failed, Reset Emualtor now... Remaining retry: %s", package_name, retry-1)
+      _reset_emulator_on_error()
+      _uninstall_android_app(package_name, retry-1)
+  else:
+    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
 
 
 def _install_android_gameloop_app(gameloop_project, retry=1):
@@ -634,12 +650,15 @@ def _install_android_gameloop_app(gameloop_project, retry=1):
   subprocess.run(args=args, check=False)
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
-  check = (retry <= 1)
-  result = subprocess.run(args=args, check=check, timeout=_CMD_TIMEOUT)
-  if result.returncode != 0:
-    logging.info("Install gameloop_project failed, Reset Emualtor now... Remaining retry: %s", retry-1)
-    _reset_emulator_on_error()
-    _install_android_gameloop_app(gameloop_project, retry-1)
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+    except:
+      logging.info("Install gameloop_project failed, Reset Emualtor now... Remaining retry: %s", retry-1)
+      _reset_emulator_on_error()
+      _install_android_gameloop_app(gameloop_project, retry-1)
+  else:
+    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
 
 
 def _run_instrumented_test():
@@ -649,9 +668,13 @@ def _run_instrumented_test():
   args = ["adb", "shell", "am", "instrument",
     "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % _GAMELOOP_PACKAGE] 
   logging.info("Running game-loop test: %s", " ".join(args))
-  result = subprocess.run(args=args, capture_output=True, text=True, check=False, timeout=_CMD_TIMEOUT) 
-  if "FAILURES!!!" in result.stdout:
-    logging.info("game-loop test result: %s", result.stdout)
+  try:
+    result = subprocess.run(args=args, capture_output=True, text=True, check=False, timeout=800) 
+    if "FAILURES!!!" in result.stdout:
+      logging.info("game-loop test result: %s", result.stdout)
+      _reset_emulator_on_error()
+  except:
+    logging.info("Running game-loop test except: timeout")
     _reset_emulator_on_error()
 
 

From f216c1f975b457f63e832337d1fe76b3a5885ee3 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Fri, 5 Nov 2021 08:13:07 -0700
Subject: [PATCH 08/13] do not force unistall gameloop

---
 scripts/gha/test_simulator.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 75c93ddd06..b4f06eeb8e 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -644,7 +644,17 @@ def _uninstall_android_app(package_name, retry=1):
 def _install_android_gameloop_app(gameloop_project, retry=1):
   os.chdir(gameloop_project)
   logging.info("cd to gameloop_project: %s", gameloop_project)
-  _uninstall_android_app("com.google.firebase.gameloop")
+  args = ["adb", "uninstall", "com.google.firebase.gameloop"]
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+    except:
+      logging.info("Uninstall gameloop_project timeout, Reset Emualtor now... Remaining retry: %s", retry-1)
+      _reset_emulator_on_error()
+      _install_android_gameloop_app(gameloop_project, retry-1)
+      return
+  else:
+    subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
   args = ["./gradlew", "clean"]
   logging.info("Clean game-loop cache: %s", " ".join(args))
   subprocess.run(args=args, check=False)

From b606a8ed4a692779acddaadab9aa30be16d672dd Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Fri, 5 Nov 2021 08:22:49 -0700
Subject: [PATCH 09/13] ./gradlew clean with timeout

---
 scripts/gha/test_simulator.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index b4f06eeb8e..53c1487644 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -657,7 +657,16 @@ def _install_android_gameloop_app(gameloop_project, retry=1):
     subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
   args = ["./gradlew", "clean"]
   logging.info("Clean game-loop cache: %s", " ".join(args))
-  subprocess.run(args=args, check=False)
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+    except:
+      logging.info("Clean game-loop cache timeout, Reset Emualtor now... Remaining retry: %s", retry-1)
+      _reset_emulator_on_error()
+      _install_android_gameloop_app(gameloop_project, retry-1)
+      return
+  else:
+    subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
   if retry > 1:

From 1bb22cce165e4c2cabe754bab06a5dd6129d56d3 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Fri, 5 Nov 2021 11:22:28 -0700
Subject: [PATCH 10/13] fix format

---
 scripts/gha/test_simulator.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 53c1487644..d9af3a1ced 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -513,7 +513,14 @@ def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
 
   command = "yes | sdkmanager --licenses"
   logging.info("Accept all licenses: %s", command)
-  subprocess.run(command, shell=True, check=False)
+  if retry > 1:
+    try:
+      subprocess.run(command, shell=True, check=False, timeout=_CMD_TIMEOUT)
+    except:
+      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
+      return
+  else:
+    subprocess.run(command, shell=True, check=False, timeout=_CMD_TIMEOUT)
 
   args = ["sdkmanager", sdk_id]
   logging.info("Download an emulator: %s", " ".join(args))
@@ -524,12 +531,18 @@ def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
       _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
       return
   else:
-      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
 
   args = ["sdkmanager", "--update"]
   logging.info("Update all installed packages: %s", " ".join(args))
-  subprocess.run(args=args, check=False)
-
+  if retry > 1:
+    try:
+      subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+    except:
+      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
+      return
+  else:
+    subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
 
 def _shutdown_emulator():
   command = "adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done"

From 0bf525b6e9dc112ddfedcd742c226f08fa8e9f00 Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Mon, 8 Nov 2021 12:00:31 -0800
Subject: [PATCH 11/13] run with retry

---
 scripts/gha/test_simulator.py | 148 +++++++++++-----------------------
 1 file changed, 49 insertions(+), 99 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index d9af3a1ced..09e5128a7c 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -94,6 +94,10 @@
 _TEST_RETRY = 3
 _CMD_TIMEOUT = 300
 
+_RESET_TYPE_NONE = 0
+_RESET_TYPE_REBOOT = 1
+_RESET_TYPE_WIPE_REBOOT = 2
+
 FLAGS = flags.FLAGS
 
 flags.DEFINE_string(
@@ -276,12 +280,12 @@ def main(argv):
       logging.error("Please set JAVA_HOME to java 8")
       return 31
 
-    _setup_android(platform_version, build_tool_version, sdk_id, _TEST_RETRY)  
+    _setup_android(platform_version, build_tool_version, sdk_id)  
 
     _create_and_boot_emulator(sdk_id)
 
     android_gameloop_project = os.path.join(current_dir, "integration_testing", "gameloop_android")
-    _install_android_gameloop_app(android_gameloop_project, _TEST_RETRY)
+    _install_android_gameloop_app(android_gameloop_project)
 
     for app_path in android_testapps:
       package_name = _get_package_name(app_path)
@@ -488,7 +492,7 @@ def _check_java_version():
   return "1.8" in java_version
 
 
-def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
+def _setup_android(platform_version, build_tool_version, sdk_id):
   android_home = os.environ["ANDROID_HOME"]
   pathlist = [os.path.join(android_home, "emulator"), 
     os.path.join(android_home, "tools"), 
@@ -502,47 +506,20 @@ def _setup_android(platform_version, build_tool_version, sdk_id, retry=1):
     "platforms;%s" % platform_version, 
     "build-tools;%s" % build_tool_version]
   logging.info("Install packages: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
-    except:
-      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
-      return
-  else:
-    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args)
 
   command = "yes | sdkmanager --licenses"
   logging.info("Accept all licenses: %s", command)
-  if retry > 1:
-    try:
-      subprocess.run(command, shell=True, check=False, timeout=_CMD_TIMEOUT)
-    except:
-      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
-      return
-  else:
-    subprocess.run(command, shell=True, check=False, timeout=_CMD_TIMEOUT)
+  _run_with_retry(command, shell=True, check=False)
 
   args = ["sdkmanager", sdk_id]
   logging.info("Download an emulator: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
-    except:
-      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
-      return
-  else:
-    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args)
 
   args = ["sdkmanager", "--update"]
   logging.info("Update all installed packages: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
-    except:
-      _setup_android(platform_version, build_tool_version, sdk_id, retry-1)
-      return
-  else:
-    subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args, check=False)
+
 
 def _shutdown_emulator():
   command = "adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done"
@@ -582,8 +559,29 @@ def _create_and_boot_emulator(sdk_id):
   else:
     time.sleep(45)
 
+def _run_with_retry(args, shell=False, check=True, timeout=_CMD_TIMEOUT, retry_time=_TEST_RETRY, retry_with=_RESET_TYPE_NONE):
+  if retry_time > 1:
+    try:
+      subprocess.run(args, shell=shell, check=check, timeout=timeout)
+    except:
+      _reset_emulator_on_error(retry_with)
+      _run_with_retry(args, shell, check, timeout, retry_time-1, retry_with)
+  else:
+    subprocess.run(args, shell=shell, check=check, timeout=timeout)
+
+
+def _reset_emulator_on_error(type=_RESET_TYPE_NONE):
+  if type == _RESET_TYPE_NONE:
+    return
+
+  if type == _RESET_TYPE_WIPE_REBOOT:
+    # wipe emulator data
+    args = ["adb", "shell", "recovery", "--wipe_data"]
+    logging.info("Erase my Emulator: %s", " ".join(args))
+    subprocess.run(args=args, check=True)
+    _reset_emulator_on_error()
 
-def _reset_emulator_on_error():
+  # reboot emulator: _RESET_TYPE_WIPE_REBOOT, _RESET_TYPE_REBOOT
   logging.info("game-loop test error!!! reboot emualtor...")
   args = ["adb", "-e", "reboot"]
   logging.info("Reboot android emulator: %s", " ".join(args))
@@ -608,10 +606,10 @@ def _get_package_name(app_path):
 
 def _run_android_gameloop_test(package_name, app_path, gameloop_project, retry=1): 
   logging.info("Running android gameloop test: %s, %s, %s", package_name, app_path, gameloop_project)
-  _install_android_app(app_path, _TEST_RETRY)
+  _install_android_app(app_path)
   _run_instrumented_test()
   log = _get_android_test_log(package_name)
-  _uninstall_android_app(package_name, _TEST_RETRY)
+  _uninstall_android_app(package_name)
   if retry > 1:
     result = test_validation.validate_results(log, test_validation.CPP)
     if not result.complete:
@@ -621,76 +619,33 @@ def _run_android_gameloop_test(package_name, app_path, gameloop_project, retry=1
   return log
 
 
-def _install_android_app(app_path, retry=1):
+def _install_android_app(app_path):
   """Install integration_test app into the emulator."""
   args = ["adb", "install", app_path]
   logging.info("Install testapp: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
-    except:
-      logging.info("Install testapp %s failed, Reset Emualtor now... Remaining retry: %s", app_path, retry-1)
-      args = ["adb", "shell", "recovery", "--wipe_data"]
-      logging.info("Erase my Emualtor: %s", " ".join(args))
-      subprocess.run(args=args, check=True)
-      _reset_emulator_on_error()
-      _install_android_app(app_path, retry-1)
-  else:
-    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args, retry_with=_RESET_TYPE_WIPE_REBOOT)
 
 
-def _uninstall_android_app(package_name, retry=1):
+def _uninstall_android_app(package_name):
   """Uninstall integration_test app from the emulator."""
   args = ["adb", "uninstall", package_name]
   logging.info("Uninstall testapp: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
-    except:
-      logging.info("Uninstall testapp %s failed, Reset Emualtor now... Remaining retry: %s", package_name, retry-1)
-      _reset_emulator_on_error()
-      _uninstall_android_app(package_name, retry-1)
-  else:
-    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args, retry_with=_RESET_TYPE_REBOOT)
 
 
-def _install_android_gameloop_app(gameloop_project, retry=1):
+def _install_android_gameloop_app(gameloop_project):
   os.chdir(gameloop_project)
   logging.info("cd to gameloop_project: %s", gameloop_project)
   args = ["adb", "uninstall", "com.google.firebase.gameloop"]
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
-    except:
-      logging.info("Uninstall gameloop_project timeout, Reset Emualtor now... Remaining retry: %s", retry-1)
-      _reset_emulator_on_error()
-      _install_android_gameloop_app(gameloop_project, retry-1)
-      return
-  else:
-    subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args, check=False, retry_with=_RESET_TYPE_REBOOT)
+
   args = ["./gradlew", "clean"]
   logging.info("Clean game-loop cache: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
-    except:
-      logging.info("Clean game-loop cache timeout, Reset Emualtor now... Remaining retry: %s", retry-1)
-      _reset_emulator_on_error()
-      _install_android_gameloop_app(gameloop_project, retry-1)
-      return
-  else:
-    subprocess.run(args=args, check=False, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args, check=False, retry_with=_RESET_TYPE_REBOOT)
+
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
-  if retry > 1:
-    try:
-      subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
-    except:
-      logging.info("Install gameloop_project failed, Reset Emualtor now... Remaining retry: %s", retry-1)
-      _reset_emulator_on_error()
-      _install_android_gameloop_app(gameloop_project, retry-1)
-  else:
-    subprocess.run(args=args, check=True, timeout=_CMD_TIMEOUT)
+  _run_with_retry(args, retry_with=_RESET_TYPE_REBOOT)
 
 
 def _run_instrumented_test():
@@ -700,14 +655,9 @@ def _run_instrumented_test():
   args = ["adb", "shell", "am", "instrument",
     "-w", "%s.test/androidx.test.runner.AndroidJUnitRunner" % _GAMELOOP_PACKAGE] 
   logging.info("Running game-loop test: %s", " ".join(args))
-  try:
-    result = subprocess.run(args=args, capture_output=True, text=True, check=False, timeout=800) 
-    if "FAILURES!!!" in result.stdout:
-      logging.info("game-loop test result: %s", result.stdout)
-      _reset_emulator_on_error()
-  except:
-    logging.info("Running game-loop test except: timeout")
-    _reset_emulator_on_error()
+  result = subprocess.run(args=args, capture_output=True, text=True, check=False) 
+  if "FAILURES!!!" in result.stdout:
+    _reset_emulator_on_error(_RESET_TYPE_REBOOT)
 
 
 def _get_android_test_log(test_package):

From e9d22bdeed023ff80150fedcf3e5e6df7b33d98b Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Mon, 8 Nov 2021 13:13:57 -0800
Subject: [PATCH 12/13] run with retry support iOS

---
 scripts/gha/test_simulator.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 09e5128a7c..7e6be017df 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -94,7 +94,10 @@
 _TEST_RETRY = 3
 _CMD_TIMEOUT = 300
 
-_RESET_TYPE_NONE = 0
+_DEVICE_NONE = "None"
+_DEVICE_ANDROID = "Android"
+_DEVICE_APPLE = "Apple"
+
 _RESET_TYPE_REBOOT = 1
 _RESET_TYPE_WIPE_REBOOT = 2
 
@@ -559,21 +562,19 @@ def _create_and_boot_emulator(sdk_id):
   else:
     time.sleep(45)
 
-def _run_with_retry(args, shell=False, check=True, timeout=_CMD_TIMEOUT, retry_time=_TEST_RETRY, retry_with=_RESET_TYPE_NONE):
+def _run_with_retry(args, shell=False, check=True, timeout=_CMD_TIMEOUT, retry_time=_TEST_RETRY, device=_DEVICE_NONE, type=_RESET_TYPE_REBOOT):
   if retry_time > 1:
     try:
       subprocess.run(args, shell=shell, check=check, timeout=timeout)
     except:
-      _reset_emulator_on_error(retry_with)
-      _run_with_retry(args, shell, check, timeout, retry_time-1, retry_with)
+      if device == _DEVICE_ANDROID:
+        _reset_emulator_on_error(type)
+      _run_with_retry(args, shell, check, timeout, retry_time-1, device, type)
   else:
     subprocess.run(args, shell=shell, check=check, timeout=timeout)
 
 
-def _reset_emulator_on_error(type=_RESET_TYPE_NONE):
-  if type == _RESET_TYPE_NONE:
-    return
-
+def _reset_emulator_on_error(type=_RESET_TYPE_REBOOT):
   if type == _RESET_TYPE_WIPE_REBOOT:
     # wipe emulator data
     args = ["adb", "shell", "recovery", "--wipe_data"]
@@ -623,29 +624,29 @@ def _install_android_app(app_path):
   """Install integration_test app into the emulator."""
   args = ["adb", "install", app_path]
   logging.info("Install testapp: %s", " ".join(args))
-  _run_with_retry(args, retry_with=_RESET_TYPE_WIPE_REBOOT)
+  _run_with_retry(args, device=_DEVICE_ANDROID, type=_RESET_TYPE_WIPE_REBOOT)
 
 
 def _uninstall_android_app(package_name):
   """Uninstall integration_test app from the emulator."""
   args = ["adb", "uninstall", package_name]
   logging.info("Uninstall testapp: %s", " ".join(args))
-  _run_with_retry(args, retry_with=_RESET_TYPE_REBOOT)
+  _run_with_retry(args, device=_DEVICE_ANDROID, type=_RESET_TYPE_REBOOT)
 
 
 def _install_android_gameloop_app(gameloop_project):
   os.chdir(gameloop_project)
   logging.info("cd to gameloop_project: %s", gameloop_project)
   args = ["adb", "uninstall", "com.google.firebase.gameloop"]
-  _run_with_retry(args, check=False, retry_with=_RESET_TYPE_REBOOT)
+  _run_with_retry(args, check=False, device=_DEVICE_ANDROID, type=_RESET_TYPE_REBOOT)
 
   args = ["./gradlew", "clean"]
   logging.info("Clean game-loop cache: %s", " ".join(args))
-  _run_with_retry(args, check=False, retry_with=_RESET_TYPE_REBOOT)
+  _run_with_retry(args, check=False, device=_DEVICE_ANDROID, type=_RESET_TYPE_REBOOT)
 
   args = ["./gradlew", "installDebug", "installDebugAndroidTest"]
   logging.info("Installing game-loop app and test: %s", " ".join(args))
-  _run_with_retry(args, retry_with=_RESET_TYPE_REBOOT)
+  _run_with_retry(args, device=_DEVICE_ANDROID, type=_RESET_TYPE_REBOOT)
 
 
 def _run_instrumented_test():

From 1411e401a4c42336e4cab834d724595d3a9acfda Mon Sep 17 00:00:00 2001
From: Mou <sunmou@google.com>
Date: Mon, 8 Nov 2021 13:28:54 -0800
Subject: [PATCH 13/13] fix bug

---
 scripts/gha/test_simulator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/gha/test_simulator.py b/scripts/gha/test_simulator.py
index 7e6be017df..a1407a478c 100644
--- a/scripts/gha/test_simulator.py
+++ b/scripts/gha/test_simulator.py
@@ -580,7 +580,6 @@ def _reset_emulator_on_error(type=_RESET_TYPE_REBOOT):
     args = ["adb", "shell", "recovery", "--wipe_data"]
     logging.info("Erase my Emulator: %s", " ".join(args))
     subprocess.run(args=args, check=True)
-    _reset_emulator_on_error()
 
   # reboot emulator: _RESET_TYPE_WIPE_REBOOT, _RESET_TYPE_REBOOT
   logging.info("game-loop test error!!! reboot emualtor...")