From 46354c09fe2999ceabcf4bd76c039dbf7526000f Mon Sep 17 00:00:00 2001
From: Jason Simmons <jason-simmons@users.noreply.github.com>
Date: Fri, 16 Feb 2024 16:56:05 -0800
Subject: [PATCH] Restore use of the API 34 device in the Firebase Test Lab
 script and handle FTL infrastructure errors in the script
 (flutter/engine#50735)

This test had been temporarily moved to an API 33 device as a workaround for an FTL infrastructure issue (see https://github.com/flutter/engine/pull/50721)

This PR resumes use of a Pixel 8/API 34 device for the FTL tests.  It also retries the test if FTL returns an error code that is known to represent an FTL infrastructure error.  If the retries fail, then the test script will not block the engine tree if all failures were caused by infrastructure.

This is similar to what the CI recipes are doing in https://flutter.googlesource.com/recipes/+/a181878fde742dacd94afca04c5a4db1b3c30b91
---
 engine/src/flutter/ci/firebase_testlab.py | 62 +++++++++++++++--------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/engine/src/flutter/ci/firebase_testlab.py b/engine/src/flutter/ci/firebase_testlab.py
index 4a88aebe5ee..d27277827a7 100755
--- a/engine/src/flutter/ci/firebase_testlab.py
+++ b/engine/src/flutter/ci/firebase_testlab.py
@@ -22,6 +22,12 @@ if 'GCP_PROJECT' not in os.environ:
   sys.exit(1)
 PROJECT = os.environ['GCP_PROJECT']
 
+# Exit codes returned by the FTL command that signal an infrastructure failure.
+FTL_INFRA_FAILURE_CODES = [1, 15, 20]
+
+# Maximum number of retries done if an infrastructure failure occurs.
+MAX_RETRY_ATTEMPTS = 2
+
 script_dir = os.path.dirname(os.path.realpath(__file__))
 buildroot_dir = os.path.abspath(os.path.join(script_dir, '..', '..'))
 out_dir = os.path.join(buildroot_dir, 'out')
@@ -54,7 +60,7 @@ def run_firebase_test(apk, results_dir):
           '--results-dir',
           results_dir,
           '--device',
-          'model=panther,version=33',
+          'model=shiba,version=34',
       ],
       stdout=subprocess.PIPE,
       stderr=subprocess.STDOUT,
@@ -106,7 +112,7 @@ def main():
   args = parser.parse_args()
 
   apks_dir = os.path.join(out_dir, args.variant, 'firebase_apks')
-  apks = glob.glob('%s/*.apk' % apks_dir)
+  apks = set(glob.glob('%s/*.apk' % apks_dir))
 
   if not apks:
     print('No APKs found at %s' % apks_dir)
@@ -115,27 +121,41 @@ def main():
   git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=script_dir)
   git_revision = byte_str_decode(git_revision)
   git_revision = git_revision.strip()
-  results = []
-  apk = None
-  for apk in apks:
-    results_dir = '%s/%s/%s' % (os.path.basename(apk), git_revision, args.build_id)
-    process = run_firebase_test(apk, results_dir)
-    results.append((results_dir, process))
 
-  for results_dir, process in results:
-    for line in iter(process.stdout.readline, ''):
-      print(line.strip())
-    return_code = process.wait()
-    if return_code != 0:
-      print('Firebase test failed with code: %s' % return_code)
-      sys.exit(return_code)
+  for retry in range(MAX_RETRY_ATTEMPTS):
+    if retry > 0:
+      print('Retrying %s' % apks)
 
-    print('Checking logcat for %s' % results_dir)
-    check_logcat(results_dir)
-    # scenario_app produces a timeline, but the android image test does not.
-    if 'scenario' in apk:
-      print('Checking timeline for %s' % results_dir)
-      check_timeline(results_dir)
+    results = []
+    for apk in sorted(apks):
+      results_dir = '%s/%s/%s' % (os.path.basename(apk), git_revision, args.build_id)
+      process = run_firebase_test(apk, results_dir)
+      results.append((apk, results_dir, process))
+
+    for apk, results_dir, process in results:
+      print('===== Test output for %s' % apk)
+      for line in iter(process.stdout.readline, ''):
+        print(line.strip())
+
+      return_code = process.wait()
+      if return_code in FTL_INFRA_FAILURE_CODES:
+        print('Firebase test %s failed with infrastructure error code: %s' % (apk, return_code))
+        continue
+      if return_code != 0:
+        print('Firebase test %s failed with code: %s' % (apk, return_code))
+        sys.exit(return_code)
+
+      print('Checking logcat for %s' % results_dir)
+      check_logcat(results_dir)
+      # scenario_app produces a timeline, but the android image test does not.
+      if 'scenario' in apk:
+        print('Checking timeline for %s' % results_dir)
+        check_timeline(results_dir)
+
+      apks.remove(apk)
+
+    if not apks:
+      break
 
   return 0