From 75bfeb8fb02a5e15c8f8683e1e4da2db7e27cae1 Mon Sep 17 00:00:00 2001
From: Matan Lurey <matanlurey@users.noreply.github.com>
Date: Tue, 27 Feb 2024 11:08:18 -0800
Subject: [PATCH] Fail lazily when 1+ Skia gold comparions fail.
 (flutter/engine#51010)

(Speculative) fix for https://github.com/flutter/flutter/issues/144238.

@zanderso Could you review from a CLI angle? I realize this code is not great, but could I do something better here?
---
 .../testing/scenario_app/bin/run_android_tests.dart       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/engine/src/flutter/testing/scenario_app/bin/run_android_tests.dart b/engine/src/flutter/testing/scenario_app/bin/run_android_tests.dart
index 88cc4623612..37db27a5b51 100644
--- a/engine/src/flutter/testing/scenario_app/bin/run_android_tests.dart
+++ b/engine/src/flutter/testing/scenario_app/bin/run_android_tests.dart
@@ -157,6 +157,7 @@ Future<void> _run({
   late final ServerSocket server;
   final List<Future<void>> pendingComparisons = <Future<void>>[];
   final List<Socket> pendingConnections = <Socket>[];
+  int comparisonsFailed = 0;
   await step('Starting server...', () async {
     server = await ServerSocket.bind(InternetAddress.anyIPv4, _tcpPort);
     if (verbose) {
@@ -186,8 +187,9 @@ Future<void> _run({
         if (isSkiaGoldClientAvailable) {
           final Future<void> comparison = skiaGoldClient!
               .addImg(fileName, goldenFile, screenshotSize: screenshot.pixelCount)
-              .catchError((dynamic err) {
-            panic(<String>['skia gold comparison failed: $err']);
+              .catchError((Object error) {
+            logWarning('skia gold comparison failed: $error');
+            comparisonsFailed++;
           });
           pendingComparisons.add(comparison);
         }
@@ -354,6 +356,8 @@ Future<void> _run({
       if (out.toString().contains('FAILURES!!!')) {
         stdout.write(out);
         panic(<String>['1 or more tests failed']);
+      } else if (comparisonsFailed > 0) {
+        panic(<String>['$comparisonsFailed Skia Gold comparisons failed']);
       }
     });
   } finally {