Retry Xcode builds if they fail due to concurrent builds running (#45608)

* Retry Xcode builds if they fail due to concurrent builds running

Fixes #40576.

* Add tests for concurrent iOS launches

* Increase number of retries to account for the initial build being slow

packages/flutter_tools/lib/src/ios/mac.dart

@@ -464,11 +464,9 @@ Future<XcodeBuildResult> buildXcodeProject({
 
   final Stopwatch sw = Stopwatch()..start();
   initialBuildStatus = logger.startProgress('Running Xcode build...', timeout: timeoutConfiguration.fastOperation);
-  final RunResult buildResult = await processUtils.run(
-    buildCommands,
-    workingDirectory: app.project.hostAppRoot.path,
-    allowReentrantFlutter: true,
-  );
+
+  final RunResult buildResult = await _runBuildWithRetries(buildCommands, app);
+
   // Notifies listener that no more output is coming.
   scriptOutputPipeFile?.writeAsStringSync('all done');
   buildSubStatus?.stop();
@@ -572,6 +570,49 @@ Future<XcodeBuildResult> buildXcodeProject({
   }
 }
 
+Future<RunResult> _runBuildWithRetries(List<String> buildCommands, BuildableIOSApp app) async {
+  int buildRetryDelaySeconds = 1;
+  int remainingTries = 8;
+
+  RunResult buildResult;
+  while (remainingTries > 0) {
+    remainingTries--;
+    buildRetryDelaySeconds *= 2;
+
+    buildResult = await processUtils.run(
+      buildCommands,
+      workingDirectory: app.project.hostAppRoot.path,
+      allowReentrantFlutter: true,
+    );
+
+    // If the result is anything other than a concurrent build failure, exit
+    // the loop after the first build.
+    if (!_isXcodeConcurrentBuildFailure(buildResult)) {
+      break;
+    }
+
+    if (remainingTries > 0) {
+      printStatus('Xcode build failed due to concurrent builds, '
+        'will retry in $buildRetryDelaySeconds seconds.');
+      await Future<void>.delayed(Duration(seconds: buildRetryDelaySeconds));
+    } else {
+      printStatus(
+        'Xcode build failed too many times due to concurrent builds, '
+        'giving up.');
+      break;
+    }
+  }
+
+  return buildResult;
+}
+
+bool _isXcodeConcurrentBuildFailure(RunResult result) {
+return result.exitCode != 0 &&
+    result.stdout != null &&
+    result.stdout.contains('database is locked') &&
+    result.stdout.contains('there are two concurrent builds running');
+}
+
 String readGeneratedXcconfig(String appPath) {
   final String generatedXcconfigPath =
       fs.path.join(fs.currentDirectory.path, appPath, 'Flutter', 'Generated.xcconfig');

packages/flutter_tools/test/general.shard/ios/devices_test.dart

@@ -462,11 +462,13 @@ void main() {
         IOSDeploy: () => mockIosDeploy,
       });
 
-      void testNonPrebuilt({
+      void testNonPrebuilt(
+        String name, {
         @required bool showBuildSettingsFlakes,
+        void Function() additionalSetup,
+        void Function() additionalExpectations,
       }) {
-        const String name = ' non-prebuilt succeeds in debug mode';
-        testUsingContext(name + ' flaky: $showBuildSettingsFlakes', () async {
+        testUsingContext('non-prebuilt succeeds in debug mode $name', () async {
           final Directory targetBuildDir =
               projectDir.childDirectory('build/ios/iphoneos/Debug-arm64');
 
@@ -525,6 +527,10 @@ void main() {
             projectDir.path,
           ]);
 
+          if (additionalSetup != null) {
+            additionalSetup();
+          }
+
           final IOSApp app = await AbsoluteBuildableIOSApp.fromProject(
             FlutterProject.fromDirectory(projectDir).ios);
           final IOSDevice device = IOSDevice('123');
@@ -550,6 +556,10 @@ void main() {
           expect(launchResult.started, isTrue);
           expect(launchResult.hasObservatory, isFalse);
           expect(await device.stopApp(mockApp), isFalse);
+
+          if (additionalExpectations != null) {
+            additionalExpectations();
+          }
         }, overrides: <Type, Generator>{
           DoctorValidatorsProvider: () => FakeIosDoctorProvider(),
           IMobileDevice: () => mockIMobileDevice,
@@ -559,8 +569,44 @@ void main() {
         });
       }
 
-      testNonPrebuilt(showBuildSettingsFlakes: false);
-      testNonPrebuilt(showBuildSettingsFlakes: true);
+      testNonPrebuilt('flaky: false', showBuildSettingsFlakes: false);
+      testNonPrebuilt('flaky: true', showBuildSettingsFlakes: true);
+      testNonPrebuilt('with concurrent build failiure',
+        showBuildSettingsFlakes: false,
+        additionalSetup: () {
+          int callCount = 0;
+          when(mockProcessManager.run(
+            argThat(allOf(
+              contains('xcodebuild'),
+              contains('-configuration'),
+              contains('Debug'),
+            )),
+            workingDirectory: anyNamed('workingDirectory'),
+            environment: anyNamed('environment'),
+          )).thenAnswer((Invocation inv) {
+            // Succeed after 2 calls.
+            if (++callCount > 2) {
+              return Future<ProcessResult>.value(ProcessResult(0, 0, '', ''));
+            }
+            // Otherwise fail with the Xcode concurrent error.
+            return Future<ProcessResult>.value(ProcessResult(
+              0,
+              1,
+              '''
+                "/Developer/Xcode/DerivedData/foo/XCBuildData/build.db":
+                database is locked
+                Possibly there are two concurrent builds running in the same filesystem location.
+                ''',
+              '',
+            ));
+          });
+        },
+        additionalExpectations: () {
+          expect(testLogger.statusText, contains('will retry in 2 seconds'));
+          expect(testLogger.statusText, contains('will retry in 4 seconds'));
+          expect(testLogger.statusText, contains('Xcode build done.'));
+        },
+      );
     });
 
     group('Process calls', () {