diff --git a/engine/src/flutter/testing/scenario_app/firebase_xctest.sh b/engine/src/flutter/testing/scenario_app/firebase_xctest.sh new file mode 100755 index 00000000000..b80d2f41979 --- /dev/null +++ b/engine/src/flutter/testing/scenario_app/firebase_xctest.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +"${BASH_SOURCE%/*}/compile_ios_aot.sh" $1 $2 + +GIT_REVISION=${3:-$(git rev-parse HEAD)} +BUILD_ID=${4:-$CIRRUS_BUILD_ID} + +pushd "${BASH_SOURCE%/*}/ios/Scenarios" +xcodebuild -project Scenarios.xcodeproj -scheme Scenarios -configuration Debug \ + -sdk iphoneos \ + -derivedDataPath DerivedData/Scenarios \ + build-for-testing + +pushd DerivedData/Scenarios/Build/Products + +zip -r scenarios.zip Debug-iphoneos Scenarios*.xctestrun + +gcloud firebase test ios run --test ./scenarios.zip \ + --device model=iphone8plus,version=12.0,locale=en_US,orientation=portrait \ + --xcode-version=10.2 \ + --results-bucket=gs://flutter_firebase_testlab \ + --results-dir=engine_scenario_test/$GIT_REVISION/$BUILD_ID \ + +popd +popd diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/.gitignore b/engine/src/flutter/testing/scenario_app/ios/Scenarios/.gitignore new file mode 100644 index 00000000000..7f350c71895 --- /dev/null +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/.gitignore @@ -0,0 +1,2 @@ +DerivedData/ + diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj index 17ef6c9a421..c7fe99847ea 100644 --- a/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 0DB7820222EA493B00E9B371 /* FlutterViewControllerTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 0DB781FC22EA2C0300E9B371 /* FlutterViewControllerTest.m */; }; 242F37A222E636DE001E83D4 /* Flutter.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4522E3B61000073EBF /* Flutter.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 242F37A322E636DE001E83D4 /* App.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4122E3B5F700073EBF /* App.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 244EA6D0230DBE8900B2D26E /* golden_platform_view_D21AP.png in Resources */ = {isa = PBXBuildFile; fileRef = 244EA6CF230DBE8900B2D26E /* golden_platform_view_D21AP.png */; }; 246B4E4222E3B5F700073EBF /* App.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4122E3B5F700073EBF /* App.framework */; }; 246B4E4622E3B61000073EBF /* Flutter.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 246B4E4522E3B61000073EBF /* Flutter.framework */; }; 248D76CC22E388370012F0C1 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 248D76CB22E388370012F0C1 /* AppDelegate.m */; }; @@ -23,7 +24,7 @@ 248D76DA22E388380012F0C1 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 248D76D922E388380012F0C1 /* main.m */; }; 248D76EF22E388380012F0C1 /* PlatformViewUITests.m in Sources */ = {isa = PBXBuildFile; fileRef = 248D76EE22E388380012F0C1 /* PlatformViewUITests.m */; }; 248FDFC422FE7CD0009CC7CD /* FlutterEngineTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 248FDFC322FE7CD0009CC7CD /* FlutterEngineTest.m */; }; - 24D47D1B230C79840069DD5E /* golden_platform_view_D21AAP.png in Resources */ = {isa = PBXBuildFile; fileRef = 24D47D1A230C79840069DD5E /* golden_platform_view_D21AAP.png */; }; + 24D47D1B230C79840069DD5E /* golden_platform_view_D211AP.png in Resources */ = {isa = PBXBuildFile; fileRef = 24D47D1A230C79840069DD5E /* golden_platform_view_D211AP.png */; }; 24D47D1D230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png in Resources */ = {isa = PBXBuildFile; fileRef = 24D47D1C230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png */; }; 24F1FB89230B4579005ACE7C /* TextPlatformView.m in Sources */ = {isa = PBXBuildFile; fileRef = 24F1FB87230B4579005ACE7C /* TextPlatformView.m */; }; /* End PBXBuildFile section */ @@ -84,6 +85,7 @@ /* Begin PBXFileReference section */ 0DB781FC22EA2C0300E9B371 /* FlutterViewControllerTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FlutterViewControllerTest.m; sourceTree = ""; }; + 244EA6CF230DBE8900B2D26E /* golden_platform_view_D21AP.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = golden_platform_view_D21AP.png; sourceTree = ""; }; 246B4E4122E3B5F700073EBF /* App.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = App.framework; sourceTree = ""; }; 246B4E4522E3B61000073EBF /* Flutter.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = Flutter.framework; sourceTree = ""; }; 248D76C722E388370012F0C1 /* Scenarios.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Scenarios.app; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -98,7 +100,7 @@ 248D76EE22E388380012F0C1 /* PlatformViewUITests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = PlatformViewUITests.m; sourceTree = ""; }; 248D76F022E388380012F0C1 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 248FDFC322FE7CD0009CC7CD /* FlutterEngineTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FlutterEngineTest.m; sourceTree = ""; }; - 24D47D1A230C79840069DD5E /* golden_platform_view_D21AAP.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = golden_platform_view_D21AAP.png; sourceTree = ""; }; + 24D47D1A230C79840069DD5E /* golden_platform_view_D211AP.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = golden_platform_view_D211AP.png; sourceTree = ""; }; 24D47D1C230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "golden_platform_view_iPhone SE_simulator.png"; sourceTree = ""; }; 24D47D1E230CA4480069DD5E /* README.md */ = {isa = PBXFileReference; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; 24F1FB87230B4579005ACE7C /* TextPlatformView.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = TextPlatformView.m; sourceTree = ""; }; @@ -182,8 +184,9 @@ 248D76ED22E388380012F0C1 /* ScenariosUITests */ = { isa = PBXGroup; children = ( + 244EA6CF230DBE8900B2D26E /* golden_platform_view_D21AP.png */, 24D47D1C230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png */, - 24D47D1A230C79840069DD5E /* golden_platform_view_D21AAP.png */, + 24D47D1A230C79840069DD5E /* golden_platform_view_D211AP.png */, 248D76EE22E388380012F0C1 /* PlatformViewUITests.m */, 248D76F022E388380012F0C1 /* Info.plist */, 24D47D1E230CA4480069DD5E /* README.md */, @@ -321,8 +324,9 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( - 24D47D1B230C79840069DD5E /* golden_platform_view_D21AAP.png in Resources */, + 24D47D1B230C79840069DD5E /* golden_platform_view_D211AP.png in Resources */, 24D47D1D230CA2700069DD5E /* golden_platform_view_iPhone SE_simulator.png in Resources */, + 244EA6D0230DBE8900B2D26E /* golden_platform_view_D21AP.png in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -424,7 +428,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.2; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -478,7 +482,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.2; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; SDKROOT = iphoneos; diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings new file mode 100644 index 00000000000..0c67376ebac --- /dev/null +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings @@ -0,0 +1,5 @@ + + + + + diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/xcshareddata/xcschemes/Scenarios.xcscheme b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/xcshareddata/xcschemes/Scenarios.xcscheme new file mode 100644 index 00000000000..a780f4aacf3 --- /dev/null +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios.xcodeproj/xcshareddata/xcschemes/Scenarios.xcscheme @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios/Info.plist b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios/Info.plist index 3c41170863c..121e6c59a99 100644 --- a/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios/Info.plist +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/Scenarios/Info.plist @@ -24,14 +24,14 @@ arm64 + UIRequiresFullScreen + UISupportedInterfaceOrientations UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight - io.flutter.embedded_views_preview - UISupportedInterfaceOrientations~ipad UIInterfaceOrientationPortrait @@ -39,5 +39,7 @@ UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight + io.flutter.embedded_views_preview + diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m index 0252f185ba7..e0a929572b8 100644 --- a/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/PlatformViewUITests.m @@ -31,15 +31,20 @@ UIImage* golden = [[UIImage alloc] initWithContentsOfFile:path]; XCUIScreenshot* screenshot = [[XCUIScreen mainScreen] screenshot]; + XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot]; + attachment.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachment]; + if (golden) { XCTAttachment* goldenAttachment = [XCTAttachment attachmentWithImage:golden]; goldenAttachment.lifetime = XCTAttachmentLifetimeKeepAlways; [self addAttachment:goldenAttachment]; + } else { + XCTFail(@"This test will fail - no golden named %@ found. Follow the steps in the " + @"README to add a new golden.", + goldenName); } - XCTAttachment* attachment = [XCTAttachment attachmentWithScreenshot:screenshot]; - attachment.lifetime = XCTAttachmentLifetimeKeepAlways; - [self addAttachment:attachment]; XCTAssertTrue([self compareImage:golden toOther:screenshot.image]); } diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/README.md b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/README.md index d5d3bc62860..70e34f3e3d6 100644 --- a/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/README.md +++ b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/README.md @@ -11,3 +11,9 @@ names are taken from the environment. New devices require running the test on the device, gathering the attachment and verifying it manually, and then adding an appropriately named file to this folder. + +If the test is attempted on a new device, the log will contain a message +indicating the file name it expected to find. The test will continue and fail, +but will contain an attachment with the expected screen shot. If the screen +shot looks good, add it with the correct name to the project and run the test +again - it should pass this time. diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D21AAP.png b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D211AP.png similarity index 100% rename from engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D21AAP.png rename to engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D211AP.png diff --git a/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D21AP.png b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D21AP.png new file mode 100644 index 00000000000..bd59044cc20 Binary files /dev/null and b/engine/src/flutter/testing/scenario_app/ios/Scenarios/ScenariosUITests/golden_platform_view_D21AP.png differ