diff --git a/engine/src/flutter/lib/web_ui/dev/README.md b/engine/src/flutter/lib/web_ui/dev/README.md index c38108e5607..ebc9211c46d 100644 --- a/engine/src/flutter/lib/web_ui/dev/README.md +++ b/engine/src/flutter/lib/web_ui/dev/README.md @@ -20,3 +20,15 @@ or: ``` felt build --watch ``` + +## Configuration files + +`chrome_lock.yaml` contains the version of Chrome we use to test Flutter for +web. Chrome is not automatically updated whenever a new release is available. +Instead, we update this file manually once in a while. + +`goldens_lock.yaml` refers to a revision in the https://github.com/flutter/goldens +repo. Screenshot tests are compared with the golden files at that revision. +When making engine changes that affect screenshots, first submit a PR to +flutter/goldens updating the screenshots. Then update this file pointing to +the new revision. diff --git a/engine/src/flutter/lib/web_ui/dev/environment.dart b/engine/src/flutter/lib/web_ui/dev/environment.dart index c1b85847c75..c6c08542324 100644 --- a/engine/src/flutter/lib/web_ui/dev/environment.dart +++ b/engine/src/flutter/lib/web_ui/dev/environment.dart @@ -103,4 +103,17 @@ class Environment { webUiRootDir.path, '.dart_tool', )); + + /// Path to the "dev" directory containing engine developer tools and + /// configuration files. + io.Directory get webUiDevDir => io.Directory(pathlib.join( + webUiRootDir.path, + 'dev', + )); + + /// Path to the clone of the flutter/goldens repository. + io.Directory get webUiGoldensRepositoryDirectory => io.Directory(pathlib.join( + webUiDartToolDir.path, + 'goldens', + )); } diff --git a/engine/src/flutter/lib/web_ui/dev/goldens.dart b/engine/src/flutter/lib/web_ui/dev/goldens.dart index d0dc5bd03d3..3ab47db707e 100644 --- a/engine/src/flutter/lib/web_ui/dev/goldens.dart +++ b/engine/src/flutter/lib/web_ui/dev/goldens.dart @@ -3,6 +3,11 @@ // found in the LICENSE file. import 'dart:io' as io; import 'package:image/image.dart'; +import 'package:path/path.dart' as path; +import 'package:yaml/yaml.dart'; + +import 'environment.dart'; +import 'utils.dart'; void main(List args) { final io.File fileA = io.File(args[0]); @@ -10,7 +15,7 @@ void main(List args) { final Image imageA = decodeNamedImage(fileA.readAsBytesSync(), 'a.png'); final Image imageB = decodeNamedImage(fileB.readAsBytesSync(), 'b.png'); final ImageDiff diff = ImageDiff(golden: imageA, other: imageB); - print('Diff: ${(diff.rate * 100).toStringAsFixed(4)}'); + print('Diff: ${(diff.rate * 100).toStringAsFixed(4)}%'); } /// This class encapsulates visually diffing an Image with any other. @@ -140,3 +145,72 @@ class ImageDiff { String getPrintableDiffFilesInfo(double diffRate, double maxRate) => '(${((diffRate) * 100).toStringAsFixed(4)}% of pixels were different. ' 'Maximum allowed rate is: ${(maxRate * 100).toStringAsFixed(4)}%).'; + +/// Fetches golden files from github.com/flutter/goldens, cloning the repository if necessary. +/// +/// The repository is cloned into web_ui/.dart_tool. +Future fetchGoldens() async { + await _GoldensRepoFetcher().fetch(); +} + +class _GoldensRepoFetcher { + String _repository; + String _revision; + + Future fetch() async { + final io.File lockFile = io.File( + path.join(environment.webUiDevDir.path, 'goldens_lock.yaml') + ); + final YamlMap lock = loadYaml(lockFile.readAsStringSync()); + _repository = lock['repository']; + _revision = lock['revision']; + + final String localRevision = await _getLocalRevision(); + if (localRevision == _revision) { + return; + } + + print('Fetching $_repository@$_revision'); + + if (!environment.webUiGoldensRepositoryDirectory.existsSync()) { + environment.webUiGoldensRepositoryDirectory.createSync(recursive: true); + await runProcess( + 'git', + ['init'], + workingDirectory: environment.webUiGoldensRepositoryDirectory.path, + mustSucceed: true, + ); + await runProcess( + 'git', + ['remote', 'add', 'origin', _repository], + workingDirectory: environment.webUiGoldensRepositoryDirectory.path, + mustSucceed: true, + ); + } + + await runProcess( + 'git', + ['fetch', 'origin', 'master'], + workingDirectory: environment.webUiGoldensRepositoryDirectory.path, + mustSucceed: true, + ); + await runProcess( + 'git', + ['checkout', _revision], + workingDirectory: environment.webUiGoldensRepositoryDirectory.path, + mustSucceed: true, + ); + } + + Future _getLocalRevision() async { + final io.File head = io.File(path.join( + environment.webUiGoldensRepositoryDirectory.path, '.git', 'HEAD' + )); + + if (!head.existsSync()) { + return null; + } + + return head.readAsStringSync().trim(); + } +} diff --git a/engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml b/engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml new file mode 100644 index 00000000000..1b72c1367e6 --- /dev/null +++ b/engine/src/flutter/lib/web_ui/dev/goldens_lock.yaml @@ -0,0 +1,2 @@ +repository: https://github.com/flutter/goldens.git +revision: dd993a32c23c5c542f083134467e7cda09cac975 diff --git a/engine/src/flutter/lib/web_ui/dev/test_platform.dart b/engine/src/flutter/lib/web_ui/dev/test_platform.dart index 339e4fa0bc1..ab2c08c675d 100644 --- a/engine/src/flutter/lib/web_ui/dev/test_platform.dart +++ b/engine/src/flutter/lib/web_ui/dev/test_platform.dart @@ -147,10 +147,28 @@ class BrowserPlatform extends PlatformPlugin { } Future _diffScreenshot(String filename, bool write, [ Map region ]) async { - const String _kGoldensDirectory = 'test/golden_files'; + String goldensDirectory; + if (filename.startsWith('__local__')) { + filename = filename.substring('__local__/'.length); + goldensDirectory = p.join( + env.environment.webUiRootDir.path, + 'test', + 'golden_files', + ); + } else { + await fetchGoldens(); + goldensDirectory = p.join( + env.environment.webUiGoldensRepositoryDirectory.path, + 'engine', + 'web', + ); + } // Bail out fast if golden doesn't exist, and user doesn't want to create it. - final File file = File(p.join(_kGoldensDirectory, filename)); + final File file = File(p.join( + goldensDirectory, + filename, + )); if (!file.existsSync() && !write) { return ''' Golden file $filename does not exist. diff --git a/engine/src/flutter/lib/web_ui/dev/utils.dart b/engine/src/flutter/lib/web_ui/dev/utils.dart index 9078648fb9f..e44909c8cdc 100644 --- a/engine/src/flutter/lib/web_ui/dev/utils.dart +++ b/engine/src/flutter/lib/web_ui/dev/utils.dart @@ -5,6 +5,7 @@ import 'dart:async'; import 'dart:io' as io; +import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'environment.dart'; @@ -31,17 +32,54 @@ class FilePath { String toString() => _absolutePath; } +/// Runs [executable] merging its output into the current process' standard out and standard error. Future runProcess( String executable, List arguments, { String workingDirectory, + bool mustSucceed: false, }) async { final io.Process process = await io.Process.start( executable, arguments, workingDirectory: workingDirectory, ); - return _forwardIOAndWait(process); + final int exitCode = await _forwardIOAndWait(process); + if (mustSucceed && exitCode != 0) { + throw ProcessException( + description: 'Sub-process failed.', + executable: executable, + arguments: arguments, + workingDirectory: workingDirectory, + exitCode: exitCode, + ); + } + return exitCode; +} + +/// Runs [executable] and returns its standard output as a string. +/// +/// If the process fails, throws a [ProcessException]. +Future evalProcess( + String executable, + List arguments, { + String workingDirectory, +}) async { + final io.ProcessResult result = await io.Process.run( + executable, + arguments, + workingDirectory: workingDirectory, + ); + if (result.exitCode != 0) { + throw ProcessException( + description: result.stderr, + executable: executable, + arguments: arguments, + workingDirectory: workingDirectory, + exitCode: result.exitCode, + ); + } + return result.stdout; } Future _forwardIOAndWait(io.Process process) { @@ -53,3 +91,31 @@ Future _forwardIOAndWait(io.Process process) { return exitCode; }); } + +@immutable +class ProcessException implements Exception { + ProcessException({ + @required this.description, + @required this.executable, + @required this.arguments, + @required this.workingDirectory, + @required this.exitCode, + }); + + final String description; + final String executable; + final List arguments; + final String workingDirectory; + final int exitCode; + + @override + String toString() { + final StringBuffer message = StringBuffer(); + message + ..writeln(description) + ..writeln('Command: $executable ${arguments.join(' ')}') + ..writeln('Working directory: ${workingDirectory ?? io.Directory.current.path}') + ..writeln('Exit code: $exitCode'); + return '$message'; + } +} diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_rrect_scuba_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_rrect_scuba_test.dart index 8d51d677599..78e2ddd5080 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_rrect_scuba_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_rrect_scuba_test.dart @@ -42,7 +42,7 @@ void main() async { } html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/canvas_rrect_round_square.png', region: region); + await matchGoldenFile('canvas_rrect_round_square.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('round rect with big radius scale down smaller radius', () async { @@ -58,7 +58,7 @@ void main() async { } html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/canvas_rrect_overlapping_radius.png', region: region); + await matchGoldenFile('canvas_rrect_overlapping_radius.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('diff round rect with big radius scale down smaller radius', () async { @@ -81,6 +81,6 @@ void main() async { } html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/canvas_drrect_overlapping_radius.png', region: region); + await matchGoldenFile('canvas_drrect_overlapping_radius.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); } diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_scuba_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_scuba_test.dart index 8fbddf0553a..4c7986e2bbe 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_scuba_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/canvas_scuba_test.dart @@ -68,7 +68,7 @@ void main() async { html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/misaligned_pixels_in_canvas_test.png', region: region); + await matchGoldenFile('misaligned_pixels_in_canvas_test.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('compensates for misalignment of the canvas', () async { @@ -83,7 +83,7 @@ void main() async { html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/misaligned_canvas_test.png', region: region); + await matchGoldenFile('misaligned_canvas_test.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('fill the whole canvas with color even when transformed', () async { @@ -94,7 +94,7 @@ void main() async { html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/bitmap_canvas_fills_color_when_transformed.png', region: region); + await matchGoldenFile('bitmap_canvas_fills_color_when_transformed.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('fill the whole canvas with paint even when transformed', () async { @@ -107,6 +107,6 @@ void main() async { html.document.body.append(canvas.rootElement); - await matchGoldenFile('engine/bitmap_canvas_fills_paint_when_transformed.png', region: region); + await matchGoldenFile('bitmap_canvas_fills_paint_when_transformed.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); } diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/compositing_scuba_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/compositing_scuba_test.dart index 148660ff78e..caed73a7f8c 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/compositing_scuba_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/compositing_scuba_test.dart @@ -34,7 +34,7 @@ void main() async { html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_shifted_clip_rect.png', region: region); + await matchGoldenFile('compositing_shifted_clip_rect.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('pushClipRect with offset and transform', () async { @@ -54,7 +54,7 @@ void main() async { html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_clip_rect_with_offset_and_transform.png', region: region); + await matchGoldenFile('compositing_clip_rect_with_offset_and_transform.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('pushClipRRect', () async { @@ -67,7 +67,7 @@ void main() async { html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_shifted_clip_rrect.png', region: region); + await matchGoldenFile('compositing_shifted_clip_rrect.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); test('pushPhysicalShape', () async { @@ -95,7 +95,7 @@ void main() async { html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_shifted_physical_shape_clip.png', region: region); + await matchGoldenFile('compositing_shifted_physical_shape_clip.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); group('Cull rect computation', () { @@ -222,7 +222,7 @@ void _testCullRectComputation() { builder.pop(); // pushClipRect html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_cull_rect_fills_layer_clip.png', region: region); + await matchGoldenFile('compositing_cull_rect_fills_layer_clip.png', region: region); final PersistedStandardPicture picture = enumeratePictures().single; expect(picture.optimalLocalCullRect, const Rect.fromLTRB(40, 40, 70, 70)); @@ -250,7 +250,7 @@ void _testCullRectComputation() { builder.pop(); // pushClipRect html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_cull_rect_intersects_clip_and_paint_bounds.png', region: region); + await matchGoldenFile('compositing_cull_rect_intersects_clip_and_paint_bounds.png', region: region); final PersistedStandardPicture picture = enumeratePictures().single; expect(picture.optimalLocalCullRect, const Rect.fromLTRB(50, 40, 70, 70)); @@ -280,7 +280,7 @@ void _testCullRectComputation() { builder.pop(); // pushClipRect html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_cull_rect_offset_inside_layer_clip.png', region: region); + await matchGoldenFile('compositing_cull_rect_offset_inside_layer_clip.png', region: region); final PersistedStandardPicture picture = enumeratePictures().single; expect(picture.optimalLocalCullRect, @@ -353,7 +353,7 @@ void _testCullRectComputation() { builder.pop(); // pushOffset html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_cull_rect_rotated.png', region: region); + await matchGoldenFile('compositing_cull_rect_rotated.png', region: region); final PersistedStandardPicture picture = enumeratePictures().single; expect( @@ -375,7 +375,7 @@ void _testCullRectComputation() { html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_clip_path.png', region: region); + await matchGoldenFile('compositing_clip_path.png', region: region); }, timeout: const Timeout(Duration(seconds: 10))); // Draw a picture inside a rotated clip. Verify that the cull rect is big @@ -477,7 +477,7 @@ void _testCullRectComputation() { builder.pop(); // pushTransform scale html.document.body.append(builder.build().webOnlyRootElement); - await matchGoldenFile('engine/compositing_3d_rotate1.png', region: region); + await matchGoldenFile('compositing_3d_rotate1.png', region: region); final PersistedStandardPicture picture = enumeratePictures().single; // TODO(https://github.com/flutter/flutter/issues/40395): diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/conic_scuba_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/conic_scuba_test.dart index 3a27a3d0dbe..452b32acffd 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/conic_scuba_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/conic_scuba_test.dart @@ -33,7 +33,7 @@ void main() async { html.document.body.append(bitmapCanvas.rootElement); canvas.apply(bitmapCanvas); - await matchGoldenFile('engine/$scubaFileName.png', region: region); + await matchGoldenFile('$scubaFileName.png', region: region); bitmapCanvas.rootElement.remove(); } diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/path_to_svg_scuba_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/path_to_svg_scuba_test.dart index 7abe5ca0c36..590cc8fe3ab 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/path_to_svg_scuba_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/path_to_svg_scuba_test.dart @@ -38,7 +38,7 @@ void main() async { canvas.apply(bitmapCanvas); - await matchGoldenFile('engine/$scubaFileName.png', region: region); + await matchGoldenFile('$scubaFileName.png', region: region); bitmapCanvas.rootElement.remove(); svgElement.remove(); diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/recording_canvas_scuba_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/recording_canvas_scuba_test.dart index d439ed70ee2..b529acadc60 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/recording_canvas_scuba_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/recording_canvas_scuba_test.dart @@ -44,7 +44,7 @@ void main() async { try { sceneElement.append(engineCanvas.rootElement); html.document.body.append(sceneElement); - await matchGoldenFile('engine/paint_bounds_for_$fileName.png', region: region); + await matchGoldenFile('paint_bounds_for_$fileName.png', region: region); } finally { // The page is reused across tests, so remove the element after taking the // Scuba screenshot. diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/scuba.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/scuba.dart index d54d0925a77..598daa11a5c 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/engine/scuba.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/engine/scuba.dart @@ -40,7 +40,7 @@ class EngineScubaTester { } Future diffScreenshot(String fileName) async { - await matchGoldenFile('engine/$fileName.png', region: ui.Rect.fromLTWH(0, 0, viewportSize.width, viewportSize.height)); + await matchGoldenFile('$fileName.png', region: ui.Rect.fromLTWH(0, 0, viewportSize.width, viewportSize.height)); } /// Prepares the DOM and inserts all the necessary nodes, then invokes scuba's diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart index aef4845c939..ddfa139452b 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart @@ -11,6 +11,6 @@ import 'package:web_engine_tester/golden_tester.dart'; void main() { test('screenshot test reports failure', () async { html.document.body.innerHtml = 'Text that does not appear on the screenshot!'; - await matchGoldenFile('smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200)); + await matchGoldenFile('__local__/smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200)); }); } diff --git a/engine/src/flutter/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart b/engine/src/flutter/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart index 98c19d0c072..c4366eb4310 100644 --- a/engine/src/flutter/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart +++ b/engine/src/flutter/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart @@ -11,6 +11,6 @@ import 'package:web_engine_tester/golden_tester.dart'; void main() { test('screenshot test reports success', () async { html.document.body.innerHtml = 'Hello world!'; - await matchGoldenFile('smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200)); + await matchGoldenFile('__local__/smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200)); }); }