diff --git a/lib/web_ui/dev/goldens.dart b/lib/web_ui/dev/goldens.dart new file mode 100644 index 00000000000..56b519c75b3 --- /dev/null +++ b/lib/web_ui/dev/goldens.dart @@ -0,0 +1,80 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:image/image.dart'; + +/// This class encapsulates visually diffing an Image with any other. +/// Both images need to be the exact same size. +class ImageDiff { + + /// The image to match + final Image golden; + + /// The image being compared + final Image other; + + /// The output of the comparison + /// Pixels in the output image can have 3 different colors depending on the comparison + /// between golden pixels and other pixels: + /// * white: when both pixels are the same + /// * red: when a pixel is found in other, but not in golden + /// * green: when a pixel is found in golden, but not in other + Image diff; + + /// The ratio of wrong pixels to all pixels in golden (between 0 and 1) + /// This gets set to 1 (100% difference) when golden and other aren't the same size. + double get rate => _wrongPixels / _pixelCount; + + ImageDiff({ Image this.golden, Image this.other }) { + _computeDiff(); + } + + int _pixelCount = 0; + int _wrongPixels = 0; + + final int _colorOk = Color.fromRgb(255, 255, 255); + final int _colorBadPixel = Color.fromRgb(255, 0, 0); + final int _colorExpectedPixel = Color.fromRgb(0, 255, 0); + + void _computeDiff() { + int goldenWidth = golden.width; + int goldenHeight = golden.height; + + _pixelCount = goldenWidth * goldenHeight; + diff = Image(goldenWidth, goldenHeight); + + if (goldenWidth == other.width && goldenHeight == other.height) { + for(int y = 0; y < goldenHeight; y++) { + for (int x = 0; x < goldenWidth; x++) { + int goldenPixel = golden.getPixel(x, y); + int otherPixel = other.getPixel(x, y); + + if (goldenPixel == otherPixel) { + diff.setPixel(x, y, _colorOk); + } else { + if (getLuminance(goldenPixel) < getLuminance(otherPixel)) { + diff.setPixel(x, y, _colorExpectedPixel); + } else { + diff.setPixel(x, y, _colorBadPixel); + } + _wrongPixels++; + } + } + } + } else { + // Images are completely different resolutions. Bail out big time. + _wrongPixels = _pixelCount; + } + } +} + +// Returns text with info about the files we just compared +String getPrintableDiffFilesInfo(diffRate, maxRate, filename, outputPath, diffPath) => ''' +(${((diffRate) * 100).toStringAsFixed(4)}% different pixels. Max: ${((maxRate) * 100).toStringAsFixed(4)}%). + +* Output image: ${outputPath} +* Diff pixels: ${diffPath} + +To update the golden file call matchGoldenFile('$filename', write: true). +'''; diff --git a/lib/web_ui/dev/test_platform.dart b/lib/web_ui/dev/test_platform.dart index f98d3f80480..79a348c06a9 100644 --- a/lib/web_ui/dev/test_platform.dart +++ b/lib/web_ui/dev/test_platform.dart @@ -10,6 +10,7 @@ import 'dart:typed_data'; import 'package:async/async.dart'; import 'package:http_multi_server/http_multi_server.dart'; +import 'package:image/image.dart'; import 'package:package_resolver/package_resolver.dart'; import 'package:path/path.dart' as p; import 'package:pedantic/pedantic.dart'; @@ -42,9 +43,13 @@ import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart' import 'chrome_installer.dart'; import 'environment.dart' as env; +import 'goldens.dart'; /// The port number Chrome exposes for debugging. const int _kChromeDevtoolsPort = 12345; +const int _kMaxScreenshotWidth = 1024; +const int _kMaxScreenshotHeight = 1024; +const double _kMaxDiffRateFailure = 1.0/100000; // 0.001% class BrowserPlatform extends PlatformPlugin { /// Starts the server. @@ -137,58 +142,80 @@ class BrowserPlatform extends PlatformPlugin { final Map requestData = json.decode(payload); final String filename = requestData['filename']; final bool write = requestData['write']; - final String result = await _diffScreenshot(filename, write); + final Map region = requestData['region']; + final String result = await _diffScreenshot(filename, write, region); return shelf.Response.ok(json.encode(result)); } - Future _diffScreenshot(String filename, bool write) async { + Future _diffScreenshot(String filename, bool write, [ Map region ]) async { const String _kGoldensDirectory = 'test/golden_files'; + + // Bail out fast if golden doesn't exist, and user doesn't want to create it. + final File file = File(p.join(_kGoldensDirectory, filename)); + if (!file.existsSync() && !write) { + return ''' +Golden file $filename does not exist. + +To automatically create this file call matchGoldenFile('$filename', write: true). +'''; + } + final wip.ChromeConnection chromeConnection = wip.ChromeConnection('localhost', _kChromeDevtoolsPort); final wip.ChromeTab chromeTab = await chromeConnection.getTab( (wip.ChromeTab chromeTab) => chromeTab.url.contains('localhost')); final wip.WipConnection wipConnection = await chromeTab.connect(); + + Map captureScreenshotParameters = null; + if (region != null) { + captureScreenshotParameters = { + 'format': 'png', + 'clip': { + 'x': region['x'], + 'y': region['y'], + 'width': region['width'], + 'height': region['height'], + 'scale': 1, // This is NOT the DPI of the page, instead it's the "zoom level". + }, + }; + } + // To tweak DPI we need to send an additional Emulation.setDeviceMetricsOverride. See: + // https://chromedevtools.github.io/devtools-protocol/tot/Emulation final wip.WipResponse response = - await wipConnection.sendCommand('Page.captureScreenshot'); - final Uint8List bytes = base64.decode(response.result['data']); - final File file = File(p.join(_kGoldensDirectory, filename)); + await wipConnection.sendCommand('Page.captureScreenshot', captureScreenshotParameters); + + // Compare screenshots + final Image screenshot = decodePng(base64.decode(response.result['data'])); + if (write) { - file.writeAsBytesSync(bytes, flush: true); + // Don't even bother with the comparison, just write and return + file.writeAsBytesSync(encodePng(screenshot), flush: true); return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; } - if (!file.existsSync()) { - return ''' - Golden file $filename does not exist. - To automatically create this file call matchGoldenFile('$filename', write: true). - '''; - } - final List goldenBytes = file.readAsBytesSync(); - final int lengths = bytes.length; - for (int i = 0; i < lengths; i++) { - if (goldenBytes[i] != bytes[i]) { - if (write) { - file.writeAsBytesSync(bytes, flush: true); - return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; - } else { - final File failedFile = File(p.join(file.parent.path, '.${p.basename(file.path)}')); - failedFile.writeAsBytesSync(bytes, flush: true); + ImageDiff diff = ImageDiff(golden: decodeNamedImage(file.readAsBytesSync(), filename), other: screenshot); - // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces - // different pictures. We need to pin Chrome versions and use a fuzzy image - // comparator. - if (Platform.environment['CIRRUS_CI'] == 'true') { - return 'OK'; - } + if (diff.rate > 0) { // Images are different, so produce some debug info + final File failedFile = File(p.join(file.parent.path, '${p.basenameWithoutExtension(file.path)}.out.png')); + failedFile.writeAsBytesSync(encodePng(screenshot), flush: true); - return ''' - Golden file ${file.path} did not match the image generated by the test. + final File failedDiff = File(p.join(file.parent.path, '${p.basenameWithoutExtension(file.path)}.diff.png')); + failedDiff.writeAsBytesSync(encodePng(diff.diff), flush:true); - The generated image was written to ${failedFile.path}. + final String printableDiffFilesInfo = getPrintableDiffFilesInfo(diff.rate, _kMaxDiffRateFailure, filename, failedFile.path, failedDiff.path); - To update the golden file call matchGoldenFile('$filename', write: true). - '''; + if (diff.rate < _kMaxDiffRateFailure) { // Warn user + print('[WARN] Golden file ${file.path} is slightly different from the image generated by this test!\n${printableDiffFilesInfo}'); + return 'OK'; // But test success! + } else { + // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces + // different pictures. We need to pin Chrome versions and use a fuzzy image + // comparator. + if (Platform.environment['CIRRUS_CI'] == 'true') { + return 'OK'; } + // Fail test + return 'Golden file ${file.path} is too different from the image generated by the test!\n${printableDiffFilesInfo}'; } } return 'OK'; @@ -867,6 +894,7 @@ class Chrome extends Browser { url.toString(), if (!debug) '--headless', if (isChromeNoSandbox) '--no-sandbox', + '--window-size=$_kMaxScreenshotWidth,$_kMaxScreenshotHeight', // When headless, this is the actual size of the viewport '--disable-extensions', '--disable-popup-blocking', '--bwsi', diff --git a/lib/web_ui/pubspec.yaml b/lib/web_ui/pubspec.yaml index 27ca873db95..f64508a0ed7 100644 --- a/lib/web_ui/pubspec.yaml +++ b/lib/web_ui/pubspec.yaml @@ -6,6 +6,7 @@ environment: dependencies: meta: 1.1.7 + image: 2.1.4 dev_dependencies: http: 0.12.0+2 diff --git a/lib/web_ui/test/golden_files/.gitignore b/lib/web_ui/test/golden_files/.gitignore index 4b2350ab78f..d41f94a58b1 100644 --- a/lib/web_ui/test/golden_files/.gitignore +++ b/lib/web_ui/test/golden_files/.gitignore @@ -1,2 +1,3 @@ # Failed screenshot files -.*.png +*.out.png +*.diff.png diff --git a/lib/web_ui/test/golden_files/smoke_test.png b/lib/web_ui/test/golden_files/smoke_test.png index c17b3ba1798..e136d923394 100644 Binary files a/lib/web_ui/test/golden_files/smoke_test.png and b/lib/web_ui/test/golden_files/smoke_test.png differ diff --git a/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart b/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart index b001b4ad1f2..aef4845c939 100644 --- a/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart +++ b/lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart @@ -5,11 +5,12 @@ import 'dart:html' as html; import 'package:test/test.dart'; +import 'package:ui/ui.dart'; import 'package:web_engine_tester/golden_tester.dart'; void main() { test('screenshot test reports failure', () async { html.document.body.innerHtml = 'Text that does not appear on the screenshot!'; - await matchGoldenFile('smoke_test.png'); + await matchGoldenFile('smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200)); }); } diff --git a/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart b/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart index 657f271eea6..98c19d0c072 100644 --- a/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart +++ b/lib/web_ui/test/golden_tests/golden_success_smoke_test.dart @@ -5,11 +5,12 @@ import 'dart:html' as html; import 'package:test/test.dart'; +import 'package:ui/ui.dart'; import 'package:web_engine_tester/golden_tester.dart'; void main() { test('screenshot test reports success', () async { html.document.body.innerHtml = 'Hello world!'; - await matchGoldenFile('smoke_test.png'); + await matchGoldenFile('smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200)); }); } diff --git a/web_sdk/web_engine_tester/lib/golden_tester.dart b/web_sdk/web_engine_tester/lib/golden_tester.dart index e4910ec2b33..369a6270b83 100644 --- a/web_sdk/web_engine_tester/lib/golden_tester.dart +++ b/web_sdk/web_engine_tester/lib/golden_tester.dart @@ -6,6 +6,8 @@ import 'dart:async'; import 'dart:convert'; import 'dart:html' as html; +import 'package:ui/ui.dart'; + import 'package:test/test.dart'; Future _callScreenshotServer(dynamic requestData) async { @@ -19,10 +21,11 @@ Future _callScreenshotServer(dynamic requestData) async { } /// Attempts to match the current browser state with the screenshot [filename]. -Future matchGoldenFile(String filename, { bool write = false }) async { +Future matchGoldenFile(String filename, { bool write = false, Rect region = null }) async { final String response = await _callScreenshotServer({ 'filename': filename, 'write': write, + 'region': region == null ? null : {'x': region.left, 'y': region.top, 'width': region.width, 'height': region.height}, }); if (response == 'OK') { // Pass diff --git a/web_sdk/web_engine_tester/lib/static/host.css b/web_sdk/web_engine_tester/lib/static/host.css index ea1e941c368..3a1f061636e 100644 --- a/web_sdk/web_engine_tester/lib/static/host.css +++ b/web_sdk/web_engine_tester/lib/static/host.css @@ -2,3 +2,7 @@ body { margin: 0; padding: 0; } + +iframe { + border: none; +}