More controls for Screenshot tests (for web) (#12284)

New features for golden tests (for web):

* Height/width of headless browser instance extracted to constants
* Remove border from the host iframe
* Added 'region' to matchGoldenFile, so screenshots can capture only
a subset of the viewport
* Added image-format-awareness (png) to screenshot comparison, so it
doesn't only look at raw bytes
* When a test fails, output a diff image alongside the unexpected
output.

lib/web_ui/dev/goldens.dart

@@ -0,0 +1,80 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:image/image.dart';
+
+/// This class encapsulates visually diffing an Image with any other.
+/// Both images need to be the exact same size.
+class ImageDiff {
+
+  /// The image to match
+  final Image golden;
+
+  /// The image being compared
+  final Image other;
+
+  /// The output of the comparison
+  /// Pixels in the output image can have 3 different colors depending on the comparison
+  /// between golden pixels and other pixels:
+  ///  * white: when both pixels are the same
+  ///  * red: when a pixel is found in other, but not in golden
+  ///  * green: when a pixel is found in golden, but not in other
+  Image diff;
+
+  /// The ratio of wrong pixels to all pixels in golden (between 0 and 1)
+  /// This gets set to 1 (100% difference) when golden and other aren't the same size.
+  double get rate => _wrongPixels / _pixelCount;
+
+  ImageDiff({ Image this.golden, Image this.other }) {
+    _computeDiff();
+  }
+
+  int _pixelCount = 0;
+  int _wrongPixels = 0;
+
+  final int _colorOk = Color.fromRgb(255, 255, 255);
+  final int _colorBadPixel = Color.fromRgb(255, 0, 0);
+  final int _colorExpectedPixel = Color.fromRgb(0, 255, 0);
+
+  void _computeDiff() {
+    int goldenWidth = golden.width;
+    int goldenHeight = golden.height;
+
+    _pixelCount = goldenWidth * goldenHeight;
+    diff = Image(goldenWidth, goldenHeight);
+
+    if (goldenWidth == other.width && goldenHeight == other.height) {
+      for(int y = 0; y < goldenHeight; y++) {
+        for (int x = 0; x < goldenWidth; x++) {
+          int goldenPixel = golden.getPixel(x, y);
+          int otherPixel = other.getPixel(x, y);
+
+          if (goldenPixel == otherPixel) {
+            diff.setPixel(x, y, _colorOk);
+          } else {
+            if (getLuminance(goldenPixel) < getLuminance(otherPixel)) {
+              diff.setPixel(x, y, _colorExpectedPixel);
+            } else {
+              diff.setPixel(x, y, _colorBadPixel);
+            }
+            _wrongPixels++;
+          }
+        }
+      }
+    } else {
+      // Images are completely different resolutions. Bail out big time.
+      _wrongPixels = _pixelCount;
+    }
+  }
+}
+
+// Returns text with info about the files we just compared
+String getPrintableDiffFilesInfo(diffRate, maxRate, filename, outputPath, diffPath) => '''
+(${((diffRate) * 100).toStringAsFixed(4)}% different pixels. Max: ${((maxRate) * 100).toStringAsFixed(4)}%).
+
+* Output image: ${outputPath}
+*  Diff pixels: ${diffPath}
+
+To update the golden file call matchGoldenFile('$filename', write: true).
+''';

lib/web_ui/dev/test_platform.dart

@@ -10,6 +10,7 @@ import 'dart:typed_data';
 
 import 'package:async/async.dart';
 import 'package:http_multi_server/http_multi_server.dart';
+import 'package:image/image.dart';
 import 'package:package_resolver/package_resolver.dart';
 import 'package:path/path.dart' as p;
 import 'package:pedantic/pedantic.dart';
@@ -42,9 +43,13 @@ import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart'
 
 import 'chrome_installer.dart';
 import 'environment.dart' as env;
+import 'goldens.dart';
 
 /// The port number Chrome exposes for debugging.
 const int _kChromeDevtoolsPort = 12345;
+const int _kMaxScreenshotWidth = 1024;
+const int _kMaxScreenshotHeight = 1024;
+const double _kMaxDiffRateFailure = 1.0/100000; // 0.001%
 
 class BrowserPlatform extends PlatformPlugin {
   /// Starts the server.
@@ -137,58 +142,80 @@ class BrowserPlatform extends PlatformPlugin {
     final Map<String, dynamic> requestData = json.decode(payload);
     final String filename = requestData['filename'];
     final bool write = requestData['write'];
-    final String result = await _diffScreenshot(filename, write);
+    final Map<String, dynamic> region = requestData['region'];
+    final String result = await _diffScreenshot(filename, write, region);
     return shelf.Response.ok(json.encode(result));
   }
 
-  Future<String> _diffScreenshot(String filename, bool write) async {
+  Future<String> _diffScreenshot(String filename, bool write, [ Map<String, dynamic> region ]) async {
     const String _kGoldensDirectory = 'test/golden_files';
+
+    // Bail out fast if golden doesn't exist, and user doesn't want to create it.
+    final File file = File(p.join(_kGoldensDirectory, filename));
+    if (!file.existsSync() && !write) {
+      return '''
+Golden file $filename does not exist.
+
+To automatically create this file call matchGoldenFile('$filename', write: true).
+''';
+    }
+
     final wip.ChromeConnection chromeConnection =
         wip.ChromeConnection('localhost', _kChromeDevtoolsPort);
     final wip.ChromeTab chromeTab = await chromeConnection.getTab(
         (wip.ChromeTab chromeTab) => chromeTab.url.contains('localhost'));
     final wip.WipConnection wipConnection = await chromeTab.connect();
+
+    Map<String, dynamic> captureScreenshotParameters = null;
+    if (region != null) {
+      captureScreenshotParameters = {
+        'format': 'png',
+        'clip': {
+          'x': region['x'],
+          'y': region['y'],
+          'width': region['width'],
+          'height': region['height'],
+          'scale': 1, // This is NOT the DPI of the page, instead it's the "zoom level".
+        },
+      };
+    }
+    // To tweak DPI we need to send an additional Emulation.setDeviceMetricsOverride. See:
+    // https://chromedevtools.github.io/devtools-protocol/tot/Emulation
     final wip.WipResponse response =
-        await wipConnection.sendCommand('Page.captureScreenshot');
-    final Uint8List bytes = base64.decode(response.result['data']);
-    final File file = File(p.join(_kGoldensDirectory, filename));
+        await wipConnection.sendCommand('Page.captureScreenshot', captureScreenshotParameters);
+
+    // Compare screenshots
+    final Image screenshot = decodePng(base64.decode(response.result['data']));
+
     if (write) {
-      file.writeAsBytesSync(bytes, flush: true);
+      // Don't even bother with the comparison, just write and return
+      file.writeAsBytesSync(encodePng(screenshot), flush: true);
       return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.';
     }
-    if (!file.existsSync()) {
-      return '''
-  Golden file $filename does not exist.
 
-  To automatically create this file call matchGoldenFile('$filename', write: true).
-  ''';
-    }
-    final List<int> goldenBytes = file.readAsBytesSync();
-    final int lengths = bytes.length;
-    for (int i = 0; i < lengths; i++) {
-      if (goldenBytes[i] != bytes[i]) {
-        if (write) {
-          file.writeAsBytesSync(bytes, flush: true);
-          return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.';
-        } else {
-          final File failedFile = File(p.join(file.parent.path, '.${p.basename(file.path)}'));
-          failedFile.writeAsBytesSync(bytes, flush: true);
+    ImageDiff diff = ImageDiff(golden: decodeNamedImage(file.readAsBytesSync(), filename), other: screenshot);
 
-          // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces
-          //                different pictures. We need to pin Chrome versions and use a fuzzy image
-          //                comparator.
-          if (Platform.environment['CIRRUS_CI'] == 'true') {
-            return 'OK';
-          }
+    if (diff.rate > 0) { // Images are different, so produce some debug info
+      final File failedFile = File(p.join(file.parent.path, '${p.basenameWithoutExtension(file.path)}.out.png'));
+      failedFile.writeAsBytesSync(encodePng(screenshot), flush: true);
 
-          return '''
-  Golden file ${file.path} did not match the image generated by the test.
+      final File failedDiff = File(p.join(file.parent.path, '${p.basenameWithoutExtension(file.path)}.diff.png'));
+      failedDiff.writeAsBytesSync(encodePng(diff.diff), flush:true);
 
-  The generated image was written to ${failedFile.path}.
+      final String printableDiffFilesInfo = getPrintableDiffFilesInfo(diff.rate, _kMaxDiffRateFailure, filename, failedFile.path, failedDiff.path);
 
-  To update the golden file call matchGoldenFile('$filename', write: true).
-  ''';
+      if (diff.rate < _kMaxDiffRateFailure) { // Warn user
+        print('[WARN] Golden file ${file.path} is slightly different from the image generated by this test!\n${printableDiffFilesInfo}');
+        return 'OK'; // But test success!
+      } else {
+        // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces
+        //                different pictures. We need to pin Chrome versions and use a fuzzy image
+        //                comparator.
+        if (Platform.environment['CIRRUS_CI'] == 'true') {
+          return 'OK';
         }
+        // Fail test
+        return 'Golden file ${file.path} is too different from the image generated by the test!\n${printableDiffFilesInfo}';
       }
     }
     return 'OK';
@@ -867,6 +894,7 @@ class Chrome extends Browser {
         url.toString(),
         if (!debug) '--headless',
         if (isChromeNoSandbox) '--no-sandbox',
+        '--window-size=$_kMaxScreenshotWidth,$_kMaxScreenshotHeight', // When headless, this is the actual size of the viewport
         '--disable-extensions',
         '--disable-popup-blocking',
         '--bwsi',

lib/web_ui/pubspec.yaml

@@ -6,6 +6,7 @@ environment:
 
 dependencies:
   meta: 1.1.7
+  image: 2.1.4
 
 dev_dependencies:
   http: 0.12.0+2

lib/web_ui/test/golden_files/.gitignore

@@ -1,2 +1,3 @@
 # Failed screenshot files
-.*.png
+*.out.png
+*.diff.png

lib/web_ui/test/golden_tests/golden_failure_smoke_test.dart

@@ -5,11 +5,12 @@
 import 'dart:html' as html;
 
 import 'package:test/test.dart';
+import 'package:ui/ui.dart';
 import 'package:web_engine_tester/golden_tester.dart';
 
 void main() {
   test('screenshot test reports failure', () async {
     html.document.body.innerHtml = 'Text that does not appear on the screenshot!';
-    await matchGoldenFile('smoke_test.png');
+    await matchGoldenFile('smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200));
   });
 }

lib/web_ui/test/golden_tests/golden_success_smoke_test.dart

@@ -5,11 +5,12 @@
 import 'dart:html' as html;
 
 import 'package:test/test.dart';
+import 'package:ui/ui.dart';
 import 'package:web_engine_tester/golden_tester.dart';
 
 void main() {
   test('screenshot test reports success', () async {
     html.document.body.innerHtml = 'Hello world!';
-    await matchGoldenFile('smoke_test.png');
+    await matchGoldenFile('smoke_test.png', region: Rect.fromLTWH(0, 0, 320, 200));
   });
 }

web_sdk/web_engine_tester/lib/golden_tester.dart

@@ -6,6 +6,8 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:html' as html;
 
+import 'package:ui/ui.dart';
+
 import 'package:test/test.dart';
 
 Future<dynamic> _callScreenshotServer(dynamic requestData) async {
@@ -19,10 +21,11 @@ Future<dynamic> _callScreenshotServer(dynamic requestData) async {
 }
 
 /// Attempts to match the current browser state with the screenshot [filename].
-Future<void> matchGoldenFile(String filename, { bool write = false }) async {
+Future<void> matchGoldenFile(String filename, { bool write = false, Rect region = null }) async {
   final String response = await _callScreenshotServer(<String, dynamic>{
     'filename': filename,
     'write': write,
+    'region': region == null ? null : {'x': region.left, 'y': region.top, 'width': region.width, 'height': region.height},
   });
   if (response == 'OK') {
     // Pass

web_sdk/web_engine_tester/lib/static/host.css

@@ -2,3 +2,7 @@ body {
   margin: 0;
   padding: 0;
 }
+
+iframe {
+  border: none;
+}