From d3c60acef82cfa5aeab70a368fa359d4e3de9e00 Mon Sep 17 00:00:00 2001 From: David Iglesias Date: Wed, 18 Sep 2019 14:58:30 -0700 Subject: [PATCH] More controls for Screenshot tests (for web) (#12284) New features for golden tests (for web): * Height/width of headless browser instance extracted to constants * Remove border from the host iframe * Added 'region' to matchGoldenFile, so screenshots can capture only a subset of the viewport * Added image-format-awareness (png) to screenshot comparison, so it doesn't only look at raw bytes * When a test fails, output a diff image alongside the unexpected output. --- lib/web_ui/dev/goldens.dart | 80 +++++++++++++++ lib/web_ui/dev/test_platform.dart | 94 ++++++++++++------ lib/web_ui/pubspec.yaml | 1 + lib/web_ui/test/golden_files/.gitignore | 3 +- lib/web_ui/test/golden_files/smoke_test.png | Bin 5228 -> 2413 bytes .../golden_failure_smoke_test.dart | 3 +- .../golden_success_smoke_test.dart | 3 +- .../web_engine_tester/lib/golden_tester.dart | 5 +- web_sdk/web_engine_tester/lib/static/host.css | 4 + 9 files changed, 156 insertions(+), 37 deletions(-) create mode 100644 lib/web_ui/dev/goldens.dart diff --git a/lib/web_ui/dev/goldens.dart b/lib/web_ui/dev/goldens.dart new file mode 100644 index 00000000000..56b519c75b3 --- /dev/null +++ b/lib/web_ui/dev/goldens.dart @@ -0,0 +1,80 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:image/image.dart'; + +/// This class encapsulates visually diffing an Image with any other. +/// Both images need to be the exact same size. +class ImageDiff { + + /// The image to match + final Image golden; + + /// The image being compared + final Image other; + + /// The output of the comparison + /// Pixels in the output image can have 3 different colors depending on the comparison + /// between golden pixels and other pixels: + /// * white: when both pixels are the same + /// * red: when a pixel is found in other, but not in golden + /// * green: when a pixel is found in golden, but not in other + Image diff; + + /// The ratio of wrong pixels to all pixels in golden (between 0 and 1) + /// This gets set to 1 (100% difference) when golden and other aren't the same size. + double get rate => _wrongPixels / _pixelCount; + + ImageDiff({ Image this.golden, Image this.other }) { + _computeDiff(); + } + + int _pixelCount = 0; + int _wrongPixels = 0; + + final int _colorOk = Color.fromRgb(255, 255, 255); + final int _colorBadPixel = Color.fromRgb(255, 0, 0); + final int _colorExpectedPixel = Color.fromRgb(0, 255, 0); + + void _computeDiff() { + int goldenWidth = golden.width; + int goldenHeight = golden.height; + + _pixelCount = goldenWidth * goldenHeight; + diff = Image(goldenWidth, goldenHeight); + + if (goldenWidth == other.width && goldenHeight == other.height) { + for(int y = 0; y < goldenHeight; y++) { + for (int x = 0; x < goldenWidth; x++) { + int goldenPixel = golden.getPixel(x, y); + int otherPixel = other.getPixel(x, y); + + if (goldenPixel == otherPixel) { + diff.setPixel(x, y, _colorOk); + } else { + if (getLuminance(goldenPixel) < getLuminance(otherPixel)) { + diff.setPixel(x, y, _colorExpectedPixel); + } else { + diff.setPixel(x, y, _colorBadPixel); + } + _wrongPixels++; + } + } + } + } else { + // Images are completely different resolutions. Bail out big time. + _wrongPixels = _pixelCount; + } + } +} + +// Returns text with info about the files we just compared +String getPrintableDiffFilesInfo(diffRate, maxRate, filename, outputPath, diffPath) => ''' +(${((diffRate) * 100).toStringAsFixed(4)}% different pixels. Max: ${((maxRate) * 100).toStringAsFixed(4)}%). + +* Output image: ${outputPath} +* Diff pixels: ${diffPath} + +To update the golden file call matchGoldenFile('$filename', write: true). +'''; diff --git a/lib/web_ui/dev/test_platform.dart b/lib/web_ui/dev/test_platform.dart index f98d3f80480..79a348c06a9 100644 --- a/lib/web_ui/dev/test_platform.dart +++ b/lib/web_ui/dev/test_platform.dart @@ -10,6 +10,7 @@ import 'dart:typed_data'; import 'package:async/async.dart'; import 'package:http_multi_server/http_multi_server.dart'; +import 'package:image/image.dart'; import 'package:package_resolver/package_resolver.dart'; import 'package:path/path.dart' as p; import 'package:pedantic/pedantic.dart'; @@ -42,9 +43,13 @@ import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart' import 'chrome_installer.dart'; import 'environment.dart' as env; +import 'goldens.dart'; /// The port number Chrome exposes for debugging. const int _kChromeDevtoolsPort = 12345; +const int _kMaxScreenshotWidth = 1024; +const int _kMaxScreenshotHeight = 1024; +const double _kMaxDiffRateFailure = 1.0/100000; // 0.001% class BrowserPlatform extends PlatformPlugin { /// Starts the server. @@ -137,58 +142,80 @@ class BrowserPlatform extends PlatformPlugin { final Map requestData = json.decode(payload); final String filename = requestData['filename']; final bool write = requestData['write']; - final String result = await _diffScreenshot(filename, write); + final Map region = requestData['region']; + final String result = await _diffScreenshot(filename, write, region); return shelf.Response.ok(json.encode(result)); } - Future _diffScreenshot(String filename, bool write) async { + Future _diffScreenshot(String filename, bool write, [ Map region ]) async { const String _kGoldensDirectory = 'test/golden_files'; + + // Bail out fast if golden doesn't exist, and user doesn't want to create it. + final File file = File(p.join(_kGoldensDirectory, filename)); + if (!file.existsSync() && !write) { + return ''' +Golden file $filename does not exist. + +To automatically create this file call matchGoldenFile('$filename', write: true). +'''; + } + final wip.ChromeConnection chromeConnection = wip.ChromeConnection('localhost', _kChromeDevtoolsPort); final wip.ChromeTab chromeTab = await chromeConnection.getTab( (wip.ChromeTab chromeTab) => chromeTab.url.contains('localhost')); final wip.WipConnection wipConnection = await chromeTab.connect(); + + Map captureScreenshotParameters = null; + if (region != null) { + captureScreenshotParameters = { + 'format': 'png', + 'clip': { + 'x': region['x'], + 'y': region['y'], + 'width': region['width'], + 'height': region['height'], + 'scale': 1, // This is NOT the DPI of the page, instead it's the "zoom level". + }, + }; + } + // To tweak DPI we need to send an additional Emulation.setDeviceMetricsOverride. See: + // https://chromedevtools.github.io/devtools-protocol/tot/Emulation final wip.WipResponse response = - await wipConnection.sendCommand('Page.captureScreenshot'); - final Uint8List bytes = base64.decode(response.result['data']); - final File file = File(p.join(_kGoldensDirectory, filename)); + await wipConnection.sendCommand('Page.captureScreenshot', captureScreenshotParameters); + + // Compare screenshots + final Image screenshot = decodePng(base64.decode(response.result['data'])); + if (write) { - file.writeAsBytesSync(bytes, flush: true); + // Don't even bother with the comparison, just write and return + file.writeAsBytesSync(encodePng(screenshot), flush: true); return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; } - if (!file.existsSync()) { - return ''' - Golden file $filename does not exist. - To automatically create this file call matchGoldenFile('$filename', write: true). - '''; - } - final List goldenBytes = file.readAsBytesSync(); - final int lengths = bytes.length; - for (int i = 0; i < lengths; i++) { - if (goldenBytes[i] != bytes[i]) { - if (write) { - file.writeAsBytesSync(bytes, flush: true); - return 'Golden file $filename was updated. You can remove "write: true" in the call to matchGoldenFile.'; - } else { - final File failedFile = File(p.join(file.parent.path, '.${p.basename(file.path)}')); - failedFile.writeAsBytesSync(bytes, flush: true); + ImageDiff diff = ImageDiff(golden: decodeNamedImage(file.readAsBytesSync(), filename), other: screenshot); - // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces - // different pictures. We need to pin Chrome versions and use a fuzzy image - // comparator. - if (Platform.environment['CIRRUS_CI'] == 'true') { - return 'OK'; - } + if (diff.rate > 0) { // Images are different, so produce some debug info + final File failedFile = File(p.join(file.parent.path, '${p.basenameWithoutExtension(file.path)}.out.png')); + failedFile.writeAsBytesSync(encodePng(screenshot), flush: true); - return ''' - Golden file ${file.path} did not match the image generated by the test. + final File failedDiff = File(p.join(file.parent.path, '${p.basenameWithoutExtension(file.path)}.diff.png')); + failedDiff.writeAsBytesSync(encodePng(diff.diff), flush:true); - The generated image was written to ${failedFile.path}. + final String printableDiffFilesInfo = getPrintableDiffFilesInfo(diff.rate, _kMaxDiffRateFailure, filename, failedFile.path, failedDiff.path); - To update the golden file call matchGoldenFile('$filename', write: true). - '''; + if (diff.rate < _kMaxDiffRateFailure) { // Warn user + print('[WARN] Golden file ${file.path} is slightly different from the image generated by this test!\n${printableDiffFilesInfo}'); + return 'OK'; // But test success! + } else { + // TODO(yjbanov): do not fail Cirrus builds. They currently fail because Chrome produces + // different pictures. We need to pin Chrome versions and use a fuzzy image + // comparator. + if (Platform.environment['CIRRUS_CI'] == 'true') { + return 'OK'; } + // Fail test + return 'Golden file ${file.path} is too different from the image generated by the test!\n${printableDiffFilesInfo}'; } } return 'OK'; @@ -867,6 +894,7 @@ class Chrome extends Browser { url.toString(), if (!debug) '--headless', if (isChromeNoSandbox) '--no-sandbox', + '--window-size=$_kMaxScreenshotWidth,$_kMaxScreenshotHeight', // When headless, this is the actual size of the viewport '--disable-extensions', '--disable-popup-blocking', '--bwsi', diff --git a/lib/web_ui/pubspec.yaml b/lib/web_ui/pubspec.yaml index 27ca873db95..f64508a0ed7 100644 --- a/lib/web_ui/pubspec.yaml +++ b/lib/web_ui/pubspec.yaml @@ -6,6 +6,7 @@ environment: dependencies: meta: 1.1.7 + image: 2.1.4 dev_dependencies: http: 0.12.0+2 diff --git a/lib/web_ui/test/golden_files/.gitignore b/lib/web_ui/test/golden_files/.gitignore index 4b2350ab78f..d41f94a58b1 100644 --- a/lib/web_ui/test/golden_files/.gitignore +++ b/lib/web_ui/test/golden_files/.gitignore @@ -1,2 +1,3 @@ # Failed screenshot files -.*.png +*.out.png +*.diff.png diff --git a/lib/web_ui/test/golden_files/smoke_test.png b/lib/web_ui/test/golden_files/smoke_test.png index c17b3ba1798f0394b3af152ffabfd7a76998c1b9..e136d923394b50c7b2a7f4beeea91524e7ac47c2 100644 GIT binary patch literal 2413 zcmeHJ`B#!zAAc&(RHk{HCbh^&t8w1U&=yM#Ethsh5G#{1m*YAP71snGCf*Z}-gJW(n7ZhaqXr{)62hAq5 zerZ#?r!Pt_cDPs_oII3aTAX_o2+f*?-f@@=POxIhV(>PjYdV(n$x|<# z%GNudDmh($t`w5ka&KBSNRwU?h0CAT^LH%jXCxmO`st4$Z15cY(xm8YcIjBQiDjJZ zvLb8t>uoLUT<_la+#)N&>SmrbkG4HM)A+=wSJQtIa5Q_Pdw$z!#PdC35tTb?S9@BF z9}l!Y*(8pmZR~WeX}QmZeQJ^<%2hP_u3(?Vl)}-!Uf%LDcFl^h^tv&kp|l!N=5D{$ zv47A$ef(;5wrL5H1&{C=zp|qDMeZ6h1r6Xn_|fG0$Si*tRAHRl>ADx&CbwcbmG#ytaxfDs<`xd&dM~s zPNx*W1m1@DkHTl@g-`VYm8+p71UWpObvA3}q=&3vk4n%yMr`LQ-XdTw7n#~4t&6h~ zbrM`*1KxkylhC20WcG?1v9AUmhaikj2dLRg8N+x1{8ht+cfD)9J{=>q;=B$Dy8r)t7lpSyY&G5Xvk1F{w2@6K6`J5dQI*z*%NlR+XyJZq{} z#w&w$80$i)#pfeZhhz2;aF-T9u*(=xJmGtJB*|Vl!LlBl$0HAFyFD95K^k9mS^0#` z4#!e=#|=EvQ?9O9DU6&QNgE4?o+W5G^6Tr6k9-H@(6 zB{iGP_dAjVs-b;qASAdnXRYTW0{G~dH+K)u#7(K`9lHbvbw6f&xJOdOMulef?Gu&Y zWZc0dXmXFkK8kiBFI6h8iyy9=`tYDwD9g25;;vV{uGoWS!uSEeJ+-(Bh{%waLM#GE zR?yK2T-aiOkz24qSB(7OR3)rs{F;p*pnT$Jt&26aUf|vSngVbC#+67(6<@f7Q}Des zPm3Y6S`AZ&T-?w#qTsECW*c!VC=a`&=kncd%w~RK1ir)L28YCK!xyw^8PN7;pSM|2 zD+828LjZuHM}*Tqh=27*V{ZP0XPl7=m;cz+s-&Rj-g8QaNl;>_Qj77P1nu)Ab9xpu zvt`j(9%_?ULz+>DF<~q+qa=t-R3mQtnv`%d{E8dQmQ0RvyKhYbJ-$0YuMYfbt-)PnWJX!XfW*^vB!&s5Sa5v8 z(7 z8MWqUJgUMw>vFg}Avmo6+3ET}6XB~LQZC2AsgBUB0+9YSzQ0`i8yTWz^#A|> literal 5228 zcmeI0X;4#H7RRsRh_o!-)zfM>AaoI7Y}qu@VGSq&8N0)g)1cqmt)~&6XAr{VNlIBngig^<`|Dd ze@`l17>j;Hi#KL2a<~^gFD4=NeFa;}0+{d4>Yu!xk7_)0uUWh6;e-1+`ui+fhO}P% z{JHp-x*1)a?6Z!2MnbnQ@qIePjFO4NH)f7+B^>a|@)!#;_$@=5BRaK!45%5rM+h1e zlgq?vXGgqhaGB9H8ppCq(E>&Xw`yw>hA&hYf7lN^53aZw0I(+0)z`CJW!a$f2>_`7G}3~1?K1S<*aX1EE!zM%ZsrC+`Zt?4 z12C}da{zwv{C%r7!c8S9V(_(HR}al{Xfy0`UL*8l4*{@C1t4u`qd<~=A5Z|_c znwiH5y=HvtHlFT)r^{!(%I66*WuQti$e3Md;rUQRLk$*Ux-zCYW=vtyc5_`^9G2T% zfJIZ=Q)IbGscWl?Lv>+Zs+AUo-onVU_(*e~(&7<3qutINfYh@&j&44#>r%^rW=o7@ z+C*(nnC-z(bai#lbyM~@%oO?_NLdkC2DjOJ?I!ZQ$|EKGZmMIvuy3WK!wuI{yeRGo zu_Gy@-lg`JXiN)uw6Bjt__04B(<(%QApEetocit1T$D?<^2 z`4Dw-aO&c#yk*5TlQjFHz&Okfo8Z!#wB;xMEiq%6^O}d=G_(smoTXTvZ-8IvDRV-F zb!B%q4F=yv5SQQiAtSk!mj3k!jAUg5A2C!P?*Ht^3-rYtJpXQ%^_yH{Ckg_KmY>^U z9awC#{=utNA$+nV=aA8P&XU)ej1&R8taVMoBFsG~!o9tFq*XSaa?O}#8+usZfuHYP za&RuJJrMx=T?;LM2yJc%?YwFkLU%Smub#<;Cs>}UJu^^4hGZn2CMw%~Co5m~qIjiG zs-JzgeTU_1U#!dWou>YW$5@gXRzr&Hqj#C3oke-lS?FTO9?UGe>^8zdwK|Q2E{(=^ z>R~9`ZQ5)#`}I(x`O1&aZEV`5C+NyCu{lQHNmT2sUR%;Aj4pj(6AsO#+J?T&Bv`|E z)7C{1g#Kfw@MYmUqHHYDwQ=@+Sl(Aq`1$9ggWjYLXaGS_mZ(3{0x6az3m?0W42Fu}NFZ8YOUCOiq($ zsS&uT->%D{*ZHrqc1Ha3KE^@X=iZRKEKHTNLfDZt?Y9bi9QSHs#j;_PLXo=c^Q-oj zjEDc@XpwdAntW+V)WE^b_E4z@!vCu8BJI<#q)j#wmu4$ZH+^TrSc{{`tQjRJd~i(r zGBaLwoaFohg7mFd-c*nl=y(y$ z2)f)v5U`zPqlC_t^>s0s<0NiF3JGm0#e8S8_SUVQZJSgiL3v8dZt~h<)Oec%)_Kt* z4Fbc>bE}9S?p&+C(EqSB_N9)}Sq+ntasyw>D?*JYnZd;UdZp>PnZtZ#gD1%cO0pPA zlKOr0>6mA852JW=(s>@9tQ;{Q9UY+el-TXdgAuJIjWkEIs1fDz8#NAd_X{NbG}Xa| z2oC#aoi7hWuuB7pGdJZ^I~n#-L6lGduQ__859#9>`KE9$Msmk+KqeYOMk$?ma`q#R z4glH>P)}lVa$V$L^%r>yqpgJVN(sw`bIU3~(o{pT(ukk3=V*NxGxrX5SAuxFZ0vcW zK@|^{u3GXad8L6ibJH)ldn}Px#&5@0CG&51#l8B55J2 zw;Ej?wP`PG8Bf-@f4(_th~lc2b1)Rm3iNw-a;zEb9CQ0gS{J(v3lLD1Mk@&E z&3%5j&Z~Mv;p2+>i|>AIlh59ESeNz_=Brv}VG;MLrYnS^;ikyH>@H85y6*g`LyHJ= zpgv(?IBK|YF9W7%J&b%Hd41Vk-N?JdG;S6C`a}XH;Hs%>X-l_xx+&?CKd_-70)Kpa z1XdS!3WBHLw$#T?zBGy-_**A;T_t01x-Dp>NVdIMQE>8w?_q}tQ7Z;9l=yk>43@0$ z _callScreenshotServer(dynamic requestData) async { @@ -19,10 +21,11 @@ Future _callScreenshotServer(dynamic requestData) async { } /// Attempts to match the current browser state with the screenshot [filename]. -Future matchGoldenFile(String filename, { bool write = false }) async { +Future matchGoldenFile(String filename, { bool write = false, Rect region = null }) async { final String response = await _callScreenshotServer({ 'filename': filename, 'write': write, + 'region': region == null ? null : {'x': region.left, 'y': region.top, 'width': region.width, 'height': region.height}, }); if (response == 'OK') { // Pass diff --git a/web_sdk/web_engine_tester/lib/static/host.css b/web_sdk/web_engine_tester/lib/static/host.css index ea1e941c368..3a1f061636e 100644 --- a/web_sdk/web_engine_tester/lib/static/host.css +++ b/web_sdk/web_engine_tester/lib/static/host.css @@ -2,3 +2,7 @@ body { margin: 0; padding: 0; } + +iframe { + border: none; +}