From 9e797d146196a1c3d2cdae389f3b40b3c04d86fc Mon Sep 17 00:00:00 2001 From: Harry Terkelsen <1961493+harryterkelsen@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:41:05 -0700 Subject: [PATCH] [canvaskit] Decode images using tag decoding (flutter/engine#53201) Prefer to decode images using the browser API rather than with CanvasKit to avoid jank when decoding. Part of deprecating the HTML renderer: https://github.com/flutter/flutter/issues/145954 [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style --- .../src/engine/canvaskit/canvaskit_api.dart | 6 +- .../lib/src/engine/canvaskit/image.dart | 470 ++++++- .../engine/canvaskit/image_web_codecs.dart | 90 +- .../lib/src/engine/canvaskit/picture.dart | 9 +- .../lib/src/engine/canvaskit/renderer.dart | 5 +- .../lib/web_ui/lib/src/engine/dom.dart | 2 +- .../lib/web_ui/lib/src/engine/html/image.dart | 4 + .../src/engine/html_image_element_codec.dart | 134 +- .../web_ui/lib/src/engine/image_decoder.dart | 37 +- .../test/canvaskit/embedded_views_test.dart | 1 + .../test/canvaskit/image_golden_test.dart | 1118 ++++------------- 11 files changed, 823 insertions(+), 1053 deletions(-) diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart index 7a847220972..99fd3671206 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart @@ -1184,10 +1184,10 @@ extension SkImageExtension on SkImage { matrix?.toJS); @JS('readPixels') - external JSUint8Array _readPixels( + external JSUint8Array? _readPixels( JSNumber srcX, JSNumber srcY, SkImageInfo imageInfo); - Uint8List readPixels(double srcX, double srcY, SkImageInfo imageInfo) => - _readPixels(srcX.toJS, srcY.toJS, imageInfo).toDart; + Uint8List? readPixels(double srcX, double srcY, SkImageInfo imageInfo) => + _readPixels(srcX.toJS, srcY.toJS, imageInfo)?.toDart; @JS('encodeToBytes') external JSUint8Array? _encodeToBytes(); diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart index 23c585f7f8a..c55dff72645 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:js_interop'; +import 'dart:math' as math; import 'dart:typed_data'; import 'package:ui/src/engine.dart'; @@ -11,17 +12,185 @@ import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; /// Instantiates a [ui.Codec] backed by an `SkAnimatedImage` from Skia. -FutureOr skiaInstantiateImageCodec(Uint8List list, - [int? targetWidth, int? targetHeight]) { - // If we have either a target width or target height, use canvaskit to decode. - if (browserSupportsImageDecoder && (targetWidth == null && targetHeight == null)) { - return CkBrowserImageDecoder.create( +Future skiaInstantiateImageCodec(Uint8List list, + [int? targetWidth, int? targetHeight, bool allowUpscaling = true]) async { + ui.Codec codec; + // ImageDecoder does not detect image type automatically. It requires us to + // tell it what the image type is. + final String contentType = tryDetectContentType(list, 'encoded image bytes'); + + if (browserSupportsImageDecoder) { + codec = await CkBrowserImageDecoder.create( data: list, + contentType: contentType, debugSource: 'encoded image bytes', ); } else { - return CkAnimatedImage.decodeFromBytes(list, 'encoded image bytes', targetWidth: targetWidth, targetHeight: targetHeight); + // TODO(harryterkelsen): If the image is animated, then use Skia to decode. + // This is currently too conservative, assuming all GIF and WEBP images are + // animated. We should detect if they are actually animated by reading the + // image headers, https://github.com/flutter/flutter/issues/151911. + if (contentType == 'image/gif' || contentType == 'image/webp') { + codec = CkAnimatedImage.decodeFromBytes(list, 'encoded image bytes', + targetWidth: targetWidth, targetHeight: targetHeight); + } else { + final DomBlob blob = createDomBlob([list.buffer]); + codec = await decodeBlobToCkImage(blob); + } } + return CkResizingCodec( + codec, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); +} + +/// A resizing codec which uses an HTML element to scale the image if +/// it is backed by an HTML Image element. +class CkResizingCodec extends ResizingCodec { + CkResizingCodec( + super.delegate, { + super.targetWidth, + super.targetHeight, + super.allowUpscaling, + }); + + @override + ui.Image scaleImage( + ui.Image image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, + }) { + final CkImage ckImage = image as CkImage; + if (ckImage.imageSource == null) { + return scaleImageIfNeeded( + image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); + } else { + return _scaleImageUsingDomCanvas( + ckImage, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); + } + } + + CkImage _scaleImageUsingDomCanvas( + CkImage image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, + }) { + assert(image.imageSource != null); + final int width = image.width; + final int height = image.height; + final BitmapSize? scaledSize = + scaledImageSize(width, height, targetWidth, targetHeight); + if (scaledSize == null) { + return image; + } + if (!allowUpscaling && + (scaledSize.width > width || scaledSize.height > height)) { + return image; + } + + final int scaledWidth = scaledSize.width; + final int scaledHeight = scaledSize.height; + + final DomOffscreenCanvas offscreenCanvas = createDomOffscreenCanvas( + scaledWidth, + scaledHeight, + ); + final DomCanvasRenderingContext2D ctx = + offscreenCanvas.getContext('2d')! as DomCanvasRenderingContext2D; + ctx.drawImage( + image.imageSource!.canvasImageSource, + 0, + 0, + width, + height, + 0, + 0, + scaledWidth, + scaledHeight, + ); + final DomImageBitmap bitmap = offscreenCanvas.transferToImageBitmap(); + final SkImage? skImage = + canvasKit.MakeLazyImageFromImageBitmap(bitmap, true); + + // Resize the canvas to 0x0 to cause the browser to eagerly reclaim its + // memory. + offscreenCanvas.width = 0; + offscreenCanvas.height = 0; + + if (skImage == null) { + domWindow.console.warn('Failed to scale image.'); + return image; + } + + return CkImage(skImage, imageSource: ImageBitmapImageSource(bitmap)); + } +} + +ui.Image createCkImageFromImageElement( + DomHTMLImageElement image, + int naturalWidth, + int naturalHeight, +) { + final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo( + image, + SkPartialImageInfo( + alphaType: canvasKit.AlphaType.Premul, + colorType: canvasKit.ColorType.RGBA_8888, + colorSpace: SkColorSpaceSRGB, + width: naturalWidth.toDouble(), + height: naturalHeight.toDouble(), + ), + ); + if (skImage == null) { + throw ImageCodecException( + 'Failed to create image from Image.decode', + ); + } + + return CkImage(skImage, imageSource: ImageElementImageSource(image)); +} + +class CkImageElementCodec extends HtmlImageElementCodec { + CkImageElementCodec(super.src); + + @override + ui.Image createImageFromHTMLImageElement( + DomHTMLImageElement image, int naturalWidth, int naturalHeight) => + createCkImageFromImageElement(image, naturalWidth, naturalHeight); +} + +class CkImageBlobCodec extends HtmlBlobCodec { + CkImageBlobCodec(super.blob); + + @override + ui.Image createImageFromHTMLImageElement( + DomHTMLImageElement image, int naturalWidth, int naturalHeight) => + createCkImageFromImageElement(image, naturalWidth, naturalHeight); +} + +/// Creates and decodes an image using HtmlImageElement. +Future decodeBlobToCkImage(DomBlob blob) async { + final CkImageBlobCodec codec = CkImageBlobCodec(blob); + await codec.decode(); + return codec; +} + +Future decodeUrlToCkImage(String src) async { + final CkImageElementCodec codec = CkImageElementCodec(src); + await codec.decode(); + return codec; } void skiaDecodeImageFromPixels( @@ -49,7 +218,9 @@ void skiaDecodeImageFromPixels( SkImageInfo( width: width.toDouble(), height: height.toDouble(), - colorType: format == ui.PixelFormat.rgba8888 ? canvasKit.ColorType.RGBA_8888 : canvasKit.ColorType.BGRA_8888, + colorType: format == ui.PixelFormat.rgba8888 + ? canvasKit.ColorType.RGBA_8888 + : canvasKit.ColorType.BGRA_8888, alphaType: canvasKit.AlphaType.Premul, colorSpace: SkColorSpaceSRGB, ), @@ -63,7 +234,8 @@ void skiaDecodeImageFromPixels( } if (targetWidth != null || targetHeight != null) { - if (validUpscale(allowUpscaling, targetWidth, targetHeight, width, height)) { + if (validUpscale( + allowUpscaling, targetWidth, targetHeight, width, height)) { return callback(scaleImage(skImage, targetWidth, targetHeight)); } } @@ -73,7 +245,8 @@ void skiaDecodeImageFromPixels( // An invalid upscale happens when allowUpscaling is false AND either the given // targetWidth is larger than the originalWidth OR the targetHeight is larger than originalHeight. -bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, int originalWidth, int originalHeight) { +bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, + int originalWidth, int originalHeight) { if (allowUpscaling) { return true; } @@ -104,42 +277,39 @@ bool validUpscale(bool allowUpscaling, int? targetWidth, int? targetHeight, int /// If either targetWidth or targetHeight is less than or equal to zero, it /// will be treated as if it is null. CkImage scaleImage(SkImage image, int? targetWidth, int? targetHeight) { - assert(targetWidth != null || targetHeight != null); - if (targetWidth != null && targetWidth <= 0) { - targetWidth = null; - } - if (targetHeight != null && targetHeight <= 0) { - targetHeight = null; - } - if (targetWidth == null && targetHeight != null) { - targetWidth = (targetHeight * (image.width() / image.height())).round(); - } else if (targetHeight == null && targetWidth != null) { - targetHeight = targetWidth ~/ (image.width() / image.height()); - } + assert(targetWidth != null || targetHeight != null); + if (targetWidth != null && targetWidth <= 0) { + targetWidth = null; + } + if (targetHeight != null && targetHeight <= 0) { + targetHeight = null; + } + if (targetWidth == null && targetHeight != null) { + targetWidth = (targetHeight * (image.width() / image.height())).round(); + } else if (targetHeight == null && targetWidth != null) { + targetHeight = targetWidth ~/ (image.width() / image.height()); + } - assert(targetWidth != null); - assert(targetHeight != null); + assert(targetWidth != null); + assert(targetHeight != null); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); + final CkPictureRecorder recorder = CkPictureRecorder(); + final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - final CkPaint paint = CkPaint(); - canvas.drawImageRect( - CkImage(image), - ui.Rect.fromLTWH(0, 0, image.width(), image.height()), - ui.Rect.fromLTWH(0, 0, targetWidth!.toDouble(), targetHeight!.toDouble()), - paint, - ); - paint.dispose(); + final CkPaint paint = CkPaint(); + canvas.drawImageRect( + CkImage(image), + ui.Rect.fromLTWH(0, 0, image.width(), image.height()), + ui.Rect.fromLTWH(0, 0, targetWidth!.toDouble(), targetHeight!.toDouble()), + paint, + ); + paint.dispose(); - final CkPicture picture = recorder.endRecording(); - final ui.Image finalImage = picture.toImageSync( - targetWidth, - targetHeight - ); + final CkPicture picture = recorder.endRecording(); + final ui.Image finalImage = picture.toImageSync(targetWidth, targetHeight); - final CkImage ckImage = finalImage as CkImage; - return ckImage; + final CkImage ckImage = finalImage as CkImage; + return ckImage; } /// Thrown when the web engine fails to decode an image, either due to a @@ -159,16 +329,35 @@ const String _kNetworkImageMessage = 'Failed to load network image.'; /// requesting from URI. Future skiaInstantiateWebImageCodec( String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { - final Uint8List list = await fetchImage(url, chunkCallback); - if (browserSupportsImageDecoder) { - return CkBrowserImageDecoder.create(data: list, debugSource: url); - } else { - return CkAnimatedImage.decodeFromBytes(list, url); + final CkImageElementCodec imageElementCodec = CkImageElementCodec(url); + try { + await imageElementCodec.decode(); + return imageElementCodec; + } on ImageCodecException { + imageElementCodec.dispose(); + final Uint8List list = await fetchImage(url, chunkCallback); + final String contentType = tryDetectContentType(list, url); + if (browserSupportsImageDecoder) { + return CkBrowserImageDecoder.create( + data: list, contentType: contentType, debugSource: url); + } else { + final DomBlob blob = createDomBlob([list.buffer]); + final CkImageBlobCodec codec = CkImageBlobCodec(blob); + + try { + await codec.decode(); + return codec; + } on ImageCodecException { + codec.dispose(); + return CkAnimatedImage.decodeFromBytes(list, url); + } + } } } /// Sends a request to fetch image data. -Future fetchImage(String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { +Future fetchImage( + String url, ui_web.ImageCodecChunkCallback? chunkCallback) async { try { final HttpFetchResponse response = await httpFetch(url); final int? contentLength = response.contentLength; @@ -199,7 +388,8 @@ Future fetchImage(String url, ui_web.ImageCodecChunkCallback? chunkCa /// Reads the [payload] in chunks using the browser's Streams API /// /// See: https://developer.mozilla.org/en-US/docs/Web/API/Streams_API -Future readChunked(HttpFetchPayload payload, int contentLength, ui_web.ImageCodecChunkCallback chunkCallback) async { +Future readChunked(HttpFetchPayload payload, int contentLength, + ui_web.ImageCodecChunkCallback chunkCallback) async { final JSUint8Array result = createUint8ArrayFromLength(contentLength); int position = 0; int cumulativeBytesLoaded = 0; @@ -214,12 +404,12 @@ Future readChunked(HttpFetchPayload payload, int contentLength, ui_we /// A [ui.Image] backed by an `SkImage` from Skia. class CkImage implements ui.Image, StackTraceDebugger { - CkImage(SkImage skImage, { this.videoFrame }) { + CkImage(SkImage skImage, {this.imageSource}) { box = CountedRef(skImage, this, 'SkImage'); _init(); } - CkImage.cloneOf(this.box, {this.videoFrame}) { + CkImage.cloneOf(this.box, {this.imageSource}) { _init(); box.ref(this); } @@ -240,13 +430,11 @@ class CkImage implements ui.Image, StackTraceDebugger { // being garbage-collected, or by an explicit call to [delete]. late final CountedRef box; - /// For browsers that support `ImageDecoder` this field holds the video frame - /// from which this image was created. - /// - /// Skia owns the video frame and will close it when it's no longer used. - /// However, Flutter co-owns the [SkImage] and therefore it's safe to access - /// the video frame until the image is [dispose]d of. - VideoFrame? videoFrame; + /// If this [CkImage] is backed by an image source (either VideoFrame, + /// element, or ImageBitmap), this is the backing image source. We read pixels + /// and byte data from the backing image source rather than from the [SkImage] + /// because of this bug: https://issues.skia.org/issues/40043810. + ImageSource? imageSource; /// The underlying Skia image object. /// @@ -270,6 +458,7 @@ class CkImage implements ui.Image, StackTraceDebugger { ui.Image.onDispose?.call(this); _disposed = true; box.unref(this); + imageSource?.close(); } @override @@ -291,7 +480,10 @@ class CkImage implements ui.Image, StackTraceDebugger { @override CkImage clone() { assert(_debugCheckIsNotDisposed()); - return CkImage.cloneOf(box, videoFrame: videoFrame?.clone()); + return CkImage.cloneOf( + box, + imageSource: imageSource, + ); } @override @@ -321,20 +513,51 @@ class CkImage implements ui.Image, StackTraceDebugger { ui.ImageByteFormat format = ui.ImageByteFormat.rawRgba, }) { assert(_debugCheckIsNotDisposed()); - // readPixelsFromVideoFrame currently does not convert I420, I444, I422 - // videoFrame formats to RGBA - if (videoFrame != null && videoFrame!.format != 'I420' && videoFrame!.format != 'I444' && videoFrame!.format != 'I422') { - return readPixelsFromVideoFrame(videoFrame!, format); + switch (imageSource) { + case ImageElementImageSource(): + final DomHTMLImageElement imageElement = + (imageSource! as ImageElementImageSource).imageElement; + return readPixelsFromDomImageSource( + imageElement, + format, + imageElement.naturalWidth.toInt(), + imageElement.naturalHeight.toInt(), + ); + case ImageBitmapImageSource(): + final DomImageBitmap imageBitmap = + (imageSource! as ImageBitmapImageSource).imageBitmap; + return readPixelsFromDomImageSource( + imageBitmap, + format, + imageBitmap.width.toDartInt, + imageBitmap.height.toDartInt, + ); + case VideoFrameImageSource(): + final VideoFrame videoFrame = + (imageSource! as VideoFrameImageSource).videoFrame; + if (videoFrame.format != 'I420' && + videoFrame.format != 'I444' && + videoFrame.format != 'I422') { + return readPixelsFromVideoFrame(videoFrame, format); + } + case null: + } + ByteData? data = _readPixelsFromSkImage(format); + data ??= _readPixelsFromImageViaSurface(format); + if (data == null) { + return Future.error('Failed to encode the image into bytes.'); } else { - return _readPixelsFromSkImage(format); + return Future.value(data); } } @override ui.ColorSpace get colorSpace => ui.ColorSpace.sRGB; - Future _readPixelsFromSkImage(ui.ImageByteFormat format) { - final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba ? canvasKit.AlphaType.Unpremul : canvasKit.AlphaType.Premul; + ByteData? _readPixelsFromSkImage(ui.ImageByteFormat format) { + final SkAlphaType alphaType = format == ui.ImageByteFormat.rawStraightRgba + ? canvasKit.AlphaType.Unpremul + : canvasKit.AlphaType.Premul; final ByteData? data = _encodeImage( skImage: skImage, format: format, @@ -342,11 +565,29 @@ class CkImage implements ui.Image, StackTraceDebugger { colorType: canvasKit.ColorType.RGBA_8888, colorSpace: SkColorSpaceSRGB, ); - if (data == null) { - return Future.error('Failed to encode the image into bytes.'); - } else { - return Future.value(data); + return data; + } + + ByteData? _readPixelsFromImageViaSurface(ui.ImageByteFormat format) { + final Surface surface = CanvasKitRenderer.instance.pictureToImageSurface; + final CkSurface ckSurface = + surface.createOrUpdateSurface(BitmapSize(width, height)); + final CkCanvas ckCanvas = ckSurface.getCanvas(); + ckCanvas.clear(const ui.Color(0x00000000)); + ckCanvas.drawImage(this, ui.Offset.zero, CkPaint()); + final SkImage skImage = ckSurface.surface.makeImageSnapshot(); + final SkImageInfo imageInfo = SkImageInfo( + alphaType: canvasKit.AlphaType.Premul, + colorType: canvasKit.ColorType.RGBA_8888, + colorSpace: SkColorSpaceSRGB, + width: width.toDouble(), + height: height.toDouble(), + ); + final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); + if (pixels == null) { + throw StateError('Unable to convert read pixels from SkImage.'); } + return pixels.buffer.asByteData(); } static ByteData? _encodeImage({ @@ -358,7 +599,8 @@ class CkImage implements ui.Image, StackTraceDebugger { }) { Uint8List? bytes; - if (format == ui.ImageByteFormat.rawRgba || format == ui.ImageByteFormat.rawStraightRgba) { + if (format == ui.ImageByteFormat.rawRgba || + format == ui.ImageByteFormat.rawStraightRgba) { final SkImageInfo imageInfo = SkImageInfo( alphaType: alphaType, colorType: colorType, @@ -380,3 +622,93 @@ class CkImage implements ui.Image, StackTraceDebugger { return '[$width\u00D7$height]'; } } + +/// Detect the content type or throw an error if content type can't be detected. +String tryDetectContentType(Uint8List data, String debugSource) { + // ImageDecoder does not detect image type automatically. It requires us to + // tell it what the image type is. + final String? contentType = detectContentType(data); + + if (contentType == null) { + final String fileHeader; + if (data.isNotEmpty) { + fileHeader = + '[${bytesToHexString(data.sublist(0, math.min(10, data.length)))}]'; + } else { + fileHeader = 'empty'; + } + throw ImageCodecException( + 'Failed to detect image file format using the file header.\n' + 'File header was $fileHeader.\n' + 'Image source: $debugSource'); + } + return contentType; +} + +sealed class ImageSource { + DomCanvasImageSource get canvasImageSource; + int get width; + int get height; + void close(); +} + +class VideoFrameImageSource extends ImageSource { + VideoFrameImageSource(this.videoFrame); + + final VideoFrame videoFrame; + + @override + void close() { + // Do nothing. Skia will close the VideoFrame when the SkImage is disposed. + } + + @override + int get height => videoFrame.displayHeight.toInt(); + + @override + int get width => videoFrame.displayWidth.toInt(); + + @override + DomCanvasImageSource get canvasImageSource => videoFrame; +} + +class ImageElementImageSource extends ImageSource { + ImageElementImageSource(this.imageElement); + + final DomHTMLImageElement imageElement; + + @override + void close() { + // There's no way to immediately close the element. Just let the + // browser garbage collect it. + } + + @override + int get height => imageElement.naturalHeight.toInt(); + + @override + int get width => imageElement.naturalWidth.toInt(); + + @override + DomCanvasImageSource get canvasImageSource => imageElement; +} + +class ImageBitmapImageSource extends ImageSource { + ImageBitmapImageSource(this.imageBitmap); + + final DomImageBitmap imageBitmap; + + @override + void close() { + imageBitmap.close(); + } + + @override + int get height => imageBitmap.height.toDartInt; + + @override + int get width => imageBitmap.width.toDartInt; + + @override + DomCanvasImageSource get canvasImageSource => imageBitmap; +} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart index f2b213c731c..f9c583aa9b3 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart @@ -11,7 +11,6 @@ import 'dart:async'; import 'dart:convert' show base64; import 'dart:js_interop'; -import 'dart:math' as math; import 'dart:typed_data'; import 'package:ui/src/engine.dart'; @@ -27,26 +26,9 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { static Future create({ required Uint8List data, + required String contentType, required String debugSource, }) async { - // ImageDecoder does not detect image type automatically. It requires us to - // tell it what the image type is. - final String? contentType = detectContentType(data); - - if (contentType == null) { - final String fileHeader; - if (data.isNotEmpty) { - fileHeader = '[${bytesToHexString(data.sublist(0, math.min(10, data.length)))}]'; - } else { - fileHeader = 'empty'; - } - throw ImageCodecException( - 'Failed to detect image file format using the file header.\n' - 'File header was $fileHeader.\n' - 'Image source: $debugSource' - ); - } - final CkBrowserImageDecoder decoder = CkBrowserImageDecoder._( contentType: contentType, dataSource: data.toJS, @@ -76,13 +58,18 @@ class CkBrowserImageDecoder extends BrowserImageDecoder { ); } - return CkImage(skImage, videoFrame: frame); + return CkImage(skImage, imageSource: VideoFrameImageSource(frame)); } } -Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFormat format) async { +Future readPixelsFromVideoFrame( + VideoFrame videoFrame, ui.ImageByteFormat format) async { if (format == ui.ImageByteFormat.png) { - final Uint8List png = await encodeVideoFrameAsPng(videoFrame); + final Uint8List png = await encodeDomImageSourceAsPng( + videoFrame, + videoFrame.displayWidth.toInt(), + videoFrame.displayHeight.toInt(), + ); return png.buffer.asByteData(); } @@ -112,6 +99,26 @@ Future readPixelsFromVideoFrame(VideoFrame videoFrame, ui.ImageByteFor return pixels.asByteData(); } +Future readPixelsFromDomImageSource( + DomCanvasImageSource imageSource, + ui.ImageByteFormat format, + int width, + int height, +) async { + if (format == ui.ImageByteFormat.png) { + final Uint8List png = await encodeDomImageSourceAsPng( + imageSource, + width, + height, + ); + return png.buffer.asByteData(); + } + + final ByteBuffer pixels = + readDomImageSourcePixelsUnmodified(imageSource, width, height); + return pixels.asByteData(); +} + /// Mutates the [pixels], converting them from BGRX/BGRA to RGBA. void _bgrToStraightRgba(ByteBuffer pixels, bool isBgrx) { final Uint8List pixelBytes = pixels.asUint8List(); @@ -159,14 +166,16 @@ void _bgrToRawRgba(ByteBuffer pixels) { } } -bool _shouldReadPixelsUnmodified(VideoFrame videoFrame, ui.ImageByteFormat format) { +bool _shouldReadPixelsUnmodified( + VideoFrame videoFrame, ui.ImageByteFormat format) { if (format == ui.ImageByteFormat.rawUnmodified) { return true; } // Do not convert if the requested format is RGBA and the video frame is // encoded as either RGBA or RGBX. - final bool isRgbFrame = videoFrame.format == 'RGBA' || videoFrame.format == 'RGBX'; + final bool isRgbFrame = + videoFrame.format == 'RGBA' || videoFrame.format == 'RGBX'; return format == ui.ImageByteFormat.rawStraightRgba && isRgbFrame; } @@ -184,13 +193,32 @@ Future readVideoFramePixelsUnmodified(VideoFrame videoFrame) async { return destination.toDart.buffer; } -Future encodeVideoFrameAsPng(VideoFrame videoFrame) async { - final int width = videoFrame.displayWidth.toInt(); - final int height = videoFrame.displayHeight.toInt(); - final DomCanvasElement canvas = createDomCanvasElement(width: width, height: - height); +ByteBuffer readDomImageSourcePixelsUnmodified( + DomCanvasImageSource imageSource, int width, int height) { + final DomCanvasElement htmlCanvas = + createDomCanvasElement(width: width, height: height); + final DomCanvasRenderingContext2D ctx = + htmlCanvas.getContext('2d')! as DomCanvasRenderingContext2D; + ctx.drawImage(imageSource, 0, 0); + final DomImageData imageData = ctx.getImageData(0, 0, width, height); + // Resize the canvas to 0x0 to cause the browser to reclaim its memory + // eagerly. + htmlCanvas.width = 0; + htmlCanvas.height = 0; + return imageData.data.buffer; +} + +Future encodeDomImageSourceAsPng( + DomCanvasImageSource imageSource, int width, int height) async { + final DomCanvasElement canvas = + createDomCanvasElement(width: width, height: height); final DomCanvasRenderingContext2D ctx = canvas.context2D; - ctx.drawImage(videoFrame, 0, 0); - final String pngBase64 = canvas.toDataURL().substring('data:image/png;base64,'.length); + ctx.drawImage(imageSource, 0, 0); + final String pngBase64 = + canvas.toDataURL().substring('data:image/png;base64,'.length); + // Resize the canvas to 0x0 to cause the browser to reclaim its memory + // eagerly. + canvas.width = 0; + canvas.height = 0; return base64.decode(pngBase64); } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart index ba30b55d2fc..addb864c441 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart @@ -101,8 +101,8 @@ class CkPicture implements ScenePicture { assert(debugCheckNotDisposed('Cannot convert picture to image.')); final Surface surface = CanvasKitRenderer.instance.pictureToImageSurface; - final CkSurface ckSurface = surface - .createOrUpdateSurface(BitmapSize(width, height)); + final CkSurface ckSurface = + surface.createOrUpdateSurface(BitmapSize(width, height)); final CkCanvas ckCanvas = ckSurface.getCanvas(); ckCanvas.clear(const ui.Color(0x00000000)); ckCanvas.drawPicture(this); @@ -114,7 +114,10 @@ class CkPicture implements ScenePicture { width: width.toDouble(), height: height.toDouble(), ); - final Uint8List pixels = skImage.readPixels(0, 0, imageInfo); + final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); + if (pixels == null) { + throw StateError('Unable to read pixels from SkImage.'); + } final SkImage? rasterImage = canvasKit.MakeImage(imageInfo, pixels, (4 * width).toDouble()); if (rasterImage == null) { diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index f94916d2732..ff5c7006f28 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -222,7 +222,8 @@ class CanvasKitRenderer implements Renderer { {int? targetWidth, int? targetHeight, bool allowUpscaling = true}) async => - skiaInstantiateImageCodec(list, targetWidth, targetHeight); + skiaInstantiateImageCodec( + list, targetWidth, targetHeight, allowUpscaling); @override Future instantiateImageCodecFromUrl(Uri uri, @@ -236,7 +237,7 @@ class CanvasKitRenderer implements Renderer { if (skImage == null) { throw Exception('Failed to convert image bitmap to an SkImage.'); } - return CkImage(skImage); + return CkImage(skImage, imageSource: ImageBitmapImageSource(imageBitmap)); } @override diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart index e632b758807..18205948582 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart @@ -1551,7 +1551,7 @@ extension DomImageDataExtension on DomImageData { @JS('ImageBitmap') @staticInterop -class DomImageBitmap {} +class DomImageBitmap implements DomCanvasImageSource {} extension DomImageBitmapExtension on DomImageBitmap { external JSNumber get width; diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/html/image.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/html/image.dart index 72c9f9ddbbd..b1a5b6d3cfb 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/html/image.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/html/image.dart @@ -98,6 +98,10 @@ class HtmlImage implements ui.Image { final DomCanvasRenderingContext2D ctx = canvas.context2D; ctx.drawImage(imgElement, 0, 0); final DomImageData imageData = ctx.getImageData(0, 0, width, height); + // Resize the canvas to 0x0 to cause the browser to reclaim its memory + // eagerly. + canvas.width = 0; + canvas.height = 0; return Future.value(imageData.data.buffer.asByteData()); default: if (imgElement.src?.startsWith('data:') ?? false) { diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/html_image_element_codec.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/html_image_element_codec.dart index b40fec7dabc..2bd6ccabc54 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/html_image_element_codec.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/html_image_element_codec.dart @@ -4,30 +4,20 @@ import 'dart:async'; +import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import 'dom.dart'; -import 'safe_browser_api.dart'; - -Object? get _jsImageDecodeFunction => getJsProperty( - getJsProperty( - getJsProperty(domWindow, 'Image'), - 'prototype', - ), - 'decode', - ); -final bool _supportsDecode = _jsImageDecodeFunction != null; - // TODO(mdebbar): Deprecate this and remove it. // https://github.com/flutter/flutter/issues/127395 typedef WebOnlyImageCodecChunkCallback = ui_web.ImageCodecChunkCallback; abstract class HtmlImageElementCodec implements ui.Codec { - HtmlImageElementCodec(this.src, {this.chunkCallback}); + HtmlImageElementCodec(this.src, {this.chunkCallback, this.debugSource}); final String src; final ui_web.ImageCodecChunkCallback? chunkCallback; + final String? debugSource; @override int get frameCount => 1; @@ -35,82 +25,58 @@ abstract class HtmlImageElementCodec implements ui.Codec { @override int get repetitionCount => 0; - @override - Future getNextFrame() async { - final Completer completer = Completer(); + /// The Image() element backing this codec. + DomHTMLImageElement? imgElement; + + /// A Future which completes when the Image element backing this codec has + /// been loaded and decoded. + Future? decodeFuture; + + Future decode() async { + if (decodeFuture != null) { + return decodeFuture; + } + final Completer completer = Completer(); + decodeFuture = completer.future; // Currently there is no way to watch decode progress, so // we add 0/100 , 100/100 progress callbacks to enable loading progress // builders to create UI. chunkCallback?.call(0, 100); - if (_supportsDecode) { - final DomHTMLImageElement imgElement = createDomHTMLImageElement(); - imgElement.src = src; - setJsProperty(imgElement, 'decoding', 'async'); + imgElement = createDomHTMLImageElement(); + imgElement!.src = src; + setJsProperty(imgElement!, 'decoding', 'async'); - // Ignoring the returned future on purpose because we're communicating - // through the `completer`. - // ignore: unawaited_futures - imgElement.decode().then((dynamic _) { - chunkCallback?.call(100, 100); - int naturalWidth = imgElement.naturalWidth.toInt(); - int naturalHeight = imgElement.naturalHeight.toInt(); - // Workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=700533. - if (naturalWidth == 0 && - naturalHeight == 0 && - ui_web.browser.browserEngine == ui_web.BrowserEngine.firefox) { - const int kDefaultImageSizeFallback = 300; - naturalWidth = kDefaultImageSizeFallback; - naturalHeight = kDefaultImageSizeFallback; - } - final ui.Image image = createImageFromHTMLImageElement( - imgElement, - naturalWidth, - naturalHeight, - ); - completer.complete(SingleFrameInfo(image)); - }).catchError((dynamic e) { - // This code path is hit on Chrome 80.0.3987.16 when too many - // images are on the page (~1000). - // Fallback here is to load using onLoad instead. - _decodeUsingOnLoad(completer); - }); - } else { - _decodeUsingOnLoad(completer); - } + // Ignoring the returned future on purpose because we're communicating + // through the `completer`. + // ignore: unawaited_futures + imgElement!.decode().then((dynamic _) { + chunkCallback?.call(100, 100); + completer.complete(); + }).catchError((dynamic e) { + completer.completeError(e.toString()); + }); return completer.future; } - void _decodeUsingOnLoad(Completer completer) { - final DomHTMLImageElement imgElement = createDomHTMLImageElement(); - // If the browser doesn't support asynchronous decoding of an image, - // then use the `onload` event to decide when it's ready to paint to the - // DOM. Unfortunately, this will cause the image to be decoded synchronously - // on the main thread, and may cause dropped framed. - late DomEventListener errorListener; - DomEventListener? loadListener; - errorListener = createDomEventListener((DomEvent event) { - if (loadListener != null) { - imgElement.removeEventListener('load', loadListener); - } - imgElement.removeEventListener('error', errorListener); - completer.completeError(event); - }); - imgElement.addEventListener('error', errorListener); - loadListener = createDomEventListener((DomEvent event) { - if (chunkCallback != null) { - chunkCallback!(100, 100); - } - imgElement.removeEventListener('load', loadListener); - imgElement.removeEventListener('error', errorListener); - final ui.Image image = createImageFromHTMLImageElement( - imgElement, - imgElement.naturalWidth.toInt(), - imgElement.naturalHeight.toInt(), - ); - completer.complete(SingleFrameInfo(image)); - }); - imgElement.addEventListener('load', loadListener); - imgElement.src = src; + @override + Future getNextFrame() async { + await decode(); + int naturalWidth = imgElement!.naturalWidth.toInt(); + int naturalHeight = imgElement!.naturalHeight.toInt(); + // Workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=700533. + if (naturalWidth == 0 && + naturalHeight == 0 && + ui_web.browser.browserEngine == ui_web.BrowserEngine.firefox) { + const int kDefaultImageSizeFallback = 300; + naturalWidth = kDefaultImageSizeFallback; + naturalHeight = kDefaultImageSizeFallback; + } + final ui.Image image = createImageFromHTMLImageElement( + imgElement!, + naturalWidth, + naturalHeight, + ); + return SingleFrameInfo(image); } /// Creates a [ui.Image] from an [HTMLImageElement] that has been loaded. @@ -125,7 +91,11 @@ abstract class HtmlImageElementCodec implements ui.Codec { } abstract class HtmlBlobCodec extends HtmlImageElementCodec { - HtmlBlobCodec(this.blob) : super(domWindow.URL.createObjectURL(blob)); + HtmlBlobCodec(this.blob) + : super( + domWindow.URL.createObjectURL(blob), + debugSource: 'encoded image bytes', + ); final DomBlob blob; diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/image_decoder.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/image_decoder.dart index 94d564952dd..ca321456731 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/image_decoder.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/image_decoder.dart @@ -381,18 +381,33 @@ class ResizingCodec implements ui.Codec { final ui.FrameInfo frameInfo = await delegate.getNextFrame(); return AnimatedImageFrameInfo( frameInfo.duration, - scaleImageIfNeeded(frameInfo.image, - targetWidth: targetWidth, - targetHeight: targetHeight, - allowUpscaling: allowUpscaling), + scaleImage( + frameInfo.image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ), ); } + ui.Image scaleImage( + ui.Image image, { + int? targetWidth, + int? targetHeight, + bool allowUpscaling = true, + }) => + scaleImageIfNeeded( + image, + targetWidth: targetWidth, + targetHeight: targetHeight, + allowUpscaling: allowUpscaling, + ); + @override int get repetitionCount => delegate.frameCount; } -ui.Size? _scaledSize( +BitmapSize? scaledImageSize( int width, int height, int? targetWidth, @@ -415,7 +430,7 @@ ui.Size? _scaledSize( } targetHeight = (height * targetWidth / width).round(); } - return ui.Size(targetWidth.toDouble(), targetHeight.toDouble()); + return BitmapSize(targetWidth, targetHeight); } ui.Image scaleImageIfNeeded( @@ -426,8 +441,8 @@ ui.Image scaleImageIfNeeded( }) { final int width = image.width; final int height = image.height; - final ui.Size? scaledSize = - _scaledSize(width, height, targetWidth, targetHeight); + final BitmapSize? scaledSize = + scaledImageSize(width, height, targetWidth, targetHeight); if (scaledSize == null) { return image; } @@ -436,8 +451,8 @@ ui.Image scaleImageIfNeeded( return image; } - final ui.Rect outputRect = - ui.Rect.fromLTWH(0, 0, scaledSize.width, scaledSize.height); + final ui.Rect outputRect = ui.Rect.fromLTWH( + 0, 0, scaledSize.width.toDouble(), scaledSize.height.toDouble()); final ui.PictureRecorder recorder = ui.PictureRecorder(); final ui.Canvas canvas = ui.Canvas(recorder, outputRect); @@ -449,7 +464,7 @@ ui.Image scaleImageIfNeeded( ); final ui.Picture picture = recorder.endRecording(); final ui.Image finalImage = - picture.toImageSync(scaledSize.width.round(), scaledSize.height.round()); + picture.toImageSync(scaledSize.width, scaledSize.height); picture.dispose(); image.dispose(); return finalImage; diff --git a/engine/src/flutter/lib/web_ui/test/canvaskit/embedded_views_test.dart b/engine/src/flutter/lib/web_ui/test/canvaskit/embedded_views_test.dart index d95f3082224..0301f2eb25f 100644 --- a/engine/src/flutter/lib/web_ui/test/canvaskit/embedded_views_test.dart +++ b/engine/src/flutter/lib/web_ui/test/canvaskit/embedded_views_test.dart @@ -597,6 +597,7 @@ void testMain() { await createPlatformView(0, 'test-platform-view'); final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( + contentType: 'image/gif', data: kAnimatedGif, debugSource: 'test', ); diff --git a/engine/src/flutter/lib/web_ui/test/canvaskit/image_golden_test.dart b/engine/src/flutter/lib/web_ui/test/canvaskit/image_golden_test.dart index 6d3ee7e2988..94d169ae7af 100644 --- a/engine/src/flutter/lib/web_ui/test/canvaskit/image_golden_test.dart +++ b/engine/src/flutter/lib/web_ui/test/canvaskit/image_golden_test.dart @@ -3,23 +3,199 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:js_interop'; import 'dart:typed_data'; import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; -import 'package:ui/ui_web/src/ui_web.dart' as ui_web; -import '../common/matchers.dart'; import 'common.dart'; import 'test_data.dart'; +List? testCodecs; + void main() { internalBootstrapBrowserTest(() => testMain); } -void testMain() { +abstract class TestCodec { + TestCodec({required this.description}); + final String description; + + ui.Codec? _cachedCodec; + + Future getCodec() async => _cachedCodec ??= await createCodec(); + + Future createCodec(); +} + +abstract class TestFileCodec extends TestCodec { + TestFileCodec.fromTestFile(this.testFile, {required super.description}); + + final String testFile; + + Future createCodecFromTestFile(String testFile); + + @override + Future createCodec() { + return createCodecFromTestFile(testFile); + } +} + +class UrlTestCodec extends TestFileCodec { + UrlTestCodec(super.testFile, this.codecFactory, String function) + : super.fromTestFile(description: 'created with $function("$testFile")'); + + final Future Function(String) codecFactory; + + @override + Future createCodecFromTestFile(String testFile) { + return codecFactory(testFile); + } +} + +class FetchTestCodec extends TestFileCodec { + FetchTestCodec( + super.testFile, + this.codecFactory, + String function, + ) : super.fromTestFile( + description: 'created with $function from bytes ' + 'fetch()\'ed from "$testFile"'); + + final Future Function(Uint8List) codecFactory; + + @override + Future createCodecFromTestFile(String testFile) async { + final HttpFetchResponse response = await httpFetch(testFile); + + if (!response.hasPayload) { + throw Exception('Unable to fetch() image test file "$testFile"'); + } + + final Uint8List responseBytes = await response.asUint8List(); + return codecFactory(responseBytes); + } +} + +class BitmapTestCodec extends TestFileCodec { + BitmapTestCodec( + super.testFile, + this.codecFactory, + String function, + ) : super.fromTestFile( + description: 'created with $function from ImageBitmap' + ' created from "$testFile"'); + + final Future Function(DomImageBitmap) codecFactory; + + @override + Future createCodecFromTestFile(String testFile) async { + final DomHTMLImageElement imageElement = createDomHTMLImageElement(); + imageElement.src = testFile; + setJsProperty(imageElement, 'decoding', 'async'); + + await imageElement.decode(); + + final DomImageBitmap bitmap = + await createImageBitmap(imageElement as JSObject, ( + x: 0, + y: 0, + width: imageElement.naturalWidth.toInt(), + height: imageElement.naturalHeight.toInt(), + )); + + final ui.Image image = await codecFactory(bitmap); + return BitmapSingleFrameCodec(bitmap, image); + } +} + +class BitmapSingleFrameCodec implements ui.Codec { + BitmapSingleFrameCodec(this.bitmap, this.image); + + final DomImageBitmap bitmap; + final ui.Image image; + + @override + void dispose() { + image.dispose(); + bitmap.close(); + } + + @override + int get frameCount => 1; + + @override + Future getNextFrame() async { + return SingleFrameInfo(image); + } + + @override + int get repetitionCount => 0; +} + +Future testMain() async { + Future> createTestCodecs( + {int testTargetWidth = 300, int testTargetHeight = 300}) async { + final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); + final List testFiles = + (await listingResponse.json() as List).cast(); + + // Sanity-check the test file list. If suddenly test files are moved or + // deleted, and the test server returns an empty list, or is missing some + // important test files, we want to know. + assert(testFiles.isNotEmpty); + assert(testFiles.any((String testFile) => testFile.endsWith('.jpg'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.png'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.gif'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.webp'))); + assert(testFiles.any((String testFile) => testFile.endsWith('.bmp'))); + + final List testCodecs = []; + for (final String testFile in testFiles) { + testCodecs.add(UrlTestCodec( + testFile, + (String file) => renderer.instantiateImageCodecFromUrl( + Uri.tryParse('/test_images/$file')!, + ), + 'renderer.instantiateImageFromUrl', + )); + testCodecs.add( + FetchTestCodec( + '/test_images/$testFile', + (Uint8List bytes) => renderer.instantiateImageCodec(bytes), + 'renderer.instantiateImageCodec', + ), + ); + testCodecs.add( + FetchTestCodec( + '/test_images/$testFile', + (Uint8List bytes) => renderer.instantiateImageCodec( + bytes, + targetWidth: testTargetWidth, + targetHeight: testTargetHeight, + ), + 'renderer.instantiateImageCodec ' + '($testTargetWidth x $testTargetHeight)', + ), + ); + testCodecs.add( + BitmapTestCodec( + 'test_images/$testFile', + (DomImageBitmap bitmap) async => + renderer.createImageFromImageBitmap(bitmap), + 'renderer.createImageFromImageBitmap', + ), + ); + } + + return testCodecs; + } + + testCodecs = await createTestCodecs(); + group('CanvasKit Images', () { setUpCanvasKitTest(withImplicitView: true); @@ -27,28 +203,100 @@ void testMain() { mockHttpFetchResponseFactory = null; }); - _testCkAnimatedImage(); - _testForImageCodecs(useBrowserImageDecoder: false); + group('Codecs', () { + for (final TestCodec testCodec in testCodecs!) { + test('${testCodec.description} can create an image', () async { + try { + final ui.Codec codec = await testCodec.getCodec(); + final ui.FrameInfo frameInfo = await codec.getNextFrame(); + final ui.Image image = frameInfo.image; + expect(image, isNotNull); + expect(image.width, isNonZero); + expect(image.height, isNonZero); + expect(image.colorSpace, isNotNull); + } catch (e) { + throw TestFailure( + 'Failed to get image for ${testCodec.description}: $e'); + } + }); - if (browserSupportsImageDecoder) { - _testForImageCodecs(useBrowserImageDecoder: true); - _testCkBrowserImageDecoder(); - } + test('${testCodec.description} can be decoded with toByteData', + () async { + ui.Image image; + try { + final ui.Codec codec = await testCodec.getCodec(); + final ui.FrameInfo frameInfo = await codec.getNextFrame(); + image = frameInfo.image; + } catch (e) { + throw TestFailure( + 'Failed to get image for ${testCodec.description}: $e'); + } + + final ByteData? byteData = await image.toByteData(); + expect( + byteData, + isNotNull, + reason: '${testCodec.description} toByteData() should not be null', + ); + expect( + byteData!.lengthInBytes, + isNonZero, + reason: '${testCodec.description} toByteData() should not be empty', + ); + expect( + byteData.buffer.asUint8List().any((int byte) => byte > 0), + isTrue, + reason: '${testCodec.description} toByteData() should ' + 'contain nonzero value', + ); + }); + } + }); + + _testCkAnimatedImage(); test('isAvif', () { expect(isAvif(Uint8List.fromList([])), isFalse); expect(isAvif(Uint8List.fromList([1, 2, 3])), isFalse); expect( isAvif(Uint8List.fromList([ - 0x00, 0x00, 0x00, 0x1c, 0x66, 0x74, 0x79, 0x70, - 0x61, 0x76, 0x69, 0x66, 0x00, 0x00, 0x00, 0x00, + 0x00, + 0x00, + 0x00, + 0x1c, + 0x66, + 0x74, + 0x79, + 0x70, + 0x61, + 0x76, + 0x69, + 0x66, + 0x00, + 0x00, + 0x00, + 0x00, ])), isTrue, ); expect( isAvif(Uint8List.fromList([ - 0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, - 0x61, 0x76, 0x69, 0x66, 0x00, 0x00, 0x00, 0x00, + 0x00, + 0x00, + 0x00, + 0x20, + 0x66, + 0x74, + 0x79, + 0x70, + 0x61, + 0x76, + 0x69, + 0x66, + 0x00, + 0x00, + 0x00, + 0x00, ])), isTrue, ); @@ -56,743 +304,14 @@ void testMain() { }, skip: isSafari); } -void _testForImageCodecs({required bool useBrowserImageDecoder}) { - final String mode = useBrowserImageDecoder ? 'webcodecs' : 'wasm'; - final List warnings = []; - late void Function(String) oldPrintWarning; - - group('($mode)', () { - setUp(() { - browserSupportsImageDecoder = useBrowserImageDecoder; - warnings.clear(); - }); - - setUpAll(() { - oldPrintWarning = printWarning; - printWarning = (String warning) { - warnings.add(warning); - }; - }); - - tearDown(() { - debugResetBrowserSupportsImageDecoder(); - }); - - tearDownAll(() { - printWarning = oldPrintWarning; - }); - - test('CkAnimatedImage can be explicitly disposed of', () { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kTransparentImage, 'test'); - expect(image.debugDisposed, isFalse); - image.dispose(); - expect(image.debugDisposed, isTrue); - - // Disallow usage after disposal - expect(() => image.frameCount, throwsAssertionError); - expect(() => image.repetitionCount, throwsAssertionError); - expect(() => image.getNextFrame(), throwsAssertionError); - - // Disallow double-dispose. - expect(() => image.dispose(), throwsAssertionError); - }); - - test('CkAnimatedImage iterates frames correctly', () async { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); - expect(image.frameCount, 3); - expect(image.repetitionCount, -1); - - final ui.FrameInfo frame1 = await image.getNextFrame(); - await expectFrameData(frame1, [255, 0, 0, 255]); - final ui.FrameInfo frame2 = await image.getNextFrame(); - await expectFrameData(frame2, [0, 255, 0, 255]); - final ui.FrameInfo frame3 = await image.getNextFrame(); - await expectFrameData(frame3, [0, 0, 255, 255]); - }); - - test('CkImage toString', () { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - expect(image.toString(), '[1×1]'); - image.dispose(); - }); - - test('CkImage can be explicitly disposed of', () { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - expect(image.debugDisposed, isFalse); - expect(image.box.isDisposed, isFalse); - image.dispose(); - expect(image.debugDisposed, isTrue); - expect(image.box.isDisposed, isTrue); - - // Disallow double-dispose. - expect(() => image.dispose(), throwsAssertionError); - }); - - test('CkImage can be explicitly disposed of when cloned', () async { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - final CountedRef box = image.box; - expect(box.refCount, 1); - expect(box.debugGetStackTraces().length, 1); - - final CkImage clone = image.clone(); - expect(box.refCount, 2); - expect(box.debugGetStackTraces().length, 2); - - expect(image.isCloneOf(clone), isTrue); - expect(box.isDisposed, isFalse); - - expect(skImage.isDeleted(), isFalse); - image.dispose(); - expect(box.refCount, 1); - expect(box.isDisposed, isFalse); - - expect(skImage.isDeleted(), isFalse); - clone.dispose(); - expect(box.refCount, 0); - expect(box.isDisposed, isTrue); - - expect(skImage.isDeleted(), isTrue); - expect(box.debugGetStackTraces().length, 0); - }); - - test('CkImage toByteData', () async { - final SkImage skImage = - canvasKit.MakeAnimatedImageFromEncoded(kTransparentImage)! - .makeImageAtCurrentFrame(); - final CkImage image = CkImage(skImage); - expect((await image.toByteData()).lengthInBytes, greaterThan(0)); - expect((await image.toByteData(format: ui.ImageByteFormat.png)).lengthInBytes, greaterThan(0)); - }); - - test('toByteData with decodeImageFromPixels on videoFrame formats', () async { - // This test ensures that toByteData() returns pixels that can be used by decodeImageFromPixels - // for the following videoFrame formats: - // [BGRX, I422, I420, I444, BGRA] - final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); - final List testFiles = (await listingResponse.json() as List).cast(); - - Future testDecodeFromPixels(Uint8List pixels, int width, int height) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - pixels, - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - ); - return completer.future; - } - - // Sanity-check the test file list. If suddenly test files are moved or - // deleted, and the test server returns an empty list, or is missing some - // important test files, we want to know. - expect(testFiles, isNotEmpty); - expect(testFiles, contains(matches(RegExp(r'.*\.jpg')))); - expect(testFiles, contains(matches(RegExp(r'.*\.png')))); - expect(testFiles, contains(matches(RegExp(r'.*\.gif')))); - expect(testFiles, contains(matches(RegExp(r'.*\.webp')))); - expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); - - for (final String testFile in testFiles) { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/$testFile'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - expect(codec.frameCount, greaterThan(0)); - expect(codec.repetitionCount, isNotNull); - - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage ckImage = frame.image as CkImage; - final ByteData imageBytes = await ckImage.toByteData(); - expect(imageBytes.lengthInBytes, greaterThan(0)); - - final Uint8List pixels = imageBytes.buffer.asUint8List(); - final ui.Image testImage = await testDecodeFromPixels(pixels, ckImage.width, ckImage.height); - expect(testImage, isNotNull); - codec.dispose(); - } - // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder. - // TODO(jacksongardner): enable on wasm - // see https://github.com/flutter/flutter/issues/118334 - }, skip: isFirefox || isSafari || isWasm); - - test('CkImage.clone also clones the VideoFrame', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - data: kAnimatedGif, - debugSource: 'test', - ); - final ui.FrameInfo frame = await image.getNextFrame(); - final CkImage ckImage = frame.image as CkImage; - expect(ckImage.videoFrame, isNotNull); - - final CkImage imageClone = ckImage.clone(); - expect(imageClone.videoFrame, isNotNull); - - final ByteData png = await imageClone.toByteData(format: ui.ImageByteFormat.png); - expect(png, isNotNull); - - // The precise PNG encoding is browser-specific, but we can check the file - // signature. - expect(detectContentType(png.buffer.asUint8List()), 'image/png'); - // TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder. - }, skip: isFirefox || isSafari); - - test('skiaInstantiateWebImageCodec loads an image from the network', - () async { - mockHttpFetchResponseFactory = (String url) async { - return MockHttpFetchResponse( - url: url, - status: 200, - payload: MockHttpFetchPayload(byteBuffer: kTransparentImage.buffer), - ); - }; - - final ui.Codec codec = await skiaInstantiateWebImageCodec( - 'http://image-server.com/picture.jpg', null); - expect(codec.frameCount, 1); - final ui.Image image = (await codec.getNextFrame()).image; - expect(image.height, 1); - expect(image.width, 1); - }); - - test('instantiateImageCodec respects target image size', - () async { - const List> targetSizes = >[ - [1, 1], - [1, 2], - [2, 3], - [3, 4], - [4, 4], - [10, 20], - ]; - - for (final List targetSize in targetSizes) { - final int targetWidth = targetSize[0]; - final int targetHeight = targetSize[1]; - - final ui.Codec codec = await ui.instantiateImageCodec( - k4x4PngImage, - targetWidth: targetWidth, - targetHeight: targetHeight, - ); - - final ui.Image image = (await codec.getNextFrame()).image; - expect(image.width, targetWidth); - expect(image.height, targetHeight); - image.dispose(); - codec.dispose(); - } - }); - - test('instantiateImageCodec with multi-frame image does not support targetWidth/targetHeight', - () async { - final ui.Codec codec = await ui.instantiateImageCodec( - kAnimatedGif, - targetWidth: 2, - targetHeight: 3, - ); - final ui.Image image = (await codec.getNextFrame()).image; - - expect( - warnings, - containsAllInOrder( - [ - 'targetWidth and targetHeight for multi-frame images not supported', - ], - ), - ); - - // expect the re-size did not happen, kAnimatedGif is [1x1] - expect(image.width, 1); - expect(image.height, 1); - image.dispose(); - codec.dispose(); - }); - - test('skiaInstantiateWebImageCodec throws exception on request error', - () async { - mockHttpFetchResponseFactory = (String url) async { - throw HttpFetchError(url, requestError: 'This is a test request error.'); - }; - - try { - await skiaInstantiateWebImageCodec('url-does-not-matter', null); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to load network image.\n' - 'Image URL: url-does-not-matter\n' - 'Trying to load an image from another domain? Find answers at:\n' - 'https://flutter.dev/docs/development/platform-integration/web-images', - ); - } - }); - - test('skiaInstantiateWebImageCodec throws exception on HTTP error', - () async { - try { - await skiaInstantiateWebImageCodec('/does-not-exist.jpg', null); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - expect( - exception.toString(), - 'ImageCodecException: Failed to load network image.\n' - 'Image URL: /does-not-exist.jpg\n' - 'Server response code: 404', - ); - } - }); - - test('skiaInstantiateWebImageCodec includes URL in the error for malformed image', - () async { - mockHttpFetchResponseFactory = (String url) async { - return MockHttpFetchResponse( - url: url, - status: 200, - payload: MockHttpFetchPayload(byteBuffer: Uint8List(0).buffer), - ); - }; - - try { - await skiaInstantiateWebImageCodec('http://image-server.com/picture.jpg', null); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: http://image-server.com/picture.jpg', - ); - } else { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was empty.\n' - 'Image source: http://image-server.com/picture.jpg', - ); - } - } - }); - - test('Reports error when failing to decode empty image data', () async { - try { - await ui.instantiateImageCodec(Uint8List(0)); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes', - ); - } else { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was empty.\n' - 'Image source: encoded image bytes', - ); - } - } - }); - - test('Reports error when failing to decode malformed image data', () async { - try { - await ui.instantiateImageCodec(Uint8List.fromList([ - 0xFF, 0xD8, 0xFF, 0xDB, 0x00, 0x00, 0x00, - ])); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes' - ); - } else { - expect( - exception.toString(), - // Browser error message is not checked as it can depend on the - // browser engine and version. - matches(RegExp( - r"ImageCodecException: Failed to decode image using the browser's ImageDecoder API.\n" - r'Image source: encoded image bytes\n' - r'Original browser error: .+' - )) - ); - } - } - }); - - test('Includes file header in the error message when fails to detect file type', () async { - try { - await ui.instantiateImageCodec(Uint8List.fromList([ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x00, - ])); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes' - ); - } else { - expect( - exception.toString(), - 'ImageCodecException: Failed to detect image file format using the file header.\n' - 'File header was [0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x00].\n' - 'Image source: encoded image bytes' - ); - } - } - }); - - test('Provides readable error message when image type is unsupported', () async { - addTearDown(() { - debugContentTypeDetector = null; - }); - debugContentTypeDetector = (_) { - return 'unsupported/image-type'; - }; - try { - await ui.instantiateImageCodec(Uint8List.fromList([ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x00, - ])); - fail('Expected to throw'); - } on ImageCodecException catch (exception) { - if (!browserSupportsImageDecoder) { - expect( - exception.toString(), - 'ImageCodecException: Failed to decode image data.\n' - 'Image source: encoded image bytes' - ); - } else { - expect( - exception.toString(), - "ImageCodecException: Image file format (unsupported/image-type) is not supported by this browser's ImageDecoder API.\n" - 'Image source: encoded image bytes' - ); - } - } - }); - - test('decodeImageFromPixels', () async { - Future testDecodeFromPixels(int width, int height) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - ); - return completer.future; - } - - final ui.Image image1 = await testDecodeFromPixels(10, 20); - expect(image1, isNotNull); - expect(image1.width, 10); - expect(image1.height, 20); - - final ui.Image image2 = await testDecodeFromPixels(40, 100); - expect(image2, isNotNull); - expect(image2.width, 40); - expect(image2.height, 100); - }); - - test('decodeImageFromPixels respects target image size', () async { - Future testDecodeFromPixels(int width, int height, int targetWidth, int targetHeight) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - targetWidth: targetWidth, - targetHeight: targetHeight, - ); - return completer.future; - } - - const List> targetSizes = >[ - [1, 1], - [1, 2], - [2, 3], - [3, 4], - [4, 4], - [10, 20], - ]; - - for (final List targetSize in targetSizes) { - final int targetWidth = targetSize[0]; - final int targetHeight = targetSize[1]; - - final ui.Image image = await testDecodeFromPixels(10, 20, targetWidth, targetHeight); - - expect(image.width, targetWidth); - expect(image.height, targetHeight); - image.dispose(); - } - }); - - test('decodeImageFromPixels upscale when allowUpscaling is false', () async { - Future testDecodeFromPixels(int width, int height) async { - final Completer completer = Completer(); - ui.decodeImageFromPixels( - Uint8List.fromList(List.filled(width * height * 4, 0)), - width, - height, - ui.PixelFormat.rgba8888, - (ui.Image image) { - completer.complete(image); - }, - targetWidth: 20, - targetHeight: 30, - allowUpscaling: false - ); - return completer.future; - } - expect(() async => testDecodeFromPixels(10, 20), throwsAssertionError); - }); - - test('Decode test images', () async { - final HttpFetchResponse listingResponse = await httpFetch('/test_images/'); - final List testFiles = (await listingResponse.json() as List).cast(); - - // Sanity-check the test file list. If suddenly test files are moved or - // deleted, and the test server returns an empty list, or is missing some - // important test files, we want to know. - expect(testFiles, isNotEmpty); - expect(testFiles, contains(matches(RegExp(r'.*\.jpg')))); - expect(testFiles, contains(matches(RegExp(r'.*\.png')))); - expect(testFiles, contains(matches(RegExp(r'.*\.gif')))); - expect(testFiles, contains(matches(RegExp(r'.*\.webp')))); - expect(testFiles, contains(matches(RegExp(r'.*\.bmp')))); - - for (final String testFile in testFiles) { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/$testFile'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - expect(codec.frameCount, greaterThan(0)); - expect(codec.repetitionCount, isNotNull); - for (int i = 0; i < codec.frameCount; i++) { - final ui.FrameInfo frame = await codec.getNextFrame(); - expect(frame.duration, isNotNull); - expect(frame.image, isNotNull); - } - codec.dispose(); - } - }); - - // Reproduces https://skbug.com/12721 - test('decoded image can be read back from picture', () async { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/mandrill_128.png'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage image = frame.image as CkImage; - - final CkImage snapshot; - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(10, 10); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawRect( - const ui.Rect.fromLTRB(5, 5, 20, 20), - CkPaint(), - ); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.drawRect( - const ui.Rect.fromLTRB(90, 90, 105, 105), - CkPaint(), - ); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - sb.pop(); - snapshot = await sb.build().toImage(150, 150) as CkImage; - } - - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawImage(snapshot, ui.Offset.zero, CkPaint()); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - - await matchSceneGolden( - 'canvaskit_read_back_decoded_image_$mode.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 150, 150)); - } - - image.dispose(); - codec.dispose(); - }); - - // This is a regression test for the issues with transferring textures from - // one GL context to another, such as: - // - // * https://github.com/flutter/flutter/issues/86809 - // * https://github.com/flutter/flutter/issues/91881 - test('the same image can be rendered on difference surfaces', () async { - ui_web.platformViewRegistry.registerViewFactory( - 'test-platform-view', - (int viewId) => createDomHTMLDivElement()..id = 'view-0', - ); - await createPlatformView(0, 'test-platform-view'); - - final ui.Codec codec = await ui.instantiateImageCodec(k4x4PngImage); - final CkImage image = (await codec.getNextFrame()).image as CkImage; - - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(4, 4); - { - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.save(); - canvas.scale(16, 16); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.restore(); - canvas.drawParagraph(makeSimpleText('1'), const ui.Offset(4, 4)); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - } - sb.addPlatformView(0, width: 100, height: 100); - sb.pushOffset(20, 20); - { - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.save(); - canvas.scale(16, 16); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.restore(); - canvas.drawParagraph(makeSimpleText('2'), const ui.Offset(2, 2)); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - } - - await matchSceneGolden( - 'canvaskit_cross_gl_context_image_$mode.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 100, 100)); - - await disposePlatformView(0); - }); - - test('toImageSync with texture-backed image', () async { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/mandrill_128.png'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage mandrill = frame.image as CkImage; - final ui.PictureRecorder recorder = ui.PictureRecorder(); - final ui.Canvas canvas = ui.Canvas(recorder); - canvas.drawImageRect( - mandrill, - const ui.Rect.fromLTWH(0, 0, 128, 128), - const ui.Rect.fromLTWH(0, 0, 128, 128), - ui.Paint(), - ); - final ui.Picture picture = recorder.endRecording(); - final ui.Image image = picture.toImageSync(50, 50); - - expect(image.width, 50); - expect(image.height, 50); - - final ByteData? data = await image.toByteData(); - expect(data, isNotNull); - expect(data!.lengthInBytes, 50 * 50 * 4); - expect(data.buffer.asUint32List().any((int byte) => byte != 0), isTrue); - - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(0, 0); - { - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.save(); - canvas.drawImage(image as CkImage, ui.Offset.zero, CkPaint()); - canvas.restore(); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - } - - await matchSceneGolden( - 'canvaskit_picture_texture_toimage.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 128, 128)); - mandrill.dispose(); - codec.dispose(); - }); - - test('decoded image can be read back from picture', () async { - final HttpFetchResponse imageResponse = await httpFetch('/test_images/mandrill_128.png'); - final Uint8List imageData = await imageResponse.asUint8List(); - final ui.Codec codec = await skiaInstantiateImageCodec(imageData); - final ui.FrameInfo frame = await codec.getNextFrame(); - final CkImage image = frame.image as CkImage; - - final CkImage snapshot; - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - sb.pushOffset(10, 10); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawRect( - const ui.Rect.fromLTRB(5, 5, 20, 20), - CkPaint(), - ); - canvas.drawImage(image, ui.Offset.zero, CkPaint()); - canvas.drawRect( - const ui.Rect.fromLTRB(90, 90, 105, 105), - CkPaint(), - ); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - sb.pop(); - snapshot = await sb.build().toImage(150, 150) as CkImage; - } - - { - final LayerSceneBuilder sb = LayerSceneBuilder(); - final CkPictureRecorder recorder = CkPictureRecorder(); - final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest); - canvas.drawImage(snapshot, ui.Offset.zero, CkPaint()); - sb.addPicture(ui.Offset.zero, recorder.endRecording()); - - await matchSceneGolden( - 'canvaskit_read_back_decoded_image_$mode.png', sb.build(), - region: const ui.Rect.fromLTRB(0, 0, 150, 150)); - } - - image.dispose(); - codec.dispose(); - }); - - test('can detect JPEG from just magic number', () async { - expect( - detectContentType( - Uint8List.fromList([0xff, 0xd8, 0xff, 0xe2, 0x0c, 0x58, 0x49, 0x43, 0x43, 0x5f])), - 'image/jpeg'); - }); - }, timeout: const Timeout.factor(10)); // These tests can take a while. Allow for a longer timeout. -} - /// Tests specific to WASM codecs bundled with CanvasKit. void _testCkAnimatedImage() { test('ImageDecoder toByteData(PNG)', () async { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + final CkAnimatedImage image = + CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? png = await frame.image.toByteData(format: ui.ImageByteFormat.png); + final ByteData? png = + await frame.image.toByteData(format: ui.ImageByteFormat.png); expect(png, isNotNull); // The precise PNG encoding is browser-specific, but we can check the file @@ -801,7 +320,8 @@ void _testCkAnimatedImage() { }); test('CkAnimatedImage toByteData(RGBA)', () async { - final CkAnimatedImage image = CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); + final CkAnimatedImage image = + CkAnimatedImage.decodeFromBytes(kAnimatedGif, 'test'); const List> expectedColors = >[ [255, 0, 0, 255], [0, 255, 0, 255], @@ -815,107 +335,3 @@ void _testCkAnimatedImage() { } }); } - -/// Tests specific to browser image codecs based functionality. -void _testCkBrowserImageDecoder() { - assert(browserSupportsImageDecoder); - - test('ImageDecoder toByteData(PNG)', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - data: kAnimatedGif, - debugSource: 'test', - ); - final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? png = await frame.image.toByteData(format: ui.ImageByteFormat.png); - expect(png, isNotNull); - - // The precise PNG encoding is browser-specific, but we can check the file - // signature. - expect(detectContentType(png!.buffer.asUint8List()), 'image/png'); - }); - - test('ImageDecoder toByteData(RGBA)', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - data: kAnimatedGif, - debugSource: 'test', - ); - const List> expectedColors = >[ - [255, 0, 0, 255], - [0, 255, 0, 255], - [0, 0, 255, 255], - ]; - for (int i = 0; i < image.frameCount; i++) { - final ui.FrameInfo frame = await image.getNextFrame(); - final ByteData? rgba = await frame.image.toByteData(); - expect(rgba, isNotNull); - expect(rgba!.buffer.asUint8List(), expectedColors[i]); - } - }); - - test('ImageDecoder expires after inactivity', () async { - const Duration testExpireDuration = Duration(milliseconds: 100); - debugOverrideWebDecoderExpireDuration(testExpireDuration); - - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - data: kAnimatedGif, - debugSource: 'test', - ); - - // ImageDecoder is initialized eagerly to populate `frameCount` and - // `repetitionCount`. - final ImageDecoder? decoder1 = image.debugCachedWebDecoder; - expect(decoder1, isNotNull); - expect(image.frameCount, 3); - expect(image.repetitionCount, -1); - - // A frame can be decoded right away. - final ui.FrameInfo frame1 = await image.getNextFrame(); - await expectFrameData(frame1, [255, 0, 0, 255]); - expect(frame1, isNotNull); - - // The cached decoder should not yet expire. - await Future.delayed(testExpireDuration ~/ 2); - expect(image.debugCachedWebDecoder, same(decoder1)); - - // Now it expires. - await Future.delayed(testExpireDuration); - expect(image.debugCachedWebDecoder, isNull); - - // A new decoder should be created upon the next frame request. - final ui.FrameInfo frame2 = await image.getNextFrame(); - - // Check that the cached decoder is indeed new. - final ImageDecoder? decoder2 = image.debugCachedWebDecoder; - expect(decoder2, isNot(same(decoder1))); - await expectFrameData(frame2, [0, 255, 0, 255]); - - // Check that the new decoder remembers the last frame index. - final ui.FrameInfo frame3 = await image.getNextFrame(); - await expectFrameData(frame3, [0, 0, 255, 255]); - - debugRestoreWebDecoderExpireDuration(); - }); - - test('ImageDecoder toByteData(translucent PNG)', () async { - final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create( - data: kTranslucentPng, - debugSource: 'test', - ); - final ui.FrameInfo frame = await image.getNextFrame(); - - ByteData? data = await frame.image.toByteData(format: ui.ImageByteFormat.rawStraightRgba); - expect(data!.buffer.asUint8List(), - [0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80, - 0x22, 0x44, 0x66, 0x80, 0x22, 0x44, 0x66, 0x80]); - - data = await frame.image.toByteData(); - expect(data!.buffer.asUint8List(), - [0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80, - 0x11, 0x22, 0x33, 0x80, 0x11, 0x22, 0x33, 0x80]); - }); -} - -Future expectFrameData(ui.FrameInfo frame, List data) async { - final ByteData frameData = (await frame.image.toByteData())!; - expect(frameData.buffer.asUint8List(), Uint8List.fromList(data)); -}