From a442e2c50fe7bb97ebd2db2df1373e3548d9062f Mon Sep 17 00:00:00 2001 From: Harry Terkelsen <1961493+harryterkelsen@users.noreply.github.com> Date: Fri, 19 Dec 2025 16:57:29 -0800 Subject: [PATCH] =?UTF-8?q?Revert=20"[reland]=20Unify=20canvas=20creation?= =?UTF-8?q?=20and=20Surface=20code=20in=20Skwasm=20and=20CanvasK=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit effb2e1ac169524ffc21b72dde571a04d6c0d898. --- .../flutter/lib/web_ui/lib/src/engine.dart | 6 +- .../canvaskit/display_canvas_factory.dart | 131 +++ .../lib/src/engine/canvaskit/image.dart | 12 +- .../canvaskit/multi_surface_rasterizer.dart | 84 ++ .../offscreen_canvas_rasterizer.dart | 59 +- .../lib/src/engine/canvaskit/picture.dart | 20 +- .../lib/src/engine/canvaskit/renderer.dart | 19 +- .../lib/src/engine/canvaskit/surface.dart | 795 +++++++++++------- .../engine/compositing/canvas_provider.dart | 110 --- .../compositing/multi_surface_rasterizer.dart | 94 --- .../src/engine/compositing/rasterizer.dart | 12 +- .../lib/src/engine/compositing/surface.dart | 115 --- .../lib/web_ui/lib/src/engine/dom.dart | 24 +- .../lib/src/engine/layer/layer_painting.dart | 3 - .../lib/src/engine/layer/layer_visitor.dart | 13 - .../lib/web_ui/lib/src/engine/renderer.dart | 8 +- .../lib/src/engine/skwasm/skwasm_impl.dart | 1 + .../src/engine/skwasm/skwasm_impl/codecs.dart | 2 +- .../src/engine/skwasm/skwasm_impl/image.dart | 12 +- .../offscreen_canvas_rasterizer.dart | 83 ++ .../engine/skwasm/skwasm_impl/picture.dart | 8 - .../skwasm/skwasm_impl/raw/raw_surface.dart | 23 +- .../engine/skwasm/skwasm_impl/renderer.dart | 25 +- .../engine/skwasm/skwasm_impl/surface.dart | 197 ++--- .../engine/skwasm/skwasm_stub/renderer.dart | 4 - .../canvaskit/bitmap_less_rendering_test.dart | 62 -- .../lib/web_ui/test/canvaskit/image_test.dart | 2 +- .../web_ui/test/canvaskit/renderer_test.dart | 12 +- .../web_ui/test/canvaskit/surface_test.dart | 350 ++++++-- .../test/common/test_initialization.dart | 1 - .../engine/compositing/rasterizer_test.dart | 12 +- .../lib/web_ui/test/engine/culling_test.dart | 2 +- .../test/ui/surface_context_lost_test.dart | 134 --- .../flutter/skwasm/library_skwasm_support.js | 269 ++---- engine/src/flutter/skwasm/skwasm_support.h | 30 +- engine/src/flutter/skwasm/surface.cc | 338 ++------ engine/src/flutter/skwasm/surface.h | 67 +- engine/src/flutter/skwasm/wrappers.h | 2 +- 38 files changed, 1399 insertions(+), 1742 deletions(-) create mode 100644 engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/display_canvas_factory.dart create mode 100644 engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart rename engine/src/flutter/lib/web_ui/lib/src/engine/{compositing => canvaskit}/offscreen_canvas_rasterizer.dart (57%) delete mode 100644 engine/src/flutter/lib/web_ui/lib/src/engine/compositing/canvas_provider.dart delete mode 100644 engine/src/flutter/lib/web_ui/lib/src/engine/compositing/multi_surface_rasterizer.dart delete mode 100644 engine/src/flutter/lib/web_ui/lib/src/engine/compositing/surface.dart create mode 100644 engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/offscreen_canvas_rasterizer.dart delete mode 100644 engine/src/flutter/lib/web_ui/test/canvaskit/bitmap_less_rendering_test.dart delete mode 100644 engine/src/flutter/lib/web_ui/test/ui/surface_context_lost_test.dart diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine.dart b/engine/src/flutter/lib/web_ui/lib/src/engine.dart index 078698fafa6..d56b9ec0602 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine.dart @@ -28,7 +28,9 @@ export 'engine/canvaskit/image_filter.dart'; export 'engine/canvaskit/image_wasm_codecs.dart'; export 'engine/canvaskit/image_web_codecs.dart'; export 'engine/canvaskit/mask_filter.dart'; +export 'engine/canvaskit/multi_surface_rasterizer.dart'; export 'engine/canvaskit/native_memory.dart'; +export 'engine/canvaskit/offscreen_canvas_rasterizer.dart'; export 'engine/canvaskit/painting.dart'; export 'engine/canvaskit/path.dart'; export 'engine/canvaskit/path_metrics.dart'; @@ -42,14 +44,10 @@ export 'engine/canvaskit/util.dart'; export 'engine/canvaskit/vertices.dart'; export 'engine/clipboard.dart'; export 'engine/color_filter.dart'; -export 'engine/compositing/canvas_provider.dart'; export 'engine/compositing/composition.dart'; export 'engine/compositing/display_canvas_factory.dart'; -export 'engine/compositing/multi_surface_rasterizer.dart'; -export 'engine/compositing/offscreen_canvas_rasterizer.dart'; export 'engine/compositing/rasterizer.dart'; export 'engine/compositing/render_canvas.dart'; -export 'engine/compositing/surface.dart'; export 'engine/configuration.dart'; export 'engine/display.dart'; export 'engine/dom.dart'; diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/display_canvas_factory.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/display_canvas_factory.dart new file mode 100644 index 00000000000..71578adfd71 --- /dev/null +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/display_canvas_factory.dart @@ -0,0 +1,131 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +import 'package:meta/meta.dart'; + +import '../../engine.dart'; + +/// Caches canvases used to display Skia-drawn content. +class DisplayCanvasFactory { + DisplayCanvasFactory({required this.createCanvas}) { + assert(() { + registerHotRestartListener(dispose); + return true; + }()); + } + + /// A function which is passed in as a constructor parameter which is used to + /// create new display canvases. + final T Function() createCanvas; + + /// The base canvas to paint on. This is the default canvas which will be + /// painted to. If there are no platform views, then this canvas will render + /// the entire scene. + late final T baseCanvas = createCanvas()..initialize(); + + /// Canvases created by this factory which are currently in use. + final List _liveCanvases = []; + + /// Canvases created by this factory which are no longer in use. These can be + /// reused. + final List _cache = []; + + /// The number of canvases which have been created by this factory. + int get _canvasCount => _liveCanvases.length + _cache.length + 1; + + /// The number of surfaces created by this factory. Used for testing. + @visibleForTesting + int get debugSurfaceCount => _canvasCount; + + /// Returns the number of cached surfaces. + /// + /// Useful in tests. + int get debugCacheSize => _cache.length; + + /// Gets a display canvas from the cache or creates a new one if there are + /// none in the cache. + T getCanvas() { + if (_cache.isNotEmpty) { + final T canvas = _cache.removeLast(); + _liveCanvases.add(canvas); + return canvas; + } else { + final T canvas = createCanvas(); + canvas.initialize(); + _liveCanvases.add(canvas); + return canvas; + } + } + + /// Releases all surfaces so they can be reused in the next frame. + /// + /// If a released surface is in the DOM, it is not removed. This allows the + /// engine to release the surfaces at the end of the frame so they are ready + /// to be used in the next frame, but still used for painting in the current + /// frame. + void releaseCanvases() { + _cache.addAll(_liveCanvases); + _liveCanvases.clear(); + } + + /// Removes all canvases except the base canvas from the DOM. + /// + /// This is called at the beginning of the frame to prepare for painting into + /// the new canvases. + void removeCanvasesFromDom() { + _cache.forEach(_removeFromDom); + _liveCanvases.forEach(_removeFromDom); + } + + /// Calls [callback] on each canvas created by this factory. + void forEachCanvas(void Function(T canvas) callback) { + callback(baseCanvas); + _cache.forEach(callback); + _liveCanvases.forEach(callback); + } + + // Removes [canvas] from the DOM. + void _removeFromDom(T canvas) { + canvas.hostElement.remove(); + } + + /// Signals that a canvas is no longer being used. It can be reused. + void releaseCanvas(T canvas) { + assert(canvas != baseCanvas, 'Attempting to release the base canvas'); + assert( + _liveCanvases.contains(canvas), + 'Attempting to release a Canvas which ' + 'was not created by this factory', + ); + canvas.hostElement.remove(); + _liveCanvases.remove(canvas); + _cache.add(canvas); + } + + /// Returns [true] if [canvas] is currently being used to paint content. + /// + /// The base canvas always counts as live. + /// + /// If a canvas is not live, then it must be in the cache and ready to be + /// reused. + bool isLive(T canvas) { + if (canvas == baseCanvas || _liveCanvases.contains(canvas)) { + return true; + } + assert(_cache.contains(canvas)); + return false; + } + + /// Dispose all canvases created by this factory. + void dispose() { + for (final T canvas in _cache) { + canvas.dispose(); + } + for (final T canvas in _liveCanvases) { + canvas.dispose(); + } + baseCanvas.dispose(); + _liveCanvases.clear(); + _cache.clear(); + } +} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart index e0de7541620..5b2a6a74e50 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/image.dart @@ -557,15 +557,11 @@ class CkImage implements ui.Image, StackTraceDebugger { ByteData? _readPixelsFromImageViaSurface(ui.ImageByteFormat format) { final Surface surface = CanvasKitRenderer.instance.pictureToImageSurface; - surface.setSize(BitmapSize(width, height)); - final ckSurface = surface as CkSurface; - final SkSurface skiaSurface = ckSurface.skSurface!; - - final ckCanvas = CkCanvas.fromSkCanvas(skiaSurface.getCanvas()); + final CkSurface ckSurface = surface.createOrUpdateSurface(BitmapSize(width, height)); + final CkCanvas ckCanvas = ckSurface.getCanvas(); ckCanvas.clear(const ui.Color(0x00000000)); ckCanvas.drawImage(this, ui.Offset.zero, CkPaint()); - final SkImage skImage = skiaSurface.makeImageSnapshot(); - + final SkImage skImage = ckSurface.surface.makeImageSnapshot(); final imageInfo = SkImageInfo( alphaType: canvasKit.AlphaType.Premul, colorType: canvasKit.ColorType.RGBA_8888, @@ -574,8 +570,6 @@ class CkImage implements ui.Image, StackTraceDebugger { height: height.toDouble(), ); final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); - skImage.delete(); - if (pixels == null) { throw StateError('Unable to convert read pixels from SkImage.'); } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart new file mode 100644 index 00000000000..fb9eb4325ea --- /dev/null +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/multi_surface_rasterizer.dart @@ -0,0 +1,84 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:ui/src/engine.dart'; +import 'package:ui/ui.dart' as ui; + +/// A Rasterizer which uses one or many on-screen WebGL contexts to display the +/// scene. This way of rendering is prone to bugs because there is a limit to +/// how many WebGL contexts can be live at one time as well as bugs in sharing +/// GL resources between the contexts. However, using [createImageBitmap] is +/// currently very slow on Firefox and Safari browsers, so directly rendering +/// to several [Surface]s is how we can achieve 60 fps on these browsers. +class MultiSurfaceRasterizer extends Rasterizer { + @override + MultiSurfaceViewRasterizer createViewRasterizer(EngineFlutterView view) { + return _viewRasterizers.putIfAbsent(view, () => MultiSurfaceViewRasterizer(view, this)); + } + + final Map _viewRasterizers = + {}; + + @override + void dispose() { + for (final MultiSurfaceViewRasterizer viewRasterizer in _viewRasterizers.values) { + viewRasterizer.dispose(); + } + _viewRasterizers.clear(); + } + + @override + void setResourceCacheMaxBytes(int bytes) { + for (final MultiSurfaceViewRasterizer viewRasterizer in _viewRasterizers.values) { + viewRasterizer.displayFactory.forEachCanvas((Surface surface) { + surface.setSkiaResourceCacheMaxBytes(bytes); + }); + } + } +} + +class MultiSurfaceViewRasterizer extends ViewRasterizer { + MultiSurfaceViewRasterizer(super.view, this.rasterizer); + + final MultiSurfaceRasterizer rasterizer; + + @override + final DisplayCanvasFactory displayFactory = DisplayCanvasFactory( + createCanvas: () => Surface(isDisplayCanvas: true), + ); + + @override + void prepareToDraw() { + displayFactory.baseCanvas.createOrUpdateSurface(currentFrameSize); + } + + Future rasterizeToCanvas(DisplayCanvas canvas, ui.Picture picture) { + final surface = canvas as Surface; + surface.createOrUpdateSurface(currentFrameSize); + surface.positionToShowFrame(currentFrameSize); + final CkCanvas skCanvas = surface.getCanvas(); + skCanvas.clear(const ui.Color(0x00000000)); + skCanvas.drawPicture(picture); + surface.flush(); + return Future.value(); + } + + @override + Future rasterize( + List displayCanvases, + List pictures, + FrameTimingRecorder? recorder, + ) async { + if (displayCanvases.length != pictures.length) { + throw ArgumentError('Called rasterize() with a different number of canvases and pictures.'); + } + final rasterizeFutures = >[]; + for (var i = 0; i < displayCanvases.length; i++) { + rasterizeFutures.add(rasterizeToCanvas(displayCanvases[i], pictures[i])); + } + recorder?.recordRasterStart(); + await Future.wait(rasterizeFutures); + recorder?.recordRasterFinish(); + } +} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/offscreen_canvas_rasterizer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/offscreen_canvas_rasterizer.dart similarity index 57% rename from engine/src/flutter/lib/web_ui/lib/src/engine/compositing/offscreen_canvas_rasterizer.dart rename to engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/offscreen_canvas_rasterizer.dart index edcc84423e7..22cc6a21cfc 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/offscreen_canvas_rasterizer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/offscreen_canvas_rasterizer.dart @@ -2,7 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:meta/meta.dart'; import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; @@ -10,22 +9,9 @@ import 'package:ui/ui.dart' as ui; /// all the rendering. It transfers bitmaps created in the OffscreenCanvas to /// one or many on-screen elements to actually display the scene. class OffscreenCanvasRasterizer extends Rasterizer { - OffscreenCanvasRasterizer( - OffscreenSurface Function(OffscreenCanvasProvider) offscreenSurfaceCreateFn, - ) : _surfaceProvider = OffscreenSurfaceProvider( - OffscreenCanvasProvider(), - offscreenSurfaceCreateFn, - ); - - final OffscreenSurfaceProvider _surfaceProvider; - - @override - @visibleForTesting - SurfaceProvider get surfaceProvider => _surfaceProvider; - /// This is an SkSurface backed by an OffScreenCanvas. This single Surface is /// used to render to many RenderCanvases to produce the rendered scene. - late final OffscreenSurface offscreenSurface = _surfaceProvider.createSurface(); + final Surface offscreenSurface = Surface(); @override OffscreenCanvasViewRasterizer createViewRasterizer(EngineFlutterView view) { @@ -37,21 +23,16 @@ class OffscreenCanvasRasterizer extends Rasterizer { @override void setResourceCacheMaxBytes(int bytes) { - _surfaceProvider.setSkiaResourceCacheMaxBytes(bytes); + offscreenSurface.setSkiaResourceCacheMaxBytes(bytes); } @override void dispose() { - _surfaceProvider.dispose(); + offscreenSurface.dispose(); for (final OffscreenCanvasViewRasterizer viewRasterizer in _viewRasterizers.values) { viewRasterizer.dispose(); } } - - @override - Surface createPictureToImageSurface() { - return _surfaceProvider.createSurface(); - } } class OffscreenCanvasViewRasterizer extends ViewRasterizer { @@ -64,9 +45,18 @@ class OffscreenCanvasViewRasterizer extends ViewRasterizer { createCanvas: () => RenderCanvas(), ); + /// Render the given [picture] so it is displayed by the given [canvas]. + Future rasterizeToCanvas(DisplayCanvas canvas, ui.Picture picture) async { + await rasterizer.offscreenSurface.rasterizeToCanvas( + currentFrameSize, + canvas as RenderCanvas, + picture, + ); + } + @override - Future prepareToDraw() { - return rasterizer.offscreenSurface.setSize(currentFrameSize); + void prepareToDraw() { + rasterizer.offscreenSurface.createOrUpdateSurface(currentFrameSize); } @override @@ -78,23 +68,12 @@ class OffscreenCanvasViewRasterizer extends ViewRasterizer { if (displayCanvases.length != pictures.length) { throw ArgumentError('Called rasterize() with a different number of canvases and pictures.'); } - recorder?.recordRasterStart(); - if (browserSupportsCreateImageBitmap) { - final List bitmaps = await rasterizer.offscreenSurface - .rasterizeToImageBitmaps(pictures); - for (var i = 0; i < displayCanvases.length; i++) { - (displayCanvases[i] as RenderCanvas).render(bitmaps[i]); - } - } else { - for (var i = 0; i < displayCanvases.length; i++) { - await rasterizer.offscreenSurface.rasterizeToCanvas(pictures[i]); - (displayCanvases[i] as RenderCanvas).renderWithNoBitmapSupport( - rasterizer.offscreenSurface.canvasImageSource, - currentFrameSize.height, - currentFrameSize, - ); - } + final rasterizeFutures = >[]; + for (var i = 0; i < displayCanvases.length; i++) { + rasterizeFutures.add(rasterizeToCanvas(displayCanvases[i], pictures[i])); } + recorder?.recordRasterStart(); + await Future.wait(rasterizeFutures); recorder?.recordRasterFinish(); } } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart index bbce2cb2e67..0d411621ffe 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/picture.dart @@ -6,7 +6,6 @@ import 'dart:typed_data'; import 'package:ui/ui.dart' as ui; -import '../compositing/surface.dart'; import '../layer/layer_painting.dart'; import '../util.dart'; import 'canvas.dart'; @@ -115,18 +114,11 @@ class CkPicture implements LayerPicture, StackTraceDebugger { assert(debugCheckNotDisposed('Cannot convert picture to image.')); final Surface surface = CanvasKitRenderer.instance.pictureToImageSurface; - surface.setSize(BitmapSize(width, height)); - final ckSurface = surface as CkSurface; - final SkSurface skiaSurface = ckSurface.skSurface!; - - final ckCanvas = CkCanvas.fromSkCanvas(skiaSurface.getCanvas()); + final CkSurface ckSurface = surface.createOrUpdateSurface(BitmapSize(width, height)); + final CkCanvas ckCanvas = ckSurface.getCanvas(); ckCanvas.clear(const ui.Color(0x00000000)); ckCanvas.drawPicture(this); - final SkImage skImage = skiaSurface.makeImageSnapshot(); - - // TODO(hterkelsen): This is a hack to get the pixels from the SkImage. - // We should be able to do this without creating a new image. This is - // a workaround for a bug in CanvasKit. + final SkImage skImage = ckSurface.surface.makeImageSnapshot(); final imageInfo = SkImageInfo( alphaType: canvasKit.AlphaType.Premul, colorType: canvasKit.ColorType.RGBA_8888, @@ -135,9 +127,8 @@ class CkPicture implements LayerPicture, StackTraceDebugger { height: height.toDouble(), ); final Uint8List? pixels = skImage.readPixels(0, 0, imageInfo); - skImage.delete(); if (pixels == null) { - throw StateError('Unable to convert read pixels from SkImage.'); + throw StateError('Unable to read pixels from SkImage.'); } final SkImage? rasterImage = canvasKit.MakeImage(imageInfo, pixels, (4 * width).toDouble()); if (rasterImage == null) { @@ -162,7 +153,4 @@ class CkPicture implements LayerPicture, StackTraceDebugger { @override StackTrace get debugStackTrace => _debugStackTrace; - - @override - bool get isDisposed => _isDisposed; } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart index 6762246be59..4861b04f4ef 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/renderer.dart @@ -32,21 +32,20 @@ class CanvasKitRenderer extends Renderer { static Rasterizer _createRasterizer() { if (configuration.canvasKitForceMultiSurfaceRasterizer || isSafari || isFirefox) { - return MultiSurfaceRasterizer( - (OnscreenCanvasProvider canvasProvider) => CkOnscreenSurface(canvasProvider), - ); + return MultiSurfaceRasterizer(); } - return OffscreenCanvasRasterizer( - (OffscreenCanvasProvider canvasProvider) => CkOffscreenSurface(canvasProvider), - ); + return OffscreenCanvasRasterizer(); } @override void debugResetRasterizer() { rasterizer = _createRasterizer(); - _pictureToImageSurface = rasterizer.createPictureToImageSurface(); } + /// A surface used specifically for `Picture.toImage` when software rendering + /// is supported. + final Surface pictureToImageSurface = Surface(); + @override Future initialize() async { _initialized ??= () async { @@ -60,7 +59,6 @@ class CanvasKitRenderer extends Renderer { windowFlutterCanvasKit = canvasKit; } rasterizer = _createRasterizer(); - _pictureToImageSurface = rasterizer.createPictureToImageSurface(); _instance = this; await super.initialize(); }(); @@ -496,9 +494,4 @@ class CanvasKitRenderer extends Renderer { } } } - - late Surface _pictureToImageSurface; - - @override - Surface get pictureToImageSurface => _pictureToImageSurface; } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/surface.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/surface.dart index d324a3bd4c2..2b2c7ff7634 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/surface.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/canvaskit/surface.dart @@ -2,338 +2,557 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:async'; import 'dart:js_interop'; -import 'dart:typed_data'; -import 'package:meta/meta.dart'; -import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; +import '../browser_detection.dart'; +import '../compositing/rasterizer.dart'; +import '../compositing/render_canvas.dart'; +import '../configuration.dart'; +import '../display.dart'; +import '../dom.dart'; +import '../platform_dispatcher.dart'; +import '../util.dart'; +import 'canvas.dart'; +import 'canvaskit_api.dart'; +import 'util.dart'; + // Only supported in profile/release mode. Allows Flutter to use MSAA but // removes the ability for disabling AA on Paint objects. const bool _kUsingMSAA = bool.fromEnvironment('flutter.canvaskit.msaa'); -/// The base class for CanvasKit surfaces, containing shared logic for context -/// management and Skia object creation. -abstract class CkSurface extends Surface { - CkSurface(this._canvasProvider) { - _canvas = _canvasProvider.acquireCanvas(_currentSize, onContextLost: onContextLost); - _maybeAttachCanvasToDom(); - _initialize(); +/// A surface which can be drawn into by the compositor. +/// +/// The underlying representation is a [CkSurface], which can be reused by +/// successive frames if they are the same size. Otherwise, a new [CkSurface] is +/// created. +class Surface extends DisplayCanvas { + Surface({this.isDisplayCanvas = false}) + : useOffscreenCanvas = Surface.offscreenCanvasSupported && !isDisplayCanvas; + + CkSurface? _surface; + + /// Returns the underlying CanvasKit Surface. Should only be used in tests. + CkSurface? debugGetCkSurface() { + var assertsEnabled = false; + assert(() { + assertsEnabled = true; + return true; + }()); + if (!assertsEnabled) { + throw StateError('debugGetCkSurface() can only be used in tests'); + } + return _surface; } - final CanvasProvider _canvasProvider; + /// Whether or not to use an `OffscreenCanvas` to back this [Surface]. + final bool useOffscreenCanvas; - BitmapSize _currentSize = const BitmapSize(1, 1); + /// If `true`, this [Surface] is used as a [DisplayCanvas]. + final bool isDisplayCanvas; - /// The underlying Skia surface object. - SkSurface? get skSurface => _skSurface; - SkSurface? _skSurface; + /// If true, forces a new WebGL context to be created, even if the window + /// size is the same. This is used to restore the UI after the browser tab + /// goes dormant and loses the GL context. + bool _forceNewContext = true; + bool get debugForceNewContext => _forceNewContext; - /// Whether or not WebGl is supported. + bool _contextLost = false; + bool get debugContextLost => _contextLost; + + /// Forces AssertionError when attempting to create a CPU-based surface. + /// Only for tests. + bool debugThrowOnSoftwareSurfaceCreation = false; + + /// A cached copy of the most recently created `webglcontextlost` listener. /// - /// This defaults to true unless `canvasKitForceCpuOnly` is set to true or - /// `webGLVersion` is -1. If Skia fails to create a GrContext, this will be - /// set to false. - @visibleForTesting - bool get supportsWebGl { - if (configuration.canvasKitForceCpuOnly) { - _fallbackToSoftwareReason = 'canvasKitForceCpuOnly is set to true'; - return false; + /// We must cache this function because each time we access the tear-off it + /// creates a new object, meaning we won't be able to remove this listener + /// later. + DomEventListener? _cachedContextLostListener; + + /// A cached copy of the most recently created `webglcontextrestored` + /// listener. + /// + /// We must cache this function because each time we access the tear-off it + /// creates a new object, meaning we won't be able to remove this listener + /// later. + DomEventListener? _cachedContextRestoredListener; + + SkGrContext? _grContext; + int? _glContext; + int? _skiaCacheBytes; + + /// The underlying OffscreenCanvas element used for this surface. + DomOffscreenCanvas? _offscreenCanvas; + + /// Returns the underlying OffscreenCanvas. Should only be used in tests. + DomOffscreenCanvas? debugGetOffscreenCanvas() { + var assertsEnabled = false; + assert(() { + assertsEnabled = true; + return true; + }()); + if (!assertsEnabled) { + throw StateError('debugGetOffscreenCanvas() can only be used in tests'); } - if (webGLVersion == -1) { - _fallbackToSoftwareReason = 'webGLVersion is -1'; - return false; - } - if (_failedToCreateGrContext) { - return false; - } - return true; + return _offscreenCanvas; } - String? _fallbackToSoftwareReason; + /// The backing this Surface in the case that OffscreenCanvas isn't + /// supported. + DomHTMLCanvasElement? _canvasElement; - /// When true, the surface will fail to create a GL context and fall back to - /// software rendering. This is useful for testing. - @visibleForTesting - static bool debugForceGLFailure = false; + /// Note, if this getter is called, then this Surface is being used as an + /// overlay and must be backed by an onscreen element. + @override + final DomElement hostElement = createDomElement('flt-canvas-container'); - bool _failedToCreateGrContext = false; + int _pixelWidth = -1; + int _pixelHeight = -1; + double _currentDevicePixelRatio = -1; + int _sampleCount = -1; + int _stencilBits = -1; + + /// Specify the GPU resource cache limits. + void setSkiaResourceCacheMaxBytes(int bytes) { + _skiaCacheBytes = bytes; + _syncCacheBytes(); + } + + void _syncCacheBytes() { + if (_skiaCacheBytes != null) { + _grContext?.setResourceCacheLimitBytes(_skiaCacheBytes!.toDouble()); + } + } + + /// The CanvasKit canvas associated with this surface. + CkCanvas getCanvas() { + return _surface!.getCanvas(); + } + + void flush() { + _surface!.flush(); + } + + Future rasterizeToCanvas( + BitmapSize bitmapSize, + RenderCanvas canvas, + ui.Picture picture, + ) async { + final CkCanvas skCanvas = getCanvas(); + skCanvas.clear(const ui.Color(0x00000000)); + skCanvas.drawPicture(picture); + flush(); + + if (browserSupportsCreateImageBitmap) { + JSObject bitmapSource; + DomImageBitmap bitmap; + if (useOffscreenCanvas) { + bitmap = _offscreenCanvas!.transferToImageBitmap(); + } else { + bitmapSource = _canvasElement!; + bitmap = await createImageBitmap(bitmapSource, ( + x: 0, + y: _pixelHeight - bitmapSize.height, + width: bitmapSize.width, + height: bitmapSize.height, + )); + } + canvas.render(bitmap); + } else { + // If the browser doesn't support `createImageBitmap` (e.g. Safari 14) + // then render using `drawImage` instead. + DomCanvasImageSource imageSource; + if (useOffscreenCanvas) { + imageSource = _offscreenCanvas! as DomCanvasImageSource; + } else { + imageSource = _canvasElement! as DomCanvasImageSource; + } + canvas.renderWithNoBitmapSupport(imageSource, _pixelHeight, bitmapSize); + } + } + + BitmapSize? _currentCanvasPhysicalSize; + + /// Sets the CSS size of the canvas so that canvas pixels are 1:1 with device + /// pixels. + void _updateLogicalHtmlCanvasSize() { + final double devicePixelRatio = EngineFlutterDisplay.instance.devicePixelRatio; + final double logicalWidth = _pixelWidth / devicePixelRatio; + final double logicalHeight = _pixelHeight / devicePixelRatio; + final DomCSSStyleDeclaration style = _canvasElement!.style; + style.width = '${logicalWidth}px'; + style.height = '${logicalHeight}px'; + _currentDevicePixelRatio = devicePixelRatio; + } + + /// The element backing this surface may be larger than the screen. + /// The Surface will draw the frame to the bottom left of the , but + /// the is, by default, positioned so that the top left corner is in + /// the top left of the window. We need to shift the canvas down so that the + /// bottom left of the is the the bottom left corner of the window. + void positionToShowFrame(BitmapSize frameSize) { + assert(isDisplayCanvas, 'Should not position Surface if not used as a render canvas'); + final double devicePixelRatio = EngineFlutterDisplay.instance.devicePixelRatio; + final double logicalHeight = _pixelHeight / devicePixelRatio; + final double logicalFrameHeight = frameSize.height / devicePixelRatio; + + // Shift the canvas up so the bottom left is in the window. + _canvasElement!.style.transform = 'translate(0px, ${logicalFrameHeight - logicalHeight}px)'; + } + + /// This is only valid after the first frame or if [ensureSurface] has been + /// called + bool get usingSoftwareBackend => + _glContext == null || + _grContext == null || + webGLVersion == -1 || + configuration.canvasKitForceCpuOnly; + + /// Ensure that the initial surface exists and has a size of at least [size]. + /// + /// If not provided, [size] defaults to 1x1. + /// + /// This also ensures that the gl/grcontext have been populated so + /// that software rendering can be detected. + void ensureSurface([BitmapSize size = const BitmapSize(1, 1)]) { + // If the GrContext hasn't been setup yet then we need to force initialization + // of the canvas and initial surface. + if (_surface != null) { + return; + } + // TODO(jonahwilliams): this is somewhat wasteful. We should probably + // eagerly setup this surface instead of delaying until the first frame? + // Or at least cache the estimated window size. + // This is the first frame we have rendered with this canvas. + createOrUpdateSurface(size); + } + + /// Creates a and SkSurface for the given [size]. + CkSurface createOrUpdateSurface(BitmapSize size) { + if (size.isEmpty) { + throw CanvasKitError('Cannot create surfaces of empty size.'); + } + + if (!_forceNewContext) { + // Check if the window is the same size as before, and if so, don't allocate + // a new canvas as the previous canvas is big enough to fit everything. + final BitmapSize? previousSurfaceSize = _surface?._size; + if (previousSurfaceSize != null && + size.width == previousSurfaceSize.width && + size.height == previousSurfaceSize.height) { + final double devicePixelRatio = EngineFlutterDisplay.instance.devicePixelRatio; + if (isDisplayCanvas && devicePixelRatio != _currentDevicePixelRatio) { + _updateLogicalHtmlCanvasSize(); + } + return _surface!; + } + + if (_currentCanvasPhysicalSize != null && + (size.width != _currentCanvasPhysicalSize!.width || + size.height != _currentCanvasPhysicalSize!.height)) { + _surface?.dispose(); + _surface = null; + _pixelWidth = size.width; + _pixelHeight = size.height; + if (useOffscreenCanvas) { + _offscreenCanvas!.width = _pixelWidth.toDouble(); + _offscreenCanvas!.height = _pixelHeight.toDouble(); + } else { + _canvasElement!.width = _pixelWidth.toDouble(); + _canvasElement!.height = _pixelHeight.toDouble(); + } + _currentCanvasPhysicalSize = BitmapSize(_pixelWidth, _pixelHeight); + if (isDisplayCanvas) { + _updateLogicalHtmlCanvasSize(); + } + } + } + + // If we reached here, then this is the first frame and we haven't made a + // surface yet, we are forcing a new context, or the size of the surface + // has changed and we need to make a new one. + _surface?.dispose(); + _surface = null; + + // Either a new context is being forced or we've never had one. + if (_forceNewContext || _currentCanvasPhysicalSize == null) { + _grContext?.releaseResourcesAndAbandonContext(); + _grContext?.delete(); + _grContext = null; + + _createNewCanvas(size); + _currentCanvasPhysicalSize = size; + } + + return _surface = _createNewSurface(size); + } + + void _contextRestoredListener(DomEvent event) { + assert( + _contextLost, + 'Received "webglcontextrestored" event but never received ' + 'a "webglcontextlost" event.', + ); + _contextLost = false; + // Force the framework to rerender the frame. + EnginePlatformDispatcher.instance.invokeOnMetricsChanged(); + event.stopPropagation(); + event.preventDefault(); + } + + void _contextLostListener(DomEvent event) { + assert( + event.target == _offscreenCanvas || event.target == _canvasElement, + 'Received a context lost event for a disposed canvas', + ); + _contextLost = true; + _forceNewContext = true; + event.preventDefault(); + } + + /// This function is expensive. + /// + /// It's better to reuse canvas if possible. + void _createNewCanvas(BitmapSize physicalSize) { + // Clear the container, if it's not empty. We're going to create a new . + if (_offscreenCanvas != null) { + _offscreenCanvas!.removeEventListener( + 'webglcontextrestored', + _cachedContextRestoredListener, + false.toJS, + ); + _offscreenCanvas!.removeEventListener( + 'webglcontextlost', + _cachedContextLostListener, + false.toJS, + ); + _offscreenCanvas = null; + _cachedContextRestoredListener = null; + _cachedContextLostListener = null; + } else if (_canvasElement != null) { + _canvasElement!.removeEventListener( + 'webglcontextrestored', + _cachedContextRestoredListener, + false.toJS, + ); + _canvasElement!.removeEventListener( + 'webglcontextlost', + _cachedContextLostListener, + false.toJS, + ); + _canvasElement!.remove(); + _canvasElement = null; + _cachedContextRestoredListener = null; + _cachedContextLostListener = null; + } + + // If `physicalSize` is not precise, use a slightly bigger canvas. This way + // we ensure that the rendred picture covers the entire browser window. + _pixelWidth = physicalSize.width; + _pixelHeight = physicalSize.height; + DomEventTarget htmlCanvas; + if (useOffscreenCanvas) { + final DomOffscreenCanvas offscreenCanvas = createDomOffscreenCanvas( + _pixelWidth, + _pixelHeight, + ); + htmlCanvas = offscreenCanvas; + _offscreenCanvas = offscreenCanvas; + _canvasElement = null; + } else { + final DomHTMLCanvasElement canvas = createDomCanvasElement( + width: _pixelWidth, + height: _pixelHeight, + ); + htmlCanvas = canvas; + _canvasElement = canvas; + _offscreenCanvas = null; + if (isDisplayCanvas) { + _canvasElement!.setAttribute('aria-hidden', 'true'); + _canvasElement!.style.position = 'absolute'; + hostElement.append(_canvasElement!); + _updateLogicalHtmlCanvasSize(); + } + } + + // When the browser tab using WebGL goes dormant the browser and/or OS may + // decide to clear GPU resources to let other tabs/programs use the GPU. + // When this happens, the browser sends the "webglcontextlost" event as a + // notification. When we receive this notification we force a new context. + // + // See also: https://www.khronos.org/webgl/wiki/HandlingContextLost + _cachedContextRestoredListener = createDomEventListener(_contextRestoredListener); + _cachedContextLostListener = createDomEventListener(_contextLostListener); + htmlCanvas.addEventListener('webglcontextlost', _cachedContextLostListener, false.toJS); + htmlCanvas.addEventListener('webglcontextrestored', _cachedContextRestoredListener, false.toJS); + _forceNewContext = false; + _contextLost = false; + + if (webGLVersion != -1 && !configuration.canvasKitForceCpuOnly) { + var glContext = 0; + final options = SkWebGLContextOptions( + // Default to no anti-aliasing. Paint commands can be explicitly + // anti-aliased by setting their `Paint` object's `antialias` property. + antialias: _kUsingMSAA ? 1 : 0, + majorVersion: webGLVersion.toDouble(), + ); + if (useOffscreenCanvas) { + glContext = canvasKit.GetOffscreenWebGLContext(_offscreenCanvas!, options).toInt(); + } else { + glContext = canvasKit.GetWebGLContext(_canvasElement!, options).toInt(); + } + + _glContext = glContext; + + if (_glContext != 0) { + _grContext = canvasKit.MakeGrContext(glContext.toDouble()); + if (_grContext == null) { + // TODO(harryterkelsen): Make this error message more descriptive by + // reporting the number of currently live Surfaces, https://github.com/flutter/flutter/issues/162868. + throw CanvasKitError( + 'Failed to initialize CanvasKit. ' + 'CanvasKit.MakeGrContext returned null.', + ); + } + if (_sampleCount == -1 || _stencilBits == -1) { + _initWebglParams(); + } + // Set the cache byte limit for this grContext, if not specified it will + // use CanvasKit's default. + _syncCacheBytes(); + } + } + } + + void _initWebglParams() { + WebGLContext gl; + if (useOffscreenCanvas) { + gl = _offscreenCanvas!.getGlContext(webGLVersion); + } else { + gl = _canvasElement!.getGlContext(webGLVersion); + } + _sampleCount = gl.getParameter(gl.samples); + _stencilBits = gl.getParameter(gl.stencilBits); + } + + CkSurface _createNewSurface(BitmapSize size) { + assert(_offscreenCanvas != null || _canvasElement != null); + if (webGLVersion == -1) { + return _makeSoftwareCanvasSurface('WebGL support not detected', size); + } else if (configuration.canvasKitForceCpuOnly) { + return _makeSoftwareCanvasSurface('CPU rendering forced by application', size); + } else if (_glContext == 0) { + return _makeSoftwareCanvasSurface('Failed to initialize WebGL context', size); + } else { + final SkSurface? skSurface = canvasKit.MakeOnScreenGLSurface( + _grContext!, + size.width.toDouble(), + size.height.toDouble(), + SkColorSpaceSRGB, + _sampleCount, + _stencilBits, + ); + + if (skSurface == null) { + return _makeSoftwareCanvasSurface('Failed to initialize WebGL surface', size); + } + + return CkSurface(skSurface, _glContext, size); + } + } static bool _didWarnAboutWebGlInitializationFailure = false; - /// The underlying GL context. Returns -1 if the context is not initialized. - @override - @visibleForTesting - int get glContext => _glContext; - int _glContext = -1; - - /// The canvas object that this surface is rendering to. - @visibleForTesting - DomEventTarget get canvas => _canvas; - late DomEventTarget _canvas; - - void _maybeAttachCanvasToDom(); - - /// A [Future] which completes when the [Surface] is initialized and ready to - /// render pictures. - @override - Future get initialized => _initialized.future; - final Completer _initialized = Completer(); - - late Completer? _handledContextLostEvent; - - /// Creates the canvas object and initializes the graphics context. - Future _initialize() async { - _createSkiaObjects(); - _initialized.complete(); - } - - /// The underlying Skia graphics context. - SkGrContext? _grContext; - - void onContextLost() { - _handledContextLostEvent?.complete(); - final DomEventTarget newCanvas = _canvasProvider.acquireCanvas( - _currentSize, - onContextLost: onContextLost, - ); - recreateContextForCanvas(newCanvas); - } - - void _recreateSkSurface() { - if (supportsWebGl) { - try { - _recreateWebGlSkSurface(); - } catch (e) { - _failedToCreateGrContext = true; - _fallbackToSoftwareReason = 'failed to create GrContext. Error: $e'; - _recreateSoftwareSkSurface(); - } - } else { - _recreateSoftwareSkSurface(); - } - } - - /// Creates the GL context and the Skia `GrContext`. - void _createGrContext() { - if (debugForceGLFailure) { - _failedToCreateGrContext = true; - _fallbackToSoftwareReason = 'debugForceGLFailure is true'; - return; - } - final options = SkWebGLContextOptions( - antialias: _kUsingMSAA ? 1 : 0, - majorVersion: webGLVersion.toDouble(), - ); - _glContext = _getGlContext(options); - _grContext = canvasKit.MakeGrContext(_glContext.toDouble()); - if (_grContext == null) { - _failedToCreateGrContext = true; - _fallbackToSoftwareReason = 'failed to create GrContext.'; - } - } - - /// Creates the underlying GL context for the canvas. - /// - /// This method is implemented by subclasses to handle their specific - /// canvas types. - int _getGlContext(SkWebGLContextOptions options); - - /// Creates the Skia objects that are backed by the canvas. - /// - /// This method is responsible for creating the `SkGrContext` and the - /// `SkSurface`. - void _createSkiaObjects() { - if (supportsWebGl) { - _createGrContext(); - } - _recreateSkSurface(); - } - - void _recreateWebGlSkSurface() { - _skSurface?.dispose(); - _skSurface = canvasKit.MakeOnScreenGLSurface( - _grContext!, - _currentSize.width.toDouble(), - _currentSize.height.toDouble(), - SkColorSpaceSRGB, - 0, - 0, - ); - if (_skSurface == null) { - throw Exception('Failed to initialize CanvasKit SkSurface.'); - } - } - - void _recreateSoftwareSkSurface() { + CkSurface _makeSoftwareCanvasSurface(String reason, BitmapSize size) { if (!_didWarnAboutWebGlInitializationFailure) { + printWarning('WARNING: Falling back to CPU-only rendering. $reason.'); _didWarnAboutWebGlInitializationFailure = true; - printWarning( - 'WARNING: Falling back to CPU-only rendering. Reason: $_fallbackToSoftwareReason', - ); } - _skSurface?.dispose(); - _skSurface = _createSoftwareSkSurface(); - if (_skSurface == null) { - throw Exception('Failed to initialize CanvasKit SkSurface.'); + + try { + assert(!debugThrowOnSoftwareSurfaceCreation); + + SkSurface surface; + if (useOffscreenCanvas) { + surface = canvasKit.MakeOffscreenSWCanvasSurface(_offscreenCanvas!); + } else { + surface = canvasKit.MakeSWCanvasSurface(_canvasElement!); + } + return CkSurface(surface, null, size); + } catch (error) { + throw CanvasKitError('Failed to create CPU-based surface: $error.'); } } - /// Creates an SkSurface for software rendering. This is used when WebGl is not - /// supported or when it fails to initialize. - SkSurface _createSoftwareSkSurface(); - - double _currentDevicePixelRatio = -1; + @override + bool get isConnected => _canvasElement!.isConnected!; @override - Future setSize(BitmapSize size) async { - final double devicePixelRatio = EngineFlutterDisplay.instance.devicePixelRatio; - if (_skSurface != null && - _currentSize == size && - devicePixelRatio == _currentDevicePixelRatio) { - return; - } - _currentDevicePixelRatio = devicePixelRatio; - _currentSize = size; - _canvasProvider.resizeCanvas(canvas, size); - _recreateSkSurface(); - } - - @override - Future recreateContextForCanvas(DomEventTarget newCanvas) async { - // The old Skia surface is now invalid and should be disposed. - _skSurface?.dispose(); - _skSurface = null; - - // The GrContext is also invalid and will be recreated by `_createSkiaObjects`. - _grContext = null; - - _canvas = newCanvas; - _maybeAttachCanvasToDom(); - _createSkiaObjects(); + void initialize() { + ensureSurface(); } @override void dispose() { - _skSurface?.dispose(); + _offscreenCanvas?.removeEventListener( + 'webglcontextlost', + _cachedContextLostListener, + false.toJS, + ); + _offscreenCanvas?.removeEventListener( + 'webglcontextrestored', + _cachedContextRestoredListener, + false.toJS, + ); + _cachedContextLostListener = null; + _cachedContextRestoredListener = null; + _surface?.dispose(); } - @override - void setSkiaResourceCacheMaxBytes(int bytes) { - _grContext?.setResourceCacheLimitBytes(bytes.toDouble()); - } - - @override - Future rasterizeImage(ui.Image image, ui.ImageByteFormat format) async { - await _initialized.future; - final ckImage = image as CkImage; - final SkSurface skSurface = _skSurface!; - final canvas = CkCanvas.fromSkCanvas(skSurface.getCanvas()); - canvas.drawImage(ckImage, ui.Offset.zero, ui.Paint()); - final SkImage snapshot = skSurface.makeImageSnapshot(); - final Uint8List? bytes = snapshot.encodeToBytes(); - snapshot.delete(); - return bytes?.buffer.asByteData(); - } - - @override - DomCanvasImageSource get canvasImageSource => canvas as DomCanvasImageSource; - - @override - Future rasterizeToCanvas(ui.Picture picture) async { - await _initialized.future; - final canvas = CkCanvas.fromSkCanvas(_skSurface!.getCanvas()); - final ckPicture = picture as CkPicture; - canvas.clear(const ui.Color(0x00000000)); - canvas.drawPicture(ckPicture); - _skSurface!.flush(); - } - - @override - Future triggerContextLoss(); - - @override - Future get handledContextLossEvent => _handledContextLostEvent!.future; + /// Safari 15 doesn't support OffscreenCanvas at all. Safari 16 supports + /// OffscreenCanvas, but only with the context2d API, not WebGL. + static bool get offscreenCanvasSupported => browserSupportsOffscreenCanvas && !isSafari; } -/// The CanvasKit implementation of [OffscreenSurface]. -class CkOffscreenSurface extends CkSurface implements OffscreenSurface { - CkOffscreenSurface(OffscreenCanvasProvider super.canvasProvider); +/// A Dart wrapper around Skia's SkSurface. +class CkSurface { + CkSurface(this.surface, this._glContext, this._size); - @override - int _getGlContext(SkWebGLContextOptions options) { - return canvasKit.GetOffscreenWebGLContext(canvas as DomOffscreenCanvas, options).toInt(); + CkCanvas getCanvas() { + assert(!_isDisposed, 'Attempting to use the canvas of a disposed surface'); + return CkCanvas.fromSkCanvas(surface.getCanvas()); } - @override - SkSurface _createSoftwareSkSurface() { - return canvasKit.MakeOffscreenSWCanvasSurface(canvas as DomOffscreenCanvas); + /// The underlying CanvasKit surface object. + /// + /// Only borrow this value temporarily. Do not store it as it may be deleted + /// at any moment. Storing it may lead to dangling pointer bugs. + final SkSurface surface; + + final BitmapSize _size; + + final int? _glContext; + + /// Flushes the graphics to be rendered on screen. + void flush() { + surface.flush(); } - @override - Future> rasterizeToImageBitmaps(List pictures) async { - await _initialized.future; - final bitmaps = []; - for (final picture in pictures) { - await rasterizeToCanvas(picture); - bitmaps.add(await createImageBitmap(_canvas)); + int? get context => _glContext; + + int width() => surface.width().ceil(); + int height() => surface.height().ceil(); + + void dispose() { + if (_isDisposed) { + return; } - return bitmaps; + surface.dispose(); + _isDisposed = true; } - @override - void _maybeAttachCanvasToDom() { - // Do not attach the OffscreenCanvas to the DOM. - } - - @override - Future triggerContextLoss() async { - _handledContextLostEvent = Completer(); - final WebGLContext gl = (canvas as DomOffscreenCanvas).getGlContext(webGLVersion); - gl.loseContextExtension.loseContext(); - } -} - -/// The CanvasKit implementation of [OnscreenSurface]. -class CkOnscreenSurface extends CkSurface implements OnscreenSurface { - CkOnscreenSurface(OnscreenCanvasProvider super.canvasProvider); - - @override - int _getGlContext(SkWebGLContextOptions options) { - return canvasKit.GetWebGLContext(canvas as DomHTMLCanvasElement, options).toInt(); - } - - @override - SkSurface _createSoftwareSkSurface() { - return canvasKit.MakeSWCanvasSurface(canvas as DomHTMLCanvasElement); - } - - final DomElement _hostElement = createDomElement('flt-canvas-container'); - - @override - DomElement get hostElement => _hostElement; - - @override - void _maybeAttachCanvasToDom() { - hostElement.appendChild(canvas as DomHTMLCanvasElement); - } - - @override - bool get isConnected => - ((canvas as JSAny?).isA()) && - (canvas as DomHTMLCanvasElement).isConnected!; - - @override - void initialize() { - // No extra initialization is required. - } - - @override - Future triggerContextLoss() async { - _handledContextLostEvent = Completer(); - final WebGLContext gl = (canvas as DomHTMLCanvasElement).getGlContext(webGLVersion); - gl.loseContextExtension.loseContext(); - } + bool _isDisposed = false; } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/canvas_provider.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/canvas_provider.dart deleted file mode 100644 index f48fa59be08..00000000000 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/canvas_provider.dart +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:ui/src/engine.dart'; -import 'package:ui/ui.dart' as ui; - -/// Manages the lifecycle of raw canvas elements, abstracting away the differences -/// between onscreen and offscreen canvases. -/// -/// This class is responsible for: -/// - Acquiring and releasing canvas elements. -/// - Resizing canvases. -/// - Attaching `webglcontextlost` event listeners and notifying the consumer. -abstract class CanvasProvider { - final Map _eventListeners = {}; - - /// Acquires a canvas element of a given `size`. - /// - /// The `onContextLost` callback will be invoked when the underlying rendering - /// context for this canvas is lost. - C acquireCanvas(BitmapSize size, {required ui.VoidCallback onContextLost}) { - final C canvas = _createCanvas(size); - final DomEventListener eventListener = createDomEventListener((DomEvent event) { - onContextLost(); - // The canvas is no longer usable. - releaseCanvas(canvas); - }); - - _eventListeners[canvas] = eventListener; - canvas.addEventListener('webglcontextlost', eventListener); - return canvas; - } - - /// Resizes the `canvas` element to the new `size`. - /// - /// This method is responsible for updating the canvas element's dimensions - /// and any associated properties (e.g., CSS styles for onscreen canvases). - void resizeCanvas(C canvas, BitmapSize size); - - /// Releases a `canvas` element, allowing it to be pooled or disposed of. - void releaseCanvas(C canvas) { - final DomEventListener? listener = _eventListeners.remove(canvas); - if (listener != null) { - canvas.removeEventListener('webglcontextlost', listener); - } - detachCanvas(canvas); - } - - /// Disposes of all canvases managed by this provider. - void dispose() { - List.from(_eventListeners.keys).forEach(releaseCanvas); - assert(_eventListeners.isEmpty); - } - - /// Creates a canvas element. - C _createCanvas(BitmapSize size); - - /// Detaches a canvas element from the DOM if necessary. - void detachCanvas(C canvas); -} - -/// A [CanvasProvider] that manages a pool of [dom.DomOffscreenCanvas] elements. -class OffscreenCanvasProvider extends CanvasProvider { - @override - DomOffscreenCanvas _createCanvas(BitmapSize size) { - return DomOffscreenCanvas(size.width, size.height); - } - - @override - void detachCanvas(DomOffscreenCanvas canvas) { - // Nothing to do for offscreen canvases. - } - - @override - void resizeCanvas(DomOffscreenCanvas canvas, BitmapSize size) { - canvas.width = size.width.toDouble(); - canvas.height = size.height.toDouble(); - } -} - -/// A [CanvasProvider] that manages [dom.DomHTMLCanvasElement] elements. -class OnscreenCanvasProvider extends CanvasProvider { - @override - DomHTMLCanvasElement _createCanvas(BitmapSize size) { - final DomHTMLCanvasElement canvas = createDomCanvasElement(); - resizeCanvas(canvas, size); - return canvas; - } - - @override - void detachCanvas(DomHTMLCanvasElement canvas) { - canvas.remove(); - } - - @override - void resizeCanvas(DomHTMLCanvasElement canvas, BitmapSize size) { - canvas.width = size.width.toDouble(); - canvas.height = size.height.toDouble(); - - // When using an onscreen canvas, we also need to update the CSS size to - // account for the device pixel ratio. - final double ratio = EngineFlutterDisplay.instance.devicePixelRatio; - final cssWidth = '${size.width / ratio}px'; - final cssHeight = '${size.height / ratio}px'; - canvas.style - ..width = cssWidth - ..height = cssHeight; - } -} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/multi_surface_rasterizer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/multi_surface_rasterizer.dart deleted file mode 100644 index a3f4fd2341c..00000000000 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/multi_surface_rasterizer.dart +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:meta/meta.dart'; -import 'package:ui/src/engine.dart'; -import 'package:ui/ui.dart' as ui; - -/// A [Rasterizer] which uses one or many on-screen WebGL contexts to display -/// the scene. This way of rendering is prone to bugs because there is a limit -/// to how many WebGL contexts can be live at one time as well as bugs in -/// sharing GL resources between the contexts. However, using -/// [createImageBitmap] is currently very slow on Firefox and Safari browsers, -/// so directly rendering to several [Surface]s is how we can achieve 60 fps on -/// these browsers. -class MultiSurfaceRasterizer extends Rasterizer { - MultiSurfaceRasterizer(OnscreenSurface Function(OnscreenCanvasProvider) onscreenSurfaceCreateFn) - : _surfaceProvider = OnscreenSurfaceProvider(OnscreenCanvasProvider(), onscreenSurfaceCreateFn); - - final OnscreenSurfaceProvider _surfaceProvider; - - @override - @visibleForTesting - SurfaceProvider get surfaceProvider => _surfaceProvider; - - @override - MultiSurfaceViewRasterizer createViewRasterizer(EngineFlutterView view) { - return _viewRasterizers.putIfAbsent( - view, - () => MultiSurfaceViewRasterizer(view, this, _surfaceProvider), - ); - } - - final Map _viewRasterizers = - {}; - - @override - void dispose() { - for (final MultiSurfaceViewRasterizer viewRasterizer in _viewRasterizers.values) { - viewRasterizer.dispose(); - } - _viewRasterizers.clear(); - _surfaceProvider.dispose(); - } - - @override - void setResourceCacheMaxBytes(int bytes) { - _surfaceProvider.setSkiaResourceCacheMaxBytes(bytes); - } - - @override - Surface createPictureToImageSurface() { - return _surfaceProvider.createSurface(); - } -} - -class MultiSurfaceViewRasterizer extends ViewRasterizer { - MultiSurfaceViewRasterizer(super.view, this.rasterizer, this.surfaceProvider); - - final MultiSurfaceRasterizer rasterizer; - final OnscreenSurfaceProvider surfaceProvider; - - @override - late final DisplayCanvasFactory displayFactory = - DisplayCanvasFactory(createCanvas: surfaceProvider.createSurface); - - @override - Future prepareToDraw() { - return displayFactory.baseCanvas.setSize(currentFrameSize); - } - - Future rasterizeToCanvas(OnscreenSurface canvas, ui.Picture picture) { - canvas.setSize(currentFrameSize); - return canvas.rasterizeToCanvas(picture); - } - - @override - Future rasterize( - List displayCanvases, - List pictures, - FrameTimingRecorder? recorder, - ) async { - if (displayCanvases.length != pictures.length) { - throw ArgumentError('Called rasterize() with a different number of canvases and pictures.'); - } - recorder?.recordRasterStart(); - final rasterizeFutures = >[]; - for (var i = 0; i < displayCanvases.length; i++) { - rasterizeFutures.add(rasterizeToCanvas(displayCanvases[i] as OnscreenSurface, pictures[i])); - } - await Future.wait(rasterizeFutures); - recorder?.recordRasterFinish(); - } -} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/rasterizer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/rasterizer.dart index fc7493d0022..17d842eaed7 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/rasterizer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/rasterizer.dart @@ -3,7 +3,6 @@ // found in the LICENSE file. import 'dart:async'; -import 'package:meta/meta.dart'; import 'package:ui/src/engine.dart'; import 'package:ui/ui.dart' as ui; @@ -13,17 +12,11 @@ abstract class Rasterizer { /// Creates a [ViewRasterizer] for a given [view]. ViewRasterizer createViewRasterizer(EngineFlutterView view); - /// Creates a [Surface] which is to be used for [Picture.toImage] calls. - Surface createPictureToImageSurface(); - /// Sets the maximum size of the resource cache to [bytes]. void setResourceCacheMaxBytes(int bytes); /// Disposes this rasterizer and all [ViewRasterizer]s that it created. void dispose(); - - @visibleForTesting - SurfaceProvider get surfaceProvider; } /// Composites Flutter content into a [FlutterView]. Manages the creation of @@ -80,7 +73,7 @@ abstract class ViewRasterizer { final bitmapSize = BitmapSize.fromSize(frameSize); currentFrameSize = bitmapSize; - await prepareToDraw(); + prepareToDraw(); viewEmbedder.frameSize = currentFrameSize; final Frame compositorFrame = context.acquireFrame(viewEmbedder); @@ -94,7 +87,7 @@ abstract class ViewRasterizer { /// /// For example, in the [OffscreenCanvasRasterizer], this ensures the backing /// [OffscreenCanvas] is the correct size to draw the frame. - Future prepareToDraw(); + void prepareToDraw(); /// Rasterizes the given [pictures] into the [displayCanvases]. /// @@ -129,6 +122,7 @@ abstract class ViewRasterizer { /// Disposes this rasterizer. void dispose() { viewEmbedder.dispose(); + displayFactory.dispose(); } /// Clears the state. Used in tests. diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/surface.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/surface.dart deleted file mode 100644 index b807f7120ce..00000000000 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/compositing/surface.dart +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:typed_data'; - -import 'package:meta/meta.dart'; -import 'package:ui/ui.dart' as ui; - -import '../dom.dart'; -import '../util.dart'; -import 'canvas_provider.dart'; -import 'rasterizer.dart'; - -/// A class which provides and manages [Surface] objects. -abstract class SurfaceProvider { - SurfaceProvider(this._canvasProvider, this._surfaceCreateFn); - - final D _canvasProvider; - final C Function(D) _surfaceCreateFn; - - final List _createdSurfaces = []; - - C createSurface() { - final C surface = _surfaceCreateFn(_canvasProvider); - if (_resourceCacheMaxBytes != null) { - surface.setSkiaResourceCacheMaxBytes(_resourceCacheMaxBytes!); - } - _createdSurfaces.add(surface); - return surface; - } - - void dispose() { - for (final C surface in _createdSurfaces) { - surface.dispose(); - } - _createdSurfaces.clear(); - } - - int? _resourceCacheMaxBytes; - - void setSkiaResourceCacheMaxBytes(int bytes) { - _resourceCacheMaxBytes = bytes; - for (final C surface in _createdSurfaces) { - surface.setSkiaResourceCacheMaxBytes(bytes); - } - } -} - -/// A [SurfaceProvider] that creates [OffscreenSurface] objects. -class OffscreenSurfaceProvider extends SurfaceProvider { - OffscreenSurfaceProvider(super.canvasProvider, super.surfaceCreateFn); -} - -/// A [SurfaceProvider] that creates [OnscreenSurface] objects. -class OnscreenSurfaceProvider extends SurfaceProvider { - OnscreenSurfaceProvider(super.canvasProvider, super.surfaceCreateFn); -} - -/// The base interface for a rendering surface. -abstract class Surface { - /// Sets the size of the underlying canvas. - Future setSize(BitmapSize size); - - /// Converts a `ui.Image` into a `ByteData` object in the specified format. - Future rasterizeImage(ui.Image image, ui.ImageByteFormat format); - - /// Sets the maximum number of bytes for the GPU resource cache. - void setSkiaResourceCacheMaxBytes(int bytes); - - /// Discards the old graphics context and creates a new one using the - /// provided canvas object. - /// - /// This is called by the `SurfaceManager` in response to a - /// `webglcontextlost` event. - Future recreateContextForCanvas(DomEventTarget newCanvas); - - /// Disposes of the surface and its resources. - void dispose(); - - /// A [Future] which completes when the [Surface] is initialized and ready to - /// render pictures. - Future get initialized; - - /// The underlying canvas used to render the pixels. - DomCanvasImageSource get canvasImageSource; - - /// Rasterizes the given [picture] to this canvas. - Future rasterizeToCanvas(ui.Picture picture); - - @visibleForTesting - int get glContext; - - @visibleForTesting - Future triggerContextLoss(); - - @visibleForTesting - Future get handledContextLossEvent; -} - -/// A rendering surface that is optimized for producing `DomImageBitmap` objects. -/// -/// This surface is not attached to the DOM and is used for off-screen rendering. -abstract class OffscreenSurface extends Surface { - /// Rasterizes the given list of [pictures] into a list of `DomImageBitmap` - /// objects. - Future> rasterizeToImageBitmaps(List pictures); -} - -/// A rendering surface that is also a `DisplayCanvas`. -/// -/// This surface renders a picture directly to an on-screen canvas that is -/// part of the DOM. -abstract class OnscreenSurface extends Surface implements DisplayCanvas {} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart index e32c32d08a0..72224d4dbc8 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/dom.dart @@ -178,6 +178,12 @@ extension type DomWindow._(JSObject _) implements DomEventTarget { /// The Trusted Types API (when available). /// See: https://developer.mozilla.org/en-US/docs/Web/API/Trusted_Types_API external DomTrustedTypePolicyFactory? get trustedTypes; + + @JS('createImageBitmap') + external JSPromise _createImageBitmap(DomImageData source); + Future createImageBitmap(DomImageData source) { + return _createImageBitmap(source).toDart.then((JSAny? value) => value! as DomImageBitmap); + } } typedef DomRequestAnimationFrameCallback = void Function(JSNumber highResTime); @@ -214,9 +220,6 @@ Future createImageBitmap( JSAny source, [ ({int x, int y, int width, int height})? bounds, ]) { - if (debugThrowOnCreateImageBitmapIfDisabled && !browserSupportsCreateImageBitmap) { - throw UnsupportedError('createImageBitmap is not supported in this browser'); - } JSPromise jsPromise; if (bounds != null) { jsPromise = _createImageBitmap(source, bounds.x, bounds.y, bounds.width, bounds.height); @@ -430,7 +433,6 @@ extension type DomElement._(JSObject _) implements DomNode { external String? getAttribute(String attributeName); external DomRect getBoundingClientRect(); external void prepend(DomNode node); - external void replaceWith(DomNode node); external DomElement? querySelector(String selectors); external DomElement? closest(String selectors); external bool matches(String selectors); @@ -802,7 +804,7 @@ extension type DomPerformanceEntry._(JSObject _) implements JSObject {} extension type DomPerformanceMeasure._(JSObject _) implements DomPerformanceEntry {} @JS('HTMLCanvasElement') -extension type DomHTMLCanvasElement._(JSObject _) implements DomHTMLElement, DomCanvasImageSource { +extension type DomHTMLCanvasElement._(JSObject _) implements DomHTMLElement { external double? width; external double? height; @@ -2020,7 +2022,7 @@ DomHTMLLabelElement createDomHTMLLabelElement() => domDocument.createElement('label') as DomHTMLLabelElement; @JS('OffscreenCanvas') -extension type DomOffscreenCanvas._(JSObject _) implements DomEventTarget, DomCanvasImageSource { +extension type DomOffscreenCanvas._(JSObject _) implements DomEventTarget { external DomOffscreenCanvas(int width, int height); external double? height; @@ -2605,16 +2607,14 @@ external JSAny? get _offscreenCanvasConstructor; bool browserSupportsOffscreenCanvas = _offscreenCanvasConstructor != null; +@JS('window.createImageBitmap') +external JSAny? get _createImageBitmapFunction; + /// Set to `true` to disable `createImageBitmap` support. Used in tests. -@visibleForTesting bool debugDisableCreateImageBitmapSupport = false; -/// Set to `true` to throw an error if `createImageBitmap` is disabled. Used in tests. -@visibleForTesting -bool debugThrowOnCreateImageBitmapIfDisabled = false; - bool get browserSupportsCreateImageBitmap => - domWindow.has('createImageBitmap') && + _createImageBitmapFunction != null && !isChrome110OrOlder && !debugDisableCreateImageBitmapSupport; diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_painting.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_painting.dart index d8653dcaffc..ab32e1d5fb7 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_painting.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_painting.dart @@ -36,9 +36,6 @@ abstract class LayerPicture implements ui.Picture { /// /// The copy points to the same underlying Skia picture as this picture. LayerPicture clone(); - - /// Returns `true` if the picture has been disposed. - bool get isDisposed; } /// A [ui.PictureRecorder] which allows callers to know if it has been disposed. diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_visitor.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_visitor.dart index 7571c04e9f5..db78649c56d 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_visitor.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/layer/layer_visitor.dart @@ -408,12 +408,6 @@ class MeasureVisitor extends LayerVisitor { @override void visitPicture(PictureLayer picture) { assert(picture.needsPainting); - if (picture.picture.isDisposed) { - // The picture layer was disposed before the picture could be painted. - // Just ignore it then. - picture.isCulled = true; - return; - } measuringCanvas.save(); measuringCanvas.translate(picture.offset.dx, picture.offset.dy); @@ -678,13 +672,6 @@ class PaintVisitor extends LayerVisitor { void visitPicture(PictureLayer picture) { assert(picture.needsPainting); - if (picture.picture.isDisposed) { - // The picture layer was disposed before the picture could be painted. - // Just ignore it then. - picture.isCulled = true; - return; - } - // For each shader mask this picture is a child of, record that it needs // to have the shader mask applied to it. for (final ShaderMaskEngineLayer shaderMask in shaderMaskStack) { diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/renderer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/renderer.dart index 12f4745704d..c250457aad7 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/renderer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/renderer.dart @@ -44,9 +44,6 @@ abstract class Renderer { late Rasterizer rasterizer; - /// A surface used specifically for `Picture.toImage`. - Surface get pictureToImageSurface; - /// Resets the [Rasterizer] to the default value. Used in tests. @visibleForTesting void debugResetRasterizer(); @@ -339,8 +336,11 @@ abstract class Renderer { void dispose() { _onViewCreatedListener.cancel(); _onViewDisposedListener.cancel(); + for (final ViewRasterizer rasterizer in rasterizers.values) { + rasterizer.dispose(); + } + rasterizers.clear(); rasterizer.dispose(); - pictureToImageSurface.dispose(); } /// Clears the state of this renderer. Used in tests. diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart index b3667e9eb3d..2587ee04216 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl.dart @@ -14,6 +14,7 @@ export 'skwasm_impl/filters.dart'; export 'skwasm_impl/font_collection.dart'; export 'skwasm_impl/image.dart'; export 'skwasm_impl/memory.dart'; +export 'skwasm_impl/offscreen_canvas_rasterizer.dart'; export 'skwasm_impl/paint.dart'; export 'skwasm_impl/paragraph.dart'; export 'skwasm_impl/path.dart'; diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart index ce82545e686..c5aae55f3a5 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart @@ -21,7 +21,7 @@ class SkwasmBrowserImageDecoder extends BrowserImageDecoder { ui.Image generateImageFromVideoFrame(VideoFrame frame) { final int width = frame.displayWidth.toInt(); final int height = frame.displayHeight.toInt(); - final surface = renderer.pictureToImageSurface as SkwasmSurface; + final SkwasmSurface surface = (renderer as SkwasmRenderer).surface; return SkwasmImage(imageCreateFromTextureSource(frame, width, height, surface.handle)); } } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart index 77196fbcaef..e0210d9dbbb 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/image.dart @@ -60,11 +60,11 @@ class SkwasmImage extends SkwasmObjectWrapper implements ui.Image { final canvas = ui.Canvas(recorder); canvas.drawImage(this, ui.Offset.zero, ui.Paint()); final picture = recorder.endRecording() as SkwasmPicture; - final surface = renderer.pictureToImageSurface as SkwasmSurface; - await surface.setSize(BitmapSize(width, height)); - final DomImageBitmap bitmap = (await surface.rasterizeToImageBitmaps([ - picture, - ])).single; + final DomImageBitmap bitmap = (await (renderer as SkwasmRenderer).surface.renderPictures( + [picture], + picture.cullRect.width.ceil(), + picture.cullRect.height.ceil(), + )).imageBitmaps.single; final DomOffscreenCanvas offscreenCanvas = createDomOffscreenCanvas( bitmap.width, bitmap.height, @@ -80,7 +80,7 @@ class SkwasmImage extends SkwasmObjectWrapper implements ui.Image { context.transferFromImageBitmap(null); return ByteData.view(arrayBuffer.toDart); } else { - return renderer.pictureToImageSurface.rasterizeImage(this, format); + return (renderer as SkwasmRenderer).surface.rasterizeImage(this, format); } } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/offscreen_canvas_rasterizer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/offscreen_canvas_rasterizer.dart new file mode 100644 index 00000000000..a0fcd2283ba --- /dev/null +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/offscreen_canvas_rasterizer.dart @@ -0,0 +1,83 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:ui/src/engine.dart'; +import 'package:ui/ui.dart' as ui; + +import '../skwasm_impl.dart'; + +/// A [Rasterizer] that uses a single GL context in an OffscreenCanvas to do +/// all the rendering. It transfers bitmaps created in the OffscreenCanvas to +/// one or many on-screen elements to actually display the scene. +class SkwasmOffscreenCanvasRasterizer extends Rasterizer { + SkwasmOffscreenCanvasRasterizer(this.offscreenSurface); + + /// This is an SkSurface backed by an OffScreenCanvas. This single Surface is + /// used to render to many RenderCanvases to produce the rendered scene. + final SkwasmSurface offscreenSurface; + + @override + SkwasmOffscreenCanvasViewRasterizer createViewRasterizer(EngineFlutterView view) { + return _viewRasterizers.putIfAbsent( + view, + () => SkwasmOffscreenCanvasViewRasterizer(view, this), + ); + } + + final Map _viewRasterizers = + {}; + + @override + void setResourceCacheMaxBytes(int bytes) { + offscreenSurface.setSkiaResourceCacheMaxBytes(bytes); + } + + @override + void dispose() { + offscreenSurface.dispose(); + for (final SkwasmOffscreenCanvasViewRasterizer viewRasterizer in _viewRasterizers.values) { + viewRasterizer.dispose(); + } + } +} + +class SkwasmOffscreenCanvasViewRasterizer extends ViewRasterizer { + SkwasmOffscreenCanvasViewRasterizer(super.view, this.rasterizer); + + final SkwasmOffscreenCanvasRasterizer rasterizer; + + @override + final DisplayCanvasFactory displayFactory = DisplayCanvasFactory( + createCanvas: () => RenderCanvas(), + ); + + @override + void prepareToDraw() { + // No need to do anything here. Skwasm sizes the surface in the `rasterize` + // call below. + } + + @override + Future rasterize( + List displayCanvases, + List pictures, + FrameTimingRecorder? recorder, + ) async { + if (displayCanvases.length != pictures.length) { + throw ArgumentError('Called rasterize() with a different number of canvases and pictures.'); + } + final RenderResult renderResult = await rasterizer.offscreenSurface.renderPictures( + pictures.cast(), + currentFrameSize.width, + currentFrameSize.height, + ); + recorder?.recordRasterStart(renderResult.rasterStartMicros); + recorder?.recordRasterFinish(renderResult.rasterEndMicros); + for (var i = 0; i < displayCanvases.length; i++) { + final renderCanvas = displayCanvases[i] as RenderCanvas; + final DomImageBitmap bitmap = renderResult.imageBitmaps[i]; + renderCanvas.render(bitmap); + } + } +} diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/picture.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/picture.dart index 095193d8ed4..791d2edc2a0 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/picture.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/picture.dart @@ -49,14 +49,6 @@ class SkwasmPicture extends SkwasmObjectWrapper implements LayerPict pictureRef(handle); return SkwasmPicture.fromHandle(handle, isClone: true); } - - @override - String toString() { - return 'SkwasmPicture(${handle.address})'; - } - - @override - bool get isDisposed => debugDisposed; } class SkwasmPictureRecorder extends SkwasmObjectWrapper diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart index 915c8cc4d0b..f5f60964eaf 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_surface.dart @@ -5,10 +5,7 @@ @DefaultAsset('skwasm') library skwasm_impl; -import 'dart:_wasm'; import 'dart:ffi'; -import 'dart:js_interop'; - import 'package:ui/src/engine/skwasm/skwasm_impl.dart'; final class RawSurface extends Opaque {} @@ -24,25 +21,9 @@ typedef CallbackId = int; @Native(symbol: 'surface_create', isLeaf: true) external SurfaceHandle surfaceCreate(); -// We use a wasm import directly here instead of @Native since this uses an externref -// in the function signature. -CallbackId surfaceSetCanvas(SurfaceHandle handle, JSAny canvas) => - surfaceSetCanvasImpl(handle.address.toWasmI32(), externRefForJSAny(canvas)).toIntUnsigned(); -@pragma('wasm:import', 'skwasm.surface_setCanvas') -external WasmI32 surfaceSetCanvasImpl(WasmI32 surfaceHandle, WasmExternRef? frame); - -@Native(symbol: 'surface_setSize', isLeaf: true) -external CallbackId surfaceSetSize(SurfaceHandle surface, int width, int height); - @Native(symbol: 'surface_getThreadId', isLeaf: true) external int surfaceGetThreadId(SurfaceHandle handle); -@Native(symbol: 'surface_getGlContext', isLeaf: true) -external int surfaceGetGlContext(SurfaceHandle handle); - -@Native(symbol: 'surface_triggerContextLoss', isLeaf: true) -external CallbackId surfaceTriggerContextLoss(SurfaceHandle handle); - @Native( symbol: 'surface_setCallbackHandler', isLeaf: true, @@ -58,13 +39,15 @@ external void surfaceDestroy(SurfaceHandle surface); ) external void surfaceSetResourceCacheLimitBytes(SurfaceHandle surface, int bytes); -@Native, Int)>( +@Native, Int, Int, Int)>( symbol: 'surface_renderPictures', isLeaf: true, ) external CallbackId surfaceRenderPictures( SurfaceHandle surface, Pointer picture, + int width, + int height, int count, ); diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart index 0acb7a0b94f..0777885f2d7 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart @@ -14,6 +14,8 @@ import 'package:ui/ui.dart' as ui; import 'package:ui/ui_web/src/ui_web.dart' as ui_web; class SkwasmRenderer extends Renderer { + late SkwasmSurface surface; + bool get isMultiThreaded => skwasmIsMultiThreaded(); bool get isWimp => skwasmIsWimp(); @@ -319,10 +321,9 @@ class SkwasmRenderer extends Renderer { } @override - FutureOr initialize() async { - rasterizer = OffscreenCanvasRasterizer( - (OffscreenCanvasProvider canvasProvider) => SkwasmSurface(canvasProvider), - ); + FutureOr initialize() { + surface = SkwasmSurface(); + rasterizer = SkwasmOffscreenCanvasRasterizer(surface); return super.initialize(); } @@ -448,7 +449,7 @@ class SkwasmRenderer extends Renderer { imageSource, imageSource.width, imageSource.height, - (pictureToImageSurface as SkwasmSurface).handle, + surface.handle, ), ); } @@ -469,12 +470,7 @@ class SkwasmRenderer extends Renderer { ))).toJSAnyShallow; } return SkwasmImage( - imageCreateFromTextureSource( - textureSource as JSObject, - width, - height, - (pictureToImageSurface as SkwasmSurface).handle, - ), + imageCreateFromTextureSource(textureSource as JSObject, width, height, surface.handle), ); } @@ -530,11 +526,6 @@ class SkwasmRenderer extends Renderer { @override void debugResetRasterizer() { - rasterizer = OffscreenCanvasRasterizer( - (OffscreenCanvasProvider canvasProvider) => SkwasmSurface(canvasProvider), - ); + rasterizer = SkwasmOffscreenCanvasRasterizer(surface); } - - @override - Surface get pictureToImageSurface => (rasterizer as OffscreenCanvasRasterizer).offscreenSurface; } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart index a9c1f64efdc..d8aab2f832c 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/surface.dart @@ -45,31 +45,26 @@ class SkwasmCallbackHandler { static SkwasmCallbackHandler instance = SkwasmCallbackHandler._(); final OnRenderCallbackHandle callbackPointer; - final Map, Zone)> _pendingCallbacks = - , Zone)>{}; + final Map> _pendingCallbacks = >{}; // Returns a future that will resolve when Skwasm calls back with the given callbackID Future registerCallback(int callbackId) { final completer = Completer(); - _pendingCallbacks[callbackId] = (completer, Zone.current); + _pendingCallbacks[callbackId] = completer; return completer.future; } void handleCallback(WasmI32 callbackId, WasmI32 context, WasmExternRef? jsContext) { - final (Completer, Zone) record = _pendingCallbacks.remove(callbackId.toIntUnsigned())!; - final Completer completer = record.$1; - final Zone zone = record.$2; - zone.run(() { - // Skwasm can either callback with a JS object (an externref) or it can call back - // with a simple integer, which usually refers to a pointer on its heap. In order - // to coerce these into a single type, we just make the completers take a JSAny - // that either contains the JS object or a JSNumber that contains the integer value. - if (!jsContext.isNull) { - completer.complete(jsContext!.toJS); - } else { - completer.complete(context.toIntUnsigned().toJS); - } - }); + // Skwasm can either callback with a JS object (an externref) or it can call back + // with a simple integer, which usually refers to a pointer on its heap. In order + // to coerce these into a single type, we just make the completers take a JSAny + // that either contains the JS object or a JSNumber that contains the integer value. + final Completer completer = _pendingCallbacks.remove(callbackId.toIntUnsigned())!; + if (!jsContext.isNull) { + completer.complete(jsContext!.toJS); + } else { + completer.complete(context.toIntUnsigned().toJS); + } } } @@ -79,72 +74,51 @@ typedef RenderResult = ({ int rasterEndMicros, }); -class SkwasmSurface implements OffscreenSurface { - factory SkwasmSurface(OffscreenCanvasProvider canvasProvider) { - final SurfaceHandle handle = withStackScope((StackScope scope) { +class SkwasmSurface { + factory SkwasmSurface() { + final SurfaceHandle surfaceHandle = withStackScope((StackScope scope) { return surfaceCreate(); }); - final surface = SkwasmSurface._fromHandle(handle, canvasProvider); + final surface = SkwasmSurface._fromHandle(surfaceHandle); + surface._initialize(); return surface; } - SkwasmSurface._fromHandle(this.handle, this._canvasProvider) - : _initializedCompleter = Completer() { - surfaceSetCallbackHandler(handle, SkwasmCallbackHandler.instance.callbackPointer); - _canvas = _canvasProvider.acquireCanvas(const BitmapSize(1, 1), onContextLost: onContextLost); - _initialize(); - } + SkwasmSurface._fromHandle(this.handle) : threadId = surfaceGetThreadId(handle); + final SurfaceHandle handle; - final OffscreenCanvasProvider _canvasProvider; - late DomOffscreenCanvas _canvas; - late SurfaceHandle handle; - double _currentDevicePixelRatio = -1; - BitmapSize _currentSize = const BitmapSize(1, 1); - Completer _initializedCompleter; - late Completer? _handledContextLostEvent; - - void onContextLost() { - if (!_initializedCompleter.isCompleted) { - _initializedCompleter.complete(); - } - _initializedCompleter = Completer(); - _handledContextLostEvent?.complete(); - final DomOffscreenCanvas newCanvas = _canvasProvider.acquireCanvas( - _currentSize, - onContextLost: onContextLost, - ); - recreateContextForCanvas(newCanvas); - } + final int threadId; void _initialize() { - final CallbackId callbackId = surfaceSetCanvas(handle, _canvas); - - SkwasmCallbackHandler.instance.registerCallback(callbackId).then((JSAny contextLostCallbackId) { - // The context may have been lost before the Surface finished - // initializing. - if (!_initializedCompleter.isCompleted) { - _initializedCompleter.complete(); - } - // Once we have transferred control of the canvas to the Skwasm Surface, - // the reference to the _canvas is no longer valid and any listeners - // attached to it will never fire. Inform the CanvasProvider that it - // should release its reference to the canvas and unregister any listeners - // attached to it. - _canvasProvider.releaseCanvas(_canvas); - SkwasmCallbackHandler.instance - .registerCallback((contextLostCallbackId as JSNumber).toDartInt) - .then((_) { - onContextLost(); - }); - }); + surfaceSetCallbackHandler(handle, SkwasmCallbackHandler.instance.callbackPointer); } - @override - Future rasterizeImage(ui.Image image, ui.ImageByteFormat format) async { - await initialized; - // Cast [image] to [SkwasmImage]. - image as SkwasmImage; - await setSize(BitmapSize(image.width, image.height)); + Future renderPictures(List pictures, int width, int height) => + withStackScope((StackScope scope) async { + final Pointer pictureHandles = scope + .allocPointerArray(pictures.length) + .cast(); + for (var i = 0; i < pictures.length; i++) { + pictureHandles[i] = pictures[i].handle; + } + final int callbackId = surfaceRenderPictures( + handle, + pictureHandles, + width, + height, + pictures.length, + ); + final rasterResult = + (await SkwasmCallbackHandler.instance.registerCallback(callbackId)) as RasterResult; + final RenderResult result = ( + imageBitmaps: rasterResult.imageBitmaps.toDart.cast(), + rasterStartMicros: (rasterResult.rasterStartMilliseconds * 1000).toInt(), + rasterEndMicros: (rasterResult.rasterEndMilliseconds * 1000).toInt(), + ); + return result; + }); + + Future rasterizeImage(SkwasmImage image, ui.ImageByteFormat format) async { final int callbackId = surfaceRasterizeImage(handle, image.handle, format.index); final int context = (await SkwasmCallbackHandler.instance.registerCallback(callbackId) as JSNumber).toDartInt; @@ -159,88 +133,11 @@ class SkwasmSurface implements OffscreenSurface { return ByteData.sublistView(output); } - @override void setSkiaResourceCacheMaxBytes(int bytes) { surfaceSetResourceCacheLimitBytes(handle, bytes); } - @override void dispose() { surfaceDestroy(handle); } - - @override - Future> rasterizeToImageBitmaps(List pictures) => - withStackScope((StackScope scope) async { - await initialized; - final Pointer pictureHandles = scope - .allocPointerArray(pictures.length) - .cast(); - for (var i = 0; i < pictures.length; i++) { - pictureHandles[i] = (pictures[i] as SkwasmPicture).handle; - } - final int callbackId = surfaceRenderPictures(handle, pictureHandles, pictures.length); - final rasterResult = - (await SkwasmCallbackHandler.instance.registerCallback(callbackId)) as RasterResult; - final RenderResult result = ( - imageBitmaps: rasterResult.imageBitmaps.toDart.cast(), - rasterStartMicros: (rasterResult.rasterStartMilliseconds * 1000).toInt(), - rasterEndMicros: (rasterResult.rasterEndMilliseconds * 1000).toInt(), - ); - return result.imageBitmaps; - }); - - @override - Future recreateContextForCanvas(DomEventTarget newCanvas) async { - _canvas = newCanvas as DomOffscreenCanvas; - _initialize(); - await initialized; - final BitmapSize lastSize = _currentSize; - // Reset _currentSize to force `setSize` to actually size the underlying - // Surface. - _currentSize = const BitmapSize(1, 1); - await setSize(lastSize); - } - - @override - Future setSize(BitmapSize size) async { - await initialized; - final double devicePixelRatio = EngineFlutterDisplay.instance.devicePixelRatio; - if (_currentSize == size && devicePixelRatio == _currentDevicePixelRatio) { - return; - } - _currentDevicePixelRatio = devicePixelRatio; - _currentSize = size; - final int callbackId = surfaceSetSize(handle, size.width, size.height); - await SkwasmCallbackHandler.instance.registerCallback(callbackId); - } - - @override - int get glContext => surfaceGetGlContext(handle); - - @override - Future get initialized => _initializedCompleter.future; - - @override - Future triggerContextLoss() async { - _handledContextLostEvent = Completer(); - final int callbackId = surfaceTriggerContextLoss(handle); - await SkwasmCallbackHandler.instance.registerCallback(callbackId); - } - - @override - Future get handledContextLossEvent => _handledContextLostEvent!.future; - - // TODO(harryterkelsen): Implement this to support MultiSurfaceRasterizer in - // Skwasm. - @override - DomCanvasImageSource get canvasImageSource => - throw StateError('canvasImageSource is not supported for SkwasmSurface'); - - // TODO(harryterkelsen): Implement this to support MultiSurfaceRasterizer in - // Skwasm. - @override - Future rasterizeToCanvas(ui.Picture picture) { - throw StateError('rasterizeToCanvas is not supported for SkwasmSurface'); - } } diff --git a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart index 2bea5e1c206..a8abdd5f17b 100644 --- a/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart +++ b/engine/src/flutter/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart @@ -326,8 +326,4 @@ class SkwasmRenderer extends Renderer { void debugResetRasterizer() { throw UnimplementedError('Skwasm not implemented on this platform.'); } - - @override - Surface get pictureToImageSurface => - throw UnimplementedError('Skwasm not implemented on this platform.'); } diff --git a/engine/src/flutter/lib/web_ui/test/canvaskit/bitmap_less_rendering_test.dart b/engine/src/flutter/lib/web_ui/test/canvaskit/bitmap_less_rendering_test.dart deleted file mode 100644 index 2f21cd8e53a..00000000000 --- a/engine/src/flutter/lib/web_ui/test/canvaskit/bitmap_less_rendering_test.dart +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:test/bootstrap/browser.dart'; -import 'package:test/test.dart'; -import 'package:ui/src/engine.dart'; -import 'package:ui/ui.dart' as ui; - -import 'common.dart'; - -void main() { - internalBootstrapBrowserTest(() => testMain); -} - -void testMain() { - group('Bitmap-less rendering', () { - setUpCanvasKitTest(withImplicitView: true); - - setUpAll(() { - debugDisableCreateImageBitmapSupport = true; - }); - - tearDownAll(() { - debugDisableCreateImageBitmapSupport = false; - }); - - test( - 'throws when createImageBitmap is not supported but rasterizeToImageBitmaps is called', - () async { - final surface = CkOffscreenSurface(OffscreenCanvasProvider()); - final pictures = []; - pictures.add(_createPicture()); - - expect(() => surface.rasterizeToImageBitmaps(pictures), throwsUnsupportedError); - }, - ); - - test('does not throw when rasterizing with a Rasterizer', () async { - final builder = ui.SceneBuilder(); - builder.addPicture(ui.Offset.zero, _createPicture()); - final ui.Scene scene = builder.build(); - final LayerTree layerTree = (scene as LayerScene).layerTree; - - final rasterizer = OffscreenCanvasRasterizer( - (OffscreenCanvasProvider canvasProvider) => CkOffscreenSurface(canvasProvider), - ); - - final OffscreenCanvasViewRasterizer viewRasterizer = rasterizer.createViewRasterizer( - EnginePlatformDispatcher.instance.implicitView!, - ); - await viewRasterizer.draw(layerTree, null); - }); - }); -} - -ui.Picture _createPicture() { - final recorder = ui.PictureRecorder(); - final canvas = ui.Canvas(recorder); - canvas.drawRect(const ui.Rect.fromLTRB(0, 0, 10, 10), ui.Paint()); - return recorder.endRecording(); -} diff --git a/engine/src/flutter/lib/web_ui/test/canvaskit/image_test.dart b/engine/src/flutter/lib/web_ui/test/canvaskit/image_test.dart index ef364930429..7fd1895e2c0 100644 --- a/engine/src/flutter/lib/web_ui/test/canvaskit/image_test.dart +++ b/engine/src/flutter/lib/web_ui/test/canvaskit/image_test.dart @@ -65,7 +65,7 @@ void testMain() { test('CkImage does not close image source too early', () async { final ImageSource imageSource = ImageBitmapImageSource( - await createImageBitmap(createBlankDomImageData(4, 4)), + await domWindow.createImageBitmap(createBlankDomImageData(4, 4)), ); final SkImage skImage1 = canvasKit.MakeAnimatedImageFromEncoded( diff --git a/engine/src/flutter/lib/web_ui/test/canvaskit/renderer_test.dart b/engine/src/flutter/lib/web_ui/test/canvaskit/renderer_test.dart index d0ddb103439..255989cb12b 100644 --- a/engine/src/flutter/lib/web_ui/test/canvaskit/renderer_test.dart +++ b/engine/src/flutter/lib/web_ui/test/canvaskit/renderer_test.dart @@ -38,14 +38,6 @@ class TestRasterizer extends Rasterizer { List treesRenderedInView(EngineFlutterView view) { return viewRasterizers[view]!.treesRendered; } - - @override - Surface createPictureToImageSurface() { - throw UnimplementedError(); - } - - @override - SurfaceProvider get surfaceProvider => throw UnimplementedError(); } class TestViewRasterizer extends ViewRasterizer { @@ -57,8 +49,8 @@ class TestViewRasterizer extends ViewRasterizer { DisplayCanvasFactory get displayFactory => throw UnimplementedError(); @override - Future prepareToDraw() { - return Future.value(); + void prepareToDraw() { + // Do nothing } @override diff --git a/engine/src/flutter/lib/web_ui/test/canvaskit/surface_test.dart b/engine/src/flutter/lib/web_ui/test/canvaskit/surface_test.dart index 2973716d4d7..61456810085 100644 --- a/engine/src/flutter/lib/web_ui/test/canvaskit/surface_test.dart +++ b/engine/src/flutter/lib/web_ui/test/canvaskit/surface_test.dart @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; + import 'package:test/bootstrap/browser.dart'; import 'package:test/test.dart'; import 'package:ui/src/engine.dart'; @@ -20,85 +23,171 @@ void testMain() { EngineFlutterDisplay.instance.debugOverrideDevicePixelRatio(1.0); }); - test('CkOnscreenSurface resizes correctly', () async { - final surfaceProvider = OnscreenSurfaceProvider( - OnscreenCanvasProvider(), - (OnscreenCanvasProvider canvasProvider) => CkOnscreenSurface(canvasProvider), - ); - final surface = surfaceProvider.createSurface() as CkOnscreenSurface; - await surface.initialized; - final canvas = surface.hostElement.children.single as DomHTMLCanvasElement; - ui.Size canvasSize = getCssSize(canvas); - - // Expect size 1x1 initially. - expect(canvas.width, 1); - expect(canvas.height, 1); - expect(canvasSize.width, 1); - expect(canvasSize.height, 1); - - await surface.setSize(const BitmapSize(9, 19)); - canvasSize = getCssSize(canvas); + test('Surface allocates canvases efficiently', () { + final surface = Surface(); + surface.createOrUpdateSurface(const BitmapSize(9, 19)); + final CkSurface originalSurface = surface.debugGetCkSurface()!; + final DomOffscreenCanvas original = surface.debugGetOffscreenCanvas()!; // Expect exact requested dimensions. - expect(canvas.width, 9); - expect(canvas.height, 19); + expect(original.width, 9); + expect(original.height, 19); + expect(originalSurface.width(), 9); + expect(originalSurface.height(), 19); + + // Shrinking causes the surface to create a new canvas with the exact + // size requested. + surface.createOrUpdateSurface(const BitmapSize(5, 15)); + final CkSurface shrunkSurface = surface.debugGetCkSurface()!; + final DomOffscreenCanvas shrunk = surface.debugGetOffscreenCanvas()!; + expect(shrunk, same(original)); + expect(shrunkSurface, isNot(same(originalSurface))); + expect(shrunkSurface.width(), 5); + expect(shrunkSurface.height(), 15); + + // The first increase will allocate a new surface to exactly the + // requested size. + surface.createOrUpdateSurface(const BitmapSize(10, 20)); + final CkSurface firstIncreaseSurface = surface.debugGetCkSurface()!; + final DomOffscreenCanvas firstIncrease = surface.debugGetOffscreenCanvas()!; + expect(firstIncrease, same(original)); + expect(firstIncreaseSurface, isNot(same(shrunkSurface))); + + // Expect exact dimensions + expect(firstIncrease.width, 10); + expect(firstIncrease.height, 20); + expect(firstIncreaseSurface.width(), 10); + expect(firstIncreaseSurface.height(), 20); + + // Subsequent increases within 40% will still allocate a new canvas. + surface.createOrUpdateSurface(const BitmapSize(11, 22)); + final CkSurface secondIncreaseSurface = surface.debugGetCkSurface()!; + final DomOffscreenCanvas secondIncrease = surface.debugGetOffscreenCanvas()!; + expect(secondIncrease, same(firstIncrease)); + expect(secondIncreaseSurface, isNot(same(firstIncreaseSurface))); + expect(secondIncreaseSurface.width(), 11); + expect(secondIncreaseSurface.height(), 22); + + // Increases beyond the 40% limit will cause a new allocation. + surface.createOrUpdateSurface(const BitmapSize(20, 40)); + final CkSurface hugeSurface = surface.debugGetCkSurface()!; + final DomOffscreenCanvas huge = surface.debugGetOffscreenCanvas()!; + expect(huge, same(secondIncrease)); + expect(hugeSurface, isNot(same(secondIncreaseSurface))); + + // Also exactly-allocated + expect(huge.width, 20); + expect(huge.height, 40); + expect(hugeSurface.width(), 20); + expect(hugeSurface.height(), 40); + + // Shrink again. Create a new surface. + surface.createOrUpdateSurface(const BitmapSize(5, 15)); + final CkSurface shrunkSurface2 = surface.debugGetCkSurface()!; + final DomOffscreenCanvas shrunk2 = surface.debugGetOffscreenCanvas()!; + expect(shrunk2, same(huge)); + expect(shrunkSurface2, isNot(same(hugeSurface))); + expect(shrunkSurface2.width(), 5); + expect(shrunkSurface2.height(), 15); + + // Doubling the DPR should halve the CSS width, height, and translation of the canvas. + // This tests https://github.com/flutter/flutter/issues/77084 + EngineFlutterDisplay.instance.debugOverrideDevicePixelRatio(2.0); + surface.createOrUpdateSurface(const BitmapSize(5, 15)); + final CkSurface dpr2Surface = surface.debugGetCkSurface()!; + final DomOffscreenCanvas dpr2Canvas = surface.debugGetOffscreenCanvas()!; + expect(dpr2Canvas, same(huge)); + expect(dpr2Surface, isNot(same(hugeSurface))); + expect(dpr2Surface.width(), 5); + expect(dpr2Surface.height(), 15); + + // Skipping on Firefox for now since Firefox headless doesn't support WebGL + // This causes issues in the test since we create a Canvas-backed surface, + // which cannot be a different size from the canvas. + // TODO(hterkelsen): See if we can give a custom size for software + // surfaces. + }, skip: isFirefox || !Surface.offscreenCanvasSupported); + + test('Surface used as DisplayCanvas resizes correctly', () { + final surface = Surface(isDisplayCanvas: true); + + surface.createOrUpdateSurface(const BitmapSize(9, 19)); + final DomHTMLCanvasElement original = getDisplayCanvas(surface); + ui.Size canvasSize = getCssSize(surface); + + // Expect exact requested dimensions. + expect(original.width, 9); + expect(original.height, 19); expect(canvasSize.width, 9); expect(canvasSize.height, 19); // Shrinking causes us to resize the canvas. - await surface.setSize(const BitmapSize(5, 15)); - canvasSize = getCssSize(canvas); - expect(canvas.width, 5); - expect(canvas.height, 15); + surface.createOrUpdateSurface(const BitmapSize(5, 15)); + final DomHTMLCanvasElement shrunk = getDisplayCanvas(surface); + canvasSize = getCssSize(surface); + expect(shrunk.width, 5); + expect(shrunk.height, 15); expect(canvasSize.width, 5); expect(canvasSize.height, 15); // Increasing the size causes us to resize the canvas. - await surface.setSize(const BitmapSize(10, 20)); - canvasSize = getCssSize(canvas); + surface.createOrUpdateSurface(const BitmapSize(10, 20)); + final DomHTMLCanvasElement firstIncrease = getDisplayCanvas(surface); + canvasSize = getCssSize(surface); + + expect(firstIncrease, same(original)); // Expect exact dimensions - expect(canvas.width, 10); - expect(canvas.height, 20); + expect(firstIncrease.width, 10); + expect(firstIncrease.height, 20); expect(canvasSize.width, 10); expect(canvasSize.height, 20); // Subsequent increases also cause canvas resizing. - await surface.setSize(const BitmapSize(11, 22)); - canvasSize = getCssSize(canvas); + surface.createOrUpdateSurface(const BitmapSize(11, 22)); + final DomHTMLCanvasElement secondIncrease = getDisplayCanvas(surface); + canvasSize = getCssSize(surface); - expect(canvas.width, 11); - expect(canvas.height, 22); + expect(secondIncrease, same(firstIncrease)); + expect(secondIncrease.width, 11); + expect(secondIncrease.height, 22); expect(canvasSize.width, 11); expect(canvasSize.height, 22); - // Increases beyond the 40% limit will cause a canvas resize. STATIC_ASSERT_FOR_WEB - await surface.setSize(const BitmapSize(20, 40)); - canvasSize = getCssSize(canvas); + // Increases beyond the 40% limit will cause a canvas resize. + surface.createOrUpdateSurface(const BitmapSize(20, 40)); + final DomHTMLCanvasElement huge = getDisplayCanvas(surface); + canvasSize = getCssSize(surface); + + expect(huge, same(secondIncrease)); // Also exact - expect(canvas.width, 20); - expect(canvas.height, 40); + expect(huge.width, 20); + expect(huge.height, 40); expect(canvasSize.width, 20); expect(canvasSize.height, 40); // Shrink again. Resize the canvas. - await surface.setSize(const BitmapSize(5, 15)); - canvasSize = getCssSize(canvas); + surface.createOrUpdateSurface(const BitmapSize(5, 15)); + final DomHTMLCanvasElement shrunk2 = getDisplayCanvas(surface); + canvasSize = getCssSize(surface); - expect(canvas.width, 5); - expect(canvas.height, 15); + expect(shrunk2, same(huge)); + expect(shrunk2.width, 5); + expect(shrunk2.height, 15); expect(canvasSize.width, 5); expect(canvasSize.height, 15); // Doubling the DPR should halve the CSS width, height, and translation of the canvas. // This tests https://github.com/flutter/flutter/issues/77084 EngineFlutterDisplay.instance.debugOverrideDevicePixelRatio(2.0); - await surface.setSize(const BitmapSize(5, 15)); - canvasSize = getCssSize(canvas); + surface.createOrUpdateSurface(const BitmapSize(5, 15)); + final DomHTMLCanvasElement dpr2Canvas = getDisplayCanvas(surface); + canvasSize = getCssSize(surface); - expect(canvas.width, 5); - expect(canvas.height, 15); + expect(dpr2Canvas, same(huge)); + expect(dpr2Canvas.width, 5); + expect(dpr2Canvas.height, 15); // Canvas is half the size in logical pixels because device pixel ratio is // 2.0. expect(canvasSize.width, 2.5); @@ -106,45 +195,160 @@ void testMain() { // Skip on wasm since same() doesn't work for JSValues. }, skip: isWasm); - test('CkOnscreenSurface falls back to software rendering', () async { - CkSurface.debugForceGLFailure = true; - final surface = CkOnscreenSurface(OnscreenCanvasProvider()); - await surface.initialized; + test( + 'Surface creates new context when WebGL context is restored', + () async { + final surface = Surface(); + expect(surface.debugForceNewContext, isTrue); + surface.createOrUpdateSurface(const BitmapSize(9, 19)); + final CkSurface before = surface.debugGetCkSurface()!; + expect(surface.debugForceNewContext, isFalse); - expect(surface.supportsWebGl, isFalse); - expect(surface.skSurface, isNotNull); - CkSurface.debugForceGLFailure = false; - }); + // Pump a timer to flush any microtasks. + await Future.delayed(Duration.zero); + surface.createOrUpdateSurface(const BitmapSize(9, 19)); + final CkSurface afterAcquireFrame = surface.debugGetCkSurface()!; + // Existing context is reused. + expect(afterAcquireFrame, same(before)); - test('CkOffscreenSurface falls back to software rendering', () async { - CkSurface.debugForceGLFailure = true; - final surface = CkOffscreenSurface(OffscreenCanvasProvider()); - await surface.initialized; + // Emulate WebGL context loss. + final DomOffscreenCanvas canvas = surface.debugGetOffscreenCanvas()!; + final WebGLContext ctx = canvas.getGlContext(2); + final WebGLLoseContextExtension loseContextExtension = ctx.loseContextExtension; + loseContextExtension.loseContext(); - expect(surface.supportsWebGl, isFalse); - expect(surface.skSurface, isNotNull); - CkSurface.debugForceGLFailure = false; - }); + // Pump a timer to allow the "lose context" event to propagate. + await Future.delayed(Duration.zero); + // We don't create a new GL context until the context is restored. + expect(surface.debugContextLost, isTrue); + final bool isContextLost = ctx.isContextLost(); + expect(isContextLost, isTrue); - test('does not recreate surface if size is the same', () async { - final surfaceProvider = OnscreenSurfaceProvider( - OnscreenCanvasProvider(), - (OnscreenCanvasProvider canvasProvider) => CkOnscreenSurface(canvasProvider), + // Emulate WebGL context restoration. + loseContextExtension.restoreContext(); + + // Pump a timer to allow the "restore context" event to propagate. + await Future.delayed(Duration.zero); + expect(surface.debugForceNewContext, isTrue); + + surface.createOrUpdateSurface(const BitmapSize(9, 19)); + final CkSurface afterContextLost = surface.debugGetCkSurface()!; + // A new context is created. + expect(afterContextLost, isNot(same(before))); + }, + // Firefox can't create a WebGL2 context in headless mode. + skip: isFirefox || !Surface.offscreenCanvasSupported, + ); + + // Regression test for https://github.com/flutter/flutter/issues/75286 + test( + 'updates canvas logical size when device-pixel ratio changes', + () { + final surface = Surface(); + surface.createOrUpdateSurface(const BitmapSize(10, 16)); + final CkSurface original = surface.debugGetCkSurface()!; + + expect(original.width(), 10); + expect(original.height(), 16); + expect(surface.debugGetOffscreenCanvas()!.width, 10); + expect(surface.debugGetOffscreenCanvas()!.height, 16); + + // Increase device-pixel ratio: this makes CSS pixels bigger, so we need + // fewer of them to cover the browser window. + EngineFlutterDisplay.instance.debugOverrideDevicePixelRatio(2.0); + surface.createOrUpdateSurface(const BitmapSize(10, 16)); + final CkSurface highDpr = surface.debugGetCkSurface()!; + expect(highDpr.width(), 10); + expect(highDpr.height(), 16); + expect(surface.debugGetOffscreenCanvas()!.width, 10); + expect(surface.debugGetOffscreenCanvas()!.height, 16); + + // Decrease device-pixel ratio: this makes CSS pixels smaller, so we need + // more of them to cover the browser window. + EngineFlutterDisplay.instance.debugOverrideDevicePixelRatio(0.5); + surface.createOrUpdateSurface(const BitmapSize(10, 16)); + final CkSurface lowDpr = surface.debugGetCkSurface()!; + expect(lowDpr.width(), 10); + expect(lowDpr.height(), 16); + expect(surface.debugGetOffscreenCanvas()!.width, 10); + expect(surface.debugGetOffscreenCanvas()!.height, 16); + + // See https://github.com/flutter/flutter/issues/77084#issuecomment-1120151172 + EngineFlutterDisplay.instance.debugOverrideDevicePixelRatio(2.0); + surface.createOrUpdateSurface(BitmapSize.fromSize(const ui.Size(9.9, 15.9))); + final CkSurface changeRatioAndSize = surface.debugGetCkSurface()!; + expect(changeRatioAndSize.width(), 10); + expect(changeRatioAndSize.height(), 16); + expect(surface.debugGetOffscreenCanvas()!.width, 10); + expect(surface.debugGetOffscreenCanvas()!.height, 16); + }, + skip: !Surface.offscreenCanvasSupported, + ); + + test('uses transferToImageBitmap for bitmap creation', () async { + final surface = Surface(); + surface.ensureSurface(const BitmapSize(10, 10)); + final DomOffscreenCanvas offscreenCanvas = surface.debugGetOffscreenCanvas()!; + final transferToImageBitmap = offscreenCanvas['transferToImageBitmap']! as JSFunction; + var transferToImageBitmapCalls = 0; + offscreenCanvas['transferToImageBitmap'] = () { + transferToImageBitmapCalls++; + return transferToImageBitmap.callAsFunction(offscreenCanvas); + }.toJS; + final renderCanvas = RenderCanvas(); + final recorder = CkPictureRecorder(); + final CkCanvas canvas = recorder.beginRecording(const ui.Rect.fromLTRB(0, 0, 10, 10)); + canvas.drawCircle( + const ui.Offset(5, 5), + 3, + CkPaint()..color = const ui.Color.fromARGB(255, 255, 0, 0), ); - final surface = surfaceProvider.createSurface() as CkOnscreenSurface; - await surface.initialized; - await surface.setSize(const BitmapSize(10, 20)); - final SkSurface? skSurface1 = surface.skSurface; - await surface.setSize(const BitmapSize(10, 20)); - final SkSurface? skSurface2 = surface.skSurface; - expect(skSurface1, same(skSurface2)); + final CkPicture picture = recorder.endRecording(); + await surface.rasterizeToCanvas(const BitmapSize(10, 10), renderCanvas, picture); + expect(transferToImageBitmapCalls, 1); + }, skip: !Surface.offscreenCanvasSupported); + + test('throws error if CanvasKit.MakeGrContext returns null', () async { + canvasKit['MakeGrContext'] = ((int glContext) => null).toJS; + final surface = Surface(); + expect(() => surface.ensureSurface(const BitmapSize(10, 10)), throwsA(isA())); + // Skipping on Firefox for now since Firefox headless doesn't support WebGL + }, skip: isFirefox); + + test('can recover from MakeSWCanvasSurface failure', () async { + debugOverrideJsConfiguration( + {'canvasKitForceCpuOnly': true}.jsify() as JsFlutterConfiguration?, + ); + addTearDown(() => debugOverrideJsConfiguration(null)); + + final surface = Surface(); + surface.debugThrowOnSoftwareSurfaceCreation = true; + expect( + () => surface.createOrUpdateSurface(const BitmapSize(12, 34)), + throwsA(isA()), + ); + await Future.delayed(Duration.zero); + + expect(surface.debugForceNewContext, isFalse); + + surface.debugThrowOnSoftwareSurfaceCreation = false; + final CkSurface ckSurface = surface.createOrUpdateSurface(const BitmapSize(12, 34)); + + expect(ckSurface.surface.width(), 12); + expect(ckSurface.surface.height(), 34); }); }); } +DomHTMLCanvasElement getDisplayCanvas(Surface surface) { + assert(surface.isDisplayCanvas); + return surface.hostElement.children.first as DomHTMLCanvasElement; +} + /// Extracts the CSS style values of 'width' and 'height' and returns them /// as a [ui.Size]. -ui.Size getCssSize(DomHTMLCanvasElement canvas) { +ui.Size getCssSize(Surface surface) { + final DomHTMLCanvasElement canvas = getDisplayCanvas(surface); final String cssWidth = canvas.style.width; final String cssHeight = canvas.style.height; // CSS width and height should be in the form 'NNNpx'. So cut off the 'px' and diff --git a/engine/src/flutter/lib/web_ui/test/common/test_initialization.dart b/engine/src/flutter/lib/web_ui/test/common/test_initialization.dart index bc456d4a06b..9461fa4784f 100644 --- a/engine/src/flutter/lib/web_ui/test/common/test_initialization.dart +++ b/engine/src/flutter/lib/web_ui/test/common/test_initialization.dart @@ -31,7 +31,6 @@ void setUpUnitTests({ {'fontFallbackBaseUrl': 'assets/fallback_fonts/'}.jsify() as engine.JsFlutterConfiguration?, ); - engine.debugThrowOnCreateImageBitmapIfDisabled = true; if (setUpTestViewDimensions) { // The following parameters are hard-coded in Flutter's test embedder. Since diff --git a/engine/src/flutter/lib/web_ui/test/engine/compositing/rasterizer_test.dart b/engine/src/flutter/lib/web_ui/test/engine/compositing/rasterizer_test.dart index 4346906ee65..c81bfc8654f 100644 --- a/engine/src/flutter/lib/web_ui/test/engine/compositing/rasterizer_test.dart +++ b/engine/src/flutter/lib/web_ui/test/engine/compositing/rasterizer_test.dart @@ -35,14 +35,6 @@ class TestRasterizer extends Rasterizer { List treesRenderedInView(EngineFlutterView view) { return viewRasterizers[view]!.treesRendered; } - - @override - Surface createPictureToImageSurface() { - throw UnimplementedError(); - } - - @override - SurfaceProvider get surfaceProvider => throw UnimplementedError(); } class TestViewRasterizer extends ViewRasterizer { @@ -54,8 +46,8 @@ class TestViewRasterizer extends ViewRasterizer { DisplayCanvasFactory get displayFactory => throw UnimplementedError(); @override - Future prepareToDraw() { - return Future.value(); + void prepareToDraw() { + // Do nothing } @override diff --git a/engine/src/flutter/lib/web_ui/test/engine/culling_test.dart b/engine/src/flutter/lib/web_ui/test/engine/culling_test.dart index 32b0b879ee0..c463d4bc69a 100644 --- a/engine/src/flutter/lib/web_ui/test/engine/culling_test.dart +++ b/engine/src/flutter/lib/web_ui/test/engine/culling_test.dart @@ -178,7 +178,7 @@ class FakeRasterizer extends ViewRasterizer { DisplayCanvasFactory get displayFactory => throw UnimplementedError(); @override - Future prepareToDraw() { + void prepareToDraw() { throw UnimplementedError(); } diff --git a/engine/src/flutter/lib/web_ui/test/ui/surface_context_lost_test.dart b/engine/src/flutter/lib/web_ui/test/ui/surface_context_lost_test.dart deleted file mode 100644 index 072109c4803..00000000000 --- a/engine/src/flutter/lib/web_ui/test/ui/surface_context_lost_test.dart +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:typed_data'; - -import 'package:test/bootstrap/browser.dart'; -import 'package:test/test.dart'; -import 'package:ui/src/engine.dart'; -import 'package:ui/ui.dart' as ui; - -import '../common/test_initialization.dart'; - -void main() { - internalBootstrapBrowserTest(() => testMain); -} - -void testMain() { - group('OffscreenSurface', () { - setUpUnitTests(); - - test( - 'creates new context when WebGL context is lost', - () async { - final Rasterizer rasterizer = renderer.rasterizer; - final surfaceProvider = rasterizer.surfaceProvider as OffscreenSurfaceProvider; - final OffscreenSurface surface = surfaceProvider.createSurface(); - await surface.initialized; - - final int initialGlContext = surface.glContext; - - await surface.triggerContextLoss(); - await surface.handledContextLossEvent; - await surface.initialized; - - // A new context is created. - expect(surface.glContext, isNot(initialGlContext)); - }, - skip: isFirefox || isSafari || !browserSupportsOffscreenCanvas, - ); - - test( - 'can still render after context is lost', - () async { - final Rasterizer rasterizer = renderer.rasterizer; - final surfaceProvider = rasterizer.surfaceProvider as OffscreenSurfaceProvider; - final OffscreenSurface surface = surfaceProvider.createSurface(); - await surface.initialized; - - await surface.setSize(const BitmapSize(10, 10)); - - // Draw a red square. - final ui.Picture redPicture = drawPicture((ui.Canvas canvas) { - canvas.drawRect( - const ui.Rect.fromLTWH(0, 0, 10, 10), - ui.Paint()..color = const ui.Color(0xFFFF0000), - ); - }); - List bitmaps = await surface.rasterizeToImageBitmaps([ - redPicture, - ]); - expect(bitmaps, hasLength(1)); - await expectBitmapColor(bitmaps.single, const ui.Color(0xFFFF0000)); - - // Lose the context. - await surface.triggerContextLoss(); - await surface.handledContextLossEvent; - await surface.initialized; - - // Draw a blue square. - final ui.Picture bluePicture = drawPicture((ui.Canvas canvas) { - canvas.drawRect( - const ui.Rect.fromLTWH(0, 0, 10, 10), - ui.Paint()..color = const ui.Color(0xFF0000FF), - ); - }); - bitmaps = await surface.rasterizeToImageBitmaps([bluePicture]); - expect(bitmaps, hasLength(1)); - await expectBitmapColor(bitmaps.single, const ui.Color(0xFF0000FF)); - }, - skip: isFirefox || isSafari || !browserSupportsOffscreenCanvas, - ); - - test( - 'can recover from multiple context losses', - () async { - final Rasterizer rasterizer = renderer.rasterizer; - final surfaceProvider = rasterizer.surfaceProvider as OffscreenSurfaceProvider; - final OffscreenSurface surface = surfaceProvider.createSurface(); - await surface.initialized; - - final int initialGlContext = surface.glContext; - - // First loss - await surface.triggerContextLoss(); - await surface.handledContextLossEvent; - await surface.initialized; - final int contextAfterFirstLoss = surface.glContext; - expect(contextAfterFirstLoss, isNot(initialGlContext)); - - // Second loss - await surface.triggerContextLoss(); - await surface.handledContextLossEvent; - await surface.initialized; - final int contextAfterSecondLoss = surface.glContext; - expect(contextAfterSecondLoss, isNot(contextAfterFirstLoss)); - }, - skip: isFirefox || isSafari || !browserSupportsOffscreenCanvas, - ); - }); -} - -ui.Picture drawPicture(void Function(ui.Canvas) drawCommands) { - final recorder = ui.PictureRecorder(); - final canvas = ui.Canvas(recorder); - drawCommands(canvas); - return recorder.endRecording(); -} - -Future expectBitmapColor(DomImageBitmap bitmap, ui.Color color) async { - final DomHTMLCanvasElement canvas = createDomCanvasElement( - width: bitmap.width, - height: bitmap.height, - ); - final DomCanvasRenderingContext2D ctx = canvas.context2D; - ctx.drawImage(bitmap, 0, 0); - final DomImageData imageData = ctx.getImageData(0, 0, 1, 1); - final Uint8ClampedList pixels = imageData.data; - expect(pixels[0], color.red); - expect(pixels[1], color.green); - expect(pixels[2], color.blue); - expect(pixels[3], color.alpha); -} diff --git a/engine/src/flutter/skwasm/library_skwasm_support.js b/engine/src/flutter/skwasm/library_skwasm_support.js index 1fdc1d7eee5..75c16f51a6a 100644 --- a/engine/src/flutter/skwasm/library_skwasm_support.js +++ b/engine/src/flutter/skwasm/library_skwasm_support.js @@ -72,10 +72,18 @@ mergeInto(LibraryManager.library, { }; } - const handleToContextLostHandlerMap = new Map(); const handleToCanvasMap = new Map(); const associatedObjectsMap = new Map(); - + _skwasm_setAssociatedObjectOnThread = function(threadId, pointer, object) { + skwasm_postMessage({ + skwasmMessage: 'setAssociatedObject', + pointer, + object, + }, [object], threadId); + }; + _skwasm_getAssociatedObject = function(pointer) { + return associatedObjectsMap.get(pointer); + }; _skwasm_connectThread = function(threadId) { const eventListener = function(data) { const skwasmMessage = data.skwasmMessage; @@ -83,35 +91,12 @@ mergeInto(LibraryManager.library, { return; } switch (skwasmMessage) { - case 'transferCanvas': - _surface_receiveCanvasOnWorker( - data.surface, - data.canvas, - data.callbackId, - ); - return; - case 'onInitialized': - _surface_onInitialized(data.surface, data.callbackId); - return; - case 'resizeSurface': - _surface_resizeOnWorker(data.surface, data.width, data.height, data.callbackId); - return; - case 'onResizeComplete': - _surface_onResizeComplete(data.surface, data.callbackId); - return; - case 'triggerContextLoss': - _surface_triggerContextLossOnWorker(data.surface, data.callbackId); - return; - case 'onContextLossTriggered': - _surface_onContextLossTriggered(data.surface, data.callbackId); - return; - case 'reportContextLost': - _surface_reportContextLost(data.surface, data.callbackId); - return; case 'renderPictures': _surface_renderPicturesOnWorker( data.surface, data.pictures, + data.width, + data.height, data.pictureCount, data.callbackId, skwasm_getCurrentTimestamp()); @@ -161,83 +146,45 @@ mergeInto(LibraryManager.library, { }; skwasm_registerMessageListener(threadId, eventListener); }; - - // Associated Objects - _skwasm_setAssociatedObjectOnThread = function(threadId, pointer, object) { + _skwasm_dispatchRenderPictures = function (threadId, surfaceHandle, pictures, width, height, pictureCount, callbackId) { skwasm_postMessage({ - skwasmMessage: 'setAssociatedObject', - pointer, - object, - }, [object], threadId); - }; - _skwasm_getAssociatedObject = function(pointer) { - return associatedObjectsMap.get(pointer); - }; - _skwasm_disposeAssociatedObjectOnThread = function(threadId, pointer) { - skwasm_postMessage({ - skwasmMessage: 'disposeAssociatedObject', - pointer, - }, [], threadId); - }; - - // Surface Lifecycle - _skwasm_dispatchDisposeSurface = function(threadId, surface) { - skwasm_postMessage({ - skwasmMessage: 'disposeSurface', - surface, - }, [], threadId); - } - - // Surface Setup - _skwasm_dispatchTransferCanvas = function (threadId, surfaceHandle, canvas, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'transferCanvas', + skwasmMessage: 'renderPictures', surface: surfaceHandle, - canvas, - callbackId, - }, [canvas], threadId); - }; - _skwasm_reportInitialized = function (surfaceHandle, contextLostCallbackId, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'onInitialized', - surface: surfaceHandle, - contextLostCallbackId, - callbackId, - }, []); - }; - - // Resizing - _skwasm_dispatchResizeSurface = function (threadId, surface, width, height, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'resizeSurface', - surface, + pictures, width, height, + pictureCount, callbackId, }, [], threadId); - } - _skwasm_reportResizeComplete = function (surfaceHandle, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'onResizeComplete', - surface: surfaceHandle, - callbackId, - }, []); + }; + _skwasm_createOffscreenCanvas = function(width, height) { + const canvas = new OffscreenCanvas(width, height); + var contextAttributes = { + majorVersion: 2, + alpha: true, + depth: true, + stencil: true, + antialias: false, + premultipliedAlpha: true, + preserveDrawingBuffer: false, + powerPreference: 'default', + failIfMajorPerformanceCaveat: false, + enableExtensionsByDefault: true, + }; + const contextHandle = GL.createContext(canvas, contextAttributes); + handleToCanvasMap.set(contextHandle, canvas); + return contextHandle; }; _skwasm_resizeCanvas = function(contextHandle, width, height) { const canvas = handleToCanvasMap.get(contextHandle); canvas.width = width; canvas.height = height; }; - - // Rendering - _skwasm_dispatchRenderPictures = function (threadId, surfaceHandle, pictures, pictureCount, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'renderPictures', - surface: surfaceHandle, - pictures, - pictureCount, - callbackId, - }, [], threadId); + _skwasm_captureImageBitmap = function (contextHandle, imageBitmaps) { + if (!imageBitmaps) imageBitmaps = Array(); + const canvas = handleToCanvasMap.get(contextHandle); + imageBitmaps.push(canvas.transferToImageBitmap()); + return imageBitmaps; }; _skwasm_resolveAndPostImages = async function (surfaceHandle, imageBitmaps, rasterStart, callbackId) { if (!imageBitmaps) imageBitmaps = Array(); @@ -251,14 +198,33 @@ mergeInto(LibraryManager.library, { rasterEnd, }, [...imageBitmaps]); }; - _skwasm_captureImageBitmap = function (contextHandle, imageBitmaps) { - if (!imageBitmaps) imageBitmaps = Array(); - const canvas = handleToCanvasMap.get(contextHandle); - imageBitmaps.push(canvas.transferToImageBitmap()); - return imageBitmaps; - }; + _skwasm_createGlTextureFromTextureSource = function(textureSource, width, height) { + const glCtx = GL.currentContext.GLctx; + const newTexture = glCtx.createTexture(); + glCtx.bindTexture(glCtx.TEXTURE_2D, newTexture); + glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); - // Image Rasterization + glCtx.texImage2D(glCtx.TEXTURE_2D, 0, glCtx.RGBA, width, height, 0, glCtx.RGBA, glCtx.UNSIGNED_BYTE, textureSource); + + glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false); + glCtx.bindTexture(glCtx.TEXTURE_2D, null); + + const textureId = GL.getNewId(GL.textures); + GL.textures[textureId] = newTexture; + return textureId; + }; + _skwasm_disposeAssociatedObjectOnThread = function(threadId, pointer) { + skwasm_postMessage({ + skwasmMessage: 'disposeAssociatedObject', + pointer, + }, [], threadId); + }; + _skwasm_dispatchDisposeSurface = function(threadId, surface) { + skwasm_postMessage({ + skwasmMessage: 'disposeSurface', + surface, + }, [], threadId); + } _skwasm_dispatchRasterizeImage = function(threadId, surface, image, format, callbackId) { skwasm_postMessage({ skwasmMessage: 'rasterizeImage', @@ -276,89 +242,6 @@ mergeInto(LibraryManager.library, { callbackId, }); } - - // Context Loss - _skwasm_dispatchTriggerContextLoss = function (threadId, surfaceHandle, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'triggerContextLoss', - surface: surfaceHandle, - callbackId, - }, [], threadId); - }; - _skwasm_reportContextLossTriggered = function (surfaceHandle, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'onContextLossTriggered', - surface: surfaceHandle, - callbackId, - }, []); - }; - _skwasm_reportContextLost = function (surfaceHandle, callbackId) { - skwasm_postMessage({ - skwasmMessage: 'reportContextLost', - surface: surfaceHandle, - callbackId, - }, []); - }; - _skwasm_triggerContextLossOnCanvas = function () { - const glCtx = GL.currentContext.GLctx; - glCtx.getExtension("WEBGL_lose_context").loseContext(); - }; - - // GL Context - _skwasm_getGlContextForCanvas = function (canvas, surfaceHandle) { - var contextAttributes = { - majorVersion: 2, - alpha: true, - depth: true, - stencil: true, - antialias: false, - premultipliedAlpha: true, - preserveDrawingBuffer: false, - powerPreference: 'default', - failIfMajorPerformanceCaveat: false, - enableExtensionsByDefault: true, - }; - const contextHandle = GL.createContext(canvas, contextAttributes); - handleToCanvasMap.set(contextHandle, canvas); - - // Register an event listener for the context lost event. - var contextLostHandler; - contextLostHandler = function (e) { - e.preventDefault(); - _surface_onContextLost(surfaceHandle); - canvas.removeEventListener('webglcontextlost', contextLostHandler); - } - canvas.addEventListener('webglcontextlost', contextLostHandler); - handleToContextLostHandlerMap.set(contextHandle, contextLostHandler); - return contextHandle; - }; - _skwasm_destroyContext = function (contextHandle) { - const canvas = handleToCanvasMap.get(contextHandle); - const handler = handleToContextLostHandlerMap.get(contextHandle); - if (canvas && handler) { - canvas.removeEventListener('webglcontextlost', handler); - } - GL.deleteContext(contextHandle); - handleToCanvasMap.delete(contextHandle); - handleToContextLostHandlerMap.delete(contextHandle); - }; - - // Texture Sources - _skwasm_createGlTextureFromTextureSource = function(textureSource, width, height) { - const glCtx = GL.currentContext.GLctx; - const newTexture = glCtx.createTexture(); - glCtx.bindTexture(glCtx.TEXTURE_2D, newTexture); - glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true); - - glCtx.texImage2D(glCtx.TEXTURE_2D, 0, glCtx.RGBA, width, height, 0, glCtx.RGBA, glCtx.UNSIGNED_BYTE, textureSource); - - glCtx.pixelStorei(glCtx.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false); - glCtx.bindTexture(glCtx.TEXTURE_2D, null); - - const textureId = GL.getNewId(GL.textures); - GL.textures[textureId] = newTexture; - return textureId; - }; }, $skwasm_registerMessageListener: function() {}, $skwasm_registerMessageListener__deps: ['$skwasm_support_setup'], @@ -376,28 +259,10 @@ mergeInto(LibraryManager.library, { skwasm_disposeAssociatedObjectOnThread__deps: ['$skwasm_support_setup'], skwasm_connectThread: function() {}, skwasm_connectThread__deps: ['$skwasm_support_setup', '$skwasm_registerMessageListener', '$skwasm_getCurrentTimestamp'], - skwasm_dispatchTransferCanvas: function () { }, - skwasm_dispatchTransferCanvas__deps: ['$skwasm_support_setup'], - skwasm_reportInitialized: function () { }, - skwasm_reportInitialized__deps: ['$skwasm_support_setup'], - skwasm_reportResizeComplete: function () { }, - skwasm_reportResizeComplete__deps: ['$skwasm_support_setup'], - skwasm_getGlContextForCanvas: function () { }, - skwasm_getGlContextForCanvas__deps: ['$skwasm_support_setup'], - skwasm_dispatchTriggerContextLoss: function () { }, - skwasm_dispatchTriggerContextLoss__deps: ['$skwasm_support_setup'], - skwasm_triggerContextLossOnCanvas: function () { }, - skwasm_triggerContextLossOnCanvas__deps: ['$skwasm_support_setup'], - skwasm_reportContextLossTriggered: function () { }, - skwasm_reportContextLossTriggered__deps: ['$skwasm_support_setup'], - skwasm_reportContextLost: function () { }, - skwasm_reportContextLost__deps: ['$skwasm_support_setup'], - skwasm_destroyContext: function () { }, - skwasm_destroyContext__deps: ['$skwasm_support_setup'], - skwasm_dispatchResizeSurface: function () { }, - skwasm_dispatchResizeSurface__deps: ['$skwasm_support_setup'], skwasm_dispatchRenderPictures: function() {}, skwasm_dispatchRenderPictures__deps: ['$skwasm_support_setup'], + skwasm_createOffscreenCanvas: function () {}, + skwasm_createOffscreenCanvas__deps: ['$skwasm_support_setup'], skwasm_resizeCanvas: function () {}, skwasm_resizeCanvas__deps: ['$skwasm_support_setup'], skwasm_captureImageBitmap: function () {}, diff --git a/engine/src/flutter/skwasm/skwasm_support.h b/engine/src/flutter/skwasm/skwasm_support.h index 06553c9849e..7b1f170a99b 100644 --- a/engine/src/flutter/skwasm/skwasm_support.h +++ b/engine/src/flutter/skwasm/skwasm_support.h @@ -32,21 +32,12 @@ extern void skwasm_connectThread(pthread_t thread_id); extern void skwasm_dispatchRenderPictures(unsigned long thread_id, Skwasm::Surface* surface, sk_sp* pictures, + int width, + int height, int count, uint32_t callback_id); -extern uint32_t skwasm_getGlContextForCanvas(SkwasmObject canvas, - Skwasm::Surface* surface); -extern void skwasm_reportInitialized(Skwasm::Surface* surface, - uint32_t callback_id, - uint32_t context_lost_callback_id); -extern void skwasm_reportResizeComplete(Skwasm::Surface* surface, - uint32_t callback_id); -extern void skwasm_dispatchResizeSurface(unsigned long thread_id, - Skwasm::Surface* surface, - int width, - int height, - uint32_t callback_id); -extern void skwasm_resizeCanvas(uint32_t contextHandle, int width, int height); +extern uint32_t skwasm_createOffscreenCanvas(int width, int height); +extern void skwasm_resizeCanvas(uint32_t context_handle, int width, int height); extern SkwasmObject skwasm_captureImageBitmap(uint32_t context_handle, SkwasmObject image_bitmaps); extern void skwasm_resolveAndPostImages(Skwasm::Surface* surface, @@ -57,19 +48,6 @@ extern unsigned int skwasm_createGlTextureFromTextureSource( SkwasmObject texture_source, int width, int height); -extern void skwasm_dispatchTriggerContextLoss(unsigned long thread_id, - Skwasm::Surface* surface, - uint32_t callback_id); -extern void skwasm_triggerContextLossOnCanvas(); -extern void skwasm_reportContextLossTriggered(Skwasm::Surface* surface, - uint32_t callback_id); -extern void skwasm_reportContextLost(Skwasm::Surface* surface, - uint32_t callback_id); -extern void skwasm_destroyContext(uint32_t context_handle); -extern void skwasm_dispatchTransferCanvas(unsigned long thread_id, - Skwasm::Surface* surface, - SkwasmObject canvas, - uint32_t callback_id); extern void skwasm_dispatchDisposeSurface(unsigned long thread_id, Skwasm::Surface* surface); extern void skwasm_dispatchRasterizeImage(unsigned long thread_id, diff --git a/engine/src/flutter/skwasm/surface.cc b/engine/src/flutter/skwasm/surface.cc index d6c99ff53fe..7649c13f35e 100644 --- a/engine/src/flutter/skwasm/surface.cc +++ b/engine/src/flutter/skwasm/surface.cc @@ -14,33 +14,6 @@ #include "flutter/skwasm/skwasm_support.h" #include "third_party/skia/include/core/SkColorSpace.h" -// This file implements the C++ side of the Skwasm Surface API. -// -// The general lifecycle of a method call that needs to be performed on the -// web worker thread is as follows: -// -// 1. The method is called on the [Surface] object on the main thread. -// This method will have the same name as the dart method that is calling it. -// It will extract the arguments, generate a callback id, and then call a -// `skwasm_dispatch*` method to send a message to the worker thread. -// 2. The `skwasm_dispatch*` method will be a javascript method in -// `library_skwasm_support.js`. This method will use `postMessage` to send a -// message to the worker thread. -// 3. The worker thread will receive the message in its `message` event -// listener. The listener will call a `surface_*OnWorker` C++ method. -// 4. The `surface_*OnWorker` method will call the corresponding `*OnWorker` -// method on the [Surface] object. This method will do the actual work of -// the method call. -// 5. When the work is complete, the `*OnWorker` method will call a -// `skwasm_report*` method. This will be a javascript method in -// `library_skwasm_support.js` which will use `postMessage` to send a -// message back to the main thread. -// 6. The main thread will receive the message in its `message` event listener. -// The listener will call an `on*` method on the C++ [Surface] object. -// 7. The `on*` method will invoke the callback handler that was registered by -// the Dart code, which will complete the future that was returned by the -// original Dart method call. - Skwasm::Surface::Surface() { if (skwasm_isSingleThreaded()) { skwasm_connectThread(0); @@ -58,45 +31,66 @@ Skwasm::Surface::Surface() { } } -// General getters are implemented in the header. - -// Lifecycle - -void Skwasm::Surface::SetCallbackHandler(CallbackHandler* callback_handler) { - assert(emscripten_is_main_browser_thread()); - callback_handler_ = callback_handler; -} - +// Worker thread only void Skwasm::Surface::Dispose() { - if (gl_context_) { - skwasm_destroyContext(gl_context_); - } delete this; } // Main thread only -uint32_t Skwasm::Surface::SetCanvas(SkwasmObject canvas) { +void Skwasm::Surface::SetResourceCacheLimit(int bytes) { + render_context_->SetResourceCacheLimit(bytes); +} + +// Main thread only +uint32_t Skwasm::Surface::RenderPictures(flutter::DisplayList** pictures, + int width, + int height, + int count) { assert(emscripten_is_main_browser_thread()); uint32_t callback_id = ++current_callback_id_; - skwasm_dispatchTransferCanvas(thread_, this, canvas, callback_id); + std::unique_ptr[]> picture_pointers = + std::make_unique[]>(count); + for (int i = 0; i < count; i++) { + picture_pointers[i] = sk_ref_sp(pictures[i]); + } + + // Releasing picturePointers here and will recreate the unique_ptr on the + // other thread See surface_renderPicturesOnWorker + skwasm_dispatchRenderPictures(thread_, this, picture_pointers.release(), + width, height, count, callback_id); return callback_id; } -void Skwasm::Surface::OnInitialized(uint32_t callback_id) { +// Main thread only +uint32_t Skwasm::Surface::RasterizeImage(flutter::DlImage* image, + Skwasm::ImageByteFormat format) { assert(emscripten_is_main_browser_thread()); - callback_handler_(callback_id, (void*)context_lost_callback_id_, - __builtin_wasm_ref_null_extern()); + uint32_t callback_id = ++current_callback_id_; + image->ref(); + + skwasm_dispatchRasterizeImage(thread_, this, image, format, callback_id); + return callback_id; +} + +std::unique_ptr +Skwasm::Surface::CreateTextureSourceWrapper( + Skwasm::SkwasmObject texture_source) { + return std::unique_ptr( + new Skwasm::TextureSourceWrapper(thread_, texture_source)); +} + +// Main thread only +void Skwasm::Surface::SetCallbackHandler( + Skwasm::Surface::CallbackHandler* callback_handler) { + assert(emscripten_is_main_browser_thread()); + callback_handler_ = callback_handler; } // Worker thread only -void Skwasm::Surface::ReceiveCanvasOnWorker(SkwasmObject canvas, - uint32_t callback_id) { - if (render_context_) { - render_context_.reset(); - } - canvas_width_ = 0; - canvas_height_ = 0; - gl_context_ = skwasm_getGlContextForCanvas(canvas, this); +void Skwasm::Surface::Init() { + // 256x256 is just an arbitrary size for the initial canvas, so that we can + // get a gl context off of it. + gl_context_ = skwasm_createOffscreenCanvas(256, 256); if (!gl_context_) { printf("Failed to create context!\n"); return; @@ -118,35 +112,12 @@ void Skwasm::Surface::ReceiveCanvasOnWorker(SkwasmObject canvas, emscripten_glGetIntegerv(GL_STENCIL_BITS, &stencil); render_context_ = Skwasm::RenderContext::Make(sample_count, stencil); - render_context_->Resize(canvas_width_, canvas_height_); + render_context_->Resize(256, 256); - context_lost_callback_id_ = ++current_callback_id_; - - skwasm_reportInitialized(this, context_lost_callback_id_, callback_id); -} - -// Resizing - -uint32_t Skwasm::Surface::SetSize(int width, int height) { - assert(emscripten_is_main_browser_thread()); - uint32_t callback_id = ++current_callback_id_; - - skwasm_dispatchResizeSurface(thread_, this, width, height, callback_id); - return callback_id; -} - -void Skwasm::Surface::OnResizeComplete(uint32_t callback_id) { - assert(emscripten_is_main_browser_thread()); - callback_handler_(callback_id, nullptr, __builtin_wasm_ref_null_extern()); -} - -void Skwasm::Surface::ResizeOnWorker(int width, - int height, - uint32_t callback_id) { - ResizeSurface(width, height); - skwasm_reportResizeComplete(this, callback_id); + is_initialized_ = true; } +// Worker thread only void Skwasm::Surface::ResizeSurface(int width, int height) { if (width != canvas_width_ || height != canvas_height_) { canvas_width_ = width; @@ -155,43 +126,35 @@ void Skwasm::Surface::ResizeSurface(int width, int height) { } } -// Rendering - -uint32_t Skwasm::Surface::RenderPictures(flutter::DisplayList** pictures, - int count) { - assert(emscripten_is_main_browser_thread()); - uint32_t callback_id = ++current_callback_id_; - std::unique_ptr[]> picture_pointers = - std::make_unique[]>(count); - for (int i = 0; i < count; i++) { - picture_pointers[i] = sk_ref_sp(pictures[i]); - } - - // Releasing picture_pointers here and will recreate the unique_ptr on the - // other thread See surface_renderPicturesOnWorker - skwasm_dispatchRenderPictures(thread_, this, picture_pointers.release(), - count, callback_id); - return callback_id; -} - -void Skwasm::Surface::OnRenderComplete(uint32_t callback_id, - SkwasmObject image_bitmap) { - assert(emscripten_is_main_browser_thread()); - callback_handler_(callback_id, nullptr, image_bitmap); +// Worker thread only +void Skwasm::Surface::RecreateSurface() { + Skwasm::makeCurrent(gl_context_); + skwasm_resizeCanvas(gl_context_, canvas_width_, canvas_height_); + render_context_->Resize(canvas_width_, canvas_height_); } +// Worker thread only void Skwasm::Surface::RenderPicturesOnWorker( sk_sp* pictures, + int width, + int height, int picture_count, uint32_t callback_id, double raster_start) { - Skwasm::makeCurrent(gl_context_); + if (!is_initialized_) { + Init(); + } + // This is initialized on the first call to `skwasm_captureImageBitmap` and // then populated with more bitmaps on subsequent calls. - SkwasmObject image_bitmap_array = __builtin_wasm_ref_null_extern(); + Skwasm::SkwasmObject image_bitmap_array = __builtin_wasm_ref_null_extern(); for (int i = 0; i < picture_count; i++) { sk_sp picture = pictures[i]; + ResizeSurface(width, height); + Skwasm::makeCurrent(gl_context_); + render_context_->RenderPicture(picture); + image_bitmap_array = skwasm_captureImageBitmap(gl_context_, image_bitmap_array); } @@ -199,30 +162,17 @@ void Skwasm::Surface::RenderPicturesOnWorker( callback_id); } -// Image Rasterization - -uint32_t Skwasm::Surface::RasterizeImage(flutter::DlImage* image, - Skwasm::ImageByteFormat format) { - assert(emscripten_is_main_browser_thread()); - uint32_t callback_id = ++current_callback_id_; - image->ref(); - - skwasm_dispatchRasterizeImage(thread_, this, image, format, callback_id); - return callback_id; -} - -void Skwasm::Surface::OnRasterizeComplete(uint32_t callback_id, SkData* data) { - assert(emscripten_is_main_browser_thread()); - callback_handler_(callback_id, data, __builtin_wasm_ref_null_extern()); -} - +// Worker thread only void Skwasm::Surface::RasterizeImageOnWorker(flutter::DlImage* image, Skwasm::ImageByteFormat format, uint32_t callback_id) { + if (!is_initialized_) { + Init(); + } + // We handle PNG encoding with browser APIs so that we can omit libpng from // skia to save binary size. assert(format != Skwasm::ImageByteFormat::png); - Skwasm::makeCurrent(gl_context_); SkAlphaType alpha_type = format == Skwasm::ImageByteFormat::rawStraightRgba ? SkAlphaType::kUnpremul_SkAlphaType : SkAlphaType::kPremul_SkAlphaType; @@ -242,6 +192,8 @@ void Skwasm::Surface::RasterizeImageOnWorker(flutter::DlImage* image, // `glReadPixels`. Once the skia bug is fixed, we should switch back to using // `SkImage::readPixels` instead. // See https://g-issues.skia.org/issues/349201915 + ResizeSurface(image->width(), image->height()); + render_context_->RenderImage(image, format); emscripten_glReadPixels(0, 0, image->width(), image->height(), GL_RGBA, @@ -251,145 +203,41 @@ void Skwasm::Surface::RasterizeImageOnWorker(flutter::DlImage* image, skwasm_postRasterizeResult(this, data.release(), callback_id); } -// Context Loss +void Skwasm::Surface::OnRasterizeComplete(uint32_t callback_id, SkData* data) { + callback_handler_(callback_id, data, __builtin_wasm_ref_null_extern()); +} -uint32_t Skwasm::Surface::TriggerContextLoss() { +// Main thread only +void Skwasm::Surface::OnRenderComplete(uint32_t callback_id, + Skwasm::SkwasmObject image_bitmap) { assert(emscripten_is_main_browser_thread()); - uint32_t callback_id = ++current_callback_id_; - skwasm_dispatchTriggerContextLoss(thread_, this, callback_id); - return callback_id; + callback_handler_(callback_id, nullptr, image_bitmap); } -void Skwasm::Surface::OnContextLossTriggered(uint32_t callback_id) { - assert(emscripten_is_main_browser_thread()); - callback_handler_(callback_id, nullptr, __builtin_wasm_ref_null_extern()); -} - -void Skwasm::Surface::ReportContextLost(uint32_t callback_id) { - assert(emscripten_is_main_browser_thread()); - callback_handler_(callback_id, nullptr, __builtin_wasm_ref_null_extern()); -} - -void Skwasm::Surface::TriggerContextLossOnWorker(uint32_t callback_id) { - Skwasm::makeCurrent(gl_context_); - skwasm_triggerContextLossOnCanvas(); - skwasm_reportContextLossTriggered(this, callback_id); -} - -void Skwasm::Surface::OnContextLost() { - if (!context_lost_callback_id_) { - printf("Received context lost event but never set callback handler!\n"); - return; - } - skwasm_reportContextLost(this, context_lost_callback_id_); -} - -// Other - -void Skwasm::Surface::SetResourceCacheLimit(int bytes) { - render_context_->SetResourceCacheLimit(bytes); -} - -std::unique_ptr -Skwasm::Surface::CreateTextureSourceWrapper(SkwasmObject texture_source) { - return std::unique_ptr( - new Skwasm::TextureSourceWrapper(thread_, texture_source)); -} - -// Private methods - -void Skwasm::Surface::RecreateSurface() { - Skwasm::makeCurrent(gl_context_); - skwasm_resizeCanvas(gl_context_, canvas_width_, canvas_height_); - render_context_->Resize(canvas_width_, canvas_height_); -} - -// TextureSourceWrapper implementation - -Skwasm::TextureSourceWrapper::TextureSourceWrapper(unsigned long thread_id, - SkwasmObject texture_source) +Skwasm::TextureSourceWrapper::TextureSourceWrapper( + unsigned long thread_id, + Skwasm::SkwasmObject texture_source) : raster_thread_id_(thread_id) { - skwasm_setAssociatedObjectOnThread(thread_id, this, texture_source); + skwasm_setAssociatedObjectOnThread(raster_thread_id_, this, texture_source); } Skwasm::TextureSourceWrapper::~TextureSourceWrapper() { skwasm_disposeAssociatedObjectOnThread(raster_thread_id_, this); } -SkwasmObject Skwasm::TextureSourceWrapper::GetTextureSource() { +Skwasm::SkwasmObject Skwasm::TextureSourceWrapper::GetTextureSource() { return skwasm_getAssociatedObject(this); } -// C-style API - SKWASM_EXPORT Skwasm::Surface* surface_create() { Skwasm::live_surface_count++; return new Skwasm::Surface(); } -SKWASM_EXPORT uint32_t surface_setCanvas(Skwasm::Surface* surface, - SkwasmObject canvas) { - // Dispatch to the worker so the canvas can be transferred to the worker. - return surface->SetCanvas(canvas); -} - -SKWASM_EXPORT void surface_receiveCanvasOnWorker(Skwasm::Surface* surface, - SkwasmObject canvas, - uint32_t callback_id) { - surface->ReceiveCanvasOnWorker(canvas, callback_id); -} - -SKWASM_EXPORT void surface_onInitialized(Skwasm::Surface* surface, - uint32_t callback_id) { - surface->OnInitialized(callback_id); -} - -SKWASM_EXPORT uint32_t surface_setSize(Skwasm::Surface* surface, - int width, - int height) { - return surface->SetSize(width, height); -} - -SKWASM_EXPORT void surface_resizeOnWorker(Skwasm::Surface* surface, - int width, - int height, - uint32_t callback_id) { - surface->ResizeOnWorker(width, height, callback_id); -} - -SKWASM_EXPORT void surface_onResizeComplete(Skwasm::Surface* surface, - uint32_t callback_id) { - surface->OnResizeComplete(callback_id); -} - SKWASM_EXPORT unsigned long surface_getThreadId(Skwasm::Surface* surface) { return surface->GetThreadId(); } -SKWASM_EXPORT EMSCRIPTEN_WEBGL_CONTEXT_HANDLE -surface_getGlContext(Skwasm::Surface* surface) { - return surface->GetGlContext(); -} - -SKWASM_EXPORT uint32_t surface_triggerContextLoss(Skwasm::Surface* surface) { - return surface->TriggerContextLoss(); -} - -SKWASM_EXPORT void surface_triggerContextLossOnWorker(Skwasm::Surface* surface, - uint32_t callback_id) { - surface->TriggerContextLossOnWorker(callback_id); -} - -SKWASM_EXPORT void surface_onContextLossTriggered(Skwasm::Surface* surface, - uint32_t callback_id) { - surface->OnContextLossTriggered(callback_id); -} - -SKWASM_EXPORT void surface_reportContextLost(Skwasm::Surface* surface, - uint32_t callback_id) { - surface->ReportContextLost(callback_id); -} - SKWASM_EXPORT void surface_setCallbackHandler( Skwasm::Surface* surface, Skwasm::Surface::CallbackHandler* callback_handler) { @@ -414,21 +262,25 @@ SKWASM_EXPORT void surface_setResourceCacheLimitBytes(Skwasm::Surface* surface, SKWASM_EXPORT uint32_t surface_renderPictures(Skwasm::Surface* surface, flutter::DisplayList** pictures, + int width, + int height, int count) { - return surface->RenderPictures(pictures, count); + return surface->RenderPictures(pictures, width, height, count); } SKWASM_EXPORT void surface_renderPicturesOnWorker( Skwasm::Surface* surface, sk_sp* pictures, + int width, + int height, int picture_count, uint32_t callback_id, double raster_start) { // This will release the pictures when they leave scope. std::unique_ptr[]> pictures_pointer = std::unique_ptr[]>(pictures); - surface->RenderPicturesOnWorker(pictures, picture_count, callback_id, - raster_start); + surface->RenderPicturesOnWorker(pictures, width, height, picture_count, + callback_id, raster_start); } SKWASM_EXPORT uint32_t surface_rasterizeImage(Skwasm::Surface* surface, @@ -449,7 +301,7 @@ SKWASM_EXPORT void surface_rasterizeImageOnWorker( // we finish creating the image bitmap, which is an asynchronous operation. SKWASM_EXPORT void surface_onRenderComplete(Skwasm::Surface* surface, uint32_t callback_id, - SkwasmObject image_bitmap) { + Skwasm::SkwasmObject image_bitmap) { surface->OnRenderComplete(callback_id, image_bitmap); } @@ -459,10 +311,6 @@ SKWASM_EXPORT void surface_onRasterizeComplete(Skwasm::Surface* surface, surface->OnRasterizeComplete(callback_id, data); } -SKWASM_EXPORT void surface_onContextLost(Skwasm::Surface* surface) { - surface->OnContextLost(); -} - SKWASM_EXPORT bool skwasm_isMultiThreaded() { return !skwasm_isSingleThreaded(); } diff --git a/engine/src/flutter/skwasm/surface.h b/engine/src/flutter/skwasm/surface.h index d1b8e3efa64..2fe04912608 100644 --- a/engine/src/flutter/skwasm/surface.h +++ b/engine/src/flutter/skwasm/surface.h @@ -5,16 +5,21 @@ #ifndef FLUTTER_SKWASM_SURFACE_H_ #define FLUTTER_SKWASM_SURFACE_H_ +#include + #include #include #include #include #include #include -#include -#include "export.h" -#include "render_context.h" -#include "wrappers.h" + +#include "flutter/display_list/image/dl_image.h" +#include "flutter/skwasm/export.h" +#include "flutter/skwasm/render_context.h" +#include "flutter/skwasm/wrappers.h" +#include "third_party/skia/include/core/SkData.h" +#include "third_party/skia/include/core/SkRefCnt.h" namespace flutter { class DisplayList; @@ -40,51 +45,35 @@ class Surface { // Main thread only Surface(); - // General getters unsigned long GetThreadId() { return thread_; } - EMSCRIPTEN_WEBGL_CONTEXT_HANDLE GetGlContext() { return gl_context_; } - // Lifecycle - void SetCallbackHandler(CallbackHandler* callback_handler); + // Main thread only void Dispose(); - - // Surface setup - uint32_t SetCanvas(SkwasmObject canvas); - void OnInitialized(uint32_t callback_id); - void ReceiveCanvasOnWorker(SkwasmObject canvas, uint32_t callback_id); - - // Resizing - uint32_t SetSize(int width, int height); - void OnResizeComplete(uint32_t callback_id); - void ResizeOnWorker(int width, int height, uint32_t callback_id); - - // Rendering - uint32_t RenderPictures(flutter::DisplayList** picture, int count); + void SetResourceCacheLimit(int bytes); + uint32_t RenderPictures(flutter::DisplayList** pictures, + int width, + int height, + int count); + uint32_t RasterizeImage(flutter::DlImage* image, ImageByteFormat format); + void SetCallbackHandler(CallbackHandler* callback_handler); void OnRenderComplete(uint32_t callback_id, SkwasmObject image_bitmap); - void RenderPicturesOnWorker(sk_sp* picture, + void OnRasterizeComplete(uint32_t callback_id, SkData* data); + + // Any thread + std::unique_ptr CreateTextureSourceWrapper( + SkwasmObject texture_source); + + // Worker thread + void RenderPicturesOnWorker(sk_sp* pictures, + int width, + int height, int picture_count, uint32_t callback_id, double raster_start); - - // Image Rasterization - uint32_t RasterizeImage(flutter::DlImage* image, ImageByteFormat format); - void OnRasterizeComplete(uint32_t callback_id, SkData* data); void RasterizeImageOnWorker(flutter::DlImage* image, ImageByteFormat format, uint32_t callback_id); - // Context Loss - uint32_t TriggerContextLoss(); - void OnContextLossTriggered(uint32_t callback_id); - void ReportContextLost(uint32_t callback_id); - void TriggerContextLossOnWorker(uint32_t callback_id); - void OnContextLost(); - - // Other - void SetResourceCacheLimit(int bytes); - std::unique_ptr CreateTextureSourceWrapper( - SkwasmObject textureSource); - private: void Init(); void ResizeSurface(int width, int height); @@ -98,8 +87,6 @@ class Surface { EMSCRIPTEN_WEBGL_CONTEXT_HANDLE gl_context_ = 0; std::unique_ptr render_context_; - uint32_t context_lost_callback_id_ = 0; - unsigned long thread_; bool is_initialized_ = false; diff --git a/engine/src/flutter/skwasm/wrappers.h b/engine/src/flutter/skwasm/wrappers.h index b142fbcf9d2..af63e815de9 100644 --- a/engine/src/flutter/skwasm/wrappers.h +++ b/engine/src/flutter/skwasm/wrappers.h @@ -29,7 +29,7 @@ inline void makeCurrent(EMSCRIPTEN_WEBGL_CONTEXT_HANDLE handle) { int result = emscripten_webgl_make_context_current(handle); if (result != EMSCRIPTEN_RESULT_SUCCESS) { - printf("make_context_current failed: %d", result); + printf("make_context failed: %d", result); } }