From feaec27593d3fa0003be37f450dce4f99bc3b959 Mon Sep 17 00:00:00 2001 From: gaaclarke <30870216+gaaclarke@users.noreply.github.com> Date: Fri, 12 Dec 2025 09:54:32 -0800 Subject: [PATCH] Implements decodeImageFromPixelsSync (#179519) fixes https://github.com/flutter/flutter/issues/178488 This doesn't implement the following. They can be implemented in later PRs. - a skia implementation (maybe won't implement?) - a web implementation - resizing - target pixel format ## Pre-launch Checklist - [x] I read the [Contributor Guide] and followed the process outlined there for submitting PRs. - [x] I read the [Tree Hygiene] wiki page, which explains my responsibilities. - [x] I read and followed the [Flutter Style Guide], including [Features we expect every widget to implement]. - [x] I signed the [CLA]. - [x] I listed at least one issue that this PR fixes in the description above. - [x] I updated/added relevant documentation (doc comments with `///`). - [x] I added new tests to check the change I am making, or this PR is [test-exempt]. - [x] I followed the [breaking change policy] and added [Data Driven Fixes] where supported. - [x] All existing and new tests are passing. If you need help, consider asking for advice on the #hackers-new channel on [Discord]. **Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed. [Contributor Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview [Tree Hygiene]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md [test-exempt]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests [Flutter Style Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md [Features we expect every widget to implement]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement [CLA]: https://cla.developers.google.com/ [flutter/tests]: https://github.com/flutter/tests [breaking change policy]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes [Discord]: https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md [Data Driven Fixes]: https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md --- engine/src/flutter/lib/gpu/texture.cc | 2 + engine/src/flutter/lib/ui/BUILD.gn | 2 + engine/src/flutter/lib/ui/dart_ui.cc | 1 + engine/src/flutter/lib/ui/painting.dart | 25 ++++ engine/src/flutter/lib/ui/painting/canvas.cc | 5 +- ...display_list_deferred_image_gpu_impeller.h | 1 + ...t_deferred_image_gpu_impeller_unittests.cc | 10 +- engine/src/flutter/lib/ui/painting/image.cc | 124 +++++++++++++++- engine/src/flutter/lib/ui/painting/image.h | 18 ++- .../pixel_deferred_image_gpu_impeller.cc | 134 ++++++++++++++++++ .../pixel_deferred_image_gpu_impeller.h | 105 ++++++++++++++ .../flutter/lib/ui/painting/testing/mocks.h | 4 + engine/src/flutter/lib/ui/snapshot_delegate.h | 3 + .../src/flutter/lib/web_ui/lib/painting.dart | 3 + engine/src/flutter/shell/common/rasterizer.cc | 6 + engine/src/flutter/shell/common/rasterizer.h | 4 + .../shell/common/snapshot_controller.h | 7 + .../common/snapshot_controller_impeller.cc | 69 +++++++++ .../common/snapshot_controller_impeller.h | 3 + .../shell/common/snapshot_controller_skia.cc | 6 + .../shell/common/snapshot_controller_skia.h | 3 + engine/src/flutter/testing/dart/BUILD.gn | 1 + .../decode_image_from_pixels_sync_test.dart | 63 ++++++++ 23 files changed, 585 insertions(+), 14 deletions(-) create mode 100644 engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.cc create mode 100644 engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.h create mode 100644 engine/src/flutter/testing/dart/decode_image_from_pixels_sync_test.dart diff --git a/engine/src/flutter/lib/gpu/texture.cc b/engine/src/flutter/lib/gpu/texture.cc index 805613775ee..56852164ced 100644 --- a/engine/src/flutter/lib/gpu/texture.cc +++ b/engine/src/flutter/lib/gpu/texture.cc @@ -6,11 +6,13 @@ #include "flutter/lib/gpu/formats.h" #include "flutter/lib/ui/painting/image.h" +#include "flutter/lib/ui/ui_dart_state.h" #include "fml/make_copyable.h" #include "fml/mapping.h" #include "impeller/core/allocator.h" #include "impeller/core/formats.h" #include "impeller/core/texture.h" + #if IMPELLER_SUPPORTS_RENDERING #include "impeller/display_list/dl_image_impeller.h" // nogncheck #endif diff --git a/engine/src/flutter/lib/ui/BUILD.gn b/engine/src/flutter/lib/ui/BUILD.gn index 725ff31ae8d..1ea3bbdc79b 100644 --- a/engine/src/flutter/lib/ui/BUILD.gn +++ b/engine/src/flutter/lib/ui/BUILD.gn @@ -200,6 +200,8 @@ source_set("ui") { "painting/image_decoder_impeller.h", "painting/image_encoding_impeller.cc", "painting/image_encoding_impeller.h", + "painting/pixel_deferred_image_gpu_impeller.cc", + "painting/pixel_deferred_image_gpu_impeller.h", ] deps += [ diff --git a/engine/src/flutter/lib/ui/dart_ui.cc b/engine/src/flutter/lib/ui/dart_ui.cc index 9b5921d58cb..04385fcf76b 100644 --- a/engine/src/flutter/lib/ui/dart_ui.cc +++ b/engine/src/flutter/lib/ui/dart_ui.cc @@ -85,6 +85,7 @@ typedef CanvasPath Path; /* Other */ \ V(FontCollection::LoadFontFromList) \ V(ImageDescriptor::initEncoded) \ + V(Image::decodeImageFromPixelsSync) \ V(ImageFilter::equals) \ V(ImmutableBuffer::init) \ V(ImmutableBuffer::initFromAsset) \ diff --git a/engine/src/flutter/lib/ui/painting.dart b/engine/src/flutter/lib/ui/painting.dart index c952939b64c..a9d1609c13d 100644 --- a/engine/src/flutter/lib/ui/painting.dart +++ b/engine/src/flutter/lib/ui/painting.dart @@ -2741,6 +2741,31 @@ void decodeImageFromPixels( }); } +/// Decodes the given [pixels] into an [Image] synchronously. +/// +/// The [pixels] are expected to be in the format specified by [format]. +/// +/// The [width] and [height] arguments specify the dimensions of the image. +/// +/// This function returns an [Image] immediately. The image might not be +/// fully decoded yet, but it can be drawn to a [Canvas]. +Image decodeImageFromPixelsSync(Uint8List pixels, int width, int height, PixelFormat format) { + final image = Image._(_Image._(), width, height); + _decodeImageFromPixelsSync(pixels, width, height, format.index, image._image); + return image; +} + +@Native( + symbol: 'Image::decodeImageFromPixelsSync', +) +external void _decodeImageFromPixelsSync( + Uint8List pixels, + int width, + int height, + int format, + _Image outImage, +); + /// Determines the winding rule that decides how the interior of a [Path] is /// calculated. /// diff --git a/engine/src/flutter/lib/ui/painting/canvas.cc b/engine/src/flutter/lib/ui/painting/canvas.cc index 64d07e8801c..d560929d1b3 100644 --- a/engine/src/flutter/lib/ui/painting/canvas.cc +++ b/engine/src/flutter/lib/ui/painting/canvas.cc @@ -378,15 +378,14 @@ void Canvas::drawArc(double left, void Canvas::drawPath(const CanvasPath* path, Dart_Handle paint_objects, Dart_Handle paint_data) { - Paint paint(paint_objects, paint_data); - - FML_DCHECK(paint.isNotNull()); if (!path) { Dart_ThrowException( ToDart("Canvas.drawPath called with non-genuine Path.")); return; } if (display_list_builder_) { + Paint paint(paint_objects, paint_data); + FML_DCHECK(paint.isNotNull()); DlPaint dl_paint; paint.paint(dl_paint, kDrawPathFlags, DlTileMode::kDecal); builder()->DrawPath(path->path(), dl_paint); diff --git a/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller.h b/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller.h index ec1b31bf111..6ade920f7f9 100644 --- a/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller.h +++ b/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller.h @@ -88,6 +88,7 @@ class DlDeferredImageGPUImpeller final : public DlImage { bool isTextureBacked() const; const std::shared_ptr texture() const { + FML_DCHECK(raster_task_runner_->RunsTasksOnCurrentThread()); return texture_; } diff --git a/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller_unittests.cc b/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller_unittests.cc index a7ac2e6780c..c8965200eff 100644 --- a/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller_unittests.cc +++ b/engine/src/flutter/lib/ui/painting/display_list_deferred_image_gpu_impeller_unittests.cc @@ -71,16 +71,18 @@ TEST(DlDeferredImageGPUImpeller, TrashesDisplayList) { snapshot_delegate_weak_ptr = snapshot_delegate->GetWeakPtr(); }); + sk_sp image; // Pause raster thread. fml::AutoResetWaitableEvent latch; - task_runner->PostTask([&latch]() { latch.Wait(); }); + task_runner->PostTask([&latch, &image]() { + latch.Wait(); + EXPECT_FALSE(image->impeller_texture()); + }); - auto image = DlDeferredImageGPUImpeller::Make( + image = DlDeferredImageGPUImpeller::Make( builder.Build(), size, SnapshotPixelFormat::kDontCare, snapshot_delegate_weak_ptr, task_runner); - EXPECT_FALSE(image->impeller_texture()); - // Unpause raster thread. latch.Signal(); diff --git a/engine/src/flutter/lib/ui/painting/image.cc b/engine/src/flutter/lib/ui/painting/image.cc index dba1a2aa410..fb6f704a641 100644 --- a/engine/src/flutter/lib/ui/painting/image.cc +++ b/engine/src/flutter/lib/ui/painting/image.cc @@ -4,18 +4,16 @@ #include "flutter/lib/ui/painting/image.h" -#include -#include #include "tonic/logging/dart_invoke.h" #if IMPELLER_SUPPORTS_RENDERING #include "flutter/lib/ui/painting/image_encoding_impeller.h" +#include "flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.h" #endif +#include "flutter/display_list/image/dl_image.h" #include "flutter/lib/ui/painting/image_encoding.h" +#include "flutter/lib/ui/ui_dart_state.h" #include "third_party/tonic/converter/dart_converter.h" -#include "third_party/tonic/dart_args.h" -#include "third_party/tonic/dart_binding_macros.h" -#include "third_party/tonic/dart_library_natives.h" namespace flutter { @@ -56,3 +54,119 @@ int CanvasImage::colorSpace() { } } // namespace flutter + +namespace flutter { + +namespace { + +int BytesPerPixel(PixelFormat pixel_format) { + switch (pixel_format) { + case PixelFormat::kRgba8888: + case PixelFormat::kBgra8888: + case PixelFormat::kRFloat32: + return 4; + case PixelFormat::kRgbaFloat32: + return 16; + } + return 4; +} + +SkColorType PixelFormatToSkColorType(PixelFormat pixel_format) { + switch (pixel_format) { + case PixelFormat::kRgba8888: + return kRGBA_8888_SkColorType; + case PixelFormat::kBgra8888: + return kBGRA_8888_SkColorType; + case PixelFormat::kRgbaFloat32: + return kRGBA_F32_SkColorType; + case PixelFormat::kRFloat32: + return kUnknown_SkColorType; + } + return kUnknown_SkColorType; +} + +// Returns only static strings. +const char* DoDecodeImageFromPixelsSync(Dart_Handle pixels_handle, + uint32_t width, + uint32_t height, + int32_t pixel_format, + Dart_Handle raw_image_handle) { + auto* dart_state = UIDartState::Current(); + if (!dart_state) { + return "Dart state is null."; + } + + if (!dart_state->IsImpellerEnabled()) { + return "decodeImageFromPixelsSync is not implemented on Skia."; + } + + if (width == 0 || height == 0) { + return "Image dimensions must be greater than zero."; + } + + if (pixel_format < 0 || + pixel_format > static_cast(kLastPixelFormat)) { + return "Invalid pixel format."; + } + PixelFormat format = static_cast(pixel_format); + + sk_sp sk_data; + sk_sp sk_image; + { + tonic::Uint8List pixels(pixels_handle); + if (!pixels.data()) { + return "Pixels must not be null."; + } + + int32_t row_bytes = width * BytesPerPixel(format); + SkColorType color_type = PixelFormatToSkColorType(format); + if (color_type == kUnknown_SkColorType) { + return "Unsupported pixel format."; + } + + SkImageInfo image_info = + SkImageInfo::Make(width, height, color_type, kUnpremul_SkAlphaType); + if (pixel_format == 2) { // rgbaFloat32 + image_info = image_info.makeAlphaType(kUnpremul_SkAlphaType); + } else { + image_info = image_info.makeAlphaType(kPremul_SkAlphaType); + } + + sk_data = SkData::MakeWithCopy(pixels.data(), pixels.num_elements()); + sk_image = SkImages::RasterFromData(image_info, sk_data, row_bytes); + if (!sk_image) { + return "Failed to create image from pixels."; + } + } + + auto snapshot_delegate = dart_state->GetSnapshotDelegate(); + auto raster_task_runner = dart_state->GetTaskRunners().GetRasterTaskRunner(); + + auto result_image = CanvasImage::Create(); + sk_sp deferred_image; + +#if IMPELLER_SUPPORTS_RENDERING + deferred_image = PixelDeferredImageGPUImpeller::Make( + sk_image, std::move(snapshot_delegate), std::move(raster_task_runner)); +#endif // IMPELLER_SUPPORTS_RENDERING + + result_image->set_image(deferred_image); + result_image->AssociateWithDartWrapper(raw_image_handle); + + return nullptr; +} +} // namespace + +void CanvasImage::decodeImageFromPixelsSync(Dart_Handle pixels_handle, + uint32_t width, + uint32_t height, + int32_t pixel_format, + Dart_Handle raw_image_handle) { + const char* error = DoDecodeImageFromPixelsSync( + pixels_handle, width, height, pixel_format, raw_image_handle); + if (error) { + Dart_ThrowException(tonic::ToDart(error)); + } +} + +} // namespace flutter diff --git a/engine/src/flutter/lib/ui/painting/image.h b/engine/src/flutter/lib/ui/painting/image.h index 23ff80c99d4..45c4a7061d6 100644 --- a/engine/src/flutter/lib/ui/painting/image.h +++ b/engine/src/flutter/lib/ui/painting/image.h @@ -7,11 +7,19 @@ #include "flutter/display_list/image/dl_image.h" #include "flutter/lib/ui/dart_wrapper.h" -#include "flutter/lib/ui/ui_dart_state.h" -#include "third_party/skia/include/core/SkImage.h" namespace flutter { +// Must be kept in sync with painting.dart. +enum class PixelFormat { + kRgba8888, + kBgra8888, + kRgbaFloat32, + kRFloat32, // kLastPixelFormat +}; + +constexpr PixelFormat kLastPixelFormat = PixelFormat::kRFloat32; + // Must be kept in sync with painting.dart. enum ColorSpace { kSRGB, @@ -35,6 +43,12 @@ class CanvasImage final : public RefCountedDartWrappable { int height() { return image_ ? image_->height() : 0; } + static void decodeImageFromPixelsSync(Dart_Handle pixels_handle, + uint32_t width, + uint32_t height, + int32_t pixel_format, + Dart_Handle raw_image_handle); + Dart_Handle toByteData(int format, Dart_Handle callback); void dispose(); diff --git a/engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.cc b/engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.cc new file mode 100644 index 00000000000..a129d5136e8 --- /dev/null +++ b/engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.cc @@ -0,0 +1,134 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.h" + +#include "flutter/fml/make_copyable.h" +#include "flutter/fml/trace_event.h" + +namespace flutter { + +sk_sp PixelDeferredImageGPUImpeller::Make( + sk_sp image, + fml::TaskRunnerAffineWeakPtr snapshot_delegate, + fml::RefPtr raster_task_runner) { + return sk_sp(new PixelDeferredImageGPUImpeller( + PixelDeferredImageGPUImpeller::ImageWrapper::Make( + std::move(image), std::move(snapshot_delegate), + std::move(raster_task_runner)))); +} + +PixelDeferredImageGPUImpeller::PixelDeferredImageGPUImpeller( + std::shared_ptr wrapper) + : wrapper_(std::move(wrapper)) {} + +PixelDeferredImageGPUImpeller::~PixelDeferredImageGPUImpeller() = default; + +sk_sp PixelDeferredImageGPUImpeller::skia_image() const { + return nullptr; +} + +std::shared_ptr +PixelDeferredImageGPUImpeller::impeller_texture() const { + if (!wrapper_) { + return nullptr; + } + return wrapper_->texture(); +} + +bool PixelDeferredImageGPUImpeller::isOpaque() const { + return false; +} + +bool PixelDeferredImageGPUImpeller::isTextureBacked() const { + return wrapper_ && wrapper_->isTextureBacked(); +} + +bool PixelDeferredImageGPUImpeller::isUIThreadSafe() const { + return true; +} + +DlISize PixelDeferredImageGPUImpeller::GetSize() const { + return wrapper_ ? wrapper_->size() : DlISize(); +} + +size_t PixelDeferredImageGPUImpeller::GetApproximateByteSize() const { + auto size = sizeof(PixelDeferredImageGPUImpeller); + if (wrapper_) { + if (wrapper_->texture()) { + size += wrapper_->texture() + ->GetTextureDescriptor() + .GetByteSizeOfBaseMipLevel(); + } else { + size += wrapper_->size().Area() * 4; + } + } + return size; +} + +std::optional PixelDeferredImageGPUImpeller::get_error() const { + return wrapper_ ? wrapper_->get_error() : std::nullopt; +} + +std::shared_ptr +PixelDeferredImageGPUImpeller::ImageWrapper::Make( + sk_sp image, + fml::TaskRunnerAffineWeakPtr snapshot_delegate, + fml::RefPtr raster_task_runner) { + auto wrapper = std::shared_ptr(new ImageWrapper( + image, std::move(snapshot_delegate), std::move(raster_task_runner))); + wrapper->SnapshotImage(std::move(image)); + return wrapper; +} + +PixelDeferredImageGPUImpeller::ImageWrapper::ImageWrapper( + const sk_sp& image, + fml::TaskRunnerAffineWeakPtr snapshot_delegate, + fml::RefPtr raster_task_runner) + : size_(DlISize(image->width(), image->height())), + snapshot_delegate_(std::move(snapshot_delegate)), + raster_task_runner_(std::move(raster_task_runner)) {} + +PixelDeferredImageGPUImpeller::ImageWrapper::~ImageWrapper() = default; + +bool PixelDeferredImageGPUImpeller::ImageWrapper::isTextureBacked() const { + return texture_ && texture_->IsValid(); +} + +void PixelDeferredImageGPUImpeller::ImageWrapper::SnapshotImage( + sk_sp image) { + fml::TaskRunner::RunNowOrPostTask( + raster_task_runner_, + fml::MakeCopyable( + [weak_this = weak_from_this(), image = std::move(image)]() mutable { + TRACE_EVENT0("flutter", "SnapshotImage (impeller)"); + auto wrapper = weak_this.lock(); + if (!wrapper) { + return; + } + FML_DCHECK(!wrapper->texture_) << "should only execute once."; + auto snapshot_delegate = wrapper->snapshot_delegate_; + if (!snapshot_delegate) { + return; + } + + // Use MakeTextureImage directly. + auto snapshot_dl_image = snapshot_delegate->MakeTextureImage( + image, SnapshotPixelFormat::kDontCare); + if (!snapshot_dl_image) { + std::scoped_lock lock(wrapper->error_mutex_); + wrapper->error_ = "Failed to create snapshot."; + return; + } + wrapper->texture_ = snapshot_dl_image->impeller_texture(); + })); +} + +std::optional +PixelDeferredImageGPUImpeller::ImageWrapper::get_error() const { + std::scoped_lock lock(error_mutex_); + return error_; +} + +} // namespace flutter diff --git a/engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.h b/engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.h new file mode 100644 index 00000000000..1310cb27cae --- /dev/null +++ b/engine/src/flutter/lib/ui/painting/pixel_deferred_image_gpu_impeller.h @@ -0,0 +1,105 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_LIB_UI_PAINTING_PIXEL_DEFERRED_IMAGE_GPU_IMPELLER_H_ +#define FLUTTER_LIB_UI_PAINTING_PIXEL_DEFERRED_IMAGE_GPU_IMPELLER_H_ + +#include +#include +#include +#include + +#include "flutter/display_list/image/dl_image.h" +#include "flutter/fml/memory/weak_ptr.h" +#include "flutter/fml/task_runner.h" +#include "flutter/lib/ui/snapshot_delegate.h" +#include "impeller/core/texture.h" +#include "third_party/skia/include/core/SkImage.h" + +namespace flutter { + +/// A deferred image that is created from pixels. +/// @see DisplayListDeferredImageGPUImpeller for another example of a deferred +/// image. +/// @see dart:ui `decodeImageFromPixelsSync` for the user of this class. +class PixelDeferredImageGPUImpeller final : public DlImage { + public: + static sk_sp Make( + sk_sp image, + fml::TaskRunnerAffineWeakPtr snapshot_delegate, + fml::RefPtr raster_task_runner); + + // |DlImage| + ~PixelDeferredImageGPUImpeller() override; + + // |DlImage| + sk_sp skia_image() const override; + + // |DlImage| + std::shared_ptr impeller_texture() const override; + + // |DlImage| + bool isOpaque() const override; + + // |DlImage| + bool isTextureBacked() const override; + + // |DlImage| + bool isUIThreadSafe() const override; + + // |DlImage| + DlISize GetSize() const override; + + // |DlImage| + size_t GetApproximateByteSize() const override; + + // |DlImage| + std::optional get_error() const override; + + private: + class ImageWrapper : public std::enable_shared_from_this { + public: + static std::shared_ptr Make( + sk_sp image, + fml::TaskRunnerAffineWeakPtr snapshot_delegate, + fml::RefPtr raster_task_runner); + + ImageWrapper( + const sk_sp& image, + fml::TaskRunnerAffineWeakPtr snapshot_delegate, + fml::RefPtr raster_task_runner); + + ~ImageWrapper(); + + std::shared_ptr texture() const { return texture_; } + + const DlISize& size() const { return size_; } + + bool isTextureBacked() const; + + std::optional get_error() const; + + private: + void SnapshotImage(sk_sp image); + + DlISize size_; + fml::TaskRunnerAffineWeakPtr snapshot_delegate_; + fml::RefPtr raster_task_runner_; + std::shared_ptr texture_; + mutable std::mutex error_mutex_; + std::optional error_; + + FML_DISALLOW_COPY_AND_ASSIGN(ImageWrapper); + }; + + explicit PixelDeferredImageGPUImpeller(std::shared_ptr wrapper); + + std::shared_ptr wrapper_; + + FML_DISALLOW_COPY_AND_ASSIGN(PixelDeferredImageGPUImpeller); +}; + +} // namespace flutter + +#endif // FLUTTER_LIB_UI_PAINTING_PIXEL_DEFERRED_IMAGE_GPU_IMPELLER_H_ diff --git a/engine/src/flutter/lib/ui/painting/testing/mocks.h b/engine/src/flutter/lib/ui/painting/testing/mocks.h index ed2af3c8b75..5d63b25bede 100644 --- a/engine/src/flutter/lib/ui/painting/testing/mocks.h +++ b/engine/src/flutter/lib/ui/painting/testing/mocks.h @@ -49,6 +49,10 @@ class MockSnapshotDelegate : public SnapshotDelegate { ConvertToRasterImage, (sk_sp), (override)); + MOCK_METHOD(sk_sp, + MakeTextureImage, + (sk_sp, SnapshotPixelFormat), + (override)); MOCK_METHOD(void, CacheRuntimeStage, (const std::shared_ptr&), diff --git a/engine/src/flutter/lib/ui/snapshot_delegate.h b/engine/src/flutter/lib/ui/snapshot_delegate.h index ae746a1b47b..65ba9030919 100644 --- a/engine/src/flutter/lib/ui/snapshot_delegate.h +++ b/engine/src/flutter/lib/ui/snapshot_delegate.h @@ -87,6 +87,9 @@ class SnapshotDelegate { DlISize picture_size, SnapshotPixelFormat pixel_format) = 0; + virtual sk_sp MakeTextureImage(sk_sp image, + SnapshotPixelFormat pixel_format) = 0; + virtual sk_sp ConvertToRasterImage(sk_sp image) = 0; /// Load and compile and initial PSO for the provided [runtime_stage]. diff --git a/engine/src/flutter/lib/web_ui/lib/painting.dart b/engine/src/flutter/lib/web_ui/lib/painting.dart index 8bc627ca296..fd305070cc5 100644 --- a/engine/src/flutter/lib/web_ui/lib/painting.dart +++ b/engine/src/flutter/lib/web_ui/lib/painting.dart @@ -791,6 +791,9 @@ void decodeImageFromPixels( allowUpscaling: allowUpscaling, ); +Image decodeImageFromPixelsSync(Uint8List pixels, int width, int height, PixelFormat format) => + throw UnimplementedError('`decodeImageFromPixelsSync` is not implemented for web targets.'); + class Shadow { const Shadow({ this.color = const Color(_kColorDefault), diff --git a/engine/src/flutter/shell/common/rasterizer.cc b/engine/src/flutter/shell/common/rasterizer.cc index 4b07d4f3c89..bfde8890c3a 100644 --- a/engine/src/flutter/shell/common/rasterizer.cc +++ b/engine/src/flutter/shell/common/rasterizer.cc @@ -457,6 +457,12 @@ sk_sp Rasterizer::ConvertToRasterImage(sk_sp image) { return snapshot_controller_->ConvertToRasterImage(image); } +sk_sp Rasterizer::MakeTextureImage(sk_sp image, + SnapshotPixelFormat pixel_format) { + TRACE_EVENT0("flutter", __FUNCTION__); + return snapshot_controller_->MakeTextureImage(image, pixel_format); +} + // |SnapshotDelegate| void Rasterizer::CacheRuntimeStage( const std::shared_ptr& runtime_stage) { diff --git a/engine/src/flutter/shell/common/rasterizer.h b/engine/src/flutter/shell/common/rasterizer.h index 190d565e8c9..a4819605e1d 100644 --- a/engine/src/flutter/shell/common/rasterizer.h +++ b/engine/src/flutter/shell/common/rasterizer.h @@ -658,6 +658,10 @@ class Rasterizer final : public SnapshotDelegate, // |SnapshotDelegate| sk_sp ConvertToRasterImage(sk_sp image) override; + // |SnapshotDelegate| + sk_sp MakeTextureImage(sk_sp image, + SnapshotPixelFormat pixel_format) override; + // |SnapshotDelegate| void CacheRuntimeStage( const std::shared_ptr& runtime_stage) override; diff --git a/engine/src/flutter/shell/common/snapshot_controller.h b/engine/src/flutter/shell/common/snapshot_controller.h index 6346797d04d..9802cb186c3 100644 --- a/engine/src/flutter/shell/common/snapshot_controller.h +++ b/engine/src/flutter/shell/common/snapshot_controller.h @@ -52,6 +52,13 @@ class SnapshotController { DlISize picture_size, SnapshotPixelFormat pixel_format) = 0; + /// Creates a texture-backed DlImage from the provided SkImage. + /// + /// This is primarily used by `decodeImageFromPixelsSync` to upload pixels + /// to the GPU synchronously. + virtual sk_sp MakeTextureImage(sk_sp image, + SnapshotPixelFormat pixel_format) = 0; + virtual sk_sp ConvertToRasterImage(sk_sp image) = 0; virtual void CacheRuntimeStage( diff --git a/engine/src/flutter/shell/common/snapshot_controller_impeller.cc b/engine/src/flutter/shell/common/snapshot_controller_impeller.cc index 02c8883cb32..6a74ffd288c 100644 --- a/engine/src/flutter/shell/common/snapshot_controller_impeller.cc +++ b/engine/src/flutter/shell/common/snapshot_controller_impeller.cc @@ -167,6 +167,75 @@ sk_sp SnapshotControllerImpeller::MakeRasterSnapshotSync( pixel_format); } +sk_sp SnapshotControllerImpeller::MakeTextureImage( + sk_sp image, + SnapshotPixelFormat pixel_format) { + auto aiks_context = GetDelegate().GetAiksContext(); + if (!aiks_context) { + return nullptr; + } + auto context = aiks_context->GetContext(); + if (!context) { + return nullptr; + } + + impeller::TextureDescriptor desc; + desc.storage_mode = impeller::StorageMode::kDevicePrivate; + desc.format = impeller::PixelFormat::kR8G8B8A8UNormInt; + desc.size = impeller::ISize(image->width(), image->height()); + desc.mip_count = 1; + + auto texture = context->GetResourceAllocator()->CreateTexture(desc); + if (!texture) { + return nullptr; + } + + size_t byte_size = image->width() * image->height() * 4; + auto buffer = context->GetResourceAllocator()->CreateBuffer( + impeller::DeviceBufferDescriptor{ + .storage_mode = impeller::StorageMode::kHostVisible, + .size = byte_size, + }); + + if (!buffer) { + return nullptr; + } + + { + uint8_t* map = buffer->OnGetContents(); + if (!map) { + return nullptr; + } + SkImageInfo info = + SkImageInfo::Make(image->width(), image->height(), + kRGBA_8888_SkColorType, kPremul_SkAlphaType); + if (!image->readPixels(info, map, image->width() * 4, 0, 0)) { + return nullptr; + } + buffer->Flush(impeller::Range(0, byte_size)); + } + + auto command_buffer = context->CreateCommandBuffer(); + if (!command_buffer) { + return nullptr; + } + auto blit_pass = command_buffer->CreateBlitPass(); + if (!blit_pass) { + return nullptr; + } + + blit_pass->AddCopy( + impeller::BufferView{buffer, impeller::Range(0, byte_size)}, texture); + blit_pass->EncodeCommands(); + + if (!context->GetCommandQueue()->Submit({command_buffer}).ok()) { + return nullptr; + } + + return impeller::DlImageImpeller::Make(texture, + DlImage::OwningContext::kRaster); +} + void SnapshotControllerImpeller::CacheRuntimeStage( const std::shared_ptr& runtime_stage) { if (!GetDelegate().IsAiksContextInitialized()) { diff --git a/engine/src/flutter/shell/common/snapshot_controller_impeller.h b/engine/src/flutter/shell/common/snapshot_controller_impeller.h index 3425b0f6bdd..096c211fdd6 100644 --- a/engine/src/flutter/shell/common/snapshot_controller_impeller.h +++ b/engine/src/flutter/shell/common/snapshot_controller_impeller.h @@ -26,6 +26,9 @@ class SnapshotControllerImpeller : public SnapshotController { DlISize picture_size, SnapshotPixelFormat pixel_format) override; + sk_sp MakeTextureImage(sk_sp image, + SnapshotPixelFormat pixel_format) override; + sk_sp ConvertToRasterImage(sk_sp image) override; void CacheRuntimeStage( diff --git a/engine/src/flutter/shell/common/snapshot_controller_skia.cc b/engine/src/flutter/shell/common/snapshot_controller_skia.cc index d77a1ca937f..33d4f1ce33b 100644 --- a/engine/src/flutter/shell/common/snapshot_controller_skia.cc +++ b/engine/src/flutter/shell/common/snapshot_controller_skia.cc @@ -148,6 +148,12 @@ sk_sp SnapshotControllerSkia::MakeRasterSnapshotSync( }); } +sk_sp SnapshotControllerSkia::MakeTextureImage( + sk_sp image, + SnapshotPixelFormat pixel_format) { + return DlImage::Make(image); +} + sk_sp SnapshotControllerSkia::ConvertToRasterImage( sk_sp image) { // If the rasterizer does not have a surface with a GrContext, then it will diff --git a/engine/src/flutter/shell/common/snapshot_controller_skia.h b/engine/src/flutter/shell/common/snapshot_controller_skia.h index 9cc9f9b0d5f..b88d8e26fe9 100644 --- a/engine/src/flutter/shell/common/snapshot_controller_skia.h +++ b/engine/src/flutter/shell/common/snapshot_controller_skia.h @@ -27,6 +27,9 @@ class SnapshotControllerSkia : public SnapshotController { DlISize size, SnapshotPixelFormat pixel_format) override; + sk_sp MakeTextureImage(sk_sp image, + SnapshotPixelFormat pixel_format) override; + virtual sk_sp ConvertToRasterImage(sk_sp image) override; void CacheRuntimeStage( diff --git a/engine/src/flutter/testing/dart/BUILD.gn b/engine/src/flutter/testing/dart/BUILD.gn index f901ae512bd..2f903b1bb90 100644 --- a/engine/src/flutter/testing/dart/BUILD.gn +++ b/engine/src/flutter/testing/dart/BUILD.gn @@ -13,6 +13,7 @@ tests = [ "color_test.dart", "compositing_test.dart", "dart_test.dart", + "decode_image_from_pixels_sync_test.dart", "encoding_test.dart", "fragment_shader_test.dart", "geometry_test.dart", diff --git a/engine/src/flutter/testing/dart/decode_image_from_pixels_sync_test.dart b/engine/src/flutter/testing/dart/decode_image_from_pixels_sync_test.dart new file mode 100644 index 00000000000..0ba9f905f64 --- /dev/null +++ b/engine/src/flutter/testing/dart/decode_image_from_pixels_sync_test.dart @@ -0,0 +1,63 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:typed_data'; +import 'dart:ui'; + +import 'package:test/test.dart'; +import 'impeller_enabled.dart'; + +void main() { + test('decodeImageFromPixelsSync decodes RGBA8888', () async { + const width = 2; + const height = 2; + // 2x2 red image + final pixels = Uint8List.fromList([ + 255, + 0, + 0, + 255, + 255, + 0, + 0, + 255, + 255, + 0, + 0, + 255, + 255, + 0, + 0, + 255, + ]); + + final Image image = decodeImageFromPixelsSync(pixels, width, height, PixelFormat.rgba8888); + + expect(image.width, width); + expect(image.height, height); + + final ByteData? data = await image.toByteData(); + expect(data, isNotNull); + final Uint8List resultPixels = data!.buffer.asUint8List(); + expect(resultPixels, pixels); + + image.dispose(); + }, skip: !impellerEnabled); + + test('decodeImageFromPixelsSync throws on invalid dimensions', () { + final pixels = Uint8List(4); + expect( + () => decodeImageFromPixelsSync(pixels, 0, 1, PixelFormat.rgba8888), + throwsA(isA()), // Throws string error from C++ + ); + }, skip: !impellerEnabled); + + test('decodeImageFromPixelsSync throws if not Impeller', () { + final pixels = Uint8List(4); + expect( + () => decodeImageFromPixelsSync(pixels, 1, 1, PixelFormat.rgba8888), + throwsA(isA()), + ); + }, skip: impellerEnabled); +}