Add native Android image decoder supported by API 28+ (flutter/engine#26746)

This commit is contained in:
Brandon DeRosier 2021-07-21 13:52:29 -07:00 committed by GitHub
parent eafe109ed0
commit e2c58529b5
26 changed files with 596 additions and 91 deletions

View File

@ -759,6 +759,8 @@ FILE: ../../../flutter/shell/platform/android/android_environment_gl.h
FILE: ../../../flutter/shell/platform/android/android_exports.lst
FILE: ../../../flutter/shell/platform/android/android_external_texture_gl.cc
FILE: ../../../flutter/shell/platform/android/android_external_texture_gl.h
FILE: ../../../flutter/shell/platform/android/android_image_generator.cc
FILE: ../../../flutter/shell/platform/android/android_image_generator.h
FILE: ../../../flutter/shell/platform/android/android_shell_holder.cc
FILE: ../../../flutter/shell/platform/android/android_shell_holder.h
FILE: ../../../flutter/shell/platform/android/android_surface_gl.cc

View File

@ -173,6 +173,17 @@ bool ClearException(JNIEnv* env) {
return true;
}
bool CheckException(JNIEnv* env) {
if (!HasException(env))
return true;
jthrowable exception = env->ExceptionOccurred();
env->ExceptionClear();
FML_LOG(ERROR) << fml::jni::GetJavaExceptionInfo(env, exception);
env->DeleteLocalRef(exception);
return false;
}
std::string GetJavaExceptionInfo(JNIEnv* env, jthrowable java_throwable) {
ScopedJavaLocalRef<jclass> throwable_clazz(
env, env->FindClass("java/lang/Throwable"));

View File

@ -42,6 +42,7 @@ bool HasException(JNIEnv* env);
bool ClearException(JNIEnv* env);
bool CheckException(JNIEnv* env);
std::string GetJavaExceptionInfo(JNIEnv* env, jthrowable java_throwable);
} // namespace jni

View File

@ -149,7 +149,7 @@ class UnknownImageGenerator : public ImageGenerator {
public:
UnknownImageGenerator() : info_(SkImageInfo::MakeUnknown()){};
~UnknownImageGenerator() = default;
const SkImageInfo& GetInfo() const { return info_; }
const SkImageInfo& GetInfo() { return info_; }
unsigned int GetFrameCount() const { return 1; }
@ -159,7 +159,7 @@ class UnknownImageGenerator : public ImageGenerator {
return {std::nullopt, 0, SkCodecAnimation::DisposalMethod::kKeep};
}
SkISize GetScaledDimensions(float scale) const {
SkISize GetScaledDimensions(float scale) {
return SkISize::Make(info_.width(), info_.height());
}
@ -167,7 +167,7 @@ class UnknownImageGenerator : public ImageGenerator {
void* pixels,
size_t row_bytes,
unsigned int frame_index,
std::optional<unsigned int> prior_frame) const {
std::optional<unsigned int> prior_frame) {
return false;
};
@ -236,7 +236,7 @@ TEST_F(ImageDecoderFixtureTest, ValidImageResultsInSuccess) {
ASSERT_GE(data->size(), 0u);
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> generator =
std::shared_ptr<ImageGenerator> generator =
registry.CreateCompatibleGenerator(data);
ASSERT_TRUE(generator);
@ -293,7 +293,7 @@ TEST_F(ImageDecoderFixtureTest, ExifDataIsRespectedOnDecode) {
ASSERT_GE(data->size(), 0u);
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> generator =
std::shared_ptr<ImageGenerator> generator =
registry.CreateCompatibleGenerator(data);
ASSERT_TRUE(generator);
@ -352,7 +352,7 @@ TEST_F(ImageDecoderFixtureTest, CanDecodeWithoutAGPUContext) {
ASSERT_GE(data->size(), 0u);
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> generator =
std::shared_ptr<ImageGenerator> generator =
registry.CreateCompatibleGenerator(data);
ASSERT_TRUE(generator);
@ -427,7 +427,7 @@ TEST_F(ImageDecoderFixtureTest, CanDecodeWithResizes) {
ASSERT_GE(data->size(), 0u);
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> generator =
std::shared_ptr<ImageGenerator> generator =
registry.CreateCompatibleGenerator(data);
ASSERT_TRUE(generator);
@ -593,7 +593,7 @@ TEST(ImageDecoderTest, VerifySimpleDecoding) {
ASSERT_EQ(SkISize::Make(600, 200), image->dimensions());
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> generator =
std::shared_ptr<ImageGenerator> generator =
registry.CreateCompatibleGenerator(data);
ASSERT_TRUE(generator);
@ -610,7 +610,7 @@ TEST(ImageDecoderTest, VerifySubpixelDecodingPreservesExifOrientation) {
auto data = OpenFixtureAsSkData("Horizontal.jpg");
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> generator =
std::shared_ptr<ImageGenerator> generator =
registry.CreateCompatibleGenerator(data);
ASSERT_TRUE(generator);
auto descriptor =
@ -662,7 +662,7 @@ TEST_F(ImageDecoderFixtureTest,
ASSERT_TRUE(gif_mapping);
ImageGeneratorRegistry registry;
std::unique_ptr<ImageGenerator> gif_generator =
std::shared_ptr<ImageGenerator> gif_generator =
registry.CreateCompatibleGenerator(gif_mapping);
ASSERT_TRUE(gif_generator);

View File

@ -47,7 +47,7 @@ ImageDescriptor::ImageDescriptor(sk_sp<SkData> buffer,
row_bytes_(row_bytes) {}
ImageDescriptor::ImageDescriptor(sk_sp<SkData> buffer,
std::unique_ptr<ImageGenerator> generator)
std::shared_ptr<ImageGenerator> generator)
: buffer_(std::move(buffer)),
generator_(std::move(generator)),
image_info_(CreateImageInfo()),
@ -83,7 +83,7 @@ void ImageDescriptor::initEncoded(Dart_NativeArguments args) {
return;
}
std::unique_ptr<ImageGenerator> generator =
auto generator =
registry->CreateCompatibleGenerator(immutable_buffer->data());
if (!generator) {
@ -140,20 +140,7 @@ void ImageDescriptor::instantiateCodec(Dart_Handle codec_handle,
}
sk_sp<SkImage> ImageDescriptor::image() const {
SkBitmap bitmap;
if (!bitmap.tryAllocPixels(image_info_)) {
FML_DLOG(ERROR) << "Failed to allocate memory for bitmap of size "
<< image_info_.computeMinByteSize() << "B";
return nullptr;
}
const auto& pixmap = bitmap.pixmap();
if (!get_pixels(pixmap)) {
FML_DLOG(ERROR) << "Failed to get pixels for image.";
return nullptr;
}
bitmap.setImmutable();
return SkImage::MakeFromBitmap(bitmap);
return generator_->GetImage();
}
bool ImageDescriptor::get_pixels(const SkPixmap& pixmap) const {

View File

@ -123,7 +123,7 @@ class ImageDescriptor : public RefCountedDartWrappable<ImageDescriptor> {
const SkImageInfo& image_info,
std::optional<size_t> row_bytes);
ImageDescriptor(sk_sp<SkData> buffer,
std::unique_ptr<ImageGenerator> generator);
std::shared_ptr<ImageGenerator> generator);
sk_sp<SkData> buffer_;
std::shared_ptr<ImageGenerator> generator_;

View File

@ -4,17 +4,38 @@
#include "flutter/lib/ui/painting/image_generator.h"
#include "flutter/fml/logging.h"
namespace flutter {
ImageGenerator::~ImageGenerator() = default;
sk_sp<SkImage> ImageGenerator::GetImage() {
SkImageInfo info = GetInfo();
SkBitmap bitmap;
if (!bitmap.tryAllocPixels(info)) {
FML_DLOG(ERROR) << "Failed to allocate memory for bitmap of size "
<< info.computeMinByteSize() << "B";
return nullptr;
}
const auto& pixmap = bitmap.pixmap();
if (!GetPixels(pixmap.info(), pixmap.writable_addr(), pixmap.rowBytes())) {
FML_DLOG(ERROR) << "Failed to get pixels for image.";
return nullptr;
}
bitmap.setImmutable();
return SkImage::MakeFromBitmap(bitmap);
}
BuiltinSkiaImageGenerator::~BuiltinSkiaImageGenerator() = default;
BuiltinSkiaImageGenerator::BuiltinSkiaImageGenerator(
std::unique_ptr<SkImageGenerator> generator)
: generator_(std::move(generator)) {}
const SkImageInfo& BuiltinSkiaImageGenerator::GetInfo() const {
const SkImageInfo& BuiltinSkiaImageGenerator::GetInfo() {
return generator_->getInfo();
}
@ -33,8 +54,7 @@ const ImageGenerator::FrameInfo BuiltinSkiaImageGenerator::GetFrameInfo(
.disposal_method = SkCodecAnimation::DisposalMethod::kKeep};
}
SkISize BuiltinSkiaImageGenerator::GetScaledDimensions(
float desired_scale) const {
SkISize BuiltinSkiaImageGenerator::GetScaledDimensions(float desired_scale) {
return generator_->getInfo().dimensions();
}
@ -43,12 +63,10 @@ bool BuiltinSkiaImageGenerator::GetPixels(
void* pixels,
size_t row_bytes,
unsigned int frame_index,
std::optional<unsigned int> prior_frame) const {
std::optional<unsigned int> prior_frame) {
return generator_->getPixels(info, pixels, row_bytes);
}
BuiltinSkiaCodecImageGenerator::~BuiltinSkiaCodecImageGenerator() = default;
std::unique_ptr<ImageGenerator> BuiltinSkiaImageGenerator::MakeFromGenerator(
std::unique_ptr<SkImageGenerator> generator) {
if (!generator) {
@ -57,6 +75,8 @@ std::unique_ptr<ImageGenerator> BuiltinSkiaImageGenerator::MakeFromGenerator(
return std::make_unique<BuiltinSkiaImageGenerator>(std::move(generator));
}
BuiltinSkiaCodecImageGenerator::~BuiltinSkiaCodecImageGenerator() = default;
BuiltinSkiaCodecImageGenerator::BuiltinSkiaCodecImageGenerator(
std::unique_ptr<SkCodec> codec)
: codec_generator_(static_cast<SkCodecImageGenerator*>(
@ -67,7 +87,7 @@ BuiltinSkiaCodecImageGenerator::BuiltinSkiaCodecImageGenerator(
: codec_generator_(static_cast<SkCodecImageGenerator*>(
SkCodecImageGenerator::MakeFromEncodedCodec(buffer).release())) {}
const SkImageInfo& BuiltinSkiaCodecImageGenerator::GetInfo() const {
const SkImageInfo& BuiltinSkiaCodecImageGenerator::GetInfo() {
return codec_generator_->getInfo();
}
@ -93,7 +113,7 @@ const ImageGenerator::FrameInfo BuiltinSkiaCodecImageGenerator::GetFrameInfo(
}
SkISize BuiltinSkiaCodecImageGenerator::GetScaledDimensions(
float desired_scale) const {
float desired_scale) {
return codec_generator_->getScaledDimensions(desired_scale);
}
@ -102,7 +122,7 @@ bool BuiltinSkiaCodecImageGenerator::GetPixels(
void* pixels,
size_t row_bytes,
unsigned int frame_index,
std::optional<unsigned int> prior_frame) const {
std::optional<unsigned int> prior_frame) {
SkCodec::Options options;
options.fFrameIndex = frame_index;
if (prior_frame.has_value()) {

View File

@ -48,7 +48,7 @@ class ImageGenerator {
/// @note This method is executed on the UI thread and used for layout
/// purposes by the framework, and so this method should not perform
/// long synchronous tasks.
virtual const SkImageInfo& GetInfo() const = 0;
virtual const SkImageInfo& GetInfo() = 0;
/// @brief Get the number of frames that the encoded image stores. This
/// method is always expected to be called before `GetFrameInfo`, as
@ -87,9 +87,11 @@ class ImageGenerator {
/// @note This method is called prior to `GetPixels` in order to query
/// for supported sizes.
/// @see `GetPixels`
virtual SkISize GetScaledDimensions(float scale) const = 0;
virtual SkISize GetScaledDimensions(float scale) = 0;
/// @brief Decode the image into a given buffer.
/// @brief Decode the image into a given buffer. This method is currently
/// always used for sub-pixel image decoding. For full-sized still
/// images, `GetImage` is always attempted first.
/// @param[in] info The desired size and color info of the decoded
/// image to be returned. The implementation of
/// `GetScaledDimensions` determines which sizes are
@ -119,7 +121,12 @@ class ImageGenerator {
void* pixels,
size_t row_bytes,
unsigned int frame_index = 0,
std::optional<unsigned int> prior_frame = std::nullopt) const = 0;
std::optional<unsigned int> prior_frame = std::nullopt) = 0;
/// @brief Creates an `SkImage` based on the current `ImageInfo` of this
/// `ImageGenerator`.
/// @return A new `SkImage` containing the decoded image data.
sk_sp<SkImage> GetImage();
};
class BuiltinSkiaImageGenerator : public ImageGenerator {
@ -129,7 +136,7 @@ class BuiltinSkiaImageGenerator : public ImageGenerator {
BuiltinSkiaImageGenerator(std::unique_ptr<SkImageGenerator> generator);
// |ImageGenerator|
const SkImageInfo& GetInfo() const override;
const SkImageInfo& GetInfo() override;
// |ImageGenerator|
unsigned int GetFrameCount() const override;
@ -142,7 +149,7 @@ class BuiltinSkiaImageGenerator : public ImageGenerator {
unsigned int frame_index) const override;
// |ImageGenerator|
SkISize GetScaledDimensions(float desired_scale) const override;
SkISize GetScaledDimensions(float desired_scale) override;
// |ImageGenerator|
bool GetPixels(
@ -150,7 +157,7 @@ class BuiltinSkiaImageGenerator : public ImageGenerator {
void* pixels,
size_t row_bytes,
unsigned int frame_index = 0,
std::optional<unsigned int> prior_frame = std::nullopt) const override;
std::optional<unsigned int> prior_frame = std::nullopt) override;
static std::unique_ptr<ImageGenerator> MakeFromGenerator(
std::unique_ptr<SkImageGenerator> generator);
@ -169,7 +176,7 @@ class BuiltinSkiaCodecImageGenerator : public ImageGenerator {
BuiltinSkiaCodecImageGenerator(sk_sp<SkData> buffer);
// |ImageGenerator|
const SkImageInfo& GetInfo() const override;
const SkImageInfo& GetInfo() override;
// |ImageGenerator|
unsigned int GetFrameCount() const override;
@ -182,7 +189,7 @@ class BuiltinSkiaCodecImageGenerator : public ImageGenerator {
unsigned int frame_index) const override;
// |ImageGenerator|
SkISize GetScaledDimensions(float desired_scale) const override;
SkISize GetScaledDimensions(float desired_scale) override;
// |ImageGenerator|
bool GetPixels(
@ -190,7 +197,7 @@ class BuiltinSkiaCodecImageGenerator : public ImageGenerator {
void* pixels,
size_t row_bytes,
unsigned int frame_index = 0,
std::optional<unsigned int> prior_frame = std::nullopt) const override;
std::optional<unsigned int> prior_frame = std::nullopt) override;
static std::unique_ptr<ImageGenerator> MakeFromData(sk_sp<SkData> data);

View File

@ -52,7 +52,7 @@ void ImageGeneratorRegistry::AddFactory(ImageGeneratorFactory factory,
{factory, priority, fml::tracing::TraceNonce()});
}
std::unique_ptr<ImageGenerator>
std::shared_ptr<ImageGenerator>
ImageGeneratorRegistry::CreateCompatibleGenerator(sk_sp<SkData> buffer) {
if (!image_generator_factories_.size()) {
FML_LOG(WARNING)
@ -64,7 +64,7 @@ ImageGeneratorRegistry::CreateCompatibleGenerator(sk_sp<SkData> buffer) {
}
for (auto& factory : image_generator_factories_) {
std::unique_ptr<ImageGenerator> result = factory.callback(buffer);
std::shared_ptr<ImageGenerator> result = factory.callback(buffer);
if (result) {
return result;
}

View File

@ -14,15 +14,19 @@
namespace flutter {
/// @brief `ImageGeneratorFactory` is the top level primitive for specifying an
/// image decoder in Flutter. When called, it should return an
/// `ImageGenerator` that typically compatible with the given input
/// data.
using ImageGeneratorFactory =
std::function<std::shared_ptr<ImageGenerator>(sk_sp<SkData> buffer)>;
/// @brief Keeps a priority-ordered registry of image generator builders to be
/// used when decoding images. This object must be created, accessed, and
/// collected on the UI thread (typically the engine or its runtime
/// controller).
class ImageGeneratorRegistry {
public:
using ImageGeneratorFactory =
std::function<std::unique_ptr<ImageGenerator>(sk_sp<SkData> buffer)>;
ImageGeneratorRegistry();
~ImageGeneratorRegistry();
@ -49,9 +53,9 @@ class ImageGeneratorRegistry {
/// @param[in] buffer The raw encoded image data.
/// @return An `ImageGenerator` that is compatible with the input buffer.
/// If no compatible `ImageGenerator` type was found, then
/// `std::unique_ptr<ImageGenerator>(nullptr)` is returned.
/// `std::shared_ptr<ImageGenerator>(nullptr)` is returned.
/// @see `ImageGenerator`
std::unique_ptr<ImageGenerator> CreateCompatibleGenerator(
std::shared_ptr<ImageGenerator> CreateCompatibleGenerator(
sk_sp<SkData> buffer);
fml::WeakPtr<ImageGeneratorRegistry> GetWeakPtr() const;

View File

@ -54,7 +54,7 @@ class FakeImageGenerator : public ImageGenerator {
SkColorType::kRGBA_8888_SkColorType,
SkAlphaType::kOpaque_SkAlphaType)){};
~FakeImageGenerator() = default;
const SkImageInfo& GetInfo() const { return info_; }
const SkImageInfo& GetInfo() { return info_; }
unsigned int GetFrameCount() const { return 1; }
@ -64,7 +64,7 @@ class FakeImageGenerator : public ImageGenerator {
return {std::nullopt, 0, SkCodecAnimation::DisposalMethod::kKeep};
}
SkISize GetScaledDimensions(float scale) const {
SkISize GetScaledDimensions(float scale) {
return SkISize::Make(info_.width(), info_.height());
}
@ -72,7 +72,7 @@ class FakeImageGenerator : public ImageGenerator {
void* pixels,
size_t row_bytes,
unsigned int frame_index,
std::optional<unsigned int> prior_frame) const {
std::optional<unsigned int> prior_frame) {
return false;
};

View File

@ -151,6 +151,10 @@ std::shared_ptr<AssetManager> Engine::GetAssetManager() {
return asset_manager_;
}
fml::WeakPtr<ImageGeneratorRegistry> Engine::GetImageGeneratorRegistry() {
return image_generator_registry_.GetWeakPtr();
}
bool Engine::UpdateAssetManager(
std::shared_ptr<AssetManager> new_asset_manager) {
if (asset_manager_ == new_asset_manager) {

View File

@ -805,6 +805,14 @@ class Engine final : public RuntimeDelegate, PointerDataDispatcher::Delegate {
// Return the asset manager associated with the current engine, or nullptr.
std::shared_ptr<AssetManager> GetAssetManager();
//----------------------------------------------------------------------------
/// @brief Get the `ImageGeneratorRegistry` associated with the current
/// engine.
///
/// @return The engine's `ImageGeneratorRegistry`.
///
fml::WeakPtr<ImageGeneratorRegistry> GetImageGeneratorRegistry();
// |PointerDataDispatcher::Delegate|
void DoDispatchPacket(std::unique_ptr<PointerDataPacket> packet,
uint64_t trace_flow_id) override;

View File

@ -163,6 +163,17 @@ void canDecompressImageFromAsset() {
List<int> getFixtureImage() native 'GetFixtureImage';
@pragma('vm:entry-point')
void canRegisterImageDecoders() {
decodeImageFromList(
// The test ImageGenerator will always behave the same regardless of input.
Uint8List(1),
(Image result) {
notifyWidthHeight(result.width, result.height);
},
);
}
void notifyLocalTime(String string) native 'NotifyLocalTime';
bool waitFixture() native 'WaitFixture';

View File

@ -1650,6 +1650,21 @@ double Shell::GetMainDisplayRefreshRate() {
return display_manager_->GetMainDisplayRefreshRate();
}
void Shell::RegisterImageDecoder(ImageGeneratorFactory factory,
int32_t priority) {
FML_DCHECK(task_runners_.GetPlatformTaskRunner()->RunsTasksOnCurrentThread());
FML_DCHECK(is_setup_);
fml::TaskRunner::RunNowOrPostTask(
task_runners_.GetUITaskRunner(),
[engine = engine_->GetWeakPtr(), factory = std::move(factory),
priority]() {
if (engine) {
engine->GetImageGeneratorRegistry()->AddFactory(factory, priority);
}
});
}
bool Shell::OnServiceProtocolGetSkSLs(
const ServiceProtocol::Handler::ServiceProtocolMap& params,
rapidjson::Document* response) {

View File

@ -25,6 +25,7 @@
#include "flutter/fml/synchronization/waitable_event.h"
#include "flutter/fml/thread.h"
#include "flutter/fml/time/time_point.h"
#include "flutter/lib/ui/painting/image_generator_registry.h"
#include "flutter/lib/ui/semantics/custom_accessibility_action.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
#include "flutter/lib/ui/volatile_path_tracker.h"
@ -368,6 +369,21 @@ class Shell final : public PlatformView::Delegate,
///
double GetMainDisplayRefreshRate();
//----------------------------------------------------------------------------
/// @brief Install a new factory that can match against and decode image
/// data.
/// @param[in] factory Callback that produces `ImageGenerator`s for
/// compatible input data.
/// @param[in] priority The priority used to determine the order in which
/// factories are tried. Higher values mean higher
/// priority. The built-in Skia decoders are installed
/// at priority 0, and so a priority > 0 takes precedent
/// over the builtin decoders. When multiple decoders
/// are added with the same priority, those which are
/// added earlier take precedent.
/// @see `CreateCompatibleGenerator`
void RegisterImageDecoder(ImageGeneratorFactory factory, int32_t priority);
private:
using ServiceProtocolHandler =
std::function<bool(const ServiceProtocol::Handler::ServiceProtocolMap&,

View File

@ -1930,6 +1930,75 @@ TEST_F(ShellTest, CanDecompressImageFromAsset) {
DestroyShell(std::move(shell));
}
/// An image generator that always creates a 1x1 single-frame green image.
class SinglePixelImageGenerator : public ImageGenerator {
public:
SinglePixelImageGenerator()
: info_(SkImageInfo::MakeN32(1, 1, SkAlphaType::kOpaque_SkAlphaType)){};
~SinglePixelImageGenerator() = default;
const SkImageInfo& GetInfo() { return info_; }
unsigned int GetFrameCount() const { return 1; }
unsigned int GetPlayCount() const { return 1; }
const ImageGenerator::FrameInfo GetFrameInfo(unsigned int frame_index) const {
return {std::nullopt, 0, SkCodecAnimation::DisposalMethod::kKeep};
}
SkISize GetScaledDimensions(float scale) {
return SkISize::Make(info_.width(), info_.height());
}
bool GetPixels(const SkImageInfo& info,
void* pixels,
size_t row_bytes,
unsigned int frame_index,
std::optional<unsigned int> prior_frame) {
assert(info.width() == 1);
assert(info.height() == 1);
assert(row_bytes == 4);
reinterpret_cast<uint32_t*>(pixels)[0] = 0x00ff00ff;
return true;
};
private:
SkImageInfo info_;
};
TEST_F(ShellTest, CanRegisterImageDecoders) {
fml::AutoResetWaitableEvent latch;
AddNativeCallback("NotifyWidthHeight", CREATE_NATIVE_ENTRY([&](auto args) {
auto width = tonic::DartConverter<int>::FromDart(
Dart_GetNativeArgument(args, 0));
auto height = tonic::DartConverter<int>::FromDart(
Dart_GetNativeArgument(args, 1));
ASSERT_EQ(width, 1);
ASSERT_EQ(height, 1);
latch.Signal();
}));
auto settings = CreateSettingsForFixture();
auto configuration = RunConfiguration::InferFromSettings(settings);
configuration.SetEntrypoint("canRegisterImageDecoders");
std::unique_ptr<Shell> shell = CreateShell(settings);
ASSERT_NE(shell.get(), nullptr);
fml::TaskRunner::RunNowOrPostTask(
shell->GetTaskRunners().GetPlatformTaskRunner(), [&shell]() {
shell->RegisterImageDecoder(
[](sk_sp<SkData> buffer) {
return std::make_unique<SinglePixelImageGenerator>();
},
100);
});
RunEngine(shell.get(), std::move(configuration));
latch.Wait();
DestroyShell(std::move(shell));
}
TEST_F(ShellTest, OnServiceProtocolGetSkSLsWorks) {
fml::ScopedTemporaryDirectory base_dir;
ASSERT_TRUE(base_dir.fd().is_valid());

View File

@ -17,6 +17,24 @@ shell_gpu_configuration("android_gpu_configuration") {
enable_metal = false
}
source_set("image_generator") {
sources = [
"android_image_generator.cc",
"android_image_generator.h",
]
deps = [
"//flutter/fml",
"//flutter/lib/ui:ui",
"//third_party/skia",
]
libs = [
"android",
"jnigraphics",
]
}
shared_library("flutter_shell_native") {
visibility = [ ":*" ]
@ -54,6 +72,7 @@ shared_library("flutter_shell_native") {
deps = [
":android_gpu_configuration",
":icudtl_object",
":image_generator",
"//flutter/assets",
"//flutter/common",
"//flutter/common/graphics",

View File

@ -0,0 +1,202 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "flutter/shell/platform/android/android_image_generator.h"
#include <android/bitmap.h>
#include <android/hardware_buffer.h>
#include "flutter/fml/platform/android/jni_util.h"
namespace flutter {
static fml::jni::ScopedJavaGlobalRef<jclass>* g_flutter_jni_class = nullptr;
static jmethodID g_decode_image_method = nullptr;
AndroidImageGenerator::~AndroidImageGenerator() = default;
AndroidImageGenerator::AndroidImageGenerator(sk_sp<SkData> data)
: data_(data), image_info_(SkImageInfo::MakeUnknown(-1, -1)) {}
const SkImageInfo& AndroidImageGenerator::GetInfo() {
header_decoded_latch_.Wait();
return image_info_;
}
unsigned int AndroidImageGenerator::GetFrameCount() const {
return 1;
}
unsigned int AndroidImageGenerator::GetPlayCount() const {
return 1;
}
const ImageGenerator::FrameInfo AndroidImageGenerator::GetFrameInfo(
unsigned int frame_index) const {
return {.required_frame = std::nullopt,
.duration = 0,
.disposal_method = SkCodecAnimation::DisposalMethod::kKeep};
}
SkISize AndroidImageGenerator::GetScaledDimensions(float desired_scale) {
return GetInfo().dimensions();
}
bool AndroidImageGenerator::GetPixels(const SkImageInfo& info,
void* pixels,
size_t row_bytes,
unsigned int frame_index,
std::optional<unsigned int> prior_frame) {
fully_decoded_latch_.Wait();
if (kRGBA_8888_SkColorType != info.colorType()) {
return false;
}
switch (info.alphaType()) {
case kOpaque_SkAlphaType:
if (kOpaque_SkAlphaType != GetInfo().alphaType()) {
return false;
}
break;
case kPremul_SkAlphaType:
break;
default:
return false;
}
// TODO(bdero): Override `GetImage()` to use `SkImage::FromAHardwareBuffer` on
// API level 30+ once it's updated to do symbol lookups and not get
// preprocessed out in Skia. This will allow for avoiding this copy in
// cases where the result image doesn't need to be resized.
memcpy(pixels, software_decoded_data_->data(),
software_decoded_data_->size());
return true;
}
void AndroidImageGenerator::DecodeImage() {
DoDecodeImage();
header_decoded_latch_.Signal();
fully_decoded_latch_.Signal();
}
void AndroidImageGenerator::DoDecodeImage() {
FML_DCHECK(g_flutter_jni_class);
FML_DCHECK(g_decode_image_method);
// Call FlutterJNI.decodeImage
JNIEnv* env = fml::jni::AttachCurrentThread();
fml::jni::ScopedJavaLocalRef<jobject> direct_buffer(
env, env->NewDirectByteBuffer(const_cast<void*>(data_->data()),
data_->size()));
fml::jni::ScopedJavaGlobalRef<jobject>* bitmap =
new fml::jni::ScopedJavaGlobalRef(
env, env->CallStaticObjectMethod(g_flutter_jni_class->obj(),
g_decode_image_method,
direct_buffer.obj(), (long)this));
FML_CHECK(fml::jni::CheckException(env));
if (bitmap->is_null()) {
return;
}
AndroidBitmapInfo info;
int status;
if ((status = AndroidBitmap_getInfo(env, bitmap->obj(), &info)) < 0) {
FML_DLOG(ERROR) << "Failed to get bitmap info, status=" << status;
return;
}
FML_DCHECK(info.format == ANDROID_BITMAP_FORMAT_RGBA_8888);
// Lock the android buffer in a shared pointer
void* pixel_lock;
if ((status = AndroidBitmap_lockPixels(env, bitmap->obj(), &pixel_lock)) <
0) {
FML_DLOG(ERROR) << "Failed to lock pixels, error=" << status;
return;
}
SkData::ReleaseProc on_release = [](const void* ptr, void* context) -> void {
fml::jni::ScopedJavaGlobalRef<jobject>* bitmap =
reinterpret_cast<fml::jni::ScopedJavaGlobalRef<jobject>*>(context);
auto env = fml::jni::AttachCurrentThread();
AndroidBitmap_unlockPixels(env, bitmap->obj());
};
software_decoded_data_ = SkData::MakeWithProc(
pixel_lock, info.width * info.height * sizeof(uint32_t), on_release,
bitmap);
}
bool AndroidImageGenerator::Register(JNIEnv* env) {
FML_DCHECK(g_flutter_jni_class->is_null());
g_flutter_jni_class = new fml::jni::ScopedJavaGlobalRef<jclass>(
env, env->FindClass("io/flutter/embedding/engine/FlutterJNI"));
FML_DCHECK(!g_flutter_jni_class->is_null());
g_decode_image_method = env->GetStaticMethodID(
g_flutter_jni_class->obj(), "decodeImage",
"(Ljava/nio/ByteBuffer;J)Landroid/graphics/Bitmap;");
FML_DCHECK(g_decode_image_method);
static const JNINativeMethod header_decoded_method = {
.name = "nativeImageHeaderCallback",
.signature = "(JII)V",
.fnPtr = reinterpret_cast<void*>(
&AndroidImageGenerator::NativeImageHeaderCallback),
};
if (env->RegisterNatives(g_flutter_jni_class->obj(), &header_decoded_method,
1) != 0) {
FML_LOG(ERROR)
<< "Failed to register FlutterJNI.nativeImageHeaderCallback method";
return false;
}
return true;
}
std::shared_ptr<ImageGenerator> AndroidImageGenerator::MakeFromData(
sk_sp<SkData> data,
fml::RefPtr<fml::TaskRunner> task_runner) {
std::shared_ptr<AndroidImageGenerator> generator(
new AndroidImageGenerator(std::move(data)));
fml::TaskRunner::RunNowOrPostTask(
task_runner, [generator]() { generator->DecodeImage(); });
if (generator->IsValidImageData()) {
return generator;
}
return nullptr;
}
void AndroidImageGenerator::NativeImageHeaderCallback(JNIEnv* env,
jclass jcaller,
jlong generator_address,
int width,
int height) {
AndroidImageGenerator* generator =
reinterpret_cast<AndroidImageGenerator*>(generator_address);
generator->image_info_ = SkImageInfo::Make(
width, height, kRGBA_8888_SkColorType, kPremul_SkAlphaType);
generator->header_decoded_latch_.Signal();
}
bool AndroidImageGenerator::IsValidImageData() {
// The generator kicks off an IO task to decode everything, and calls to
// "GetInfo()" block until either the header has been decoded or decoding has
// failed, whichever is sooner. The decoder is initialized with a width and
// height of -1 and will update the dimensions if the image is able to be
// decoded.
return GetInfo().height() != -1;
}
} // namespace flutter

View File

@ -0,0 +1,84 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef FLUTTER_SHELL_PLATFORM_ANDROID_ANDROID_IMAGE_GENERATOR_H_
#define FLUTTER_SHELL_PLATFORM_ANDROID_ANDROID_IMAGE_GENERATOR_H_
#include <jni.h>
#include "flutter/fml/memory/ref_ptr.h"
#include "flutter/fml/synchronization/waitable_event.h"
#include "flutter/fml/task_runner.h"
#include "flutter/lib/ui/painting/image_generator.h"
namespace flutter {
class AndroidImageGenerator : public ImageGenerator {
private:
AndroidImageGenerator(sk_sp<SkData> buffer);
public:
~AndroidImageGenerator();
// |ImageGenerator|
const SkImageInfo& GetInfo() override;
// |ImageGenerator|
unsigned int GetFrameCount() const override;
// |ImageGenerator|
unsigned int GetPlayCount() const override;
// |ImageGenerator|
const ImageGenerator::FrameInfo GetFrameInfo(
unsigned int frame_index) const override;
// |ImageGenerator|
SkISize GetScaledDimensions(float desired_scale) override;
// |ImageGenerator|
bool GetPixels(
const SkImageInfo& info,
void* pixels,
size_t row_bytes,
unsigned int frame_index = 0,
std::optional<unsigned int> prior_frame = std::nullopt) override;
void DecodeImage();
static bool Register(JNIEnv* env);
static std::shared_ptr<ImageGenerator> MakeFromData(
sk_sp<SkData> data,
fml::RefPtr<fml::TaskRunner> task_runner);
static void NativeImageHeaderCallback(JNIEnv* env,
jclass jcaller,
jlong generator_pointer,
int width,
int height);
private:
sk_sp<SkData> data_;
sk_sp<SkData> software_decoded_data_;
SkImageInfo image_info_;
/// Blocks until the header of the image has been decoded and the image
/// dimensions have been determined.
fml::ManualResetWaitableEvent header_decoded_latch_;
/// Blocks until the image has been fully decoded.
fml::ManualResetWaitableEvent fully_decoded_latch_;
void DoDecodeImage();
bool IsValidImageData();
FML_DISALLOW_COPY_ASSIGN_AND_MOVE(AndroidImageGenerator);
};
} // namespace flutter
#endif // FLUTTER_SHELL_PLATFORM_ANDROID_ANDROID_IMAGE_GENERATOR_H_

View File

@ -19,10 +19,13 @@
#include "flutter/fml/logging.h"
#include "flutter/fml/make_copyable.h"
#include "flutter/fml/message_loop.h"
#include "flutter/fml/native_library.h"
#include "flutter/fml/platform/android/jni_util.h"
#include "flutter/lib/ui/painting/image_generator_registry.h"
#include "flutter/shell/common/rasterizer.h"
#include "flutter/shell/common/run_configuration.h"
#include "flutter/shell/common/thread_host.h"
#include "flutter/shell/platform/android/android_image_generator.h"
#include "flutter/shell/platform/android/context/android_context.h"
#include "flutter/shell/platform/android/platform_view_android.h"
@ -135,6 +138,13 @@ AndroidShellHolder::AndroidShellHolder(
FML_LOG(ERROR) << "Failed to set Workers task runner priority";
}
});
shell_->RegisterImageDecoder(
[runner = task_runners.GetIOTaskRunner()](sk_sp<SkData> buffer) {
return AndroidImageGenerator::MakeFromData(buffer, runner);
},
-1);
FML_DLOG(INFO) << "Registered Android SDK image decoder (API level 28+)";
}
platform_view_ = weak_platform_view;
@ -303,4 +313,5 @@ std::optional<RunConfiguration> AndroidShellHolder::BuildRunConfiguration(
}
return config;
}
} // namespace flutter

View File

@ -128,6 +128,8 @@ class AndroidShellHolder {
const std::string& entrypoint,
const std::string& libraryUrl) const;
bool IsNDKImageDecoderAvailable();
FML_DISALLOW_COPY_AND_ASSIGN(AndroidShellHolder);
};

View File

@ -7,9 +7,12 @@ package io.flutter.embedding.engine;
import android.content.Context;
import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.graphics.ColorSpace;
import android.graphics.ImageDecoder;
import android.graphics.SurfaceTexture;
import android.os.Build;
import android.os.Looper;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceHolder;
import androidx.annotation.Keep;
@ -30,6 +33,7 @@ import io.flutter.plugin.platform.PlatformViewsController;
import io.flutter.util.Preconditions;
import io.flutter.view.AccessibilityBridge;
import io.flutter.view.FlutterCallbackInformation;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
@ -407,6 +411,45 @@ public class FlutterJNI {
flutterUiDisplayListeners.remove(listener);
}
public static native void nativeImageHeaderCallback(
long imageGeneratorPointer, int width, int height);
/**
* Called by native as a fallback method of image decoding. There are other ways to decode images
* on lower API levels, they involve copying the native data _and_ do not support any additional
* formats, whereas ImageDecoder supports HEIF images. Unlike most other methods called from
* native, this method is expected to be called on a worker thread, since it only uses thread safe
* methods and may take multiple frames to complete.
*/
@SuppressWarnings("unused")
@VisibleForTesting
@Nullable
public static Bitmap decodeImage(@NonNull ByteBuffer buffer, long imageGeneratorAddress) {
if (Build.VERSION.SDK_INT >= 28) {
ImageDecoder.Source source = ImageDecoder.createSource(buffer);
try {
return ImageDecoder.decodeBitmap(
source,
(decoder, info, src) -> {
// i.e. ARGB_8888
decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB));
// TODO(bdero): Switch to ALLOCATOR_HARDWARE for devices that have
// `AndroidBitmap_getHardwareBuffer` (API 30+) available once Skia supports
// `SkImage::MakeFromAHardwareBuffer` via dynamic lookups:
// https://skia-review.googlesource.com/c/skia/+/428960
decoder.setAllocator(ImageDecoder.ALLOCATOR_SOFTWARE);
Size size = info.getSize();
nativeImageHeaderCallback(imageGeneratorAddress, size.getWidth(), size.getHeight());
});
} catch (IOException e) {
Log.e(TAG, "Failed to decode image", e);
return null;
}
}
return null;
}
// Called by native to notify first Flutter frame rendered.
@SuppressWarnings("unused")
@VisibleForTesting

View File

@ -3,6 +3,7 @@
// found in the LICENSE file.
#include "flutter/fml/platform/android/jni_util.h"
#include "flutter/shell/platform/android/android_image_generator.h"
#include "flutter/shell/platform/android/flutter_main.h"
#include "flutter/shell/platform/android/platform_view_android.h"
#include "flutter/shell/platform/android/vsync_waiter_android.h"
@ -27,5 +28,9 @@ JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) {
result = flutter::VsyncWaiterAndroid::Register(env);
FML_CHECK(result);
// Register AndroidImageDecoder.
result = flutter::AndroidImageGenerator::Register(env);
FML_CHECK(result);
return JNI_VERSION_1_4;
}

View File

@ -37,22 +37,6 @@
namespace flutter {
namespace {
bool CheckException(JNIEnv* env) {
if (env->ExceptionCheck() == JNI_FALSE) {
return true;
}
jthrowable exception = env->ExceptionOccurred();
env->ExceptionClear();
FML_LOG(ERROR) << fml::jni::GetJavaExceptionInfo(env, exception);
env->DeleteLocalRef(exception);
return false;
}
} // anonymous namespace
static fml::jni::ScopedJavaGlobalRef<jclass>* g_flutter_callback_info_class =
nullptr;
@ -1119,7 +1103,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewHandlePlatformMessage(
java_channel.obj(), nullptr, responseId);
}
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewHandlePlatformMessageResponse(
@ -1150,7 +1134,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewHandlePlatformMessageResponse(
data_array.obj());
}
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewUpdateSemantics(
@ -1175,7 +1159,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewUpdateSemantics(
direct_buffer.obj(), jstrings.obj(),
jstring_attribute_args.obj());
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewUpdateCustomAccessibilityActions(
@ -1199,7 +1183,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewUpdateCustomAccessibilityActions(
g_update_custom_accessibility_actions_method,
direct_actions_buffer.obj(), jstrings.obj());
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewOnFirstFrame() {
@ -1212,7 +1196,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewOnFirstFrame() {
env->CallVoidMethod(java_object.obj(), g_on_first_frame_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewOnPreEngineRestart() {
@ -1225,7 +1209,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewOnPreEngineRestart() {
env->CallVoidMethod(java_object.obj(), g_on_engine_restart_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::SurfaceTextureAttachToGLContext(
@ -1242,7 +1226,7 @@ void PlatformViewAndroidJNIImpl::SurfaceTextureAttachToGLContext(
env->CallVoidMethod(surface_texture_local_ref.obj(),
g_attach_to_gl_context_method, textureId);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::SurfaceTextureUpdateTexImage(
@ -1258,7 +1242,7 @@ void PlatformViewAndroidJNIImpl::SurfaceTextureUpdateTexImage(
env->CallVoidMethod(surface_texture_local_ref.obj(),
g_update_tex_image_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
// The bounds we set for the canvas are post composition.
@ -1289,7 +1273,7 @@ void PlatformViewAndroidJNIImpl::SurfaceTextureGetTransformMatrix(
env->CallVoidMethod(surface_texture_local_ref.obj(),
g_get_transform_matrix_method, transformMatrix.obj());
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
float* m = env->GetFloatArrayElements(transformMatrix.obj(), nullptr);
float scaleX = m[0], scaleY = m[5];
@ -1316,7 +1300,7 @@ void PlatformViewAndroidJNIImpl::SurfaceTextureDetachFromGLContext(
env->CallVoidMethod(surface_texture_local_ref.obj(),
g_detach_from_gl_context_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewOnDisplayPlatformView(
@ -1395,7 +1379,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewOnDisplayPlatformView(
view_id, x, y, width, height, viewWidth, viewHeight,
mutatorsStack);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewDisplayOverlaySurface(
@ -1414,7 +1398,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewDisplayOverlaySurface(
env->CallVoidMethod(java_object.obj(), g_on_display_overlay_surface_method,
surface_id, x, y, width, height);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewBeginFrame() {
@ -1427,7 +1411,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewBeginFrame() {
env->CallVoidMethod(java_object.obj(), g_on_begin_frame_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
void PlatformViewAndroidJNIImpl::FlutterViewEndFrame() {
@ -1440,7 +1424,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewEndFrame() {
env->CallVoidMethod(java_object.obj(), g_on_end_frame_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
std::unique_ptr<PlatformViewAndroidJNI::OverlayMetadata>
@ -1455,7 +1439,7 @@ PlatformViewAndroidJNIImpl::FlutterViewCreateOverlaySurface() {
fml::jni::ScopedJavaLocalRef<jobject> overlay(
env, env->CallObjectMethod(java_object.obj(),
g_create_overlay_surface_method));
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
if (overlay.is_null()) {
return std::make_unique<PlatformViewAndroidJNI::OverlayMetadata>(0,
@ -1485,7 +1469,7 @@ void PlatformViewAndroidJNIImpl::FlutterViewDestroyOverlaySurfaces() {
env->CallVoidMethod(java_object.obj(), g_destroy_overlay_surfaces_method);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
}
std::unique_ptr<std::vector<std::string>>
@ -1506,7 +1490,7 @@ PlatformViewAndroidJNIImpl::FlutterViewComputePlatformResolvedLocale(
java_object.obj(), g_compute_platform_resolved_locale_method,
j_locales_data.obj()));
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
int length = env->GetArrayLength(result);
for (int i = 0; i < length; i++) {
@ -1545,7 +1529,7 @@ bool PlatformViewAndroidJNIImpl::RequestDartDeferredLibrary(
env->CallVoidMethod(java_object.obj(), g_request_dart_deferred_library_method,
loading_unit_id);
FML_CHECK(CheckException(env));
FML_CHECK(fml::jni::CheckException(env));
return true;
}

View File

@ -81,7 +81,7 @@ void main() {
final Codec codec = await descriptor.instantiateCodec();
expect(codec.frameCount, 1);
}, skip: !(Platform.isIOS || Platform.isMacOS || Platform.isWindows));
}, skip: !(Platform.isAndroid || Platform.isIOS || Platform.isMacOS || Platform.isWindows));
}
Future<Uint8List> readFile(String fileName, ) async {