// Copyright 2015 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "flutter/shell/platform/android/platform_view_android.h" #include #include #include #include #include #include "base/android/jni_array.h" #include "base/android/jni_string.h" #include "flutter/common/threads.h" #include "flutter/runtime/dart_service_isolate.h" #include "flutter/shell/gpu/gpu_rasterizer.h" #include "flutter/shell/platform/android/android_surface_gl.h" #include "flutter/shell/platform/android/vsync_waiter_android.h" #include "jni/FlutterView_jni.h" #include "lib/ftl/functional/make_copyable.h" #if SHELL_ENABLE_VULKAN #include "flutter/shell/platform/android/android_surface_vulkan.h" #endif // SHELL_ENABLE_VULKAN namespace shell { class PlatformMessageResponseAndroid : public blink::PlatformMessageResponse { FRIEND_MAKE_REF_COUNTED(PlatformMessageResponseAndroid); public: void Complete(std::vector data) override { ftl::RefPtr self(this); blink::Threads::Platform()->PostTask( ftl::MakeCopyable([ self, data = std::move(data) ]() mutable { if (!self->view_) return; static_cast(self->view_.get()) ->HandlePlatformMessageResponse(self->response_id_, std::move(data)); })); } void CompleteWithError() override { Complete(std::vector()); } private: PlatformMessageResponseAndroid(int response_id, ftl::WeakPtr view) : response_id_(response_id), view_(view) {} int response_id_; ftl::WeakPtr view_; }; static std::unique_ptr InitializePlatformSurfaceGL() { const PlatformView::SurfaceConfig offscreen_config = { .red_bits = 8, .green_bits = 8, .blue_bits = 8, .alpha_bits = 8, .depth_bits = 0, .stencil_bits = 0, }; auto surface = std::make_unique(offscreen_config); return surface->IsOffscreenContextValid() ? std::move(surface) : nullptr; } static std::unique_ptr InitializePlatformSurfaceVulkan() { #if SHELL_ENABLE_VULKAN auto surface = std::make_unique(); return surface->IsValid() ? std::move(surface) : nullptr; #else // SHELL_ENABLE_VULKAN return nullptr; #endif // SHELL_ENABLE_VULKAN } static std::unique_ptr InitializePlatformSurface() { if (auto surface = InitializePlatformSurfaceVulkan()) { FTL_DLOG(INFO) << "Vulkan surface initialized."; return surface; } FTL_DLOG(INFO) << "Could not initialize Vulkan surface. Falling back to OpenGL."; if (auto surface = InitializePlatformSurfaceGL()) { FTL_DLOG(INFO) << "GL surface initialized."; return surface; } FTL_CHECK(false) << "Could not initialize either the Vulkan or OpenGL " "surface backends. Flutter requires a GPU to render."; return nullptr; } PlatformViewAndroid::PlatformViewAndroid() : PlatformView(std::make_unique(nullptr)), android_surface_(InitializePlatformSurface()) { CreateEngine(); // Eagerly setup the IO thread context. We have already setup the surface. SetupResourceContextOnIOThread(); UpdateThreadPriorities(); PostAddToShellTask(); } PlatformViewAndroid::~PlatformViewAndroid() = default; void PlatformViewAndroid::Detach(JNIEnv* env, jobject obj) { ReleaseSurface(); delete this; } void PlatformViewAndroid::SurfaceCreated(JNIEnv* env, jobject obj, jobject jsurface, jint backgroundColor) { // Note: This frame ensures that any local references used by // ANativeWindow_fromSurface are released immediately. This is needed as a // workaround for https://code.google.com/p/android/issues/detail?id=68174 base::android::ScopedJavaLocalFrame scoped_local_reference_frame(env); auto native_window = ftl::MakeRefCounted( ANativeWindow_fromSurface(env, jsurface)); if (!native_window->IsValid()) { return; } if (!android_surface_->SetNativeWindow(native_window)) { return; } std::unique_ptr gpu_surface = android_surface_->CreateGPUSurface(); if (gpu_surface == nullptr || !gpu_surface->IsValid()) { return; } NotifyCreated(std::move(gpu_surface), [ this, backgroundColor, native_window_size = native_window->GetSize() ] { rasterizer().Clear(backgroundColor, native_window_size); }); } void PlatformViewAndroid::SurfaceChanged(JNIEnv* env, jobject obj, jint width, jint height) { blink::Threads::Gpu()->PostTask([this, width, height]() { if (android_surface_) { android_surface_->OnScreenSurfaceResize(SkISize::Make(width, height)); } }); } void PlatformViewAndroid::UpdateThreadPriorities() { blink::Threads::Gpu()->PostTask( []() { ::setpriority(PRIO_PROCESS, gettid(), -2); }); blink::Threads::UI()->PostTask( []() { ::setpriority(PRIO_PROCESS, gettid(), -1); }); } void PlatformViewAndroid::SurfaceDestroyed(JNIEnv* env, jobject obj) { ReleaseSurface(); } void PlatformViewAndroid::RunBundleAndSnapshot(JNIEnv* env, jobject obj, jstring java_bundle_path, jstring java_snapshot_override) { std::string bundle_path = base::android::ConvertJavaStringToUTF8(env, java_bundle_path); std::string snapshot_override = java_snapshot_override ? base::android::ConvertJavaStringToUTF8(env, java_snapshot_override) : ""; blink::Threads::UI()->PostTask( [ engine = engine_->GetWeakPtr(), bundle_path, snapshot_override ] { if (engine) engine->RunBundleAndSnapshot(bundle_path, snapshot_override); }); } void PlatformViewAndroid::RunBundleAndSource(JNIEnv* env, jobject obj, jstring java_bundle_path, jstring java_main, jstring java_packages) { std::string bundle_path = base::android::ConvertJavaStringToUTF8(env, java_bundle_path); std::string main = base::android::ConvertJavaStringToUTF8(env, java_main); std::string packages = base::android::ConvertJavaStringToUTF8(env, java_packages); blink::Threads::UI()->PostTask( [ engine = engine_->GetWeakPtr(), bundle_path, main, packages ] { if (engine) engine->RunBundleAndSource(bundle_path, main, packages); }); } void PlatformViewAndroid::SetViewportMetrics(JNIEnv* env, jobject obj, jfloat device_pixel_ratio, jint physical_width, jint physical_height, jint physical_padding_top, jint physical_padding_right, jint physical_padding_bottom, jint physical_padding_left) { blink::ViewportMetrics metrics; metrics.device_pixel_ratio = device_pixel_ratio; metrics.physical_width = physical_width; metrics.physical_height = physical_height; metrics.physical_padding_top = physical_padding_top; metrics.physical_padding_right = physical_padding_right; metrics.physical_padding_bottom = physical_padding_bottom; metrics.physical_padding_left = physical_padding_left; blink::Threads::UI()->PostTask([ engine = engine_->GetWeakPtr(), metrics ] { if (engine) engine->SetViewportMetrics(metrics); }); } void PlatformViewAndroid::DispatchPlatformMessage(JNIEnv* env, jobject obj, jstring java_name, jstring java_message_data, jint response_id) { std::string name = base::android::ConvertJavaStringToUTF8(env, java_name); std::string data; if (java_message_data) data = base::android::ConvertJavaStringToUTF8(env, java_message_data); ftl::RefPtr response; if (response_id) { response = ftl::MakeRefCounted( response_id, GetWeakPtr()); } const uint8_t* buffer = reinterpret_cast(data.data()); PlatformView::DispatchPlatformMessage( ftl::MakeRefCounted( std::move(name), std::vector(buffer, buffer + data.size()), std::move(response))); } void PlatformViewAndroid::DispatchPointerDataPacket(JNIEnv* env, jobject obj, jobject buffer, jint position) { uint8_t* data = static_cast(env->GetDirectBufferAddress(buffer)); blink::Threads::UI()->PostTask(ftl::MakeCopyable([ engine = engine_->GetWeakPtr(), packet = std::make_unique(data, position) ] { if (engine.get()) engine->DispatchPointerDataPacket(*packet); })); } void PlatformViewAndroid::InvokePlatformMessageResponseCallback( JNIEnv* env, jobject obj, jint response_id, jstring java_response) { if (!response_id) return; auto it = pending_responses_.find(response_id); if (it == pending_responses_.end()) return; std::string response; if (java_response) response = base::android::ConvertJavaStringToUTF8(env, java_response); auto message_response = std::move(it->second); pending_responses_.erase(it); // TODO(abarth): There's an extra copy here. message_response->Complete( std::vector(response.data(), response.data() + response.size())); } void PlatformViewAndroid::HandlePlatformMessage( ftl::RefPtr message) { JNIEnv* env = base::android::AttachCurrentThread(); base::android::ScopedJavaLocalRef view = flutter_view_.get(env); if (view.is_null()) return; int response_id = 0; if (auto response = message->response()) { response_id = next_response_id_++; pending_responses_[response_id] = response; } auto data = message->data(); base::StringPiece message_data(reinterpret_cast(data.data()), data.size()); auto java_channel = base::android::ConvertUTF8ToJavaString(env, message->channel()); auto java_message_data = base::android::ConvertUTF8ToJavaString(env, message_data); message = nullptr; // This call can re-enter in InvokePlatformMessageResponseCallback. Java_FlutterView_handlePlatformMessage(env, view.obj(), java_channel.obj(), java_message_data.obj(), response_id); } void PlatformViewAndroid::HandlePlatformMessageResponse( int response_id, std::vector data) { JNIEnv* env = base::android::AttachCurrentThread(); base::android::ScopedJavaLocalRef view = flutter_view_.get(env); if (view.is_null()) return; base::StringPiece message_data(reinterpret_cast(data.data()), data.size()); auto java_message_data = base::android::ConvertUTF8ToJavaString(env, message_data); Java_FlutterView_handlePlatformMessageResponse(env, view.obj(), response_id, java_message_data.obj()); } void PlatformViewAndroid::DispatchSemanticsAction(JNIEnv* env, jobject obj, jint id, jint action) { PlatformView::DispatchSemanticsAction( id, static_cast(action)); } void PlatformViewAndroid::SetSemanticsEnabled(JNIEnv* env, jobject obj, jboolean enabled) { PlatformView::SetSemanticsEnabled(enabled); } void PlatformViewAndroid::ReleaseSurface() { NotifyDestroyed(); android_surface_->TeardownOnScreenContext(); } VsyncWaiter* PlatformViewAndroid::GetVsyncWaiter() { if (!vsync_waiter_) vsync_waiter_ = std::make_unique(); return vsync_waiter_.get(); } bool PlatformViewAndroid::ResourceContextMakeCurrent() { return android_surface_->ResourceContextMakeCurrent(); } void PlatformViewAndroid::UpdateSemantics( std::vector update) { constexpr size_t kBytesPerNode = 25 * sizeof(int32_t); constexpr size_t kBytesPerChild = sizeof(int32_t); JNIEnv* env = base::android::AttachCurrentThread(); { base::android::ScopedJavaLocalRef view = flutter_view_.get(env); if (view.is_null()) return; size_t num_bytes = 0; for (const blink::SemanticsNode& node : update) { num_bytes += kBytesPerNode; num_bytes += node.children.size() * kBytesPerChild; } std::vector buffer(num_bytes); int32_t* buffer_int32 = reinterpret_cast(&buffer[0]); float* buffer_float32 = reinterpret_cast(&buffer[0]); std::vector strings; size_t position = 0; for (const blink::SemanticsNode& node : update) { buffer_int32[position++] = node.id; buffer_int32[position++] = node.flags; buffer_int32[position++] = node.actions; if (node.label.empty()) { buffer_int32[position++] = -1; } else { buffer_int32[position++] = strings.size(); strings.push_back(node.label); } buffer_float32[position++] = node.rect.left(); buffer_float32[position++] = node.rect.top(); buffer_float32[position++] = node.rect.right(); buffer_float32[position++] = node.rect.bottom(); node.transform.asColMajorf(&buffer_float32[position]); position += 16; buffer_int32[position++] = node.children.size(); for (int32_t child : node.children) buffer_int32[position++] = child; } Java_FlutterView_updateSemantics( env, view.obj(), env->NewDirectByteBuffer(buffer.data(), buffer.size()), base::android::ToJavaArrayOfStrings(env, strings).obj()); } } void PlatformViewAndroid::RunFromSource(const std::string& assets_directory, const std::string& main, const std::string& packages) { FTL_CHECK(base::android::IsVMInitialized()); JNIEnv* env = base::android::AttachCurrentThread(); FTL_CHECK(env); { base::android::ScopedJavaLocalRef local_flutter_view = flutter_view_.get(env); if (local_flutter_view.is_null()) { // Collected. return; } // Grab the class of the flutter view. jclass flutter_view_class = env->GetObjectClass(local_flutter_view.obj()); FTL_CHECK(flutter_view_class); // Grab the runFromSource method id. jmethodID run_from_source_method_id = env->GetMethodID( flutter_view_class, "runFromSource", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V"); FTL_CHECK(run_from_source_method_id); // Invoke runFromSource on the Android UI thread. jstring java_assets_directory = env->NewStringUTF(assets_directory.c_str()); FTL_CHECK(java_assets_directory); jstring java_main = env->NewStringUTF(main.c_str()); FTL_CHECK(java_main); jstring java_packages = env->NewStringUTF(packages.c_str()); FTL_CHECK(java_packages); env->CallVoidMethod(local_flutter_view.obj(), run_from_source_method_id, java_assets_directory, java_main, java_packages); } // Detaching from the VM deletes any stray local references. base::android::DetachFromVM(); } base::android::ScopedJavaLocalRef PlatformViewAndroid::GetBitmap( JNIEnv* env, jobject obj) { // Render the last frame to an array of pixels on the GPU thread. // The pixels will be returned as a global JNI reference to an int array. ftl::AutoResetWaitableEvent latch; jobject pixels_ref = nullptr; SkISize frame_size; blink::Threads::Gpu()->PostTask([this, &latch, &pixels_ref, &frame_size]() { GetBitmapGpuTask(&pixels_ref, &frame_size); latch.Signal(); }); latch.Wait(); // Convert the pixel array to an Android bitmap. if (pixels_ref == nullptr) return base::android::ScopedJavaLocalRef(); base::android::ScopedJavaGlobalRef pixels(env, pixels_ref); jclass bitmap_class = env->FindClass("android/graphics/Bitmap"); FTL_CHECK(bitmap_class); jmethodID create_bitmap = env->GetStaticMethodID( bitmap_class, "createBitmap", "([IIILandroid/graphics/Bitmap$Config;)Landroid/graphics/Bitmap;"); FTL_CHECK(create_bitmap); jclass bitmap_config_class = env->FindClass("android/graphics/Bitmap$Config"); FTL_CHECK(bitmap_config_class); jmethodID bitmap_config_value_of = env->GetStaticMethodID( bitmap_config_class, "valueOf", "(Ljava/lang/String;)Landroid/graphics/Bitmap$Config;"); FTL_CHECK(bitmap_config_value_of); jstring argb = env->NewStringUTF("ARGB_8888"); FTL_CHECK(argb); jobject bitmap_config = env->CallStaticObjectMethod( bitmap_config_class, bitmap_config_value_of, argb); FTL_CHECK(bitmap_config); jobject bitmap = env->CallStaticObjectMethod( bitmap_class, create_bitmap, pixels.obj(), frame_size.width(), frame_size.height(), bitmap_config); return base::android::ScopedJavaLocalRef(env, bitmap); } void PlatformViewAndroid::GetBitmapGpuTask(jobject* pixels_out, SkISize* size_out) { flow::LayerTree* layer_tree = rasterizer_->GetLastLayerTree(); if (layer_tree == nullptr) return; JNIEnv* env = base::android::AttachCurrentThread(); FTL_CHECK(env); const SkISize& frame_size = layer_tree->frame_size(); jsize pixels_size = frame_size.width() * frame_size.height(); jintArray pixels_array = env->NewIntArray(pixels_size); FTL_CHECK(pixels_array); jint* pixels = env->GetIntArrayElements(pixels_array, nullptr); FTL_CHECK(pixels); SkImageInfo image_info = SkImageInfo::Make(frame_size.width(), frame_size.height(), kRGBA_8888_SkColorType, kPremul_SkAlphaType); sk_sp surface = SkSurface::MakeRasterDirect( image_info, pixels, frame_size.width() * sizeof(jint)); flow::CompositorContext compositor_context(nullptr); SkCanvas* canvas = surface->getCanvas(); flow::CompositorContext::ScopedFrame frame = compositor_context.AcquireFrame(nullptr, canvas, false); canvas->clear(SK_ColorBLACK); layer_tree->Raster(frame); canvas->flush(); // Our configuration of Skia does not support rendering to the // BitmapConfig.ARGB_8888 format expected by android.graphics.Bitmap. // Convert from kRGBA_8888 to kBGRA_8888 (equivalent to ARGB_8888). for (int i = 0; i < pixels_size; i++) { uint8_t* bytes = reinterpret_cast(pixels + i); std::swap(bytes[0], bytes[2]); } env->ReleaseIntArrayElements(pixels_array, pixels, 0); *pixels_out = env->NewGlobalRef(pixels_array); *size_out = frame_size; base::android::DetachFromVM(); } jstring GetObservatoryUri(JNIEnv* env, jclass clazz) { return env->NewStringUTF( blink::DartServiceIsolate::GetObservatoryUri().c_str()); } bool PlatformViewAndroid::Register(JNIEnv* env) { return RegisterNativesImpl(env); } static jlong Attach(JNIEnv* env, jclass clazz, jobject flutterView) { PlatformViewAndroid* view = new PlatformViewAndroid(); // Create a weak reference to the flutterView Java object so that we can make // calls into it later. view->set_flutter_view(JavaObjectWeakGlobalRef(env, flutterView)); return reinterpret_cast(view); } } // namespace shell