From 5a61efcfffcab99ae6bbe0f2c7b27d6947e029fd Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Thu, 11 Jan 2024 15:32:54 -0800 Subject: [PATCH] Add `SurfaceTextureSurfaceProducer` (flutter/engine#49653) Unblocks https://github.com/flutter/flutter/issues/139702. The next step will be local validation that this API works for at least a few of our internal plugins (video_player, camera). --- .../ci/licenses_golden/licenses_flutter | 1 + .../flutter/shell/platform/android/BUILD.gn | 1 + .../engine/renderer/FlutterRenderer.java | 46 ++++++++-- .../SurfaceTextureSurfaceProducer.java | 84 +++++++++++++++++++ .../io/flutter/view/TextureRegistry.java | 12 +++ .../SurfaceTextureSurfaceProducerTest.java | 61 ++++++++++++++ 6 files changed, 199 insertions(+), 6 deletions(-) create mode 100644 engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java create mode 100644 engine/src/flutter/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java diff --git a/engine/src/flutter/ci/licenses_golden/licenses_flutter b/engine/src/flutter/ci/licenses_golden/licenses_flutter index 8ab8045245f..8b069e99fc6 100644 --- a/engine/src/flutter/ci/licenses_golden/licenses_flutter +++ b/engine/src/flutter/ci/licenses_golden/licenses_flutter @@ -9247,6 +9247,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/plugin FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterUiDisplayListener.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/RenderSurface.java +FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/systemchannels/AccessibilityChannel.java FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/systemchannels/DeferredComponentChannel.java diff --git a/engine/src/flutter/shell/platform/android/BUILD.gn b/engine/src/flutter/shell/platform/android/BUILD.gn index e0ce4c8b4ee..f1fdf7c4ae7 100644 --- a/engine/src/flutter/shell/platform/android/BUILD.gn +++ b/engine/src/flutter/shell/platform/android/BUILD.gn @@ -266,6 +266,7 @@ android_java_sources = [ "io/flutter/embedding/engine/renderer/FlutterRenderer.java", "io/flutter/embedding/engine/renderer/FlutterUiDisplayListener.java", "io/flutter/embedding/engine/renderer/RenderSurface.java", + "io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java", "io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java", "io/flutter/embedding/engine/systemchannels/AccessibilityChannel.java", "io/flutter/embedding/engine/systemchannels/DeferredComponentChannel.java", diff --git a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java index e670bce481b..dd67c7f7185 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java @@ -47,6 +47,16 @@ import java.util.concurrent.atomic.AtomicLong; * io.flutter.embedding.android.FlutterTextureView} are implementations of {@link RenderSurface}. */ public class FlutterRenderer implements TextureRegistry { + /** + * Whether to always use GL textures for {@link FlutterRenderer#createSurfaceProducer()}. + * + *

This is a debug-only API intended for local development. For example, when using a newer + * Android device (that normally would use {@link ImageReaderSurfaceProducer}, but wanting to test + * the OpenGLES/{@link SurfaceTextureSurfaceProducer} code branch. This flag has undefined + * behavior if set to true while running in a Vulkan (Impeller) context. + */ + @VisibleForTesting static boolean debugForceSurfaceProducerGlTextures = false; + private static final String TAG = "FlutterRenderer"; @NonNull private final FlutterJNI flutterJNI; @@ -162,12 +172,36 @@ public class FlutterRenderer implements TextureRegistry { @NonNull @Override public SurfaceProducer createSurfaceProducer() { - // TODO(matanl, johnmccutchan): Implement a SurfaceTexture version and switch on whether or - // not impeller is enabled. - final ImageReaderSurfaceProducer entry = - new ImageReaderSurfaceProducer(nextTextureId.getAndIncrement()); - Log.v(TAG, "New SurfaceProducer ID: " + entry.id()); - registerImageTexture(entry.id(), entry); + // Prior to Impeller, Flutter on Android *only* ran on OpenGLES (via Skia). That meant that + // plugins (i.e. end-users) either explicitly created a SurfaceTexture (via + // createX/registerX) or an ImageTexture (via createX/registerX). + // + // In an Impeller world, which for the first time uses (if available) a Vulkan rendering + // backend, it is no longer possible (at least not trivially) to render an OpenGLES-provided + // texture (SurfaceTexture) in a Vulkan context. + // + // This function picks the "best" rendering surface based on the Android runtime, and + // provides a consumer-agnostic SurfaceProducer (which in turn vends a Surface), and has + // plugins (i.e. end-users) use the Surface instead, letting us "hide" the consumer-side + // of the implementation. + // + // tl;dr: If ImageTexture is available, we use it, otherwise we use a SurfaceTexture. + // Coincidentally, if ImageTexture is available, we are also on an Android version that is + // running Vulkan, so we don't have to worry about it not being supported. + final long id = nextTextureId.getAndIncrement(); + final SurfaceProducer entry; + if (!debugForceSurfaceProducerGlTextures && Build.VERSION.SDK_INT >= 29) { + final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id); + registerImageTexture(id, producer); + Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id); + entry = producer; + } else { + final SurfaceTextureSurfaceProducer producer = + new SurfaceTextureSurfaceProducer(id, handler, flutterJNI); + registerSurfaceTexture(producer.getSurfaceTexture()); + Log.v(TAG, "New SurfaceTextureSurfaceProducer ID: " + id); + entry = producer; + } return entry; } diff --git a/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java new file mode 100644 index 00000000000..50205f51d54 --- /dev/null +++ b/engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java @@ -0,0 +1,84 @@ +package io.flutter.embedding.engine.renderer; + +import android.graphics.SurfaceTexture; +import android.os.Handler; +import android.view.Surface; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import io.flutter.embedding.engine.FlutterJNI; +import io.flutter.view.TextureRegistry; + +/** Uses a {@link android.graphics.SurfaceTexture} to populate the texture registry. */ +final class SurfaceTextureSurfaceProducer + implements TextureRegistry.SurfaceProducer, TextureRegistry.GLTextureConsumer { + private final long id; + private int requestBufferWidth; + private int requestedBufferHeight; + private boolean released; + @Nullable private Surface surface; + @NonNull private final SurfaceTexture texture; + @NonNull private final Handler handler; + @NonNull private final FlutterJNI flutterJNI; + + SurfaceTextureSurfaceProducer(long id, @NonNull Handler handler, @NonNull FlutterJNI flutterJNI) { + this.id = id; + this.handler = handler; + this.flutterJNI = flutterJNI; + this.texture = new SurfaceTexture(0); + } + + @Override + protected void finalize() throws Throwable { + try { + if (released) { + return; + } + release(); + handler.post(new FlutterRenderer.TextureFinalizerRunnable(id, flutterJNI)); + } finally { + super.finalize(); + } + } + + @Override + public long id() { + return id; + } + + @Override + public void release() { + texture.release(); + released = true; + } + + @Override + @NonNull + public SurfaceTexture getSurfaceTexture() { + return texture; + } + + @Override + public void setSize(int width, int height) { + requestBufferWidth = width; + requestedBufferHeight = height; + getSurfaceTexture().setDefaultBufferSize(width, height); + } + + @Override + public int getWidth() { + return requestBufferWidth; + } + + @Override + public int getHeight() { + return requestedBufferHeight; + } + + @Override + public Surface getSurface() { + if (surface == null) { + surface = new Surface(texture); + } + return surface; + } +} diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/TextureRegistry.java b/engine/src/flutter/shell/platform/android/io/flutter/view/TextureRegistry.java index 361f46ceb00..7387e39a379 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/TextureRegistry.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/TextureRegistry.java @@ -159,6 +159,18 @@ public interface TextureRegistry { * * @return Image or null. */ + @Nullable public Image acquireLatestImage(); } + + @Keep + interface GLTextureConsumer extends TextureEntry { + /** + * Retrieve the last GL texture produced. + * + * @return SurfaceTexture. + */ + @NonNull + public SurfaceTexture getSurfaceTexture(); + } } diff --git a/engine/src/flutter/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java b/engine/src/flutter/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java new file mode 100644 index 00000000000..81aae388806 --- /dev/null +++ b/engine/src/flutter/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java @@ -0,0 +1,61 @@ +package io.flutter.embedding.engine.renderer; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.robolectric.Shadows.shadowOf; + +import android.annotation.TargetApi; +import android.graphics.Canvas; +import android.os.Handler; +import android.os.Looper; +import android.view.Surface; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import io.flutter.embedding.engine.FlutterJNI; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +@TargetApi(26) +public final class SurfaceTextureSurfaceProducerTest { + private final FlutterJNI fakeJNI = mock(FlutterJNI.class); + + @Test + public void createsSurfaceTextureOfGivenSizeAndResizesWhenRequested() { + // Create a surface and set the initial size. + final Handler handler = new Handler(Looper.getMainLooper()); + final SurfaceTextureSurfaceProducer producer = + new SurfaceTextureSurfaceProducer(0, handler, fakeJNI); + final Surface surface = producer.getSurface(); + AtomicInteger frames = new AtomicInteger(); + producer + .getSurfaceTexture() + .setOnFrameAvailableListener( + (texture) -> { + if (texture.isReleased()) { + return; + } + frames.getAndIncrement(); + }); + producer.setSize(100, 200); + + // Draw. + Canvas canvas = surface.lockHardwareCanvas(); + canvas.drawARGB(255, 255, 0, 0); + surface.unlockCanvasAndPost(canvas); + shadowOf(Looper.getMainLooper()).idle(); + assertEquals(frames.get(), 1); + + // Resize and redraw. + producer.setSize(400, 800); + canvas = surface.lockHardwareCanvas(); + canvas.drawARGB(255, 255, 0, 0); + surface.unlockCanvasAndPost(canvas); + shadowOf(Looper.getMainLooper()).idle(); + assertEquals(frames.get(), 2); + + // Done. + fakeJNI.detachFromNativeAndReleaseResources(); + producer.release(); + } +}