Add SurfaceTextureSurfaceProducer (flutter/engine#49653)

Unblocks https://github.com/flutter/flutter/issues/139702.

The next step will be local validation that this API works for at least a few of our internal plugins (video_player, camera).

engine/src/flutter/ci/licenses_golden/licenses_flutter

@@ -9247,6 +9247,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/plugin
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterUiDisplayListener.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/RenderSurface.java
+FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/systemchannels/AccessibilityChannel.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/engine/systemchannels/DeferredComponentChannel.java

engine/src/flutter/shell/platform/android/BUILD.gn

@@ -266,6 +266,7 @@ android_java_sources = [
   "io/flutter/embedding/engine/renderer/FlutterRenderer.java",
   "io/flutter/embedding/engine/renderer/FlutterUiDisplayListener.java",
   "io/flutter/embedding/engine/renderer/RenderSurface.java",
+  "io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java",
   "io/flutter/embedding/engine/renderer/SurfaceTextureWrapper.java",
   "io/flutter/embedding/engine/systemchannels/AccessibilityChannel.java",
   "io/flutter/embedding/engine/systemchannels/DeferredComponentChannel.java",

engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java

@@ -47,6 +47,16 @@ import java.util.concurrent.atomic.AtomicLong;
  * io.flutter.embedding.android.FlutterTextureView} are implementations of {@link RenderSurface}.
  */
 public class FlutterRenderer implements TextureRegistry {
+  /**
+   * Whether to always use GL textures for {@link FlutterRenderer#createSurfaceProducer()}.
+   *
+   * <p>This is a debug-only API intended for local development. For example, when using a newer
+   * Android device (that normally would use {@link ImageReaderSurfaceProducer}, but wanting to test
+   * the OpenGLES/{@link SurfaceTextureSurfaceProducer} code branch. This flag has undefined
+   * behavior if set to true while running in a Vulkan (Impeller) context.
+   */
+  @VisibleForTesting static boolean debugForceSurfaceProducerGlTextures = false;
+
   private static final String TAG = "FlutterRenderer";
 
   @NonNull private final FlutterJNI flutterJNI;
@@ -162,12 +172,36 @@ public class FlutterRenderer implements TextureRegistry {
   @NonNull
   @Override
   public SurfaceProducer createSurfaceProducer() {
-    // TODO(matanl, johnmccutchan): Implement a SurfaceTexture version and switch on whether or
-    // not impeller is enabled.
-    final ImageReaderSurfaceProducer entry =
-        new ImageReaderSurfaceProducer(nextTextureId.getAndIncrement());
-    Log.v(TAG, "New SurfaceProducer ID: " + entry.id());
-    registerImageTexture(entry.id(), entry);
+    // Prior to Impeller, Flutter on Android *only* ran on OpenGLES (via Skia). That meant that
+    // plugins (i.e. end-users) either explicitly created a SurfaceTexture (via
+    // createX/registerX) or an ImageTexture (via createX/registerX).
+    //
+    // In an Impeller world, which for the first time uses (if available) a Vulkan rendering
+    // backend, it is no longer possible (at least not trivially) to render an OpenGLES-provided
+    // texture (SurfaceTexture) in a Vulkan context.
+    //
+    // This function picks the "best" rendering surface based on the Android runtime, and
+    // provides a consumer-agnostic SurfaceProducer (which in turn vends a Surface), and has
+    // plugins (i.e. end-users) use the Surface instead, letting us "hide" the consumer-side
+    // of the implementation.
+    //
+    // tl;dr: If ImageTexture is available, we use it, otherwise we use a SurfaceTexture.
+    // Coincidentally, if ImageTexture is available, we are also on an Android version that is
+    // running Vulkan, so we don't have to worry about it not being supported.
+    final long id = nextTextureId.getAndIncrement();
+    final SurfaceProducer entry;
+    if (!debugForceSurfaceProducerGlTextures && Build.VERSION.SDK_INT >= 29) {
+      final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id);
+      registerImageTexture(id, producer);
+      Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id);
+      entry = producer;
+    } else {
+      final SurfaceTextureSurfaceProducer producer =
+          new SurfaceTextureSurfaceProducer(id, handler, flutterJNI);
+      registerSurfaceTexture(producer.getSurfaceTexture());
+      Log.v(TAG, "New SurfaceTextureSurfaceProducer ID: " + id);
+      entry = producer;
+    }
     return entry;
   }
 

engine/src/flutter/shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java

@@ -0,0 +1,84 @@
+package io.flutter.embedding.engine.renderer;
+
+import android.graphics.SurfaceTexture;
+import android.os.Handler;
+import android.view.Surface;
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import io.flutter.embedding.engine.FlutterJNI;
+import io.flutter.view.TextureRegistry;
+
+/** Uses a {@link android.graphics.SurfaceTexture} to populate the texture registry. */
+final class SurfaceTextureSurfaceProducer
+    implements TextureRegistry.SurfaceProducer, TextureRegistry.GLTextureConsumer {
+  private final long id;
+  private int requestBufferWidth;
+  private int requestedBufferHeight;
+  private boolean released;
+  @Nullable private Surface surface;
+  @NonNull private final SurfaceTexture texture;
+  @NonNull private final Handler handler;
+  @NonNull private final FlutterJNI flutterJNI;
+
+  SurfaceTextureSurfaceProducer(long id, @NonNull Handler handler, @NonNull FlutterJNI flutterJNI) {
+    this.id = id;
+    this.handler = handler;
+    this.flutterJNI = flutterJNI;
+    this.texture = new SurfaceTexture(0);
+  }
+
+  @Override
+  protected void finalize() throws Throwable {
+    try {
+      if (released) {
+        return;
+      }
+      release();
+      handler.post(new FlutterRenderer.TextureFinalizerRunnable(id, flutterJNI));
+    } finally {
+      super.finalize();
+    }
+  }
+
+  @Override
+  public long id() {
+    return id;
+  }
+
+  @Override
+  public void release() {
+    texture.release();
+    released = true;
+  }
+
+  @Override
+  @NonNull
+  public SurfaceTexture getSurfaceTexture() {
+    return texture;
+  }
+
+  @Override
+  public void setSize(int width, int height) {
+    requestBufferWidth = width;
+    requestedBufferHeight = height;
+    getSurfaceTexture().setDefaultBufferSize(width, height);
+  }
+
+  @Override
+  public int getWidth() {
+    return requestBufferWidth;
+  }
+
+  @Override
+  public int getHeight() {
+    return requestedBufferHeight;
+  }
+
+  @Override
+  public Surface getSurface() {
+    if (surface == null) {
+      surface = new Surface(texture);
+    }
+    return surface;
+  }
+}

engine/src/flutter/shell/platform/android/io/flutter/view/TextureRegistry.java

@@ -159,6 +159,18 @@ public interface TextureRegistry {
      *
      * @return Image or null.
      */
+    @Nullable
     public Image acquireLatestImage();
   }
+
+  @Keep
+  interface GLTextureConsumer extends TextureEntry {
+    /**
+     * Retrieve the last GL texture produced.
+     *
+     * @return SurfaceTexture.
+     */
+    @NonNull
+    public SurfaceTexture getSurfaceTexture();
+  }
 }

engine/src/flutter/shell/platform/android/test/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducerTest.java

@@ -0,0 +1,61 @@
+package io.flutter.embedding.engine.renderer;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.robolectric.Shadows.shadowOf;
+
+import android.annotation.TargetApi;
+import android.graphics.Canvas;
+import android.os.Handler;
+import android.os.Looper;
+import android.view.Surface;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import io.flutter.embedding.engine.FlutterJNI;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(AndroidJUnit4.class)
+@TargetApi(26)
+public final class SurfaceTextureSurfaceProducerTest {
+  private final FlutterJNI fakeJNI = mock(FlutterJNI.class);
+
+  @Test
+  public void createsSurfaceTextureOfGivenSizeAndResizesWhenRequested() {
+    // Create a surface and set the initial size.
+    final Handler handler = new Handler(Looper.getMainLooper());
+    final SurfaceTextureSurfaceProducer producer =
+        new SurfaceTextureSurfaceProducer(0, handler, fakeJNI);
+    final Surface surface = producer.getSurface();
+    AtomicInteger frames = new AtomicInteger();
+    producer
+        .getSurfaceTexture()
+        .setOnFrameAvailableListener(
+            (texture) -> {
+              if (texture.isReleased()) {
+                return;
+              }
+              frames.getAndIncrement();
+            });
+    producer.setSize(100, 200);
+
+    // Draw.
+    Canvas canvas = surface.lockHardwareCanvas();
+    canvas.drawARGB(255, 255, 0, 0);
+    surface.unlockCanvasAndPost(canvas);
+    shadowOf(Looper.getMainLooper()).idle();
+    assertEquals(frames.get(), 1);
+
+    // Resize and redraw.
+    producer.setSize(400, 800);
+    canvas = surface.lockHardwareCanvas();
+    canvas.drawARGB(255, 255, 0, 0);
+    surface.unlockCanvasAndPost(canvas);
+    shadowOf(Looper.getMainLooper()).idle();
+    assertEquals(frames.get(), 2);
+
+    // Done.
+    fakeJNI.detachFromNativeAndReleaseResources();
+    producer.release();
+  }
+}