Add SurfaceProducer.Callback lifecycle hooks (flutter#53280)

Work towards flutter#148417.

Author: matanlurey

DEPS

@@ -785,7 +785,7 @@ deps = {
      'packages': [
        {
         'package': 'flutter/android/embedding_bundle',
-        'version': 'last_updated:2023-08-11T11:35:44-0700'
+        'version': 'last_updated:2024-06-12T14:15:49-0700'
        }
      ],
      'condition': 'download_android_deps',

shell/platform/android/BUILD.gn

@@ -374,6 +374,7 @@ embedding_dependencies_jars = [
   "//third_party/android_embedding_dependencies/lib/lifecycle-common-java8-2.2.0.jar",
   "//third_party/android_embedding_dependencies/lib/lifecycle-livedata-2.0.0.jar",
   "//third_party/android_embedding_dependencies/lib/lifecycle-livedata-core-2.0.0.jar",
+  "//third_party/android_embedding_dependencies/lib/lifecycle-process-2.2.0.jar",
   "//third_party/android_embedding_dependencies/lib/lifecycle-runtime-2.2.0.jar",
   "//third_party/android_embedding_dependencies/lib/lifecycle-viewmodel-2.1.0.jar",
   "//third_party/android_embedding_dependencies/lib/loader-1.0.0.jar",

shell/platform/android/build.gradle

@@ -62,4 +62,3 @@ android {
         implementation "org.mockito:mockito-android:$mockitoVersion"
     }
 }
-

shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java

@@ -24,6 +24,9 @@
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
+import androidx.lifecycle.DefaultLifecycleObserver;
+import androidx.lifecycle.LifecycleOwner;
+import androidx.lifecycle.ProcessLifecycleOwner;
 import io.flutter.Log;
 import io.flutter.embedding.engine.FlutterJNI;
 import io.flutter.view.TextureRegistry;
@@ -78,6 +81,8 @@ public class FlutterRenderer implements TextureRegistry {
   private final Set<WeakReference<TextureRegistry.OnTrimMemoryListener>> onTrimMemoryListeners =
       new HashSet<>();
 
+  @NonNull private final List<ImageReaderSurfaceProducer> imageReaderProducers = new ArrayList<>();
+
   @NonNull
   private final FlutterUiDisplayListener flutterUiDisplayListener =
       new FlutterUiDisplayListener() {
@@ -95,6 +100,20 @@ public void onFlutterUiNoLongerDisplayed() {
   public FlutterRenderer(@NonNull FlutterJNI flutterJNI) {
     this.flutterJNI = flutterJNI;
     this.flutterJNI.addIsDisplayingFlutterUiListener(flutterUiDisplayListener);
+    ProcessLifecycleOwner.get()
+        .getLifecycle()
+        .addObserver(
+            new DefaultLifecycleObserver() {
+              @Override
+              public void onResume(@NonNull LifecycleOwner owner) {
+                Log.v(TAG, "onResume called; notifying SurfaceProducers");
+                for (ImageReaderSurfaceProducer producer : imageReaderProducers) {
+                  if (producer.callback != null) {
+                    producer.callback.onSurfaceCreated();
+                  }
+                }
+              }
+            });
   }
 
   /**
@@ -197,6 +216,7 @@ public SurfaceProducer createSurfaceProducer() {
       final ImageReaderSurfaceProducer producer = new ImageReaderSurfaceProducer(id);
       registerImageTexture(id, producer);
       addOnTrimMemoryListener(producer);
+      imageReaderProducers.add(producer);
       Log.v(TAG, "New ImageReaderSurfaceProducer ID: " + id);
       entry = producer;
     } else {
@@ -453,6 +473,7 @@ final class ImageReaderSurfaceProducer
         new HashMap<ImageReader, PerImageReader>();
     private PerImage lastDequeuedImage = null;
     private PerImageReader lastReaderDequeuedFrom = null;
+    private Callback callback = null;
 
     /** Internal class: state held per Image produced by ImageReaders. */
     private class PerImage {
@@ -673,11 +694,15 @@ public void onTrimMemory(int level) {
       }
       cleanup();
       createNewReader = true;
+      if (this.callback != null) {
+        this.callback.onSurfaceDestroyed();
+      }
     }
 
     private void releaseInternal() {
       cleanup();
       released = true;
+      imageReaderProducers.remove(this);
     }
 
     private void cleanup() {
@@ -732,6 +757,11 @@ private void maybeWaitOnFence(Image image) {
       this.id = id;
     }
 
+    @Override
+    public void setCallback(Callback callback) {
+      this.callback = callback;
+    }
+
     @Override
     public long id() {
       return id;

shell/platform/android/io/flutter/embedding/engine/renderer/SurfaceTextureSurfaceProducer.java

@@ -55,6 +55,11 @@ public void release() {
     released = true;
   }
 
+  @Override
+  public void setCallback(Callback callback) {
+    // Intentionally blank: SurfaceTextures don't get platform notifications or cleanup.
+  }
+
   @Override
   @NonNull
   public SurfaceTexture getSurfaceTexture() {

shell/platform/android/io/flutter/view/FlutterView.java

@@ -902,6 +902,7 @@ public ImageTextureEntry createImageTexture() {
     throw new UnsupportedOperationException("Image textures are not supported in this mode.");
   }
 
+  @NonNull
   @Override
   public SurfaceProducer createSurfaceProducer() {
     throw new UnsupportedOperationException(

shell/platform/android/io/flutter/view/TextureRegistry.java

@@ -62,7 +62,7 @@ interface TextureEntry {
     /** @return The identity of this texture. */
     long id();
 
-    /** Deregisters and releases all resources . */
+    /** De-registers and releases all resources . */
     void release();
   }
 
@@ -79,18 +79,52 @@ interface SurfaceProducer extends TextureEntry {
     int getHeight();
 
     /**
-     * Get a Surface that can be used to update the texture contents.
+     * Direct access to the surface object.
      *
-     * <p>NOTE: You should not cache the returned surface but instead invoke getSurface each time
-     * you need to draw. The surface may change when the texture is resized or has its format
+     * <p>When using this API, you will usually need to implement {@link SurfaceProducer.Callback}
+     * and provide it to {@link #setCallback(Callback)} in order to be notified when an existing
+     * surface has been destroyed (such as when the application goes to the background) or a new
+     * surface has been created (such as when the application is resumed back to the foreground).
+     *
+     * <p>NOTE: You should not cache the returned surface but instead invoke {@code getSurface} each
+     * time you need to draw. The surface may change when the texture is resized or has its format
      * changed.
      *
      * @return a Surface to use for a drawing target for various APIs.
      */
     Surface getSurface();
 
+    /**
+     * Sets a callback that is notified when a previously created {@link Surface} returned by {@link
+     * SurfaceProducer#getSurface()} is no longer valid, either due to being destroyed or being
+     * changed.
+     *
+     * @param callback The callback to notify, or null to remove the callback.
+     */
+    void setCallback(Callback callback);
+
+    /** Callback invoked by {@link #setCallback(Callback)}. */
+    interface Callback {
+      /**
+       * Invoked when a previous surface is now invalid and a new surface is now available.
+       *
+       * <p>Typically plugins will use this callback as a signal to redraw, such as due to the
+       * texture being resized, the format being changed, or the application being resumed after
+       * being suspended in the background.
+       */
+      void onSurfaceCreated();
+
+      /**
+       * Invoked when a previous surface is now invalid.
+       *
+       * <p>Typically plugins will use this callback as a signal to release resources.
+       */
+      void onSurfaceDestroyed();
+    }
+
+    /** This method is not officially part of the public API surface and will be deprecated. */
     void scheduleFrame();
-  };
+  }
 
   /** A registry entry for a managed SurfaceTexture. */
   @Keep
@@ -144,7 +178,7 @@ interface ImageConsumer {
      * @return Image or null.
      */
     @Nullable
-    public Image acquireLatestImage();
+    Image acquireLatestImage();
   }
 
   @Keep
@@ -155,6 +189,6 @@ interface GLTextureConsumer {
      * @return SurfaceTexture.
      */
     @NonNull
-    public SurfaceTexture getSurfaceTexture();
+    SurfaceTexture getSurfaceTexture();
   }
 }

shell/platform/android/test/io/flutter/embedding/engine/renderer/FlutterRendererTest.java

@@ -23,6 +23,8 @@
 import android.os.Looper;
 import android.view.Surface;
 import androidx.lifecycle.Lifecycle;
+import androidx.lifecycle.LifecycleRegistry;
+import androidx.lifecycle.ProcessLifecycleOwner;
 import androidx.test.ext.junit.rules.ActivityScenarioRule;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import io.flutter.embedding.android.FlutterActivity;
@@ -728,8 +730,46 @@ public void SurfaceTextureSurfaceProducerCreatesAConnectedTexture() {
   }
 
   @Test
-  public void CanLaunchActivityUsingFlutterEngine() {
-    // This is a placeholder test that will be used to test lifecycle events w/ SurfaceProducer.
-    scenarioRule.getScenario().moveToState(Lifecycle.State.RESUMED);
+  public void ImageReaderSurfaceProducerIsDestroyedOnTrimMemory() {
+    FlutterRenderer flutterRenderer = engineRule.getFlutterEngine().getRenderer();
+    TextureRegistry.SurfaceProducer producer = flutterRenderer.createSurfaceProducer();
+
+    // Create and set a mock callback.
+    TextureRegistry.SurfaceProducer.Callback callback =
+        mock(TextureRegistry.SurfaceProducer.Callback.class);
+    producer.setCallback(callback);
+
+    // Trim memory.
+    ((FlutterRenderer.ImageReaderSurfaceProducer) producer).onTrimMemory(40);
+
+    // Verify.
+    verify(callback).onSurfaceDestroyed();
+  }
+
+  @Test
+  public void ImageReaderSurfaceProducerIsCreatedOnLifecycleResume() throws Exception {
+    FlutterRenderer flutterRenderer = engineRule.getFlutterEngine().getRenderer();
+    TextureRegistry.SurfaceProducer producer = flutterRenderer.createSurfaceProducer();
+
+    // Create a callback.
+    CountDownLatch latch = new CountDownLatch(1);
+    TextureRegistry.SurfaceProducer.Callback callback =
+        new TextureRegistry.SurfaceProducer.Callback() {
+          @Override
+          public void onSurfaceCreated() {
+            latch.countDown();
+          }
+
+          @Override
+          public void onSurfaceDestroyed() {}
+        };
+    producer.setCallback(callback);
+
+    // Trigger a resume.
+    ((LifecycleRegistry) ProcessLifecycleOwner.get().getLifecycle())
+        .setCurrentState(Lifecycle.State.RESUMED);
+
+    // Verify.
+    latch.await();
   }
 }

shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java

@@ -1582,13 +1582,16 @@ private static void attachToFlutterView(
         new TextureRegistry() {
           public void TextureRegistry() {}
 
+          @NonNull
           @Override
           public SurfaceTextureEntry createSurfaceTexture() {
             return registerSurfaceTexture(mock(SurfaceTexture.class));
           }
 
+          @NonNull
           @Override
-          public SurfaceTextureEntry registerSurfaceTexture(SurfaceTexture surfaceTexture) {
+          public SurfaceTextureEntry registerSurfaceTexture(
+              @NonNull SurfaceTexture surfaceTexture) {
             return new SurfaceTextureEntry() {
               @NonNull
               @Override
@@ -1606,6 +1609,7 @@ public void release() {}
             };
           }
 
+          @NonNull
           @Override
           public ImageTextureEntry createImageTexture() {
             return new ImageTextureEntry() {
@@ -1622,9 +1626,13 @@ public void pushImage(Image image) {}
             };
           }
 
+          @NonNull
           @Override
           public SurfaceProducer createSurfaceProducer() {
             return new SurfaceProducer() {
+              @Override
+              public void setCallback(SurfaceProducer.Callback cb) {}
+
               @Override
               public long id() {
                 return 0;

testing/scenario_app/android/app/gradle.lockfile

@@ -28,6 +28,7 @@ androidx.lifecycle:lifecycle-common-java8:2.2.0=debugAndroidTestCompileClasspath
 androidx.lifecycle:lifecycle-common:2.3.1=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath
 androidx.lifecycle:lifecycle-livedata-core:2.0.0=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath
 androidx.lifecycle:lifecycle-livedata:2.0.0=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath
+androidx.lifecycle:lifecycle-process:2.2.0=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath
 androidx.lifecycle:lifecycle-runtime:2.2.0=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath
 androidx.lifecycle:lifecycle-viewmodel:2.1.0=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath
 androidx.loader:loader:1.0.0=debugAndroidTestCompileClasspath,debugCompileClasspath,debugImplementationDependenciesMetadata,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestRuntimeClasspath,implementationDependenciesMetadata,releaseCompileClasspath,releaseImplementationDependenciesMetadata,releaseRuntimeClasspath,releaseUnitTestCompileClasspath,releaseUnitTestRuntimeClasspath