fix(ui): capture TextureView and SurfaceView pixels in Android screenshots

.nx/version-plans/version-plan-1792417333384.md

@@ -0,0 +1,5 @@
+---
+__default__: minor
+---
+
+On Android 8 and later, `screen.screenshot()` captures what a TextureView or a SurfaceView shows, such as a GPU-rendered canvas, a video or a camera preview, instead of transparent pixels. An on-screen capture now copies what the window shows, so effects only the GPU renderer draws, such as elevation shadows and `filter` blurs, appear in it, and an Android baseline recorded without them needs recording again; a capture whose copy fails, or whose window is not shown, as while the screen is off, resolves `null` instead of a different image. An element under a scale or rotation transform is captured in the box it covers on the screen, as on iOS. An element partly off the screen, scaled to nothing or detached from the window is still drawn by itself, at its scroll offset, so a TextureView or a SurfaceView in it still comes out transparent; on Android 8.0 and 8.1 a window with elevation is drawn from its view tree, as before, with its SurfaceViews behind it; and a SurfaceView set above its window with `setZOrderOnTop` is still left out.

apps/playground/android/app/src/main/java/com/harnessplayground/MainApplication.kt

@@ -15,6 +15,7 @@ class MainApplication : Application(), ReactApplication {
       packageList =
         PackageList(this).packages.apply {
           add(PlaygroundCrashPackage())
+          add(PaintedSurfacesPackage())
         },
     )
   }

apps/playground/android/app/src/main/java/com/harnessplayground/PaintedSurfacesPackage.kt

@@ -0,0 +1,206 @@
+package com.harnessplayground
+
+import android.content.Context
+import android.graphics.Color
+import android.graphics.Paint
+import android.graphics.SurfaceTexture
+import android.media.MediaPlayer
+import android.view.SurfaceHolder
+import android.view.SurfaceView
+import android.view.TextureView
+import android.view.View
+import com.facebook.react.ReactPackage
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.uimanager.PixelUtil
+import com.facebook.react.uimanager.SimpleViewManager
+import com.facebook.react.uimanager.ThemedReactContext
+import com.facebook.react.uimanager.ViewManager
+import com.facebook.react.uimanager.annotations.ReactProp
+
+// Views whose pixels live on a surface of their own, as a GPU-rendered view's
+// do. Each fills its surface from the CPU or a decoder and labels itself once
+// that frame is what it shows.
+
+class SolidColorTextureView(context: Context) :
+  TextureView(context),
+  TextureView.SurfaceTextureListener {
+  init {
+    surfaceTextureListener = this
+  }
+
+  override fun onSurfaceTextureAvailable(surface: SurfaceTexture, width: Int, height: Int) {
+    val canvas = lockCanvas() ?: return
+    canvas.drawColor(Color.RED)
+    unlockCanvasAndPost(canvas)
+  }
+
+  override fun onSurfaceTextureUpdated(surface: SurfaceTexture) {
+    contentDescription = PAINTED_LABEL
+  }
+
+  override fun onSurfaceTextureSizeChanged(surface: SurfaceTexture, width: Int, height: Int) {}
+
+  override fun onSurfaceTextureDestroyed(surface: SurfaceTexture) = true
+
+  companion object {
+    const val PAINTED_LABEL = "texture-view-painted"
+  }
+}
+
+class SolidColorSurfaceView(context: Context) :
+  SurfaceView(context),
+  SurfaceHolder.Callback {
+  init {
+    holder.addCallback(this)
+  }
+
+  override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {
+    val canvas = holder.lockCanvas() ?: return
+    canvas.drawColor(Color.BLUE)
+    holder.unlockCanvasAndPost(canvas)
+    contentDescription = PAINTED_LABEL
+  }
+
+  override fun surfaceCreated(holder: SurfaceHolder) {}
+
+  override fun surfaceDestroyed(holder: SurfaceHolder) {}
+
+  companion object {
+    const val PAINTED_LABEL = "surface-view-painted"
+  }
+}
+
+// Blue on its left half and yellow on its right, so a capture shows which part
+// of the frame it copied and at what scale.
+class SplitColorSurfaceView(context: Context) :
+  SurfaceView(context),
+  SurfaceHolder.Callback {
+  init {
+    holder.addCallback(this)
+  }
+
+  override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {
+    val canvas = holder.lockCanvas() ?: return
+    canvas.drawColor(Color.BLUE)
+    canvas.drawRect(width / 2f, 0f, width.toFloat(), height.toFloat(), Paint().apply { color = Color.YELLOW })
+    holder.unlockCanvasAndPost(canvas)
+    contentDescription = PAINTED_LABEL
+  }
+
+  override fun surfaceCreated(holder: SurfaceHolder) {}
+
+  override fun surfaceDestroyed(holder: SurfaceHolder) {}
+
+  companion object {
+    const val PAINTED_LABEL = "split-surface-view-painted"
+  }
+}
+
+// A video whose 320 x 180 frame, blue on its left half and yellow on its right,
+// is larger than the view: the decoder queues buffers at the frame's size and
+// the screen shows them scaled to the view, as a video player's do.
+class SplitColorVideoView(context: Context) :
+  SurfaceView(context),
+  SurfaceHolder.Callback {
+  private var player: MediaPlayer? = null
+
+  init {
+    holder.addCallback(this)
+  }
+
+  override fun surfaceCreated(holder: SurfaceHolder) {
+    player =
+      MediaPlayer.create(context, R.raw.split_color)?.apply {
+        setDisplay(holder)
+        setOnInfoListener { _, what, _ ->
+          if (what == MediaPlayer.MEDIA_INFO_VIDEO_RENDERING_START) {
+            this@SplitColorVideoView.contentDescription = PAINTED_LABEL
+          }
+          false
+        }
+        start()
+      }
+  }
+
+  override fun surfaceChanged(holder: SurfaceHolder, format: Int, width: Int, height: Int) {}
+
+  override fun surfaceDestroyed(holder: SurfaceHolder) {
+    player?.release()
+    player = null
+  }
+
+  companion object {
+    const val PAINTED_LABEL = "split-video-painted"
+  }
+}
+
+// Moves the window's root view down while it is mounted, as an app's slide or
+// shake effect does, and puts it back when it is unmounted.
+class RootShiftView(context: Context) : View(context) {
+  private var shiftedRoot: View? = null
+
+  var rootTranslationY = 0f
+    set(value) {
+      field = value
+      shiftedRoot?.translationY = value
+    }
+
+  override fun onAttachedToWindow() {
+    super.onAttachedToWindow()
+    shiftedRoot = rootView.also { root -> root.translationY = rootTranslationY }
+  }
+
+  override fun onDetachedFromWindow() {
+    shiftedRoot?.translationY = 0f
+    shiftedRoot = null
+    super.onDetachedFromWindow()
+  }
+}
+
+class SolidColorTextureViewManager : SimpleViewManager<SolidColorTextureView>() {
+  override fun getName() = "SolidColorTextureView"
+
+  override fun createViewInstance(context: ThemedReactContext) = SolidColorTextureView(context)
+}
+
+class SolidColorSurfaceViewManager : SimpleViewManager<SolidColorSurfaceView>() {
+  override fun getName() = "SolidColorSurfaceView"
+
+  override fun createViewInstance(context: ThemedReactContext) = SolidColorSurfaceView(context)
+}
+
+class SplitColorSurfaceViewManager : SimpleViewManager<SplitColorSurfaceView>() {
+  override fun getName() = "SplitColorSurfaceView"
+
+  override fun createViewInstance(context: ThemedReactContext) = SplitColorSurfaceView(context)
+}
+
+class SplitColorVideoViewManager : SimpleViewManager<SplitColorVideoView>() {
+  override fun getName() = "SplitColorVideoView"
+
+  override fun createViewInstance(context: ThemedReactContext) = SplitColorVideoView(context)
+}
+
+class RootShiftViewManager : SimpleViewManager<RootShiftView>() {
+  override fun getName() = "RootShiftView"
+
+  override fun createViewInstance(context: ThemedReactContext) = RootShiftView(context)
+
+  @ReactProp(name = "rootTranslationY")
+  fun setRootTranslationY(view: RootShiftView, value: Float) {
+    view.rootTranslationY = PixelUtil.toPixelFromDIP(value)
+  }
+}
+
+class PaintedSurfacesPackage : ReactPackage {
+  override fun createViewManagers(
+    reactContext: ReactApplicationContext,
+  ): List<ViewManager<*, *>> =
+    listOf(
+      SolidColorTextureViewManager(),
+      SolidColorSurfaceViewManager(),
+      SplitColorSurfaceViewManager(),
+      SplitColorVideoViewManager(),
+      RootShiftViewManager(),
+    )
+}