feat(replay): Add ReplayFrameObserver for snapshot testing

.github/workflows/integration-tests-ui.yml

@@ -73,6 +73,28 @@ jobs:
         if: env.SAUCE_USERNAME != null
 
 
+      - name: Install Sentry CLI
+        if: ${{ !cancelled() && env.SAUCE_USERNAME != null }}
+        run: curl -sL https://sentry.io/get-cli/ | bash
+
+      - name: Upload Replay Snapshots to Sentry
+        if: ${{ !cancelled() && env.SAUCE_USERNAME != null }}
+        run: |
+          shopt -s globstar nullglob
+          pngs=(artifacts/**/*.png)
+          if [ ${#pngs[@]} -gt 0 ]; then
+            mkdir -p replay-snapshots
+            cp "${pngs[@]}" replay-snapshots/
+            sentry-cli build snapshots ./replay-snapshots \
+              --app-id sentry-android-replay
+          else
+            echo "No replay snapshot files found, skipping upload"
+          fi
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+          SENTRY_ORG: sentry-sdks
+          SENTRY_PROJECT: sentry-android
+
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3

.sauce/sentry-uitest-android-ui.yml

@@ -32,4 +32,5 @@ artifacts:
     when: always
     match:
       - junit.xml
+      - "*.png"
     directory: ./artifacts/

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## Unreleased
+
+### Features
+
+- Session Replay: Add `ReplayFrameObserver` for observing captured replay frames ([#5386](https://github.com/getsentry/sentry-java/pull/5386))
+
+  ```kotlin
+  SentryAndroid.init(context) { options ->
+    options.sessionReplay.frameObserver =
+      SentryReplayOptions.ReplayFrameObserver { hint, frameTimestamp, screenName ->
+        val bitmap = hint.getAs(TypeCheckHint.REPLAY_FRAME_BITMAP, Bitmap::class.java)
+        if (bitmap != null) {
+          try {
+            // Process the masked replay frame
+            myAnalyzer.processFrame(bitmap, frameTimestamp, screenName)
+          } finally {
+            bitmap.recycle()
+          }
+        }
+      }
+  }
+  ```
+
 ## 8.42.0
 
 ### Features

sentry-android-integration-tests/sentry-uitest-android/build.gradle.kts

@@ -83,6 +83,10 @@ android {
 
 val applySentryIntegrations = System.getenv("APPLY_SENTRY_INTEGRATIONS")?.toBoolean() ?: true
 
+if (applySentryIntegrations) {
+  android.sourceSets["androidTest"].java.srcDirs("src/androidTestReplay/java")
+}
+
 dependencies {
   implementation(
     kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)

sentry-android-integration-tests/sentry-uitest-android/src/androidTestReplay/java/io/sentry/uitest/android/ReplaySnapshotTest.kt

@@ -0,0 +1,71 @@
+package io.sentry.uitest.android
+
+import android.graphics.Bitmap
+import android.os.Environment
+import androidx.lifecycle.Lifecycle
+import androidx.test.core.app.launchActivity
+import io.sentry.SentryReplayOptions
+import io.sentry.TypeCheckHint
+import java.io.File
+import java.util.concurrent.CopyOnWriteArraySet
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import kotlin.test.Test
+import kotlin.test.assertTrue
+import org.hamcrest.CoreMatchers.`is`
+import org.junit.Assume.assumeThat
+import org.junit.Before
+
+class ReplaySnapshotTest : BaseUiTest() {
+
+  @Before
+  fun setup() {
+    // GH Actions emulators don't support capturing screenshots for replay
+    @Suppress("KotlinConstantConditions")
+    assumeThat(BuildConfig.ENVIRONMENT != "github", `is`(true))
+  }
+
+  @Test
+  fun captureComposeReplayFrameSnapshots() {
+    val snapshotsDir =
+      File(
+          Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS),
+          "sauce_labs_custom_screenshots",
+        )
+        .apply {
+          deleteRecursively()
+          mkdirs()
+        }
+    val frameReceived = CountDownLatch(1)
+    val capturedScreens = CopyOnWriteArraySet<String>()
+
+    val activityScenario = launchActivity<ComposeActivity>()
+    activityScenario.moveToState(Lifecycle.State.RESUMED)
+
+    initSentry {
+      it.sessionReplay.sessionSampleRate = 1.0
+      it.sessionReplay.frameObserver =
+        SentryReplayOptions.ReplayFrameObserver { hint, frameTimestamp, screenName ->
+          val bitmap =
+            hint.getAs(TypeCheckHint.REPLAY_FRAME_BITMAP, Bitmap::class.java)
+              ?: return@ReplayFrameObserver
+          val name = screenName ?: "unknown"
+          if (capturedScreens.add(name)) {
+            val file = File(snapshotsDir, "${name}_$frameTimestamp.png")
+            file.outputStream().use { out -> bitmap.compress(Bitmap.CompressFormat.PNG, 100, out) }
+          }
+          bitmap.recycle()
+          frameReceived.countDown()
+        }
+    }
+
+    assertTrue(frameReceived.await(10, TimeUnit.SECONDS), "Expected at least one replay frame")
+    assertTrue(capturedScreens.isNotEmpty(), "Expected at least one screen captured")
+
+    val files = snapshotsDir.listFiles()?.filter { it.extension == "png" } ?: emptyList()
+    assertTrue(files.isNotEmpty(), "Expected snapshot PNG files on disk")
+    assertTrue(files.all { it.length() > 0 }, "Snapshot files should not be empty")
+
+    activityScenario.moveToState(Lifecycle.State.DESTROYED)
+  }
+}

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt

@@ -2,11 +2,13 @@ package io.sentry.android.replay
 
 import android.content.Context
 import android.graphics.Bitmap
+import android.graphics.BitmapFactory
 import android.os.Build
 import android.view.MotionEvent
 import io.sentry.Breadcrumb
 import io.sentry.DataCategory.All
 import io.sentry.DataCategory.Replay
+import io.sentry.Hint
 import io.sentry.IConnectionStatusProvider.ConnectionStatus
 import io.sentry.IConnectionStatusProvider.ConnectionStatus.DISCONNECTED
 import io.sentry.IConnectionStatusProvider.IConnectionStatusObserver
@@ -17,8 +19,10 @@ import io.sentry.ReplayBreadcrumbConverter
 import io.sentry.ReplayController
 import io.sentry.SentryIntegrationPackageStorage
 import io.sentry.SentryLevel.DEBUG
+import io.sentry.SentryLevel.ERROR
 import io.sentry.SentryLevel.INFO
 import io.sentry.SentryOptions
+import io.sentry.TypeCheckHint
 import io.sentry.android.replay.ReplayState.CLOSED
 import io.sentry.android.replay.ReplayState.PAUSED
 import io.sentry.android.replay.ReplayState.RESUMED
@@ -308,13 +312,45 @@ public class ReplayIntegration(
     var screen: String? = null
     scopes?.configureScope { screen = it.screen?.substringAfterLast('.') }
     captureStrategy?.onScreenshotRecorded(bitmap) { frameTimeStamp ->
+      val observer = options.sessionReplay.frameObserver
+      if (observer != null) {
+        val copy = bitmap.copy(bitmap.config!!, false)
+        if (copy != null) {
+          try {
+            val hint = Hint()
+            hint.set(TypeCheckHint.REPLAY_FRAME_BITMAP, copy)
+            observer.onMaskedFrameCaptured(hint, frameTimeStamp, screen)
+          } catch (e: Throwable) {
+            options.logger.log(ERROR, "Error in ReplayFrameObserver", e)
+            copy.recycle()
+          }
+        }
+      }
       addFrame(bitmap, frameTimeStamp, screen)
     }
     checkCanRecord()
   }
 
   override fun onScreenshotRecorded(screenshot: File, frameTimestamp: Long) {
-    captureStrategy?.onScreenshotRecorded { _ -> addFrame(screenshot, frameTimestamp) }
+    var screen: String? = null
+    scopes?.configureScope { screen = it.screen?.substringAfterLast('.') }
+    captureStrategy?.onScreenshotRecorded { _ ->
+      val observer = options.sessionReplay.frameObserver
+      if (observer != null) {
+        val bitmap = BitmapFactory.decodeFile(screenshot.absolutePath)
+        if (bitmap != null) {
+          try {
+            val hint = Hint()
+            hint.set(TypeCheckHint.REPLAY_FRAME_BITMAP, bitmap)
+            observer.onMaskedFrameCaptured(hint, frameTimestamp, screen)
+          } catch (e: Throwable) {
+            options.logger.log(ERROR, "Error in ReplayFrameObserver", e)
+            bitmap.recycle()
+          }
+        }
+      }
+      addFrame(screenshot, frameTimestamp, screen)
+    }
     checkCanRecord()
   }
 

sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayIntegrationTest.kt

@@ -18,6 +18,8 @@ import io.sentry.SentryEvent
 import io.sentry.SentryIntegrationPackageStorage
 import io.sentry.SentryOptions
 import io.sentry.SentryReplayEvent.ReplayType
+import io.sentry.SentryReplayOptions
+import io.sentry.TypeCheckHint
 import io.sentry.android.replay.ReplayCache.Companion.ONGOING_SEGMENT
 import io.sentry.android.replay.ReplayCache.Companion.SEGMENT_KEY_BIT_RATE
 import io.sentry.android.replay.ReplayCache.Companion.SEGMENT_KEY_FRAME_RATE
@@ -63,6 +65,7 @@ import org.mockito.kotlin.anyOrNull
 import org.mockito.kotlin.argThat
 import org.mockito.kotlin.check
 import org.mockito.kotlin.doAnswer
+import org.mockito.kotlin.doReturn
 import org.mockito.kotlin.eq
 import org.mockito.kotlin.mock
 import org.mockito.kotlin.never
@@ -969,6 +972,106 @@ class ReplayIntegrationTest {
     assertFalse(replay.isDebugMaskingOverlayEnabled)
   }
 
+  @Test
+  fun `snapshot observer is invoked with bitmap and metadata`() {
+    var callbackInvoked = false
+    var receivedTimestamp = 0L
+    var receivedScreen: String? = null
+    var receivedBitmap: Bitmap? = null
+
+    val captureStrategy =
+      mock<CaptureStrategy> {
+        doAnswer {
+            ((it.arguments[1] as ReplayCache.(frameTimestamp: Long) -> Unit)).invoke(
+              fixture.replayCache,
+              1720693523997,
+            )
+          }
+          .whenever(mock)
+          .onScreenshotRecorded(anyOrNull<Bitmap>(), any())
+      }
+    val replay = fixture.getSut(context, replayCaptureStrategyProvider = { captureStrategy })
+
+    fixture.scopes.configureScope { it.screen = "MainActivity" }
+    replay.register(fixture.scopes, fixture.options)
+    replay.start()
+
+    fixture.options.sessionReplay.frameObserver =
+      SentryReplayOptions.ReplayFrameObserver { hint, frameTimestamp, screenName ->
+        callbackInvoked = true
+        receivedTimestamp = frameTimestamp
+        receivedScreen = screenName
+        receivedBitmap = hint.getAs(TypeCheckHint.REPLAY_FRAME_BITMAP, Bitmap::class.java)
+      }
+
+    val copyBitmap = mock<Bitmap>()
+    val sourceBitmap =
+      mock<Bitmap> {
+        on { config } doReturn ARGB_8888
+        on { copy(any(), any()) } doReturn copyBitmap
+      }
+    replay.onScreenshotRecorded(sourceBitmap)
+
+    assertTrue(callbackInvoked)
+    assertEquals(1720693523997, receivedTimestamp)
+    assertEquals("MainActivity", receivedScreen)
+    assertEquals(copyBitmap, receivedBitmap)
+  }
+
+  @Test
+  fun `snapshot observer exception does not prevent frame storage`() {
+    val captureStrategy =
+      mock<CaptureStrategy> {
+        doAnswer {
+            ((it.arguments[1] as ReplayCache.(frameTimestamp: Long) -> Unit)).invoke(
+              fixture.replayCache,
+              1720693523997,
+            )
+          }
+          .whenever(mock)
+          .onScreenshotRecorded(anyOrNull<Bitmap>(), any())
+      }
+    val replay = fixture.getSut(context, replayCaptureStrategyProvider = { captureStrategy })
+
+    replay.register(fixture.scopes, fixture.options)
+    replay.start()
+
+    fixture.options.sessionReplay.frameObserver =
+      SentryReplayOptions.ReplayFrameObserver { _, _, _ -> throw RuntimeException("test") }
+
+    val sourceBitmap =
+      mock<Bitmap> {
+        on { config } doReturn ARGB_8888
+        on { copy(any(), any()) } doReturn mock<Bitmap>()
+      }
+    replay.onScreenshotRecorded(sourceBitmap)
+
+    verify(fixture.replayCache).addFrame(any<Bitmap>(), any(), anyOrNull())
+  }
+
+  @Test
+  fun `snapshot observer is not invoked when null`() {
+    val captureStrategy =
+      mock<CaptureStrategy> {
+        doAnswer {
+            ((it.arguments[1] as ReplayCache.(frameTimestamp: Long) -> Unit)).invoke(
+              fixture.replayCache,
+              1720693523997,
+            )
+          }
+          .whenever(mock)
+          .onScreenshotRecorded(anyOrNull<Bitmap>(), any())
+      }
+    val replay = fixture.getSut(context, replayCaptureStrategyProvider = { captureStrategy })
+
+    replay.register(fixture.scopes, fixture.options)
+    replay.start()
+
+    replay.onScreenshotRecorded(mock<Bitmap>())
+
+    verify(fixture.replayCache).addFrame(any<Bitmap>(), any(), anyOrNull())
+  }
+
   private fun getSessionCaptureStrategy(options: SentryOptions): SessionCaptureStrategy =
     SessionCaptureStrategy(
       options,

sentry/api/sentry.api

@@ -4063,6 +4063,7 @@ public final class io/sentry/SentryReplayOptions : io/sentry/SentryMaskingOption
 	public fun addUnmaskViewClass (Ljava/lang/String;)V
 	public fun getBeforeErrorSampling ()Lio/sentry/SentryReplayOptions$BeforeErrorSamplingCallback;
 	public fun getErrorReplayDuration ()J
+	public fun getFrameObserver ()Lio/sentry/SentryReplayOptions$ReplayFrameObserver;
 	public fun getFrameRate ()I
 	public fun getNetworkDetailAllowUrls ()Ljava/util/List;
 	public fun getNetworkDetailDenyUrls ()Ljava/util/List;
@@ -4085,6 +4086,7 @@ public final class io/sentry/SentryReplayOptions : io/sentry/SentryMaskingOption
 	public fun setBeforeErrorSampling (Lio/sentry/SentryReplayOptions$BeforeErrorSamplingCallback;)V
 	public fun setCaptureSurfaceViews (Z)V
 	public fun setDebug (Z)V
+	public fun setFrameObserver (Lio/sentry/SentryReplayOptions$ReplayFrameObserver;)V
 	public fun setMaskAllImages (Z)V
 	public fun setMaskAllText (Z)V
 	public fun setNetworkCaptureBodies (Z)V
@@ -4105,6 +4107,10 @@ public abstract interface class io/sentry/SentryReplayOptions$BeforeErrorSamplin
 	public abstract fun execute (Lio/sentry/SentryEvent;Lio/sentry/Hint;)Z
 }
 
+public abstract interface class io/sentry/SentryReplayOptions$ReplayFrameObserver {
+	public abstract fun onMaskedFrameCaptured (Lio/sentry/Hint;JLjava/lang/String;)V
+}
+
 public final class io/sentry/SentryReplayOptions$SentryReplayQuality : java/lang/Enum {
 	public static final field HIGH Lio/sentry/SentryReplayOptions$SentryReplayQuality;
 	public static final field LOW Lio/sentry/SentryReplayOptions$SentryReplayQuality;
@@ -4651,6 +4657,7 @@ public final class io/sentry/TypeCheckHint {
 	public static final field OKHTTP_RESPONSE Ljava/lang/String;
 	public static final field OPEN_FEIGN_REQUEST Ljava/lang/String;
 	public static final field OPEN_FEIGN_RESPONSE Ljava/lang/String;
+	public static final field REPLAY_FRAME_BITMAP Ljava/lang/String;
 	public static final field SENTRY_DART_SDK_NAME Ljava/lang/String;
 	public static final field SENTRY_DOTNET_SDK_NAME Ljava/lang/String;
 	public static final field SENTRY_EVENT_DROP_REASON Ljava/lang/String;