From 7979ebd0298ba48507d9988a119e2c5f8287d6f2 Mon Sep 17 00:00:00 2001 From: Ole-Morten Duesund Date: Thu, 5 Mar 2026 13:58:17 +0100 Subject: [PATCH] Fix CameraX GL surface: render-on-demand, crop-to-fill, lifecycle - Switch GLSurfaceView from RENDERMODE_CONTINUOUSLY to RENDERMODE_WHEN_DIRTY with OnFrameAvailableListener, halving GPU work - Add crop-to-fill aspect ratio correction so camera preview is not stretched on displays with non-16:9 aspect ratios - Add LifecycleEventObserver to pause/resume GLSurfaceView with Activity lifecycle, preventing background rendering and GL context issues Co-Authored-By: Claude Opus 4.6 --- .../no/naiv/tiltshift/camera/CameraManager.kt | 4 + .../tiltshift/effect/TiltShiftRenderer.kt | 79 ++++++++++++++++--- .../java/no/naiv/tiltshift/ui/CameraScreen.kt | 42 ++++++++-- 3 files changed, 107 insertions(+), 18 deletions(-) diff --git a/app/src/main/java/no/naiv/tiltshift/camera/CameraManager.kt b/app/src/main/java/no/naiv/tiltshift/camera/CameraManager.kt index 9c440ac..00f9107 100644 --- a/app/src/main/java/no/naiv/tiltshift/camera/CameraManager.kt +++ b/app/src/main/java/no/naiv/tiltshift/camera/CameraManager.kt @@ -59,6 +59,9 @@ class CameraManager(private val context: Context) { private val _isFrontCamera = MutableStateFlow(false) val isFrontCamera: StateFlow = _isFrontCamera.asStateFlow() + private val _previewResolution = MutableStateFlow(Size(0, 0)) + val previewResolution: StateFlow = _previewResolution.asStateFlow() + /** Background executor for image capture callbacks to avoid blocking the main thread. */ private val captureExecutor: ExecutorService = Executors.newSingleThreadExecutor() @@ -161,6 +164,7 @@ class CameraManager(private val context: Context) { } surfaceSize = request.resolution + _previewResolution.value = surfaceSize surfaceTexture.setDefaultBufferSize(surfaceSize.width, surfaceSize.height) val surface = Surface(surfaceTexture) diff --git a/app/src/main/java/no/naiv/tiltshift/effect/TiltShiftRenderer.kt b/app/src/main/java/no/naiv/tiltshift/effect/TiltShiftRenderer.kt index b3bf963..3258fa8 100644 --- a/app/src/main/java/no/naiv/tiltshift/effect/TiltShiftRenderer.kt +++ b/app/src/main/java/no/naiv/tiltshift/effect/TiltShiftRenderer.kt @@ -19,7 +19,8 @@ import javax.microedition.khronos.opengles.GL10 */ class TiltShiftRenderer( private val context: Context, - private val onSurfaceTextureAvailable: (SurfaceTexture) -> Unit + private val onSurfaceTextureAvailable: (SurfaceTexture) -> Unit, + private val onFrameAvailable: () -> Unit ) : GLSurfaceView.Renderer { private lateinit var shader: TiltShiftShader @@ -39,13 +40,13 @@ class TiltShiftRenderer( @Volatile private var isFrontCamera: Boolean = false - // Quad vertices (full screen) - private val vertices = floatArrayOf( - -1f, -1f, // Bottom left - 1f, -1f, // Bottom right - -1f, 1f, // Top left - 1f, 1f // Top right - ) + // Camera resolution for aspect ratio correction (set from UI thread) + @Volatile + private var cameraWidth: Int = 0 + @Volatile + private var cameraHeight: Int = 0 + @Volatile + private var vertexBufferDirty: Boolean = false // Texture coordinates rotated 90° for portrait mode (back camera) // (Camera sensors are landscape-oriented, we rotate to portrait) @@ -75,11 +76,12 @@ class TiltShiftRenderer( shader = TiltShiftShader(context) shader.initialize() - // Create vertex buffer - vertexBuffer = ByteBuffer.allocateDirect(vertices.size * 4) + // Allocate vertex buffer (8 floats = 4 vertices × 2 components) + vertexBuffer = ByteBuffer.allocateDirect(8 * 4) .order(ByteOrder.nativeOrder()) .asFloatBuffer() - .put(vertices) + // Fill with default full-screen quad; will be recomputed when camera resolution is known + vertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)) vertexBuffer.position(0) // Create texture coordinate buffer @@ -102,6 +104,7 @@ class TiltShiftRenderer( // Create SurfaceTexture for camera frames surfaceTexture = SurfaceTexture(cameraTextureId).also { + it.setOnFrameAvailableListener { onFrameAvailable() } onSurfaceTextureAvailable(it) } } @@ -110,12 +113,19 @@ class TiltShiftRenderer( GLES20.glViewport(0, 0, width, height) surfaceWidth = width surfaceHeight = height + vertexBufferDirty = true } override fun onDrawFrame(gl: GL10?) { // Update texture with latest camera frame surfaceTexture?.updateTexImage() + // Recompute vertex buffer for crop-to-fill when camera or surface dimensions change + if (vertexBufferDirty) { + recomputeVertices() + vertexBufferDirty = false + } + // Update texture coordinate buffer if camera changed if (updateTexCoordBuffer) { texCoordBuffer.clear() @@ -182,6 +192,53 @@ class TiltShiftRenderer( @Volatile private var updateTexCoordBuffer = false + /** + * Sets the camera preview resolution for crop-to-fill aspect ratio correction. + * Thread-safe — vertex buffer is recomputed on the next frame. + */ + fun setCameraResolution(width: Int, height: Int) { + if (cameraWidth != width || cameraHeight != height) { + cameraWidth = width + cameraHeight = height + vertexBufferDirty = true + } + } + + /** + * Recomputes vertex positions to achieve crop-to-fill. + * + * The camera sensor is landscape; after the 90° rotation applied via texture coordinates, + * the effective portrait dimensions are (cameraHeight × cameraWidth). We scale the vertex + * quad so the camera frame fills the screen without stretching — the GPU clips the overflow. + */ + private fun recomputeVertices() { + var scaleX = 1f + var scaleY = 1f + + if (cameraWidth > 0 && cameraHeight > 0 && surfaceWidth > 0 && surfaceHeight > 0) { + // After 90° rotation: portrait width = cameraHeight, portrait height = cameraWidth + val cameraRatio = cameraHeight.toFloat() / cameraWidth + val screenRatio = surfaceWidth.toFloat() / surfaceHeight + + if (cameraRatio > screenRatio) { + // Camera wider than screen → crop sides + scaleX = cameraRatio / screenRatio + } else { + // Camera taller than screen → crop top/bottom + scaleY = screenRatio / cameraRatio + } + } + + vertexBuffer.clear() + vertexBuffer.put(floatArrayOf( + -scaleX, -scaleY, + scaleX, -scaleY, + -scaleX, scaleY, + scaleX, scaleY + )) + vertexBuffer.position(0) + } + /** * Releases OpenGL resources. * Must be called from GL thread. diff --git a/app/src/main/java/no/naiv/tiltshift/ui/CameraScreen.kt b/app/src/main/java/no/naiv/tiltshift/ui/CameraScreen.kt index 80ed80b..8012961 100644 --- a/app/src/main/java/no/naiv/tiltshift/ui/CameraScreen.kt +++ b/app/src/main/java/no/naiv/tiltshift/ui/CameraScreen.kt @@ -74,6 +74,8 @@ import androidx.compose.ui.semantics.stateDescription import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.ui.viewinterop.AndroidView +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleEventObserver import androidx.lifecycle.compose.LocalLifecycleOwner import androidx.lifecycle.viewmodel.compose.viewModel import kotlinx.coroutines.flow.collectLatest @@ -116,6 +118,7 @@ fun CameraScreen( val minZoom by viewModel.cameraManager.minZoomRatio.collectAsState() val maxZoom by viewModel.cameraManager.maxZoomRatio.collectAsState() val isFrontCamera by viewModel.cameraManager.isFrontCamera.collectAsState() + val previewResolution by viewModel.cameraManager.previewResolution.collectAsState() val cameraError by viewModel.cameraManager.error.collectAsState() // Gallery picker @@ -161,6 +164,14 @@ fun CameraScreen( glSurfaceView?.requestRender() } + // Update renderer with camera preview resolution for crop-to-fill + LaunchedEffect(previewResolution) { + if (previewResolution.width > 0) { + renderer?.setCameraResolution(previewResolution.width, previewResolution.height) + glSurfaceView?.requestRender() + } + } + // Start camera when surface texture is available LaunchedEffect(surfaceTexture) { surfaceTexture?.let { @@ -172,14 +183,28 @@ fun CameraScreen( LaunchedEffect(isGalleryPreview) { if (isGalleryPreview) { glSurfaceView?.onPause() - } else { + } else if (lifecycleOwner.lifecycle.currentState.isAtLeast(Lifecycle.State.RESUMED)) { glSurfaceView?.onResume() } } - // Cleanup GL resources on GL thread (ViewModel handles its own cleanup in onCleared) - DisposableEffect(Unit) { + // Tie GLSurfaceView lifecycle to Activity lifecycle to prevent background rendering + val currentIsGalleryPreview by rememberUpdatedState(isGalleryPreview) + DisposableEffect(lifecycleOwner) { + val observer = LifecycleEventObserver { _, event -> + when (event) { + Lifecycle.Event.ON_RESUME -> { + if (!currentIsGalleryPreview) { + glSurfaceView?.onResume() + } + } + Lifecycle.Event.ON_PAUSE -> glSurfaceView?.onPause() + else -> {} + } + } + lifecycleOwner.lifecycle.addObserver(observer) onDispose { + lifecycleOwner.lifecycle.removeObserver(observer) glSurfaceView?.queueEvent { renderer?.release() } } } @@ -208,13 +233,16 @@ fun CameraScreen( GLSurfaceView(ctx).apply { setEGLContextClientVersion(2) - val newRenderer = TiltShiftRenderer(ctx) { st -> - surfaceTexture = st - } + val view = this + val newRenderer = TiltShiftRenderer( + context = ctx, + onSurfaceTextureAvailable = { st -> surfaceTexture = st }, + onFrameAvailable = { view.requestRender() } + ) renderer = newRenderer setRenderer(newRenderer) - renderMode = GLSurfaceView.RENDERMODE_CONTINUOUSLY + renderMode = GLSurfaceView.RENDERMODE_WHEN_DIRTY glSurfaceView = this }