package no.naiv.tiltshift.effect import android.content.Context import android.graphics.SurfaceTexture import android.opengl.GLES11Ext import android.opengl.GLES20 import android.opengl.GLSurfaceView import android.opengl.Matrix import android.util.Log import android.view.Surface import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.FloatBuffer import javax.microedition.khronos.egl.EGLConfig import javax.microedition.khronos.opengles.GL10 /** * OpenGL renderer for applying tilt-shift effect to camera preview * using a two-pass separable Gaussian blur. * * Rendering pipeline (3 draw calls per frame): * 1. **Passthrough**: camera texture → FBO-A (handles coordinate transform via vertex/texcoord) * 2. **Horizontal blur**: FBO-A → FBO-B (13-tap Gaussian, tilt-shift mask) * 3. **Vertical blur**: FBO-B → screen (13-tap Gaussian, tilt-shift mask) * * The passthrough decouples the camera's rotated coordinate system from the blur * passes, which work entirely in screen space. */ class TiltShiftRenderer( private val context: Context, private val onSurfaceTextureAvailable: (SurfaceTexture) -> Unit, private val onFrameAvailable: () -> Unit ) : GLSurfaceView.Renderer { companion object { private const val TAG = "TiltShiftRenderer" } private lateinit var shader: TiltShiftShader private var surfaceTexture: SurfaceTexture? = null private var cameraTextureId: Int = 0 // Camera quad: crop-to-fill vertices, standard texcoords (rotation comes from texMatrix) private lateinit var cameraVertexBuffer: FloatBuffer // Fullscreen quad for blur passes (no crop, standard texcoords) private lateinit var fullscreenVertexBuffer: FloatBuffer private lateinit var fullscreenTexCoordBuffer: FloatBuffer private var surfaceWidth: Int = 0 private var surfaceHeight: Int = 0 // FBO resources: one framebuffer, two color textures for ping-pong private var fboId: Int = 0 private var fboTexA: Int = 0 private var fboTexB: Int = 0 // SurfaceTexture transform matrix, refreshed each frame on the GL thread. private val texMatrix = FloatArray(16) // Sensor-to-buffer matrix from SurfaceTexture before display-rotation correction. private val sensorMatrix = FloatArray(16) // Current effect parameters (updated from UI thread) @Volatile var blurParameters: BlurParameters = BlurParameters.DEFAULT @Volatile private var isFrontCamera: Boolean = false // Camera resolution for aspect ratio correction (set from UI thread) @Volatile private var cameraWidth: Int = 0 @Volatile private var cameraHeight: Int = 0 /** Display rotation as a Surface.ROTATION_* constant; affects effective aspect. */ @Volatile private var displayRotation: Int = Surface.ROTATION_0 @Volatile private var vertexBufferDirty: Boolean = false override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) { GLES20.glClearColor(0f, 0f, 0f, 1f) shader = TiltShiftShader(context) shader.initialize() // Camera quad vertex buffer (crop-to-fill, recomputed when resolution is known) cameraVertexBuffer = allocateFloatBuffer(8) cameraVertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)) cameraVertexBuffer.position(0) // Fullscreen quad for blur passes (standard coords). The same buffer is reused // for the camera passthrough texcoords — rotation is applied via uTexMatrix. fullscreenVertexBuffer = allocateFloatBuffer(8) fullscreenVertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)) fullscreenVertexBuffer.position(0) fullscreenTexCoordBuffer = allocateFloatBuffer(8) fullscreenTexCoordBuffer.put(floatArrayOf(0f, 0f, 1f, 0f, 0f, 1f, 1f, 1f)) fullscreenTexCoordBuffer.position(0) // Create camera texture val textures = IntArray(1) GLES20.glGenTextures(1, textures, 0) cameraTextureId = textures[0] GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, cameraTextureId) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) // Create SurfaceTexture for camera frames surfaceTexture = SurfaceTexture(cameraTextureId).also { it.setOnFrameAvailableListener { onFrameAvailable() } onSurfaceTextureAvailable(it) } } override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) { GLES20.glViewport(0, 0, width, height) surfaceWidth = width surfaceHeight = height vertexBufferDirty = true recreateFBOs(width, height) } override fun onDrawFrame(gl: GL10?) { val st = surfaceTexture st?.updateTexImage() // SurfaceTexture's transform matrix only handles the sensor-to-buffer // orientation; for a custom SurfaceProvider it does NOT vary with // Preview.targetRotation. Apply the display-rotation correction here. st?.getTransformMatrix(sensorMatrix) composeTexMatrix() if (vertexBufferDirty) { recomputeVertices() vertexBufferDirty = false } val params = blurParameters // --- Pass 1: Camera → FBO-A (passthrough with crop-to-fill) --- GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboId) GLES20.glFramebufferTexture2D( GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, fboTexA, 0 ) GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) shader.usePassthrough(cameraTextureId, texMatrix, isFrontCamera) drawQuad( shader.passthroughPositionLoc, shader.passthroughTexCoordLoc, cameraVertexBuffer, fullscreenTexCoordBuffer ) // --- Pass 2: FBO-A → FBO-B (horizontal blur) --- GLES20.glFramebufferTexture2D( GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, fboTexB, 0 ) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) shader.useBlurPass(fboTexA, params, surfaceWidth, surfaceHeight, 1f, 0f) drawQuad( shader.blurPositionLoc, shader.blurTexCoordLoc, fullscreenVertexBuffer, fullscreenTexCoordBuffer ) // --- Pass 3: FBO-B → screen (vertical blur) --- GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0) GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) shader.useBlurPass(fboTexB, params, surfaceWidth, surfaceHeight, 0f, 1f) drawQuad( shader.blurPositionLoc, shader.blurTexCoordLoc, fullscreenVertexBuffer, fullscreenTexCoordBuffer ) } fun updateParameters(params: BlurParameters) { blurParameters = params } fun setFrontCamera(front: Boolean) { isFrontCamera = front } fun setCameraResolution(width: Int, height: Int) { if (cameraWidth != width || cameraHeight != height) { cameraWidth = width cameraHeight = height vertexBufferDirty = true } } /** Updates the display rotation so crop-to-fill picks the right effective aspect. */ fun setDisplayRotation(rotation: Int) { if (displayRotation != rotation) { displayRotation = rotation vertexBufferDirty = true } } /** * Combines the sensor-to-buffer matrix with a rotation that compensates for * the activity's current rotation. The activity rotates with the device * (screenOrientation="fullSensor"), so the GL clip-space "up" direction * tracks the device rather than the world. To keep world-up at screen-up * regardless of orientation, rotate the texcoord sampling pattern by the * inverse of the activity rotation. Without this correction, the two * landscape orientations would render the same matrix and one would appear * upside-down on a real device. */ private fun composeTexMatrix() { // Inverse of the activity rotation: rotating the sampling pattern by // -activityAngle puts the world-aligned point originally at screen P // at the same screen P after the activity has rotated. val angle = when (displayRotation) { Surface.ROTATION_90 -> -90f Surface.ROTATION_180 -> 180f Surface.ROTATION_270 -> 90f else -> 0f } if (angle == 0f) { System.arraycopy(sensorMatrix, 0, texMatrix, 0, 16) return } // Build rotation around the (0.5, 0.5) texcoord center. Matrix.setIdentityM(texMatrix, 0) Matrix.translateM(texMatrix, 0, 0.5f, 0.5f, 0f) Matrix.rotateM(texMatrix, 0, angle, 0f, 0f, 1f) Matrix.translateM(texMatrix, 0, -0.5f, -0.5f, 0f) // texMatrix = sensorMatrix * rotation (rotation is applied to the // texcoord first, then the sensor-to-buffer transform). val rot = texMatrix.copyOf() Matrix.multiplyMM(texMatrix, 0, sensorMatrix, 0, rot, 0) } fun release() { shader.release() surfaceTexture?.release() surfaceTexture = null if (cameraTextureId != 0) { GLES20.glDeleteTextures(1, intArrayOf(cameraTextureId), 0) cameraTextureId = 0 } deleteFBOs() } // --- Private helpers --- private fun drawQuad( positionLoc: Int, texCoordLoc: Int, vertices: FloatBuffer, texCoords: FloatBuffer ) { GLES20.glEnableVertexAttribArray(positionLoc) GLES20.glVertexAttribPointer(positionLoc, 2, GLES20.GL_FLOAT, false, 0, vertices) GLES20.glEnableVertexAttribArray(texCoordLoc) GLES20.glVertexAttribPointer(texCoordLoc, 2, GLES20.GL_FLOAT, false, 0, texCoords) GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) GLES20.glDisableVertexAttribArray(positionLoc) GLES20.glDisableVertexAttribArray(texCoordLoc) } /** * Recomputes camera vertex positions to achieve crop-to-fill. * * The camera buffer is the sensor's native landscape resolution. The texMatrix * rotates it to match the display, so the *effective* displayed dimensions * depend on the display rotation: in portrait the buffer is rotated 90° * (effective width = cameraHeight), in landscape it is unrotated. * We scale the vertex quad so the frame fills the surface without stretching — * the GPU clips the overflow. */ private fun recomputeVertices() { var scaleX = 1f var scaleY = 1f if (cameraWidth > 0 && cameraHeight > 0 && surfaceWidth > 0 && surfaceHeight > 0) { val isPortrait = displayRotation == Surface.ROTATION_0 || displayRotation == Surface.ROTATION_180 val effectiveW = if (isPortrait) cameraHeight else cameraWidth val effectiveH = if (isPortrait) cameraWidth else cameraHeight val cameraRatio = effectiveW.toFloat() / effectiveH val screenRatio = surfaceWidth.toFloat() / surfaceHeight if (cameraRatio > screenRatio) { scaleX = cameraRatio / screenRatio } else { scaleY = screenRatio / cameraRatio } } cameraVertexBuffer.clear() cameraVertexBuffer.put(floatArrayOf( -scaleX, -scaleY, scaleX, -scaleY, -scaleX, scaleY, scaleX, scaleY )) cameraVertexBuffer.position(0) } private fun recreateFBOs(width: Int, height: Int) { deleteFBOs() // Create two color textures for ping-pong val texIds = IntArray(2) GLES20.glGenTextures(2, texIds, 0) fboTexA = texIds[0] fboTexB = texIds[1] for (texId in texIds) { GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) GLES20.glTexImage2D( GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, width, height, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null ) } // Create single FBO (we swap the attached texture for ping-pong) val fbos = IntArray(1) GLES20.glGenFramebuffers(1, fbos, 0) fboId = fbos[0] // Verify with texture A GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboId) GLES20.glFramebufferTexture2D( GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, fboTexA, 0 ) val status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) if (status != GLES20.GL_FRAMEBUFFER_COMPLETE) { Log.e(TAG, "FBO incomplete: $status") } GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0) } private fun deleteFBOs() { if (fboId != 0) { GLES20.glDeleteFramebuffers(1, intArrayOf(fboId), 0) fboId = 0 } if (fboTexA != 0 || fboTexB != 0) { GLES20.glDeleteTextures(2, intArrayOf(fboTexA, fboTexB), 0) fboTexA = 0 fboTexB = 0 } } private fun allocateFloatBuffer(floatCount: Int): FloatBuffer { return ByteBuffer.allocateDirect(floatCount * 4) .order(ByteOrder.nativeOrder()) .asFloatBuffer() } }