package no.naiv.tiltshift.effect import android.content.Context import android.graphics.SurfaceTexture import android.opengl.GLES11Ext import android.opengl.GLES20 import android.opengl.GLSurfaceView import android.util.Log import android.view.Surface import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.FloatBuffer import javax.microedition.khronos.egl.EGLConfig import javax.microedition.khronos.opengles.GL10 /** * OpenGL renderer for applying tilt-shift effect to camera preview * using a two-pass separable Gaussian blur. * * Rendering pipeline (3 draw calls per frame): * 1. **Passthrough**: camera texture → FBO-A (handles coordinate transform via vertex/texcoord) * 2. **Horizontal blur**: FBO-A → FBO-B (13-tap Gaussian, tilt-shift mask) * 3. **Vertical blur**: FBO-B → screen (13-tap Gaussian, tilt-shift mask) * * The passthrough decouples the camera's rotated coordinate system from the blur * passes, which work entirely in screen space. */ class TiltShiftRenderer( private val context: Context, private val onSurfaceTextureAvailable: (SurfaceTexture) -> Unit, private val onFrameAvailable: () -> Unit ) : GLSurfaceView.Renderer { companion object { private const val TAG = "TiltShiftRenderer" } private lateinit var shader: TiltShiftShader private var surfaceTexture: SurfaceTexture? = null private var cameraTextureId: Int = 0 // Camera quad: crop-to-fill vertices + rotated texcoords (pass 1 only) private lateinit var cameraVertexBuffer: FloatBuffer private lateinit var cameraTexCoordBuffer: FloatBuffer // Fullscreen quad for blur passes (no crop, standard texcoords) private lateinit var fullscreenVertexBuffer: FloatBuffer private lateinit var fullscreenTexCoordBuffer: FloatBuffer private var surfaceWidth: Int = 0 private var surfaceHeight: Int = 0 // FBO resources: one framebuffer, two color textures for ping-pong private var fboId: Int = 0 private var fboTexA: Int = 0 private var fboTexB: Int = 0 // Current effect parameters (updated from UI thread) @Volatile var blurParameters: BlurParameters = BlurParameters.DEFAULT @Volatile private var isFrontCamera: Boolean = false // Camera resolution for aspect ratio correction (set from UI thread) @Volatile private var cameraWidth: Int = 0 @Volatile private var cameraHeight: Int = 0 @Volatile private var vertexBufferDirty: Boolean = false // Texture coordinates for the back camera, indexed by Surface.ROTATION_*. // The base orientation (index 0) applies the 90° CCW rotation that maps // the landscape sensor frame to a portrait display. Indices 1/2/3 layer // additional CCW rotations on top so the activity's rotation is // compensated and world-up stays at clip-space-top. private val texCoordsBackByRotation = arrayOf( floatArrayOf(1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f), // ROTATION_0 floatArrayOf(1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f), // ROTATION_90 floatArrayOf(0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f), // ROTATION_180 floatArrayOf(0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f) // ROTATION_270 ) // Front camera variants: same as back, but horizontally mirrored // for the natural selfie view. private val texCoordsFrontByRotation = arrayOf( floatArrayOf(0f, 1f, 0f, 0f, 1f, 1f, 1f, 0f), // ROTATION_0 floatArrayOf(0f, 0f, 1f, 0f, 0f, 1f, 1f, 1f), // ROTATION_90 floatArrayOf(1f, 0f, 1f, 1f, 0f, 0f, 0f, 1f), // ROTATION_180 floatArrayOf(1f, 1f, 0f, 1f, 1f, 0f, 0f, 0f) // ROTATION_270 ) @Volatile private var displayRotation: Int = Surface.ROTATION_0 @Volatile private var currentTexCoords = texCoordsBackByRotation[Surface.ROTATION_0] @Volatile private var updateTexCoordBuffer = false override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) { GLES20.glClearColor(0f, 0f, 0f, 1f) shader = TiltShiftShader(context) shader.initialize() // Camera quad vertex buffer (crop-to-fill, recomputed when resolution is known) cameraVertexBuffer = allocateFloatBuffer(8) cameraVertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)) cameraVertexBuffer.position(0) // Camera texcoord buffer (rotated for portrait) cameraTexCoordBuffer = allocateFloatBuffer(8) cameraTexCoordBuffer.put(currentTexCoords) cameraTexCoordBuffer.position(0) // Fullscreen quad for blur passes (standard coords) fullscreenVertexBuffer = allocateFloatBuffer(8) fullscreenVertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)) fullscreenVertexBuffer.position(0) fullscreenTexCoordBuffer = allocateFloatBuffer(8) fullscreenTexCoordBuffer.put(floatArrayOf(0f, 0f, 1f, 0f, 0f, 1f, 1f, 1f)) fullscreenTexCoordBuffer.position(0) // Create camera texture val textures = IntArray(1) GLES20.glGenTextures(1, textures, 0) cameraTextureId = textures[0] GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, cameraTextureId) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) // Create SurfaceTexture for camera frames surfaceTexture = SurfaceTexture(cameraTextureId).also { it.setOnFrameAvailableListener { onFrameAvailable() } onSurfaceTextureAvailable(it) } } override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) { GLES20.glViewport(0, 0, width, height) surfaceWidth = width surfaceHeight = height vertexBufferDirty = true recreateFBOs(width, height) } override fun onDrawFrame(gl: GL10?) { surfaceTexture?.updateTexImage() if (vertexBufferDirty) { recomputeVertices() vertexBufferDirty = false } if (updateTexCoordBuffer) { cameraTexCoordBuffer.clear() cameraTexCoordBuffer.put(currentTexCoords) cameraTexCoordBuffer.position(0) updateTexCoordBuffer = false } val params = blurParameters // --- Pass 1: Camera → FBO-A (passthrough with crop-to-fill) --- GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboId) GLES20.glFramebufferTexture2D( GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, fboTexA, 0 ) GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) shader.usePassthrough(cameraTextureId) drawQuad( shader.passthroughPositionLoc, shader.passthroughTexCoordLoc, cameraVertexBuffer, cameraTexCoordBuffer ) // --- Pass 2: FBO-A → FBO-B (horizontal blur) --- GLES20.glFramebufferTexture2D( GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, fboTexB, 0 ) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) shader.useBlurPass(fboTexA, params, surfaceWidth, surfaceHeight, 1f, 0f) drawQuad( shader.blurPositionLoc, shader.blurTexCoordLoc, fullscreenVertexBuffer, fullscreenTexCoordBuffer ) // --- Pass 3: FBO-B → screen (vertical blur) --- GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0) GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight) GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT) shader.useBlurPass(fboTexB, params, surfaceWidth, surfaceHeight, 0f, 1f) drawQuad( shader.blurPositionLoc, shader.blurTexCoordLoc, fullscreenVertexBuffer, fullscreenTexCoordBuffer ) } fun updateParameters(params: BlurParameters) { blurParameters = params } fun setFrontCamera(front: Boolean) { if (isFrontCamera != front) { isFrontCamera = front refreshTexCoords() } } fun setCameraResolution(width: Int, height: Int) { if (cameraWidth != width || cameraHeight != height) { cameraWidth = width cameraHeight = height vertexBufferDirty = true } } /** * Updates the active display rotation. The texture-coordinate buffer is * rebuilt so the camera image stays world-aligned as the activity rotates * with the device under screenOrientation="fullSensor", and the * crop-to-fill math picks the correct effective aspect ratio. */ fun setDisplayRotation(rotation: Int) { if (displayRotation != rotation) { displayRotation = rotation refreshTexCoords() vertexBufferDirty = true } } private fun refreshTexCoords() { val table = if (isFrontCamera) texCoordsFrontByRotation else texCoordsBackByRotation val idx = displayRotation.coerceIn(0, table.size - 1) currentTexCoords = table[idx] updateTexCoordBuffer = true } fun release() { shader.release() surfaceTexture?.release() surfaceTexture = null if (cameraTextureId != 0) { GLES20.glDeleteTextures(1, intArrayOf(cameraTextureId), 0) cameraTextureId = 0 } deleteFBOs() } // --- Private helpers --- private fun drawQuad( positionLoc: Int, texCoordLoc: Int, vertices: FloatBuffer, texCoords: FloatBuffer ) { GLES20.glEnableVertexAttribArray(positionLoc) GLES20.glVertexAttribPointer(positionLoc, 2, GLES20.GL_FLOAT, false, 0, vertices) GLES20.glEnableVertexAttribArray(texCoordLoc) GLES20.glVertexAttribPointer(texCoordLoc, 2, GLES20.GL_FLOAT, false, 0, texCoords) GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) GLES20.glDisableVertexAttribArray(positionLoc) GLES20.glDisableVertexAttribArray(texCoordLoc) } /** * Recomputes camera vertex positions to achieve crop-to-fill. * * The camera sensor is landscape; after the orientation-dependent texcoord * rotation, the effective dimensions seen on screen are either swapped * (portrait orientations) or kept (landscape orientations). We scale the * vertex quad so the camera frame fills the surface without stretching — * the GPU clips the overflow. */ private fun recomputeVertices() { var scaleX = 1f var scaleY = 1f if (cameraWidth > 0 && cameraHeight > 0 && surfaceWidth > 0 && surfaceHeight > 0) { val isPortrait = displayRotation == Surface.ROTATION_0 || displayRotation == Surface.ROTATION_180 val effectiveW = if (isPortrait) cameraHeight else cameraWidth val effectiveH = if (isPortrait) cameraWidth else cameraHeight val cameraRatio = effectiveW.toFloat() / effectiveH val screenRatio = surfaceWidth.toFloat() / surfaceHeight if (cameraRatio > screenRatio) { scaleX = cameraRatio / screenRatio } else { scaleY = screenRatio / cameraRatio } } cameraVertexBuffer.clear() cameraVertexBuffer.put(floatArrayOf( -scaleX, -scaleY, scaleX, -scaleY, -scaleX, scaleY, scaleX, scaleY )) cameraVertexBuffer.position(0) } private fun recreateFBOs(width: Int, height: Int) { deleteFBOs() // Create two color textures for ping-pong val texIds = IntArray(2) GLES20.glGenTextures(2, texIds, 0) fboTexA = texIds[0] fboTexB = texIds[1] for (texId in texIds) { GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE) GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE) GLES20.glTexImage2D( GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, width, height, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null ) } // Create single FBO (we swap the attached texture for ping-pong) val fbos = IntArray(1) GLES20.glGenFramebuffers(1, fbos, 0) fboId = fbos[0] // Verify with texture A GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboId) GLES20.glFramebufferTexture2D( GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, fboTexA, 0 ) val status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER) if (status != GLES20.GL_FRAMEBUFFER_COMPLETE) { Log.e(TAG, "FBO incomplete: $status") } GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0) } private fun deleteFBOs() { if (fboId != 0) { GLES20.glDeleteFramebuffers(1, intArrayOf(fboId), 0) fboId = 0 } if (fboTexA != 0 || fboTexB != 0) { GLES20.glDeleteTextures(2, intArrayOf(fboTexA, fboTexB), 0) fboTexA = 0 fboTexB = 0 } } private fun allocateFloatBuffer(floatCount: Int): FloatBuffer { return ByteBuffer.allocateDirect(floatCount * 4) .order(ByteOrder.nativeOrder()) .asFloatBuffer() } }