tilt-shift-camera/app/src/main/java/no/naiv/tiltshift/effect/TiltShiftRenderer.kt
Ole-Morten Duesund a2dfa7db3d Make camera image follow device rotation (4-orientation texcoord table)
Re-add landscape support, this time via four precomputed texcoord
buffers — one per Surface.ROTATION_* — instead of going through
SurfaceTexture.getTransformMatrix() (which doesn't honour
Preview.targetRotation for custom SurfaceProviders) or the manual
matrix composition attempts in v1.1.6–1.1.11.

For each device orientation the renderer picks the texcoord set that
both compensates for the 90° CW sensor mount and the activity's own
rotation under screenOrientation="fullSensor", so world-up stays at
clip-space-top. recomputeVertices swaps effective camera dimensions
between portrait and landscape so crop-to-fill picks the right aspect.

Verified empirically in the emulator across all four Display.rotation
values (sky-yellow band always lands at the top of the screen).

Bump to 1.1.12.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-11 16:12:29 +02:00

376 lines
14 KiB
Kotlin

package no.naiv.tiltshift.effect
import android.content.Context
import android.graphics.SurfaceTexture
import android.opengl.GLES11Ext
import android.opengl.GLES20
import android.opengl.GLSurfaceView
import android.util.Log
import android.view.Surface
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
import javax.microedition.khronos.egl.EGLConfig
import javax.microedition.khronos.opengles.GL10
/**
* OpenGL renderer for applying tilt-shift effect to camera preview
* using a two-pass separable Gaussian blur.
*
* Rendering pipeline (3 draw calls per frame):
* 1. **Passthrough**: camera texture → FBO-A (handles coordinate transform via vertex/texcoord)
* 2. **Horizontal blur**: FBO-A → FBO-B (13-tap Gaussian, tilt-shift mask)
* 3. **Vertical blur**: FBO-B → screen (13-tap Gaussian, tilt-shift mask)
*
* The passthrough decouples the camera's rotated coordinate system from the blur
* passes, which work entirely in screen space.
*/
class TiltShiftRenderer(
private val context: Context,
private val onSurfaceTextureAvailable: (SurfaceTexture) -> Unit,
private val onFrameAvailable: () -> Unit
) : GLSurfaceView.Renderer {
companion object {
private const val TAG = "TiltShiftRenderer"
}
private lateinit var shader: TiltShiftShader
private var surfaceTexture: SurfaceTexture? = null
private var cameraTextureId: Int = 0
// Camera quad: crop-to-fill vertices + rotated texcoords (pass 1 only)
private lateinit var cameraVertexBuffer: FloatBuffer
private lateinit var cameraTexCoordBuffer: FloatBuffer
// Fullscreen quad for blur passes (no crop, standard texcoords)
private lateinit var fullscreenVertexBuffer: FloatBuffer
private lateinit var fullscreenTexCoordBuffer: FloatBuffer
private var surfaceWidth: Int = 0
private var surfaceHeight: Int = 0
// FBO resources: one framebuffer, two color textures for ping-pong
private var fboId: Int = 0
private var fboTexA: Int = 0
private var fboTexB: Int = 0
// Current effect parameters (updated from UI thread)
@Volatile
var blurParameters: BlurParameters = BlurParameters.DEFAULT
@Volatile
private var isFrontCamera: Boolean = false
// Camera resolution for aspect ratio correction (set from UI thread)
@Volatile
private var cameraWidth: Int = 0
@Volatile
private var cameraHeight: Int = 0
@Volatile
private var vertexBufferDirty: Boolean = false
// Texture coordinates for the back camera, indexed by Surface.ROTATION_*.
// The base orientation (index 0) applies the 90° CCW rotation that maps
// the landscape sensor frame to a portrait display. Indices 1/2/3 layer
// additional CCW rotations on top so the activity's rotation is
// compensated and world-up stays at clip-space-top.
private val texCoordsBackByRotation = arrayOf(
floatArrayOf(1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f), // ROTATION_0
floatArrayOf(1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f), // ROTATION_90
floatArrayOf(0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f), // ROTATION_180
floatArrayOf(0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f) // ROTATION_270
)
// Front camera variants: same as back, but horizontally mirrored
// for the natural selfie view.
private val texCoordsFrontByRotation = arrayOf(
floatArrayOf(0f, 1f, 0f, 0f, 1f, 1f, 1f, 0f), // ROTATION_0
floatArrayOf(0f, 0f, 1f, 0f, 0f, 1f, 1f, 1f), // ROTATION_90
floatArrayOf(1f, 0f, 1f, 1f, 0f, 0f, 0f, 1f), // ROTATION_180
floatArrayOf(1f, 1f, 0f, 1f, 1f, 0f, 0f, 0f) // ROTATION_270
)
@Volatile
private var displayRotation: Int = Surface.ROTATION_0
@Volatile
private var currentTexCoords = texCoordsBackByRotation[Surface.ROTATION_0]
@Volatile
private var updateTexCoordBuffer = false
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
GLES20.glClearColor(0f, 0f, 0f, 1f)
shader = TiltShiftShader(context)
shader.initialize()
// Camera quad vertex buffer (crop-to-fill, recomputed when resolution is known)
cameraVertexBuffer = allocateFloatBuffer(8)
cameraVertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f))
cameraVertexBuffer.position(0)
// Camera texcoord buffer (rotated for portrait)
cameraTexCoordBuffer = allocateFloatBuffer(8)
cameraTexCoordBuffer.put(currentTexCoords)
cameraTexCoordBuffer.position(0)
// Fullscreen quad for blur passes (standard coords)
fullscreenVertexBuffer = allocateFloatBuffer(8)
fullscreenVertexBuffer.put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f))
fullscreenVertexBuffer.position(0)
fullscreenTexCoordBuffer = allocateFloatBuffer(8)
fullscreenTexCoordBuffer.put(floatArrayOf(0f, 0f, 1f, 0f, 0f, 1f, 1f, 1f))
fullscreenTexCoordBuffer.position(0)
// Create camera texture
val textures = IntArray(1)
GLES20.glGenTextures(1, textures, 0)
cameraTextureId = textures[0]
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, cameraTextureId)
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
// Create SurfaceTexture for camera frames
surfaceTexture = SurfaceTexture(cameraTextureId).also {
it.setOnFrameAvailableListener { onFrameAvailable() }
onSurfaceTextureAvailable(it)
}
}
override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
GLES20.glViewport(0, 0, width, height)
surfaceWidth = width
surfaceHeight = height
vertexBufferDirty = true
recreateFBOs(width, height)
}
override fun onDrawFrame(gl: GL10?) {
surfaceTexture?.updateTexImage()
if (vertexBufferDirty) {
recomputeVertices()
vertexBufferDirty = false
}
if (updateTexCoordBuffer) {
cameraTexCoordBuffer.clear()
cameraTexCoordBuffer.put(currentTexCoords)
cameraTexCoordBuffer.position(0)
updateTexCoordBuffer = false
}
val params = blurParameters
// --- Pass 1: Camera → FBO-A (passthrough with crop-to-fill) ---
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboId)
GLES20.glFramebufferTexture2D(
GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, fboTexA, 0
)
GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight)
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
shader.usePassthrough(cameraTextureId)
drawQuad(
shader.passthroughPositionLoc, shader.passthroughTexCoordLoc,
cameraVertexBuffer, cameraTexCoordBuffer
)
// --- Pass 2: FBO-A → FBO-B (horizontal blur) ---
GLES20.glFramebufferTexture2D(
GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, fboTexB, 0
)
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
shader.useBlurPass(fboTexA, params, surfaceWidth, surfaceHeight, 1f, 0f)
drawQuad(
shader.blurPositionLoc, shader.blurTexCoordLoc,
fullscreenVertexBuffer, fullscreenTexCoordBuffer
)
// --- Pass 3: FBO-B → screen (vertical blur) ---
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight)
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
shader.useBlurPass(fboTexB, params, surfaceWidth, surfaceHeight, 0f, 1f)
drawQuad(
shader.blurPositionLoc, shader.blurTexCoordLoc,
fullscreenVertexBuffer, fullscreenTexCoordBuffer
)
}
fun updateParameters(params: BlurParameters) {
blurParameters = params
}
fun setFrontCamera(front: Boolean) {
if (isFrontCamera != front) {
isFrontCamera = front
refreshTexCoords()
}
}
fun setCameraResolution(width: Int, height: Int) {
if (cameraWidth != width || cameraHeight != height) {
cameraWidth = width
cameraHeight = height
vertexBufferDirty = true
}
}
/**
* Updates the active display rotation. The texture-coordinate buffer is
* rebuilt so the camera image stays world-aligned as the activity rotates
* with the device under screenOrientation="fullSensor", and the
* crop-to-fill math picks the correct effective aspect ratio.
*/
fun setDisplayRotation(rotation: Int) {
if (displayRotation != rotation) {
displayRotation = rotation
refreshTexCoords()
vertexBufferDirty = true
}
}
private fun refreshTexCoords() {
val table = if (isFrontCamera) texCoordsFrontByRotation else texCoordsBackByRotation
val idx = displayRotation.coerceIn(0, table.size - 1)
currentTexCoords = table[idx]
updateTexCoordBuffer = true
}
fun release() {
shader.release()
surfaceTexture?.release()
surfaceTexture = null
if (cameraTextureId != 0) {
GLES20.glDeleteTextures(1, intArrayOf(cameraTextureId), 0)
cameraTextureId = 0
}
deleteFBOs()
}
// --- Private helpers ---
private fun drawQuad(
positionLoc: Int,
texCoordLoc: Int,
vertices: FloatBuffer,
texCoords: FloatBuffer
) {
GLES20.glEnableVertexAttribArray(positionLoc)
GLES20.glVertexAttribPointer(positionLoc, 2, GLES20.GL_FLOAT, false, 0, vertices)
GLES20.glEnableVertexAttribArray(texCoordLoc)
GLES20.glVertexAttribPointer(texCoordLoc, 2, GLES20.GL_FLOAT, false, 0, texCoords)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
GLES20.glDisableVertexAttribArray(positionLoc)
GLES20.glDisableVertexAttribArray(texCoordLoc)
}
/**
* Recomputes camera vertex positions to achieve crop-to-fill.
*
* The camera sensor is landscape; after the orientation-dependent texcoord
* rotation, the effective dimensions seen on screen are either swapped
* (portrait orientations) or kept (landscape orientations). We scale the
* vertex quad so the camera frame fills the surface without stretching —
* the GPU clips the overflow.
*/
private fun recomputeVertices() {
var scaleX = 1f
var scaleY = 1f
if (cameraWidth > 0 && cameraHeight > 0 && surfaceWidth > 0 && surfaceHeight > 0) {
val isPortrait = displayRotation == Surface.ROTATION_0 ||
displayRotation == Surface.ROTATION_180
val effectiveW = if (isPortrait) cameraHeight else cameraWidth
val effectiveH = if (isPortrait) cameraWidth else cameraHeight
val cameraRatio = effectiveW.toFloat() / effectiveH
val screenRatio = surfaceWidth.toFloat() / surfaceHeight
if (cameraRatio > screenRatio) {
scaleX = cameraRatio / screenRatio
} else {
scaleY = screenRatio / cameraRatio
}
}
cameraVertexBuffer.clear()
cameraVertexBuffer.put(floatArrayOf(
-scaleX, -scaleY,
scaleX, -scaleY,
-scaleX, scaleY,
scaleX, scaleY
))
cameraVertexBuffer.position(0)
}
private fun recreateFBOs(width: Int, height: Int) {
deleteFBOs()
// Create two color textures for ping-pong
val texIds = IntArray(2)
GLES20.glGenTextures(2, texIds, 0)
fboTexA = texIds[0]
fboTexB = texIds[1]
for (texId in texIds) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA,
width, height, 0,
GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null
)
}
// Create single FBO (we swap the attached texture for ping-pong)
val fbos = IntArray(1)
GLES20.glGenFramebuffers(1, fbos, 0)
fboId = fbos[0]
// Verify with texture A
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, fboId)
GLES20.glFramebufferTexture2D(
GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, fboTexA, 0
)
val status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER)
if (status != GLES20.GL_FRAMEBUFFER_COMPLETE) {
Log.e(TAG, "FBO incomplete: $status")
}
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0)
}
private fun deleteFBOs() {
if (fboId != 0) {
GLES20.glDeleteFramebuffers(1, intArrayOf(fboId), 0)
fboId = 0
}
if (fboTexA != 0 || fboTexB != 0) {
GLES20.glDeleteTextures(2, intArrayOf(fboTexA, fboTexB), 0)
fboTexA = 0
fboTexB = 0
}
}
private fun allocateFloatBuffer(floatCount: Int): FloatBuffer {
return ByteBuffer.allocateDirect(floatCount * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
}
}