Android: remote transcription via whisper.cpp server (Phase 2)

- new Transcriber interface makes the recorder service engine-agnostic; WhisperEngine (local JNI) and new RemoteWhisperEngine (POST /inference, verbose_json, in-memory WAV upload via new WavEncoder) implement it - SessionConfig carries serverUrl; when set, BOTH the live pass and the final pass transcribe remotely (server owns the model, e.g. large-v3 on GPU); local engine remains the offline fallback - UI: Transcribe on: phone/server dropdown + server URL field, persisted in SharedPreferences; model controls grey out in server mode; manual Transcribe also routes to the server - network security config permits cleartext HTTP for user-configured LAN/tailnet servers (documented; HTTPS works either way) - WavEncoder round-trip unit test (11 total green); validated phone -> Tailscale -> R9700 with large-v3 before the app-side change
2026-09-07 13:26:02 +02:00
parent a1df9d4037
commit 50d00286d7
10 changed files with 328 additions and 44 deletions
@@ -0,0 +1,38 @@
+package com.meetrec.core.recording
+
+/**
+ * Encodes mono float32 audio into an in-memory 16-bit PCM WAV — used to
+ * upload audio to a remote whisper.cpp server.
+ */
+object WavEncoder {
+
+    fun encodePcm16(
+        audio: FloatArray,
+        sampleRate: Int = Audio.TARGET_RATE,
+        channels: Int = 1,
+    ): ByteArray {
+        val data = ByteArray(audio.size * channels * 2)
+        for (i in audio.indices) {
+            val v = (audio[i].coerceIn(-1f, 1f) * 32767f).toInt()
+            data[2 * i] = (v and 0xFF).toByte()
+            data[2 * i + 1] = ((v shr 8) and 0xFF).toByte()
+        }
+        val header = le32(0x46464952) +                    // "RIFF"
+            le32(36 + data.size) +
+            le32(0x45564157) +                             // "WAVE"
+            le32(0x20746D66) + le32(16) + le16(1) +         // "fmt ", PCM
+            le16(channels) + le32(sampleRate) +
+            le32(sampleRate * channels * 2) + le16(channels * 2) + le16(16) +
+            le32(0x61746164) + le32(data.size)              // "data"
+        return header + data
+    }
+
+    private fun le32(v: Int) = byteArrayOf(
+        (v and 0xFF).toByte(), ((v shr 8) and 0xFF).toByte(),
+        ((v shr 16) and 0xFF).toByte(), ((v shr 24) and 0xFF).toByte(),
+    )
+
+    private fun le16(v: Int) = byteArrayOf(
+        (v and 0xFF).toByte(), ((v shr 8) and 0xFF).toByte(),
+    )
+}
@@ -73,6 +73,22 @@ class WavRoundTripTest {
        assertTrue(f.delete())
    }

+    @Test
+    fun `wav encoder output round trips through wav reader`() {
+        val rate = 16000
+        val x = sine(1.0, rate, 440, 0.5f)
+        val bytes = WavEncoder.encodePcm16(x, rate)
+        assertEquals(44 + 2 * x.size, bytes.size)
+        assertEquals("RIFF", String(bytes, 0, 4))
+        assertEquals("WAVE", String(bytes, 8, 4))
+        val back = WavReader.read(bytes.inputStream())
+        assertEquals(x.size, back.size)
+        for (i in x.indices) {
+            assertEquals(x[i], back[i], 1f / 16000f)   // int16 quantization
+        }
+        assertEquals(440.0, dominantFreq(back, rate), 15.0)
+    }
+
    @Test
    fun `resampler preserves length ratio and tone`() {
        for (from in intArrayOf(44100, 48000)) {
@@ -30,6 +30,7 @@ android {
 }

 dependencies {
+    implementation(project(":core:recording"))   // WavEncoder/Audio for uploads
    // TranscriptFiles is pure Kotlin and unit tested on the JVM.
    testImplementation(libs.junit)
 }
@@ -0,0 +1,120 @@
+package com.meetrec.core.whisper
+
+import com.meetrec.core.recording.Audio
+import com.meetrec.core.recording.WavEncoder
+import java.io.IOException
+import java.net.HttpURLConnection
+import java.net.URL
+import org.json.JSONObject
+
+/** Common engine surface so callers can be engine-agnostic. */
+interface Transcriber {
+    /**
+     * Transcribes 16 kHz mono float32 audio. [threads]/[beamSize] may be
+     * ignored by remote engines (the server owns its model and beam
+     * config). Blocking — call off the UI thread.
+     */
+    fun transcribe(
+        audio: FloatArray,
+        threads: Int,
+        beamSize: Int,
+        language: String?,
+    ): List<WhisperEngine.Segment>
+}
+
+/**
+ * Transcription via a remote whisper.cpp server (POST /inference), e.g.
+ * the Docker Compose stack in server/whisper-server/. The model lives on
+ * the server — beam size is a server-start setting in whisper.cpp
+ * v1.9.3 (no per-request override), so beamSize/threads are ignored.
+ */
+class RemoteWhisperEngine(baseUrl: String) : Transcriber {
+
+    private val base: String = baseUrl.trim().trimEnd('/')
+
+    init {
+        require(base.startsWith("http://") || base.startsWith("https://")) {
+            "server URL must start with http:// or https:// — got: $baseUrl"
+        }
+    }
+
+    override fun transcribe(
+        audio: FloatArray,
+        threads: Int,
+        beamSize: Int,
+        language: String?,
+    ): List<WhisperEngine.Segment> {
+        if (audio.isEmpty()) return emptyList()
+
+        val wav = WavEncoder.encodePcm16(audio, Audio.TARGET_RATE)
+        val body = multipart(
+            mapOf(
+                "response_format" to "verbose_json",
+                "language" to (language ?: "auto"),
+            ),
+            wav,
+        )
+
+        val conn = (URL("$base/inference").openConnection() as HttpURLConnection).apply {
+            requestMethod = "POST"
+            doOutput = true
+            connectTimeout = 15_000
+            readTimeout = 300_000
+            setFixedLengthStreamingMode(body.size)
+            setRequestProperty(
+                "Content-Type",
+                "multipart/form-data; boundary=$BOUNDARY",
+            )
+        }
+        try {
+            conn.outputStream.use { it.write(body) }
+            val code = conn.responseCode
+            val stream = if (code in 200..299) conn.inputStream else conn.errorStream
+            val resp = stream?.use { it.readBytes() } ?: ByteArray(0)
+            if (code !in 200..299) {
+                throw IOException(
+                    "server returned $code: " +
+                        String(resp, 0, minOf(resp.size, 200)),
+                )
+            }
+            return parseSegments(String(resp))
+        } catch (e: IOException) {
+            throw IOException("whisper server unreachable at $base: ${e.message}", e)
+        } finally {
+            conn.disconnect()
+        }
+    }
+
+    private fun multipart(fields: Map<String, String>, wav: ByteArray): ByteArray {
+        val head = fields.entries.joinToString("") { (k, v) ->
+            "--$BOUNDARY\r\n" +
+                "Content-Disposition: form-data; name=\"$k\"\r\n\r\n$v\r\n"
+        }.toByteArray() +
+            ("--$BOUNDARY\r\n" +
+                "Content-Disposition: form-data; name=\"file\"; " +
+                "filename=\"audio.wav\"\r\n" +
+                "Content-Type: audio/wav\r\n\r\n").toByteArray()
+        val tail = "\r\n--$BOUNDARY--\r\n".toByteArray()
+        return head + wav + tail
+    }
+
+    private fun parseSegments(json: String): List<WhisperEngine.Segment> {
+        val segments = JSONObject(json).optJSONArray("segments") ?: return emptyList()
+        return (0 until segments.length()).mapNotNull { i ->
+            val o = segments.getJSONObject(i)
+            val text = o.optString("text", "").trim()
+            if (text.isEmpty()) null
+            else {
+                WhisperEngine.Segment(
+                    startMs = (o.optDouble("start", 0.0) * 1000).toLong(),
+                    endMs = (o.optDouble("end", 0.0) * 1000).toLong(),
+                    text = text,
+                )
+            }
+        }
+    }
+
+    private companion object {
+        const val BOUNDARY = "meetrec-android-7c1a9b8f"
+    }
+}
@@ -15,7 +15,7 @@ class WhisperEngine private constructor(
    val modelPath: String,
    /** Model role name ("tiny", "base", …) derived from the GGML file. */
    val modelName: String,
-) : AutoCloseable {
+) : Transcriber, AutoCloseable {

    data class Segment(val startMs: Long, val endMs: Long, val text: String)

@@ -31,11 +31,11 @@ class WhisperEngine private constructor(
     * [language] is an ISO code like "en"/"de", or null to autodetect.
     */
    @Synchronized
-    fun transcribe(
+    override fun transcribe(
        audio: FloatArray,
-        threads: Int = DEFAULT_THREADS,
-        beamSize: Int = 5,
-        language: String? = null,
+        threads: Int,
+        beamSize: Int,
+        language: String?,
    ): List<Segment> {
        check(!closed) { "engine is closed" }
        check(ptr != 0L) { "engine failed to load" }