From 50d00286d7518b42d992f13ee9f9933dfe168c57 Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Mon, 7 Sep 2026 13:26:02 +0200 Subject: [PATCH] Android: remote transcription via whisper.cpp server (Phase 2) - new Transcriber interface makes the recorder service engine-agnostic; WhisperEngine (local JNI) and new RemoteWhisperEngine (POST /inference, verbose_json, in-memory WAV upload via new WavEncoder) implement it - SessionConfig carries serverUrl; when set, BOTH the live pass and the final pass transcribe remotely (server owns the model, e.g. large-v3 on GPU); local engine remains the offline fallback - UI: Transcribe on: phone/server dropdown + server URL field, persisted in SharedPreferences; model controls grey out in server mode; manual Transcribe also routes to the server - network security config permits cleartext HTTP for user-configured LAN/tailnet servers (documented; HTTPS works either way) - WavEncoder round-trip unit test (11 total green); validated phone -> Tailscale -> R9700 with large-v3 before the app-side change --- android/README.md | 37 +++--- android/app/src/main/AndroidManifest.xml | 1 + .../java/com/meetrec/android/MainActivity.kt | 103 +++++++++++++-- .../com/meetrec/android/RecorderService.kt | 35 +++-- .../main/res/xml/network_security_config.xml | 11 ++ .../com/meetrec/core/recording/WavEncoder.kt | 38 ++++++ .../core/recording/WavRoundTripTest.kt | 16 +++ android/core/whisper/build.gradle.kts | 1 + .../core/whisper/RemoteWhisperEngine.kt | 120 ++++++++++++++++++ .../com/meetrec/core/whisper/WhisperEngine.kt | 10 +- 10 files changed, 328 insertions(+), 44 deletions(-) create mode 100644 android/app/src/main/res/xml/network_security_config.xml create mode 100644 android/core/recording/src/main/java/com/meetrec/core/recording/WavEncoder.kt create mode 100644 android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt diff --git a/android/README.md b/android/README.md index a9e43cb..6c0fb58 100644 --- a/android/README.md +++ b/android/README.md @@ -4,10 +4,22 @@ Native Android app with the same functionality as the desktop meetrec: record meetings and transcribe them on-device with Whisper. Nothing leaves the phone — no cloud, no telemetry. -Status: **M2 (live transcript + automatic final pass)** — record a meeting -and watch the live rolling transcript while recording; on Stop the final -(better) pass runs automatically and saves `.txt`/`.srt`/`.json` next to -the WAV, mirroring the desktop two-pass design. +Status: **M2.5 (remote transcription)** — record on the phone while a +remote whisper.cpp server (GPU) provides live and final transcripts. +The phone's local whisper.cpp engine remains as the offline fallback. + +## Transcription targets + +**phone** — the app's local whisper.cpp engine (JNI/NEON): pick a model +(tiny/base for live, e.g. small for the final pass). Measured on the +Fairphone 6: ~0.6–0.8× realtime, so keep local models small. + +**server** — a remote [whisper.cpp server](../server/whisper-server/) +(Docker Compose, Vulkan GPU) transcribes BOTH the live pass and the final +pass; the model lives on the server (e.g. large-v3). The phone only +records and displays. Set the server URL in the app (persisted; e.g. +`http://100.103.83.12:8085` over Tailscale — the phone needs Tailscale +too). Local model selection is greyed out in this mode. ## Requirements @@ -30,20 +42,15 @@ release tag so the JNI layer never breaks on upstream churn. ## Test on device -1. Launch **MeetRec**, pick a model (`tiny` is fine for a first test) and tap - **Download** (model comes from Hugging Face; tiny is ~75 MB). -2. Tap **Load engine**. -3. Tap **Record** — live transcript lines appear every few seconds (they use - the cheap *live model*, `tiny`/`base`, on a 60 s rolling window). -4. Tap **Stop** — the final pass runs automatically with the selected model - and writes `.txt/.srt/.json` next to the WAV. +1. Pick **Transcribe on: phone or server**. For server: enter the URL + (needs Tailscale on the phone for a Tailscale-only server). +2. Phone mode: Download + Load engine first (tiny is fine for a start). +3. Tap **Record** — live transcript lines appear every few seconds. +4. Tap **Stop** — the final pass runs automatically and writes + `.txt/.srt/.json` next to the WAV. 5. Share the transcript via the share sheet. You can also pick any PCM WAV file and transcribe it manually. -Tip: use `tiny`/`base` as the live model (they keep up on a phone CPU) and -a larger model like `small` for the final pass. The **Live model** dropdown -also has `off` to disable live transcription (longest battery life). - Expect roughly realtime transcription with `tiny`/`base` on the Fairphone 6's CPU; `small` is noticeably slower — use it for final passes only (the live/final split comes with the recorder milestones). diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index ac61d9c..8a2b8e2 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -12,6 +12,7 @@ diff --git a/android/app/src/main/java/com/meetrec/android/MainActivity.kt b/android/app/src/main/java/com/meetrec/android/MainActivity.kt index 9f87044..2f793b5 100644 --- a/android/app/src/main/java/com/meetrec/android/MainActivity.kt +++ b/android/app/src/main/java/com/meetrec/android/MainActivity.kt @@ -1,6 +1,7 @@ package com.meetrec.android import android.Manifest +import android.content.Context import android.content.Intent import android.content.pm.PackageManager import android.os.Build @@ -42,6 +43,8 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.unit.dp import com.meetrec.core.recording.WavReader +import com.meetrec.core.whisper.RemoteWhisperEngine +import com.meetrec.core.whisper.Transcriber import com.meetrec.core.whisper.WhisperEngine import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -66,6 +69,25 @@ fun MeetRecScreen() { val context = LocalContext.current val scope = rememberCoroutineScope() + val prefs = remember { context.getSharedPreferences("meetrec", Context.MODE_PRIVATE) } + var transcribeOn by remember { + mutableStateOf(prefs.getString("transcribe_on", "phone") ?: "phone") + } + var transcribeMenu by remember { mutableStateOf(false) } + var serverUrl by remember { + mutableStateOf( + prefs.getString("server_url", "http://100.103.83.12:8085") + ?: "http://100.103.83.12:8085", + ) + } + + LaunchedEffect(transcribeOn) { + prefs.edit().putString("transcribe_on", transcribeOn).apply() + } + LaunchedEffect(serverUrl) { + prefs.edit().putString("server_url", serverUrl).apply() + } + var model by remember { mutableStateOf("tiny") } var modelMenu by remember { mutableStateOf(false) } var language by remember { mutableStateOf("auto") } @@ -163,11 +185,22 @@ fun MeetRecScreen() { } if (missing.isEmpty()) { RecorderService.session = SessionConfig( - finalEngine = engine, + finalEngine = if (transcribeOn == "phone") engine else null, + serverUrl = if (transcribeOn == "server") { + serverUrl.trim().ifBlank { null } + } else { + null + }, language = language.takeIf { it != "auto" }, - liveModel = liveModel.takeIf { it != "off" }, + liveModel = if (transcribeOn == "phone") { + liveModel.takeIf { it != "off" } + } else { + null + }, ) - if (engine == null) { + if (transcribeOn == "server") { + status = "Recording — live and final pass on ${serverUrl.trim()}" + } else if (engine == null) { status = "Recording without transcription — load an engine first " + "for the automatic final pass" } @@ -213,6 +246,42 @@ fun MeetRecScreen() { ) { Text("MeetRec", style = MaterialTheme.typography.headlineSmall) + // transcription target: the phone's local engines or a remote server + ExposedDropdownMenuBox( + expanded = transcribeMenu, + onExpandedChange = { transcribeMenu = it }, + modifier = Modifier.fillMaxWidth(0.6f), + ) { + OutlinedTextField( + value = transcribeOn, + onValueChange = {}, + readOnly = true, + label = { Text("Transcribe on") }, + trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(transcribeMenu) }, + modifier = Modifier.menuAnchor().fillMaxWidth(), + ) + ExposedDropdownMenu( + expanded = transcribeMenu, + onDismissRequest = { transcribeMenu = false }, + ) { + listOf("phone", "server").forEach { choice -> + DropdownMenuItem( + text = { Text(choice) }, + onClick = { transcribeOn = choice; transcribeMenu = false }, + ) + } + } + } + if (transcribeOn == "server") { + OutlinedTextField( + value = serverUrl, + onValueChange = { serverUrl = it }, + label = { Text("Server URL") }, + placeholder = { Text("http://100.103.83.12:8085") }, + modifier = Modifier.fillMaxWidth(), + ) + } + // model + language Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { ExposedDropdownMenuBox( @@ -224,7 +293,8 @@ fun MeetRecScreen() { value = model, onValueChange = {}, readOnly = true, - label = { Text("Model") }, + enabled = transcribeOn == "phone", + label = { Text(if (transcribeOn == "server") "Model (n/a)" else "Model") }, trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(modelMenu) }, modifier = Modifier.menuAnchor().fillMaxWidth(), ) @@ -261,7 +331,7 @@ fun MeetRecScreen() { } } - // live model + // live model (local engine only; the server owns its own models) Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { ExposedDropdownMenuBox( expanded = liveMenu, @@ -272,7 +342,10 @@ fun MeetRecScreen() { value = liveModel, onValueChange = {}, readOnly = true, - label = { Text("Live model") }, + enabled = transcribeOn == "phone", + label = { + Text(if (transcribeOn == "server") "Live model (n/a)" else "Live model") + }, trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(liveMenu) }, modifier = Modifier.menuAnchor().fillMaxWidth(), ) @@ -299,7 +372,7 @@ fun MeetRecScreen() { status = "Model $model ready at ${modelFile.name}" } }, - enabled = !busy && !modelFile.isFile, + enabled = !busy && !modelFile.isFile && transcribeOn == "phone", ) { Text("Download") } Button( onClick = { @@ -311,7 +384,7 @@ fun MeetRecScreen() { status = "$model loaded (${WhisperEngine.DEFAULT_THREADS} threads)" } }, - enabled = !busy && modelFile.isFile, + enabled = !busy && modelFile.isFile && transcribeOn == "phone", ) { Text("Load engine") } } @@ -352,13 +425,18 @@ fun MeetRecScreen() { Button( onClick = { val samples = wavSamples - val eng = engine - if (samples != null && eng != null) { + if (samples != null && (engine != null || transcribeOn == "server")) { runBusy { + val t: Transcriber = if (transcribeOn == "server") { + RemoteWhisperEngine(serverUrl.trim()) + } else { + engine!! + } val start = System.currentTimeMillis() val segs = withContext(Dispatchers.Default) { - eng.transcribe( + t.transcribe( samples, + WhisperEngine.DEFAULT_THREADS, beamSize = 5, language = language.takeIf { it != "auto" }, ) @@ -370,7 +448,8 @@ fun MeetRecScreen() { } } }, - enabled = !busy && engine != null && wavSamples != null, + enabled = !busy && wavSamples != null && + (engine != null || transcribeOn == "server"), ) { Text("Transcribe") } } diff --git a/android/app/src/main/java/com/meetrec/android/RecorderService.kt b/android/app/src/main/java/com/meetrec/android/RecorderService.kt index d6d76fc..5ee0b2f 100644 --- a/android/app/src/main/java/com/meetrec/android/RecorderService.kt +++ b/android/app/src/main/java/com/meetrec/android/RecorderService.kt @@ -15,6 +15,8 @@ import android.util.Log import androidx.core.app.NotificationCompat import com.meetrec.core.recording.MeetingRecorder import com.meetrec.core.recording.WavReader +import com.meetrec.core.whisper.RemoteWhisperEngine +import com.meetrec.core.whisper.Transcriber import com.meetrec.core.whisper.TranscriptFiles import com.meetrec.core.whisper.WhisperEngine import java.io.File @@ -37,11 +39,13 @@ data class LiveLine(val startMs: Long, val text: String) /** * Engine/session for the next recording; set by the UI right before - * starting the service. finalEngine may be null (record only); - * liveModel may be null (no live transcript). + * starting the service. When [serverUrl] is set, BOTH passes transcribe + * remotely (model lives on the server); otherwise [finalEngine] and the + * local [liveModel] are used. All engines may be null — record only. */ data class SessionConfig( val finalEngine: WhisperEngine?, + val serverUrl: String?, val language: String?, val liveModel: String?, val liveIntervalSec: Int = 8, @@ -148,19 +152,23 @@ class RecorderService : Service() { rec.start() val cfg = session - if (cfg?.liveModel != null) { + if (cfg?.liveModel != null || cfg?.serverUrl != null) { liveJob = scope.launch { liveLoop(rec, cfg) } } } /** * Rolling-window live transcription (desktop live_worker, Android - * edition). Loads the live model off the main thread, reuses the - * final engine when both roles use the same model. + * edition): the server engine when configured, otherwise the local + * live model (loaded off the main thread, reusing the final engine + * when both roles use the same model). */ private suspend fun liveLoop(rec: MeetingRecorder, cfg: SessionConfig) { - val engine: WhisperEngine = try { - if (cfg.liveModel == cfg.finalEngine?.modelName) { + val transcriber: Transcriber = try { + if (cfg.serverUrl != null) { + Log.i(TAG, "live: remote engine at ${cfg.serverUrl}") + RemoteWhisperEngine(cfg.serverUrl) + } else if (cfg.liveModel == cfg.finalEngine?.modelName) { Log.i(TAG, "live: reusing final engine (${cfg.liveModel})") cfg.finalEngine!! } else { @@ -190,8 +198,9 @@ class RecorderService : Service() { val snap = rec.snapshot() ?: continue // < 2 s captured so far try { val t0 = SystemClock.elapsedRealtime() - val segs = engine.transcribe( + val segs = transcriber.transcribe( snap.audio, + WhisperEngine.DEFAULT_THREADS, beamSize = 1, language = cfg.language, ) @@ -248,8 +257,9 @@ class RecorderService : Service() { liveJob = null val cfg = session - val engine = cfg?.finalEngine - if (engine == null) { + val transcriber: Transcriber? = + cfg?.serverUrl?.let { RemoteWhisperEngine(it) } ?: cfg?.finalEngine + if (transcriber == null) { _state.value = RecordingState.Finished(file, durationMs, sampleRate) cleanup() return @@ -262,10 +272,11 @@ class RecorderService : Service() { val samples = withContext(Dispatchers.IO) { file.inputStream().use { WavReader.read(it) } } - val segments = engine.transcribe( + val segments = transcriber.transcribe( samples, + WhisperEngine.DEFAULT_THREADS, beamSize = 5, - language = cfg.language, + language = cfg?.language, ) Log.i(TAG, "final: ${segments.size} segments in " + "${SystemClock.elapsedRealtime() - t0} ms") diff --git a/android/app/src/main/res/xml/network_security_config.xml b/android/app/src/main/res/xml/network_security_config.xml new file mode 100644 index 0000000..610e186 --- /dev/null +++ b/android/app/src/main/res/xml/network_security_config.xml @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/android/core/recording/src/main/java/com/meetrec/core/recording/WavEncoder.kt b/android/core/recording/src/main/java/com/meetrec/core/recording/WavEncoder.kt new file mode 100644 index 0000000..4201f68 --- /dev/null +++ b/android/core/recording/src/main/java/com/meetrec/core/recording/WavEncoder.kt @@ -0,0 +1,38 @@ +package com.meetrec.core.recording + +/** + * Encodes mono float32 audio into an in-memory 16-bit PCM WAV — used to + * upload audio to a remote whisper.cpp server. + */ +object WavEncoder { + + fun encodePcm16( + audio: FloatArray, + sampleRate: Int = Audio.TARGET_RATE, + channels: Int = 1, + ): ByteArray { + val data = ByteArray(audio.size * channels * 2) + for (i in audio.indices) { + val v = (audio[i].coerceIn(-1f, 1f) * 32767f).toInt() + data[2 * i] = (v and 0xFF).toByte() + data[2 * i + 1] = ((v shr 8) and 0xFF).toByte() + } + val header = le32(0x46464952) + // "RIFF" + le32(36 + data.size) + + le32(0x45564157) + // "WAVE" + le32(0x20746D66) + le32(16) + le16(1) + // "fmt ", PCM + le16(channels) + le32(sampleRate) + + le32(sampleRate * channels * 2) + le16(channels * 2) + le16(16) + + le32(0x61746164) + le32(data.size) // "data" + return header + data + } + + private fun le32(v: Int) = byteArrayOf( + (v and 0xFF).toByte(), ((v shr 8) and 0xFF).toByte(), + ((v shr 16) and 0xFF).toByte(), ((v shr 24) and 0xFF).toByte(), + ) + + private fun le16(v: Int) = byteArrayOf( + (v and 0xFF).toByte(), ((v shr 8) and 0xFF).toByte(), + ) +} \ No newline at end of file diff --git a/android/core/recording/src/test/java/com/meetrec/core/recording/WavRoundTripTest.kt b/android/core/recording/src/test/java/com/meetrec/core/recording/WavRoundTripTest.kt index e74e3cc..cad0063 100644 --- a/android/core/recording/src/test/java/com/meetrec/core/recording/WavRoundTripTest.kt +++ b/android/core/recording/src/test/java/com/meetrec/core/recording/WavRoundTripTest.kt @@ -73,6 +73,22 @@ class WavRoundTripTest { assertTrue(f.delete()) } + @Test + fun `wav encoder output round trips through wav reader`() { + val rate = 16000 + val x = sine(1.0, rate, 440, 0.5f) + val bytes = WavEncoder.encodePcm16(x, rate) + assertEquals(44 + 2 * x.size, bytes.size) + assertEquals("RIFF", String(bytes, 0, 4)) + assertEquals("WAVE", String(bytes, 8, 4)) + val back = WavReader.read(bytes.inputStream()) + assertEquals(x.size, back.size) + for (i in x.indices) { + assertEquals(x[i], back[i], 1f / 16000f) // int16 quantization + } + assertEquals(440.0, dominantFreq(back, rate), 15.0) + } + @Test fun `resampler preserves length ratio and tone`() { for (from in intArrayOf(44100, 48000)) { diff --git a/android/core/whisper/build.gradle.kts b/android/core/whisper/build.gradle.kts index 6bd1bb8..81bfb94 100644 --- a/android/core/whisper/build.gradle.kts +++ b/android/core/whisper/build.gradle.kts @@ -30,6 +30,7 @@ android { } dependencies { + implementation(project(":core:recording")) // WavEncoder/Audio for uploads // TranscriptFiles is pure Kotlin and unit tested on the JVM. testImplementation(libs.junit) } \ No newline at end of file diff --git a/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt b/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt new file mode 100644 index 0000000..62e48ab --- /dev/null +++ b/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt @@ -0,0 +1,120 @@ +package com.meetrec.core.whisper + +import com.meetrec.core.recording.Audio +import com.meetrec.core.recording.WavEncoder +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import org.json.JSONObject + +/** Common engine surface so callers can be engine-agnostic. */ +interface Transcriber { + /** + * Transcribes 16 kHz mono float32 audio. [threads]/[beamSize] may be + * ignored by remote engines (the server owns its model and beam + * config). Blocking — call off the UI thread. + */ + fun transcribe( + audio: FloatArray, + threads: Int, + beamSize: Int, + language: String?, + ): List +} + +/** + * Transcription via a remote whisper.cpp server (POST /inference), e.g. + * the Docker Compose stack in server/whisper-server/. The model lives on + * the server — beam size is a server-start setting in whisper.cpp + * v1.9.3 (no per-request override), so beamSize/threads are ignored. + */ +class RemoteWhisperEngine(baseUrl: String) : Transcriber { + + private val base: String = baseUrl.trim().trimEnd('/') + + init { + require(base.startsWith("http://") || base.startsWith("https://")) { + "server URL must start with http:// or https:// — got: $baseUrl" + } + } + + override fun transcribe( + audio: FloatArray, + threads: Int, + beamSize: Int, + language: String?, + ): List { + if (audio.isEmpty()) return emptyList() + + val wav = WavEncoder.encodePcm16(audio, Audio.TARGET_RATE) + val body = multipart( + mapOf( + "response_format" to "verbose_json", + "language" to (language ?: "auto"), + ), + wav, + ) + + val conn = (URL("$base/inference").openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + doOutput = true + connectTimeout = 15_000 + readTimeout = 300_000 + setFixedLengthStreamingMode(body.size) + setRequestProperty( + "Content-Type", + "multipart/form-data; boundary=$BOUNDARY", + ) + } + try { + conn.outputStream.use { it.write(body) } + val code = conn.responseCode + val stream = if (code in 200..299) conn.inputStream else conn.errorStream + val resp = stream?.use { it.readBytes() } ?: ByteArray(0) + if (code !in 200..299) { + throw IOException( + "server returned $code: " + + String(resp, 0, minOf(resp.size, 200)), + ) + } + return parseSegments(String(resp)) + } catch (e: IOException) { + throw IOException("whisper server unreachable at $base: ${e.message}", e) + } finally { + conn.disconnect() + } + } + + private fun multipart(fields: Map, wav: ByteArray): ByteArray { + val head = fields.entries.joinToString("") { (k, v) -> + "--$BOUNDARY\r\n" + + "Content-Disposition: form-data; name=\"$k\"\r\n\r\n$v\r\n" + }.toByteArray() + + ("--$BOUNDARY\r\n" + + "Content-Disposition: form-data; name=\"file\"; " + + "filename=\"audio.wav\"\r\n" + + "Content-Type: audio/wav\r\n\r\n").toByteArray() + val tail = "\r\n--$BOUNDARY--\r\n".toByteArray() + return head + wav + tail + } + + private fun parseSegments(json: String): List { + val segments = JSONObject(json).optJSONArray("segments") ?: return emptyList() + return (0 until segments.length()).mapNotNull { i -> + val o = segments.getJSONObject(i) + val text = o.optString("text", "").trim() + if (text.isEmpty()) null + else { + WhisperEngine.Segment( + startMs = (o.optDouble("start", 0.0) * 1000).toLong(), + endMs = (o.optDouble("end", 0.0) * 1000).toLong(), + text = text, + ) + } + } + } + + private companion object { + const val BOUNDARY = "meetrec-android-7c1a9b8f" + } +} \ No newline at end of file diff --git a/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt b/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt index 4b1f9d0..10ac9d3 100644 --- a/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt +++ b/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt @@ -15,7 +15,7 @@ class WhisperEngine private constructor( val modelPath: String, /** Model role name ("tiny", "base", …) derived from the GGML file. */ val modelName: String, -) : AutoCloseable { +) : Transcriber, AutoCloseable { data class Segment(val startMs: Long, val endMs: Long, val text: String) @@ -31,11 +31,11 @@ class WhisperEngine private constructor( * [language] is an ISO code like "en"/"de", or null to autodetect. */ @Synchronized - fun transcribe( + override fun transcribe( audio: FloatArray, - threads: Int = DEFAULT_THREADS, - beamSize: Int = 5, - language: String? = null, + threads: Int, + beamSize: Int, + language: String?, ): List { check(!closed) { "engine is closed" } check(ptr != 0L) { "engine failed to load" }