fegger/meetrec
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Android: remote transcription via whisper.cpp server (Phase 2)

Browse Source 
- new Transcriber interface makes the recorder service engine-agnostic;
  WhisperEngine (local JNI) and new RemoteWhisperEngine (POST /inference,
  verbose_json, in-memory WAV upload via new WavEncoder) implement it
- SessionConfig carries serverUrl; when set, BOTH the live pass and the
  final pass transcribe remotely (server owns the model, e.g. large-v3
  on GPU); local engine remains the offline fallback
- UI: Transcribe on: phone/server dropdown + server URL field,
  persisted in SharedPreferences; model controls grey out in server
  mode; manual Transcribe also routes to the server
- network security config permits cleartext HTTP for user-configured
  LAN/tailnet servers (documented; HTTPS works either way)
- WavEncoder round-trip unit test (11 total green); validated phone ->
  Tailscale -> R9700 with large-v3 before the app-side change
This commit is contained in:
Florian Egger
2026-09-07 13:26:02 +02:00
parent a1df9d4037
commit 50d00286d7
10 changed files with 328 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android/app/src/main/AndroidManifest.xml
+1
View File
@@ -12,6 +12,7 @@
<application
android:icon="@mipmap/ic_launcher"
android:label="MeetRec"
android:networkSecurityConfig="@xml/network_security_config"
android:theme="@android:style/Theme.Material.Light.NoActionBar"
android:allowBackup="true">
android/app/src/main/java/com/meetrec/android/MainActivity.kt
+91 -12
View File
@@ -1,6 +1,7 @@
package com.meetrec.android
import android.Manifest
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.os.Build
@@ -42,6 +43,8 @@ import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.unit.dp
import com.meetrec.core.recording.WavReader
import com.meetrec.core.whisper.RemoteWhisperEngine
import com.meetrec.core.whisper.Transcriber
import com.meetrec.core.whisper.WhisperEngine
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
@@ -66,6 +69,25 @@ fun MeetRecScreen() {
val context = LocalContext.current
val scope = rememberCoroutineScope()
val prefs = remember { context.getSharedPreferences("meetrec", Context.MODE_PRIVATE) }
var transcribeOn by remember {
mutableStateOf(prefs.getString("transcribe_on", "phone") ?: "phone")
}
var transcribeMenu by remember { mutableStateOf(false) }
var serverUrl by remember {
mutableStateOf(
prefs.getString("server_url", "http://100.103.83.12:8085")
?: "http://100.103.83.12:8085",
)
}
LaunchedEffect(transcribeOn) {
prefs.edit().putString("transcribe_on", transcribeOn).apply()
}
LaunchedEffect(serverUrl) {
prefs.edit().putString("server_url", serverUrl).apply()
}
var model by remember { mutableStateOf("tiny") }
var modelMenu by remember { mutableStateOf(false) }
var language by remember { mutableStateOf("auto") }
@@ -163,11 +185,22 @@ fun MeetRecScreen() {
}
if (missing.isEmpty()) {
RecorderService.session = SessionConfig(
finalEngine = engine,
finalEngine = if (transcribeOn == "phone") engine else null,
serverUrl = if (transcribeOn == "server") {
serverUrl.trim().ifBlank { null }
} else {
null
},
language = language.takeIf { it != "auto" },
liveModel = liveModel.takeIf { it != "off" },
liveModel = if (transcribeOn == "phone") {
liveModel.takeIf { it != "off" }
} else {
null
},
)
if (engine == null) {
if (transcribeOn == "server") {
status = "Recording — live and final pass on ${serverUrl.trim()}"
} else if (engine == null) {
status = "Recording without transcription — load an engine first " +
"for the automatic final pass"
}
@@ -213,6 +246,42 @@ fun MeetRecScreen() {
) {
Text("MeetRec", style = MaterialTheme.typography.headlineSmall)
// transcription target: the phone's local engines or a remote server
ExposedDropdownMenuBox(
expanded = transcribeMenu,
onExpandedChange = { transcribeMenu = it },
modifier = Modifier.fillMaxWidth(0.6f),
) {
OutlinedTextField(
value = transcribeOn,
onValueChange = {},
readOnly = true,
label = { Text("Transcribe on") },
trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(transcribeMenu) },
modifier = Modifier.menuAnchor().fillMaxWidth(),
)
ExposedDropdownMenu(
expanded = transcribeMenu,
onDismissRequest = { transcribeMenu = false },
) {
listOf("phone", "server").forEach { choice ->
DropdownMenuItem(
text = { Text(choice) },
onClick = { transcribeOn = choice; transcribeMenu = false },
)
}
}
}
if (transcribeOn == "server") {
OutlinedTextField(
value = serverUrl,
onValueChange = { serverUrl = it },
label = { Text("Server URL") },
placeholder = { Text("http://100.103.83.12:8085") },
modifier = Modifier.fillMaxWidth(),
)
}
// model + language
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
ExposedDropdownMenuBox(
@@ -224,7 +293,8 @@ fun MeetRecScreen() {
value = model,
onValueChange = {},
readOnly = true,
label = { Text("Model") },
enabled = transcribeOn == "phone",
label = { Text(if (transcribeOn == "server") "Model (n/a)" else "Model") },
trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(modelMenu) },
modifier = Modifier.menuAnchor().fillMaxWidth(),
)
@@ -261,7 +331,7 @@ fun MeetRecScreen() {
}
}
// live model
// live model (local engine only; the server owns its own models)
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
ExposedDropdownMenuBox(
expanded = liveMenu,
@@ -272,7 +342,10 @@ fun MeetRecScreen() {
value = liveModel,
onValueChange = {},
readOnly = true,
label = { Text("Live model") },
enabled = transcribeOn == "phone",
label = {
Text(if (transcribeOn == "server") "Live model (n/a)" else "Live model")
},
trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(liveMenu) },
modifier = Modifier.menuAnchor().fillMaxWidth(),
)
@@ -299,7 +372,7 @@ fun MeetRecScreen() {
status = "Model $model ready at ${modelFile.name}"
}
},
enabled = !busy && !modelFile.isFile,
enabled = !busy && !modelFile.isFile && transcribeOn == "phone",
) { Text("Download") }
Button(
onClick = {
@@ -311,7 +384,7 @@ fun MeetRecScreen() {
status = "$model loaded (${WhisperEngine.DEFAULT_THREADS} threads)"
}
},
enabled = !busy && modelFile.isFile,
enabled = !busy && modelFile.isFile && transcribeOn == "phone",
) { Text("Load engine") }
}
@@ -352,13 +425,18 @@ fun MeetRecScreen() {
Button(
onClick = {
val samples = wavSamples
val eng = engine
if (samples != null && eng != null) {
if (samples != null && (engine != null || transcribeOn == "server")) {
runBusy {
val t: Transcriber = if (transcribeOn == "server") {
RemoteWhisperEngine(serverUrl.trim())
} else {
engine!!
}
val start = System.currentTimeMillis()
val segs = withContext(Dispatchers.Default) {
eng.transcribe(
t.transcribe(
samples,
WhisperEngine.DEFAULT_THREADS,
beamSize = 5,
language = language.takeIf { it != "auto" },
)
@@ -370,7 +448,8 @@ fun MeetRecScreen() {
}
}
},
enabled = !busy && engine != null && wavSamples != null,
enabled = !busy && wavSamples != null &&
(engine != null || transcribeOn == "server"),
) { Text("Transcribe") }
}
android/app/src/main/java/com/meetrec/android/RecorderService.kt
+23 -12
View File
@@ -15,6 +15,8 @@ import android.util.Log
import androidx.core.app.NotificationCompat
import com.meetrec.core.recording.MeetingRecorder
import com.meetrec.core.recording.WavReader
import com.meetrec.core.whisper.RemoteWhisperEngine
import com.meetrec.core.whisper.Transcriber
import com.meetrec.core.whisper.TranscriptFiles
import com.meetrec.core.whisper.WhisperEngine
import java.io.File
@@ -37,11 +39,13 @@ data class LiveLine(val startMs: Long, val text: String)
/**
* Engine/session for the next recording; set by the UI right before
* starting the service. finalEngine may be null (record only);
* liveModel may be null (no live transcript).
* starting the service. When [serverUrl] is set, BOTH passes transcribe
* remotely (model lives on the server); otherwise [finalEngine] and the
* local [liveModel] are used. All engines may be null — record only.
*/
data class SessionConfig(
val finalEngine: WhisperEngine?,
val serverUrl: String?,
val language: String?,
val liveModel: String?,
val liveIntervalSec: Int = 8,
@@ -148,19 +152,23 @@ class RecorderService : Service() {
rec.start()
val cfg = session
if (cfg?.liveModel != null) {
if (cfg?.liveModel != null || cfg?.serverUrl != null) {
liveJob = scope.launch { liveLoop(rec, cfg) }
}
}
/**
* Rolling-window live transcription (desktop live_worker, Android
* edition). Loads the live model off the main thread, reuses the
* final engine when both roles use the same model.
* edition): the server engine when configured, otherwise the local
* live model (loaded off the main thread, reusing the final engine
* when both roles use the same model).
*/
private suspend fun liveLoop(rec: MeetingRecorder, cfg: SessionConfig) {
val engine: WhisperEngine = try {
if (cfg.liveModel == cfg.finalEngine?.modelName) {
val transcriber: Transcriber = try {
if (cfg.serverUrl != null) {
Log.i(TAG, "live: remote engine at ${cfg.serverUrl}")
RemoteWhisperEngine(cfg.serverUrl)
} else if (cfg.liveModel == cfg.finalEngine?.modelName) {
Log.i(TAG, "live: reusing final engine (${cfg.liveModel})")
cfg.finalEngine!!
} else {
@@ -190,8 +198,9 @@ class RecorderService : Service() {
val snap = rec.snapshot() ?: continue // < 2 s captured so far
try {
val t0 = SystemClock.elapsedRealtime()
val segs = engine.transcribe(
val segs = transcriber.transcribe(
snap.audio,
WhisperEngine.DEFAULT_THREADS,
beamSize = 1,
language = cfg.language,
)
@@ -248,8 +257,9 @@ class RecorderService : Service() {
liveJob = null
val cfg = session
val engine = cfg?.finalEngine
if (engine == null) {
val transcriber: Transcriber? =
cfg?.serverUrl?.let { RemoteWhisperEngine(it) } ?: cfg?.finalEngine
if (transcriber == null) {
_state.value = RecordingState.Finished(file, durationMs, sampleRate)
cleanup()
return
@@ -262,10 +272,11 @@ class RecorderService : Service() {
val samples = withContext(Dispatchers.IO) {
file.inputStream().use { WavReader.read(it) }
}
val segments = engine.transcribe(
val segments = transcriber.transcribe(
samples,
WhisperEngine.DEFAULT_THREADS,
beamSize = 5,
language = cfg.language,
language = cfg?.language,
)
Log.i(TAG, "final: ${segments.size} segments in " +
"${SystemClock.elapsedRealtime() - t0} ms")
android/app/src/main/res/xml/network_security_config.xml
+11
View File
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
<!--
The transcription server URL is user-configured and is typically a
plain-HTTP host on a private tailnet/LAN (e.g. a whisper.cpp
server behind Tailscale). HTTPS URLs work regardless of this
setting. If you ever point the app at a server outside a trusted
network, put TLS in front of it instead.
-->
<base-config cleartextTrafficPermitted="true" />
</network-security-config>