Android M2: live rolling transcript and automatic final pass
- RecorderService: rolling-window live pass with a dedicated live model (off/tiny/base, beam 1, 8 s ticks, time-based dedupe) and an automatic final pass with the selected model (beam 5) writing txt/srt/json - TranscriptFiles in core/whisper: desktop-compatible outputs, 5 JVM tests - WhisperEngine exposes modelName; UI: live-model dropdown, merged transcript view, share sheet; falls back to manual path without engine - live loop failures now logged (MeetRec tag) and surfaced in the UI (was silently swallowed), plus final-pass timing logs On-device measurements (Fairphone 6): tiny live ~0.6x realtime, small final ~0.8x realtime — motivates the planned whisper-server engine.
This commit is contained in:
@@ -49,8 +49,9 @@ import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* M1 screen: record a meeting (foreground service), then transcribe the
|
||||
* recording on device. Live transcript arrives with M2.
|
||||
* Record a meeting with live transcript (cheap model, rolling window) and
|
||||
* an automatic final pass with the selected model on stop — mirroring the
|
||||
* desktop app's two-pass design.
|
||||
*/
|
||||
class MainActivity : ComponentActivity() {
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
@@ -69,6 +70,8 @@ fun MeetRecScreen() {
|
||||
var modelMenu by remember { mutableStateOf(false) }
|
||||
var language by remember { mutableStateOf("auto") }
|
||||
var langMenu by remember { mutableStateOf(false) }
|
||||
var liveModel by remember { mutableStateOf("tiny") }
|
||||
var liveMenu by remember { mutableStateOf(false) }
|
||||
var status by remember { mutableStateOf("Select a model, download and load it.") }
|
||||
var busy by remember { mutableStateOf(false) }
|
||||
var engine by remember { mutableStateOf<WhisperEngine?>(null) }
|
||||
@@ -80,6 +83,7 @@ fun MeetRecScreen() {
|
||||
val modelFile = ModelDownloader.modelFile(model, modelsDir)
|
||||
|
||||
val recState by RecorderService.state.collectAsState()
|
||||
val live by RecorderService.liveLines.collectAsState()
|
||||
|
||||
fun <T> runBusy(block: suspend () -> T) {
|
||||
if (busy) return
|
||||
@@ -95,23 +99,35 @@ fun MeetRecScreen() {
|
||||
}
|
||||
}
|
||||
|
||||
// A finished recording is auto-loaded for transcription.
|
||||
suspend fun loadManualWav(file: File) {
|
||||
val samples = withContext(Dispatchers.IO) {
|
||||
file.inputStream().use { WavReader.read(it) }
|
||||
}
|
||||
wavName = file.name
|
||||
wavSamples = samples
|
||||
status = "${file.name} loaded — press Transcribe"
|
||||
}
|
||||
|
||||
// Final result: show the transcript and note the saved files.
|
||||
LaunchedEffect(recState) {
|
||||
val fin = recState as? RecordingState.Finished ?: return@LaunchedEffect
|
||||
runBusy {
|
||||
val samples = withContext(Dispatchers.IO) {
|
||||
fin.file.inputStream().use { WavReader.read(it) }
|
||||
segments = emptyList()
|
||||
wavSamples = null
|
||||
if (fin.segments.isNotEmpty()) {
|
||||
segments = fin.segments
|
||||
status = if (fin.error != null) {
|
||||
"Final pass failed: ${fin.error}"
|
||||
} else {
|
||||
"Saved ${fin.outputs.size} transcript files next to ${fin.file.name}"
|
||||
}
|
||||
wavName = fin.file.name
|
||||
wavSamples = samples
|
||||
segments = emptyList()
|
||||
status = "Recorded ${fmtMs(fin.durationMs)} at ${fin.sampleRate} Hz" +
|
||||
" — ready to transcribe"
|
||||
} else if (fin.error != null) {
|
||||
status = "Final pass failed: ${fin.error} — the WAV is saved"
|
||||
runBusy { loadManualWav(fin.file) }
|
||||
} else {
|
||||
// no engine was loaded — offer the manual transcribe path
|
||||
runBusy { loadManualWav(fin.file) }
|
||||
}
|
||||
}
|
||||
(recState as? RecordingState.Failed)?.let {
|
||||
LaunchedEffect(it) { status = "Recording failed: ${it.message}" }
|
||||
}
|
||||
|
||||
fun startRecording() = context.startForegroundService(
|
||||
Intent(context, RecorderService::class.java)
|
||||
@@ -145,7 +161,20 @@ fun MeetRecScreen() {
|
||||
val missing = wanted.filter {
|
||||
context.checkSelfPermission(it) != PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
if (missing.isEmpty()) startRecording() else permLauncher.launch(missing.toTypedArray())
|
||||
if (missing.isEmpty()) {
|
||||
RecorderService.session = SessionConfig(
|
||||
finalEngine = engine,
|
||||
language = language.takeIf { it != "auto" },
|
||||
liveModel = liveModel.takeIf { it != "off" },
|
||||
)
|
||||
if (engine == null) {
|
||||
status = "Recording without transcription — load an engine first " +
|
||||
"for the automatic final pass"
|
||||
}
|
||||
startRecording()
|
||||
} else {
|
||||
permLauncher.launch(missing.toTypedArray())
|
||||
}
|
||||
}
|
||||
|
||||
val pickWav = rememberLauncherForActivityResult(
|
||||
@@ -167,6 +196,17 @@ fun MeetRecScreen() {
|
||||
}
|
||||
}
|
||||
|
||||
// what the transcript list shows: live lines while recording, else segments
|
||||
val transcript: List<Pair<Long, String>> =
|
||||
if (recState is RecordingState.Recording) {
|
||||
live.map { it.startMs to it.text }
|
||||
} else {
|
||||
segments.map { it.startMs to it.text }
|
||||
}
|
||||
|
||||
val recording = recState as? RecordingState.Recording
|
||||
val finalizing = recState is RecordingState.Finalizing
|
||||
|
||||
Column(
|
||||
modifier = Modifier.fillMaxSize().padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(12.dp),
|
||||
@@ -221,6 +261,32 @@ fun MeetRecScreen() {
|
||||
}
|
||||
}
|
||||
|
||||
// live model
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
ExposedDropdownMenuBox(
|
||||
expanded = liveMenu,
|
||||
onExpandedChange = { liveMenu = it },
|
||||
modifier = Modifier.weight(1f),
|
||||
) {
|
||||
OutlinedTextField(
|
||||
value = liveModel,
|
||||
onValueChange = {},
|
||||
readOnly = true,
|
||||
label = { Text("Live model") },
|
||||
trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(liveMenu) },
|
||||
modifier = Modifier.menuAnchor().fillMaxWidth(),
|
||||
)
|
||||
ExposedDropdownMenu(expanded = liveMenu, onDismissRequest = { liveMenu = false }) {
|
||||
LIVE_MODELS.forEach { name ->
|
||||
DropdownMenuItem(
|
||||
text = { Text(name) },
|
||||
onClick = { liveModel = name; liveMenu = false },
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// model actions
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
Button(
|
||||
@@ -250,11 +316,10 @@ fun MeetRecScreen() {
|
||||
}
|
||||
|
||||
// recording
|
||||
val recording = recState as? RecordingState.Recording
|
||||
Button(
|
||||
onClick = { toggleRecording() },
|
||||
enabled = !busy || recording != null,
|
||||
colors = if (recording == null) {
|
||||
enabled = !finalizing && (!busy || recording != null),
|
||||
colors = if (recording == null && !finalizing) {
|
||||
ButtonDefaults.buttonColors()
|
||||
} else {
|
||||
ButtonDefaults.buttonColors(
|
||||
@@ -264,10 +329,10 @@ fun MeetRecScreen() {
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
) {
|
||||
Text(
|
||||
if (recording == null) {
|
||||
"\u25CF Record"
|
||||
} else {
|
||||
"\u25A0 Stop (${fmtMs(recording.elapsedMs)})"
|
||||
when {
|
||||
recording != null -> "\u25A0 Stop (${fmtMs(recording.elapsedMs)})"
|
||||
finalizing -> "Transcribing\u2026"
|
||||
else -> "\u25CF Record"
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -278,7 +343,7 @@ fun MeetRecScreen() {
|
||||
)
|
||||
}
|
||||
|
||||
// transcribe actions
|
||||
// manual transcribe path (or re-transcribe of picked files)
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
OutlinedButton(
|
||||
onClick = { pickWav.launch(arrayOf("audio/*", "application/octet-stream")) },
|
||||
@@ -309,11 +374,9 @@ fun MeetRecScreen() {
|
||||
) { Text("Transcribe") }
|
||||
}
|
||||
|
||||
if (segments.isNotEmpty()) {
|
||||
if (transcript.isNotEmpty()) {
|
||||
OutlinedButton(onClick = {
|
||||
val text = segments.joinToString("\n") {
|
||||
"[${fmtMs(it.startMs)}] ${it.text}"
|
||||
}
|
||||
val text = transcript.joinToString("\n") { "[${fmtMs(it.first)}] ${it.second}" }
|
||||
context.startActivity(
|
||||
Intent.createChooser(
|
||||
Intent(Intent.ACTION_SEND)
|
||||
@@ -326,16 +389,16 @@ fun MeetRecScreen() {
|
||||
}
|
||||
|
||||
Text(status, style = MaterialTheme.typography.bodySmall)
|
||||
if (busy) CircularProgressIndicator()
|
||||
if (busy || finalizing) CircularProgressIndicator()
|
||||
|
||||
LazyColumn(modifier = Modifier.fillMaxWidth().weight(1f)) {
|
||||
items(segments) { seg ->
|
||||
items(transcript) { (startMs, text) ->
|
||||
Column(Modifier.fillMaxWidth().padding(vertical = 4.dp)) {
|
||||
Text(
|
||||
"${fmtMs(seg.startMs)} – ${fmtMs(seg.endMs)}",
|
||||
fmtMs(startMs),
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
)
|
||||
Text(seg.text, style = MaterialTheme.typography.bodyMedium)
|
||||
Text(text, style = MaterialTheme.typography.bodyMedium)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -343,6 +406,7 @@ fun MeetRecScreen() {
|
||||
}
|
||||
|
||||
private val LANGUAGES = listOf("auto", "en", "de")
|
||||
private val LIVE_MODELS = listOf("off", "tiny", "base")
|
||||
|
||||
private fun fmtMs(ms: Long): String {
|
||||
val total = ms / 1000
|
||||
|
||||
@@ -10,34 +10,79 @@ import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.pm.ServiceInfo
|
||||
import android.os.IBinder
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import androidx.core.app.NotificationCompat
|
||||
import com.meetrec.core.recording.MeetingRecorder
|
||||
import com.meetrec.core.recording.WavReader
|
||||
import com.meetrec.core.whisper.TranscriptFiles
|
||||
import com.meetrec.core.whisper.WhisperEngine
|
||||
import java.io.File
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Date
|
||||
import java.util.Locale
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
/** One line of the live rolling transcript, with its absolute start time. */
|
||||
data class LiveLine(val startMs: Long, val text: String)
|
||||
|
||||
/**
|
||||
* Engine/session for the next recording; set by the UI right before
|
||||
* starting the service. finalEngine may be null (record only);
|
||||
* liveModel may be null (no live transcript).
|
||||
*/
|
||||
data class SessionConfig(
|
||||
val finalEngine: WhisperEngine?,
|
||||
val language: String?,
|
||||
val liveModel: String?,
|
||||
val liveIntervalSec: Int = 8,
|
||||
)
|
||||
|
||||
/** Recording state published to the UI (collect RecorderService.state). */
|
||||
sealed interface RecordingState {
|
||||
data object Idle : RecordingState
|
||||
|
||||
data class Recording(val file: File, val elapsedMs: Long, val level: Float) :
|
||||
RecordingState
|
||||
|
||||
data class Finalizing(val file: File, val durationMs: Long) : RecordingState
|
||||
|
||||
data class Finished(
|
||||
val file: File,
|
||||
val durationMs: Long,
|
||||
val sampleRate: Int,
|
||||
/** Final transcript; empty when no engine was loaded at stop time. */
|
||||
val segments: List<WhisperEngine.Segment> = emptyList(),
|
||||
/** Transcript files written next to the WAV. */
|
||||
val outputs: List<File> = emptyList(),
|
||||
/** Set when the final pass itself failed (the WAV is still saved). */
|
||||
val error: String? = null,
|
||||
) : RecordingState
|
||||
|
||||
data class Failed(val message: String) : RecordingState
|
||||
}
|
||||
|
||||
/**
|
||||
* Foreground service (type: microphone) that owns the MeetingRecorder,
|
||||
* mirroring the desktop recorder lifecycle. Keeps recording alive while
|
||||
* the app is backgrounded and posts an ongoing notification.
|
||||
* Foreground service (type: microphone) owning the full record →
|
||||
* transcribe flow, mirroring the desktop app:
|
||||
*
|
||||
* - records at the input's native rate to WAV,
|
||||
* - while recording, transcribes the 60 s rolling window every
|
||||
* `liveIntervalSec` with the (cheap) live model at beam 1,
|
||||
* - on stop, runs the final pass with the selected model at beam 5 and
|
||||
* writes .txt/.srt/.json next to the WAV.
|
||||
*
|
||||
* The UI configures [session] before starting the service. Everything
|
||||
* works without an engine too — recording is never blocked on models.
|
||||
*/
|
||||
class RecorderService : Service() {
|
||||
|
||||
@@ -46,13 +91,24 @@ class RecorderService : Service() {
|
||||
const val ACTION_STOP = "com.meetrec.android.action.STOP"
|
||||
private const val CHANNEL_ID = "meetrec_recording"
|
||||
private const val NOTIFICATION_ID = 1
|
||||
private const val TAG = "MeetRec"
|
||||
|
||||
@Volatile
|
||||
var session: SessionConfig? = null
|
||||
|
||||
val state: StateFlow<RecordingState> = MutableStateFlow(RecordingState.Idle)
|
||||
private val _state = state as MutableStateFlow<RecordingState>
|
||||
|
||||
/** Live transcript lines while recording (cleared on start/finish). */
|
||||
val liveLines: StateFlow<List<LiveLine>> =
|
||||
MutableStateFlow<List<LiveLine>>(emptyList())
|
||||
private val _liveLines = liveLines as MutableStateFlow<List<LiveLine>>
|
||||
}
|
||||
|
||||
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private var recorder: MeetingRecorder? = null
|
||||
private var currentFile: File? = null
|
||||
private var liveJob: Job? = null
|
||||
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
|
||||
@@ -87,8 +143,86 @@ class RecorderService : Service() {
|
||||
|
||||
val rec = MeetingRecorder(currentFile!!, listener)
|
||||
recorder = rec
|
||||
_liveLines.value = emptyList()
|
||||
_state.value = RecordingState.Recording(currentFile!!, 0, 0f)
|
||||
rec.start()
|
||||
|
||||
val cfg = session
|
||||
if (cfg?.liveModel != null) {
|
||||
liveJob = scope.launch { liveLoop(rec, cfg) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rolling-window live transcription (desktop live_worker, Android
|
||||
* edition). Loads the live model off the main thread, reuses the
|
||||
* final engine when both roles use the same model.
|
||||
*/
|
||||
private suspend fun liveLoop(rec: MeetingRecorder, cfg: SessionConfig) {
|
||||
val engine: WhisperEngine = try {
|
||||
if (cfg.liveModel == cfg.finalEngine?.modelName) {
|
||||
Log.i(TAG, "live: reusing final engine (${cfg.liveModel})")
|
||||
cfg.finalEngine!!
|
||||
} else {
|
||||
withContext(Dispatchers.Default) {
|
||||
val f = ModelDownloader.modelFile(cfg.liveModel!!, File(filesDir, "models"))
|
||||
check(f.isFile) {
|
||||
"live model ${cfg.liveModel} not downloaded (get it first)"
|
||||
}
|
||||
Log.i(TAG, "live: loading ${cfg.liveModel} (${f.name})")
|
||||
WhisperEngine.load(f.absolutePath).also {
|
||||
Log.i(TAG, "live: ${cfg.liveModel} loaded")
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
// Live transcript is optional — keep recording regardless.
|
||||
Log.w(TAG, "live: engine unavailable: ${e.message}")
|
||||
_liveLines.value = listOf(LiveLine(0, "live transcript off: ${e.message}"))
|
||||
return
|
||||
}
|
||||
|
||||
var lastEndMs = 0L
|
||||
var errors = 0
|
||||
while (rec.isRunning) {
|
||||
delay(cfg.liveIntervalSec * 1000L)
|
||||
if (!rec.isRunning) break
|
||||
val snap = rec.snapshot() ?: continue // < 2 s captured so far
|
||||
try {
|
||||
val t0 = SystemClock.elapsedRealtime()
|
||||
val segs = engine.transcribe(
|
||||
snap.audio,
|
||||
beamSize = 1,
|
||||
language = cfg.language,
|
||||
)
|
||||
val tookMs = SystemClock.elapsedRealtime() - t0
|
||||
val baseMs = (snap.startSec * 1000).toLong()
|
||||
val fresh = segs.mapNotNull { s ->
|
||||
val absEnd = baseMs + s.endMs
|
||||
if (s.text.isBlank() || absEnd <= lastEndMs) null
|
||||
else {
|
||||
lastEndMs = maxOf(lastEndMs, absEnd)
|
||||
LiveLine(baseMs + s.startMs, s.text)
|
||||
}
|
||||
}
|
||||
Log.i(
|
||||
TAG, "live: window=${snap.audio.size / 16000}s " +
|
||||
"segs=${segs.size} fresh=${fresh.size} took=${tookMs}ms",
|
||||
)
|
||||
if (fresh.isNotEmpty()) {
|
||||
_liveLines.value = _liveLines.value + fresh
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "live: transcribe failed: ${e.message}", e)
|
||||
if (errors == 0) {
|
||||
_liveLines.value = _liveLines.value +
|
||||
LiveLine(0, "live error: ${e.message}")
|
||||
}
|
||||
errors++
|
||||
// one failed window must not kill the loop
|
||||
}
|
||||
}
|
||||
Log.i(TAG, "live: loop finished (errors=$errors)")
|
||||
}
|
||||
|
||||
private val listener = object : MeetingRecorder.Listener {
|
||||
@@ -97,8 +231,7 @@ class RecorderService : Service() {
|
||||
}
|
||||
|
||||
override fun onFinish(file: File, durationMs: Long, sampleRate: Int) {
|
||||
_state.value = RecordingState.Finished(file, durationMs, sampleRate)
|
||||
cleanup()
|
||||
scope.launch { finishRecording(file, durationMs, sampleRate) }
|
||||
}
|
||||
|
||||
override fun onError(message: String) {
|
||||
@@ -107,7 +240,47 @@ class RecorderService : Service() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Final pass + outputs, after the WAV is safely closed. */
|
||||
private suspend fun finishRecording(file: File, durationMs: Long, sampleRate: Int) {
|
||||
// let the live loop drain its current window first
|
||||
liveJob?.cancel()
|
||||
liveJob?.join()
|
||||
liveJob = null
|
||||
|
||||
val cfg = session
|
||||
val engine = cfg?.finalEngine
|
||||
if (engine == null) {
|
||||
_state.value = RecordingState.Finished(file, durationMs, sampleRate)
|
||||
cleanup()
|
||||
return
|
||||
}
|
||||
|
||||
_state.value = RecordingState.Finalizing(file, durationMs)
|
||||
updateNotification("Transcribing meeting…")
|
||||
try {
|
||||
val t0 = SystemClock.elapsedRealtime()
|
||||
val samples = withContext(Dispatchers.IO) {
|
||||
file.inputStream().use { WavReader.read(it) }
|
||||
}
|
||||
val segments = engine.transcribe(
|
||||
samples,
|
||||
beamSize = 5,
|
||||
language = cfg.language,
|
||||
)
|
||||
Log.i(TAG, "final: ${segments.size} segments in " +
|
||||
"${SystemClock.elapsedRealtime() - t0} ms")
|
||||
val outputs = TranscriptFiles.writeAll(file, durationMs, segments)
|
||||
_state.value = RecordingState.Finished(file, durationMs, sampleRate, segments, outputs)
|
||||
} catch (e: Exception) {
|
||||
_state.value = RecordingState.Finished(
|
||||
file, durationMs, sampleRate, error = e.message,
|
||||
)
|
||||
}
|
||||
cleanup()
|
||||
}
|
||||
|
||||
private fun cleanup() {
|
||||
session = null
|
||||
stopForeground(STOP_FOREGROUND_REMOVE)
|
||||
stopSelf()
|
||||
recorder = null
|
||||
@@ -116,6 +289,7 @@ class RecorderService : Service() {
|
||||
override fun onDestroy() {
|
||||
recorder?.stop()
|
||||
recorder = null
|
||||
scope.cancel()
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
@@ -144,4 +318,9 @@ class RecorderService : Service() {
|
||||
.setContentIntent(pi)
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun updateNotification(text: String) {
|
||||
getSystemService(NotificationManager::class.java)
|
||||
.notify(NOTIFICATION_ID, notification(text))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user