diff --git a/android/app/src/main/java/com/meetrec/android/RecordScreen.kt b/android/app/src/main/java/com/meetrec/android/RecordScreen.kt index 78c9281..a0c8e06 100644 --- a/android/app/src/main/java/com/meetrec/android/RecordScreen.kt +++ b/android/app/src/main/java/com/meetrec/android/RecordScreen.kt @@ -67,6 +67,7 @@ fun RecordScreen() { val storageUrl = prefs.getString("storage_url", "") ?: "" val language = prefs.getString("language", "auto") ?: "auto" val liveModel = prefs.getString("live_model", "tiny") ?: "tiny" + val diarizeUrl = prefs.getString("diarize_url", "") ?: "" var agendaText by remember { mutableStateOf(prefs.getString("agenda", "") ?: "") } var status by remember { mutableStateOf("Ready.") } @@ -182,6 +183,7 @@ fun RecordScreen() { }, storageUrl = storageUrl.trim().ifBlank { null }, agenda = agendaText.lines().map { it.trim() }.filter { it.isNotBlank() }, + diarizeUrl = diarizeUrl.trim().ifBlank { null }, ) if (transcribeOn == "server") { status = "Recording — live and final pass on ${serverUrl.trim()}" diff --git a/android/app/src/main/java/com/meetrec/android/RecorderService.kt b/android/app/src/main/java/com/meetrec/android/RecorderService.kt index 7e13e0a..7525e11 100644 --- a/android/app/src/main/java/com/meetrec/android/RecorderService.kt +++ b/android/app/src/main/java/com/meetrec/android/RecorderService.kt @@ -15,6 +15,7 @@ import android.util.Log import androidx.core.app.NotificationCompat import com.meetrec.core.recording.MeetingRecorder import com.meetrec.core.recording.WavReader +import com.meetrec.core.whisper.Diarization import com.meetrec.core.whisper.RemoteWhisperEngine import com.meetrec.core.whisper.Transcriber import com.meetrec.core.whisper.TranscriptFiles @@ -63,6 +64,8 @@ data class SessionConfig( val liveIntervalSec: Int = 8, val storageUrl: String? = null, val agenda: List = emptyList(), + /** tinydiarize server; the final pass is speaker-labeled via its turns. */ + val diarizeUrl: String? = null, ) /** Recording state published to the UI (collect RecorderService.state). */ @@ -294,7 +297,7 @@ class RecorderService : Service() { val samples = withContext(Dispatchers.IO) { file.inputStream().use { WavReader.read(it) } } - val segments = transcriber.transcribe( + var segments = transcriber.transcribe( samples, WhisperEngine.DEFAULT_THREADS, beamSize = 5, @@ -302,6 +305,21 @@ class RecorderService : Service() { ) Log.i(TAG, "final: ${segments.size} segments in " + "${SystemClock.elapsedRealtime() - t0} ms") + // optional speaker labeling: a second pass on the tinydiarize + // server contributes only the TURN TIMES, which are merged + // onto the quality transcript as "Sprecher 1/2" labels. + if (cfg?.diarizeUrl != null) { + try { + val dt0 = SystemClock.elapsedRealtime() + val turns = RemoteWhisperEngine(cfg.diarizeUrl, diarize = true) + .transcribe(samples, WhisperEngine.DEFAULT_THREADS, 5, cfg.language) + segments = Diarization.merge(segments, turns) + Log.i(TAG, "diarize: ${turns.count { it.speakerTurnNext }} turns " + + "in ${SystemClock.elapsedRealtime() - dt0} ms") + } catch (e: Exception) { + Log.w(TAG, "diarize pass failed: ${e.message}") + } + } outputs = TranscriptFiles.writeAll(file, durationMs, segments) _state.value = RecordingState.Finished(file, durationMs, sampleRate, segments, outputs) } catch (e: Exception) { diff --git a/android/app/src/main/java/com/meetrec/android/SettingsScreen.kt b/android/app/src/main/java/com/meetrec/android/SettingsScreen.kt index c9c4dc3..71d8452 100644 --- a/android/app/src/main/java/com/meetrec/android/SettingsScreen.kt +++ b/android/app/src/main/java/com/meetrec/android/SettingsScreen.kt @@ -72,6 +72,7 @@ fun SettingsScreen() { var langMenu by remember { mutableStateOf(false) } var liveModel by remember { mutableStateOf(prefs.getString("live_model", "tiny") ?: "tiny") } var liveMenu by remember { mutableStateOf(false) } + var diarizeUrl by remember { mutableStateOf(prefs.getString("diarize_url", "") ?: "") } var status by remember { mutableStateOf("") } var busy by remember { mutableStateOf(false) } @@ -84,6 +85,7 @@ fun SettingsScreen() { LaunchedEffect(model) { prefs.edit().putString("model", model).apply() } LaunchedEffect(language) { prefs.edit().putString("language", language).apply() } LaunchedEffect(liveModel) { prefs.edit().putString("live_model", liveModel).apply() } + LaunchedEffect(diarizeUrl) { prefs.edit().putString("diarize_url", diarizeUrl).apply() } fun runBusy(block: suspend () -> T) { if (busy) return @@ -265,10 +267,20 @@ fun SettingsScreen() { Text(status, style = MaterialTheme.typography.bodySmall) if (busy) CircularProgressIndicator() + OutlinedTextField( + value = diarizeUrl, + onValueChange = { diarizeUrl = it }, + label = { Text("Diarize server URL") }, + placeholder = { Text("http://100.103.83.12:8086") }, + modifier = Modifier.fillMaxWidth(), + ) + Text( - "Transcribe on: phone uses the local engine (download + load), " + - "server sends both passes to the whisper.cpp server. Recordings " + - "upload to the library when the Library URL is set.", + "Speaker labels: when a diarize server is set, a second " + + "(English-trained, 2-speaker) tinydiarize pass marks the speaker " + + "changes; meetrec merges them onto the transcript as " + + "Sprecher 1/2. Best-effort on non-English audio; an empty URL " + + "disables it.", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant, ) diff --git a/android/core/whisper/src/main/java/com/meetrec/core/whisper/Diarization.kt b/android/core/whisper/src/main/java/com/meetrec/core/whisper/Diarization.kt new file mode 100644 index 0000000..afea2ea --- /dev/null +++ b/android/core/whisper/src/main/java/com/meetrec/core/whisper/Diarization.kt @@ -0,0 +1,82 @@ +package com.meetrec.core.whisper + +/** + * Two-pass speaker labeling ("Sprecher 1/Sprecher 2"): the quality + * transcript comes from the main model; the tinydiarize (tdrz) pass + * only contributes the TIMES at which the speaker switches. + * + * tinydiarize is 2-speaker, English-trained and experimental — on + * non-English audio turn detection is best-effort. If no turns are + * detected the transcript is returned unchanged (no wrong labels). + */ +object Diarization { + + /** + * Merges [segments] (quality transcript) with [turns] (tdrz pass, + * whose [WhisperEngine.Segment.speakerTurnNext] flags mark a speaker + * switch after that segment). Returns segments labeled + * "Sprecher N: ", split at turn times when a turn falls inside + * a quality segment. + */ + fun merge( + segments: List, + turns: List, + ): List { + val turnTimes = turns.filter { it.speakerTurnNext } + .map { it.endMs } + .sorted() + if (turnTimes.isEmpty()) { + return segments + } + + var speaker = 1 + var turnIdx = 0 + val out = mutableListOf() + + for (seg in segments) { + // consume all turns that happened before this segment + while (turnIdx < turnTimes.size && turnTimes[turnIdx] <= seg.startMs) { + speaker = flip(speaker) + turnIdx++ + } + // split this segment at turn times that fall inside it + var start = seg.startMs + var text = seg.text + while (turnIdx < turnTimes.size && turnTimes[turnIdx] < seg.endMs) { + val t = turnTimes[turnIdx] + val (head, rest) = splitText(text, seg.startMs, seg.endMs, t) + out.add( + WhisperEngine.Segment(start, t, "Sprecher $speaker: ${head.trim()}"), + ) + speaker = flip(speaker) + turnIdx++ + start = t + text = rest + } + out.add( + WhisperEngine.Segment( + start, seg.endMs, "Sprecher $speaker: ${text.trim()}", + ), + ) + } + return out + } + + private fun flip(speaker: Int) = if (speaker == 1) 2 else 1 + + /** + * Splits [text] at the position corresponding to time [tMs], assuming + * text distributes proportionally over the segment's duration. + */ + private fun splitText( + text: String, + startMs: Long, + endMs: Long, + tMs: Long, + ): Pair { + val span = (endMs - startMs).coerceAtLeast(1) + val frac = ((tMs - startMs).toDouble() / span).coerceIn(0.0, 1.0) + val cut = (text.length * frac).toInt().coerceIn(0, text.length) + return text.substring(0, cut) to text.substring(cut) + } +} \ No newline at end of file diff --git a/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt b/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt index 62e48ab..9010651 100644 --- a/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt +++ b/android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt @@ -28,7 +28,7 @@ interface Transcriber { * the server — beam size is a server-start setting in whisper.cpp * v1.9.3 (no per-request override), so beamSize/threads are ignored. */ -class RemoteWhisperEngine(baseUrl: String) : Transcriber { +class RemoteWhisperEngine(baseUrl: String, private val diarize: Boolean = false) : Transcriber { private val base: String = baseUrl.trim().trimEnd('/') @@ -47,13 +47,14 @@ class RemoteWhisperEngine(baseUrl: String) : Transcriber { if (audio.isEmpty()) return emptyList() val wav = WavEncoder.encodePcm16(audio, Audio.TARGET_RATE) - val body = multipart( - mapOf( - "response_format" to "verbose_json", - "language" to (language ?: "auto"), - ), - wav, - ) + val fields = buildMap { + put("response_format", "verbose_json") + put("language", language ?: "auto") + if (diarize) { + put("tinydiarize", "true") + } + } + val body = multipart(fields, wav) val conn = (URL("$base/inference").openConnection() as HttpURLConnection).apply { requestMethod = "POST" @@ -109,6 +110,7 @@ class RemoteWhisperEngine(baseUrl: String) : Transcriber { startMs = (o.optDouble("start", 0.0) * 1000).toLong(), endMs = (o.optDouble("end", 0.0) * 1000).toLong(), text = text, + speakerTurnNext = o.optBoolean("speaker_turn_next", false), ) } } diff --git a/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt b/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt index 10ac9d3..4164804 100644 --- a/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt +++ b/android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt @@ -17,7 +17,13 @@ class WhisperEngine private constructor( val modelName: String, ) : Transcriber, AutoCloseable { - data class Segment(val startMs: Long, val endMs: Long, val text: String) + data class Segment( + val startMs: Long, + val endMs: Long, + val text: String, + /** tinydiarize: a speaker turn occurs after this segment. */ + val speakerTurnNext: Boolean = false, + ) @Volatile private var closed = false diff --git a/android/core/whisper/src/test/java/com/meetrec/core/whisper/DiarizationTest.kt b/android/core/whisper/src/test/java/com/meetrec/core/whisper/DiarizationTest.kt new file mode 100644 index 0000000..798b4a5 --- /dev/null +++ b/android/core/whisper/src/test/java/com/meetrec/core/whisper/DiarizationTest.kt @@ -0,0 +1,84 @@ +package com.meetrec.core.whisper + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class DiarizationTest { + + private fun seg(a: Long, b: Long, text: String, turn: Boolean = false) = + WhisperEngine.Segment(a, b, text, turn) + + @Test + fun `turns between segments label alternating speakers`() { + val quality = listOf( + seg(0, 5000, "Hallo zusammen."), + seg(5100, 10000, "Ja hallo, ich hab eine Frage."), + seg(10100, 15000, "Aber gerne doch."), + ) + val tdrz = listOf( + seg(0, 5000, "ignored", turn = true), + seg(5100, 10000, "ignored"), + ) + val out = Diarization.merge(quality, tdrz) + assertEquals(3, out.size) + assertEquals("Sprecher 1: Hallo zusammen.", out[0].text) + assertEquals("Sprecher 2: Ja hallo, ich hab eine Frage.", out[1].text) + assertEquals("Sprecher 2: Aber gerne doch.", out[2].text) + // times preserved + assertEquals(0L to 5000L, out[0].startMs to out[0].endMs) + assertEquals(10100L to 15000L, out[2].startMs to out[2].endMs) + } + + @Test + fun `turn inside a segment splits it with both labels`() { + val quality = listOf(seg(0, 10000, "abcdefghij")) + val tdrz = listOf(seg(0, 6000, "ignored", turn = true)) + val out = Diarization.merge(quality, tdrz) + assertEquals(2, out.size) + assertEquals(0L, out[0].startMs) + assertEquals(6000L, out[0].endMs) + assertEquals("Sprecher 1: abcdef", out[0].text) + assertEquals(6000L, out[1].startMs) + assertEquals(10000L, out[1].endMs) + assertEquals("Sprecher 2: ghij", out[1].text) + } + + @Test + fun `no turns returns segments unlabeled`() { + val quality = listOf(seg(0, 5000, "Hallo zusammen.")) + val tdrz = listOf(seg(0, 5000, "ignored")) + val out = Diarization.merge(quality, tdrz) + assertEquals(quality, out) // untouched, no wrong labels + } + + @Test + fun `multiple turns flip back and forth`() { + val quality = listOf( + seg(0, 4000, "Eins"), seg(4100, 8000, "Zwei"), seg(8100, 12000, "Drei"), + ) + val tdrz = listOf( + seg(0, 4000, "x", turn = true), + seg(4100, 8000, "y", turn = true), + ) + val out = Diarization.merge(quality, tdrz) + assertEquals(listOf(1, 2, 1), out.map { speakerOf(it) }) + } + + @Test + fun `turn after the last segment is ignored`() { + val quality = listOf(seg(0, 5000, "Eins")) + val tdrz = listOf(seg(0, 5000, "ignored", turn = true)) + val out = Diarization.merge(quality, tdrz) + assertEquals(1, out.size) + assertEquals("Sprecher 1: Eins", out[0].text) + } + + @Test + fun `empty quality transcript stays empty`() { + assertTrue(Diarization.merge(emptyList(), listOf(seg(0, 1000, "x"))).isEmpty()) + } + + private fun speakerOf(s: WhisperEngine.Segment): Int = + s.text.removePrefix("Sprecher ").take(1).toInt() +} \ No newline at end of file diff --git a/server/whisper-server/Dockerfile b/server/whisper-server/Dockerfile index a581df0..483267f 100644 --- a/server/whisper-server/Dockerfile +++ b/server/whisper-server/Dockerfile @@ -19,6 +19,10 @@ ARG WHISPER_TAG=v1.9.3 RUN git clone --depth 1 --branch ${WHISPER_TAG} \ https://github.com/ggml-org/whisper.cpp /src +# expose tinydiarize speaker_turn_next in verbose_json (see patch header) +COPY speaker-turn.patch /src/ +RUN git -C /src apply speaker-turn.patch + RUN cmake -S /src -B /src/build -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=OFF \ -DWHISPER_BUILD_EXAMPLES=ON \ diff --git a/server/whisper-server/docker-compose.yml b/server/whisper-server/docker-compose.yml index 3932926..0e82e3b 100644 --- a/server/whisper-server/docker-compose.yml +++ b/server/whisper-server/docker-compose.yml @@ -1,14 +1,16 @@ -# whisper.cpp inference server for meetrec, with Vulkan GPU support +# whisper.cpp inference servers for meetrec, with Vulkan GPU support # (AMD Radeon AI PRO R9700), reachable over Tailscale. # # RENDER_GID=$(getent group render | cut -d: -f3) docker compose up -d --build # -# Security: whisper.cpp's server has NO authentication. The port below is -# bound ONLY to the Tailscale interface (100.103.83.12), so the API is -# never exposed to the LAN or the internet. Tailscale must own that IP -# before the container starts, otherwise the bind fails — start order: -# tailscale first, then `docker compose up -d`. If you would rather -# tolerate LAN exposure, use "8085:8085" instead. +# Two services: +# whisper-server port 8085 — large-v3, quality transcripts +# whisper-server-tdrz port 8086 — small.en-tdrz, tinydiarize speaker +# turns (English-trained, 2-speaker, best-effort) +# +# The image is patched to expose `speaker_turn_next` per segment in +# verbose_json (speaker-turn.patch); clients merge the turn times onto +# the better transcript as "Sprecher 1/2" labels. services: whisper-server: @@ -45,4 +47,41 @@ services: interval: 30s timeout: 5s retries: 3 + start_period: 60s + + whisper-server-tdrz: + # tinydiarize variant: English-only small.en-tdrz, used ONLY for its + # speaker-turn timestamps (meetrec merges them onto the better + # transcript from the main server). 2-speaker detection, best-effort + # on non-English audio. + build: . + image: meetrec-whisper-server:latest + container_name: whisper-server-tdrz + restart: unless-stopped + + environment: + MODEL: small.en-tdrz + MODEL_URL: https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-small.en-tdrz.bin + THREADS: 8 + HOST: 0.0.0.0 + PORT: 8086 + TDRZ: 1 + + volumes: + - ./models:/models # shared with the main server (distinct files) + + ports: + - "100.103.83.12:8086:8086" # tailscale-only + + devices: + - /dev/dri:/dev/dri + group_add: + - video + - "${RENDER_GID:-110}" + + healthcheck: + test: ["CMD-SHELL", "curl -s -o /dev/null http://localhost:8086/ || exit 1"] + interval: 30s + timeout: 5s + retries: 3 start_period: 60s \ No newline at end of file diff --git a/server/whisper-server/entrypoint.sh b/server/whisper-server/entrypoint.sh index 122feec..9d19832 100644 --- a/server/whisper-server/entrypoint.sh +++ b/server/whisper-server/entrypoint.sh @@ -15,7 +15,8 @@ HOST="${HOST:-0.0.0.0}" PORT="${PORT:-8085}" MODEL_DIR="${MODEL_DIR:-/models}" FILE="$MODEL_DIR/ggml-$MODEL.bin" -URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin" +# default to the ggerganov collection; TDRZ models live elsewhere +URL="${MODEL_URL:-https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin}" # smallest supported model (ggml-tiny.bin) is ~75 MB MIN_SIZE=50000000 @@ -58,6 +59,9 @@ GPU_FLAGS="" if [ "${NO_GPU:-0}" = "1" ]; then GPU_FLAGS="-ng" fi +if [ "${TDRZ:-0}" = "1" ]; then + GPU_FLAGS="$GPU_FLAGS -tdrz" +fi echo "starting whisper-server: model=$MODEL threads=$THREADS host=$HOST port=$PORT gpu=auto" exec whisper-server \ diff --git a/server/whisper-server/speaker-turn.patch b/server/whisper-server/speaker-turn.patch new file mode 100644 index 0000000..2e2555a --- /dev/null +++ b/server/whisper-server/speaker-turn.patch @@ -0,0 +1,15 @@ +diff --git a/examples/server/server.cpp b/examples/server/server.cpp +index b87ef27..9e13ceb 100644 +--- a/examples/server/server.cpp ++++ b/examples/server/server.cpp +@@ -1090,6 +1090,10 @@ int main(int argc, char ** argv) { + segment["end"] = whisper_full_get_segment_t1(ctx, i) * 0.01; + } + ++ if (params.tinydiarize) { ++ segment["speaker_turn_next"] = whisper_full_get_segment_speaker_turn_next(ctx, i); ++ } ++ + if (params.diarize && pcmf32s.size() == 2) { + segment["speaker"] = estimate_diarization_speaker( + pcmf32s,