M3d: speaker labels via tinydiarize two-pass merge

- whisper-server stack: second container (port 8086) running the English-trained small.en-tdrz model with -tdrz; image patched (speaker-turn.patch) to expose speaker_turn_next per segment in verbose_json like the cli example does - core/whisper Diarization: merges the tdrz pass's TURN TIMES onto the quality transcript as alternating 'Sprecher 1/2:' labels, splitting segments when a turn falls inside them; no turns detected = no labels (never mislabels); 6 unit tests - RemoteWhisperEngine gains a diarize flag (sends tinydiarize=true, parses speaker_turn_next); WhisperEngine.Segment carries the flag - RecorderService: optional second pass on the diarize server after the final pass; failures keep the unlabeled transcript - Settings: Diarize server URL (persisted; empty disables) - validated infrastructure locally: patched image builds, tdrz model downloads from akashmjn/tinydiarize-whisper.cpp, speaker_turn_next present in responses; synthetic espeak audio does not trigger the model's turn tokens — real two-person speech needed for the end-to-end check
2026-09-08 17:07:31 +02:00
parent 6a400e842f
commit 76611af7e0
11 changed files with 289 additions and 21 deletions
@@ -0,0 +1,82 @@
+package com.meetrec.core.whisper
+
+/**
+ * Two-pass speaker labeling ("Sprecher 1/Sprecher 2"): the quality
+ * transcript comes from the main model; the tinydiarize (tdrz) pass
+ * only contributes the TIMES at which the speaker switches.
+ *
+ * tinydiarize is 2-speaker, English-trained and experimental — on
+ * non-English audio turn detection is best-effort. If no turns are
+ * detected the transcript is returned unchanged (no wrong labels).
+ */
+object Diarization {
+
+    /**
+     * Merges [segments] (quality transcript) with [turns] (tdrz pass,
+     * whose [WhisperEngine.Segment.speakerTurnNext] flags mark a speaker
+     * switch after that segment). Returns segments labeled
+     * "Sprecher N: <text>", split at turn times when a turn falls inside
+     * a quality segment.
+     */
+    fun merge(
+        segments: List<WhisperEngine.Segment>,
+        turns: List<WhisperEngine.Segment>,
+    ): List<WhisperEngine.Segment> {
+        val turnTimes = turns.filter { it.speakerTurnNext }
+            .map { it.endMs }
+            .sorted()
+        if (turnTimes.isEmpty()) {
+            return segments
+        }
+
+        var speaker = 1
+        var turnIdx = 0
+        val out = mutableListOf<WhisperEngine.Segment>()
+
+        for (seg in segments) {
+            // consume all turns that happened before this segment
+            while (turnIdx < turnTimes.size && turnTimes[turnIdx] <= seg.startMs) {
+                speaker = flip(speaker)
+                turnIdx++
+            }
+            // split this segment at turn times that fall inside it
+            var start = seg.startMs
+            var text = seg.text
+            while (turnIdx < turnTimes.size && turnTimes[turnIdx] < seg.endMs) {
+                val t = turnTimes[turnIdx]
+                val (head, rest) = splitText(text, seg.startMs, seg.endMs, t)
+                out.add(
+                    WhisperEngine.Segment(start, t, "Sprecher $speaker: ${head.trim()}"),
+                )
+                speaker = flip(speaker)
+                turnIdx++
+                start = t
+                text = rest
+            }
+            out.add(
+                WhisperEngine.Segment(
+                    start, seg.endMs, "Sprecher $speaker: ${text.trim()}",
+                ),
+            )
+        }
+        return out
+    }
+
+    private fun flip(speaker: Int) = if (speaker == 1) 2 else 1
+
+    /**
+     * Splits [text] at the position corresponding to time [tMs], assuming
+     * text distributes proportionally over the segment's duration.
+     */
+    private fun splitText(
+        text: String,
+        startMs: Long,
+        endMs: Long,
+        tMs: Long,
+    ): Pair<String, String> {
+        val span = (endMs - startMs).coerceAtLeast(1)
+        val frac = ((tMs - startMs).toDouble() / span).coerceIn(0.0, 1.0)
+        val cut = (text.length * frac).toInt().coerceIn(0, text.length)
+        return text.substring(0, cut) to text.substring(cut)
+    }
+}
@@ -28,7 +28,7 @@ interface Transcriber {
 * the server — beam size is a server-start setting in whisper.cpp
 * v1.9.3 (no per-request override), so beamSize/threads are ignored.
 */
-class RemoteWhisperEngine(baseUrl: String) : Transcriber {
+class RemoteWhisperEngine(baseUrl: String, private val diarize: Boolean = false) : Transcriber {

    private val base: String = baseUrl.trim().trimEnd('/')

@@ -47,13 +47,14 @@ class RemoteWhisperEngine(baseUrl: String) : Transcriber {
        if (audio.isEmpty()) return emptyList()

        val wav = WavEncoder.encodePcm16(audio, Audio.TARGET_RATE)
-        val body = multipart(
-            mapOf(
-                "response_format" to "verbose_json",
-                "language" to (language ?: "auto"),
-            ),
-            wav,
-        )
+        val fields = buildMap {
+            put("response_format", "verbose_json")
+            put("language", language ?: "auto")
+            if (diarize) {
+                put("tinydiarize", "true")
+            }
+        }
+        val body = multipart(fields, wav)

        val conn = (URL("$base/inference").openConnection() as HttpURLConnection).apply {
            requestMethod = "POST"
@@ -109,6 +110,7 @@ class RemoteWhisperEngine(baseUrl: String) : Transcriber {
                    startMs = (o.optDouble("start", 0.0) * 1000).toLong(),
                    endMs = (o.optDouble("end", 0.0) * 1000).toLong(),
                    text = text,
+                    speakerTurnNext = o.optBoolean("speaker_turn_next", false),
                )
            }
        }
@@ -17,7 +17,13 @@ class WhisperEngine private constructor(
    val modelName: String,
 ) : Transcriber, AutoCloseable {

-    data class Segment(val startMs: Long, val endMs: Long, val text: String)
+    data class Segment(
+        val startMs: Long,
+        val endMs: Long,
+        val text: String,
+        /** tinydiarize: a speaker turn occurs after this segment. */
+        val speakerTurnNext: Boolean = false,
+    )

    @Volatile
    private var closed = false
@@ -0,0 +1,84 @@
+package com.meetrec.core.whisper
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class DiarizationTest {
+
+    private fun seg(a: Long, b: Long, text: String, turn: Boolean = false) =
+        WhisperEngine.Segment(a, b, text, turn)
+
+    @Test
+    fun `turns between segments label alternating speakers`() {
+        val quality = listOf(
+            seg(0, 5000, "Hallo zusammen."),
+            seg(5100, 10000, "Ja hallo, ich hab eine Frage."),
+            seg(10100, 15000, "Aber gerne doch."),
+        )
+        val tdrz = listOf(
+            seg(0, 5000, "ignored", turn = true),
+            seg(5100, 10000, "ignored"),
+        )
+        val out = Diarization.merge(quality, tdrz)
+        assertEquals(3, out.size)
+        assertEquals("Sprecher 1: Hallo zusammen.", out[0].text)
+        assertEquals("Sprecher 2: Ja hallo, ich hab eine Frage.", out[1].text)
+        assertEquals("Sprecher 2: Aber gerne doch.", out[2].text)
+        // times preserved
+        assertEquals(0L to 5000L, out[0].startMs to out[0].endMs)
+        assertEquals(10100L to 15000L, out[2].startMs to out[2].endMs)
+    }
+
+    @Test
+    fun `turn inside a segment splits it with both labels`() {
+        val quality = listOf(seg(0, 10000, "abcdefghij"))
+        val tdrz = listOf(seg(0, 6000, "ignored", turn = true))
+        val out = Diarization.merge(quality, tdrz)
+        assertEquals(2, out.size)
+        assertEquals(0L, out[0].startMs)
+        assertEquals(6000L, out[0].endMs)
+        assertEquals("Sprecher 1: abcdef", out[0].text)
+        assertEquals(6000L, out[1].startMs)
+        assertEquals(10000L, out[1].endMs)
+        assertEquals("Sprecher 2: ghij", out[1].text)
+    }
+
+    @Test
+    fun `no turns returns segments unlabeled`() {
+        val quality = listOf(seg(0, 5000, "Hallo zusammen."))
+        val tdrz = listOf(seg(0, 5000, "ignored"))
+        val out = Diarization.merge(quality, tdrz)
+        assertEquals(quality, out)   // untouched, no wrong labels
+    }
+
+    @Test
+    fun `multiple turns flip back and forth`() {
+        val quality = listOf(
+            seg(0, 4000, "Eins"), seg(4100, 8000, "Zwei"), seg(8100, 12000, "Drei"),
+        )
+        val tdrz = listOf(
+            seg(0, 4000, "x", turn = true),
+            seg(4100, 8000, "y", turn = true),
+        )
+        val out = Diarization.merge(quality, tdrz)
+        assertEquals(listOf(1, 2, 1), out.map { speakerOf(it) })
+    }
+
+    @Test
+    fun `turn after the last segment is ignored`() {
+        val quality = listOf(seg(0, 5000, "Eins"))
+        val tdrz = listOf(seg(0, 5000, "ignored", turn = true))
+        val out = Diarization.merge(quality, tdrz)
+        assertEquals(1, out.size)
+        assertEquals("Sprecher 1: Eins", out[0].text)
+    }
+
+    @Test
+    fun `empty quality transcript stays empty`() {
+        assertTrue(Diarization.merge(emptyList(), listOf(seg(0, 1000, "x"))).isEmpty())
+    }
+
+    private fun speakerOf(s: WhisperEngine.Segment): Int =
+        s.text.removePrefix("Sprecher ").take(1).toInt()
+}