fegger/meetrec
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

M3d: speaker labels via tinydiarize two-pass merge

Browse Source 
- whisper-server stack: second container (port 8086) running the
  English-trained small.en-tdrz model with -tdrz; image patched
  (speaker-turn.patch) to expose speaker_turn_next per segment in
  verbose_json like the cli example does
- core/whisper Diarization: merges the tdrz pass's TURN TIMES onto the
  quality transcript as alternating 'Sprecher 1/2:' labels, splitting
  segments when a turn falls inside them; no turns detected = no
  labels (never mislabels); 6 unit tests
- RemoteWhisperEngine gains a diarize flag (sends tinydiarize=true,
  parses speaker_turn_next); WhisperEngine.Segment carries the flag
- RecorderService: optional second pass on the diarize server after the
  final pass; failures keep the unlabeled transcript
- Settings: Diarize server URL (persisted; empty disables)
- validated infrastructure locally: patched image builds, tdrz model
  downloads from akashmjn/tinydiarize-whisper.cpp, speaker_turn_next
  present in responses; synthetic espeak audio does not trigger the
  model's turn tokens — real two-person speech needed for the
  end-to-end check
This commit is contained in:
Florian Egger
2026-09-08 17:07:31 +02:00
parent 6a400e842f
commit 76611af7e0
11 changed files with 289 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android/app/src/main/java/com/meetrec/android/RecordScreen.kt
+2
View File
@@ -67,6 +67,7 @@ fun RecordScreen() {
val storageUrl = prefs.getString("storage_url", "") ?: ""
val language = prefs.getString("language", "auto") ?: "auto"
val liveModel = prefs.getString("live_model", "tiny") ?: "tiny"
val diarizeUrl = prefs.getString("diarize_url", "") ?: ""
var agendaText by remember { mutableStateOf(prefs.getString("agenda", "") ?: "") }
var status by remember { mutableStateOf("Ready.") }
@@ -182,6 +183,7 @@ fun RecordScreen() {
},
storageUrl = storageUrl.trim().ifBlank { null },
agenda = agendaText.lines().map { it.trim() }.filter { it.isNotBlank() },
diarizeUrl = diarizeUrl.trim().ifBlank { null },
)
if (transcribeOn == "server") {
status = "Recording — live and final pass on ${serverUrl.trim()}"
android/app/src/main/java/com/meetrec/android/RecorderService.kt
+19 -1
View File
@@ -15,6 +15,7 @@ import android.util.Log
import androidx.core.app.NotificationCompat
import com.meetrec.core.recording.MeetingRecorder
import com.meetrec.core.recording.WavReader
import com.meetrec.core.whisper.Diarization
import com.meetrec.core.whisper.RemoteWhisperEngine
import com.meetrec.core.whisper.Transcriber
import com.meetrec.core.whisper.TranscriptFiles
@@ -63,6 +64,8 @@ data class SessionConfig(
val liveIntervalSec: Int = 8,
val storageUrl: String? = null,
val agenda: List<String> = emptyList(),
/** tinydiarize server; the final pass is speaker-labeled via its turns. */
val diarizeUrl: String? = null,
)
/** Recording state published to the UI (collect RecorderService.state). */
@@ -294,7 +297,7 @@ class RecorderService : Service() {
val samples = withContext(Dispatchers.IO) {
file.inputStream().use { WavReader.read(it) }
}
val segments = transcriber.transcribe(
var segments = transcriber.transcribe(
samples,
WhisperEngine.DEFAULT_THREADS,
beamSize = 5,
@@ -302,6 +305,21 @@ class RecorderService : Service() {
)
Log.i(TAG, "final: ${segments.size} segments in " +
"${SystemClock.elapsedRealtime() - t0} ms")
// optional speaker labeling: a second pass on the tinydiarize
// server contributes only the TURN TIMES, which are merged
// onto the quality transcript as "Sprecher 1/2" labels.
if (cfg?.diarizeUrl != null) {
try {
val dt0 = SystemClock.elapsedRealtime()
val turns = RemoteWhisperEngine(cfg.diarizeUrl, diarize = true)
.transcribe(samples, WhisperEngine.DEFAULT_THREADS, 5, cfg.language)
segments = Diarization.merge(segments, turns)
Log.i(TAG, "diarize: ${turns.count { it.speakerTurnNext }} turns " +
"in ${SystemClock.elapsedRealtime() - dt0} ms")
} catch (e: Exception) {
Log.w(TAG, "diarize pass failed: ${e.message}")
}
}
outputs = TranscriptFiles.writeAll(file, durationMs, segments)
_state.value = RecordingState.Finished(file, durationMs, sampleRate, segments, outputs)
} catch (e: Exception) {
android/app/src/main/java/com/meetrec/android/SettingsScreen.kt
+15 -3
View File
@@ -72,6 +72,7 @@ fun SettingsScreen() {
var langMenu by remember { mutableStateOf(false) }
var liveModel by remember { mutableStateOf(prefs.getString("live_model", "tiny") ?: "tiny") }
var liveMenu by remember { mutableStateOf(false) }
var diarizeUrl by remember { mutableStateOf(prefs.getString("diarize_url", "") ?: "") }
var status by remember { mutableStateOf("") }
var busy by remember { mutableStateOf(false) }
@@ -84,6 +85,7 @@ fun SettingsScreen() {
LaunchedEffect(model) { prefs.edit().putString("model", model).apply() }
LaunchedEffect(language) { prefs.edit().putString("language", language).apply() }
LaunchedEffect(liveModel) { prefs.edit().putString("live_model", liveModel).apply() }
LaunchedEffect(diarizeUrl) { prefs.edit().putString("diarize_url", diarizeUrl).apply() }
fun <T> runBusy(block: suspend () -> T) {
if (busy) return
@@ -265,10 +267,20 @@ fun SettingsScreen() {
Text(status, style = MaterialTheme.typography.bodySmall)
if (busy) CircularProgressIndicator()
OutlinedTextField(
value = diarizeUrl,
onValueChange = { diarizeUrl = it },
label = { Text("Diarize server URL") },
placeholder = { Text("http://100.103.83.12:8086") },
modifier = Modifier.fillMaxWidth(),
)
Text(
"Transcribe on: phone uses the local engine (download + load), " +
"server sends both passes to the whisper.cpp server. Recordings " +
"upload to the library when the Library URL is set.",
"Speaker labels: when a diarize server is set, a second " +
"(English-trained, 2-speaker) tinydiarize pass marks the speaker " +
"changes; meetrec merges them onto the transcript as " +
"Sprecher 1/2. Best-effort on non-English audio; an empty URL " +
"disables it.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
android/core/whisper/src/main/java/com/meetrec/core/whisper/Diarization.kt
+82
View File
@@ -0,0 +1,82 @@
package com.meetrec.core.whisper
/**
* Two-pass speaker labeling ("Sprecher 1/Sprecher 2"): the quality
* transcript comes from the main model; the tinydiarize (tdrz) pass
* only contributes the TIMES at which the speaker switches.
*
* tinydiarize is 2-speaker, English-trained and experimental — on
* non-English audio turn detection is best-effort. If no turns are
* detected the transcript is returned unchanged (no wrong labels).
*/
object Diarization {
/**
* Merges [segments] (quality transcript) with [turns] (tdrz pass,
* whose [WhisperEngine.Segment.speakerTurnNext] flags mark a speaker
* switch after that segment). Returns segments labeled
* "Sprecher N: <text>", split at turn times when a turn falls inside
* a quality segment.
*/
fun merge(
segments: List<WhisperEngine.Segment>,
turns: List<WhisperEngine.Segment>,
): List<WhisperEngine.Segment> {
val turnTimes = turns.filter { it.speakerTurnNext }
.map { it.endMs }
.sorted()
if (turnTimes.isEmpty()) {
return segments
}
var speaker = 1
var turnIdx = 0
val out = mutableListOf<WhisperEngine.Segment>()
for (seg in segments) {
// consume all turns that happened before this segment
while (turnIdx < turnTimes.size && turnTimes[turnIdx] <= seg.startMs) {
speaker = flip(speaker)
turnIdx++
}
// split this segment at turn times that fall inside it
var start = seg.startMs
var text = seg.text
while (turnIdx < turnTimes.size && turnTimes[turnIdx] < seg.endMs) {
val t = turnTimes[turnIdx]
val (head, rest) = splitText(text, seg.startMs, seg.endMs, t)
out.add(
WhisperEngine.Segment(start, t, "Sprecher $speaker: ${head.trim()}"),
)
speaker = flip(speaker)
turnIdx++
start = t
text = rest
}
out.add(
WhisperEngine.Segment(
start, seg.endMs, "Sprecher $speaker: ${text.trim()}",
),
)
}
return out
}
private fun flip(speaker: Int) = if (speaker == 1) 2 else 1
/**
* Splits [text] at the position corresponding to time [tMs], assuming
* text distributes proportionally over the segment's duration.
*/
private fun splitText(
text: String,
startMs: Long,
endMs: Long,
tMs: Long,
): Pair<String, String> {
val span = (endMs - startMs).coerceAtLeast(1)
val frac = ((tMs - startMs).toDouble() / span).coerceIn(0.0, 1.0)
val cut = (text.length * frac).toInt().coerceIn(0, text.length)
return text.substring(0, cut) to text.substring(cut)
}
}
android/core/whisper/src/main/java/com/meetrec/core/whisper/RemoteWhisperEngine.kt
+10 -8
View File
@@ -28,7 +28,7 @@ interface Transcriber {
* the server — beam size is a server-start setting in whisper.cpp
* v1.9.3 (no per-request override), so beamSize/threads are ignored.
*/
class RemoteWhisperEngine(baseUrl: String) : Transcriber {
class RemoteWhisperEngine(baseUrl: String, private val diarize: Boolean = false) : Transcriber {
private val base: String = baseUrl.trim().trimEnd('/')
@@ -47,13 +47,14 @@ class RemoteWhisperEngine(baseUrl: String) : Transcriber {
if (audio.isEmpty()) return emptyList()
val wav = WavEncoder.encodePcm16(audio, Audio.TARGET_RATE)
val body = multipart(
mapOf(
"response_format" to "verbose_json",
"language" to (language ?: "auto"),
),
wav,
)
val fields = buildMap {
put("response_format", "verbose_json")
put("language", language ?: "auto")
if (diarize) {
put("tinydiarize", "true")
}
}
val body = multipart(fields, wav)
val conn = (URL("$base/inference").openConnection() as HttpURLConnection).apply {
requestMethod = "POST"
@@ -109,6 +110,7 @@ class RemoteWhisperEngine(baseUrl: String) : Transcriber {
startMs = (o.optDouble("start", 0.0) * 1000).toLong(),
endMs = (o.optDouble("end", 0.0) * 1000).toLong(),
text = text,
speakerTurnNext = o.optBoolean("speaker_turn_next", false),
)
}
}
android/core/whisper/src/main/java/com/meetrec/core/whisper/WhisperEngine.kt
+7 -1
View File
@@ -17,7 +17,13 @@ class WhisperEngine private constructor(
val modelName: String,
) : Transcriber, AutoCloseable {
data class Segment(val startMs: Long, val endMs: Long, val text: String)
data class Segment(
val startMs: Long,
val endMs: Long,
val text: String,
/** tinydiarize: a speaker turn occurs after this segment. */
val speakerTurnNext: Boolean = false,
)
@Volatile
private var closed = false
android/core/whisper/src/test/java/com/meetrec/core/whisper/DiarizationTest.kt
+84
View File
@@ -0,0 +1,84 @@
package com.meetrec.core.whisper
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
class DiarizationTest {
private fun seg(a: Long, b: Long, text: String, turn: Boolean = false) =
WhisperEngine.Segment(a, b, text, turn)
@Test
fun `turns between segments label alternating speakers`() {
val quality = listOf(
seg(0, 5000, "Hallo zusammen."),
seg(5100, 10000, "Ja hallo, ich hab eine Frage."),
seg(10100, 15000, "Aber gerne doch."),
)
val tdrz = listOf(
seg(0, 5000, "ignored", turn = true),
seg(5100, 10000, "ignored"),
)
val out = Diarization.merge(quality, tdrz)
assertEquals(3, out.size)
assertEquals("Sprecher 1: Hallo zusammen.", out[0].text)
assertEquals("Sprecher 2: Ja hallo, ich hab eine Frage.", out[1].text)
assertEquals("Sprecher 2: Aber gerne doch.", out[2].text)
// times preserved
assertEquals(0L to 5000L, out[0].startMs to out[0].endMs)
assertEquals(10100L to 15000L, out[2].startMs to out[2].endMs)
}
@Test
fun `turn inside a segment splits it with both labels`() {
val quality = listOf(seg(0, 10000, "abcdefghij"))
val tdrz = listOf(seg(0, 6000, "ignored", turn = true))
val out = Diarization.merge(quality, tdrz)
assertEquals(2, out.size)
assertEquals(0L, out[0].startMs)
assertEquals(6000L, out[0].endMs)
assertEquals("Sprecher 1: abcdef", out[0].text)
assertEquals(6000L, out[1].startMs)
assertEquals(10000L, out[1].endMs)
assertEquals("Sprecher 2: ghij", out[1].text)
}
@Test
fun `no turns returns segments unlabeled`() {
val quality = listOf(seg(0, 5000, "Hallo zusammen."))
val tdrz = listOf(seg(0, 5000, "ignored"))
val out = Diarization.merge(quality, tdrz)
assertEquals(quality, out) // untouched, no wrong labels
}
@Test
fun `multiple turns flip back and forth`() {
val quality = listOf(
seg(0, 4000, "Eins"), seg(4100, 8000, "Zwei"), seg(8100, 12000, "Drei"),
)
val tdrz = listOf(
seg(0, 4000, "x", turn = true),
seg(4100, 8000, "y", turn = true),
)
val out = Diarization.merge(quality, tdrz)
assertEquals(listOf(1, 2, 1), out.map { speakerOf(it) })
}
@Test
fun `turn after the last segment is ignored`() {
val quality = listOf(seg(0, 5000, "Eins"))
val tdrz = listOf(seg(0, 5000, "ignored", turn = true))
val out = Diarization.merge(quality, tdrz)
assertEquals(1, out.size)
assertEquals("Sprecher 1: Eins", out[0].text)
}
@Test
fun `empty quality transcript stays empty`() {
assertTrue(Diarization.merge(emptyList(), listOf(seg(0, 1000, "x"))).isEmpty())
}
private fun speakerOf(s: WhisperEngine.Segment): Int =
s.text.removePrefix("Sprecher ").take(1).toInt()
}
server/whisper-server/Dockerfile
+4
View File
@@ -19,6 +19,10 @@ ARG WHISPER_TAG=v1.9.3
RUN git clone --depth 1 --branch ${WHISPER_TAG} \
https://github.com/ggml-org/whisper.cpp /src
# expose tinydiarize speaker_turn_next in verbose_json (see patch header)
COPY speaker-turn.patch /src/
RUN git -C /src apply speaker-turn.patch
RUN cmake -S /src -B /src/build -DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=OFF \
-DWHISPER_BUILD_EXAMPLES=ON \
server/whisper-server/docker-compose.yml
+46 -7
View File
@@ -1,14 +1,16 @@
# whisper.cpp inference server for meetrec, with Vulkan GPU support
# whisper.cpp inference servers for meetrec, with Vulkan GPU support
# (AMD Radeon AI PRO R9700), reachable over Tailscale.
#
# RENDER_GID=$(getent group render | cut -d: -f3) docker compose up -d --build
#
# Security: whisper.cpp's server has NO authentication. The port below is
# bound ONLY to the Tailscale interface (100.103.83.12), so the API is
# never exposed to the LAN or the internet. Tailscale must own that IP
# before the container starts, otherwise the bind fails — start order:
# tailscale first, then `docker compose up -d`. If you would rather
# tolerate LAN exposure, use "8085:8085" instead.
# Two services:
# whisper-server port 8085 — large-v3, quality transcripts
# whisper-server-tdrz port 8086 — small.en-tdrz, tinydiarize speaker
# turns (English-trained, 2-speaker, best-effort)
#
# The image is patched to expose `speaker_turn_next` per segment in
# verbose_json (speaker-turn.patch); clients merge the turn times onto
# the better transcript as "Sprecher 1/2" labels.
services:
whisper-server:
@@ -45,4 +47,41 @@ services:
interval: 30s
timeout: 5s
retries: 3
start_period: 60s
whisper-server-tdrz:
# tinydiarize variant: English-only small.en-tdrz, used ONLY for its
# speaker-turn timestamps (meetrec merges them onto the better
# transcript from the main server). 2-speaker detection, best-effort
# on non-English audio.
build: .
image: meetrec-whisper-server:latest
container_name: whisper-server-tdrz
restart: unless-stopped
environment:
MODEL: small.en-tdrz
MODEL_URL: https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-small.en-tdrz.bin
THREADS: 8
HOST: 0.0.0.0
PORT: 8086
TDRZ: 1
volumes:
- ./models:/models # shared with the main server (distinct files)
ports:
- "100.103.83.12:8086:8086" # tailscale-only
devices:
- /dev/dri:/dev/dri
group_add:
- video
- "${RENDER_GID:-110}"
healthcheck:
test: ["CMD-SHELL", "curl -s -o /dev/null http://localhost:8086/ || exit 1"]
interval: 30s
timeout: 5s
retries: 3
start_period: 60s
server/whisper-server/entrypoint.sh
+5 -1
View File
@@ -15,7 +15,8 @@ HOST="${HOST:-0.0.0.0}"
PORT="${PORT:-8085}"
MODEL_DIR="${MODEL_DIR:-/models}"
FILE="$MODEL_DIR/ggml-$MODEL.bin"
URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin"
# default to the ggerganov collection; TDRZ models live elsewhere
URL="${MODEL_URL:-https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin}"
# smallest supported model (ggml-tiny.bin) is ~75 MB
MIN_SIZE=50000000
@@ -58,6 +59,9 @@ GPU_FLAGS=""
if [ "${NO_GPU:-0}" = "1" ]; then
GPU_FLAGS="-ng"
fi
if [ "${TDRZ:-0}" = "1" ]; then
GPU_FLAGS="$GPU_FLAGS -tdrz"
fi
echo "starting whisper-server: model=$MODEL threads=$THREADS host=$HOST port=$PORT gpu=auto"
exec whisper-server \
server/whisper-server/speaker-turn.patch
+15
View File
@@ -0,0 +1,15 @@
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index b87ef27..9e13ceb 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -1090,6 +1090,10 @@ int main(int argc, char ** argv) {
segment["end"] = whisper_full_get_segment_t1(ctx, i) * 0.01;
}
+ if (params.tinydiarize) {
+ segment["speaker_turn_next"] = whisper_full_get_segment_speaker_turn_next(ctx, i);
+ }
+
if (params.diarize && pcmf32s.size() == 2) {
segment["speaker"] = estimate_diarization_speaker(
pcmf32s,