M3d: speaker labels via tinydiarize two-pass merge

- whisper-server stack: second container (port 8086) running the
  English-trained small.en-tdrz model with -tdrz; image patched
  (speaker-turn.patch) to expose speaker_turn_next per segment in
  verbose_json like the cli example does
- core/whisper Diarization: merges the tdrz pass's TURN TIMES onto the
  quality transcript as alternating 'Sprecher 1/2:' labels, splitting
  segments when a turn falls inside them; no turns detected = no
  labels (never mislabels); 6 unit tests
- RemoteWhisperEngine gains a diarize flag (sends tinydiarize=true,
  parses speaker_turn_next); WhisperEngine.Segment carries the flag
- RecorderService: optional second pass on the diarize server after the
  final pass; failures keep the unlabeled transcript
- Settings: Diarize server URL (persisted; empty disables)
- validated infrastructure locally: patched image builds, tdrz model
  downloads from akashmjn/tinydiarize-whisper.cpp, speaker_turn_next
  present in responses; synthetic espeak audio does not trigger the
  model's turn tokens — real two-person speech needed for the
  end-to-end check
This commit is contained in:
2026-09-08 17:07:31 +02:00
parent 6a400e842f
commit 76611af7e0
11 changed files with 289 additions and 21 deletions
+4
View File
@@ -19,6 +19,10 @@ ARG WHISPER_TAG=v1.9.3
RUN git clone --depth 1 --branch ${WHISPER_TAG} \
https://github.com/ggml-org/whisper.cpp /src
# expose tinydiarize speaker_turn_next in verbose_json (see patch header)
COPY speaker-turn.patch /src/
RUN git -C /src apply speaker-turn.patch
RUN cmake -S /src -B /src/build -DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=OFF \
-DWHISPER_BUILD_EXAMPLES=ON \
+46 -7
View File
@@ -1,14 +1,16 @@
# whisper.cpp inference server for meetrec, with Vulkan GPU support
# whisper.cpp inference servers for meetrec, with Vulkan GPU support
# (AMD Radeon AI PRO R9700), reachable over Tailscale.
#
# RENDER_GID=$(getent group render | cut -d: -f3) docker compose up -d --build
#
# Security: whisper.cpp's server has NO authentication. The port below is
# bound ONLY to the Tailscale interface (100.103.83.12), so the API is
# never exposed to the LAN or the internet. Tailscale must own that IP
# before the container starts, otherwise the bind fails — start order:
# tailscale first, then `docker compose up -d`. If you would rather
# tolerate LAN exposure, use "8085:8085" instead.
# Two services:
# whisper-server port 8085 — large-v3, quality transcripts
# whisper-server-tdrz port 8086 — small.en-tdrz, tinydiarize speaker
# turns (English-trained, 2-speaker, best-effort)
#
# The image is patched to expose `speaker_turn_next` per segment in
# verbose_json (speaker-turn.patch); clients merge the turn times onto
# the better transcript as "Sprecher 1/2" labels.
services:
whisper-server:
@@ -45,4 +47,41 @@ services:
interval: 30s
timeout: 5s
retries: 3
start_period: 60s
whisper-server-tdrz:
# tinydiarize variant: English-only small.en-tdrz, used ONLY for its
# speaker-turn timestamps (meetrec merges them onto the better
# transcript from the main server). 2-speaker detection, best-effort
# on non-English audio.
build: .
image: meetrec-whisper-server:latest
container_name: whisper-server-tdrz
restart: unless-stopped
environment:
MODEL: small.en-tdrz
MODEL_URL: https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-small.en-tdrz.bin
THREADS: 8
HOST: 0.0.0.0
PORT: 8086
TDRZ: 1
volumes:
- ./models:/models # shared with the main server (distinct files)
ports:
- "100.103.83.12:8086:8086" # tailscale-only
devices:
- /dev/dri:/dev/dri
group_add:
- video
- "${RENDER_GID:-110}"
healthcheck:
test: ["CMD-SHELL", "curl -s -o /dev/null http://localhost:8086/ || exit 1"]
interval: 30s
timeout: 5s
retries: 3
start_period: 60s
+5 -1
View File
@@ -15,7 +15,8 @@ HOST="${HOST:-0.0.0.0}"
PORT="${PORT:-8085}"
MODEL_DIR="${MODEL_DIR:-/models}"
FILE="$MODEL_DIR/ggml-$MODEL.bin"
URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin"
# default to the ggerganov collection; TDRZ models live elsewhere
URL="${MODEL_URL:-https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin}"
# smallest supported model (ggml-tiny.bin) is ~75 MB
MIN_SIZE=50000000
@@ -58,6 +59,9 @@ GPU_FLAGS=""
if [ "${NO_GPU:-0}" = "1" ]; then
GPU_FLAGS="-ng"
fi
if [ "${TDRZ:-0}" = "1" ]; then
GPU_FLAGS="$GPU_FLAGS -tdrz"
fi
echo "starting whisper-server: model=$MODEL threads=$THREADS host=$HOST port=$PORT gpu=auto"
exec whisper-server \
+15
View File
@@ -0,0 +1,15 @@
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index b87ef27..9e13ceb 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -1090,6 +1090,10 @@ int main(int argc, char ** argv) {
segment["end"] = whisper_full_get_segment_t1(ctx, i) * 0.01;
}
+ if (params.tinydiarize) {
+ segment["speaker_turn_next"] = whisper_full_get_segment_speaker_turn_next(ctx, i);
+ }
+
if (params.diarize && pcmf32s.size() == 2) {
segment["speaker"] = estimate_diarization_speaker(
pcmf32s,