M3d: speaker labels via tinydiarize two-pass merge
- whisper-server stack: second container (port 8086) running the English-trained small.en-tdrz model with -tdrz; image patched (speaker-turn.patch) to expose speaker_turn_next per segment in verbose_json like the cli example does - core/whisper Diarization: merges the tdrz pass's TURN TIMES onto the quality transcript as alternating 'Sprecher 1/2:' labels, splitting segments when a turn falls inside them; no turns detected = no labels (never mislabels); 6 unit tests - RemoteWhisperEngine gains a diarize flag (sends tinydiarize=true, parses speaker_turn_next); WhisperEngine.Segment carries the flag - RecorderService: optional second pass on the diarize server after the final pass; failures keep the unlabeled transcript - Settings: Diarize server URL (persisted; empty disables) - validated infrastructure locally: patched image builds, tdrz model downloads from akashmjn/tinydiarize-whisper.cpp, speaker_turn_next present in responses; synthetic espeak audio does not trigger the model's turn tokens — real two-person speech needed for the end-to-end check
This commit is contained in:
@@ -19,6 +19,10 @@ ARG WHISPER_TAG=v1.9.3
|
||||
RUN git clone --depth 1 --branch ${WHISPER_TAG} \
|
||||
https://github.com/ggml-org/whisper.cpp /src
|
||||
|
||||
# expose tinydiarize speaker_turn_next in verbose_json (see patch header)
|
||||
COPY speaker-turn.patch /src/
|
||||
RUN git -C /src apply speaker-turn.patch
|
||||
|
||||
RUN cmake -S /src -B /src/build -DCMAKE_BUILD_TYPE=Release \
|
||||
-DBUILD_SHARED_LIBS=OFF \
|
||||
-DWHISPER_BUILD_EXAMPLES=ON \
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
# whisper.cpp inference server for meetrec, with Vulkan GPU support
|
||||
# whisper.cpp inference servers for meetrec, with Vulkan GPU support
|
||||
# (AMD Radeon AI PRO R9700), reachable over Tailscale.
|
||||
#
|
||||
# RENDER_GID=$(getent group render | cut -d: -f3) docker compose up -d --build
|
||||
#
|
||||
# Security: whisper.cpp's server has NO authentication. The port below is
|
||||
# bound ONLY to the Tailscale interface (100.103.83.12), so the API is
|
||||
# never exposed to the LAN or the internet. Tailscale must own that IP
|
||||
# before the container starts, otherwise the bind fails — start order:
|
||||
# tailscale first, then `docker compose up -d`. If you would rather
|
||||
# tolerate LAN exposure, use "8085:8085" instead.
|
||||
# Two services:
|
||||
# whisper-server port 8085 — large-v3, quality transcripts
|
||||
# whisper-server-tdrz port 8086 — small.en-tdrz, tinydiarize speaker
|
||||
# turns (English-trained, 2-speaker, best-effort)
|
||||
#
|
||||
# The image is patched to expose `speaker_turn_next` per segment in
|
||||
# verbose_json (speaker-turn.patch); clients merge the turn times onto
|
||||
# the better transcript as "Sprecher 1/2" labels.
|
||||
|
||||
services:
|
||||
whisper-server:
|
||||
@@ -45,4 +47,41 @@ services:
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
whisper-server-tdrz:
|
||||
# tinydiarize variant: English-only small.en-tdrz, used ONLY for its
|
||||
# speaker-turn timestamps (meetrec merges them onto the better
|
||||
# transcript from the main server). 2-speaker detection, best-effort
|
||||
# on non-English audio.
|
||||
build: .
|
||||
image: meetrec-whisper-server:latest
|
||||
container_name: whisper-server-tdrz
|
||||
restart: unless-stopped
|
||||
|
||||
environment:
|
||||
MODEL: small.en-tdrz
|
||||
MODEL_URL: https://huggingface.co/akashmjn/tinydiarize-whisper.cpp/resolve/main/ggml-small.en-tdrz.bin
|
||||
THREADS: 8
|
||||
HOST: 0.0.0.0
|
||||
PORT: 8086
|
||||
TDRZ: 1
|
||||
|
||||
volumes:
|
||||
- ./models:/models # shared with the main server (distinct files)
|
||||
|
||||
ports:
|
||||
- "100.103.83.12:8086:8086" # tailscale-only
|
||||
|
||||
devices:
|
||||
- /dev/dri:/dev/dri
|
||||
group_add:
|
||||
- video
|
||||
- "${RENDER_GID:-110}"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -s -o /dev/null http://localhost:8086/ || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
@@ -15,7 +15,8 @@ HOST="${HOST:-0.0.0.0}"
|
||||
PORT="${PORT:-8085}"
|
||||
MODEL_DIR="${MODEL_DIR:-/models}"
|
||||
FILE="$MODEL_DIR/ggml-$MODEL.bin"
|
||||
URL="https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin"
|
||||
# default to the ggerganov collection; TDRZ models live elsewhere
|
||||
URL="${MODEL_URL:-https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin}"
|
||||
|
||||
# smallest supported model (ggml-tiny.bin) is ~75 MB
|
||||
MIN_SIZE=50000000
|
||||
@@ -58,6 +59,9 @@ GPU_FLAGS=""
|
||||
if [ "${NO_GPU:-0}" = "1" ]; then
|
||||
GPU_FLAGS="-ng"
|
||||
fi
|
||||
if [ "${TDRZ:-0}" = "1" ]; then
|
||||
GPU_FLAGS="$GPU_FLAGS -tdrz"
|
||||
fi
|
||||
|
||||
echo "starting whisper-server: model=$MODEL threads=$THREADS host=$HOST port=$PORT gpu=auto"
|
||||
exec whisper-server \
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
|
||||
index b87ef27..9e13ceb 100644
|
||||
--- a/examples/server/server.cpp
|
||||
+++ b/examples/server/server.cpp
|
||||
@@ -1090,6 +1090,10 @@ int main(int argc, char ** argv) {
|
||||
segment["end"] = whisper_full_get_segment_t1(ctx, i) * 0.01;
|
||||
}
|
||||
|
||||
+ if (params.tinydiarize) {
|
||||
+ segment["speaker_turn_next"] = whisper_full_get_segment_speaker_turn_next(ctx, i);
|
||||
+ }
|
||||
+
|
||||
if (params.diarize && pcmf32s.size() == 2) {
|
||||
segment["speaker"] = estimate_diarization_speaker(
|
||||
pcmf32s,
|
||||
Reference in New Issue
Block a user