From ef59fc0fb4e111e03885327259555a519e24720a Mon Sep 17 00:00:00 2001 From: Florian Egger Date: Mon, 7 Sep 2026 12:17:40 +0200 Subject: [PATCH] Add dockerized whisper.cpp inference server (Vulkan GPU, Tailscale) - server/whisper-server: compose stack built from the pinned whisper.cpp v1.9.3 release, same as the Android JNI layer - Vulkan GPU backend (AMD Radeon AI PRO R9700 / RADV) with transparent CPU fallback and NO_GPU override; GGML models auto-download on first start (MODEL env, default large-v3) - API bound to the Tailscale interface only (100.103.83.12:8080) since whisper-server has no authentication; render-group GID passthrough for /dev/dri - validated locally: image builds, entrypoint downloads tiny, POST /inference returns verbose_json with language + segments; GPU-less fallback confirmed --- server/whisper-server/.dockerignore | 7 +++ server/whisper-server/Dockerfile | 43 ++++++++++++++ server/whisper-server/README.md | 71 ++++++++++++++++++++++++ server/whisper-server/docker-compose.yml | 48 ++++++++++++++++ server/whisper-server/entrypoint.sh | 35 ++++++++++++ 5 files changed, 204 insertions(+) create mode 100644 server/whisper-server/.dockerignore create mode 100644 server/whisper-server/Dockerfile create mode 100644 server/whisper-server/README.md create mode 100644 server/whisper-server/docker-compose.yml create mode 100644 server/whisper-server/entrypoint.sh diff --git a/server/whisper-server/.dockerignore b/server/whisper-server/.dockerignore new file mode 100644 index 0000000..761c7ba --- /dev/null +++ b/server/whisper-server/.dockerignore @@ -0,0 +1,7 @@ +models/ +*.bin +*.part +.git +Dockerfile +docker-compose.yml +README.md \ No newline at end of file diff --git a/server/whisper-server/Dockerfile b/server/whisper-server/Dockerfile new file mode 100644 index 0000000..e90f28a --- /dev/null +++ b/server/whisper-server/Dockerfile @@ -0,0 +1,43 @@ +# whisper.cpp inference server for meetrec. +# +# Built from the same pinned whisper.cpp release (v1.9.3) as the Android +# app's JNI layer, compiled statically WITH the Vulkan GPU backend for +# AMD GPUs (Radeon AI PRO R9700 / RDNA, also NVIDIA/Intel). Without a GPU +# present it transparently falls back to CPU. +# +# Note: ggml builds with -march=native by default, so build the image ON +# the machine that will run it (docker compose build on the server). + +FROM debian:12-slim AS build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git cmake build-essential ca-certificates curl \ + libvulkan-dev glslang-tools \ + && rm -rf /var/lib/apt/lists/* + +ARG WHISPER_TAG=v1.9.3 +RUN git clone --depth 1 --branch ${WHISPER_TAG} \ + https://github.com/ggml-org/whisper.cpp /src + +RUN cmake -S /src -B /src/build -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DWHISPER_BUILD_EXAMPLES=ON \ + -DWHISPER_BUILD_TESTS=OFF \ + -DWHISPER_BUILD_SERVER=ON \ + -DWHISPER_VULKAN=ON \ + && cmake --build /src/build --target whisper-server -j"$(nproc)" + +FROM debian:12-slim + +# libvulkan1 + mesa-vulkan-drivers: the AMD RADV Vulkan driver used by +# the R9700. GPU access is granted via /dev/dri in docker-compose.yml. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 ca-certificates curl libvulkan1 mesa-vulkan-drivers \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /src/build/bin/whisper-server /usr/local/bin/whisper-server +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] \ No newline at end of file diff --git a/server/whisper-server/README.md b/server/whisper-server/README.md new file mode 100644 index 0000000..197206d --- /dev/null +++ b/server/whisper-server/README.md @@ -0,0 +1,71 @@ +# whisper.cpp server (Docker Compose) + +The transcription backend for meetrec clients: the same pinned whisper.cpp +release (v1.9.3) as the Android app, exposed as an HTTP inference API with +**Vulkan GPU support** (AMD Radeon AI PRO R9700) and reachable over +Tailscale at `100.103.83.12:8080`. + +The Fairphone 6 transcribes at roughly 0.6–0.8× realtime on-device; the +R9700 (Strix Halo, RDNA 3.5, ~256 GB/s shared memory) is bandwidth-bound +friendly for Whisper — expect large-v3 at many times realtime. + +## Start + +```sh +RENDER_GID=$(getent group render | cut -d: -f3) docker compose up -d --build +docker compose logs -f # watch the model download, then "running" +``` + +The `RENDER_GID` lookup passes the host's render group into the container +so the GPU device is accessible. Verify the GPU is actually used from the +startup log — it should print `ggml_vulkan: Found 1 Vulkan devices` (and +`VULKAN = 1` in the system info); if the GPU is unavailable the server +transparently falls back to CPU. + +Configuration lives in `docker-compose.yml`: + +| Env | Default | Meaning | +| ---------- | --------- | ------------------------------------------ | +| `MODEL` | `large-v3`| `tiny`/`base`/`small`/`medium`/`large-v3` (downloaded to `./models` on first start) | +| `THREADS` | `8` | CPU threads per inference | +| `PORT` | `8080` | Port inside the container | + +## Try it + +```sh +curl http://100.103.83.12:8080/inference \ + -F file=@meeting.wav \ + -F response_format=verbose_json \ + -F language=auto +``` + +`POST /inference` accepts multipart fields `file` (PCM WAV, any rate), +`language` (`auto` supported), `response_format` +(`text`/`json`/`srt`/`vtt`/`verbose_json`), `temperature`. With +`verbose_json` the response carries the detected language and segments +with `start`/`end`/`text` (seconds). + +Notes: + +- **Beam size is a server-start setting** (v1.9.3 has no per-request + override), so the live/final beam split of the clients doesn't apply + here — one beam for all requests. +- The server also has a `/load` endpoint to swap models at runtime. +- **No authentication**: the compose file binds `100.103.83.12:8080` + (Tailscale interface only) for that reason. Do not switch this to + `0.0.0.0` unless the host is otherwise firewalled. +- The image builds with CPU feature auto-detection (`-march=native`): + build it on the machine that runs it (`--build` from the server, not + by exporting an image from another host). +- **GPU backend**: Vulkan via the RADV driver (mesa-vulkan-drivers in the + image). For maximum performance a ROCm/HIP build is the alternative + (heavier image, needs a ROCm base image and `gfx1151` target support + for Strix Halo) — add later if Vulkan benchmarks are insufficient. +- `NO_GPU=1` in the environment forces CPU-only inference. + +## Client status + +- Desktop `meetrec`: a `whisper-server` engine is planned + (`--engine whisper-server --server-url http://100.103.83.12:8080`). +- Android app: a remote engine option is planned (phone records, server + transcribes; local JNI stays as the offline fallback). \ No newline at end of file diff --git a/server/whisper-server/docker-compose.yml b/server/whisper-server/docker-compose.yml new file mode 100644 index 0000000..5e3018d --- /dev/null +++ b/server/whisper-server/docker-compose.yml @@ -0,0 +1,48 @@ +# whisper.cpp inference server for meetrec, with Vulkan GPU support +# (AMD Radeon AI PRO R9700), reachable over Tailscale. +# +# RENDER_GID=$(getent group render | cut -d: -f3) docker compose up -d --build +# +# Security: whisper.cpp's server has NO authentication. The port below is +# bound ONLY to the Tailscale interface (100.103.83.12), so the API is +# never exposed to the LAN or the internet. Tailscale must own that IP +# before the container starts, otherwise the bind fails — start order: +# tailscale first, then `docker compose up -d`. If you would rather +# tolerate LAN exposure, use "8080:8080" instead. + +services: + whisper-server: + build: . + image: meetrec-whisper-server:latest + container_name: whisper-server + restart: unless-stopped + + environment: + # tiny | base | small | medium | large-v3 (downloads on first start) + MODEL: large-v3 + THREADS: 8 + HOST: 0.0.0.0 # inside the container; the host bind is below + PORT: 8080 + # NO_GPU: 1 # force CPU if the GPU ever misbehaves + + volumes: + - ./models:/models # keep downloads across container rebuilds + + ports: + - "100.103.83.12:8080:8080" # tailscale-only; see note above + + # GPU passthrough: the render group must match the host's GID, hence + # the RENDER_GID env var in the comment at the top of this file. + devices: + - /dev/dri:/dev/dri + group_add: + - video + - "${RENDER_GID:-110}" + + healthcheck: + # any HTTP response counts as healthy (the server has no /health) + test: ["CMD-SHELL", "curl -s -o /dev/null http://localhost:8080/ || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s \ No newline at end of file diff --git a/server/whisper-server/entrypoint.sh b/server/whisper-server/entrypoint.sh new file mode 100644 index 0000000..72a20fc --- /dev/null +++ b/server/whisper-server/entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Downloads the configured GGML model on first start, then runs whisper-server. +# Uses the GPU (Vulkan) automatically when one is present; set NO_GPU=1 +# to force CPU. +set -eu + +MODEL="${MODEL:-small}" +THREADS="${THREADS:-4}" +HOST="${HOST:-0.0.0.0}" +PORT="${PORT:-8080}" +MODEL_DIR="${MODEL_DIR:-/models}" +FILE="$MODEL_DIR/ggml-$MODEL.bin" + +mkdir -p "$MODEL_DIR" +if [ ! -s "$FILE" ]; then + echo "downloading ggml-$MODEL.bin to $FILE ..." + curl -fL --retry 3 -C - -o "$FILE.part" \ + "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$MODEL.bin" + mv "$FILE.part" "$FILE" + echo "done." +fi + +GPU_FLAGS="" +if [ "${NO_GPU:-0}" = "1" ]; then + GPU_FLAGS="-ng" +fi + +echo "starting whisper-server: model=$MODEL threads=$THREADS host=$HOST port=$PORT gpu=auto" +exec whisper-server \ + -m "$FILE" \ + -l auto \ + -t "$THREADS" \ + $GPU_FLAGS \ + --host "$HOST" \ + --port "$PORT" \ No newline at end of file