diff --git a/README.md b/README.md index c87bead..4b338da 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,9 @@ Key options: | `--language` | force a language code, e.g. `en`, `de` (default: autodetect) | | `--device` | `auto` / `cpu` / `cuda` (default `auto`); faster-whisper only | | `--compute-type`| `int8` / `int8_float16` / `float16` / `float32` (default `int8`); faster-whisper only | +| `--agenda-file` | text file with agenda items, one per line — uploaded for the coverage check | +| `--diarize-url` | tinydiarize server URL → transcript gets `Sprecher 1/2:` labels (or `MEETREC_DIARIZE_URL`) | +| `--library-url` | meetrec-server URL → uploads the recording and triggers summary + agenda check (or `MEETREC_LIBRARY_URL`) | `Ctrl+C` stops recording and runs the final transcription. @@ -138,6 +141,22 @@ Notes on `whisper-server`: - Run your own with the Docker Compose stack in `server/whisper-server/` — Vulkan GPU (AMD/NVIDIA/Intel), model auto-download, Tailscale-only binding. - The server has no authentication: keep it on Tailscale or behind a VPN/firewall. - `verbose_json` reports language names ("german") rather than ISO codes. + +## Meeting protocol (library, summary, agenda, speakers) + +The desktop app has feature parity with the Android app's M3 set: + +```sh +MEETREC_SERVER_URL=http://100.103.83.12:8085 \ +MEETREC_LIBRARY_URL=http://100.103.83.12:8090 \ +MEETREC_DIARIZE_URL=http://100.103.83.12:8086 \ + meetrec-cli --engine whisper-server --agenda-file agenda.txt -o meeting +``` + +- `--library-url` uploads the WAV + transcripts after the final pass; the server then generates the German summary (Ollama, gemma4:12b) and checks which agenda items were discussed (browse everything in the Android app's Library tab). +- `--agenda-file` supplies the agenda items (one per line). +- `--diarize-url` adds a second tinydiarize pass whose speaker-turn times are merged onto the transcript as `Sprecher 1:/Sprecher 2:` labels (2 speakers, best-effort on non-English audio; failures keep the transcript unlabeled). +- The GUI has the same options as fields (Library URL, Diarize URL, agenda editor). - `--device` / `--compute-type` do not apply (GPU vs. CPU is decided by the whisper.cpp build). ## Project layout diff --git a/meetrec.py b/meetrec.py index 89ea9dc..c752f1c 100644 --- a/meetrec.py +++ b/meetrec.py @@ -273,17 +273,28 @@ def live_worker(model, rec: Recorder, args, live_srt_path: str, def final_transcribe(model, wav_path: str, out_stem: str, language, - on_info=None, on_segment=None, on_done=None): - """Full-pass transcription of the whole recording. + on_info=None, on_segment=None, on_done=None, + diarize_url=None, agenda=None, library_url=None, + device_name="desktop"): + """Full-pass transcription of the whole recording, with optional + speaker labeling (tinydiarize server) and library upload. on_info(msg), on_segment(start, end, text) and on_done() are optional - callbacks; when absent the same progress is printed to stdout.""" + callbacks; when absent the same progress is printed to stdout. + Returns (items, info): items are (start, end, text) tuples of the + final — possibly speaker-labeled — transcript.""" def info(msg): if on_info is not None: on_info(msg) else: print(msg, flush=True) + def emit(a, b, text): + if on_segment is not None: + on_segment(a, b, text) + else: + print(f"[{fmt_hms(a)}] {text}", flush=True) + info("\nRunning final transcription (this can take a while)...") segments, info_ = model.transcribe( wav_path, beam_size=5, vad_filter=True, language=language, @@ -293,13 +304,26 @@ def final_transcribe(model, wav_path: str, out_stem: str, language, f"duration: {fmt_hms(info_.duration)}") items = [] - for s in segments: # generator: streams as it decodes - text = s.text.strip() - items.append((s.start, s.end, text)) - if on_segment is not None: - on_segment(s.start, s.end, text) - else: - print(f"[{fmt_hms(s.start)}] {text}", flush=True) + if diarize_url: + for s in segments: # collect first: labels need the whole pass + items.append((s.start, s.end, s.text.strip())) + info("Running speaker-diarization pass (tinydiarize server)...") + try: + tdrz = WhisperServerEngine(diarize_url, diarize=True) + turns = tdrz.transcribe(wav_path, language=language) + labeled = diarize_merge( + [SimpleNamespace(start=a, end=b, text=t) for a, b, t in items], + turns) + items = [(s.start, s.end, s.text) for s in labeled] + except Exception as e: + info(f"diarize pass failed (continuing unlabeled): {e}") + for a, b, text in items: + emit(a, b, text) + else: + for s in segments: # generator: streams as it decodes + text = s.text.strip() + items.append((s.start, s.end, text)) + emit(s.start, s.end, text) full = " ".join(t for _, _, t in items) with open(out_stem + ".txt", "w") as f: @@ -313,8 +337,21 @@ def final_transcribe(model, wav_path: str, out_stem: str, language, for a, b, t in items], }, f, ensure_ascii=False, indent=2) info(f"\nwrote: {out_stem}.wav {out_stem}.txt {out_stem}.srt {out_stem}.json") + + if library_url: + info("Uploading recording to the library...") + try: + rid = upload_recording( + library_url, wav_path, out_stem, info_.duration, + getattr(info_, "language", None), device_name, agenda or []) + info(f"Uploaded to library as {rid} — summary and agenda check " + "follow automatically on the server.") + except Exception as e: + info(f"upload failed: {e}") + if on_done is not None: on_done() + return items, info_ # -------------------------------------------------------------------------- @@ -499,27 +536,29 @@ class WhisperCppEngine: MULTIPART_BOUNDARY = "meetrec-9f3e1c7a5b2d" -def _multipart_body(fields: dict, file_name: str, file_obj) -> bytes: - """Build a multipart/form-data body with stdlib only.""" +def _multipart_body(fields: dict, files) -> bytes: + """Build a multipart/form-data body with stdlib only. + files: list of (field_name, filename, file_obj) tuples.""" parts = [] for name, value in fields.items(): parts.append( f"--{MULTIPART_BOUNDARY}\r\n" f'Content-Disposition: form-data; name="{name}"\r\n\r\n' f"{value}\r\n".encode()) - parts.append( - f"--{MULTIPART_BOUNDARY}\r\n" - f'Content-Disposition: form-data; name="file"; ' - f'filename="{file_name}"\r\n' - f"Content-Type: audio/wav\r\n\r\n".encode()) - parts.append(file_obj.read()) - parts.append(b"\r\n") + for name, fname, fh in files: + parts.append( + f"--{MULTIPART_BOUNDARY}\r\n" + f'Content-Disposition: form-data; name="{name}"; ' + f'filename="{fname}"\r\n' + f"Content-Type: application/octet-stream\r\n\r\n".encode()) + parts.append(fh.read()) + parts.append(b"\r\n") parts.append(f"--{MULTIPART_BOUNDARY}--\r\n".encode()) return b"".join(parts) class WhisperServerEngine: - """Transcription via a remote whisper.cpp server (whisper-server). + """Transcription via a remote whisper.cpp server (POST /inference). Use the Docker Compose stack in server/whisper-server/ to run one (Vulkan GPU on AMD, or CPU). The model lives on the server, so @@ -529,10 +568,15 @@ class WhisperServerEngine: vad_filter/vad_parameters are likewise accepted but unused; the server runs its own pipeline. Note: verbose_json reports language names like "german" rather than ISO codes. + + With diarize=True (a tinydiarize server, see whisper-server-tdrz) + segments carry .speaker_turn_next and their text is NOT used for + labeling — use diarize_merge() for that. """ - def __init__(self, server_url: str): + def __init__(self, server_url: str, diarize: bool = False): self.server_url = server_url.rstrip("/") + self.diarize = diarize self.model_name = None # the server owns the model def transcribe(self, audio, beam_size=5, vad_filter=False, @@ -543,11 +587,15 @@ class WhisperServerEngine: else: wav = os.path.join(td, "in.wav") write_wav_float(wav, audio) - with open(wav, "rb") as f: - body = _multipart_body( - {"response_format": "verbose_json", - "language": language or "auto"}, - "audio.wav", f) + files = [("file", "audio.wav", open(wav, "rb"))] + fields = {"response_format": "verbose_json", + "language": language or "auto"} + if self.diarize: + fields["tinydiarize"] = "true" + try: + body = _multipart_body(fields, files) + finally: + files[0][2].close() req = urllib.request.Request( self.server_url + "/inference", data=body, headers={"Content-Type": @@ -563,7 +611,8 @@ class WhisperServerEngine: f"{self.server_url}: {e.reason}") from None segs = [SimpleNamespace(start=s["start"], end=s["end"], - text=s["text"].strip()) + text=s["text"].strip(), + speaker_turn_next=bool(s.get("speaker_turn_next", False))) for s in data.get("segments", []) if s.get("text", "").strip()] info = SimpleNamespace( @@ -574,6 +623,76 @@ class WhisperServerEngine: return segs, info +def diarize_merge(segments, turn_segments): + """Label segments with alternating "Sprecher 1/2:" using the TURN + TIMES of a tinydiarize pass; segments in which a turn falls are split. + Input is returned unchanged when no turns were detected (never + mislabels). Mirrors the Android app's Diarization object.""" + turns = sorted(t.end for t in turn_segments + if getattr(t, "speaker_turn_next", False)) + if not turns: + return list(segments) + + def seg(a, b, text): + return SimpleNamespace(start=a, end=b, text=text) + + speaker, ti = 1, 0 + out = [] + for s in segments: + while ti < len(turns) and turns[ti] <= s.start: + speaker = 3 - speaker + ti += 1 + start, text = s.start, s.text + while ti < len(turns) and turns[ti] < s.end: + t = turns[ti] + frac = (t - s.start) / max(s.end - s.start, 1e-9) + cut = int(len(text) * frac) + out.append(seg(start, t, f"Sprecher {speaker}: {text[:cut].strip()}")) + speaker = 3 - speaker + ti += 1 + start, text = t, text[cut:] + out.append(seg(start, s.end, f"Sprecher {speaker}: {text.strip()}")) + return out + + +def upload_recording(library_url: str, wav_path: str, out_stem: str, + duration: float, language, device: str, agenda) -> str: + """Upload a recording bundle to the meetrec-server library; + returns the new recording id. The server then generates the summary + and agenda coverage automatically (Ollama).""" + files = [] + for ext in ("wav", "txt", "srt", "json"): + p = wav_path if ext == "wav" else out_stem + "." + ext + if os.path.isfile(p): + files.append((ext, "meeting." + ext, open(p, "rb"))) + started = time.strftime("%Y%m%d-%H%M%S", + time.localtime(os.path.getmtime(wav_path))) + fields = { + "started_at": started, + "duration_ms": int(duration * 1000), + "language": language or "", + "device": device, + "agenda": json.dumps(list(agenda or []), ensure_ascii=False), + } + try: + body = _multipart_body(fields, files) + finally: + for _, _, fh in files: + fh.close() + req = urllib.request.Request( + library_url.rstrip("/") + "/api/recordings", data=body, method="POST", + headers={"Content-Type": "multipart/form-data; boundary=" + MULTIPART_BOUNDARY}) + try: + with urllib.request.urlopen(req, timeout=300) as r: + return json.load(r)["id"] + except urllib.error.HTTPError as e: + raise RuntimeError(f"library upload failed: {e.code} " + f"{e.read()[:200]!r}") from None + except urllib.error.URLError as e: + raise RuntimeError(f"cannot reach library at {library_url}: " + f"{e.reason}") from None + + def load_engine(model_name, engine="faster-whisper", device="auto", compute_type="int8", server_url=None): """Create a transcription engine. See the Engines section above.""" @@ -630,6 +749,18 @@ def main(): "(default: int8); faster-whisper only") ap.add_argument("--list-sources", action="store_true", help="list available input devices and exit") + ap.add_argument("--agenda-file", default=None, metavar="PATH", + help="text file with agenda items, one per line — " + "uploaded with the recording for the coverage " + "check (see --library-url)") + ap.add_argument("--diarize-url", default=os.environ.get("MEETREC_DIARIZE_URL"), + help="tinydiarize whisper-server URL for speaker labels " + "(Sprecher 1/2), e.g. http://100.103.83.12:8086 " + "(or set MEETREC_DIARIZE_URL)") + ap.add_argument("--library-url", default=os.environ.get("MEETREC_LIBRARY_URL"), + help="meetrec-server URL; uploads the recording after " + "transcription and triggers summary + agenda check " + "on the server (or set MEETREC_LIBRARY_URL)") args = ap.parse_args() if args.list_sources: @@ -678,7 +809,13 @@ def main(): rec.stop() if worker: worker.join(timeout=60) - final_transcribe(model, args.output + ".wav", args.output, args.language) + agenda = [] + if args.agenda_file: + with open(args.agenda_file) as f: + agenda = [ln.strip() for ln in f if ln.strip()] + final_transcribe(model, args.output + ".wav", args.output, args.language, + diarize_url=args.diarize_url, agenda=agenda, + library_url=args.library_url) if __name__ == "__main__": diff --git a/meetrec_gui.py b/meetrec_gui.py index 88b9325..0cfdd03 100644 --- a/meetrec_gui.py +++ b/meetrec_gui.py @@ -23,13 +23,14 @@ from types import SimpleNamespace from PySide6.QtCore import Qt, QThread, QTimer, Signal from PySide6.QtWidgets import ( QApplication, QCheckBox, QComboBox, QHBoxLayout, QLabel, QLineEdit, - QMessageBox, QProgressBar, QPushButton, QSpinBox, QTextEdit, - QVBoxLayout, QWidget, + QMessageBox, QPlainTextEdit, QProgressBar, QPushButton, QSpinBox, + QTextEdit, QVBoxLayout, QWidget, ) import sounddevice as sd import meetrec +import os from meetrec import ( Recorder, final_transcribe, @@ -77,12 +78,16 @@ class FinalWorker(QThread): done = Signal(str) # output stem failed = Signal(str) - def __init__(self, model, wav_path, out_stem, language, parent=None): + def __init__(self, model, wav_path, out_stem, language, + diarize_url=None, agenda=None, library_url=None, parent=None): super().__init__(parent) self.model = model self.wav_path = wav_path self.out_stem = out_stem self.language = language + self.diarize_url = diarize_url + self.agenda = agenda + self.library_url = library_url def run(self): try: @@ -91,6 +96,9 @@ class FinalWorker(QThread): on_info=self.info.emit, on_segment=self.segment.emit, on_done=lambda: self.done.emit(self.out_stem), + diarize_url=self.diarize_url, + agenda=self.agenda, + library_url=self.library_url, ) except Exception as e: self.failed.emit(str(e)) @@ -206,6 +214,22 @@ class MeetRecWindow(QWidget): r2c.addWidget(self.server_url, 1) root.addLayout(r2c) + # row 2d: library + diarization servers + r2d = QHBoxLayout() + self.library_url = QLineEdit(os.environ.get("MEETREC_LIBRARY_URL", "")) + self.library_url.setPlaceholderText("Library URL (meetrec-server)") + self.diarize_url = QLineEdit(os.environ.get("MEETREC_DIARIZE_URL", "")) + self.diarize_url.setPlaceholderText("Diarize URL (Sprecher 1/2)") + r2d.addWidget(self.library_url, 1) + r2d.addWidget(self.diarize_url, 1) + root.addLayout(r2d) + + # agenda items (one per line), used for the server-side coverage check + self.agenda_edit = QPlainTextEdit() + self.agenda_edit.setPlaceholderText("Agenda (one item per line)") + self.agenda_edit.setFixedHeight(64) + root.addWidget(self.agenda_edit) + # row 3: live options + output name r3 = QHBoxLayout() r3.addWidget(QLabel("Live:")) @@ -354,6 +378,10 @@ class MeetRecWindow(QWidget): output=self.out.text().strip() or "meeting", live=live, language=None if lang in ("", "autodetect") else lang, + library_url=self.library_url.text().strip() or None, + diarize_url=self.diarize_url.text().strip() or None, + agenda=[ln.strip() for ln in self.agenda_edit.toPlainText() + .splitlines() if ln.strip()], ) self._set_inputs_enabled(False) self.transcript.clear() @@ -455,6 +483,9 @@ class MeetRecWindow(QWidget): self._args.output + ".wav", self._args.output, self._args.language, + diarize_url=self._args.diarize_url, + agenda=self._args.agenda, + library_url=self._args.library_url, parent=self) self.worker.info.connect(lambda m: self.status.setText(m)) self.worker.segment.connect(self._final_segment)