M1+M2: RAG-Pipeline mit verbindlichem Grounding

agent/-Paket: Ingest (601 Layer-2-Eintraege -> 3005 Chunks, FTS5-BM25 +
Vektoren-Cache), Hybrid-Retrieval (RRF, Stand-Boost, cross_ref-Erweiterung),
Ollama-Client (embed/chat, think-Flag-Fallback, kurzes Connect-Budget),
Systemprompt mit Zitierpflicht, Post-Validierung (zitierte IDs gemaess
Retrieved-Set, 1x Regenerierung, dann Verweigerung), FastAPI (/ask, /health,
/reindex), CLI, Goldset (31 Fragen, IDs gegen kb.json verifiziert, inkl.
ATZ-Konfliktfall + 4 Verweigerungsfaelle), Eval-Suite, Test-Chat.

41 Offline-Tests gruen. Baseline BM25-only: Hit-Rate 0,871 / Recall@8 0,855 /
MRR 0,476. Hybrid-Messung, Antwortmodus-Eval und Modell-Bake-off (M3) auf
dem Host ausstaendig (Ollama aus der Zed-Sandbox nicht erreichbar).

MEMORY.md und planung.md Umsetzungsstand aktualisiert.

agent/kb.py

@@ -0,0 +1,164 @@
+"""Layer-2-Wissensbasis: Parsing (Frontmatter + H2-Sektionen) und kb.json-Gate.
+
+Die Layer-2-Frontmatter ist die Single Source of Truth; kb.json ist deren
+generierte, validierte Projektion. Der Gate bricht den Ingest bei Abweichung
+ab (Fehlermeldung nennt die Regenerierung der Registry als nächsten Schritt).
+"""
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+ID_RE = re.compile(r"^(lb|wk)-[a-z0-9]+-\d+$")
+STAND_RE = re.compile(r"^\d{4}-\d{2}$")
+REQUIRED_KEYS = (
+    "id", "batch", "title", "work", "chapter", "topic", "author",
+    "stand", "source", "legal_bases", "tags", "cross_refs",
+)
+
+
+class KbValidationError(Exception):
+    """Wissensbasis oder Registry ist inkonsistent — Ingest wird abgebrochen."""
+
+
+@dataclass
+class Section:
+    title: str
+    text: str
+
+
+@dataclass
+class KBEntry:
+    id: str
+    title: str
+    work: str
+    chapter: str
+    topic: str
+    author: str
+    stand: str
+    batch: int
+    source: dict
+    legal_bases: list
+    tags: list
+    cross_refs: list
+    path: Path
+    sections: list = field(default_factory=list)
+
+
+def parse_frontmatter(raw: str) -> tuple[dict, str]:
+    lines = raw.splitlines()
+    if not lines or lines[0].strip() != "---":
+        raise KbValidationError("missing frontmatter delimiter '---'")
+    for i in range(1, len(lines)):
+        if lines[i].strip() == "---":
+            meta = yaml.safe_load("\n".join(lines[1:i]))
+            body = "\n".join(lines[i + 1:])
+            break
+    else:
+        raise KbValidationError("unterminated frontmatter")
+    if not isinstance(meta, dict):
+        raise KbValidationError("frontmatter is not a mapping")
+    return meta, body
+
+
+def split_sections(body: str) -> list[Section]:
+    """H2-Sektionen als Chunks; H1-Titel und Quellzeile fallen weg."""
+    sections: list[Section] = []
+    current_title: str | None = None
+    current: list[str] = []
+    for line in body.splitlines():
+        if line.startswith("## "):
+            if current_title is not None:
+                sections.append(Section(current_title, "\n".join(current).strip()))
+            current_title = line[3:].strip()
+            current = []
+        elif line.startswith("# "):
+            continue
+        elif current_title is not None:
+            current.append(line)
+    if current_title is not None:
+        sections.append(Section(current_title, "\n".join(current).strip()))
+    return [s for s in sections if s.text]
+
+
+def load_entry(path: Path) -> KBEntry:
+    meta, body = parse_frontmatter(path.read_text(encoding="utf-8"))
+    missing = [k for k in REQUIRED_KEYS if k not in meta]
+    if missing:
+        raise KbValidationError(f"{path.name}: missing frontmatter keys {missing}")
+    for key in ("id", "title", "work", "chapter", "topic", "author", "stand"):
+        if not isinstance(meta[key], str) or not meta[key].strip():
+            raise KbValidationError(f"{path.name}: empty '{key}'")
+    if not ID_RE.match(meta["id"]):
+        raise KbValidationError(f"{path.name}: invalid id '{meta['id']}'")
+    if not STAND_RE.match(meta["stand"]):
+        raise KbValidationError(f"{path.name}: stand '{meta['stand']}' not YYYY-MM")
+    if not isinstance(meta["source"], dict) or not {"pdf", "text"} <= set(meta["source"]):
+        raise KbValidationError(f"{path.name}: source needs pdf+text")
+    for key in ("legal_bases", "tags", "cross_refs"):
+        if not isinstance(meta[key], list):
+            raise KbValidationError(f"{path.name}: '{key}' must be a list")
+    entry = KBEntry(
+        id=meta["id"],
+        title=meta["title"],
+        work=meta["work"],
+        chapter=meta["chapter"],
+        topic=meta["topic"],
+        author=meta["author"],
+        stand=meta["stand"],
+        batch=int(meta["batch"]),
+        source=dict(meta["source"]),
+        legal_bases=list(meta["legal_bases"]),
+        tags=list(meta["tags"]),
+        cross_refs=list(meta["cross_refs"]),
+        path=path,
+    )
+    entry.sections = split_sections(body)
+    return entry
+
+
+def load_kb(kb_dir: str | Path, verify_registry: bool = True) -> list[KBEntry]:
+    """Lädt alle Layer-2-Einträge, prüft Cross-Ref-Integrität und das kb.json-Gate."""
+    root = Path(kb_dir)
+    doc_dir = root / "dokumente"
+    paths = sorted(doc_dir.glob("*.md"))
+    if not paths:
+        raise KbValidationError(f"no Layer-2 documents under {doc_dir}")
+    entries: dict[str, KBEntry] = {}
+    for p in paths:
+        e = load_entry(p)
+        if e.id in entries:
+            raise KbValidationError(
+                f"duplicate id {e.id}: {entries[e.id].path.name} and {p.name}"
+            )
+        entries[e.id] = e
+    for e in entries.values():
+        dangling = [ref for ref in e.cross_refs if ref not in entries]
+        if dangling:
+            raise KbValidationError(f"{e.id}: dangling cross_refs {dangling}")
+    if verify_registry:
+        _registry_gate(root, entries)
+    return list(entries.values())
+
+
+def _registry_gate(root: Path, entries: dict[str, KBEntry]) -> None:
+    reg_path = root / "kb.json"
+    if not reg_path.is_file():
+        raise KbValidationError("kb.json missing — regenerate with --registry")
+    registry = json.loads(reg_path.read_text(encoding="utf-8"))
+    reg_ids = [e.get("id") for e in registry.get("entries", []) if e.get("id")]
+    reg_set = set(reg_ids)
+    kb_set = set(entries)
+    if reg_set != kb_set:
+        kb_only = sorted(kb_set - reg_set)[:5]
+        reg_only = sorted(reg_set - kb_set)[:5]
+        raise KbValidationError(
+            "kb.json out of sync with Layer 2 "
+            f"(docs-only: {kb_only}, registry-only: {reg_only}; "
+            f"registry n={registry.get('n_entries')}, docs n={len(entries)}) — "
+            "regenerate the registry first (build_lexis_kb.py --registry)"
+        )