pv-agent/tools/build_registry.py

"""Registry-Generator für die Wissensbasis (alle vier ID-Räume lb/wk/kv/ris).

Ersetzt in diesem Repo das im Schwesterprojekt verbliebene
`build_lexis_kb.py --registry`: lädt alle Layer-2-Einträge über
`agent.kb.load_kb` (Schema-Validierung + Cross-Ref-Integrität), prüft
zusätzlich topic<->ID-Präfix-Konsistenz und Batch-Plausibilität und
schreibt `kb.json` + `INDEX.md` neu. Beide Dateien sind generiert —
nie manuell editieren.
"""
from __future__ import annotations

import json
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from agent.kb import load_kb  # noqa: E402
from tools.kb_common import ID_SPACES, NEW_CLUSTERS, load_topic_prefix_map  # noqa: E402

KB_DIR = ROOT / "wissensbasis"


def space_of(entry_id: str) -> str:
    return entry_id.split("-")[0]


def collect(entries) -> tuple[dict, list[str]]:
    """Validiert und aggregiert; gibt (Cluster-Map, Fehler) zurück."""
    errors: list[str] = []
    topic_prefix = load_topic_prefix_map(KB_DIR / "kb.json")
    for e in entries:
        prefix = e.id.split("-")[1]
        known = topic_prefix.get(e.topic)
        if known != prefix:
            errors.append(
                f"{e.id}: topic '{e.topic}' erwartet Präfix '{known}', "
                f"ID hat '{prefix}'"
            )
        if e.batch < 1:
            errors.append(f"{e.id}: batch {e.batch} < 1")
        if not e.sections:
            errors.append(f"{e.id}: keine H2-Sektionen (Chunking würde leer laufen)")
    return topic_prefix, errors


def build_kb_json(entries, topic_prefix: dict, cluster_names: dict) -> dict:
    entries_sorted = sorted(entries, key=lambda e: e.path.name)
    sources = Counter(e.work for e in entries)
    batches = Counter((space_of(e.id), e.batch) for e in entries)
    clusters = Counter(e.topic for e in entries)
    return {
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "n_entries": len(entries),
        "sources": [
            {"work": work, "n": n}
            for work, n in sorted(sources.items(), key=lambda kv: -kv[1])
        ],
        "batches": [
            {"source": space, "batch": batch, "n": n}
            for (space, batch), n in sorted(batches.items())
        ],
        "layers": {
            "curated": "wissensbasis/dokumente",
            "fulltext": ".lexis360/md + .wiku/md (lokal, unversioniert — lizenzierte Quellen)",
            "raw": ".firecrawl/wko-kv/docs (WKO-KV-Seiten) + .ris (Gesetze) — lokal, unversioniert; kv/ris-Layer-2 ist quellentreu generiert",
        },
        "schema": {
            "id": "lb-<cluster-prefix>-<nn> (Lexis) | wk-<cluster-prefix>-<nn> (WIKU) | "
                  "kv-kvt-<nnn> (WKO-KV, Cluster kollektivvertraege) | "
                  "ris-<cluster-prefix>-<nn> (RIS-Gesetz)",
            "stand": "YYYY-MM (Stand der Quelle; KV: Geltungsbeginn)",
            "legal_bases": "Rechtsgrundlagen as cited in the source",
            "cross_refs": "ids of related kb entries (alle ID-Räume dürfen kreuzen)",
            "generated_files": "kv_*.md / ris_*.md sind Tool-Output (tools/ingest_sources.py) — keine manuelle Kuratierung",
        },
        "clusters": [
            {
                "slug": slug,
                "name": cluster_names.get(slug, slug),
                "prefix": topic_prefix[slug],
                "n": n,
            }
            for slug, n in sorted(clusters.items())
        ],
        "entries": [
            {
                "id": e.id,
                "batch": e.batch,
                "title": e.title,
                "work": e.work,
                "chapter": e.chapter,
                "topic": e.topic,
                "author": e.author,
                "stand": e.stand,
                "source": e.source,
                "legal_bases": e.legal_bases,
                "tags": e.tags,
                "cross_refs": e.cross_refs,
            }
            for e in entries_sorted
        ],
    }


def build_index_md(entries, topic_prefix: dict, cluster_names: dict) -> str:
    by_space: dict[str, list] = defaultdict(list)
    for e in entries:
        by_space[space_of(e.id)].append(e)

    n_clusters = len({e.topic for e in entries})
    header = (
        "# Index — Wissensbasis Personalverrechnung\n\n"
        f"Generiert am {datetime.now(timezone.utc).date().isoformat()} · "
        f"{len(entries)} Einträge · {n_clusters} Cluster. Quellen: "
        "Lexis Briefings Personalrecht (Lexis 360, lizenzierter Export), "
        "WIKU Personal (Fachbroschüren, Arbeitsunterlagen, Casebooks, "
        "„WIKU Personal aktuell“), WKO.at – Kollektivvertrag (KV-Dokumente, "
        "quellentreu generiert) und RIS – Rechtsinformationssystem des Bundes "
        "(Gesetzes-§-Auschnitte, quellentreu generiert). Rohquellen lokal/"
        "unversioniert (`.lexis360/`, `.wiku/`, `.firecrawl/`, `.ris/`). "
        "Werte gelten je zum Quell-Stand (`stand` im Frontmatter); Widersprüche "
        "und offene Punkte sind mit ⚠/❓ in den Einträgen selbst dokumentiert. "
        "Schema & Konventionen: `README.md`.\n"
    )
    out = [header]
    for space in ("lb", "wk", "kv", "ris"):
        group = by_space.get(space)
        if not group:
            continue
        label = ID_SPACES[space]
        out.append(f"\n# {label} — {len(group)} Einträge\n")
        by_topic: dict[str, list] = defaultdict(list)
        for e in group:
            by_topic[e.topic].append(e)
        for topic in sorted(by_topic, key=lambda t: -len(by_topic[t])):
            rows = sorted(by_topic[topic], key=lambda e: e.id)
            stands = sorted(e.stand for e in rows)
            out.append(
                f"\n## {cluster_names.get(topic, topic)} (`topic: {topic}`) — "
                f"{len(rows)} Einträge · Stand {stands[0]} bis {stands[-1]}\n"
            )
            out.append("\n| ID | Titel | Autor | Stand | Datei |")
            out.append("|---|---|---|---|---|")
            for e in rows:
                out.append(
                    f"| {e.id} | {e.title} | {e.author} | {e.stand} | {e.path.name} |"
                )
            out.append("")
    return "\n".join(out) + "\n"


def main() -> int:
    entries = load_kb(KB_DIR, verify_registry=False)
    topic_prefix, errors = collect(entries)
    if errors:
        print("VALIDIERUNGSFEHLER:")
        for err in errors[:40]:
            print(" -", err)
        if len(errors) > 40:
            print(f" ... und {len(errors) - 40} weitere")
        return 1

    cluster_names = {
        slug: name for slug, (_prefix, name) in NEW_CLUSTERS.items()
    }
    # Anzeigenamen bestehender Cluster aus der bestehenden kb.json übernehmen.
    old = json.loads((KB_DIR / "kb.json").read_text(encoding="utf-8"))
    for c in old.get("clusters", []):
        cluster_names.setdefault(c["slug"], c.get("name", c["slug"]))

    kb_json = build_kb_json(entries, topic_prefix, cluster_names)
    (KB_DIR / "kb.json").write_text(
        json.dumps(kb_json, ensure_ascii=False, indent=1) + "\n", encoding="utf-8"
    )
    (KB_DIR / "INDEX.md").write_text(
        build_index_md(entries, topic_prefix, cluster_names), encoding="utf-8"
    )

    print(f"Registry neu generiert: {len(entries)} Einträge, "
          f"{len(kb_json['clusters'])} Cluster")
    for s in kb_json["sources"]:
        print(f"  {s['work']}: {s['n']}")
    print(f"  kb.json ({(KB_DIR / 'kb.json').stat().st_size // 1024} KB), "
          f"INDEX.md ({(KB_DIR / 'INDEX.md').stat().st_size // 1024} KB)")

    # Konsistenz-Gate gegen die frische Registry bestätigen.
    load_kb(KB_DIR, verify_registry=True)
    print("Gate-Check: kb.json <-> Layer 2 konsistent.")
    return 0


if __name__ == "__main__":
    sys.exit(main())