199 lines
8.0 KiB
Python
199 lines
8.0 KiB
Python
"""Registry-Generator für die Wissensbasis (alle fünf ID-Räume lb/wk/kv/ris/rj).
|
||
|
||
Ersetzt in diesem Repo das im Schwesterprojekt verbliebene
|
||
`build_lexis_kb.py --registry`: lädt alle Layer-2-Einträge über
|
||
`agent.kb.load_kb` (Schema-Validierung + Cross-Ref-Integrität), prüft
|
||
zusätzlich topic<->ID-Präfix-Konsistenz und Batch-Plausibilität und
|
||
schreibt `kb.json` + `INDEX.md` neu. Beide Dateien sind generiert —
|
||
nie manuell editieren.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from agent.kb import load_kb # noqa: E402
|
||
from tools.kb_common import ID_SPACES, NEW_CLUSTERS, load_topic_prefix_map # noqa: E402
|
||
|
||
KB_DIR = ROOT / "wissensbasis"
|
||
|
||
|
||
def space_of(entry_id: str) -> str:
|
||
return entry_id.split("-")[0]
|
||
|
||
|
||
def collect(entries) -> tuple[dict, list[str]]:
|
||
"""Validiert und aggregiert; gibt (Cluster-Map, Fehler) zurück."""
|
||
errors: list[str] = []
|
||
topic_prefix = load_topic_prefix_map(KB_DIR / "kb.json")
|
||
for e in entries:
|
||
prefix = e.id.split("-")[1]
|
||
known = topic_prefix.get(e.topic)
|
||
if known != prefix:
|
||
errors.append(
|
||
f"{e.id}: topic '{e.topic}' erwartet Präfix '{known}', "
|
||
f"ID hat '{prefix}'"
|
||
)
|
||
if e.batch < 1:
|
||
errors.append(f"{e.id}: batch {e.batch} < 1")
|
||
if not e.sections:
|
||
errors.append(f"{e.id}: keine H2-Sektionen (Chunking würde leer laufen)")
|
||
return topic_prefix, errors
|
||
|
||
|
||
def build_kb_json(entries, topic_prefix: dict, cluster_names: dict) -> dict:
|
||
entries_sorted = sorted(entries, key=lambda e: e.path.name)
|
||
sources = Counter(e.work for e in entries)
|
||
batches = Counter((space_of(e.id), e.batch) for e in entries)
|
||
clusters = Counter(e.topic for e in entries)
|
||
return {
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"n_entries": len(entries),
|
||
"sources": [
|
||
{"work": work, "n": n}
|
||
for work, n in sorted(sources.items(), key=lambda kv: -kv[1])
|
||
],
|
||
"batches": [
|
||
{"source": space, "batch": batch, "n": n}
|
||
for (space, batch), n in sorted(batches.items())
|
||
],
|
||
"layers": {
|
||
"curated": "wissensbasis/dokumente",
|
||
"fulltext": ".lexis360/md + .wiku/md (lokal, unversioniert — lizenzierte Quellen)",
|
||
"raw": ".firecrawl/wko-kv/docs (WKO-KV-Seiten) + .ris (Gesetze) + .rechtsprechung (Entscheidungen/Normauszüge) — lokal, unversioniert; kv/ris/rj-Layer-2 ist quellentreu generiert",
|
||
},
|
||
"schema": {
|
||
"id": "lb-<cluster-prefix>-<nn> (Lexis) | wk-<cluster-prefix>-<nn> (WIKU) | "
|
||
"kv-kvt-<nnn> (WKO-KV, Cluster kollektivvertraege) | "
|
||
"ris-<cluster-prefix>-<nn> (RIS-Gesetz) | rj-rjs-<nnn> (Rechtsprechung/Normauszug)",
|
||
"stand": "YYYY-MM (Stand der Quelle; KV: Geltungsbeginn)",
|
||
"legal_bases": "Rechtsgrundlagen as cited in the source",
|
||
"cross_refs": "ids of related kb entries (alle ID-Räume dürfen kreuzen)",
|
||
"generated_files": "kv_*.md / ris_*.md / rj_*.md sind Tool-Output (tools/ingest_sources.py) — keine manuelle Kuratierung",
|
||
},
|
||
"clusters": [
|
||
{
|
||
"slug": slug,
|
||
"name": cluster_names.get(slug, slug),
|
||
"prefix": topic_prefix[slug],
|
||
"n": n,
|
||
}
|
||
for slug, n in sorted(clusters.items())
|
||
],
|
||
"entries": [
|
||
{
|
||
"id": e.id,
|
||
"batch": e.batch,
|
||
"title": e.title,
|
||
"work": e.work,
|
||
"chapter": e.chapter,
|
||
"topic": e.topic,
|
||
"author": e.author,
|
||
"stand": e.stand,
|
||
"source": e.source,
|
||
"legal_bases": e.legal_bases,
|
||
"tags": e.tags,
|
||
"cross_refs": e.cross_refs,
|
||
}
|
||
for e in entries_sorted
|
||
],
|
||
}
|
||
|
||
|
||
def build_index_md(entries, topic_prefix: dict, cluster_names: dict) -> str:
|
||
by_space: dict[str, list] = defaultdict(list)
|
||
for e in entries:
|
||
by_space[space_of(e.id)].append(e)
|
||
|
||
n_clusters = len({e.topic for e in entries})
|
||
header = (
|
||
"# Index — Wissensbasis Personalverrechnung\n\n"
|
||
f"Generiert am {datetime.now(timezone.utc).date().isoformat()} · "
|
||
f"{len(entries)} Einträge · {n_clusters} Cluster. Quellen: "
|
||
"Lexis Briefings Personalrecht (Lexis 360, lizenzierter Export), "
|
||
"WIKU Personal (Fachbroschüren, Arbeitsunterlagen, Casebooks, "
|
||
"„WIKU Personal aktuell“), WKO.at – Kollektivvertrag (KV-Dokumente, "
|
||
"quellentreu generiert), RIS – Rechtsinformationssystem des Bundes "
|
||
"(Gesetzes-§-Auschnitte, quellentreu generiert) sowie Rechtsprechung "
|
||
"(RIS-OGD-Entscheidungen und Normauszüge; EuGH-Textwiedergaben separat "
|
||
"als nicht amtlich markiert). Rohquellen lokal/unversioniert "
|
||
"(`.lexis360/`, `.wiku/`, `.firecrawl/`, `.ris/`, `.rechtsprechung/`). "
|
||
"Werte gelten je zum Quell-Stand (`stand` im Frontmatter); Widersprüche "
|
||
"und offene Punkte sind mit ⚠/❓ in den Einträgen selbst dokumentiert. "
|
||
"Schema & Konventionen: `README.md`.\n"
|
||
)
|
||
out = [header]
|
||
for space in ("lb", "wk", "kv", "ris", "rj"):
|
||
group = by_space.get(space)
|
||
if not group:
|
||
continue
|
||
label = ID_SPACES[space]
|
||
out.append(f"\n# {label} — {len(group)} Einträge\n")
|
||
by_topic: dict[str, list] = defaultdict(list)
|
||
for e in group:
|
||
by_topic[e.topic].append(e)
|
||
for topic in sorted(by_topic, key=lambda t: -len(by_topic[t])):
|
||
rows = sorted(by_topic[topic], key=lambda e: e.id)
|
||
stands = sorted(e.stand for e in rows)
|
||
out.append(
|
||
f"\n## {cluster_names.get(topic, topic)} (`topic: {topic}`) — "
|
||
f"{len(rows)} Einträge · Stand {stands[0]} bis {stands[-1]}\n"
|
||
)
|
||
out.append("\n| ID | Titel | Autor | Stand | Datei |")
|
||
out.append("|---|---|---|---|---|")
|
||
for e in rows:
|
||
out.append(
|
||
f"| {e.id} | {e.title} | {e.author} | {e.stand} | {e.path.name} |"
|
||
)
|
||
out.append("")
|
||
return "\n".join(out) + "\n"
|
||
|
||
|
||
def main() -> int:
|
||
entries = load_kb(KB_DIR, verify_registry=False)
|
||
topic_prefix, errors = collect(entries)
|
||
if errors:
|
||
print("VALIDIERUNGSFEHLER:")
|
||
for err in errors[:40]:
|
||
print(" -", err)
|
||
if len(errors) > 40:
|
||
print(f" ... und {len(errors) - 40} weitere")
|
||
return 1
|
||
|
||
cluster_names = {
|
||
slug: name for slug, (_prefix, name) in NEW_CLUSTERS.items()
|
||
}
|
||
# Anzeigenamen bestehender Cluster aus der bestehenden kb.json übernehmen.
|
||
old = json.loads((KB_DIR / "kb.json").read_text(encoding="utf-8"))
|
||
for c in old.get("clusters", []):
|
||
cluster_names.setdefault(c["slug"], c.get("name", c["slug"]))
|
||
|
||
kb_json = build_kb_json(entries, topic_prefix, cluster_names)
|
||
(KB_DIR / "kb.json").write_text(
|
||
json.dumps(kb_json, ensure_ascii=False, indent=1) + "\n", encoding="utf-8"
|
||
)
|
||
(KB_DIR / "INDEX.md").write_text(
|
||
build_index_md(entries, topic_prefix, cluster_names), encoding="utf-8"
|
||
)
|
||
|
||
print(f"Registry neu generiert: {len(entries)} Einträge, "
|
||
f"{len(kb_json['clusters'])} Cluster")
|
||
for s in kb_json["sources"]:
|
||
print(f" {s['work']}: {s['n']}")
|
||
print(f" kb.json ({(KB_DIR / 'kb.json').stat().st_size // 1024} KB), "
|
||
f"INDEX.md ({(KB_DIR / 'INDEX.md').stat().st_size // 1024} KB)")
|
||
|
||
# Konsistenz-Gate gegen die frische Registry bestätigen.
|
||
load_kb(KB_DIR, verify_registry=True)
|
||
print("Gate-Check: kb.json <-> Layer 2 konsistent.")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main()) |