KV/RIS-Erweiterung: 614 WKO-KV-Dokumente + 59 RIS-Gesetze (D9)
- Korpus 601 -> 1274 Layer-2-Eintraege: kv-kvt-001...614 (ein Cluster kollektivvertraege, Branche als Tag) und ris-<cluster-prefix>-nn auf der bestehenden Cluster-Map + 7 neue Cluster (zvr/avr/agg/lst/abo/ nso/kvt); LAW_MAP dokumentiert die 59 Gesetz-Zuordnungen. - kv/ris-Eintraege sind quellentreu generiert (D9) - Gesetze sind amtliche Werke, KV-Lohntabellen zahlenexakt; Tool-Output in tools/ (ingest_sources.py, build_registry.py, kb_common.py), eingefrorene ID-Kataloge tools/catalogs/*.json (nur Metadaten). - kb.json/INDEX.md regeneriert (1274 Eintraege, 76 Cluster); agent/kb.py: ID-Raeume kv|ris, source akzeptiert html-only. - Tests 41 -> 49 (Konverter, LAW_MAP-Abdeckung, neue ID-Raeume, Korpus-Integrationszahl).
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
"""Registry-Generator für die Wissensbasis (alle vier ID-Räume lb/wk/kv/ris).
|
||||
|
||||
Ersetzt in diesem Repo das im Schwesterprojekt verbliebene
|
||||
`build_lexis_kb.py --registry`: lädt alle Layer-2-Einträge über
|
||||
`agent.kb.load_kb` (Schema-Validierung + Cross-Ref-Integrität), prüft
|
||||
zusätzlich topic<->ID-Präfix-Konsistenz und Batch-Plausibilität und
|
||||
schreibt `kb.json` + `INDEX.md` neu. Beide Dateien sind generiert —
|
||||
nie manuell editieren.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from agent.kb import load_kb # noqa: E402
|
||||
from tools.kb_common import ID_SPACES, NEW_CLUSTERS, load_topic_prefix_map # noqa: E402
|
||||
|
||||
KB_DIR = ROOT / "wissensbasis"
|
||||
|
||||
|
||||
def space_of(entry_id: str) -> str:
|
||||
return entry_id.split("-")[0]
|
||||
|
||||
|
||||
def collect(entries) -> tuple[dict, list[str]]:
|
||||
"""Validiert und aggregiert; gibt (Cluster-Map, Fehler) zurück."""
|
||||
errors: list[str] = []
|
||||
topic_prefix = load_topic_prefix_map(KB_DIR / "kb.json")
|
||||
for e in entries:
|
||||
prefix = e.id.split("-")[1]
|
||||
known = topic_prefix.get(e.topic)
|
||||
if known != prefix:
|
||||
errors.append(
|
||||
f"{e.id}: topic '{e.topic}' erwartet Präfix '{known}', "
|
||||
f"ID hat '{prefix}'"
|
||||
)
|
||||
if e.batch < 1:
|
||||
errors.append(f"{e.id}: batch {e.batch} < 1")
|
||||
if not e.sections:
|
||||
errors.append(f"{e.id}: keine H2-Sektionen (Chunking würde leer laufen)")
|
||||
return topic_prefix, errors
|
||||
|
||||
|
||||
def build_kb_json(entries, topic_prefix: dict, cluster_names: dict) -> dict:
|
||||
entries_sorted = sorted(entries, key=lambda e: e.path.name)
|
||||
sources = Counter(e.work for e in entries)
|
||||
batches = Counter((space_of(e.id), e.batch) for e in entries)
|
||||
clusters = Counter(e.topic for e in entries)
|
||||
return {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"n_entries": len(entries),
|
||||
"sources": [
|
||||
{"work": work, "n": n}
|
||||
for work, n in sorted(sources.items(), key=lambda kv: -kv[1])
|
||||
],
|
||||
"batches": [
|
||||
{"source": space, "batch": batch, "n": n}
|
||||
for (space, batch), n in sorted(batches.items())
|
||||
],
|
||||
"layers": {
|
||||
"curated": "wissensbasis/dokumente",
|
||||
"fulltext": ".lexis360/md + .wiku/md (lokal, unversioniert — lizenzierte Quellen)",
|
||||
"raw": ".firecrawl/wko-kv/docs (WKO-KV-Seiten) + .ris (Gesetze) — lokal, unversioniert; kv/ris-Layer-2 ist quellentreu generiert",
|
||||
},
|
||||
"schema": {
|
||||
"id": "lb-<cluster-prefix>-<nn> (Lexis) | wk-<cluster-prefix>-<nn> (WIKU) | "
|
||||
"kv-kvt-<nnn> (WKO-KV, Cluster kollektivvertraege) | "
|
||||
"ris-<cluster-prefix>-<nn> (RIS-Gesetz)",
|
||||
"stand": "YYYY-MM (Stand der Quelle; KV: Geltungsbeginn)",
|
||||
"legal_bases": "Rechtsgrundlagen as cited in the source",
|
||||
"cross_refs": "ids of related kb entries (alle ID-Räume dürfen kreuzen)",
|
||||
"generated_files": "kv_*.md / ris_*.md sind Tool-Output (tools/ingest_sources.py) — keine manuelle Kuratierung",
|
||||
},
|
||||
"clusters": [
|
||||
{
|
||||
"slug": slug,
|
||||
"name": cluster_names.get(slug, slug),
|
||||
"prefix": topic_prefix[slug],
|
||||
"n": n,
|
||||
}
|
||||
for slug, n in sorted(clusters.items())
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"id": e.id,
|
||||
"batch": e.batch,
|
||||
"title": e.title,
|
||||
"work": e.work,
|
||||
"chapter": e.chapter,
|
||||
"topic": e.topic,
|
||||
"author": e.author,
|
||||
"stand": e.stand,
|
||||
"source": e.source,
|
||||
"legal_bases": e.legal_bases,
|
||||
"tags": e.tags,
|
||||
"cross_refs": e.cross_refs,
|
||||
}
|
||||
for e in entries_sorted
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_index_md(entries, topic_prefix: dict, cluster_names: dict) -> str:
|
||||
by_space: dict[str, list] = defaultdict(list)
|
||||
for e in entries:
|
||||
by_space[space_of(e.id)].append(e)
|
||||
|
||||
n_clusters = len({e.topic for e in entries})
|
||||
header = (
|
||||
"# Index — Wissensbasis Personalverrechnung\n\n"
|
||||
f"Generiert am {datetime.now(timezone.utc).date().isoformat()} · "
|
||||
f"{len(entries)} Einträge · {n_clusters} Cluster. Quellen: "
|
||||
"Lexis Briefings Personalrecht (Lexis 360, lizenzierter Export), "
|
||||
"WIKU Personal (Fachbroschüren, Arbeitsunterlagen, Casebooks, "
|
||||
"„WIKU Personal aktuell“), WKO.at – Kollektivvertrag (KV-Dokumente, "
|
||||
"quellentreu generiert) und RIS – Rechtsinformationssystem des Bundes "
|
||||
"(Gesetzes-§-Auschnitte, quellentreu generiert). Rohquellen lokal/"
|
||||
"unversioniert (`.lexis360/`, `.wiku/`, `.firecrawl/`, `.ris/`). "
|
||||
"Werte gelten je zum Quell-Stand (`stand` im Frontmatter); Widersprüche "
|
||||
"und offene Punkte sind mit ⚠/❓ in den Einträgen selbst dokumentiert. "
|
||||
"Schema & Konventionen: `README.md`.\n"
|
||||
)
|
||||
out = [header]
|
||||
for space in ("lb", "wk", "kv", "ris"):
|
||||
group = by_space.get(space)
|
||||
if not group:
|
||||
continue
|
||||
label = ID_SPACES[space]
|
||||
out.append(f"\n# {label} — {len(group)} Einträge\n")
|
||||
by_topic: dict[str, list] = defaultdict(list)
|
||||
for e in group:
|
||||
by_topic[e.topic].append(e)
|
||||
for topic in sorted(by_topic, key=lambda t: -len(by_topic[t])):
|
||||
rows = sorted(by_topic[topic], key=lambda e: e.id)
|
||||
stands = sorted(e.stand for e in rows)
|
||||
out.append(
|
||||
f"\n## {cluster_names.get(topic, topic)} (`topic: {topic}`) — "
|
||||
f"{len(rows)} Einträge · Stand {stands[0]} bis {stands[-1]}\n"
|
||||
)
|
||||
out.append("\n| ID | Titel | Autor | Stand | Datei |")
|
||||
out.append("|---|---|---|---|---|")
|
||||
for e in rows:
|
||||
out.append(
|
||||
f"| {e.id} | {e.title} | {e.author} | {e.stand} | {e.path.name} |"
|
||||
)
|
||||
out.append("")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
entries = load_kb(KB_DIR, verify_registry=False)
|
||||
topic_prefix, errors = collect(entries)
|
||||
if errors:
|
||||
print("VALIDIERUNGSFEHLER:")
|
||||
for err in errors[:40]:
|
||||
print(" -", err)
|
||||
if len(errors) > 40:
|
||||
print(f" ... und {len(errors) - 40} weitere")
|
||||
return 1
|
||||
|
||||
cluster_names = {
|
||||
slug: name for slug, (_prefix, name) in NEW_CLUSTERS.items()
|
||||
}
|
||||
# Anzeigenamen bestehender Cluster aus der bestehenden kb.json übernehmen.
|
||||
old = json.loads((KB_DIR / "kb.json").read_text(encoding="utf-8"))
|
||||
for c in old.get("clusters", []):
|
||||
cluster_names.setdefault(c["slug"], c.get("name", c["slug"]))
|
||||
|
||||
kb_json = build_kb_json(entries, topic_prefix, cluster_names)
|
||||
(KB_DIR / "kb.json").write_text(
|
||||
json.dumps(kb_json, ensure_ascii=False, indent=1) + "\n", encoding="utf-8"
|
||||
)
|
||||
(KB_DIR / "INDEX.md").write_text(
|
||||
build_index_md(entries, topic_prefix, cluster_names), encoding="utf-8"
|
||||
)
|
||||
|
||||
print(f"Registry neu generiert: {len(entries)} Einträge, "
|
||||
f"{len(kb_json['clusters'])} Cluster")
|
||||
for s in kb_json["sources"]:
|
||||
print(f" {s['work']}: {s['n']}")
|
||||
print(f" kb.json ({(KB_DIR / 'kb.json').stat().st_size // 1024} KB), "
|
||||
f"INDEX.md ({(KB_DIR / 'INDEX.md').stat().st_size // 1024} KB)")
|
||||
|
||||
# Konsistenz-Gate gegen die frische Registry bestätigen.
|
||||
load_kb(KB_DIR, verify_registry=True)
|
||||
print("Gate-Check: kb.json <-> Layer 2 konsistent.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user