KV/RIS-Erweiterung: 614 WKO-KV-Dokumente + 59 RIS-Gesetze (D9)
- Korpus 601 -> 1274 Layer-2-Eintraege: kv-kvt-001...614 (ein Cluster kollektivvertraege, Branche als Tag) und ris-<cluster-prefix>-nn auf der bestehenden Cluster-Map + 7 neue Cluster (zvr/avr/agg/lst/abo/ nso/kvt); LAW_MAP dokumentiert die 59 Gesetz-Zuordnungen. - kv/ris-Eintraege sind quellentreu generiert (D9) - Gesetze sind amtliche Werke, KV-Lohntabellen zahlenexakt; Tool-Output in tools/ (ingest_sources.py, build_registry.py, kb_common.py), eingefrorene ID-Kataloge tools/catalogs/*.json (nur Metadaten). - kb.json/INDEX.md regeneriert (1274 Eintraege, 76 Cluster); agent/kb.py: ID-Raeume kv|ris, source akzeptiert html-only. - Tests 41 -> 49 (Konverter, LAW_MAP-Abdeckung, neue ID-Raeume, Korpus-Integrationszahl).
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
"""Registry-Generator für die Wissensbasis (alle vier ID-Räume lb/wk/kv/ris).
|
||||
|
||||
Ersetzt in diesem Repo das im Schwesterprojekt verbliebene
|
||||
`build_lexis_kb.py --registry`: lädt alle Layer-2-Einträge über
|
||||
`agent.kb.load_kb` (Schema-Validierung + Cross-Ref-Integrität), prüft
|
||||
zusätzlich topic<->ID-Präfix-Konsistenz und Batch-Plausibilität und
|
||||
schreibt `kb.json` + `INDEX.md` neu. Beide Dateien sind generiert —
|
||||
nie manuell editieren.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from agent.kb import load_kb # noqa: E402
|
||||
from tools.kb_common import ID_SPACES, NEW_CLUSTERS, load_topic_prefix_map # noqa: E402
|
||||
|
||||
KB_DIR = ROOT / "wissensbasis"
|
||||
|
||||
|
||||
def space_of(entry_id: str) -> str:
|
||||
return entry_id.split("-")[0]
|
||||
|
||||
|
||||
def collect(entries) -> tuple[dict, list[str]]:
|
||||
"""Validiert und aggregiert; gibt (Cluster-Map, Fehler) zurück."""
|
||||
errors: list[str] = []
|
||||
topic_prefix = load_topic_prefix_map(KB_DIR / "kb.json")
|
||||
for e in entries:
|
||||
prefix = e.id.split("-")[1]
|
||||
known = topic_prefix.get(e.topic)
|
||||
if known != prefix:
|
||||
errors.append(
|
||||
f"{e.id}: topic '{e.topic}' erwartet Präfix '{known}', "
|
||||
f"ID hat '{prefix}'"
|
||||
)
|
||||
if e.batch < 1:
|
||||
errors.append(f"{e.id}: batch {e.batch} < 1")
|
||||
if not e.sections:
|
||||
errors.append(f"{e.id}: keine H2-Sektionen (Chunking würde leer laufen)")
|
||||
return topic_prefix, errors
|
||||
|
||||
|
||||
def build_kb_json(entries, topic_prefix: dict, cluster_names: dict) -> dict:
|
||||
entries_sorted = sorted(entries, key=lambda e: e.path.name)
|
||||
sources = Counter(e.work for e in entries)
|
||||
batches = Counter((space_of(e.id), e.batch) for e in entries)
|
||||
clusters = Counter(e.topic for e in entries)
|
||||
return {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"n_entries": len(entries),
|
||||
"sources": [
|
||||
{"work": work, "n": n}
|
||||
for work, n in sorted(sources.items(), key=lambda kv: -kv[1])
|
||||
],
|
||||
"batches": [
|
||||
{"source": space, "batch": batch, "n": n}
|
||||
for (space, batch), n in sorted(batches.items())
|
||||
],
|
||||
"layers": {
|
||||
"curated": "wissensbasis/dokumente",
|
||||
"fulltext": ".lexis360/md + .wiku/md (lokal, unversioniert — lizenzierte Quellen)",
|
||||
"raw": ".firecrawl/wko-kv/docs (WKO-KV-Seiten) + .ris (Gesetze) — lokal, unversioniert; kv/ris-Layer-2 ist quellentreu generiert",
|
||||
},
|
||||
"schema": {
|
||||
"id": "lb-<cluster-prefix>-<nn> (Lexis) | wk-<cluster-prefix>-<nn> (WIKU) | "
|
||||
"kv-kvt-<nnn> (WKO-KV, Cluster kollektivvertraege) | "
|
||||
"ris-<cluster-prefix>-<nn> (RIS-Gesetz)",
|
||||
"stand": "YYYY-MM (Stand der Quelle; KV: Geltungsbeginn)",
|
||||
"legal_bases": "Rechtsgrundlagen as cited in the source",
|
||||
"cross_refs": "ids of related kb entries (alle ID-Räume dürfen kreuzen)",
|
||||
"generated_files": "kv_*.md / ris_*.md sind Tool-Output (tools/ingest_sources.py) — keine manuelle Kuratierung",
|
||||
},
|
||||
"clusters": [
|
||||
{
|
||||
"slug": slug,
|
||||
"name": cluster_names.get(slug, slug),
|
||||
"prefix": topic_prefix[slug],
|
||||
"n": n,
|
||||
}
|
||||
for slug, n in sorted(clusters.items())
|
||||
],
|
||||
"entries": [
|
||||
{
|
||||
"id": e.id,
|
||||
"batch": e.batch,
|
||||
"title": e.title,
|
||||
"work": e.work,
|
||||
"chapter": e.chapter,
|
||||
"topic": e.topic,
|
||||
"author": e.author,
|
||||
"stand": e.stand,
|
||||
"source": e.source,
|
||||
"legal_bases": e.legal_bases,
|
||||
"tags": e.tags,
|
||||
"cross_refs": e.cross_refs,
|
||||
}
|
||||
for e in entries_sorted
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_index_md(entries, topic_prefix: dict, cluster_names: dict) -> str:
|
||||
by_space: dict[str, list] = defaultdict(list)
|
||||
for e in entries:
|
||||
by_space[space_of(e.id)].append(e)
|
||||
|
||||
n_clusters = len({e.topic for e in entries})
|
||||
header = (
|
||||
"# Index — Wissensbasis Personalverrechnung\n\n"
|
||||
f"Generiert am {datetime.now(timezone.utc).date().isoformat()} · "
|
||||
f"{len(entries)} Einträge · {n_clusters} Cluster. Quellen: "
|
||||
"Lexis Briefings Personalrecht (Lexis 360, lizenzierter Export), "
|
||||
"WIKU Personal (Fachbroschüren, Arbeitsunterlagen, Casebooks, "
|
||||
"„WIKU Personal aktuell“), WKO.at – Kollektivvertrag (KV-Dokumente, "
|
||||
"quellentreu generiert) und RIS – Rechtsinformationssystem des Bundes "
|
||||
"(Gesetzes-§-Auschnitte, quellentreu generiert). Rohquellen lokal/"
|
||||
"unversioniert (`.lexis360/`, `.wiku/`, `.firecrawl/`, `.ris/`). "
|
||||
"Werte gelten je zum Quell-Stand (`stand` im Frontmatter); Widersprüche "
|
||||
"und offene Punkte sind mit ⚠/❓ in den Einträgen selbst dokumentiert. "
|
||||
"Schema & Konventionen: `README.md`.\n"
|
||||
)
|
||||
out = [header]
|
||||
for space in ("lb", "wk", "kv", "ris"):
|
||||
group = by_space.get(space)
|
||||
if not group:
|
||||
continue
|
||||
label = ID_SPACES[space]
|
||||
out.append(f"\n# {label} — {len(group)} Einträge\n")
|
||||
by_topic: dict[str, list] = defaultdict(list)
|
||||
for e in group:
|
||||
by_topic[e.topic].append(e)
|
||||
for topic in sorted(by_topic, key=lambda t: -len(by_topic[t])):
|
||||
rows = sorted(by_topic[topic], key=lambda e: e.id)
|
||||
stands = sorted(e.stand for e in rows)
|
||||
out.append(
|
||||
f"\n## {cluster_names.get(topic, topic)} (`topic: {topic}`) — "
|
||||
f"{len(rows)} Einträge · Stand {stands[0]} bis {stands[-1]}\n"
|
||||
)
|
||||
out.append("\n| ID | Titel | Autor | Stand | Datei |")
|
||||
out.append("|---|---|---|---|---|")
|
||||
for e in rows:
|
||||
out.append(
|
||||
f"| {e.id} | {e.title} | {e.author} | {e.stand} | {e.path.name} |"
|
||||
)
|
||||
out.append("")
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
entries = load_kb(KB_DIR, verify_registry=False)
|
||||
topic_prefix, errors = collect(entries)
|
||||
if errors:
|
||||
print("VALIDIERUNGSFEHLER:")
|
||||
for err in errors[:40]:
|
||||
print(" -", err)
|
||||
if len(errors) > 40:
|
||||
print(f" ... und {len(errors) - 40} weitere")
|
||||
return 1
|
||||
|
||||
cluster_names = {
|
||||
slug: name for slug, (_prefix, name) in NEW_CLUSTERS.items()
|
||||
}
|
||||
# Anzeigenamen bestehender Cluster aus der bestehenden kb.json übernehmen.
|
||||
old = json.loads((KB_DIR / "kb.json").read_text(encoding="utf-8"))
|
||||
for c in old.get("clusters", []):
|
||||
cluster_names.setdefault(c["slug"], c.get("name", c["slug"]))
|
||||
|
||||
kb_json = build_kb_json(entries, topic_prefix, cluster_names)
|
||||
(KB_DIR / "kb.json").write_text(
|
||||
json.dumps(kb_json, ensure_ascii=False, indent=1) + "\n", encoding="utf-8"
|
||||
)
|
||||
(KB_DIR / "INDEX.md").write_text(
|
||||
build_index_md(entries, topic_prefix, cluster_names), encoding="utf-8"
|
||||
)
|
||||
|
||||
print(f"Registry neu generiert: {len(entries)} Einträge, "
|
||||
f"{len(kb_json['clusters'])} Cluster")
|
||||
for s in kb_json["sources"]:
|
||||
print(f" {s['work']}: {s['n']}")
|
||||
print(f" kb.json ({(KB_DIR / 'kb.json').stat().st_size // 1024} KB), "
|
||||
f"INDEX.md ({(KB_DIR / 'INDEX.md').stat().st_size // 1024} KB)")
|
||||
|
||||
# Konsistenz-Gate gegen die frische Registry bestätigen.
|
||||
load_kb(KB_DIR, verify_registry=True)
|
||||
print("Gate-Check: kb.json <-> Layer 2 konsistent.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,361 @@
|
||||
{
|
||||
"entries": {
|
||||
"ABGB.md": {
|
||||
"gnr": "10001622",
|
||||
"id": "ris-zvr-01",
|
||||
"stand": "2026-09",
|
||||
"title": "ABGB – Allgemeines bürgerliches Gesetzbuch"
|
||||
},
|
||||
"APG.md": {
|
||||
"gnr": "20003831",
|
||||
"id": "ris-pvs-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Allgemeines Pensionsgesetz (APG)"
|
||||
},
|
||||
"APSG.md": {
|
||||
"gnr": "10008788",
|
||||
"id": "ris-azm-01",
|
||||
"stand": "2026-09",
|
||||
"title": "APSG – Arbeitsplatzsicherungsgesetz"
|
||||
},
|
||||
"APflG.md": {
|
||||
"gnr": "20009604",
|
||||
"id": "ris-leh-01",
|
||||
"stand": "2026-09",
|
||||
"title": "APflG – Ausbildungspflichtgesetz"
|
||||
},
|
||||
"ARG.md": {
|
||||
"gnr": "10008541",
|
||||
"id": "ris-rhz-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitsruhegesetz (ARG)"
|
||||
},
|
||||
"ASGG.md": {
|
||||
"gnr": "10000813",
|
||||
"id": "ris-agg-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeits- und Sozialgerichtsgesetz (ASGG)"
|
||||
},
|
||||
"ASVG.md": {
|
||||
"gnr": "10008147",
|
||||
"id": "ris-sva-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Allgemeines Sozialversicherungsgesetz (ASVG)"
|
||||
},
|
||||
"ASchG.md": {
|
||||
"gnr": "10008910",
|
||||
"id": "ris-asc-01",
|
||||
"stand": "2026-09",
|
||||
"title": "ArbeitnehmerInnenschutzgesetz (ASchG)"
|
||||
},
|
||||
"AVRAG.md": {
|
||||
"gnr": "10008872",
|
||||
"id": "ris-avr-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitsvertragsrechts-Anpassungsgesetz (AVRAG)"
|
||||
},
|
||||
"AZG.md": {
|
||||
"gnr": "10008238",
|
||||
"id": "ris-azg-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitszeitgesetz (AZG)"
|
||||
},
|
||||
"AktG.md": {
|
||||
"gnr": "10002070",
|
||||
"id": "ris-vst-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Aktiengesetz (AktG)"
|
||||
},
|
||||
"AlVG.md": {
|
||||
"gnr": "10008407",
|
||||
"id": "ris-atz-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitslosenversicherungsgesetz 1977 (AlVG)"
|
||||
},
|
||||
"AngG.md": {
|
||||
"gnr": "10008069",
|
||||
"id": "ris-bnd-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Angestelltengesetz (AngG)"
|
||||
},
|
||||
"ArbIG.md": {
|
||||
"gnr": "10008840",
|
||||
"id": "ris-nso-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitsinspektionsgesetz 1993 (ArbIG)"
|
||||
},
|
||||
"ArbVG.md": {
|
||||
"gnr": "10008329",
|
||||
"id": "ris-brt-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitsverfassungsgesetz (ArbVG)"
|
||||
},
|
||||
"AÜG.md": {
|
||||
"gnr": "10008655",
|
||||
"id": "ris-aug-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Arbeitskräfteüberlassungsgesetz (AÜG)"
|
||||
},
|
||||
"BAG.md": {
|
||||
"gnr": "10006276",
|
||||
"id": "ris-leh-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Berufsausbildungsgesetz (BAG)"
|
||||
},
|
||||
"BAO.md": {
|
||||
"gnr": "10003940",
|
||||
"id": "ris-abo-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Bundesabgabenordnung (BAO)"
|
||||
},
|
||||
"BBG.md": {
|
||||
"gnr": "10008713",
|
||||
"id": "ris-beh-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Bundesbehindertengesetz (BBG)"
|
||||
},
|
||||
"BEinstG.md": {
|
||||
"gnr": "10008253",
|
||||
"id": "ris-beh-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Behinderteneinstellungsgesetz (BEinstG)"
|
||||
},
|
||||
"BMSVG.md": {
|
||||
"gnr": "20002088",
|
||||
"id": "ris-vor-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Betriebliches Mitarbeiter- und Selbständigenvorsorgegesetz (BMSVG)"
|
||||
},
|
||||
"BS-V.md": {
|
||||
"gnr": "10009121",
|
||||
"id": "ris-asc-02",
|
||||
"stand": "2026-09",
|
||||
"title": "BS-V – Bildschirmarbeitsverordnung"
|
||||
},
|
||||
"BSVG.md": {
|
||||
"gnr": "10008431",
|
||||
"id": "ris-bsv-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Bauern-Sozialversicherungsgesetz (BSVG)"
|
||||
},
|
||||
"BUAG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-end-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Bauarbeiter-Urlaubs- und Abfertigungsgesetz (BUAG)"
|
||||
},
|
||||
"BauV.md": {
|
||||
"gnr": "10008904",
|
||||
"id": "ris-asc-03",
|
||||
"stand": "2026-09",
|
||||
"title": "BauV – Bauarbeiterschutzverordnung"
|
||||
},
|
||||
"DHG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-dnh-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Dienstnehmerhaftpflichtgesetz (DHG)"
|
||||
},
|
||||
"EFZG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-krs-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Entgeltfortzahlungsgesetz (EFZG)"
|
||||
},
|
||||
"EO.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-pfa-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Exekutionsordnung (EO)"
|
||||
},
|
||||
"EStG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-lst-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Einkommensteuergesetz 1988 (EStG)"
|
||||
},
|
||||
"FLAG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-kbg-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Familienlastenausgleichsgesetz 1967 (FLAG)"
|
||||
},
|
||||
"FamZeitbG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-pap-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Familienzeitbonusgesetz (FamZeitbG)"
|
||||
},
|
||||
"FlexKapGG.md": {
|
||||
"gnr": "20012473",
|
||||
"id": "ris-vor-03",
|
||||
"stand": "2026-09",
|
||||
"title": "Flexible-Kapitalgesellschafts-Gesetz (FlexKapGG)"
|
||||
},
|
||||
"GSVG.md": {
|
||||
"gnr": "10008422",
|
||||
"id": "ris-gsv-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Gewerbliches Sozialversicherungsgesetz (GSVG)"
|
||||
},
|
||||
"GewO.md": {
|
||||
"gnr": "10007517",
|
||||
"id": "ris-gwe-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Gewerbeordnung 1994 (GewO)"
|
||||
},
|
||||
"GlBG.md": {
|
||||
"gnr": "20003395",
|
||||
"id": "ris-glb-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Gleichbehandlungsgesetz (GlBG)"
|
||||
},
|
||||
"GmbHG.md": {
|
||||
"gnr": "10001720",
|
||||
"id": "ris-gsf-01",
|
||||
"stand": "2026-09",
|
||||
"title": "GmbH-Gesetz (GmbHG)"
|
||||
},
|
||||
"IESG.md": {
|
||||
"gnr": "10008418",
|
||||
"id": "ris-ins-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Insolvenz-Entgeltsicherungsgesetz (IESG)"
|
||||
},
|
||||
"IPRG.md": {
|
||||
"gnr": "10002426",
|
||||
"id": "ris-zvr-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Bundesgesetz über das internationale Privatrecht (IPRG)"
|
||||
},
|
||||
"KBGG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-kbg-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Kinderbetreuungsgeldgesetz (KBGG)"
|
||||
},
|
||||
"KJBG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-jug-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Kinder- und Jugendlichen-Beschäftigungsgesetz 1987 (KJBG)"
|
||||
},
|
||||
"KSchG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-zvr-03",
|
||||
"stand": "2026-09",
|
||||
"title": "Konsumentenschutzgesetz (KSchG)"
|
||||
},
|
||||
"LAG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-bnd-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Landarbeitsgesetz 2021 (LAG)"
|
||||
},
|
||||
"LSD-BG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-lsd-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Lohn- und Sozialdumping-Bekämpfungsgesetz (LSD-BG)"
|
||||
},
|
||||
"MSchG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-sch-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Mutterschutzgesetz 1979 (MSchG)"
|
||||
},
|
||||
"MSchV.md": {
|
||||
"gnr": "20010035",
|
||||
"id": "ris-sch-02",
|
||||
"stand": "2026-09",
|
||||
"title": "MSchV – Verordnung über die Beschäftigungsbeschränkungen für werdende und stillende Mütter (Mutterschutzverordnung)"
|
||||
},
|
||||
"NSchG.md": {
|
||||
"gnr": "10008502",
|
||||
"id": "ris-nsc-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Nachtschwerarbeitsgesetz (NSchG)"
|
||||
},
|
||||
"PKG.md": {
|
||||
"gnr": "10007055",
|
||||
"id": "ris-vor-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Pensionskassengesetz (PKG)"
|
||||
},
|
||||
"RStDG.md": {
|
||||
"gnr": "10008187",
|
||||
"id": "ris-nso-02",
|
||||
"stand": "2026-09",
|
||||
"title": "RStDG – Richter- und Staatsanwaltschaftsdienstgesetz"
|
||||
},
|
||||
"Sachbezugswerteverordnung.md": {
|
||||
"gnr": "20001641",
|
||||
"id": "ris-sac-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Sachbezugswerteverordnung – Sachbezugswerteverordnung"
|
||||
},
|
||||
"Schwerarbeitsverordnung.md": {
|
||||
"gnr": "20004642",
|
||||
"id": "ris-swa-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Verordnung der Bundesministerin für soziale Sicherheit, Generationen und Konsumentenschutz über besondere Belastungen bei Schwerarbeit (Schwerarbeitsverordnung) (Schwerarbeitsverordnung)"
|
||||
},
|
||||
"StGB.md": {
|
||||
"gnr": "10002296",
|
||||
"id": "ris-nso-03",
|
||||
"stand": "2026-09",
|
||||
"title": "Strafgesetzbuch (StGB)"
|
||||
},
|
||||
"StPO.md": {
|
||||
"gnr": "10002326",
|
||||
"id": "ris-nso-04",
|
||||
"stand": "2026-09",
|
||||
"title": "Strafprozeßordnung 1975 (StPO)"
|
||||
},
|
||||
"TAG.md": {
|
||||
"gnr": "20007012",
|
||||
"id": "ris-url-02",
|
||||
"stand": "2026-09",
|
||||
"title": "Theaterarbeitsgesetz (TAG)"
|
||||
},
|
||||
"UGB.md": {
|
||||
"gnr": "10001702",
|
||||
"id": "ris-nso-05",
|
||||
"stand": "2026-09",
|
||||
"title": "Unternehmensgesetzbuch (UGB)"
|
||||
},
|
||||
"UrlG.md": {
|
||||
"gnr": "10008376",
|
||||
"id": "ris-url-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Urlaubsgesetz (UrlG)"
|
||||
},
|
||||
"VBG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-url-03",
|
||||
"stand": "2026-09",
|
||||
"title": "Vertragsbedienstetengesetz 1948 (VBG)"
|
||||
},
|
||||
"VKG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-kar-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Väter-Karenzgesetz (VKG)"
|
||||
},
|
||||
"ZDG.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-prd-01",
|
||||
"stand": "2026-09",
|
||||
"title": "Zivildienstgesetz 1986 (ZDG)"
|
||||
},
|
||||
"ZPO.md": {
|
||||
"gnr": null,
|
||||
"id": "ris-zvr-04",
|
||||
"stand": "2026-09",
|
||||
"title": "Zivilprozessordnung (ZPO)"
|
||||
}
|
||||
},
|
||||
"next_seq": 1,
|
||||
"source": "ris",
|
||||
"updated": "2026-09-14T22:28:40.683188+00:00"
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
"""Intake der Quellen `kv` (WKO.at-KV-Seiten) und `ris` (RIS-Gesetze) in die
|
||||
Layer-2-Wissensbasis.
|
||||
|
||||
Abweichend von den lizenzierten Quellen (Lexis/WIKU: Eigene-Worte-Kuratierung)
|
||||
werden kv/ris-Einträge **quellentreu generiert** (Entscheidung 2026-09-15):
|
||||
Gesetze sind amtliche Werke, KV-Lohntabellen müssen zahlenexakt bleiben.
|
||||
Die generierten Dateien (`kv_*.md`, `ris_*.md`) sind damit Tool-Output —
|
||||
keine manuelle Kuratierung, bei Änderungen neu generieren.
|
||||
|
||||
IDs werden aus den Katalogen `tools/catalogs/<source>_catalog.json` vergeben
|
||||
und danach eingefroren (Wissensbasis-Konvention: nie wiederverwenden).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from bs4 import BeautifulSoup, Comment, Tag
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from pathlib import Path
|
||||
|
||||
from tools.kb_common import ( # noqa: E402
|
||||
DATE_RE,
|
||||
LAW_MAP,
|
||||
WORK_KV,
|
||||
WORK_RIS,
|
||||
YEAR_RE,
|
||||
ascii_slug,
|
||||
load_topic_prefix_map,
|
||||
stand_from_date,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
KV_DIR = ROOT / ".firecrawl" / "wko-kv" / "docs"
|
||||
RIS_DIR = ROOT / ".ris"
|
||||
DOCS = ROOT / "wissensbasis" / "dokumente"
|
||||
CATALOGS = ROOT / "tools" / "catalogs"
|
||||
CRAWL_STAND = "2026-09" # Beschaffungsstand der beiden Quellen
|
||||
|
||||
|
||||
TOPIC_PREFIX_FULL: dict[str, str] | None = None
|
||||
|
||||
|
||||
def topic_prefix(topic: str) -> str:
|
||||
global TOPIC_PREFIX_FULL
|
||||
if TOPIC_PREFIX_FULL is None:
|
||||
TOPIC_PREFIX_FULL = load_topic_prefix_map(ROOT / "wissensbasis" / "kb.json")
|
||||
return TOPIC_PREFIX_FULL[topic]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Kataloge (eingefrorene IDs)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def load_catalog(source: str) -> dict:
|
||||
path = CATALOGS / f"{source}_catalog.json"
|
||||
if path.is_file():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return {"source": source, "updated": None, "next_seq": 1, "entries": {}}
|
||||
|
||||
|
||||
def save_catalog(catalog: dict) -> None:
|
||||
catalog["updated"] = datetime.now(timezone.utc).isoformat()
|
||||
CATALOGS.mkdir(parents=True, exist_ok=True)
|
||||
path = CATALOGS / f"{catalog['source']}_catalog.json"
|
||||
path.write_text(
|
||||
json.dumps(catalog, ensure_ascii=False, indent=1, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# WKO-KV: HTML -> Markdown
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
NOISE_TAGS = ("script", "style", "button", "svg", "nav", "iframe", "form", "img")
|
||||
NOISE_CLASSES = re.compile(
|
||||
r"social|share|related|teaser|breadcrumb|pagination|print|meta-details", re.I
|
||||
)
|
||||
|
||||
|
||||
def prep_soup(html: str) -> BeautifulSoup:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup.find_all(NOISE_TAGS):
|
||||
tag.decompose()
|
||||
for br in soup.find_all("br"):
|
||||
br.replace_with("\n")
|
||||
return soup
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
"""Zeilenweise strippen, interne Whitespace-Läufe kollabieren."""
|
||||
lines = []
|
||||
for line in text.split("\n"):
|
||||
line = re.sub(r"[ \t\xa0]+", " ", line).strip()
|
||||
if line:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def cell_text(cell: Tag) -> str:
|
||||
text = cell.get_text(" ")
|
||||
return re.sub(r"\s+", " ", text).strip().replace("|", "\\|")
|
||||
|
||||
|
||||
def table_to_md(table: Tag) -> str:
|
||||
rows = []
|
||||
for tr in table.find_all("tr"):
|
||||
if tr.find_parent("table") is not table:
|
||||
continue # verschachtelte Tabellen nur einmal ausgeben
|
||||
cells = [cell_text(c) for c in tr.find_all(["td", "th"], recursive=False)]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if not rows:
|
||||
return ""
|
||||
width = max(len(r) for r in rows)
|
||||
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||
out = ["| " + " | ".join(rows[0]) + " |"]
|
||||
out.append("| " + " | ".join(["---"] * width) + " |")
|
||||
for r in rows[1:]:
|
||||
out.append("| " + " | ".join(r) + " |")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def list_to_md(el: Tag, depth: int = 0) -> list[str]:
|
||||
lines = []
|
||||
indent = " " * depth
|
||||
ordered = el.name == "ol"
|
||||
for i, li in enumerate(el.find_all("li", recursive=False), 1):
|
||||
own = []
|
||||
for child in li.children:
|
||||
if isinstance(child, Tag) and child.name in ("ul", "ol"):
|
||||
continue
|
||||
own.append(child.get_text(" ") if isinstance(child, Tag) else str(child))
|
||||
text = clean_text("".join(own))
|
||||
marker = f"{i}. " if ordered else "- "
|
||||
if text:
|
||||
lines.append(f"{indent}{marker}{text.replace(chr(10), chr(10) + indent + ' ')}")
|
||||
for sub in li.find_all(["ul", "ol"], recursive=False):
|
||||
lines.extend(list_to_md(sub, depth + 1))
|
||||
return lines
|
||||
|
||||
|
||||
def _is_toc_paragraph(p: Tag) -> bool:
|
||||
"""Absatz, der ausschließlich einen In-Seiten-Anker-Link enthält."""
|
||||
links = p.find_all("a", href=re.compile(r"^#"))
|
||||
if not links:
|
||||
return False
|
||||
rest = clean_text(p.get_text(" "))
|
||||
for a in links:
|
||||
rest = rest.replace(clean_text(a.get_text(" ")), "", 1)
|
||||
return not rest
|
||||
|
||||
|
||||
def blocks_to_md(el: Tag) -> list[str]:
|
||||
"""Rekursiver Block-Konverter; liefert Markdown-Zeilen."""
|
||||
lines: list[str] = []
|
||||
skip_toc = False # TOC-Absätze nach „Inhalt“-Überschrift auslassen
|
||||
for child in el.children:
|
||||
if isinstance(child, Comment):
|
||||
continue
|
||||
if isinstance(child, str):
|
||||
text = clean_text(child)
|
||||
if text:
|
||||
lines.extend([text, ""])
|
||||
continue
|
||||
if not isinstance(child, Tag):
|
||||
continue
|
||||
classes = " ".join(child.get("class", []))
|
||||
if child.name in NOISE_TAGS or NOISE_CLASSES.search(classes):
|
||||
continue
|
||||
if child.name in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
title = clean_text(child.get_text(" "))
|
||||
if not title:
|
||||
continue
|
||||
if title.casefold() == "inhalt":
|
||||
skip_toc = True
|
||||
continue
|
||||
prefix = "###" if child.name in ("h5", "h6") else "##"
|
||||
lines.extend([f"{prefix} {title}", ""])
|
||||
skip_toc = False
|
||||
continue
|
||||
if child.name == "p" and skip_toc and _is_toc_paragraph(child):
|
||||
continue # TOC-Einträge (reine Anker-Links) auslassen
|
||||
skip_toc = False
|
||||
if child.name in ("ul", "ol"):
|
||||
lines.extend(list_to_md(child))
|
||||
lines.append("")
|
||||
continue
|
||||
if child.name == "table":
|
||||
md = table_to_md(child)
|
||||
if md:
|
||||
lines.extend([md, ""])
|
||||
continue
|
||||
if child.name == "p":
|
||||
text = clean_text(child.get_text())
|
||||
if text:
|
||||
lines.extend([text, ""])
|
||||
continue
|
||||
if child.name == "dl":
|
||||
for dt in child.find_all("dt"):
|
||||
dd = dt.find_next_sibling("dd")
|
||||
if dd is not None:
|
||||
lines.append(f"- **{clean_text(dt.get_text(' '))}:** "
|
||||
f"{clean_text(dd.get_text(' '))}")
|
||||
lines.append("")
|
||||
continue
|
||||
if child.name in ("div", "section", "article", "main", "header", "footer"):
|
||||
lines.extend(blocks_to_md(child))
|
||||
continue
|
||||
# blockquote, pre, hr, sonstiges: Text durchreichen
|
||||
text = clean_text(child.get_text())
|
||||
if text:
|
||||
lines.extend([text, ""])
|
||||
return lines
|
||||
|
||||
|
||||
def parse_kv_meta(main: Tag) -> list[tuple[str, str]]:
|
||||
"""dt/dd-Paare aus dem meta-details-Block."""
|
||||
pairs = []
|
||||
box = main.find("div", class_="meta-details")
|
||||
if box is None:
|
||||
return pairs
|
||||
for dt in box.find_all("dt"):
|
||||
dd = dt.find_next_sibling("dd")
|
||||
label = clean_text(dt.get_text(" ")).rstrip(":")
|
||||
value = clean_text(dd.get_text(" ")) if dd is not None else ""
|
||||
if label:
|
||||
pairs.append((label, value))
|
||||
return pairs
|
||||
|
||||
|
||||
DOCTYPE_RULES: list[tuple[str, str]] = [
|
||||
("zusatzkollektivvertrag", "Zusatz-KV"),
|
||||
("zusatz-kv", "Zusatz-KV"),
|
||||
("zusatzvereinbarung", "Zusatz-KV"),
|
||||
("rahmenkollektivvertrag", "Rahmen-KV"),
|
||||
("rahmen-kv", "Rahmen-KV"),
|
||||
("generalkollektivvertrag", "Generalkollektivvertrag"),
|
||||
("general-kv", "Generalkollektivvertrag"),
|
||||
("mantelvertrag", "Mantelvertrag"),
|
||||
("berechnungsbeispiele", "Berechnungsbeispiele"),
|
||||
("fragen", "Fragen & Antworten"),
|
||||
("kommentar", "Kommentar"),
|
||||
("interpretation", "Interpretation"),
|
||||
("erläuterung", "Erläuterungen"),
|
||||
("handlungsempfehlung", "Handlungsempfehlung"),
|
||||
("empfehlung", "Empfehlung"),
|
||||
("information", "Information"),
|
||||
("erklärung", "Erklärung"),
|
||||
("klarstellung", "Klarstellung"),
|
||||
("änderung", "Änderung"),
|
||||
("anhang", "Anhang/Anlage"),
|
||||
("anlage", "Anhang/Anlage"),
|
||||
("sozialpartnervereinbarung", "Vereinbarung"),
|
||||
("vereinbarung", "Vereinbarung"),
|
||||
("punktation", "Vereinbarung"),
|
||||
("überleitungsschema", "Tabelle/Schema"),
|
||||
("umstiegsdienstzettel", "Tabelle/Schema"),
|
||||
("rückzahlungstabelle", "Tabelle/Schema"),
|
||||
("dienst- und besoldungsordnung", "Dienst- und Besoldungsordnung"),
|
||||
("gehaltsordnung", "Gehaltsordnung"),
|
||||
("gehaltsordnungen", "Gehaltsordnung"),
|
||||
("gehaltstabelle", "Gehaltsordnung"),
|
||||
("gehaltstafel", "Gehaltsordnung"),
|
||||
("lohnordnung", "Lohnordnung"),
|
||||
("lohntafeln", "Lohnordnung"),
|
||||
("lohntafel", "Lohnordnung"),
|
||||
("lohn- gehaltsordnung", "Lohn-/Gehaltsordnung"),
|
||||
("lohn-gehaltsordnung", "Lohn-/Gehaltsordnung"),
|
||||
("lohn- und gehaltsordnung", "Lohn-/Gehaltsordnung"),
|
||||
("heimarbeitstarif", "Heimarbeitstarif"),
|
||||
("kollektivvertragsabschluss", "KV-Abschluss"),
|
||||
("kollektivvertrag", "Kollektivvertrag"),
|
||||
]
|
||||
|
||||
|
||||
def classify_doctype(title: str) -> str:
|
||||
low = title.casefold()
|
||||
for needle, doctype in DOCTYPE_RULES:
|
||||
if low.startswith(needle):
|
||||
return doctype
|
||||
for needle, doctype in DOCTYPE_RULES:
|
||||
if needle in low:
|
||||
return doctype
|
||||
return "Sonstiges KV-Dokument"
|
||||
|
||||
|
||||
def derive_stand(title: str, meta: list[tuple[str, str]], filename: str) -> tuple[str, str]:
|
||||
"""(stand, tags) — explizites Datum > Titel-Datum > Titel-Jahr >
|
||||
Dateiname-Jahr > Beschaffungsstand."""
|
||||
for label, value in meta:
|
||||
if "geltungsdauer" in label.casefold():
|
||||
m = DATE_RE.search(value)
|
||||
if m:
|
||||
return stand_from_date(m.groups()), []
|
||||
m = YEAR_RE.search(value)
|
||||
if m:
|
||||
return f"{m.group(1)}-01", ["stand-jahr"]
|
||||
m = DATE_RE.search(title)
|
||||
if m:
|
||||
return stand_from_date(m.groups()), []
|
||||
m = YEAR_RE.search(title)
|
||||
if m:
|
||||
return f"{m.group(1)}-01", ["stand-jahr"]
|
||||
m = YEAR_RE.search(filename)
|
||||
if m:
|
||||
return f"{m.group(1)}-01", ["stand-jahr"]
|
||||
return CRAWL_STAND, ["stand-geschaetzt"]
|
||||
|
||||
|
||||
def kv_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None:
|
||||
html = path.read_text(encoding="utf-8")
|
||||
soup = prep_soup(html)
|
||||
og_title = soup.find("meta", property="og:title")
|
||||
og_url = soup.find("meta", property="og:url")
|
||||
if og_title is None or og_url is None:
|
||||
stats["no_og"].append(path.name)
|
||||
return None
|
||||
title = clean_text(og_title["content"])
|
||||
url = og_url["content"]
|
||||
main = soup.find("main", class_="col-lg-8")
|
||||
if main is None:
|
||||
stats["no_main"].append(path.name)
|
||||
return None
|
||||
meta = parse_kv_meta(main)
|
||||
body_blocks = blocks_to_md(main)
|
||||
|
||||
key = path.name
|
||||
entry = catalog["entries"].get(key)
|
||||
if entry is None:
|
||||
seq = catalog["next_seq"]
|
||||
catalog["next_seq"] = seq + 1
|
||||
slug = "kv_" + path.stem.lstrip("-")
|
||||
entry = {"id": f"kv-kvt-{seq:03d}", "slug": slug}
|
||||
catalog["entries"][key] = entry
|
||||
kid, slug = entry["id"], entry["slug"]
|
||||
|
||||
stand, stand_tags = derive_stand(title, meta, path.name)
|
||||
doctype = classify_doctype(title)
|
||||
year = YEAR_RE.search(path.stem)
|
||||
tags = ["kollektivvertrag", ascii_slug(doctype)]
|
||||
if year:
|
||||
tags.append(f"jahr-{year.group(1)}")
|
||||
tags.extend(stand_tags)
|
||||
|
||||
meta_rows = "\n".join(f"| {label} | {value} |" for label, value in meta)
|
||||
meta_section = ""
|
||||
if meta_rows:
|
||||
meta_section = (
|
||||
"## Geltungsbereich (WKO-Angaben)\n\n"
|
||||
"| Merkmal | Angabe |\n|---|---|\n" + meta_rows + "\n\n"
|
||||
)
|
||||
body = "\n".join(body_blocks).strip()
|
||||
md = (
|
||||
f"---\n"
|
||||
f"id: {kid}\n"
|
||||
f"batch: 1\n"
|
||||
f"title: {json.dumps(title, ensure_ascii=False)}\n"
|
||||
f"work: {json.dumps(WORK_KV, ensure_ascii=False)}\n"
|
||||
f"chapter: {json.dumps(doctype, ensure_ascii=False)}\n"
|
||||
f"topic: kollektivvertraege\n"
|
||||
f"author: \"WKO\"\n"
|
||||
f"stand: {stand}\n"
|
||||
f"source:\n"
|
||||
f" html: \".firecrawl/wko-kv/docs/{path.name}\"\n"
|
||||
f"legal_bases: []\n"
|
||||
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
|
||||
f"cross_refs: []\n"
|
||||
f"---\n\n"
|
||||
f"# {title}\n\n"
|
||||
f"*{WORK_KV}, WKO, Stand {stand} ({kid}). Quelle: {url}*\n\n"
|
||||
f"{meta_section}"
|
||||
f"{body}\n"
|
||||
)
|
||||
entry.update({"title": title, "stand": stand, "doctype": doctype, "url": url})
|
||||
stats["kv_written"] += 1
|
||||
return slug, md
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# RIS: Markdown -> Layer-2
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
GNR_RE = re.compile(r"Gesetzesnummer[:\s]+(\d+)")
|
||||
ISO_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})")
|
||||
|
||||
|
||||
def ris_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None:
|
||||
key = path.name
|
||||
base = path.stem
|
||||
law = LAW_MAP.get(base)
|
||||
if law is None:
|
||||
stats["ris_unmapped"].append(base)
|
||||
return None
|
||||
topic, chapter, nn = law
|
||||
|
||||
raw = path.read_text(encoding="utf-8").replace("\xa0", " ")
|
||||
lines = raw.split("\n")
|
||||
title = lines[0].lstrip("# ").strip()
|
||||
preamble, sections = raw, ""
|
||||
m = re.search(r"^## .*$", raw, flags=re.M)
|
||||
if m:
|
||||
preamble, sections = raw[: m.start()], raw[m.start():]
|
||||
preamble_lines = [ln for ln in preamble.split("\n")[1:] if ln.strip()]
|
||||
|
||||
gnr = GNR_RE.search(preamble)
|
||||
stand_match = ISO_DATE_RE.search(preamble)
|
||||
stand = stand_match.group(1)[:7] if stand_match else CRAWL_STAND
|
||||
|
||||
entry = catalog["entries"].get(key)
|
||||
if entry is None:
|
||||
prefix = topic_prefix(topic)
|
||||
kid = f"ris-{prefix}-{nn:02d}"
|
||||
entry = {"id": kid}
|
||||
catalog["entries"][key] = entry
|
||||
kid = entry["id"]
|
||||
|
||||
abbr = extract_law_abbreviation(title, base)
|
||||
tags = ["gesetz", ascii_slug(abbr), ascii_slug(topic)]
|
||||
md = (
|
||||
f"---\n"
|
||||
f"id: {kid}\n"
|
||||
f"batch: 1\n"
|
||||
f"title: {json.dumps(title, ensure_ascii=False)}\n"
|
||||
f"work: {json.dumps(WORK_RIS, ensure_ascii=False)}\n"
|
||||
f"chapter: {json.dumps(chapter, ensure_ascii=False)}\n"
|
||||
f"topic: {topic}\n"
|
||||
f"author: \"RIS (Bundeskanzleramt)\"\n"
|
||||
f"stand: {stand}\n"
|
||||
f"source:\n"
|
||||
f" text: \".ris/{path.name}\"\n"
|
||||
f"legal_bases: {json.dumps([abbr], ensure_ascii=False)}\n"
|
||||
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
|
||||
f"cross_refs: []\n"
|
||||
f"---\n\n"
|
||||
f"# {title}\n\n"
|
||||
+ "\n".join(preamble_lines).strip() + "\n\n"
|
||||
f"*Wissensbasis: {WORK_RIS}, Stand {stand} ({kid}).*\n\n"
|
||||
f"{sections.strip()}\n"
|
||||
)
|
||||
entry.update({"title": title, "stand": stand, "gnr": gnr.group(1) if gnr else None})
|
||||
stats["ris_written"] += 1
|
||||
return f"ris_{ascii_slug(base)}", md
|
||||
|
||||
|
||||
def extract_law_abbreviation(title: str, base: str) -> str:
|
||||
"""Abkürzung aus dem Titel: 'X (ABK)' am Ende oder 'ABK – Name' am Anfang."""
|
||||
parens = re.findall(r"\(([^()]+)\)\s*$", title.strip())
|
||||
if parens:
|
||||
return parens[-1].strip()
|
||||
m = re.match(r"^([A-Za-zÄÖÜäöüß\-]+)\s*[–-]\s", title.strip())
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
return base
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# CLI
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def write_entries(results: list[tuple[str, str]], dry: bool) -> int:
|
||||
n = 0
|
||||
for slug, md in results:
|
||||
target = DOCS / f"{slug}.md"
|
||||
if not dry:
|
||||
if target.is_file() and target.read_text(encoding="utf-8") == md:
|
||||
continue
|
||||
target.write_text(md, encoding="utf-8")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def run(source: str, dry: bool, limit: int | None, single: str | None) -> int:
|
||||
stats: dict = {"kv_written": 0, "ris_written": 0, "no_og": [], "no_main": [],
|
||||
"ris_unmapped": []}
|
||||
written = 0
|
||||
|
||||
if source in ("kv", "all"):
|
||||
catalog = load_catalog("kv")
|
||||
files = sorted(KV_DIR.glob("*.html"))
|
||||
if single:
|
||||
files = [f for f in files if single in f.name]
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
results = []
|
||||
for f in files:
|
||||
res = kv_entry(f, catalog, stats)
|
||||
if res:
|
||||
results.append(res)
|
||||
written += write_entries(results, dry)
|
||||
if not dry:
|
||||
save_catalog(catalog)
|
||||
|
||||
if source in ("ris", "all"):
|
||||
catalog = load_catalog("ris")
|
||||
files = sorted(RIS_DIR.glob("*.md"))
|
||||
if single:
|
||||
files = [f for f in files if single in f.name]
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
results = []
|
||||
for f in files:
|
||||
res = ris_entry(f, catalog, stats)
|
||||
if res:
|
||||
results.append(res)
|
||||
written += write_entries(results, dry)
|
||||
if not dry:
|
||||
save_catalog(catalog)
|
||||
|
||||
print(f"source={source} dry={dry} written={written}")
|
||||
print(f" kv: {stats['kv_written']} ris: {stats['ris_written']}")
|
||||
for key in ("no_og", "no_main", "ris_unmapped"):
|
||||
if stats[key]:
|
||||
print(f" {key}: {stats[key]}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--source", choices=("kv", "ris", "all"), default="all")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--limit", type=int, default=None)
|
||||
ap.add_argument("--file", default=None, help="Teilstring des Quelldateinamens")
|
||||
args = ap.parse_args()
|
||||
return run(args.source, args.dry_run, args.limit, args.file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+23
-1
@@ -14,6 +14,7 @@ Cluster oder landen bewusst im Sammel-Cluster `normen-sonstige`.
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# Alle ID-Räume des Korpus (Schema-Beschreibung für kb.json).
|
||||
ID_SPACES = {
|
||||
@@ -87,7 +88,7 @@ LAW_MAP: dict[str, tuple[str, str, int]] = {
|
||||
"GSVG": ("gsvg-fsvg", "Sozialversicherungsrecht", 1),
|
||||
"GewO": ("gewerbe", "Gewerberecht", 1),
|
||||
"GlBG": ("gleichbehandlung", "Arbeitsrecht", 1),
|
||||
"GmbHH_PLACEHOLDER": ("geschaftsfuhrer", "Gesellschaftsrecht", 1),
|
||||
"GmbHG": ("geschaftsfuhrer", "Gesellschaftsrecht", 1),
|
||||
"IESG": ("insolvenz-betriebsubergang", "Insolvenzrecht", 1),
|
||||
"KJBG": ("jugendliche", "Jugendschutz", 1),
|
||||
"LSD-BG": ("lohndumping", "Arbeitsrecht", 1),
|
||||
@@ -107,6 +108,27 @@ DATE_RE = re.compile(r"(\d{1,2})\.\s*(\d{1,2})\.\s*(\d{4})")
|
||||
YEAR_RE = re.compile(r"(20\d\d)")
|
||||
|
||||
|
||||
def load_topic_prefix_map(kb_json_path) -> dict[str, str]:
|
||||
"""topic -> ID-Präfix über ALLE Cluster: aus den bestehenden Layer-2-
|
||||
Einträgen (kb.json) abgeleitet, ergänzt um die neuen Cluster."""
|
||||
import json
|
||||
|
||||
kb = json.loads(Path(kb_json_path).read_text(encoding="utf-8"))
|
||||
mapping: dict[str, str] = {}
|
||||
for e in kb["entries"]:
|
||||
prefix = e["id"].split("-")[1]
|
||||
known = mapping.get(e["topic"])
|
||||
if known is not None and known != prefix:
|
||||
raise SystemExit(
|
||||
f"inkonsistente Cluster-Map: topic '{e['topic']}' hat Präfixe "
|
||||
f"{known} und {prefix}"
|
||||
)
|
||||
mapping[e["topic"]] = prefix
|
||||
for slug, (prefix, _name) in NEW_CLUSTERS.items():
|
||||
mapping[slug] = prefix
|
||||
return mapping
|
||||
|
||||
|
||||
def ascii_slug(text: str) -> str:
|
||||
"""Wissensbasis-Konvention: Umlaute auf Basisbuchstabe (ü→u, nicht ue),
|
||||
ß→ss, Rest klein; alles außer a-z0-9 und Bindestrich fällt weg."""
|
||||
|
||||
Reference in New Issue
Block a user