Files
pv-agent/tools/ingest_sources.py
T
fegger ac060c4977 KV/RIS-Erweiterung: 614 WKO-KV-Dokumente + 59 RIS-Gesetze (D9)
- Korpus 601 -> 1274 Layer-2-Eintraege: kv-kvt-001...614 (ein Cluster
  kollektivvertraege, Branche als Tag) und ris-<cluster-prefix>-nn auf
  der bestehenden Cluster-Map + 7 neue Cluster (zvr/avr/agg/lst/abo/
  nso/kvt); LAW_MAP dokumentiert die 59 Gesetz-Zuordnungen.
- kv/ris-Eintraege sind quellentreu generiert (D9) - Gesetze sind
  amtliche Werke, KV-Lohntabellen zahlenexakt; Tool-Output in
  tools/ (ingest_sources.py, build_registry.py, kb_common.py),
  eingefrorene ID-Kataloge tools/catalogs/*.json (nur Metadaten).
- kb.json/INDEX.md regeneriert (1274 Eintraege, 76 Cluster);
  agent/kb.py: ID-Raeume kv|ris, source akzeptiert html-only.
- Tests 41 -> 49 (Konverter, LAW_MAP-Abdeckung, neue ID-Raeume,
  Korpus-Integrationszahl).
2026-09-15 07:44:56 +02:00

535 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Intake der Quellen `kv` (WKO.at-KV-Seiten) und `ris` (RIS-Gesetze) in die
Layer-2-Wissensbasis.
Abweichend von den lizenzierten Quellen (Lexis/WIKU: Eigene-Worte-Kuratierung)
werden kv/ris-Einträge **quellentreu generiert** (Entscheidung 2026-09-15):
Gesetze sind amtliche Werke, KV-Lohntabellen müssen zahlenexakt bleiben.
Die generierten Dateien (`kv_*.md`, `ris_*.md`) sind damit Tool-Output —
keine manuelle Kuratierung, bei Änderungen neu generieren.
IDs werden aus den Katalogen `tools/catalogs/<source>_catalog.json` vergeben
und danach eingefroren (Wissensbasis-Konvention: nie wiederverwenden).
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from datetime import datetime, timezone
from pathlib import Path
from bs4 import BeautifulSoup, Comment, Tag
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pathlib import Path
from tools.kb_common import ( # noqa: E402
DATE_RE,
LAW_MAP,
WORK_KV,
WORK_RIS,
YEAR_RE,
ascii_slug,
load_topic_prefix_map,
stand_from_date,
)
ROOT = Path(__file__).resolve().parents[1]
KV_DIR = ROOT / ".firecrawl" / "wko-kv" / "docs"
RIS_DIR = ROOT / ".ris"
DOCS = ROOT / "wissensbasis" / "dokumente"
CATALOGS = ROOT / "tools" / "catalogs"
CRAWL_STAND = "2026-09" # Beschaffungsstand der beiden Quellen
TOPIC_PREFIX_FULL: dict[str, str] | None = None
def topic_prefix(topic: str) -> str:
global TOPIC_PREFIX_FULL
if TOPIC_PREFIX_FULL is None:
TOPIC_PREFIX_FULL = load_topic_prefix_map(ROOT / "wissensbasis" / "kb.json")
return TOPIC_PREFIX_FULL[topic]
# --------------------------------------------------------------------------
# Kataloge (eingefrorene IDs)
# --------------------------------------------------------------------------
def load_catalog(source: str) -> dict:
path = CATALOGS / f"{source}_catalog.json"
if path.is_file():
return json.loads(path.read_text(encoding="utf-8"))
return {"source": source, "updated": None, "next_seq": 1, "entries": {}}
def save_catalog(catalog: dict) -> None:
catalog["updated"] = datetime.now(timezone.utc).isoformat()
CATALOGS.mkdir(parents=True, exist_ok=True)
path = CATALOGS / f"{catalog['source']}_catalog.json"
path.write_text(
json.dumps(catalog, ensure_ascii=False, indent=1, sort_keys=True) + "\n",
encoding="utf-8",
)
# --------------------------------------------------------------------------
# WKO-KV: HTML -> Markdown
# --------------------------------------------------------------------------
NOISE_TAGS = ("script", "style", "button", "svg", "nav", "iframe", "form", "img")
NOISE_CLASSES = re.compile(
r"social|share|related|teaser|breadcrumb|pagination|print|meta-details", re.I
)
def prep_soup(html: str) -> BeautifulSoup:
soup = BeautifulSoup(html, "html.parser")
for tag in soup.find_all(NOISE_TAGS):
tag.decompose()
for br in soup.find_all("br"):
br.replace_with("\n")
return soup
def clean_text(text: str) -> str:
"""Zeilenweise strippen, interne Whitespace-Läufe kollabieren."""
lines = []
for line in text.split("\n"):
line = re.sub(r"[ \t\xa0]+", " ", line).strip()
if line:
lines.append(line)
return "\n".join(lines)
def cell_text(cell: Tag) -> str:
text = cell.get_text(" ")
return re.sub(r"\s+", " ", text).strip().replace("|", "\\|")
def table_to_md(table: Tag) -> str:
rows = []
for tr in table.find_all("tr"):
if tr.find_parent("table") is not table:
continue # verschachtelte Tabellen nur einmal ausgeben
cells = [cell_text(c) for c in tr.find_all(["td", "th"], recursive=False)]
if cells:
rows.append(cells)
if not rows:
return ""
width = max(len(r) for r in rows)
rows = [r + [""] * (width - len(r)) for r in rows]
out = ["| " + " | ".join(rows[0]) + " |"]
out.append("| " + " | ".join(["---"] * width) + " |")
for r in rows[1:]:
out.append("| " + " | ".join(r) + " |")
return "\n".join(out)
def list_to_md(el: Tag, depth: int = 0) -> list[str]:
lines = []
indent = " " * depth
ordered = el.name == "ol"
for i, li in enumerate(el.find_all("li", recursive=False), 1):
own = []
for child in li.children:
if isinstance(child, Tag) and child.name in ("ul", "ol"):
continue
own.append(child.get_text(" ") if isinstance(child, Tag) else str(child))
text = clean_text("".join(own))
marker = f"{i}. " if ordered else "- "
if text:
lines.append(f"{indent}{marker}{text.replace(chr(10), chr(10) + indent + ' ')}")
for sub in li.find_all(["ul", "ol"], recursive=False):
lines.extend(list_to_md(sub, depth + 1))
return lines
def _is_toc_paragraph(p: Tag) -> bool:
"""Absatz, der ausschließlich einen In-Seiten-Anker-Link enthält."""
links = p.find_all("a", href=re.compile(r"^#"))
if not links:
return False
rest = clean_text(p.get_text(" "))
for a in links:
rest = rest.replace(clean_text(a.get_text(" ")), "", 1)
return not rest
def blocks_to_md(el: Tag) -> list[str]:
"""Rekursiver Block-Konverter; liefert Markdown-Zeilen."""
lines: list[str] = []
skip_toc = False # TOC-Absätze nach „Inhalt“-Überschrift auslassen
for child in el.children:
if isinstance(child, Comment):
continue
if isinstance(child, str):
text = clean_text(child)
if text:
lines.extend([text, ""])
continue
if not isinstance(child, Tag):
continue
classes = " ".join(child.get("class", []))
if child.name in NOISE_TAGS or NOISE_CLASSES.search(classes):
continue
if child.name in ("h1", "h2", "h3", "h4", "h5", "h6"):
title = clean_text(child.get_text(" "))
if not title:
continue
if title.casefold() == "inhalt":
skip_toc = True
continue
prefix = "###" if child.name in ("h5", "h6") else "##"
lines.extend([f"{prefix} {title}", ""])
skip_toc = False
continue
if child.name == "p" and skip_toc and _is_toc_paragraph(child):
continue # TOC-Einträge (reine Anker-Links) auslassen
skip_toc = False
if child.name in ("ul", "ol"):
lines.extend(list_to_md(child))
lines.append("")
continue
if child.name == "table":
md = table_to_md(child)
if md:
lines.extend([md, ""])
continue
if child.name == "p":
text = clean_text(child.get_text())
if text:
lines.extend([text, ""])
continue
if child.name == "dl":
for dt in child.find_all("dt"):
dd = dt.find_next_sibling("dd")
if dd is not None:
lines.append(f"- **{clean_text(dt.get_text(' '))}:** "
f"{clean_text(dd.get_text(' '))}")
lines.append("")
continue
if child.name in ("div", "section", "article", "main", "header", "footer"):
lines.extend(blocks_to_md(child))
continue
# blockquote, pre, hr, sonstiges: Text durchreichen
text = clean_text(child.get_text())
if text:
lines.extend([text, ""])
return lines
def parse_kv_meta(main: Tag) -> list[tuple[str, str]]:
"""dt/dd-Paare aus dem meta-details-Block."""
pairs = []
box = main.find("div", class_="meta-details")
if box is None:
return pairs
for dt in box.find_all("dt"):
dd = dt.find_next_sibling("dd")
label = clean_text(dt.get_text(" ")).rstrip(":")
value = clean_text(dd.get_text(" ")) if dd is not None else ""
if label:
pairs.append((label, value))
return pairs
DOCTYPE_RULES: list[tuple[str, str]] = [
("zusatzkollektivvertrag", "Zusatz-KV"),
("zusatz-kv", "Zusatz-KV"),
("zusatzvereinbarung", "Zusatz-KV"),
("rahmenkollektivvertrag", "Rahmen-KV"),
("rahmen-kv", "Rahmen-KV"),
("generalkollektivvertrag", "Generalkollektivvertrag"),
("general-kv", "Generalkollektivvertrag"),
("mantelvertrag", "Mantelvertrag"),
("berechnungsbeispiele", "Berechnungsbeispiele"),
("fragen", "Fragen & Antworten"),
("kommentar", "Kommentar"),
("interpretation", "Interpretation"),
("erläuterung", "Erläuterungen"),
("handlungsempfehlung", "Handlungsempfehlung"),
("empfehlung", "Empfehlung"),
("information", "Information"),
("erklärung", "Erklärung"),
("klarstellung", "Klarstellung"),
("änderung", "Änderung"),
("anhang", "Anhang/Anlage"),
("anlage", "Anhang/Anlage"),
("sozialpartnervereinbarung", "Vereinbarung"),
("vereinbarung", "Vereinbarung"),
("punktation", "Vereinbarung"),
("überleitungsschema", "Tabelle/Schema"),
("umstiegsdienstzettel", "Tabelle/Schema"),
("rückzahlungstabelle", "Tabelle/Schema"),
("dienst- und besoldungsordnung", "Dienst- und Besoldungsordnung"),
("gehaltsordnung", "Gehaltsordnung"),
("gehaltsordnungen", "Gehaltsordnung"),
("gehaltstabelle", "Gehaltsordnung"),
("gehaltstafel", "Gehaltsordnung"),
("lohnordnung", "Lohnordnung"),
("lohntafeln", "Lohnordnung"),
("lohntafel", "Lohnordnung"),
("lohn- gehaltsordnung", "Lohn-/Gehaltsordnung"),
("lohn-gehaltsordnung", "Lohn-/Gehaltsordnung"),
("lohn- und gehaltsordnung", "Lohn-/Gehaltsordnung"),
("heimarbeitstarif", "Heimarbeitstarif"),
("kollektivvertragsabschluss", "KV-Abschluss"),
("kollektivvertrag", "Kollektivvertrag"),
]
def classify_doctype(title: str) -> str:
low = title.casefold()
for needle, doctype in DOCTYPE_RULES:
if low.startswith(needle):
return doctype
for needle, doctype in DOCTYPE_RULES:
if needle in low:
return doctype
return "Sonstiges KV-Dokument"
def derive_stand(title: str, meta: list[tuple[str, str]], filename: str) -> tuple[str, str]:
"""(stand, tags) — explizites Datum > Titel-Datum > Titel-Jahr >
Dateiname-Jahr > Beschaffungsstand."""
for label, value in meta:
if "geltungsdauer" in label.casefold():
m = DATE_RE.search(value)
if m:
return stand_from_date(m.groups()), []
m = YEAR_RE.search(value)
if m:
return f"{m.group(1)}-01", ["stand-jahr"]
m = DATE_RE.search(title)
if m:
return stand_from_date(m.groups()), []
m = YEAR_RE.search(title)
if m:
return f"{m.group(1)}-01", ["stand-jahr"]
m = YEAR_RE.search(filename)
if m:
return f"{m.group(1)}-01", ["stand-jahr"]
return CRAWL_STAND, ["stand-geschaetzt"]
def kv_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None:
html = path.read_text(encoding="utf-8")
soup = prep_soup(html)
og_title = soup.find("meta", property="og:title")
og_url = soup.find("meta", property="og:url")
if og_title is None or og_url is None:
stats["no_og"].append(path.name)
return None
title = clean_text(og_title["content"])
url = og_url["content"]
main = soup.find("main", class_="col-lg-8")
if main is None:
stats["no_main"].append(path.name)
return None
meta = parse_kv_meta(main)
body_blocks = blocks_to_md(main)
key = path.name
entry = catalog["entries"].get(key)
if entry is None:
seq = catalog["next_seq"]
catalog["next_seq"] = seq + 1
slug = "kv_" + path.stem.lstrip("-")
entry = {"id": f"kv-kvt-{seq:03d}", "slug": slug}
catalog["entries"][key] = entry
kid, slug = entry["id"], entry["slug"]
stand, stand_tags = derive_stand(title, meta, path.name)
doctype = classify_doctype(title)
year = YEAR_RE.search(path.stem)
tags = ["kollektivvertrag", ascii_slug(doctype)]
if year:
tags.append(f"jahr-{year.group(1)}")
tags.extend(stand_tags)
meta_rows = "\n".join(f"| {label} | {value} |" for label, value in meta)
meta_section = ""
if meta_rows:
meta_section = (
"## Geltungsbereich (WKO-Angaben)\n\n"
"| Merkmal | Angabe |\n|---|---|\n" + meta_rows + "\n\n"
)
body = "\n".join(body_blocks).strip()
md = (
f"---\n"
f"id: {kid}\n"
f"batch: 1\n"
f"title: {json.dumps(title, ensure_ascii=False)}\n"
f"work: {json.dumps(WORK_KV, ensure_ascii=False)}\n"
f"chapter: {json.dumps(doctype, ensure_ascii=False)}\n"
f"topic: kollektivvertraege\n"
f"author: \"WKO\"\n"
f"stand: {stand}\n"
f"source:\n"
f" html: \".firecrawl/wko-kv/docs/{path.name}\"\n"
f"legal_bases: []\n"
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
f"cross_refs: []\n"
f"---\n\n"
f"# {title}\n\n"
f"*{WORK_KV}, WKO, Stand {stand} ({kid}). Quelle: {url}*\n\n"
f"{meta_section}"
f"{body}\n"
)
entry.update({"title": title, "stand": stand, "doctype": doctype, "url": url})
stats["kv_written"] += 1
return slug, md
# --------------------------------------------------------------------------
# RIS: Markdown -> Layer-2
# --------------------------------------------------------------------------
GNR_RE = re.compile(r"Gesetzesnummer[:\s]+(\d+)")
ISO_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})")
def ris_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None:
key = path.name
base = path.stem
law = LAW_MAP.get(base)
if law is None:
stats["ris_unmapped"].append(base)
return None
topic, chapter, nn = law
raw = path.read_text(encoding="utf-8").replace("\xa0", " ")
lines = raw.split("\n")
title = lines[0].lstrip("# ").strip()
preamble, sections = raw, ""
m = re.search(r"^## .*$", raw, flags=re.M)
if m:
preamble, sections = raw[: m.start()], raw[m.start():]
preamble_lines = [ln for ln in preamble.split("\n")[1:] if ln.strip()]
gnr = GNR_RE.search(preamble)
stand_match = ISO_DATE_RE.search(preamble)
stand = stand_match.group(1)[:7] if stand_match else CRAWL_STAND
entry = catalog["entries"].get(key)
if entry is None:
prefix = topic_prefix(topic)
kid = f"ris-{prefix}-{nn:02d}"
entry = {"id": kid}
catalog["entries"][key] = entry
kid = entry["id"]
abbr = extract_law_abbreviation(title, base)
tags = ["gesetz", ascii_slug(abbr), ascii_slug(topic)]
md = (
f"---\n"
f"id: {kid}\n"
f"batch: 1\n"
f"title: {json.dumps(title, ensure_ascii=False)}\n"
f"work: {json.dumps(WORK_RIS, ensure_ascii=False)}\n"
f"chapter: {json.dumps(chapter, ensure_ascii=False)}\n"
f"topic: {topic}\n"
f"author: \"RIS (Bundeskanzleramt)\"\n"
f"stand: {stand}\n"
f"source:\n"
f" text: \".ris/{path.name}\"\n"
f"legal_bases: {json.dumps([abbr], ensure_ascii=False)}\n"
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
f"cross_refs: []\n"
f"---\n\n"
f"# {title}\n\n"
+ "\n".join(preamble_lines).strip() + "\n\n"
f"*Wissensbasis: {WORK_RIS}, Stand {stand} ({kid}).*\n\n"
f"{sections.strip()}\n"
)
entry.update({"title": title, "stand": stand, "gnr": gnr.group(1) if gnr else None})
stats["ris_written"] += 1
return f"ris_{ascii_slug(base)}", md
def extract_law_abbreviation(title: str, base: str) -> str:
"""Abkürzung aus dem Titel: 'X (ABK)' am Ende oder 'ABK Name' am Anfang."""
parens = re.findall(r"\(([^()]+)\)\s*$", title.strip())
if parens:
return parens[-1].strip()
m = re.match(r"^([A-Za-zÄÖÜäöüß\-]+)\s*[-]\s", title.strip())
if m:
return m.group(1).strip()
return base
# --------------------------------------------------------------------------
# CLI
# --------------------------------------------------------------------------
def write_entries(results: list[tuple[str, str]], dry: bool) -> int:
n = 0
for slug, md in results:
target = DOCS / f"{slug}.md"
if not dry:
if target.is_file() and target.read_text(encoding="utf-8") == md:
continue
target.write_text(md, encoding="utf-8")
n += 1
return n
def run(source: str, dry: bool, limit: int | None, single: str | None) -> int:
stats: dict = {"kv_written": 0, "ris_written": 0, "no_og": [], "no_main": [],
"ris_unmapped": []}
written = 0
if source in ("kv", "all"):
catalog = load_catalog("kv")
files = sorted(KV_DIR.glob("*.html"))
if single:
files = [f for f in files if single in f.name]
if limit:
files = files[:limit]
results = []
for f in files:
res = kv_entry(f, catalog, stats)
if res:
results.append(res)
written += write_entries(results, dry)
if not dry:
save_catalog(catalog)
if source in ("ris", "all"):
catalog = load_catalog("ris")
files = sorted(RIS_DIR.glob("*.md"))
if single:
files = [f for f in files if single in f.name]
if limit:
files = files[:limit]
results = []
for f in files:
res = ris_entry(f, catalog, stats)
if res:
results.append(res)
written += write_entries(results, dry)
if not dry:
save_catalog(catalog)
print(f"source={source} dry={dry} written={written}")
print(f" kv: {stats['kv_written']} ris: {stats['ris_written']}")
for key in ("no_og", "no_main", "ris_unmapped"):
if stats[key]:
print(f" {key}: {stats[key]}")
return 0
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--source", choices=("kv", "ris", "all"), default="all")
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--limit", type=int, default=None)
ap.add_argument("--file", default=None, help="Teilstring des Quelldateinamens")
args = ap.parse_args()
return run(args.source, args.dry_run, args.limit, args.file)
if __name__ == "__main__":
sys.exit(main())