"""Intake der Quellen `kv` (WKO.at-KV-Seiten) und `ris` (RIS-Gesetze) in die Layer-2-Wissensbasis. Abweichend von den lizenzierten Quellen (Lexis/WIKU: Eigene-Worte-Kuratierung) werden kv/ris-Einträge **quellentreu generiert** (Entscheidung 2026-09-15): Gesetze sind amtliche Werke, KV-Lohntabellen müssen zahlenexakt bleiben. Die generierten Dateien (`kv_*.md`, `ris_*.md`) sind damit Tool-Output — keine manuelle Kuratierung, bei Änderungen neu generieren. IDs werden aus den Katalogen `tools/catalogs/_catalog.json` vergeben und danach eingefroren (Wissensbasis-Konvention: nie wiederverwenden). """ from __future__ import annotations import argparse import json import re import sys from datetime import datetime, timezone from pathlib import Path from bs4 import BeautifulSoup, Comment, Tag sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from pathlib import Path from tools.kb_common import ( # noqa: E402 DATE_RE, LAW_MAP, WORK_KV, WORK_RIS, YEAR_RE, ascii_slug, load_topic_prefix_map, stand_from_date, ) ROOT = Path(__file__).resolve().parents[1] KV_DIR = ROOT / ".firecrawl" / "wko-kv" / "docs" RIS_DIR = ROOT / ".ris" DOCS = ROOT / "wissensbasis" / "dokumente" CATALOGS = ROOT / "tools" / "catalogs" CRAWL_STAND = "2026-09" # Beschaffungsstand der beiden Quellen TOPIC_PREFIX_FULL: dict[str, str] | None = None def topic_prefix(topic: str) -> str: global TOPIC_PREFIX_FULL if TOPIC_PREFIX_FULL is None: TOPIC_PREFIX_FULL = load_topic_prefix_map(ROOT / "wissensbasis" / "kb.json") return TOPIC_PREFIX_FULL[topic] # -------------------------------------------------------------------------- # Kataloge (eingefrorene IDs) # -------------------------------------------------------------------------- def load_catalog(source: str) -> dict: path = CATALOGS / f"{source}_catalog.json" if path.is_file(): return json.loads(path.read_text(encoding="utf-8")) return {"source": source, "updated": None, "next_seq": 1, "entries": {}} def save_catalog(catalog: dict) -> None: catalog["updated"] = datetime.now(timezone.utc).isoformat() CATALOGS.mkdir(parents=True, exist_ok=True) path = CATALOGS / f"{catalog['source']}_catalog.json" path.write_text( json.dumps(catalog, ensure_ascii=False, indent=1, sort_keys=True) + "\n", encoding="utf-8", ) # -------------------------------------------------------------------------- # WKO-KV: HTML -> Markdown # -------------------------------------------------------------------------- NOISE_TAGS = ("script", "style", "button", "svg", "nav", "iframe", "form", "img") NOISE_CLASSES = re.compile( r"social|share|related|teaser|breadcrumb|pagination|print|meta-details", re.I ) def prep_soup(html: str) -> BeautifulSoup: soup = BeautifulSoup(html, "html.parser") for tag in soup.find_all(NOISE_TAGS): tag.decompose() for br in soup.find_all("br"): br.replace_with("\n") return soup def clean_text(text: str) -> str: """Zeilenweise strippen, interne Whitespace-Läufe kollabieren.""" lines = [] for line in text.split("\n"): line = re.sub(r"[ \t\xa0]+", " ", line).strip() if line: lines.append(line) return "\n".join(lines) def cell_text(cell: Tag) -> str: text = cell.get_text(" ") return re.sub(r"\s+", " ", text).strip().replace("|", "\\|") def table_to_md(table: Tag) -> str: rows = [] for tr in table.find_all("tr"): if tr.find_parent("table") is not table: continue # verschachtelte Tabellen nur einmal ausgeben cells = [cell_text(c) for c in tr.find_all(["td", "th"], recursive=False)] if cells: rows.append(cells) if not rows: return "" width = max(len(r) for r in rows) rows = [r + [""] * (width - len(r)) for r in rows] out = ["| " + " | ".join(rows[0]) + " |"] out.append("| " + " | ".join(["---"] * width) + " |") for r in rows[1:]: out.append("| " + " | ".join(r) + " |") return "\n".join(out) def list_to_md(el: Tag, depth: int = 0) -> list[str]: lines = [] indent = " " * depth ordered = el.name == "ol" for i, li in enumerate(el.find_all("li", recursive=False), 1): own = [] for child in li.children: if isinstance(child, Tag) and child.name in ("ul", "ol"): continue own.append(child.get_text(" ") if isinstance(child, Tag) else str(child)) text = clean_text("".join(own)) marker = f"{i}. " if ordered else "- " if text: lines.append(f"{indent}{marker}{text.replace(chr(10), chr(10) + indent + ' ')}") for sub in li.find_all(["ul", "ol"], recursive=False): lines.extend(list_to_md(sub, depth + 1)) return lines def _is_toc_paragraph(p: Tag) -> bool: """Absatz, der ausschließlich einen In-Seiten-Anker-Link enthält.""" links = p.find_all("a", href=re.compile(r"^#")) if not links: return False rest = clean_text(p.get_text(" ")) for a in links: rest = rest.replace(clean_text(a.get_text(" ")), "", 1) return not rest def blocks_to_md(el: Tag) -> list[str]: """Rekursiver Block-Konverter; liefert Markdown-Zeilen.""" lines: list[str] = [] skip_toc = False # TOC-Absätze nach „Inhalt“-Überschrift auslassen for child in el.children: if isinstance(child, Comment): continue if isinstance(child, str): text = clean_text(child) if text: lines.extend([text, ""]) continue if not isinstance(child, Tag): continue classes = " ".join(child.get("class", [])) if child.name in NOISE_TAGS or NOISE_CLASSES.search(classes): continue if child.name in ("h1", "h2", "h3", "h4", "h5", "h6"): title = clean_text(child.get_text(" ")) if not title: continue if title.casefold() == "inhalt": skip_toc = True continue prefix = "###" if child.name in ("h5", "h6") else "##" lines.extend([f"{prefix} {title}", ""]) skip_toc = False continue if child.name == "p" and skip_toc and _is_toc_paragraph(child): continue # TOC-Einträge (reine Anker-Links) auslassen skip_toc = False if child.name in ("ul", "ol"): lines.extend(list_to_md(child)) lines.append("") continue if child.name == "table": md = table_to_md(child) if md: lines.extend([md, ""]) continue if child.name == "p": text = clean_text(child.get_text()) if text: lines.extend([text, ""]) continue if child.name == "dl": for dt in child.find_all("dt"): dd = dt.find_next_sibling("dd") if dd is not None: lines.append(f"- **{clean_text(dt.get_text(' '))}:** " f"{clean_text(dd.get_text(' '))}") lines.append("") continue if child.name in ("div", "section", "article", "main", "header", "footer"): lines.extend(blocks_to_md(child)) continue # blockquote, pre, hr, sonstiges: Text durchreichen text = clean_text(child.get_text()) if text: lines.extend([text, ""]) return lines def parse_kv_meta(main: Tag) -> list[tuple[str, str]]: """dt/dd-Paare aus dem meta-details-Block.""" pairs = [] box = main.find("div", class_="meta-details") if box is None: return pairs for dt in box.find_all("dt"): dd = dt.find_next_sibling("dd") label = clean_text(dt.get_text(" ")).rstrip(":") value = clean_text(dd.get_text(" ")) if dd is not None else "" if label: pairs.append((label, value)) return pairs DOCTYPE_RULES: list[tuple[str, str]] = [ ("zusatzkollektivvertrag", "Zusatz-KV"), ("zusatz-kv", "Zusatz-KV"), ("zusatzvereinbarung", "Zusatz-KV"), ("rahmenkollektivvertrag", "Rahmen-KV"), ("rahmen-kv", "Rahmen-KV"), ("generalkollektivvertrag", "Generalkollektivvertrag"), ("general-kv", "Generalkollektivvertrag"), ("mantelvertrag", "Mantelvertrag"), ("berechnungsbeispiele", "Berechnungsbeispiele"), ("fragen", "Fragen & Antworten"), ("kommentar", "Kommentar"), ("interpretation", "Interpretation"), ("erläuterung", "Erläuterungen"), ("handlungsempfehlung", "Handlungsempfehlung"), ("empfehlung", "Empfehlung"), ("information", "Information"), ("erklärung", "Erklärung"), ("klarstellung", "Klarstellung"), ("änderung", "Änderung"), ("anhang", "Anhang/Anlage"), ("anlage", "Anhang/Anlage"), ("sozialpartnervereinbarung", "Vereinbarung"), ("vereinbarung", "Vereinbarung"), ("punktation", "Vereinbarung"), ("überleitungsschema", "Tabelle/Schema"), ("umstiegsdienstzettel", "Tabelle/Schema"), ("rückzahlungstabelle", "Tabelle/Schema"), ("dienst- und besoldungsordnung", "Dienst- und Besoldungsordnung"), ("gehaltsordnung", "Gehaltsordnung"), ("gehaltsordnungen", "Gehaltsordnung"), ("gehaltstabelle", "Gehaltsordnung"), ("gehaltstafel", "Gehaltsordnung"), ("lohnordnung", "Lohnordnung"), ("lohntafeln", "Lohnordnung"), ("lohntafel", "Lohnordnung"), ("lohn- gehaltsordnung", "Lohn-/Gehaltsordnung"), ("lohn-gehaltsordnung", "Lohn-/Gehaltsordnung"), ("lohn- und gehaltsordnung", "Lohn-/Gehaltsordnung"), ("heimarbeitstarif", "Heimarbeitstarif"), ("kollektivvertragsabschluss", "KV-Abschluss"), ("kollektivvertrag", "Kollektivvertrag"), ] def classify_doctype(title: str) -> str: low = title.casefold() for needle, doctype in DOCTYPE_RULES: if low.startswith(needle): return doctype for needle, doctype in DOCTYPE_RULES: if needle in low: return doctype return "Sonstiges KV-Dokument" def derive_stand(title: str, meta: list[tuple[str, str]], filename: str) -> tuple[str, str]: """(stand, tags) — explizites Datum > Titel-Datum > Titel-Jahr > Dateiname-Jahr > Beschaffungsstand.""" for label, value in meta: if "geltungsdauer" in label.casefold(): m = DATE_RE.search(value) if m: return stand_from_date(m.groups()), [] m = YEAR_RE.search(value) if m: return f"{m.group(1)}-01", ["stand-jahr"] m = DATE_RE.search(title) if m: return stand_from_date(m.groups()), [] m = YEAR_RE.search(title) if m: return f"{m.group(1)}-01", ["stand-jahr"] m = YEAR_RE.search(filename) if m: return f"{m.group(1)}-01", ["stand-jahr"] return CRAWL_STAND, ["stand-geschaetzt"] def kv_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None: html = path.read_text(encoding="utf-8") soup = prep_soup(html) og_title = soup.find("meta", property="og:title") og_url = soup.find("meta", property="og:url") if og_title is None or og_url is None: stats["no_og"].append(path.name) return None title = clean_text(og_title["content"]) url = og_url["content"] main = soup.find("main", class_="col-lg-8") if main is None: stats["no_main"].append(path.name) return None meta = parse_kv_meta(main) body_blocks = blocks_to_md(main) key = path.name entry = catalog["entries"].get(key) if entry is None: seq = catalog["next_seq"] catalog["next_seq"] = seq + 1 slug = "kv_" + path.stem.lstrip("-") entry = {"id": f"kv-kvt-{seq:03d}", "slug": slug} catalog["entries"][key] = entry kid, slug = entry["id"], entry["slug"] stand, stand_tags = derive_stand(title, meta, path.name) doctype = classify_doctype(title) year = YEAR_RE.search(path.stem) tags = ["kollektivvertrag", ascii_slug(doctype)] if year: tags.append(f"jahr-{year.group(1)}") tags.extend(stand_tags) meta_rows = "\n".join(f"| {label} | {value} |" for label, value in meta) meta_section = "" if meta_rows: meta_section = ( "## Geltungsbereich (WKO-Angaben)\n\n" "| Merkmal | Angabe |\n|---|---|\n" + meta_rows + "\n\n" ) body = "\n".join(body_blocks).strip() md = ( f"---\n" f"id: {kid}\n" f"batch: 1\n" f"title: {json.dumps(title, ensure_ascii=False)}\n" f"work: {json.dumps(WORK_KV, ensure_ascii=False)}\n" f"chapter: {json.dumps(doctype, ensure_ascii=False)}\n" f"topic: kollektivvertraege\n" f"author: \"WKO\"\n" f"stand: {stand}\n" f"source:\n" f" html: \".firecrawl/wko-kv/docs/{path.name}\"\n" f"legal_bases: []\n" f"tags: {json.dumps(tags, ensure_ascii=False)}\n" f"cross_refs: []\n" f"---\n\n" f"# {title}\n\n" f"*{WORK_KV}, WKO, Stand {stand} ({kid}). Quelle: {url}*\n\n" f"{meta_section}" f"{body}\n" ) entry.update({"title": title, "stand": stand, "doctype": doctype, "url": url}) stats["kv_written"] += 1 return slug, md # -------------------------------------------------------------------------- # RIS: Markdown -> Layer-2 # -------------------------------------------------------------------------- GNR_RE = re.compile(r"Gesetzesnummer[:\s]+(\d+)") ISO_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})") def ris_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None: key = path.name base = path.stem law = LAW_MAP.get(base) if law is None: stats["ris_unmapped"].append(base) return None topic, chapter, nn = law raw = path.read_text(encoding="utf-8").replace("\xa0", " ") lines = raw.split("\n") title = lines[0].lstrip("# ").strip() preamble, sections = raw, "" m = re.search(r"^## .*$", raw, flags=re.M) if m: preamble, sections = raw[: m.start()], raw[m.start():] preamble_lines = [ln for ln in preamble.split("\n")[1:] if ln.strip()] gnr = GNR_RE.search(preamble) stand_match = ISO_DATE_RE.search(preamble) stand = stand_match.group(1)[:7] if stand_match else CRAWL_STAND entry = catalog["entries"].get(key) if entry is None: prefix = topic_prefix(topic) kid = f"ris-{prefix}-{nn:02d}" entry = {"id": kid} catalog["entries"][key] = entry kid = entry["id"] abbr = extract_law_abbreviation(title, base) tags = ["gesetz", ascii_slug(abbr), ascii_slug(topic)] md = ( f"---\n" f"id: {kid}\n" f"batch: 1\n" f"title: {json.dumps(title, ensure_ascii=False)}\n" f"work: {json.dumps(WORK_RIS, ensure_ascii=False)}\n" f"chapter: {json.dumps(chapter, ensure_ascii=False)}\n" f"topic: {topic}\n" f"author: \"RIS (Bundeskanzleramt)\"\n" f"stand: {stand}\n" f"source:\n" f" text: \".ris/{path.name}\"\n" f"legal_bases: {json.dumps([abbr], ensure_ascii=False)}\n" f"tags: {json.dumps(tags, ensure_ascii=False)}\n" f"cross_refs: []\n" f"---\n\n" f"# {title}\n\n" + "\n".join(preamble_lines).strip() + "\n\n" f"*Wissensbasis: {WORK_RIS}, Stand {stand} ({kid}).*\n\n" f"{sections.strip()}\n" ) entry.update({"title": title, "stand": stand, "gnr": gnr.group(1) if gnr else None}) stats["ris_written"] += 1 return f"ris_{ascii_slug(base)}", md def extract_law_abbreviation(title: str, base: str) -> str: """Abkürzung aus dem Titel: 'X (ABK)' am Ende oder 'ABK – Name' am Anfang.""" parens = re.findall(r"\(([^()]+)\)\s*$", title.strip()) if parens: return parens[-1].strip() m = re.match(r"^([A-Za-zÄÖÜäöüß\-]+)\s*[–-]\s", title.strip()) if m: return m.group(1).strip() return base # -------------------------------------------------------------------------- # CLI # -------------------------------------------------------------------------- def write_entries(results: list[tuple[str, str]], dry: bool) -> int: n = 0 for slug, md in results: target = DOCS / f"{slug}.md" if not dry: if target.is_file() and target.read_text(encoding="utf-8") == md: continue target.write_text(md, encoding="utf-8") n += 1 return n def run(source: str, dry: bool, limit: int | None, single: str | None) -> int: stats: dict = {"kv_written": 0, "ris_written": 0, "no_og": [], "no_main": [], "ris_unmapped": []} written = 0 if source in ("kv", "all"): catalog = load_catalog("kv") files = sorted(KV_DIR.glob("*.html")) if single: files = [f for f in files if single in f.name] if limit: files = files[:limit] results = [] for f in files: res = kv_entry(f, catalog, stats) if res: results.append(res) written += write_entries(results, dry) if not dry: save_catalog(catalog) if source in ("ris", "all"): catalog = load_catalog("ris") files = sorted(RIS_DIR.glob("*.md")) if single: files = [f for f in files if single in f.name] if limit: files = files[:limit] results = [] for f in files: res = ris_entry(f, catalog, stats) if res: results.append(res) written += write_entries(results, dry) if not dry: save_catalog(catalog) print(f"source={source} dry={dry} written={written}") print(f" kv: {stats['kv_written']} ris: {stats['ris_written']}") for key in ("no_og", "no_main", "ris_unmapped"): if stats[key]: print(f" {key}: {stats[key]}") return 0 def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--source", choices=("kv", "ris", "all"), default="all") ap.add_argument("--dry-run", action="store_true") ap.add_argument("--limit", type=int, default=None) ap.add_argument("--file", default=None, help="Teilstring des Quelldateinamens") args = ap.parse_args() return run(args.source, args.dry_run, args.limit, args.file) if __name__ == "__main__": sys.exit(main())