KV/RIS-Erweiterung: 614 WKO-KV-Dokumente + 59 RIS-Gesetze (D9)
- Korpus 601 -> 1274 Layer-2-Eintraege: kv-kvt-001...614 (ein Cluster kollektivvertraege, Branche als Tag) und ris-<cluster-prefix>-nn auf der bestehenden Cluster-Map + 7 neue Cluster (zvr/avr/agg/lst/abo/ nso/kvt); LAW_MAP dokumentiert die 59 Gesetz-Zuordnungen. - kv/ris-Eintraege sind quellentreu generiert (D9) - Gesetze sind amtliche Werke, KV-Lohntabellen zahlenexakt; Tool-Output in tools/ (ingest_sources.py, build_registry.py, kb_common.py), eingefrorene ID-Kataloge tools/catalogs/*.json (nur Metadaten). - kb.json/INDEX.md regeneriert (1274 Eintraege, 76 Cluster); agent/kb.py: ID-Raeume kv|ris, source akzeptiert html-only. - Tests 41 -> 49 (Konverter, LAW_MAP-Abdeckung, neue ID-Raeume, Korpus-Integrationszahl).
This commit is contained in:
@@ -0,0 +1,535 @@
|
||||
"""Intake der Quellen `kv` (WKO.at-KV-Seiten) und `ris` (RIS-Gesetze) in die
|
||||
Layer-2-Wissensbasis.
|
||||
|
||||
Abweichend von den lizenzierten Quellen (Lexis/WIKU: Eigene-Worte-Kuratierung)
|
||||
werden kv/ris-Einträge **quellentreu generiert** (Entscheidung 2026-09-15):
|
||||
Gesetze sind amtliche Werke, KV-Lohntabellen müssen zahlenexakt bleiben.
|
||||
Die generierten Dateien (`kv_*.md`, `ris_*.md`) sind damit Tool-Output —
|
||||
keine manuelle Kuratierung, bei Änderungen neu generieren.
|
||||
|
||||
IDs werden aus den Katalogen `tools/catalogs/<source>_catalog.json` vergeben
|
||||
und danach eingefroren (Wissensbasis-Konvention: nie wiederverwenden).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from bs4 import BeautifulSoup, Comment, Tag
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from pathlib import Path
|
||||
|
||||
from tools.kb_common import ( # noqa: E402
|
||||
DATE_RE,
|
||||
LAW_MAP,
|
||||
WORK_KV,
|
||||
WORK_RIS,
|
||||
YEAR_RE,
|
||||
ascii_slug,
|
||||
load_topic_prefix_map,
|
||||
stand_from_date,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
KV_DIR = ROOT / ".firecrawl" / "wko-kv" / "docs"
|
||||
RIS_DIR = ROOT / ".ris"
|
||||
DOCS = ROOT / "wissensbasis" / "dokumente"
|
||||
CATALOGS = ROOT / "tools" / "catalogs"
|
||||
CRAWL_STAND = "2026-09" # Beschaffungsstand der beiden Quellen
|
||||
|
||||
|
||||
TOPIC_PREFIX_FULL: dict[str, str] | None = None
|
||||
|
||||
|
||||
def topic_prefix(topic: str) -> str:
|
||||
global TOPIC_PREFIX_FULL
|
||||
if TOPIC_PREFIX_FULL is None:
|
||||
TOPIC_PREFIX_FULL = load_topic_prefix_map(ROOT / "wissensbasis" / "kb.json")
|
||||
return TOPIC_PREFIX_FULL[topic]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Kataloge (eingefrorene IDs)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def load_catalog(source: str) -> dict:
|
||||
path = CATALOGS / f"{source}_catalog.json"
|
||||
if path.is_file():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return {"source": source, "updated": None, "next_seq": 1, "entries": {}}
|
||||
|
||||
|
||||
def save_catalog(catalog: dict) -> None:
|
||||
catalog["updated"] = datetime.now(timezone.utc).isoformat()
|
||||
CATALOGS.mkdir(parents=True, exist_ok=True)
|
||||
path = CATALOGS / f"{catalog['source']}_catalog.json"
|
||||
path.write_text(
|
||||
json.dumps(catalog, ensure_ascii=False, indent=1, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# WKO-KV: HTML -> Markdown
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
NOISE_TAGS = ("script", "style", "button", "svg", "nav", "iframe", "form", "img")
|
||||
NOISE_CLASSES = re.compile(
|
||||
r"social|share|related|teaser|breadcrumb|pagination|print|meta-details", re.I
|
||||
)
|
||||
|
||||
|
||||
def prep_soup(html: str) -> BeautifulSoup:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup.find_all(NOISE_TAGS):
|
||||
tag.decompose()
|
||||
for br in soup.find_all("br"):
|
||||
br.replace_with("\n")
|
||||
return soup
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
"""Zeilenweise strippen, interne Whitespace-Läufe kollabieren."""
|
||||
lines = []
|
||||
for line in text.split("\n"):
|
||||
line = re.sub(r"[ \t\xa0]+", " ", line).strip()
|
||||
if line:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def cell_text(cell: Tag) -> str:
|
||||
text = cell.get_text(" ")
|
||||
return re.sub(r"\s+", " ", text).strip().replace("|", "\\|")
|
||||
|
||||
|
||||
def table_to_md(table: Tag) -> str:
|
||||
rows = []
|
||||
for tr in table.find_all("tr"):
|
||||
if tr.find_parent("table") is not table:
|
||||
continue # verschachtelte Tabellen nur einmal ausgeben
|
||||
cells = [cell_text(c) for c in tr.find_all(["td", "th"], recursive=False)]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if not rows:
|
||||
return ""
|
||||
width = max(len(r) for r in rows)
|
||||
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||
out = ["| " + " | ".join(rows[0]) + " |"]
|
||||
out.append("| " + " | ".join(["---"] * width) + " |")
|
||||
for r in rows[1:]:
|
||||
out.append("| " + " | ".join(r) + " |")
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def list_to_md(el: Tag, depth: int = 0) -> list[str]:
|
||||
lines = []
|
||||
indent = " " * depth
|
||||
ordered = el.name == "ol"
|
||||
for i, li in enumerate(el.find_all("li", recursive=False), 1):
|
||||
own = []
|
||||
for child in li.children:
|
||||
if isinstance(child, Tag) and child.name in ("ul", "ol"):
|
||||
continue
|
||||
own.append(child.get_text(" ") if isinstance(child, Tag) else str(child))
|
||||
text = clean_text("".join(own))
|
||||
marker = f"{i}. " if ordered else "- "
|
||||
if text:
|
||||
lines.append(f"{indent}{marker}{text.replace(chr(10), chr(10) + indent + ' ')}")
|
||||
for sub in li.find_all(["ul", "ol"], recursive=False):
|
||||
lines.extend(list_to_md(sub, depth + 1))
|
||||
return lines
|
||||
|
||||
|
||||
def _is_toc_paragraph(p: Tag) -> bool:
|
||||
"""Absatz, der ausschließlich einen In-Seiten-Anker-Link enthält."""
|
||||
links = p.find_all("a", href=re.compile(r"^#"))
|
||||
if not links:
|
||||
return False
|
||||
rest = clean_text(p.get_text(" "))
|
||||
for a in links:
|
||||
rest = rest.replace(clean_text(a.get_text(" ")), "", 1)
|
||||
return not rest
|
||||
|
||||
|
||||
def blocks_to_md(el: Tag) -> list[str]:
|
||||
"""Rekursiver Block-Konverter; liefert Markdown-Zeilen."""
|
||||
lines: list[str] = []
|
||||
skip_toc = False # TOC-Absätze nach „Inhalt“-Überschrift auslassen
|
||||
for child in el.children:
|
||||
if isinstance(child, Comment):
|
||||
continue
|
||||
if isinstance(child, str):
|
||||
text = clean_text(child)
|
||||
if text:
|
||||
lines.extend([text, ""])
|
||||
continue
|
||||
if not isinstance(child, Tag):
|
||||
continue
|
||||
classes = " ".join(child.get("class", []))
|
||||
if child.name in NOISE_TAGS or NOISE_CLASSES.search(classes):
|
||||
continue
|
||||
if child.name in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
title = clean_text(child.get_text(" "))
|
||||
if not title:
|
||||
continue
|
||||
if title.casefold() == "inhalt":
|
||||
skip_toc = True
|
||||
continue
|
||||
prefix = "###" if child.name in ("h5", "h6") else "##"
|
||||
lines.extend([f"{prefix} {title}", ""])
|
||||
skip_toc = False
|
||||
continue
|
||||
if child.name == "p" and skip_toc and _is_toc_paragraph(child):
|
||||
continue # TOC-Einträge (reine Anker-Links) auslassen
|
||||
skip_toc = False
|
||||
if child.name in ("ul", "ol"):
|
||||
lines.extend(list_to_md(child))
|
||||
lines.append("")
|
||||
continue
|
||||
if child.name == "table":
|
||||
md = table_to_md(child)
|
||||
if md:
|
||||
lines.extend([md, ""])
|
||||
continue
|
||||
if child.name == "p":
|
||||
text = clean_text(child.get_text())
|
||||
if text:
|
||||
lines.extend([text, ""])
|
||||
continue
|
||||
if child.name == "dl":
|
||||
for dt in child.find_all("dt"):
|
||||
dd = dt.find_next_sibling("dd")
|
||||
if dd is not None:
|
||||
lines.append(f"- **{clean_text(dt.get_text(' '))}:** "
|
||||
f"{clean_text(dd.get_text(' '))}")
|
||||
lines.append("")
|
||||
continue
|
||||
if child.name in ("div", "section", "article", "main", "header", "footer"):
|
||||
lines.extend(blocks_to_md(child))
|
||||
continue
|
||||
# blockquote, pre, hr, sonstiges: Text durchreichen
|
||||
text = clean_text(child.get_text())
|
||||
if text:
|
||||
lines.extend([text, ""])
|
||||
return lines
|
||||
|
||||
|
||||
def parse_kv_meta(main: Tag) -> list[tuple[str, str]]:
|
||||
"""dt/dd-Paare aus dem meta-details-Block."""
|
||||
pairs = []
|
||||
box = main.find("div", class_="meta-details")
|
||||
if box is None:
|
||||
return pairs
|
||||
for dt in box.find_all("dt"):
|
||||
dd = dt.find_next_sibling("dd")
|
||||
label = clean_text(dt.get_text(" ")).rstrip(":")
|
||||
value = clean_text(dd.get_text(" ")) if dd is not None else ""
|
||||
if label:
|
||||
pairs.append((label, value))
|
||||
return pairs
|
||||
|
||||
|
||||
DOCTYPE_RULES: list[tuple[str, str]] = [
|
||||
("zusatzkollektivvertrag", "Zusatz-KV"),
|
||||
("zusatz-kv", "Zusatz-KV"),
|
||||
("zusatzvereinbarung", "Zusatz-KV"),
|
||||
("rahmenkollektivvertrag", "Rahmen-KV"),
|
||||
("rahmen-kv", "Rahmen-KV"),
|
||||
("generalkollektivvertrag", "Generalkollektivvertrag"),
|
||||
("general-kv", "Generalkollektivvertrag"),
|
||||
("mantelvertrag", "Mantelvertrag"),
|
||||
("berechnungsbeispiele", "Berechnungsbeispiele"),
|
||||
("fragen", "Fragen & Antworten"),
|
||||
("kommentar", "Kommentar"),
|
||||
("interpretation", "Interpretation"),
|
||||
("erläuterung", "Erläuterungen"),
|
||||
("handlungsempfehlung", "Handlungsempfehlung"),
|
||||
("empfehlung", "Empfehlung"),
|
||||
("information", "Information"),
|
||||
("erklärung", "Erklärung"),
|
||||
("klarstellung", "Klarstellung"),
|
||||
("änderung", "Änderung"),
|
||||
("anhang", "Anhang/Anlage"),
|
||||
("anlage", "Anhang/Anlage"),
|
||||
("sozialpartnervereinbarung", "Vereinbarung"),
|
||||
("vereinbarung", "Vereinbarung"),
|
||||
("punktation", "Vereinbarung"),
|
||||
("überleitungsschema", "Tabelle/Schema"),
|
||||
("umstiegsdienstzettel", "Tabelle/Schema"),
|
||||
("rückzahlungstabelle", "Tabelle/Schema"),
|
||||
("dienst- und besoldungsordnung", "Dienst- und Besoldungsordnung"),
|
||||
("gehaltsordnung", "Gehaltsordnung"),
|
||||
("gehaltsordnungen", "Gehaltsordnung"),
|
||||
("gehaltstabelle", "Gehaltsordnung"),
|
||||
("gehaltstafel", "Gehaltsordnung"),
|
||||
("lohnordnung", "Lohnordnung"),
|
||||
("lohntafeln", "Lohnordnung"),
|
||||
("lohntafel", "Lohnordnung"),
|
||||
("lohn- gehaltsordnung", "Lohn-/Gehaltsordnung"),
|
||||
("lohn-gehaltsordnung", "Lohn-/Gehaltsordnung"),
|
||||
("lohn- und gehaltsordnung", "Lohn-/Gehaltsordnung"),
|
||||
("heimarbeitstarif", "Heimarbeitstarif"),
|
||||
("kollektivvertragsabschluss", "KV-Abschluss"),
|
||||
("kollektivvertrag", "Kollektivvertrag"),
|
||||
]
|
||||
|
||||
|
||||
def classify_doctype(title: str) -> str:
|
||||
low = title.casefold()
|
||||
for needle, doctype in DOCTYPE_RULES:
|
||||
if low.startswith(needle):
|
||||
return doctype
|
||||
for needle, doctype in DOCTYPE_RULES:
|
||||
if needle in low:
|
||||
return doctype
|
||||
return "Sonstiges KV-Dokument"
|
||||
|
||||
|
||||
def derive_stand(title: str, meta: list[tuple[str, str]], filename: str) -> tuple[str, str]:
|
||||
"""(stand, tags) — explizites Datum > Titel-Datum > Titel-Jahr >
|
||||
Dateiname-Jahr > Beschaffungsstand."""
|
||||
for label, value in meta:
|
||||
if "geltungsdauer" in label.casefold():
|
||||
m = DATE_RE.search(value)
|
||||
if m:
|
||||
return stand_from_date(m.groups()), []
|
||||
m = YEAR_RE.search(value)
|
||||
if m:
|
||||
return f"{m.group(1)}-01", ["stand-jahr"]
|
||||
m = DATE_RE.search(title)
|
||||
if m:
|
||||
return stand_from_date(m.groups()), []
|
||||
m = YEAR_RE.search(title)
|
||||
if m:
|
||||
return f"{m.group(1)}-01", ["stand-jahr"]
|
||||
m = YEAR_RE.search(filename)
|
||||
if m:
|
||||
return f"{m.group(1)}-01", ["stand-jahr"]
|
||||
return CRAWL_STAND, ["stand-geschaetzt"]
|
||||
|
||||
|
||||
def kv_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None:
|
||||
html = path.read_text(encoding="utf-8")
|
||||
soup = prep_soup(html)
|
||||
og_title = soup.find("meta", property="og:title")
|
||||
og_url = soup.find("meta", property="og:url")
|
||||
if og_title is None or og_url is None:
|
||||
stats["no_og"].append(path.name)
|
||||
return None
|
||||
title = clean_text(og_title["content"])
|
||||
url = og_url["content"]
|
||||
main = soup.find("main", class_="col-lg-8")
|
||||
if main is None:
|
||||
stats["no_main"].append(path.name)
|
||||
return None
|
||||
meta = parse_kv_meta(main)
|
||||
body_blocks = blocks_to_md(main)
|
||||
|
||||
key = path.name
|
||||
entry = catalog["entries"].get(key)
|
||||
if entry is None:
|
||||
seq = catalog["next_seq"]
|
||||
catalog["next_seq"] = seq + 1
|
||||
slug = "kv_" + path.stem.lstrip("-")
|
||||
entry = {"id": f"kv-kvt-{seq:03d}", "slug": slug}
|
||||
catalog["entries"][key] = entry
|
||||
kid, slug = entry["id"], entry["slug"]
|
||||
|
||||
stand, stand_tags = derive_stand(title, meta, path.name)
|
||||
doctype = classify_doctype(title)
|
||||
year = YEAR_RE.search(path.stem)
|
||||
tags = ["kollektivvertrag", ascii_slug(doctype)]
|
||||
if year:
|
||||
tags.append(f"jahr-{year.group(1)}")
|
||||
tags.extend(stand_tags)
|
||||
|
||||
meta_rows = "\n".join(f"| {label} | {value} |" for label, value in meta)
|
||||
meta_section = ""
|
||||
if meta_rows:
|
||||
meta_section = (
|
||||
"## Geltungsbereich (WKO-Angaben)\n\n"
|
||||
"| Merkmal | Angabe |\n|---|---|\n" + meta_rows + "\n\n"
|
||||
)
|
||||
body = "\n".join(body_blocks).strip()
|
||||
md = (
|
||||
f"---\n"
|
||||
f"id: {kid}\n"
|
||||
f"batch: 1\n"
|
||||
f"title: {json.dumps(title, ensure_ascii=False)}\n"
|
||||
f"work: {json.dumps(WORK_KV, ensure_ascii=False)}\n"
|
||||
f"chapter: {json.dumps(doctype, ensure_ascii=False)}\n"
|
||||
f"topic: kollektivvertraege\n"
|
||||
f"author: \"WKO\"\n"
|
||||
f"stand: {stand}\n"
|
||||
f"source:\n"
|
||||
f" html: \".firecrawl/wko-kv/docs/{path.name}\"\n"
|
||||
f"legal_bases: []\n"
|
||||
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
|
||||
f"cross_refs: []\n"
|
||||
f"---\n\n"
|
||||
f"# {title}\n\n"
|
||||
f"*{WORK_KV}, WKO, Stand {stand} ({kid}). Quelle: {url}*\n\n"
|
||||
f"{meta_section}"
|
||||
f"{body}\n"
|
||||
)
|
||||
entry.update({"title": title, "stand": stand, "doctype": doctype, "url": url})
|
||||
stats["kv_written"] += 1
|
||||
return slug, md
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# RIS: Markdown -> Layer-2
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
GNR_RE = re.compile(r"Gesetzesnummer[:\s]+(\d+)")
|
||||
ISO_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})")
|
||||
|
||||
|
||||
def ris_entry(path: Path, catalog: dict, stats: dict) -> tuple[str, str] | None:
|
||||
key = path.name
|
||||
base = path.stem
|
||||
law = LAW_MAP.get(base)
|
||||
if law is None:
|
||||
stats["ris_unmapped"].append(base)
|
||||
return None
|
||||
topic, chapter, nn = law
|
||||
|
||||
raw = path.read_text(encoding="utf-8").replace("\xa0", " ")
|
||||
lines = raw.split("\n")
|
||||
title = lines[0].lstrip("# ").strip()
|
||||
preamble, sections = raw, ""
|
||||
m = re.search(r"^## .*$", raw, flags=re.M)
|
||||
if m:
|
||||
preamble, sections = raw[: m.start()], raw[m.start():]
|
||||
preamble_lines = [ln for ln in preamble.split("\n")[1:] if ln.strip()]
|
||||
|
||||
gnr = GNR_RE.search(preamble)
|
||||
stand_match = ISO_DATE_RE.search(preamble)
|
||||
stand = stand_match.group(1)[:7] if stand_match else CRAWL_STAND
|
||||
|
||||
entry = catalog["entries"].get(key)
|
||||
if entry is None:
|
||||
prefix = topic_prefix(topic)
|
||||
kid = f"ris-{prefix}-{nn:02d}"
|
||||
entry = {"id": kid}
|
||||
catalog["entries"][key] = entry
|
||||
kid = entry["id"]
|
||||
|
||||
abbr = extract_law_abbreviation(title, base)
|
||||
tags = ["gesetz", ascii_slug(abbr), ascii_slug(topic)]
|
||||
md = (
|
||||
f"---\n"
|
||||
f"id: {kid}\n"
|
||||
f"batch: 1\n"
|
||||
f"title: {json.dumps(title, ensure_ascii=False)}\n"
|
||||
f"work: {json.dumps(WORK_RIS, ensure_ascii=False)}\n"
|
||||
f"chapter: {json.dumps(chapter, ensure_ascii=False)}\n"
|
||||
f"topic: {topic}\n"
|
||||
f"author: \"RIS (Bundeskanzleramt)\"\n"
|
||||
f"stand: {stand}\n"
|
||||
f"source:\n"
|
||||
f" text: \".ris/{path.name}\"\n"
|
||||
f"legal_bases: {json.dumps([abbr], ensure_ascii=False)}\n"
|
||||
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
|
||||
f"cross_refs: []\n"
|
||||
f"---\n\n"
|
||||
f"# {title}\n\n"
|
||||
+ "\n".join(preamble_lines).strip() + "\n\n"
|
||||
f"*Wissensbasis: {WORK_RIS}, Stand {stand} ({kid}).*\n\n"
|
||||
f"{sections.strip()}\n"
|
||||
)
|
||||
entry.update({"title": title, "stand": stand, "gnr": gnr.group(1) if gnr else None})
|
||||
stats["ris_written"] += 1
|
||||
return f"ris_{ascii_slug(base)}", md
|
||||
|
||||
|
||||
def extract_law_abbreviation(title: str, base: str) -> str:
|
||||
"""Abkürzung aus dem Titel: 'X (ABK)' am Ende oder 'ABK – Name' am Anfang."""
|
||||
parens = re.findall(r"\(([^()]+)\)\s*$", title.strip())
|
||||
if parens:
|
||||
return parens[-1].strip()
|
||||
m = re.match(r"^([A-Za-zÄÖÜäöüß\-]+)\s*[–-]\s", title.strip())
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
return base
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# CLI
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def write_entries(results: list[tuple[str, str]], dry: bool) -> int:
|
||||
n = 0
|
||||
for slug, md in results:
|
||||
target = DOCS / f"{slug}.md"
|
||||
if not dry:
|
||||
if target.is_file() and target.read_text(encoding="utf-8") == md:
|
||||
continue
|
||||
target.write_text(md, encoding="utf-8")
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def run(source: str, dry: bool, limit: int | None, single: str | None) -> int:
|
||||
stats: dict = {"kv_written": 0, "ris_written": 0, "no_og": [], "no_main": [],
|
||||
"ris_unmapped": []}
|
||||
written = 0
|
||||
|
||||
if source in ("kv", "all"):
|
||||
catalog = load_catalog("kv")
|
||||
files = sorted(KV_DIR.glob("*.html"))
|
||||
if single:
|
||||
files = [f for f in files if single in f.name]
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
results = []
|
||||
for f in files:
|
||||
res = kv_entry(f, catalog, stats)
|
||||
if res:
|
||||
results.append(res)
|
||||
written += write_entries(results, dry)
|
||||
if not dry:
|
||||
save_catalog(catalog)
|
||||
|
||||
if source in ("ris", "all"):
|
||||
catalog = load_catalog("ris")
|
||||
files = sorted(RIS_DIR.glob("*.md"))
|
||||
if single:
|
||||
files = [f for f in files if single in f.name]
|
||||
if limit:
|
||||
files = files[:limit]
|
||||
results = []
|
||||
for f in files:
|
||||
res = ris_entry(f, catalog, stats)
|
||||
if res:
|
||||
results.append(res)
|
||||
written += write_entries(results, dry)
|
||||
if not dry:
|
||||
save_catalog(catalog)
|
||||
|
||||
print(f"source={source} dry={dry} written={written}")
|
||||
print(f" kv: {stats['kv_written']} ris: {stats['ris_written']}")
|
||||
for key in ("no_og", "no_main", "ris_unmapped"):
|
||||
if stats[key]:
|
||||
print(f" {key}: {stats[key]}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--source", choices=("kv", "ris", "all"), default="all")
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--limit", type=int, default=None)
|
||||
ap.add_argument("--file", default=None, help="Teilstring des Quelldateinamens")
|
||||
args = ap.parse_args()
|
||||
return run(args.source, args.dry_run, args.limit, args.file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user