fegger
27f3b4b7f9
Zehnter Lexis 360-Intake (64 Briefings aus den Sektionsordnern beispiele / muster / uebersichten_checklisten, Quell-Staende 2020-12 bis 2026-08), erweitert das Korpus auf 665 kuratierte Eintraege in 69 Clustern. Dasselbe Werk, dieselbe Breadcrumb-Form — die Pipeline globt jetzt die drei Subordner; die neue no-author-Breadcrumb-Variante (Muster-Exporte "Kapitel > Thema · <Monat> <Jahr>") defaultet auf "Lexis Redaktion". - 13 bestehende Cluster erweitert (keine neuen): beendigungsarten bnd 44→66 (Muster: Kuendigung/Aenderung/ Aufloesung/Entlassung/Fristablauf/Befristung/Verwarnung/ Dienstfreistellung, 13 Checklisten, Anspruchsuebersicht Tod), endabrechnung end 25→34 (Beispiele: Zeitschulden, Viertel-/Zwoelftelregelung, Voll-Ueberttritt, Gleitzeitguthaben, UEL; Checklisten: Ausbildungskosten, Kuendigungsentschaedigung; Muster: Vergleich, Vordienstzeiten), uberstunden ues 4→9, telearbeit tel 6→11, beendigungsphase-sonstiges bso 10→16, arbeitszeitgrenzen azg 4→9, arbeitszeitmodelle azm 7→10, entsendung grz 8→11, beschaftigungsformen bes 8→10, sonderzahlungen son 5→6, schwerarbeit swa 5→6, arbeitskrafteuberlassung aug 2→3, urlaub url 16→17 - alte Quell-Staende (2020-12 bis 2025-12) mit "⚠ Alter Stand" in den Eintraegen markiert - Pipeline: BATCH_LEXIS 10, SOURCES["lexis"].pdf_subdirs, BC_LINE_NOAUTHOR_RE + Slug-Kollisionsguard in extract_lexis() - RUNBOOK (Batch-10-Paragraf) und Agent-Memory aktualisiert; --registry und --check beide gruen (635/635, 0 problems)
1602 lines
62 KiB
Python
1602 lines
62 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the Personalverrechnung knowledge base (Layer-1 extraction +
|
||
kb.json registry).
|
||
|
||
Two licensed source corpora feed ONE shared curated knowledge base under
|
||
personalverrechnung/wissensbasis/ (development reference and future
|
||
copilot corpus):
|
||
|
||
lexis .lexis360/*.pdf PDF exports of "Lexis Briefings Personalrecht"
|
||
(LexisNexis AT, licensed Lexis 360 subscription);
|
||
batch 10 adds the same work's sections in
|
||
subfolders: .lexis360/{beispiele,muster,
|
||
uebersichten_checklisten}/
|
||
wiku .wiku/*.pdf WIKU Personal publications (Fachbroschüren,
|
||
Arbeitsunterlagen, Casebooks, the periodical
|
||
"WIKU Personal aktuell")
|
||
|
||
Licensing (decision D1, 2026-09-10): the PDF exports and the extracted
|
||
full texts stay local and unversioned (.lexis360/, .wiku/ are gitignored),
|
||
like .firecrawl/. Only the curated entries under wissensbasis/ (own
|
||
words, short quotes, source citations) are versioned.
|
||
|
||
WIKU has no Lexis breadcrumbs: its metadata (title, chapter, stand, work,
|
||
cluster) comes from the explicit WIKU_PINS table (one publication = one
|
||
entry, per the integration model), and Layer-2 files get a "wiku_"
|
||
filename prefix in the shared dokumente/ directory.
|
||
|
||
Run from the repository root:
|
||
|
||
python3 personalverrechnung/tools/build_lexis_kb.py --extract [--source lexis|wiku|all]
|
||
python3 personalverrechnung/tools/build_lexis_kb.py --registry # kb.json (shared corpus)
|
||
python3 personalverrechnung/tools/build_lexis_kb.py --check [--source lexis|wiku|all]
|
||
|
||
Outputs:
|
||
Layer 1 (unversioned, licensed raw texts):
|
||
.lexis360/md/<slug>.md full text + extraction frontmatter
|
||
.lexis360/md/_catalog.json parsed metadata of all sources
|
||
.wiku/md/<slug>.md full text + extraction frontmatter
|
||
.wiku/md/_catalog.json parsed metadata of all sources
|
||
Layer 2 (versioned, curated by hand):
|
||
personalverrechnung/wissensbasis/dokumente/<slug>.md (Lexis)
|
||
personalverrechnung/wissensbasis/dokumente/wiku_<slug>.md (WIKU)
|
||
personalverrechnung/wissensbasis/kb.json (generated)
|
||
|
||
Frontmatter schema of curated entries (subset, no PyYAML needed):
|
||
id (lb-<cluster>-<nn> Lexis | wk-<cluster>-<nn> WIKU), batch, title,
|
||
work, chapter, topic (cluster slug), author, stand (YYYY-MM),
|
||
source{pdf,text}, legal_bases[], tags[], cross_refs[]
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import datetime
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
CURATED_DIR = ROOT / "personalverrechnung" / "wissensbasis" / "dokumente"
|
||
KB_JSON = CURATED_DIR.parent / "kb.json"
|
||
|
||
WORK_LEXIS = "Lexis Briefings Personalrecht"
|
||
BATCH_LEXIS = 10
|
||
BATCH_WIKU = 1
|
||
WIKU_SLUG_PREFIX = "wiku_"
|
||
WIKU_WORKS = (
|
||
"WIKU Fachbroschüre",
|
||
"WIKU Arbeitsunterlage",
|
||
"WIKU Casebook",
|
||
"WIKU Personal aktuell",
|
||
)
|
||
|
||
# One shared curated corpus (wissensbasis/ + kb.json); the `work` field
|
||
# distinguishes the licensed sources.
|
||
SOURCES: dict[str, dict] = {
|
||
"lexis": {
|
||
"work": WORK_LEXIS,
|
||
"pdf_dir": ROOT / ".lexis360",
|
||
# batch 10 — same work, same breadcrumb format, section exports in
|
||
# subfolders (Muster, Beispiele, Übersichten & Checklisten)
|
||
"pdf_subdirs": ("beispiele", "muster", "uebersichten_checklisten"),
|
||
"md_dir": ROOT / ".lexis360" / "md",
|
||
"pdf_glob": "Lexis360_*.pdf",
|
||
"id_prefix": "lb",
|
||
"max_batch": BATCH_LEXIS,
|
||
},
|
||
"wiku": {
|
||
"work": None, # per-publication family, see WIKU_PINS
|
||
"pdf_dir": ROOT / ".wiku",
|
||
"md_dir": ROOT / ".wiku" / "md",
|
||
"pdf_glob": "*.pdf",
|
||
"id_prefix": "wk",
|
||
"max_batch": BATCH_WIKU,
|
||
},
|
||
}
|
||
|
||
MONTHS = {
|
||
"jänner": 1, "januar": 1, "februar": 2, "märz": 3, "april": 4,
|
||
"mai": 5, "juni": 6, "juli": 7, "august": 8, "september": 9,
|
||
"oktober": 10, "november": 11, "dezember": 12,
|
||
}
|
||
|
||
# Explicit breadcrumb-topic -> cluster mapping (pinned from a first
|
||
# --extract pass; keep deterministic and auditable). Keys are the
|
||
# ascii_slug() form of the parsed breadcrumb topic: diacritics STRIPPED
|
||
# (ü->u, ä->a, ß->ss), non-alnum -> hyphen — matching the umlaut-dropped
|
||
# style of the Lexis360 export filenames ("beschaftigung", not
|
||
# "beschaeftigung").
|
||
TOPIC_MAP: dict[str, str] = {
|
||
"altersteilzeit": "atz",
|
||
"lehrlinge": "leh",
|
||
"schnupperlehre": "leh",
|
||
"jugendliche": "jug",
|
||
"kinder": "jug",
|
||
"ferialpraktikanten-volontare": "pra",
|
||
"geringfugige-beschaftigung": "bes",
|
||
"fallweise-beschaftigung": "bes",
|
||
"freier-dienstvertrag": "bes",
|
||
"werkvertrag": "bes",
|
||
"echtes-dienstverhaltnis": "bes",
|
||
"abgrenzung-arbeiter-angestellte": "bes",
|
||
"teilzeitbeschaftigung": "tzb",
|
||
"bildungsteilzeit": "tzb",
|
||
# batch 2
|
||
"abrechnungsperiode-fur-bezuge": "ent",
|
||
"rechtsgrundlagen": "ent",
|
||
"drittlohn": "ent",
|
||
"lohn-gehalt": "ent",
|
||
"behinderte-arbeitnehmer": "beh",
|
||
"geschaftsfuhrer": "gsf",
|
||
"leitende-angestellte": "lei",
|
||
"vorstandsmitglieder": "vst",
|
||
"sachbezuge-freiwillige-sozialleistungen": "sac",
|
||
"diverse-pramien": "prm",
|
||
"provisionen": "prm",
|
||
# batch 3
|
||
"dienstreise-reisekosten": "rei",
|
||
"ruckerstattung-ausbildungskosten": "ent",
|
||
"vorsorgeleistungen": "vor",
|
||
"sonderzahlungen": "son",
|
||
"zulagen-und-zuschlage": "zus",
|
||
"pendlerforderung": "pen",
|
||
"sachzuwendungen-geschenke": "ges",
|
||
# batch 4
|
||
"bekampfung-von-lohn-und-sozialdumping": "lsd",
|
||
"gplb": "gpl",
|
||
"lohnnebenkosten": "lnk",
|
||
"lohnpfandung": "pfa",
|
||
"lohnverrechnung": "lvr",
|
||
"nachzahlungen": "naz",
|
||
# batch 5
|
||
"arbeitszeitgrenzen": "azg",
|
||
"arbeitszeitmodelle": "azm",
|
||
"melde-und-informationspflichten": "mip",
|
||
"nachtschwerarbeit": "nsc",
|
||
"ruhezeiten": "rhz",
|
||
"schwangerschaft": "sch",
|
||
"uberstunden": "ues",
|
||
"zeiterfassung": "zer",
|
||
# batch 6
|
||
"arbeitnehmerschutz-geltungsbereich-gefahrenevaluierung": "asc",
|
||
"arbeitsstatte-mittel-stoffe": "asc",
|
||
"arbeitsunfall": "asc",
|
||
"bauarbeitenkoordinationsgesetz": "asc",
|
||
"bildschirmarbeit": "asc",
|
||
"gesundheitsuberwachung": "asc",
|
||
"hitzeschutz": "asc",
|
||
"praventivdienste": "asc",
|
||
"sicherheitsvertrauenspersonen": "asc",
|
||
"strafbarkeit-aschg": "asc",
|
||
"dienstnehmerhaftung-schadenersatz": "dnh",
|
||
"gleichbehandlung": "glb",
|
||
"mobbing": "mob",
|
||
"sachnutzung-von-firmeneigentum": "fir",
|
||
"weisungen-an-die-arbeitnehmer": "wei",
|
||
"schwerarbeit": "swa",
|
||
# batch 7
|
||
"arbeitskrafteuberlassung": "aug",
|
||
"grenzuberschreitender-einsatz-von-arbeitskraften": "grz",
|
||
"standortverlegung": "grz",
|
||
"teleworking-heimarbeit": "tel",
|
||
"dienstverhinderungsgrunde-auf-arbeitgeberseite": "dvh",
|
||
"sonstige-dienstverhinderungsgrunde": "dvh",
|
||
"streik": "dvh",
|
||
"entgeltfortzahlung-an-feiertagen": "efz",
|
||
"pflegefreistellung": "pfl",
|
||
"prasenzdienst-zivildienst": "prd",
|
||
"krankenstand": "krs",
|
||
"wiedereingliederungsteilzeit": "wie",
|
||
"arbeitsunfall-und-berufskrankheit": "asc",
|
||
"karenz": "kar",
|
||
"elternteilzeit": "elt",
|
||
"kinderbetreuungsgeld": "kbg",
|
||
"beihilfen": "kbg",
|
||
"geburt-schutzfrist": "msf",
|
||
"wochengeld": "msf",
|
||
"papamonat-und-familienzeitbonus": "pap",
|
||
"bildungskarenz": "kzs",
|
||
"familienhospizkarenz": "kzs",
|
||
"pflegekarenz-pflegeteilzeit": "kzs",
|
||
"kinderrehabilitation": "kzs",
|
||
"urlaubsablose": "url",
|
||
"urlaubsanspruch-ausmass": "url",
|
||
"urlaubsaufzeichnungen": "url",
|
||
"urlaubsentgelt": "url",
|
||
"urlaubsersatzleistung": "url",
|
||
"urlaubsverbrauch": "url",
|
||
# batch 9 — chapter "Sozialversicherung" (all topic groups unique to
|
||
# this chapter, so plain pins suffice; gsv also carries the GSVG
|
||
# self-employed severance briefing which the bare "abfertigung"
|
||
# keyword would otherwise steal for the end cluster)
|
||
"beitragsrecht-asvg": "sva",
|
||
"beitragsrecht-bauerliche-sozialversicherung": "bsv",
|
||
"beitragsrecht-gsvg": "gsv",
|
||
"gesellschaftsformen-und-sv-rechtliche-zuordnung": "gsv",
|
||
"krankenversicherung-asvg": "kvs",
|
||
"neue-selbststandige-nach-gsvg": "gsv",
|
||
"pensionsversicherung": "pvs",
|
||
"abfertigung-fur-selbststandige": "gsv",
|
||
# wiku batch 1 — WIKU-specific topic groups (WIKU publications are
|
||
# pinned via WIKU_PINS; these entries complete the four-structure set
|
||
# for the new clusters per the curation conventions)
|
||
"personal-aktuell": "akt",
|
||
"aktuell": "akt",
|
||
"gastgewerbe": "gwe",
|
||
"gueterbefoerderungsgewerbe": "gwe",
|
||
"gewerbe": "gwe",
|
||
}
|
||
|
||
# Chapter-qualified topic pins, checked BEFORE the plain TOPIC_MAP:
|
||
# the same topic string can legitimately appear under two different
|
||
# chapters and belong to two different clusters (batch-8 lesson:
|
||
# "Sonderzahlungen", "Provisionen", "Pramien", "Urlaubsersatzleistung"
|
||
# exist both in "Entgelt: Anspruch & Abrechnung" and in
|
||
# "Beendigungsansprueche & Endabrechnung"). Keys are
|
||
# "<chapter-ascii>/<topic-ascii>" (ascii_slug() form, & -> hyphen).
|
||
CHAPTER_TOPIC_MAP: dict[str, str] = {
|
||
# batch 8 — chapter "Beendigungsarten" -> bnd (Kündigungsschutz and
|
||
# the remaining Kündigungsarten sub-topics are covered by the plain
|
||
# TOPIC_MAP entry "kündigung" -> bnd below)
|
||
"beendigungsarten/auflosung-in-probezeit": "bnd",
|
||
"beendigungsarten/austritt-allgemein": "bnd",
|
||
"beendigungsarten/austrittsgrunde": "bnd",
|
||
"beendigungsarten/einvernehmliche-losung": "bnd",
|
||
"beendigungsarten/entlassung-allgemein": "bnd",
|
||
"beendigungsarten/entlassungsgrunde": "bnd",
|
||
"beendigungsarten/kundigung": "bnd",
|
||
"beendigungsarten/kundigungsschutz": "bnd",
|
||
"beendigungsarten/tod-des-arbeitgebers": "bnd",
|
||
"beendigungsarten/tod-des-arbeitnehmers": "bnd",
|
||
"beendigungsarten/zeitablauf-bei-befristung": "bnd",
|
||
# batch 8 — chapter "Betriebsrat & Betriebsvereinbarungen" -> brt
|
||
"betriebsrat-betriebsvereinbarungen/belegschaftsvertretungen-betriebs-und-arbeitnehmerbegriff": "brt",
|
||
"betriebsrat-betriebsvereinbarungen/befugnisse-des-betriebsrates": "brt",
|
||
"betriebsrat-betriebsvereinbarungen/betriebsvereinbarungen": "brt",
|
||
"betriebsrat-betriebsvereinbarungen/rechtsstellung-der-betriebsratsmitglieder": "brt",
|
||
# batch 8 — chapter "Beendigungsansprueche & Endabrechnung" -> end
|
||
"beendigungsansprueche-endabrechnung/abfertigung-alt": "end",
|
||
"beendigungsansprueche-endabrechnung/abfertigung-neu": "end",
|
||
"beendigungsansprueche-endabrechnung/aufrechnung-mit-noch-offenen-vorschussen": "end",
|
||
"beendigungsansprueche-endabrechnung/ausbildungskostenruckersatz": "end",
|
||
"beendigungsansprueche-endabrechnung/auszahlung-normal-mehr-und-uberstunden-zeitguthaben": "end",
|
||
"beendigungsansprueche-endabrechnung/freiwillige-abfertigung-abfindung": "end",
|
||
"beendigungsansprueche-endabrechnung/kundigungsentschadigung": "end",
|
||
"beendigungsansprueche-endabrechnung/outplacement": "end",
|
||
"beendigungsansprueche-endabrechnung/pramien": "end",
|
||
"beendigungsansprueche-endabrechnung/provisionen": "end",
|
||
"beendigungsansprueche-endabrechnung/ruckzahlung-von-darlehen": "end",
|
||
"beendigungsansprueche-endabrechnung/sonderzahlungen": "end",
|
||
"beendigungsansprueche-endabrechnung/urlaubsersatzleistung": "end",
|
||
"beendigungsansprueche-endabrechnung/vergleich-vergleichssummen": "end",
|
||
# batch 8 — chapter "Unternehmensauflösung & Betriebsübergang" -> ins
|
||
"unternehmensaufloesung-betriebsubergang/betriebsubergang": "ins",
|
||
"unternehmensaufloesung-betriebsubergang/insolvenz-des-arbeitgebers": "ins",
|
||
# batch 8 — chapter "Meldungen & Verpflichtungen" -> mel
|
||
"meldungen-verpflichtungen/arbeitsrecht-und-datenschutz": "mel",
|
||
"meldungen-verpflichtungen/auflage-und-aushangpflichten": "mel",
|
||
"meldungen-verpflichtungen/informationspflichten": "mel",
|
||
"meldungen-verpflichtungen/meldepflichten": "mel",
|
||
"meldungen-verpflichtungen/schriftform-im-arbeitsrecht": "mel",
|
||
# batch 8 — chapter "Beendigungsphase - Sonstiges" -> bso
|
||
"beendigungsphase-sonstiges/abmeldung-sozialversicherung": "bso",
|
||
"beendigungsphase-sonstiges/arbeitslosengeld": "bso",
|
||
"beendigungsphase-sonstiges/asg-verfahren": "bso",
|
||
"beendigungsphase-sonstiges/auflosungsabgabe": "bso",
|
||
"beendigungsphase-sonstiges/dienstfreistellung-wahrend-kundigungsfrist": "bso",
|
||
"beendigungsphase-sonstiges/dienstzeugnis": "bso",
|
||
"beendigungsphase-sonstiges/konkurrenzklausel-konventionalstrafe": "bso",
|
||
"beendigungsphase-sonstiges/postensuche": "bso",
|
||
"beendigungsphase-sonstiges/sozialplan": "bso",
|
||
}
|
||
|
||
# Fallback keyword scan over the ASCII topic + slug, checked after the
|
||
# explicit map. Order matters (first match wins): specific stems first,
|
||
# generic ones last - the fallback must not steal docs from specific
|
||
# clusters (batch-4 lesson: iesg_zuschlag etc. were caught by generic
|
||
# "zuschlag" before the topic was pinned).
|
||
KEYWORDS: list[tuple[str, str]] = [
|
||
# wiku batch 1 (specific stems first; WIKU docs are pinned via
|
||
# WIKU_PINS — these fallbacks cover Lexis breadcrumbs only)
|
||
("gueterbefoerderung", "gwe"),
|
||
("gastgewerbe", "gwe"),
|
||
("personal-aktuell", "akt"),
|
||
# batch 9 (specific stems MUST precede the batch-8 "abfertigung"/
|
||
# generic keywords: the GSVG severance briefing contains
|
||
# "abfertigung" but belongs to gsv, not end; chapter topics are
|
||
# already pinned in TOPIC_MAP above)
|
||
("bauerliche", "bsv"),
|
||
("pensionsversicherung", "pvs"),
|
||
("pensionsart", "pvs"),
|
||
("pensionsberechnung", "pvs"),
|
||
("pensionskonto", "pvs"),
|
||
("pensionsanpassung", "pvs"),
|
||
("teilpension", "pvs"),
|
||
("krankenversicherung-asvg", "kvs"),
|
||
("krankenversicherung", "kvs"),
|
||
("gsvg", "gsv"),
|
||
("fsvg", "gsv"),
|
||
("selbststandige", "gsv"),
|
||
# batch 8 (specific stems MUST precede the generic beendigungsarten/
|
||
# insolvenz/entgelt/urlaub/meldepflicht keywords; chapter-qualified
|
||
# topics are already pinned in CHAPTER_TOPIC_MAP above)
|
||
("kundigungsentschadigung", "end"),
|
||
("urlaubserersatzleistung", "end"),
|
||
("auszahlung-normal-mehr", "end"),
|
||
("abfertigung", "end"),
|
||
("aufrechnung", "end"),
|
||
("ausbildungskostenruckersatz", "end"),
|
||
("ruckzahlung", "end"),
|
||
("outplacement", "end"),
|
||
("vergleich", "end"),
|
||
("beendigung", "end"),
|
||
("insolvenz", "ins"),
|
||
("betriebsubergang", "ins"),
|
||
("auflosungsabgabe", "bso"),
|
||
("arbeitslosengeld", "bso"),
|
||
("abmeldung", "bso"),
|
||
("konkurrenzklausel", "bso"),
|
||
("konventionalstrafe", "bso"),
|
||
("sozialplan", "bso"),
|
||
("postensuche", "bso"),
|
||
("dienstzeugnis", "bso"),
|
||
("dienstfreistellung", "bso"),
|
||
("datenschutz", "mel"),
|
||
("betriebsrat", "brt"),
|
||
("betriebsvereinbarung", "brt"),
|
||
("belegschaftsvertretung", "brt"),
|
||
("austritt", "bnd"),
|
||
("entlassung", "bnd"),
|
||
("einvernehmliche", "bnd"),
|
||
("zeitablauf", "bnd"),
|
||
("kundigung", "bnd"),
|
||
# batch 7 (specific stems MUST precede the generic kinder-/teilzeit-/
|
||
# entgelt-meldepflicht keywords to prevent fallback misroutes)
|
||
("arbeitskrafteuberlassung", "aug"),
|
||
("entsendung", "grz"),
|
||
("auslandstatigkeit", "grz"),
|
||
("auslandisch", "grz"),
|
||
("standortverlegung", "grz"),
|
||
("telearbeit", "tel"),
|
||
("homeoffice", "tel"),
|
||
("heimarbeit", "tel"),
|
||
("dienstverhinderungsgrund", "dvh"),
|
||
("streik", "dvh"),
|
||
("entgeltfortzahlung-an-feiertagen", "efz"),
|
||
("krankenstand", "krs"),
|
||
("krankengeld", "krs"),
|
||
("krankenentgelt", "krs"),
|
||
("wiedereingliederung", "wie"),
|
||
("elternteilzeit", "elt"),
|
||
("kinderbetreuungsgeld", "kbg"),
|
||
("kinderrehabilitation", "kzs"),
|
||
("bildungskarenz", "kzs"),
|
||
("familienhospizkarenz", "kzs"),
|
||
("pflegekarenz", "kzs"),
|
||
("karenz", "kar"),
|
||
("geburt", "msf"),
|
||
("schutzfrist", "msf"),
|
||
("wochengeld", "msf"),
|
||
("papamonat", "pap"),
|
||
("familienzeitbonus", "pap"),
|
||
("pflegefreistellung", "pfl"),
|
||
("prasenzdienst", "prd"),
|
||
("zivildienst", "prd"),
|
||
("urlaub", "url"),
|
||
# batch 6
|
||
("arbeitnehmerschutz", "asc"),
|
||
("aschg", "asc"),
|
||
("gefahrenevaluierung", "asc"),
|
||
("arbeitsstatte", "asc"),
|
||
("arbeitsmittel", "asc"),
|
||
("arbeitsstoffe", "asc"),
|
||
("arbeitsunfall", "asc"),
|
||
("praventivdienst", "asc"),
|
||
("sicherheitsvertrauensperson", "asc"),
|
||
("bildschirmarbeit", "asc"),
|
||
("hitzeschutz", "asc"),
|
||
("bauarbeitenkoordination", "asc"),
|
||
("dienstnehmerhaftung", "dnh"),
|
||
("mankohaftung", "dnh"),
|
||
("gleichbehandlung", "glb"),
|
||
("diskriminierung", "glb"),
|
||
("mobbing", "mob"),
|
||
("firmeneigentum", "fir"),
|
||
("sachnutzung", "fir"),
|
||
("weisung", "wei"),
|
||
("ordnungsvorschrift", "wei"),
|
||
("bekleidungsvorschrift", "wei"),
|
||
("sorgfaltspflicht", "wei"),
|
||
("schwerarbeit", "swa"),
|
||
# batch 5
|
||
("arbeitszeitgrenzen", "azg"),
|
||
("hochstgrenzen", "azg"),
|
||
("normalarbeitszeit", "azg"),
|
||
("arbeitszeitmodell", "azm"),
|
||
("gleitzeit", "azm"),
|
||
("schichtarbeit", "azm"),
|
||
("kurzarbeit", "azm"),
|
||
("arbeitsbereitschaft", "azm"),
|
||
("rufbereitschaft", "azm"),
|
||
("meldepflicht", "mip"),
|
||
("aushang", "mip"),
|
||
("auflagepflicht", "mip"),
|
||
("nachtschwerarbeit", "nsc"),
|
||
("ruhezeit", "rhz"),
|
||
("ruhepause", "rhz"),
|
||
("feiertagsruhe", "rhz"),
|
||
("schwanger", "sch"),
|
||
("mutterschutz", "sch"),
|
||
("uberstunden", "ues"),
|
||
("zeiterfassung", "zer"),
|
||
# batch 4
|
||
("lohnpfandung", "pfa"),
|
||
("pfandung", "pfa"),
|
||
("lohndumping", "lsd"),
|
||
("unterentlohnung", "lsd"),
|
||
("sicherungsmittel", "lsd"),
|
||
("gplb", "gpl"),
|
||
("dienstgeberbeitrag", "lnk"),
|
||
("arbeiterkammerumlage", "lnk"),
|
||
("kommunalsteuer", "lnk"),
|
||
("wohnbau", "lnk"),
|
||
("iesg", "lnk"),
|
||
("lohnzettel", "lvr"),
|
||
("aufrollung", "lvr"),
|
||
("e-card", "lvr"),
|
||
("nachzahlung", "naz"),
|
||
# batches 1-3
|
||
("altersteilzeit", "atz"),
|
||
("lehrling", "leh"),
|
||
("schnupperlehre", "leh"),
|
||
("lehrverhaeltnis", "leh"),
|
||
("jugend", "jug"),
|
||
("kinder", "jug"),
|
||
("ferialpraktikant", "pra"),
|
||
("ferialpraktikum", "pra"),
|
||
("volontar", "pra"),
|
||
("au-pair", "pra"),
|
||
("bildungsteilzeit", "tzb"),
|
||
("teilzeit", "tzb"),
|
||
("geringfugig", "bes"),
|
||
("fallweise", "bes"),
|
||
("dienstvertrag", "bes"),
|
||
("werkvertrag", "bes"),
|
||
("arbeitsverhaltnis", "bes"),
|
||
("behinderte", "beh"),
|
||
("geschaftsfuhrer", "gsf"),
|
||
("vorstand", "vst"),
|
||
("sachbezug", "sac"),
|
||
("pramie", "prm"),
|
||
("provision", "prm"),
|
||
("dienstreise", "rei"),
|
||
("reisekosten", "rei"),
|
||
("taggeld", "rei"),
|
||
("nachtigung", "rei"),
|
||
("pendler", "pen"),
|
||
("sonderzahlung", "son"),
|
||
("zuschlag", "zus"),
|
||
("zulage", "zus"),
|
||
("vorsorge", "vor"),
|
||
("bmsvg", "vor"),
|
||
("mitarbeiterbeteiligung", "vor"),
|
||
("betriebspension", "vor"),
|
||
("geschenk", "ges"),
|
||
("drittlohn", "ent"),
|
||
("entgelt", "ent"),
|
||
("ausbildungskosten", "ent"),
|
||
]
|
||
|
||
# Cluster registry: ID prefix -> display name (kb.json, INDEX).
|
||
CLUSTERS: dict[str, str] = {
|
||
"atz": "Altersteilzeit",
|
||
"leh": "Lehrverhältnis / Lehrlinge",
|
||
"jug": "Jugendarbeit / Jugendschutz",
|
||
"pra": "Ferialpraktikanten, Volontäre, Au-pair",
|
||
"bes": "Beschäftigungsformen & Abgrenzung",
|
||
"tzb": "Teilzeit & Bildungsteilzeit",
|
||
# batch 2
|
||
"beh": "Behinderte Arbeitnehmer",
|
||
"ent": "Entgelt: Anspruch & Abrechnung",
|
||
"gsf": "Geschäftsführer",
|
||
"lei": "Leitende Angestellte",
|
||
"prm": "Prämien & Provisionen",
|
||
"sac": "Sachbezüge & freiwillige Sozialleistungen",
|
||
"vst": "Vorstandsmitglieder (Vorstand)",
|
||
# batch 3
|
||
"rei": "Dienstreise & Reisekosten",
|
||
"vor": "Vorsorgeleistungen (BMSVG, Betriebspension, Beteiligung)",
|
||
"son": "Sonderzahlungen",
|
||
"zus": "Zulagen und Zuschläge",
|
||
"pen": "Pendlerförderung",
|
||
"ges": "Sachzuwendungen & Geschenke",
|
||
# batch 4
|
||
"lsd": "Bekämpfung von Lohn- und Sozialdumping (LSD-BG)",
|
||
"gpl": "GPLB",
|
||
"lnk": "Lohnnebenkosten (Dienstgeberbeiträge)",
|
||
"pfa": "Lohnpfändung",
|
||
"lvr": "Lohnverrechnung (Meldewesen, Abrechnungsfragen)",
|
||
"naz": "Nachzahlungen",
|
||
# batch 5
|
||
"azg": "Arbeitszeitgrenzen",
|
||
"azm": "Arbeitszeitmodelle (Gleitzeit, Schicht, Kurzarbeit)",
|
||
"mip": "Melde- und Informationspflichten",
|
||
"nsc": "Nachtschwerarbeit",
|
||
"rhz": "Ruhezeiten",
|
||
"sch": "Schwangerschaft (Mutterschutz)",
|
||
"ues": "Überstunden",
|
||
"zer": "Zeiterfassung",
|
||
# batch 6
|
||
"asc": "Arbeitnehmerschutz (ASchG)",
|
||
"dnh": "Dienstnehmerhaftung (Schadenersatz)",
|
||
"glb": "Gleichbehandlung & Diskriminierung",
|
||
"mob": "Mobbing",
|
||
"fir": "Firmeneigentum & Sachnutzung",
|
||
"wei": "Weisungen an die Arbeitnehmer",
|
||
"swa": "Schwerarbeit",
|
||
# batch 7
|
||
"aug": "Arbeitskräfteüberlassung (AÜG)",
|
||
"grz": "Grenzüberschreitender Einsatz & Entsendung",
|
||
"tel": "Teleworking & Heimarbeit",
|
||
"dvh": "Dienstverhinderungsgründe & Streik",
|
||
"efz": "Entgeltfortzahlung an Feiertagen",
|
||
"pfl": "Pflegefreistellung",
|
||
"prd": "Präsenzdienst & Zivildienst",
|
||
"krs": "Krankenstand & Entgeltfortzahlung",
|
||
"wie": "Wiedereingliederung (Geld, Teilzeit)",
|
||
"kar": "Karenz (Elternkarenz)",
|
||
"elt": "Elternteilzeit",
|
||
"kbg": "Kinderbetreuungsgeld & Beihilfen",
|
||
"msf": "Geburt, Schutzfrist & Wochengeld",
|
||
"pap": "Papamonat & Familienzeitbonus",
|
||
"kzs": "Karenz-Sonderformen (Bildungs-, Hospiz-, Pflegekarenz)",
|
||
"url": "Urlaub (Anspruch, Entgelt, Verbrauch)",
|
||
# batch 8
|
||
"bnd": "Beendigungsarten (Austritt, Entlassung, Kündigung)",
|
||
"brt": "Betriebsrat & Betriebsvereinbarungen",
|
||
"end": "Beendigungsansprüche & Endabrechnung (Abfertigung, Vergleich, Stundenguthaben)",
|
||
"ins": "Unternehmensauflösung & Betriebsübergang (Insolvenz)",
|
||
"mel": "Meldungen & Verpflichtungen (Beendigung, Datenschutz)",
|
||
"bso": "Beendigungsphase – Sonstiges (Zeugnis, Konkurrenzklausel, Sozialplan, Postensuche)",
|
||
# batch 9
|
||
"sva": "Sozialversicherungsrecht: Beitragsrecht ASVG (Beitragsgrundlagen, Beitragssätze)",
|
||
"bsv": "Bäuerliche Sozialversicherung (BSVG)",
|
||
"gsv": "GSVG/FSVG: Selbständige, neue Selbstständige, Abfertigung",
|
||
"kvs": "Krankenversicherung ASVG (Leistungen)",
|
||
"pvs": "Pensionsversicherung (Pensionsarten, Pensionskonto)",
|
||
# wiku batch 1
|
||
"akt": "WIKU Personal aktuell (Neuerungen, Urteile, Praxis)",
|
||
"gwe": "Gewerbebezogene Personalverrechnung (Güterbeförderung, Gastgewerbe)",
|
||
}
|
||
|
||
# Frontmatter `topic` values (descriptive ASCII slugs, per approved D2
|
||
# schema example) -> cluster ID prefix. The 3-letter prefix lives in the
|
||
# id; `topic` carries the human-filterable slug.
|
||
TOPIC_TO_PREFIX: dict[str, str] = {
|
||
"altersteilzeit": "atz",
|
||
"lehrlinge": "leh",
|
||
"jugendliche": "jug",
|
||
"ferialpraktikanten": "pra",
|
||
"beschaftigungsformen": "bes",
|
||
"teilzeit": "tzb",
|
||
# batch 2
|
||
"behinderte": "beh",
|
||
"entgelt": "ent",
|
||
"geschaftsfuhrer": "gsf",
|
||
"leitende-angestellte": "lei",
|
||
"pramien": "prm",
|
||
"sachbezuge": "sac",
|
||
"vorstand": "vst",
|
||
# batch 3
|
||
"reisekosten": "rei",
|
||
"vorsorgeleistungen": "vor",
|
||
"sonderzahlungen": "son",
|
||
"zuschlage": "zus",
|
||
"pendlerforderung": "pen",
|
||
"geschenke": "ges",
|
||
# batch 4
|
||
"lohndumping": "lsd",
|
||
"gplb": "gpl",
|
||
"lohnnebenkosten": "lnk",
|
||
"lohnpfandung": "pfa",
|
||
"lohnverrechnung": "lvr",
|
||
"nachzahlungen": "naz",
|
||
# batch 5
|
||
"arbeitszeitgrenzen": "azg",
|
||
"arbeitszeitmodelle": "azm",
|
||
"meldepflichten": "mip",
|
||
"nachtschwerarbeit": "nsc",
|
||
"ruhezeiten": "rhz",
|
||
"schwangerschaft": "sch",
|
||
"uberstunden": "ues",
|
||
"zeiterfassung": "zer",
|
||
# batch 6
|
||
"arbeitnehmerschutz": "asc",
|
||
"dienstnehmerhaftung": "dnh",
|
||
"gleichbehandlung": "glb",
|
||
"mobbing": "mob",
|
||
"firmeneigentum": "fir",
|
||
"weisungen": "wei",
|
||
"schwerarbeit": "swa",
|
||
# batch 7
|
||
"arbeitskrafteuberlassung": "aug",
|
||
"entsendung": "grz",
|
||
"telearbeit": "tel",
|
||
"dienstverhinderung": "dvh",
|
||
"entgeltfortzahlung": "efz",
|
||
"pflegefreistellung": "pfl",
|
||
"prasenzdienst": "prd",
|
||
"krankenstand": "krs",
|
||
"wiedereingliederung": "wie",
|
||
"karenz": "kar",
|
||
"elternteilzeit": "elt",
|
||
"kinderbetreuungsgeld": "kbg",
|
||
"schutzfrist": "msf",
|
||
"familienzeit": "pap",
|
||
"karenzsonderformen": "kzs",
|
||
"urlaub": "url",
|
||
# batch 8
|
||
"beendigungsarten": "bnd",
|
||
"betriebsrat": "brt",
|
||
"endabrechnung": "end",
|
||
"insolvenz-betriebsubergang": "ins",
|
||
"meldungen-verpflichtungen": "mel",
|
||
"beendigungsphase-sonstiges": "bso",
|
||
# batch 9
|
||
"beitragsrecht-asvg": "sva",
|
||
"bauerliche-sozialversicherung": "bsv",
|
||
"gsvg-fsvg": "gsv",
|
||
"krankenversicherung": "kvs",
|
||
"pension": "pvs",
|
||
# wiku batch 1
|
||
"aktuell": "akt",
|
||
"gewerbe": "gwe",
|
||
}
|
||
|
||
# WIKU batch 1 — explicit pins per PDF filename (one publication = one
|
||
# entry, per the WIKU integration model). WIKU publications have no Lexis
|
||
# breadcrumbs, so slug, cluster, stand, work family, chapter and title are
|
||
# pinned here (deterministic and auditable — a PDF without a pin is a
|
||
# hard error at --extract). `stand` sources: title page / filename for
|
||
# the 18 brochures/working papers/casebooks; PDF CreationDate month for
|
||
# the 12 "WIKU Personal aktuell" 2026 issues (content-consistent). All
|
||
# 30 publications: author Wilhelm Kurzböck.
|
||
WIKU_PINS: dict[str, dict] = {
|
||
"Arbeitsunterlage Betrieblichen Vorsorge - Version 2026-04.pdf": {
|
||
"slug": "betriebliche-vorsorge-2026-04",
|
||
"cluster": "vor",
|
||
"stand": "2026-04",
|
||
"work": "WIKU Arbeitsunterlage",
|
||
"chapter": "Betriebliche Vorsorge",
|
||
"title": "Alles zur Betrieblichen Vorsorge",
|
||
},
|
||
"Arbeitsunterlage Mutterschutz, Karenz, KBG, Familienzeit, Elternteilzeit 2025-09.pdf": {
|
||
"slug": "mutterschutz-karenz-kbg-familienzeit-elternteilzeit-2025-09",
|
||
"cluster": "kar",
|
||
"stand": "2025-09",
|
||
"work": "WIKU Arbeitsunterlage",
|
||
"chapter": "Mutterschutz, Karenz, KBG, Familienzeit, Elternteilzeit",
|
||
"title": "Mutterschutz – Karenz – Familienzeit – Kinderbetreuungsgeld – Elternteilzeit",
|
||
},
|
||
"Beendigung von Dienstverhältnissen aus Sicht der PV_ 2026-03.pdf": {
|
||
"slug": "beendigung-dienstverhaeltnisse-2026-03",
|
||
"cluster": "bnd",
|
||
"stand": "2026-03",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Beendigung von Dienstverhältnissen",
|
||
"title": "Beendigung von Dienstverhältnissen aus Sicht der Personalverrechnung",
|
||
},
|
||
"Casebook - gelöste Praxisfälle LV aus 01 bis 04_2026.pdf": {
|
||
"slug": "casebook-lohnverrechnung-01-04-2026",
|
||
"cluster": "lvr",
|
||
"stand": "2026-04",
|
||
"work": "WIKU Casebook",
|
||
"chapter": "Casebook Lohnverrechnung 2026 (Ausgaben 01–04)",
|
||
"title": "Casebook – gelöste LV-Problemfälle (01 bis 04/2026)",
|
||
},
|
||
"Casebook - gelöste Praxisfälle LV aus 01 bis 08-2026.pdf": {
|
||
"slug": "casebook-lohnverrechnung-01-08-2026",
|
||
"cluster": "lvr",
|
||
"stand": "2026-08",
|
||
"work": "WIKU Casebook",
|
||
"chapter": "Casebook Lohnverrechnung 2026 (Ausgaben 01–08)",
|
||
"title": "Casebook – gelöste LV-Problemfälle (01 bis 08/2026)",
|
||
},
|
||
"Dienstverhinderungen - 2026-08.pdf": {
|
||
"slug": "dienstverhinderungen-2026-08",
|
||
"cluster": "dvh",
|
||
"stand": "2026-08",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Dienstverhinderungen",
|
||
"title": "Spezialfragen zu Dienstverhinderungen aus Sicht der Personalverrechnung",
|
||
},
|
||
"Dienstverträge - freie Dienstverträge - Werkverträge - Aushilfen - Stand 2026-01.pdf": {
|
||
"slug": "dienstvertraege-werkvertraege-aushilfen-2026-01",
|
||
"cluster": "bes",
|
||
"stand": "2026-01",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Dienstverträge, freie Dienstverträge, Werkverträge, Aushilfen",
|
||
"title": "Dienstverträge – freie Dienstverträge – Werkverträge – Aushilfen",
|
||
},
|
||
"Fachbroschüre Dienstreise aus Sicht der Personalverrechnung_2026-01.pdf": {
|
||
"slug": "dienstreise-2026-01",
|
||
"cluster": "rei",
|
||
"stand": "2026-01",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Dienstreise",
|
||
"title": "Dienstreise aus Sicht der Personalverrechnung",
|
||
},
|
||
"GmbH-Geschäftsführer und AG-Vorstandsmitglieder aus Sicht der Personalverrechnung - Stand 2023-07.pdf": {
|
||
"slug": "gmbh-gf-ag-vorstaende-2023-07",
|
||
"cluster": "gsf",
|
||
"stand": "2023-07",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "GmbH-Geschäftsführer, AG-Vorstände",
|
||
"title": "GmbH-Geschäftsführer und AG-Vorstände aus Sicht der Personalverrechnung",
|
||
},
|
||
"Grenzüberschreitende Arbeitnehmereinsätze aus Sicht der PV_ 2026-05.pdf": {
|
||
"slug": "grenzuerschreitende-einsaetze-2026-05",
|
||
"cluster": "grz",
|
||
"stand": "2026-05",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Grenzüberschreitende Arbeitnehmereinsätze",
|
||
"title": "Grenzüberschreitende Arbeitnehmereinsätze aus Sicht der Personalverrechnung",
|
||
},
|
||
"Insolvenz aus Sicht der Personalverrechnung - Fachbroschüre - Stand 2024-05.pdf": {
|
||
"slug": "insolvenz-2024-05",
|
||
"cluster": "ins",
|
||
"stand": "2024-05",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Insolvenz",
|
||
"title": "Insolvenz aus Sicht der Personalverrechnung",
|
||
},
|
||
"Lohndumping aus Sicht der Personalverrechnung - Arbeitsunterlage - Stand 2025-10.pdf": {
|
||
"slug": "lohndumping-2025-10",
|
||
"cluster": "lsd",
|
||
"stand": "2025-10",
|
||
"work": "WIKU Arbeitsunterlage",
|
||
"chapter": "Lohn- und Sozialdumping",
|
||
"title": "Lohndumping aus Sicht der Personalverrechnung",
|
||
},
|
||
"Lohnpfändung aus Sicht der Personalverrechnung - 2026-02.pdf": {
|
||
"slug": "lohnpfandung-2026-02",
|
||
"cluster": "pfa",
|
||
"stand": "2026-02",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Lohnpfändung",
|
||
"title": "Lohnpfändung aus Sicht der Personalverrechnung",
|
||
},
|
||
"Personalverrechnung im Güterbeförderungsgewerbe_ 2026-04.pdf": {
|
||
"slug": "gueterbefoerderungsgewerbe-2026-04",
|
||
"cluster": "gwe",
|
||
"stand": "2026-04",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Güterbeförderungsgewerbe",
|
||
"title": "Personalverrechnung im Güterbeförderungsgewerbe",
|
||
},
|
||
"Spezialfragen zum Thema Arbeitszeit - Stand 2023-09.pdf": {
|
||
"slug": "spezialfragen-arbeitszeit-2023-09",
|
||
"cluster": "azg",
|
||
"stand": "2023-09",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Arbeitszeit",
|
||
"title": "Spezialfragen zum Thema Arbeitszeit (ohne Teilzeit)",
|
||
},
|
||
"Spezialfragen zu Sachbezügen aus Sicht der PV_2026-08.pdf": {
|
||
"slug": "spezialfragen-sachbezuege-2026-08",
|
||
"cluster": "sac",
|
||
"stand": "2026-08",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Sachbezüge",
|
||
"title": "Spezialfragen zu Sachbezügen aus Sicht der Personalverrechnung",
|
||
},
|
||
"Spezialfragen zu Sonderzahlungen und sonstigen Bezügen aus Sicht der PV_2026-07.pdf": {
|
||
"slug": "spezialfragen-sonderzahlungen-2026-07",
|
||
"cluster": "son",
|
||
"stand": "2026-07",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Sonderzahlungen & sonstige Bezüge",
|
||
"title": "Spezialfragen zu Sonderzahlungen und sonstigen Bezügen",
|
||
},
|
||
"WIKU-Fachbroschüre Gastgewerbe aus Sicht der Personalverrechnung - Stand per 2025-05.pdf": {
|
||
"slug": "gastgewerbe-2025-05",
|
||
"cluster": "gwe",
|
||
"stand": "2025-05",
|
||
"work": "WIKU Fachbroschüre",
|
||
"chapter": "Gastgewerbe",
|
||
"title": "Gastgewerbe aus Sicht der Personalverrechnung",
|
||
},
|
||
# WIKU Personal aktuell 2026 — stand = CreationDate month (see table
|
||
# docstring); issues 4-5 and 8-9 are combined editions (DOPPELAUSGABE)
|
||
"WIKU Personal aktuell 2026, Nr. 1.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-1",
|
||
"cluster": "akt",
|
||
"stand": "2026-01",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 1",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 1",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 2.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-2",
|
||
"cluster": "akt",
|
||
"stand": "2026-02",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 2",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 2",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 3.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-3",
|
||
"cluster": "akt",
|
||
"stand": "2026-02",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 3",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 3",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 4-5.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-4-5",
|
||
"cluster": "akt",
|
||
"stand": "2026-03",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 4-5 (Doppelheft)",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 4-5 (Doppelheft)",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 6.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-6",
|
||
"cluster": "akt",
|
||
"stand": "2026-04",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 6",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 6",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 7.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-7",
|
||
"cluster": "akt",
|
||
"stand": "2026-04",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 7",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 7",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 8 - 9.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-8-9",
|
||
"cluster": "akt",
|
||
"stand": "2026-05",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 8-9 (Doppelheft)",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 8-9 (Doppelheft)",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 10.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-10",
|
||
"cluster": "akt",
|
||
"stand": "2026-06",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 10",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 10",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 11.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-11",
|
||
"cluster": "akt",
|
||
"stand": "2026-06",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 11",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 11",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 12.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-12",
|
||
"cluster": "akt",
|
||
"stand": "2026-07",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 12",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 12",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 13.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-13",
|
||
"cluster": "akt",
|
||
"stand": "2026-08",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 13",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 13",
|
||
},
|
||
"WIKU Personal aktuell 2026, Nr. 14.pdf": {
|
||
"slug": "personal-aktuell-2026-nr-14",
|
||
"cluster": "akt",
|
||
"stand": "2026-09",
|
||
"work": "WIKU Personal aktuell",
|
||
"chapter": "WIKU Personal aktuell — 2026, Nr. 14",
|
||
"title": "WIKU Personal aktuell 2026, Nr. 14",
|
||
},
|
||
}
|
||
|
||
HEADER_RE = re.compile(r"^\s*Lexis 360®\s*$")
|
||
MARKER = "Lexis Briefings Personalrecht"
|
||
# Breadcrumb: "Kapitel > Thema · Autor(en) · <Monat> <Jahr>". Dossier-style
|
||
# exports merge the breadcrumb and the body into ONE line, so trailing
|
||
# text is allowed and ignored by the (?P<rest>...) tail.
|
||
BC_LINE_RE = re.compile(
|
||
r"^\s*(?P<chapter>[^>·]+?)\s*>\s*(?P<topic>.+?)\s*·\s*(?P<author>.+?)"
|
||
r"\s*·\s*(?P<month>[A-ZÄÖÜ][a-zäöüß]+)\s+(?P<year>\d{4})(?P<rest>.*)$"
|
||
)
|
||
# Breadcrumb variant WITHOUT an author segment (batch-10 Muster exports:
|
||
# "Kapitel > Thema · <Monat> <Jahr>"). The author then defaults to the
|
||
# editorial attribution in parse_meta().
|
||
BC_LINE_NOAUTHOR_RE = re.compile(
|
||
r"^\s*(?P<chapter>[^>·]+?)\s*>\s*(?P<topic>.+?)\s*·\s*"
|
||
r"(?P<month>[A-ZÄÖÜ][a-zäöüß]+)\s+(?P<year>\d{4})(?P<rest>.*)$"
|
||
)
|
||
STAND_RE = re.compile(r"^\d{4}-\d{2}$")
|
||
ID_RE = re.compile(r"^(lb|wk)-[a-z]{3}-\d{2}$")
|
||
|
||
# WIKU page-footer artefact: two short numbers separated by wide gaps
|
||
# (e.g. "09 5"), i.e. page/section counters from the WIKU
|
||
# typesetting — dropped in Layer 1, everything else kept (full text).
|
||
WIKU_PAGE_RE = re.compile(r"^\s*\d{1,2}\s{3,}\d{1,2}\s*$")
|
||
|
||
|
||
def ascii_slug(text: str) -> str:
|
||
"""ASCII form used for topic matching: diacritics stripped (NFKD,
|
||
ü->u, ä->a), ß->ss, lowercase, non-alphanumeric -> hyphen. This
|
||
matches the umlaut-dropped style of the Lexis360 export filenames."""
|
||
text = unicodedata.normalize("NFKD", text)
|
||
text = "".join(c for c in text if not unicodedata.combining(c))
|
||
text = text.lower().replace("ß", "ss")
|
||
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
|
||
return text
|
||
|
||
|
||
def parse_pdf(pdf: Path) -> tuple[str, int, str]:
|
||
"""Return (text, pages, pdfinfo title)."""
|
||
proc = subprocess.run(
|
||
["pdftotext", "-layout", str(pdf), "-"],
|
||
capture_output=True, text=True, check=True,
|
||
)
|
||
pages = 0
|
||
title = ""
|
||
info = subprocess.run(
|
||
["pdfinfo", str(pdf)], capture_output=True, text=True, check=True,
|
||
)
|
||
for line in info.stdout.splitlines():
|
||
if line.startswith("Pages:"):
|
||
pages = int(line.split(":", 1)[1].strip())
|
||
if line.startswith("Title:"):
|
||
title = line.split(":", 1)[1].strip()
|
||
return proc.stdout, pages, title
|
||
|
||
|
||
def clean_text(raw: str) -> str:
|
||
lines = [ln for ln in raw.splitlines() if not HEADER_RE.match(ln)]
|
||
text = "\n".join(lines).replace("\x0c", "")
|
||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||
return text.strip() + "\n"
|
||
|
||
|
||
def clean_wiku_text(raw: str) -> str:
|
||
"""WIKU-specific cleanup: drop page-counter footer lines, keep the
|
||
full text (ad blocks stay in Layer 1 — curation ignores them)."""
|
||
lines = [ln for ln in raw.splitlines() if not WIKU_PAGE_RE.match(ln)]
|
||
text = "\n".join(lines).replace("\x0c", "")
|
||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||
return text.strip() + "\n"
|
||
|
||
|
||
def _match_bc(line: str):
|
||
"""Try the author-carrying breadcrumb regex first, then the
|
||
no-author variant (batch 10)."""
|
||
for rx in (BC_LINE_RE, BC_LINE_NOAUTHOR_RE):
|
||
m = rx.match(line)
|
||
if m:
|
||
return m
|
||
return None
|
||
|
||
|
||
def parse_meta(text: str, pdf_title: str = "") -> dict:
|
||
"""Parse title + breadcrumb from the first page.
|
||
|
||
Layout: a line with the work marker, the chapter title, then the
|
||
breadcrumb "Kapitel > Thema · Autor · <Monat> <Jahr>" (or the
|
||
batch-10 no-author variant "Kapitel > Thema · <Monat> <Jahr>").
|
||
Dossier-style exports merge breadcrumb and body into one line; the
|
||
regex tail (?P<rest>) tolerates that. If no separate title line
|
||
exists, fall back to the pdfinfo Title.
|
||
"""
|
||
lines = text.splitlines()
|
||
marker_idx = None
|
||
for i, ln in enumerate(lines[:20]):
|
||
if MARKER in ln:
|
||
marker_idx = i
|
||
break
|
||
if marker_idx is None:
|
||
raise ValueError("marker 'Lexis Briefings Personalrecht' not found")
|
||
title_lines: list[str] = []
|
||
m = None
|
||
scan = lines[marker_idx + 1:]
|
||
for pos, ln in enumerate(scan):
|
||
cand = _match_bc(ln)
|
||
if cand is None and ("·" in ln or " > " in ln):
|
||
# dossier-style exports wrap the breadcrumb ("· Juli\n2026",
|
||
# "... ·\nAutor · Juli 2026", or the topic split BEFORE the
|
||
# first "·": "… > Auszahlung Normal-, Mehr- und\nÜberstunden;
|
||
# Zeitguthaben · Autor · Juli 2026" — batch-8 variant). Retry
|
||
# with up to two following lines joined in.
|
||
joined = ln.rstrip()
|
||
for nxt in scan[pos + 1:pos + 3]:
|
||
joined = f"{joined} {nxt.strip()}".strip()
|
||
cand = _match_bc(joined)
|
||
if cand:
|
||
break
|
||
if cand:
|
||
m = cand
|
||
break
|
||
if not ln.strip():
|
||
continue
|
||
title_lines.append(ln.strip())
|
||
if len(title_lines) > 4:
|
||
raise ValueError("breadcrumb not found near page header")
|
||
if m is None:
|
||
raise ValueError("breadcrumb not found")
|
||
title = " ".join(t for t in title_lines if t) or pdf_title
|
||
if not title:
|
||
raise ValueError("no title (neither in text nor in pdfinfo)")
|
||
month_no = MONTHS[m.group("month").lower()]
|
||
return {
|
||
"title": title,
|
||
"chapter": m.group("chapter").strip(),
|
||
"topic_group": m.group("topic").strip(),
|
||
# batch-10 no-author variant: editorial default attribution
|
||
"author": m.groupdict().get("author") or "Lexis Redaktion",
|
||
"stand": f"{m.group('year')}-{month_no:02d}",
|
||
"stand_human": f"{m.group('month')} {m.group('year')}",
|
||
}
|
||
|
||
|
||
def resolve_cluster(chapter_ascii: str, topic_ascii: str, slug: str) -> str | None:
|
||
# 1. chapter-qualified pin (same topic, different chapter -> different
|
||
# cluster; batch-8 lesson)
|
||
pair = f"{chapter_ascii}/{topic_ascii}"
|
||
if pair in CHAPTER_TOPIC_MAP:
|
||
return CHAPTER_TOPIC_MAP[pair]
|
||
# 2. plain topic pin
|
||
if topic_ascii in TOPIC_MAP:
|
||
return TOPIC_MAP[topic_ascii]
|
||
# 3. keyword fallback over topic + slug
|
||
for keyword, cluster in KEYWORDS:
|
||
if keyword in topic_ascii or keyword in slug:
|
||
return cluster
|
||
return None
|
||
|
||
|
||
def load_previous_ids(md_dir: Path) -> dict[str, tuple[str, int]]:
|
||
"""slug -> (id, batch) from the source's previous _catalog.json.
|
||
|
||
IDs are frozen once assigned: a later batch must never renumber
|
||
existing entries, because cross_refs and kb.json consumers rely on
|
||
stable IDs. New documents continue after the highest existing
|
||
number in their cluster; numbers of removed documents are retired,
|
||
never reused. Each source (Lexis `lb-`, WIKU `wk-`) keeps its own
|
||
catalog and therefore its own frozen ID space.
|
||
"""
|
||
path = md_dir / "_catalog.json"
|
||
if not path.is_file():
|
||
return {}
|
||
try:
|
||
prev = json.loads(path.read_text(encoding="utf-8"))
|
||
except json.JSONDecodeError:
|
||
return {}
|
||
out = {}
|
||
for e in prev.get("entries", []):
|
||
if "slug" in e and "id" in e:
|
||
out[e["slug"]] = (e["id"], int(e.get("batch", 1)))
|
||
return out
|
||
|
||
|
||
def assign_ids(catalog: list[dict],
|
||
prev: dict[str, tuple[str, int]],
|
||
id_prefix: str) -> list[str]:
|
||
"""Freeze prev ids, assign new sequential ids (namespace `id_prefix`);
|
||
returns warnings for frozen ids whose prefix no longer matches the
|
||
(re)pinned cluster."""
|
||
id_warnings: list[str] = []
|
||
counters: dict[str, int] = {}
|
||
for id_, _batch in prev.values():
|
||
m = ID_RE.match(id_)
|
||
if m and id_[:2] == id_prefix:
|
||
prefix, num = id_[3:6], int(id_[-2:])
|
||
counters[prefix] = max(counters.get(prefix, 0), num)
|
||
for entry in catalog:
|
||
if slug_prev := prev.get(entry["slug"]):
|
||
entry["id"], entry["batch"] = slug_prev
|
||
if entry["cluster"] and entry["id"][3:6] != entry["cluster"]:
|
||
id_warnings.append(
|
||
f"{entry['slug']}: frozen id {entry['id']} no longer matches "
|
||
f"cluster '{entry['cluster']}' - retire and reassign before "
|
||
f"curation")
|
||
for entry in sorted(catalog, key=lambda e: (e["cluster"] or "zzz", e["slug"])):
|
||
if entry.get("id") or entry["cluster"] is None:
|
||
continue
|
||
cluster = entry["cluster"]
|
||
counters[cluster] = counters.get(cluster, 0) + 1
|
||
entry["id"] = f"{id_prefix}-{cluster}-{counters[cluster]:02d}"
|
||
return id_warnings
|
||
|
||
|
||
def write_layer1(entry: dict, text: str, md_dir: Path) -> None:
|
||
md = md_dir / f"{entry['slug']}.md"
|
||
fm = [
|
||
"---",
|
||
f"id: {entry['id']}",
|
||
f"batch: {entry['batch']}",
|
||
f'title: "{entry["title"]}"',
|
||
f'work: "{entry["work"]}"',
|
||
f'chapter: "{entry["chapter"]}"',
|
||
f'topic_group: "{entry["topic_group"]}"',
|
||
f"cluster: {entry['cluster']}",
|
||
f'author: "{entry["author"]}"',
|
||
f"stand: {entry['stand']}",
|
||
f'pdf: "{entry["pdf_rel"]}"',
|
||
f"pages: {entry['pages']}",
|
||
"---",
|
||
"",
|
||
]
|
||
md.write_text("\n".join(fm) + text, encoding="utf-8")
|
||
|
||
|
||
def extract_lexis() -> int:
|
||
src = SOURCES["lexis"]
|
||
if not src["pdf_dir"].is_dir():
|
||
sys.exit(f"missing {src['pdf_dir']}")
|
||
src["md_dir"].mkdir(exist_ok=True)
|
||
pdfs: list[Path] = []
|
||
for sub in (None, *src.get("pdf_subdirs", ())):
|
||
pdir = src["pdf_dir"] / sub if sub else src["pdf_dir"]
|
||
if pdir.is_dir():
|
||
pdfs.extend(sorted(pdir.glob(src["pdf_glob"])))
|
||
catalog: list[dict] = []
|
||
texts: dict[str, str] = {}
|
||
warnings: list[str] = []
|
||
seen_slugs: set[str] = set()
|
||
for pdf in pdfs:
|
||
slug = pdf.stem[len("Lexis360_"):]
|
||
if slug in seen_slugs:
|
||
warnings.append(f"{pdf.name}: slug {slug!r} collides - skipped")
|
||
continue
|
||
seen_slugs.add(slug)
|
||
raw, pages, pdf_title = parse_pdf(pdf)
|
||
text = clean_text(raw)
|
||
texts[slug] = text
|
||
try:
|
||
meta = parse_meta(text, pdf_title)
|
||
except ValueError as exc:
|
||
warnings.append(f"{pdf.name}: {exc}")
|
||
continue
|
||
entry = {
|
||
"slug": slug,
|
||
"source": "lexis",
|
||
"pdf_rel": pdf.relative_to(ROOT).as_posix(),
|
||
"md_rel": f".lexis360/md/{slug}.md",
|
||
"work": WORK_LEXIS,
|
||
"pages": pages,
|
||
"chars": len(text),
|
||
"batch": BATCH_LEXIS,
|
||
**meta,
|
||
}
|
||
topic_ascii = ascii_slug(entry["topic_group"])
|
||
entry["topic_group_ascii"] = topic_ascii
|
||
chapter_ascii = ascii_slug(entry["chapter"])
|
||
entry["cluster"] = resolve_cluster(chapter_ascii, topic_ascii, slug)
|
||
if entry["cluster"] is None:
|
||
warnings.append(f"{slug}: no cluster for topic {topic_ascii!r}")
|
||
catalog.append(entry)
|
||
id_warnings = assign_ids(catalog, load_previous_ids(src["md_dir"]), "lb")
|
||
warnings.extend(id_warnings)
|
||
for entry in catalog:
|
||
if entry["cluster"] is not None:
|
||
write_layer1(entry, texts[entry["slug"]], src["md_dir"])
|
||
report = {
|
||
"source": "lexis",
|
||
"generated_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
|
||
"n_pdfs": len(pdfs),
|
||
"n_entries": len(catalog),
|
||
"n_unclustered": sum(1 for e in catalog if e["cluster"] is None),
|
||
"entries": catalog,
|
||
"warnings": warnings,
|
||
}
|
||
(src["md_dir"] / "_catalog.json").write_text(
|
||
json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print(f"[lexis] extracted {len(catalog)}/{len(pdfs)} "
|
||
f"({report['n_unclustered']} unclustered)")
|
||
topics = sorted({e["topic_group"] for e in catalog})
|
||
print(f"distinct breadcrumb topics ({len(topics)}):")
|
||
for t in topics:
|
||
print(f" - {t}")
|
||
for w in warnings:
|
||
print(f"WARNING: {w}")
|
||
return 0 if not warnings else 1
|
||
|
||
|
||
def extract_wiku() -> int:
|
||
"""WIKU Layer 1: pin-driven extraction (no breadcrumbs — see
|
||
WIKU_PINS). A PDF without a pin is a hard warning: it is NOT
|
||
extracted (never guess metadata for licensed content)."""
|
||
src = SOURCES["wiku"]
|
||
if not src["pdf_dir"].is_dir():
|
||
sys.exit(f"missing {src['pdf_dir']}")
|
||
src["md_dir"].mkdir(parents=True, exist_ok=True)
|
||
pdfs = sorted(src["pdf_dir"].glob(src["pdf_glob"]))
|
||
catalog: list[dict] = []
|
||
texts: dict[str, str] = {}
|
||
warnings: list[str] = []
|
||
for pdf in pdfs:
|
||
pin = WIKU_PINS.get(pdf.name)
|
||
if pin is None:
|
||
warnings.append(
|
||
f"{pdf.name}: no WIKU_PINS entry (slug/cluster/stand/work/"
|
||
f"chapter/title) - add the pin before extracting")
|
||
continue
|
||
slug = pin["slug"]
|
||
raw, pages, _pdf_title = parse_pdf(pdf)
|
||
text = clean_wiku_text(raw)
|
||
texts[slug] = text
|
||
entry = {
|
||
"slug": slug,
|
||
"source": "wiku",
|
||
"pdf_rel": f".wiku/{pdf.name}",
|
||
"md_rel": f".wiku/md/{slug}.md",
|
||
"work": pin["work"],
|
||
"pages": pages,
|
||
"chars": len(text),
|
||
"batch": BATCH_WIKU,
|
||
"title": pin["title"],
|
||
"chapter": pin["chapter"],
|
||
"topic_group": pin["chapter"],
|
||
"topic_group_ascii": ascii_slug(pin["chapter"]),
|
||
"author": pin.get("author", "Wilhelm Kurzböck"),
|
||
"stand": pin["stand"],
|
||
"stand_human": pin["stand"],
|
||
"cluster": pin["cluster"],
|
||
}
|
||
catalog.append(entry)
|
||
id_warnings = assign_ids(catalog, load_previous_ids(src["md_dir"]), "wk")
|
||
warnings.extend(id_warnings)
|
||
for entry in catalog:
|
||
if entry["cluster"] is not None:
|
||
write_layer1(entry, texts[entry["slug"]], src["md_dir"])
|
||
report = {
|
||
"source": "wiku",
|
||
"generated_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
|
||
"n_pdfs": len(pdfs),
|
||
"n_entries": len(catalog),
|
||
"n_unclustered": sum(1 for e in catalog if e["cluster"] is None),
|
||
"entries": catalog,
|
||
"warnings": warnings,
|
||
}
|
||
(src["md_dir"] / "_catalog.json").write_text(
|
||
json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
print(f"[wiku] extracted {len(catalog)}/{len(pdfs)} "
|
||
f"({report['n_unclustered']} unclustered)")
|
||
works = sorted({e["work"] for e in catalog})
|
||
print(f"distinct works ({len(works)}):")
|
||
for w in works:
|
||
print(f" - {w}")
|
||
for w in warnings:
|
||
print(f"WARNING: {w}")
|
||
return 0 if not warnings else 1
|
||
|
||
|
||
def extract(source: str) -> int:
|
||
if source == "lexis":
|
||
return extract_lexis()
|
||
if source == "wiku":
|
||
return extract_wiku()
|
||
rc = 0
|
||
for s in ("lexis", "wiku"):
|
||
rc |= {"lexis": extract_lexis, "wiku": extract_wiku}[s]()
|
||
return rc
|
||
|
||
|
||
# --- frontmatter (controlled YAML subset) ----------------------------------
|
||
|
||
def fm_value(raw: str):
|
||
raw = raw.strip()
|
||
if raw.startswith('"') and raw.endswith('"'):
|
||
return raw[1:-1]
|
||
if raw.startswith("[") and raw.endswith("]"):
|
||
inner = raw[1:-1].strip()
|
||
if not inner:
|
||
return []
|
||
return [v.strip().strip('"') for v in inner.split(",")]
|
||
if re.fullmatch(r"-?\d+", raw):
|
||
return int(raw)
|
||
return raw
|
||
|
||
|
||
def parse_frontmatter(text: str) -> dict:
|
||
if not text.startswith("---\n"):
|
||
raise ValueError("no frontmatter")
|
||
end = text.index("\n---\n", 4)
|
||
lines = text[4:end].splitlines()
|
||
data: dict = {}
|
||
current = None
|
||
for ln in lines:
|
||
if not ln.strip():
|
||
continue
|
||
if ln.startswith(" ") and current:
|
||
key, _, raw = ln.strip().partition(":")
|
||
data[current][key] = fm_value(raw)
|
||
elif ln.startswith("\t"):
|
||
raise ValueError("tabs not allowed in frontmatter")
|
||
else:
|
||
key, _, raw = ln.partition(":")
|
||
key = key.strip()
|
||
if raw.strip() == "":
|
||
data[key] = {}
|
||
current = key
|
||
else:
|
||
data[key] = fm_value(raw)
|
||
current = None
|
||
return data
|
||
|
||
|
||
REQUIRED = [
|
||
"id", "batch", "title", "work", "chapter", "topic", "author", "stand",
|
||
"legal_bases", "tags",
|
||
]
|
||
|
||
|
||
def validate(entry: dict, all_ids: set[str], source: str) -> list[str]:
|
||
errors = []
|
||
for key in REQUIRED:
|
||
if key not in entry:
|
||
errors.append(f"{source}: missing key {key!r}")
|
||
src = entry.get("source", {})
|
||
for key in ("pdf", "text"):
|
||
if key not in src:
|
||
errors.append(f"{source}: missing source.{key}")
|
||
id_ = str(entry.get("id", ""))
|
||
if not ID_RE.match(id_):
|
||
errors.append(f"{source}: bad id {entry.get('id')!r}")
|
||
id_ = ""
|
||
if id_:
|
||
id_space = id_[:2] # "lb" (Lexis) or "wk" (WIKU)
|
||
work = str(entry.get("work", ""))
|
||
if id_space == "lb":
|
||
if work != WORK_LEXIS:
|
||
errors.append(
|
||
f"{source}: lb id requires work {WORK_LEXIS!r}, got {work!r}")
|
||
else:
|
||
if work not in WIKU_WORKS:
|
||
errors.append(
|
||
f"{source}: wk id requires work in "
|
||
f"{list(WIKU_WORKS)}, got {work!r}")
|
||
if str(entry.get("stand", "")) and not STAND_RE.match(str(entry["stand"])):
|
||
errors.append(f"{source}: stand must be YYYY-MM, got {entry['stand']!r}")
|
||
for ref in entry.get("cross_refs", []):
|
||
if ref not in all_ids:
|
||
errors.append(f"{source}: dangling cross_ref {ref!r}")
|
||
max_batch = BATCH_LEXIS if id_.startswith("lb") else BATCH_WIKU
|
||
try:
|
||
batch_no = int(entry["batch"])
|
||
except (KeyError, TypeError, ValueError):
|
||
errors.append(f"{source}: batch must be an integer 1..{max_batch}")
|
||
else:
|
||
if not 1 <= batch_no <= max_batch:
|
||
errors.append(f"{source}: batch must be in 1..{max_batch}, got {batch_no}")
|
||
if entry.get("topic") not in TOPIC_TO_PREFIX:
|
||
errors.append(
|
||
f"{source}: topic must be one of {sorted(TOPIC_TO_PREFIX)}, "
|
||
f"got {entry.get('topic')!r}")
|
||
if id_ and id_[3:6] != TOPIC_TO_PREFIX.get(str(entry.get("topic")), ""):
|
||
errors.append(
|
||
f"{source}: id prefix does not match topic {entry.get('topic')!r}")
|
||
return errors
|
||
|
||
|
||
def _entry_source(e: dict) -> str:
|
||
return "wiku" if str(e.get("id", "")).startswith("wk-") else "lexis"
|
||
|
||
|
||
def write_index(entries: list[dict], clusters: dict) -> None:
|
||
"""Generate wissensbasis/INDEX.md from curated frontmatter (derived,
|
||
never hand-edited — regenerate with --registry). One shared corpus,
|
||
sectioned per source (Lexis first, then WIKU), then per cluster."""
|
||
today = datetime.date.today().isoformat()
|
||
by_source: dict[str, list[dict]] = {}
|
||
for e in entries:
|
||
by_source.setdefault(_entry_source(e), []).append(e)
|
||
n_topics = len({str(e["topic"]) for e in entries})
|
||
header = (
|
||
f"# Index — Wissensbasis Personalverrechnung\n\n"
|
||
f"Generiert am {today} · {len(entries)} Einträge · {n_topics} Cluster. "
|
||
f"Quellen: {WORK_LEXIS} (Lexis 360, lizenzierter Export) und "
|
||
"WIKU Personal (Fachbroschüren, Arbeitsunterlagen, Casebooks, "
|
||
"WIKU Personal aktuell). Volltexte lokal/unversioniert "
|
||
"(`.lexis360/`, `.wiku/`). Werte gelten je zum Quell-Stand "
|
||
"(`stand` im Frontmatter); Widersprüche und offene Punkte sind "
|
||
"mit ⚠/❓ in den Einträgen selbst dokumentiert. Schema & "
|
||
"Konventionen: `README.md`.\n"
|
||
)
|
||
lines = [header]
|
||
source_labels = {
|
||
"lexis": f"{WORK_LEXIS} (Lexis 360-Exporte)",
|
||
"wiku": "WIKU Personal",
|
||
}
|
||
for src in ("lexis", "wiku"):
|
||
src_entries = by_source.get(src, [])
|
||
if not src_entries:
|
||
continue
|
||
lines.append(f"\n# {source_labels[src]} — {len(src_entries)} Einträge\n")
|
||
by_topic: dict[str, list[dict]] = {}
|
||
for e in src_entries:
|
||
by_topic.setdefault(str(e["topic"]), []).append(e)
|
||
for topic, cluster_entries in sorted(by_topic.items()):
|
||
stands = sorted(str(e["stand"]) for e in cluster_entries)
|
||
prefix = TOPIC_TO_PREFIX.get(topic)
|
||
name = CLUSTERS.get(prefix, topic)
|
||
lines.append(
|
||
f"\n## {name} (`topic: {topic}`) — {len(cluster_entries)} "
|
||
f"Einträge · Stand {stands[0]} bis {stands[-1]}\n")
|
||
lines.append("| ID | Titel | Autor | Stand | Datei |")
|
||
lines.append("|---|---|---|---|---|")
|
||
for e in sorted(cluster_entries, key=lambda x: str(x["id"])):
|
||
fname = str(e["source"]["text"]).rsplit("/", 1)[-1]
|
||
lines.append(
|
||
f"| {e['id']} | {e['title']} | {e['author']} | {e['stand']} "
|
||
f"| {fname} |")
|
||
path = CURATED_DIR.parent / "INDEX.md"
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
print(f"INDEX.md written: {len(entries)} entries")
|
||
|
||
|
||
def registry() -> int:
|
||
CURATED_DIR.mkdir(parents=True, exist_ok=True)
|
||
docs = sorted(CURATED_DIR.glob("*.md"))
|
||
if not docs:
|
||
sys.exit(f"no curated docs in {CURATED_DIR}")
|
||
entries = []
|
||
errors: list[str] = []
|
||
for doc in docs:
|
||
try:
|
||
fm = parse_frontmatter(doc.read_text(encoding="utf-8"))
|
||
except ValueError as exc:
|
||
errors.append(f"{doc.name}: {exc}")
|
||
continue
|
||
entries.append(fm)
|
||
all_ids = {str(e.get("id")) for e in entries if e.get("id")}
|
||
for fm, doc in zip(entries, docs):
|
||
errors.extend(validate(fm, all_ids, doc.name))
|
||
if errors:
|
||
for e in errors:
|
||
print(f"ERROR: {e}")
|
||
sys.exit(1)
|
||
entries.sort(key=lambda e: str(e.get("id")))
|
||
clusters = {}
|
||
batches: dict[tuple[str, int], int] = {}
|
||
works: dict[str, int] = {}
|
||
for e in entries:
|
||
clusters[e["topic"]] = clusters.get(e["topic"], 0) + 1
|
||
b = int(e.get("batch", 1))
|
||
space = str(e.get("id", ""))[:2]
|
||
batches[(space, b)] = batches.get((space, b), 0) + 1
|
||
w = str(e.get("work", ""))
|
||
works[w] = works.get(w, 0) + 1
|
||
out = {
|
||
"generated_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"),
|
||
"n_entries": len(entries),
|
||
"sources": [
|
||
{"work": w, "n": n} for w, n in sorted(works.items())
|
||
],
|
||
"batches": [
|
||
{"source": "lexis" if space == "lb" else "wiku",
|
||
"batch": b, "n": n}
|
||
for (space, b), n in sorted(batches.items())
|
||
],
|
||
"layers": {
|
||
"curated": "personalverrechnung/wissensbasis/dokumente",
|
||
"fulltext": ".lexis360/md + .wiku/md (local, unversioned - licensed source texts)",
|
||
"pdf": ".lexis360 + .wiku (local, unversioned - licensed PDF exports)",
|
||
},
|
||
"schema": {
|
||
"id": "lb-<cluster-prefix>-<nn> (Lexis) | wk-<cluster-prefix>-<nn> (WIKU)",
|
||
"stand": "YYYY-MM (Stand der Quelle)",
|
||
"legal_bases": "Rechtsgrundlagen as cited in the source",
|
||
"cross_refs": "ids of related kb entries (lb-* and wk-* may cross-ref)",
|
||
},
|
||
"clusters": [
|
||
{"slug": slug,
|
||
"name": CLUSTERS.get(TOPIC_TO_PREFIX.get(slug, ""), slug),
|
||
"n": n}
|
||
for slug, n in sorted(clusters.items())
|
||
],
|
||
"entries": entries,
|
||
}
|
||
KB_JSON.parent.mkdir(parents=True, exist_ok=True)
|
||
KB_JSON.write_text(
|
||
json.dumps(out, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||
write_index(entries, clusters)
|
||
print(f"kb.json written: {len(entries)} entries, {len(clusters)} clusters")
|
||
return 0
|
||
|
||
|
||
def curated_slugs_for(source: str) -> set[str]:
|
||
"""Layer-2 slugs of one source: Lexis = files WITHOUT the `wiku_`
|
||
prefix (slug = stem), WIKU = files WITH the prefix (slug = stem
|
||
minus prefix)."""
|
||
if not CURATED_DIR.is_dir():
|
||
return set()
|
||
slugs = set()
|
||
for p in CURATED_DIR.glob("*.md"):
|
||
if source == "lexis" and p.name.startswith(WIKU_SLUG_PREFIX):
|
||
continue
|
||
if source == "wiku" and not p.name.startswith(WIKU_SLUG_PREFIX):
|
||
continue
|
||
name = p.name
|
||
# strip "wiku_" prefix AND the .md suffix (slug = bare stem)
|
||
slugs.add(p.stem[len(WIKU_SLUG_PREFIX):] if name.startswith(WIKU_SLUG_PREFIX) else p.stem)
|
||
return slugs
|
||
|
||
|
||
def check(source: str) -> int:
|
||
"""QA: curated docs 1:1 with the source's extraction catalog, sources
|
||
exist."""
|
||
problems: list[str] = []
|
||
src = SOURCES[source]
|
||
catalog_path = src["md_dir"] / "_catalog.json"
|
||
if not catalog_path.is_file():
|
||
sys.exit(f"run --extract --source {source} first "
|
||
f"(missing {catalog_path})")
|
||
catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
|
||
cat_slugs = {e["slug"]: e for e in catalog["entries"]}
|
||
cur_slugs = curated_slugs_for(source)
|
||
for slug in sorted(set(cat_slugs) - cur_slugs):
|
||
problems.append(f"not curated: {slug}")
|
||
for slug in sorted(cur_slugs - set(cat_slugs)):
|
||
problems.append(f"curated but no source: {slug}")
|
||
for slug in sorted(set(cat_slugs) & cur_slugs):
|
||
text_path = ROOT / cat_slugs[slug]["md_rel"]
|
||
if not text_path.is_file():
|
||
problems.append(f"layer-1 text missing: {text_path}")
|
||
pdf_path = ROOT / cat_slugs[slug]["pdf_rel"]
|
||
if not pdf_path.is_file():
|
||
problems.append(f"source pdf missing: {pdf_path}")
|
||
for p in problems:
|
||
print(f"ERROR: {p}")
|
||
print(f"[{source}] check: {len(cat_slugs)} sources, {len(cur_slugs)} "
|
||
f"curated, {len(problems)} problems")
|
||
return 1 if problems else 0
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
group = parser.add_mutually_exclusive_group(required=True)
|
||
group.add_argument("--extract", action="store_true",
|
||
help="extract PDFs into <source> Layer-1 md")
|
||
group.add_argument("--registry", action="store_true",
|
||
help="generate kb.json from curated frontmatter "
|
||
"(shared corpus)")
|
||
group.add_argument("--check", action="store_true",
|
||
help="QA: curated docs vs extraction catalog")
|
||
parser.add_argument("--source", choices=("lexis", "wiku", "all"),
|
||
default="lexis",
|
||
help="source for --extract/--check "
|
||
"(default: lexis; 'all' = both)")
|
||
args = parser.parse_args()
|
||
if args.source == "all":
|
||
sources = ("lexis", "wiku")
|
||
else:
|
||
sources = (args.source,)
|
||
if args.extract:
|
||
rc = 0
|
||
for s in sources:
|
||
rc |= extract(s)
|
||
return rc
|
||
if args.registry:
|
||
return registry()
|
||
rc = 0
|
||
for s in sources:
|
||
rc |= check(s)
|
||
return rc
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main()) |