#!/usr/bin/env python3 """Build the Personalverrechnung knowledge base (Layer-1 extraction + kb.json registry). Two licensed source corpora feed ONE shared curated knowledge base under personalverrechnung/wissensbasis/ (development reference and future copilot corpus): lexis .lexis360/*.pdf PDF exports of "Lexis Briefings Personalrecht" (LexisNexis AT, licensed Lexis 360 subscription); batch 10 adds the same work's sections in subfolders: .lexis360/{beispiele,muster, uebersichten_checklisten}/ wiku .wiku/*.pdf WIKU Personal publications (Fachbroschüren, Arbeitsunterlagen, Casebooks, the periodical "WIKU Personal aktuell") Licensing (decision D1, 2026-09-10): the PDF exports and the extracted full texts stay local and unversioned (.lexis360/, .wiku/ are gitignored), like .firecrawl/. Only the curated entries under wissensbasis/ (own words, short quotes, source citations) are versioned. WIKU has no Lexis breadcrumbs: its metadata (title, chapter, stand, work, cluster) comes from the explicit WIKU_PINS table (one publication = one entry, per the integration model), and Layer-2 files get a "wiku_" filename prefix in the shared dokumente/ directory. Run from the repository root: python3 personalverrechnung/tools/build_lexis_kb.py --extract [--source lexis|wiku|all] python3 personalverrechnung/tools/build_lexis_kb.py --registry # kb.json (shared corpus) python3 personalverrechnung/tools/build_lexis_kb.py --check [--source lexis|wiku|all] Outputs: Layer 1 (unversioned, licensed raw texts): .lexis360/md/.md full text + extraction frontmatter .lexis360/md/_catalog.json parsed metadata of all sources .wiku/md/.md full text + extraction frontmatter .wiku/md/_catalog.json parsed metadata of all sources Layer 2 (versioned, curated by hand): personalverrechnung/wissensbasis/dokumente/.md (Lexis) personalverrechnung/wissensbasis/dokumente/wiku_.md (WIKU) personalverrechnung/wissensbasis/kb.json (generated) Frontmatter schema of curated entries (subset, no PyYAML needed): id (lb-- Lexis | wk-- WIKU), batch, title, work, chapter, topic (cluster slug), author, stand (YYYY-MM), source{pdf,text}, legal_bases[], tags[], cross_refs[] """ from __future__ import annotations import argparse import datetime import json import re import subprocess import sys import unicodedata from pathlib import Path ROOT = Path(__file__).resolve().parents[2] CURATED_DIR = ROOT / "personalverrechnung" / "wissensbasis" / "dokumente" KB_JSON = CURATED_DIR.parent / "kb.json" WORK_LEXIS = "Lexis Briefings Personalrecht" BATCH_LEXIS = 10 BATCH_WIKU = 1 WIKU_SLUG_PREFIX = "wiku_" WIKU_WORKS = ( "WIKU Fachbroschüre", "WIKU Arbeitsunterlage", "WIKU Casebook", "WIKU Personal aktuell", ) # One shared curated corpus (wissensbasis/ + kb.json); the `work` field # distinguishes the licensed sources. SOURCES: dict[str, dict] = { "lexis": { "work": WORK_LEXIS, "pdf_dir": ROOT / ".lexis360", # batch 10 — same work, same breadcrumb format, section exports in # subfolders (Muster, Beispiele, Übersichten & Checklisten) "pdf_subdirs": ("beispiele", "muster", "uebersichten_checklisten"), "md_dir": ROOT / ".lexis360" / "md", "pdf_glob": "Lexis360_*.pdf", "id_prefix": "lb", "max_batch": BATCH_LEXIS, }, "wiku": { "work": None, # per-publication family, see WIKU_PINS "pdf_dir": ROOT / ".wiku", "md_dir": ROOT / ".wiku" / "md", "pdf_glob": "*.pdf", "id_prefix": "wk", "max_batch": BATCH_WIKU, }, } MONTHS = { "jänner": 1, "januar": 1, "februar": 2, "märz": 3, "april": 4, "mai": 5, "juni": 6, "juli": 7, "august": 8, "september": 9, "oktober": 10, "november": 11, "dezember": 12, } # Explicit breadcrumb-topic -> cluster mapping (pinned from a first # --extract pass; keep deterministic and auditable). Keys are the # ascii_slug() form of the parsed breadcrumb topic: diacritics STRIPPED # (ü->u, ä->a, ß->ss), non-alnum -> hyphen — matching the umlaut-dropped # style of the Lexis360 export filenames ("beschaftigung", not # "beschaeftigung"). TOPIC_MAP: dict[str, str] = { "altersteilzeit": "atz", "lehrlinge": "leh", "schnupperlehre": "leh", "jugendliche": "jug", "kinder": "jug", "ferialpraktikanten-volontare": "pra", "geringfugige-beschaftigung": "bes", "fallweise-beschaftigung": "bes", "freier-dienstvertrag": "bes", "werkvertrag": "bes", "echtes-dienstverhaltnis": "bes", "abgrenzung-arbeiter-angestellte": "bes", "teilzeitbeschaftigung": "tzb", "bildungsteilzeit": "tzb", # batch 2 "abrechnungsperiode-fur-bezuge": "ent", "rechtsgrundlagen": "ent", "drittlohn": "ent", "lohn-gehalt": "ent", "behinderte-arbeitnehmer": "beh", "geschaftsfuhrer": "gsf", "leitende-angestellte": "lei", "vorstandsmitglieder": "vst", "sachbezuge-freiwillige-sozialleistungen": "sac", "diverse-pramien": "prm", "provisionen": "prm", # batch 3 "dienstreise-reisekosten": "rei", "ruckerstattung-ausbildungskosten": "ent", "vorsorgeleistungen": "vor", "sonderzahlungen": "son", "zulagen-und-zuschlage": "zus", "pendlerforderung": "pen", "sachzuwendungen-geschenke": "ges", # batch 4 "bekampfung-von-lohn-und-sozialdumping": "lsd", "gplb": "gpl", "lohnnebenkosten": "lnk", "lohnpfandung": "pfa", "lohnverrechnung": "lvr", "nachzahlungen": "naz", # batch 5 "arbeitszeitgrenzen": "azg", "arbeitszeitmodelle": "azm", "melde-und-informationspflichten": "mip", "nachtschwerarbeit": "nsc", "ruhezeiten": "rhz", "schwangerschaft": "sch", "uberstunden": "ues", "zeiterfassung": "zer", # batch 6 "arbeitnehmerschutz-geltungsbereich-gefahrenevaluierung": "asc", "arbeitsstatte-mittel-stoffe": "asc", "arbeitsunfall": "asc", "bauarbeitenkoordinationsgesetz": "asc", "bildschirmarbeit": "asc", "gesundheitsuberwachung": "asc", "hitzeschutz": "asc", "praventivdienste": "asc", "sicherheitsvertrauenspersonen": "asc", "strafbarkeit-aschg": "asc", "dienstnehmerhaftung-schadenersatz": "dnh", "gleichbehandlung": "glb", "mobbing": "mob", "sachnutzung-von-firmeneigentum": "fir", "weisungen-an-die-arbeitnehmer": "wei", "schwerarbeit": "swa", # batch 7 "arbeitskrafteuberlassung": "aug", "grenzuberschreitender-einsatz-von-arbeitskraften": "grz", "standortverlegung": "grz", "teleworking-heimarbeit": "tel", "dienstverhinderungsgrunde-auf-arbeitgeberseite": "dvh", "sonstige-dienstverhinderungsgrunde": "dvh", "streik": "dvh", "entgeltfortzahlung-an-feiertagen": "efz", "pflegefreistellung": "pfl", "prasenzdienst-zivildienst": "prd", "krankenstand": "krs", "wiedereingliederungsteilzeit": "wie", "arbeitsunfall-und-berufskrankheit": "asc", "karenz": "kar", "elternteilzeit": "elt", "kinderbetreuungsgeld": "kbg", "beihilfen": "kbg", "geburt-schutzfrist": "msf", "wochengeld": "msf", "papamonat-und-familienzeitbonus": "pap", "bildungskarenz": "kzs", "familienhospizkarenz": "kzs", "pflegekarenz-pflegeteilzeit": "kzs", "kinderrehabilitation": "kzs", "urlaubsablose": "url", "urlaubsanspruch-ausmass": "url", "urlaubsaufzeichnungen": "url", "urlaubsentgelt": "url", "urlaubsersatzleistung": "url", "urlaubsverbrauch": "url", # batch 9 — chapter "Sozialversicherung" (all topic groups unique to # this chapter, so plain pins suffice; gsv also carries the GSVG # self-employed severance briefing which the bare "abfertigung" # keyword would otherwise steal for the end cluster) "beitragsrecht-asvg": "sva", "beitragsrecht-bauerliche-sozialversicherung": "bsv", "beitragsrecht-gsvg": "gsv", "gesellschaftsformen-und-sv-rechtliche-zuordnung": "gsv", "krankenversicherung-asvg": "kvs", "neue-selbststandige-nach-gsvg": "gsv", "pensionsversicherung": "pvs", "abfertigung-fur-selbststandige": "gsv", # wiku batch 1 — WIKU-specific topic groups (WIKU publications are # pinned via WIKU_PINS; these entries complete the four-structure set # for the new clusters per the curation conventions) "personal-aktuell": "akt", "aktuell": "akt", "gastgewerbe": "gwe", "gueterbefoerderungsgewerbe": "gwe", "gewerbe": "gwe", } # Chapter-qualified topic pins, checked BEFORE the plain TOPIC_MAP: # the same topic string can legitimately appear under two different # chapters and belong to two different clusters (batch-8 lesson: # "Sonderzahlungen", "Provisionen", "Pramien", "Urlaubsersatzleistung" # exist both in "Entgelt: Anspruch & Abrechnung" and in # "Beendigungsansprueche & Endabrechnung"). Keys are # "/" (ascii_slug() form, & -> hyphen). CHAPTER_TOPIC_MAP: dict[str, str] = { # batch 8 — chapter "Beendigungsarten" -> bnd (Kündigungsschutz and # the remaining Kündigungsarten sub-topics are covered by the plain # TOPIC_MAP entry "kündigung" -> bnd below) "beendigungsarten/auflosung-in-probezeit": "bnd", "beendigungsarten/austritt-allgemein": "bnd", "beendigungsarten/austrittsgrunde": "bnd", "beendigungsarten/einvernehmliche-losung": "bnd", "beendigungsarten/entlassung-allgemein": "bnd", "beendigungsarten/entlassungsgrunde": "bnd", "beendigungsarten/kundigung": "bnd", "beendigungsarten/kundigungsschutz": "bnd", "beendigungsarten/tod-des-arbeitgebers": "bnd", "beendigungsarten/tod-des-arbeitnehmers": "bnd", "beendigungsarten/zeitablauf-bei-befristung": "bnd", # batch 8 — chapter "Betriebsrat & Betriebsvereinbarungen" -> brt "betriebsrat-betriebsvereinbarungen/belegschaftsvertretungen-betriebs-und-arbeitnehmerbegriff": "brt", "betriebsrat-betriebsvereinbarungen/befugnisse-des-betriebsrates": "brt", "betriebsrat-betriebsvereinbarungen/betriebsvereinbarungen": "brt", "betriebsrat-betriebsvereinbarungen/rechtsstellung-der-betriebsratsmitglieder": "brt", # batch 8 — chapter "Beendigungsansprueche & Endabrechnung" -> end "beendigungsansprueche-endabrechnung/abfertigung-alt": "end", "beendigungsansprueche-endabrechnung/abfertigung-neu": "end", "beendigungsansprueche-endabrechnung/aufrechnung-mit-noch-offenen-vorschussen": "end", "beendigungsansprueche-endabrechnung/ausbildungskostenruckersatz": "end", "beendigungsansprueche-endabrechnung/auszahlung-normal-mehr-und-uberstunden-zeitguthaben": "end", "beendigungsansprueche-endabrechnung/freiwillige-abfertigung-abfindung": "end", "beendigungsansprueche-endabrechnung/kundigungsentschadigung": "end", "beendigungsansprueche-endabrechnung/outplacement": "end", "beendigungsansprueche-endabrechnung/pramien": "end", "beendigungsansprueche-endabrechnung/provisionen": "end", "beendigungsansprueche-endabrechnung/ruckzahlung-von-darlehen": "end", "beendigungsansprueche-endabrechnung/sonderzahlungen": "end", "beendigungsansprueche-endabrechnung/urlaubsersatzleistung": "end", "beendigungsansprueche-endabrechnung/vergleich-vergleichssummen": "end", # batch 8 — chapter "Unternehmensauflösung & Betriebsübergang" -> ins "unternehmensaufloesung-betriebsubergang/betriebsubergang": "ins", "unternehmensaufloesung-betriebsubergang/insolvenz-des-arbeitgebers": "ins", # batch 8 — chapter "Meldungen & Verpflichtungen" -> mel "meldungen-verpflichtungen/arbeitsrecht-und-datenschutz": "mel", "meldungen-verpflichtungen/auflage-und-aushangpflichten": "mel", "meldungen-verpflichtungen/informationspflichten": "mel", "meldungen-verpflichtungen/meldepflichten": "mel", "meldungen-verpflichtungen/schriftform-im-arbeitsrecht": "mel", # batch 8 — chapter "Beendigungsphase - Sonstiges" -> bso "beendigungsphase-sonstiges/abmeldung-sozialversicherung": "bso", "beendigungsphase-sonstiges/arbeitslosengeld": "bso", "beendigungsphase-sonstiges/asg-verfahren": "bso", "beendigungsphase-sonstiges/auflosungsabgabe": "bso", "beendigungsphase-sonstiges/dienstfreistellung-wahrend-kundigungsfrist": "bso", "beendigungsphase-sonstiges/dienstzeugnis": "bso", "beendigungsphase-sonstiges/konkurrenzklausel-konventionalstrafe": "bso", "beendigungsphase-sonstiges/postensuche": "bso", "beendigungsphase-sonstiges/sozialplan": "bso", } # Fallback keyword scan over the ASCII topic + slug, checked after the # explicit map. Order matters (first match wins): specific stems first, # generic ones last - the fallback must not steal docs from specific # clusters (batch-4 lesson: iesg_zuschlag etc. were caught by generic # "zuschlag" before the topic was pinned). KEYWORDS: list[tuple[str, str]] = [ # wiku batch 1 (specific stems first; WIKU docs are pinned via # WIKU_PINS — these fallbacks cover Lexis breadcrumbs only) ("gueterbefoerderung", "gwe"), ("gastgewerbe", "gwe"), ("personal-aktuell", "akt"), # batch 9 (specific stems MUST precede the batch-8 "abfertigung"/ # generic keywords: the GSVG severance briefing contains # "abfertigung" but belongs to gsv, not end; chapter topics are # already pinned in TOPIC_MAP above) ("bauerliche", "bsv"), ("pensionsversicherung", "pvs"), ("pensionsart", "pvs"), ("pensionsberechnung", "pvs"), ("pensionskonto", "pvs"), ("pensionsanpassung", "pvs"), ("teilpension", "pvs"), ("krankenversicherung-asvg", "kvs"), ("krankenversicherung", "kvs"), ("gsvg", "gsv"), ("fsvg", "gsv"), ("selbststandige", "gsv"), # batch 8 (specific stems MUST precede the generic beendigungsarten/ # insolvenz/entgelt/urlaub/meldepflicht keywords; chapter-qualified # topics are already pinned in CHAPTER_TOPIC_MAP above) ("kundigungsentschadigung", "end"), ("urlaubserersatzleistung", "end"), ("auszahlung-normal-mehr", "end"), ("abfertigung", "end"), ("aufrechnung", "end"), ("ausbildungskostenruckersatz", "end"), ("ruckzahlung", "end"), ("outplacement", "end"), ("vergleich", "end"), ("beendigung", "end"), ("insolvenz", "ins"), ("betriebsubergang", "ins"), ("auflosungsabgabe", "bso"), ("arbeitslosengeld", "bso"), ("abmeldung", "bso"), ("konkurrenzklausel", "bso"), ("konventionalstrafe", "bso"), ("sozialplan", "bso"), ("postensuche", "bso"), ("dienstzeugnis", "bso"), ("dienstfreistellung", "bso"), ("datenschutz", "mel"), ("betriebsrat", "brt"), ("betriebsvereinbarung", "brt"), ("belegschaftsvertretung", "brt"), ("austritt", "bnd"), ("entlassung", "bnd"), ("einvernehmliche", "bnd"), ("zeitablauf", "bnd"), ("kundigung", "bnd"), # batch 7 (specific stems MUST precede the generic kinder-/teilzeit-/ # entgelt-meldepflicht keywords to prevent fallback misroutes) ("arbeitskrafteuberlassung", "aug"), ("entsendung", "grz"), ("auslandstatigkeit", "grz"), ("auslandisch", "grz"), ("standortverlegung", "grz"), ("telearbeit", "tel"), ("homeoffice", "tel"), ("heimarbeit", "tel"), ("dienstverhinderungsgrund", "dvh"), ("streik", "dvh"), ("entgeltfortzahlung-an-feiertagen", "efz"), ("krankenstand", "krs"), ("krankengeld", "krs"), ("krankenentgelt", "krs"), ("wiedereingliederung", "wie"), ("elternteilzeit", "elt"), ("kinderbetreuungsgeld", "kbg"), ("kinderrehabilitation", "kzs"), ("bildungskarenz", "kzs"), ("familienhospizkarenz", "kzs"), ("pflegekarenz", "kzs"), ("karenz", "kar"), ("geburt", "msf"), ("schutzfrist", "msf"), ("wochengeld", "msf"), ("papamonat", "pap"), ("familienzeitbonus", "pap"), ("pflegefreistellung", "pfl"), ("prasenzdienst", "prd"), ("zivildienst", "prd"), ("urlaub", "url"), # batch 6 ("arbeitnehmerschutz", "asc"), ("aschg", "asc"), ("gefahrenevaluierung", "asc"), ("arbeitsstatte", "asc"), ("arbeitsmittel", "asc"), ("arbeitsstoffe", "asc"), ("arbeitsunfall", "asc"), ("praventivdienst", "asc"), ("sicherheitsvertrauensperson", "asc"), ("bildschirmarbeit", "asc"), ("hitzeschutz", "asc"), ("bauarbeitenkoordination", "asc"), ("dienstnehmerhaftung", "dnh"), ("mankohaftung", "dnh"), ("gleichbehandlung", "glb"), ("diskriminierung", "glb"), ("mobbing", "mob"), ("firmeneigentum", "fir"), ("sachnutzung", "fir"), ("weisung", "wei"), ("ordnungsvorschrift", "wei"), ("bekleidungsvorschrift", "wei"), ("sorgfaltspflicht", "wei"), ("schwerarbeit", "swa"), # batch 5 ("arbeitszeitgrenzen", "azg"), ("hochstgrenzen", "azg"), ("normalarbeitszeit", "azg"), ("arbeitszeitmodell", "azm"), ("gleitzeit", "azm"), ("schichtarbeit", "azm"), ("kurzarbeit", "azm"), ("arbeitsbereitschaft", "azm"), ("rufbereitschaft", "azm"), ("meldepflicht", "mip"), ("aushang", "mip"), ("auflagepflicht", "mip"), ("nachtschwerarbeit", "nsc"), ("ruhezeit", "rhz"), ("ruhepause", "rhz"), ("feiertagsruhe", "rhz"), ("schwanger", "sch"), ("mutterschutz", "sch"), ("uberstunden", "ues"), ("zeiterfassung", "zer"), # batch 4 ("lohnpfandung", "pfa"), ("pfandung", "pfa"), ("lohndumping", "lsd"), ("unterentlohnung", "lsd"), ("sicherungsmittel", "lsd"), ("gplb", "gpl"), ("dienstgeberbeitrag", "lnk"), ("arbeiterkammerumlage", "lnk"), ("kommunalsteuer", "lnk"), ("wohnbau", "lnk"), ("iesg", "lnk"), ("lohnzettel", "lvr"), ("aufrollung", "lvr"), ("e-card", "lvr"), ("nachzahlung", "naz"), # batches 1-3 ("altersteilzeit", "atz"), ("lehrling", "leh"), ("schnupperlehre", "leh"), ("lehrverhaeltnis", "leh"), ("jugend", "jug"), ("kinder", "jug"), ("ferialpraktikant", "pra"), ("ferialpraktikum", "pra"), ("volontar", "pra"), ("au-pair", "pra"), ("bildungsteilzeit", "tzb"), ("teilzeit", "tzb"), ("geringfugig", "bes"), ("fallweise", "bes"), ("dienstvertrag", "bes"), ("werkvertrag", "bes"), ("arbeitsverhaltnis", "bes"), ("behinderte", "beh"), ("geschaftsfuhrer", "gsf"), ("vorstand", "vst"), ("sachbezug", "sac"), ("pramie", "prm"), ("provision", "prm"), ("dienstreise", "rei"), ("reisekosten", "rei"), ("taggeld", "rei"), ("nachtigung", "rei"), ("pendler", "pen"), ("sonderzahlung", "son"), ("zuschlag", "zus"), ("zulage", "zus"), ("vorsorge", "vor"), ("bmsvg", "vor"), ("mitarbeiterbeteiligung", "vor"), ("betriebspension", "vor"), ("geschenk", "ges"), ("drittlohn", "ent"), ("entgelt", "ent"), ("ausbildungskosten", "ent"), ] # Cluster registry: ID prefix -> display name (kb.json, INDEX). CLUSTERS: dict[str, str] = { "atz": "Altersteilzeit", "leh": "Lehrverhältnis / Lehrlinge", "jug": "Jugendarbeit / Jugendschutz", "pra": "Ferialpraktikanten, Volontäre, Au-pair", "bes": "Beschäftigungsformen & Abgrenzung", "tzb": "Teilzeit & Bildungsteilzeit", # batch 2 "beh": "Behinderte Arbeitnehmer", "ent": "Entgelt: Anspruch & Abrechnung", "gsf": "Geschäftsführer", "lei": "Leitende Angestellte", "prm": "Prämien & Provisionen", "sac": "Sachbezüge & freiwillige Sozialleistungen", "vst": "Vorstandsmitglieder (Vorstand)", # batch 3 "rei": "Dienstreise & Reisekosten", "vor": "Vorsorgeleistungen (BMSVG, Betriebspension, Beteiligung)", "son": "Sonderzahlungen", "zus": "Zulagen und Zuschläge", "pen": "Pendlerförderung", "ges": "Sachzuwendungen & Geschenke", # batch 4 "lsd": "Bekämpfung von Lohn- und Sozialdumping (LSD-BG)", "gpl": "GPLB", "lnk": "Lohnnebenkosten (Dienstgeberbeiträge)", "pfa": "Lohnpfändung", "lvr": "Lohnverrechnung (Meldewesen, Abrechnungsfragen)", "naz": "Nachzahlungen", # batch 5 "azg": "Arbeitszeitgrenzen", "azm": "Arbeitszeitmodelle (Gleitzeit, Schicht, Kurzarbeit)", "mip": "Melde- und Informationspflichten", "nsc": "Nachtschwerarbeit", "rhz": "Ruhezeiten", "sch": "Schwangerschaft (Mutterschutz)", "ues": "Überstunden", "zer": "Zeiterfassung", # batch 6 "asc": "Arbeitnehmerschutz (ASchG)", "dnh": "Dienstnehmerhaftung (Schadenersatz)", "glb": "Gleichbehandlung & Diskriminierung", "mob": "Mobbing", "fir": "Firmeneigentum & Sachnutzung", "wei": "Weisungen an die Arbeitnehmer", "swa": "Schwerarbeit", # batch 7 "aug": "Arbeitskräfteüberlassung (AÜG)", "grz": "Grenzüberschreitender Einsatz & Entsendung", "tel": "Teleworking & Heimarbeit", "dvh": "Dienstverhinderungsgründe & Streik", "efz": "Entgeltfortzahlung an Feiertagen", "pfl": "Pflegefreistellung", "prd": "Präsenzdienst & Zivildienst", "krs": "Krankenstand & Entgeltfortzahlung", "wie": "Wiedereingliederung (Geld, Teilzeit)", "kar": "Karenz (Elternkarenz)", "elt": "Elternteilzeit", "kbg": "Kinderbetreuungsgeld & Beihilfen", "msf": "Geburt, Schutzfrist & Wochengeld", "pap": "Papamonat & Familienzeitbonus", "kzs": "Karenz-Sonderformen (Bildungs-, Hospiz-, Pflegekarenz)", "url": "Urlaub (Anspruch, Entgelt, Verbrauch)", # batch 8 "bnd": "Beendigungsarten (Austritt, Entlassung, Kündigung)", "brt": "Betriebsrat & Betriebsvereinbarungen", "end": "Beendigungsansprüche & Endabrechnung (Abfertigung, Vergleich, Stundenguthaben)", "ins": "Unternehmensauflösung & Betriebsübergang (Insolvenz)", "mel": "Meldungen & Verpflichtungen (Beendigung, Datenschutz)", "bso": "Beendigungsphase – Sonstiges (Zeugnis, Konkurrenzklausel, Sozialplan, Postensuche)", # batch 9 "sva": "Sozialversicherungsrecht: Beitragsrecht ASVG (Beitragsgrundlagen, Beitragssätze)", "bsv": "Bäuerliche Sozialversicherung (BSVG)", "gsv": "GSVG/FSVG: Selbständige, neue Selbstständige, Abfertigung", "kvs": "Krankenversicherung ASVG (Leistungen)", "pvs": "Pensionsversicherung (Pensionsarten, Pensionskonto)", # wiku batch 1 "akt": "WIKU Personal aktuell (Neuerungen, Urteile, Praxis)", "gwe": "Gewerbebezogene Personalverrechnung (Güterbeförderung, Gastgewerbe)", } # Frontmatter `topic` values (descriptive ASCII slugs, per approved D2 # schema example) -> cluster ID prefix. The 3-letter prefix lives in the # id; `topic` carries the human-filterable slug. TOPIC_TO_PREFIX: dict[str, str] = { "altersteilzeit": "atz", "lehrlinge": "leh", "jugendliche": "jug", "ferialpraktikanten": "pra", "beschaftigungsformen": "bes", "teilzeit": "tzb", # batch 2 "behinderte": "beh", "entgelt": "ent", "geschaftsfuhrer": "gsf", "leitende-angestellte": "lei", "pramien": "prm", "sachbezuge": "sac", "vorstand": "vst", # batch 3 "reisekosten": "rei", "vorsorgeleistungen": "vor", "sonderzahlungen": "son", "zuschlage": "zus", "pendlerforderung": "pen", "geschenke": "ges", # batch 4 "lohndumping": "lsd", "gplb": "gpl", "lohnnebenkosten": "lnk", "lohnpfandung": "pfa", "lohnverrechnung": "lvr", "nachzahlungen": "naz", # batch 5 "arbeitszeitgrenzen": "azg", "arbeitszeitmodelle": "azm", "meldepflichten": "mip", "nachtschwerarbeit": "nsc", "ruhezeiten": "rhz", "schwangerschaft": "sch", "uberstunden": "ues", "zeiterfassung": "zer", # batch 6 "arbeitnehmerschutz": "asc", "dienstnehmerhaftung": "dnh", "gleichbehandlung": "glb", "mobbing": "mob", "firmeneigentum": "fir", "weisungen": "wei", "schwerarbeit": "swa", # batch 7 "arbeitskrafteuberlassung": "aug", "entsendung": "grz", "telearbeit": "tel", "dienstverhinderung": "dvh", "entgeltfortzahlung": "efz", "pflegefreistellung": "pfl", "prasenzdienst": "prd", "krankenstand": "krs", "wiedereingliederung": "wie", "karenz": "kar", "elternteilzeit": "elt", "kinderbetreuungsgeld": "kbg", "schutzfrist": "msf", "familienzeit": "pap", "karenzsonderformen": "kzs", "urlaub": "url", # batch 8 "beendigungsarten": "bnd", "betriebsrat": "brt", "endabrechnung": "end", "insolvenz-betriebsubergang": "ins", "meldungen-verpflichtungen": "mel", "beendigungsphase-sonstiges": "bso", # batch 9 "beitragsrecht-asvg": "sva", "bauerliche-sozialversicherung": "bsv", "gsvg-fsvg": "gsv", "krankenversicherung": "kvs", "pension": "pvs", # wiku batch 1 "aktuell": "akt", "gewerbe": "gwe", } # WIKU batch 1 — explicit pins per PDF filename (one publication = one # entry, per the WIKU integration model). WIKU publications have no Lexis # breadcrumbs, so slug, cluster, stand, work family, chapter and title are # pinned here (deterministic and auditable — a PDF without a pin is a # hard error at --extract). `stand` sources: title page / filename for # the 18 brochures/working papers/casebooks; PDF CreationDate month for # the 12 "WIKU Personal aktuell" 2026 issues (content-consistent). All # 30 publications: author Wilhelm Kurzböck. WIKU_PINS: dict[str, dict] = { "Arbeitsunterlage Betrieblichen Vorsorge - Version 2026-04.pdf": { "slug": "betriebliche-vorsorge-2026-04", "cluster": "vor", "stand": "2026-04", "work": "WIKU Arbeitsunterlage", "chapter": "Betriebliche Vorsorge", "title": "Alles zur Betrieblichen Vorsorge", }, "Arbeitsunterlage Mutterschutz, Karenz, KBG, Familienzeit, Elternteilzeit 2025-09.pdf": { "slug": "mutterschutz-karenz-kbg-familienzeit-elternteilzeit-2025-09", "cluster": "kar", "stand": "2025-09", "work": "WIKU Arbeitsunterlage", "chapter": "Mutterschutz, Karenz, KBG, Familienzeit, Elternteilzeit", "title": "Mutterschutz – Karenz – Familienzeit – Kinderbetreuungsgeld – Elternteilzeit", }, "Beendigung von Dienstverhältnissen aus Sicht der PV_ 2026-03.pdf": { "slug": "beendigung-dienstverhaeltnisse-2026-03", "cluster": "bnd", "stand": "2026-03", "work": "WIKU Fachbroschüre", "chapter": "Beendigung von Dienstverhältnissen", "title": "Beendigung von Dienstverhältnissen aus Sicht der Personalverrechnung", }, "Casebook - gelöste Praxisfälle LV aus 01 bis 04_2026.pdf": { "slug": "casebook-lohnverrechnung-01-04-2026", "cluster": "lvr", "stand": "2026-04", "work": "WIKU Casebook", "chapter": "Casebook Lohnverrechnung 2026 (Ausgaben 01–04)", "title": "Casebook – gelöste LV-Problemfälle (01 bis 04/2026)", }, "Casebook - gelöste Praxisfälle LV aus 01 bis 08-2026.pdf": { "slug": "casebook-lohnverrechnung-01-08-2026", "cluster": "lvr", "stand": "2026-08", "work": "WIKU Casebook", "chapter": "Casebook Lohnverrechnung 2026 (Ausgaben 01–08)", "title": "Casebook – gelöste LV-Problemfälle (01 bis 08/2026)", }, "Dienstverhinderungen - 2026-08.pdf": { "slug": "dienstverhinderungen-2026-08", "cluster": "dvh", "stand": "2026-08", "work": "WIKU Fachbroschüre", "chapter": "Dienstverhinderungen", "title": "Spezialfragen zu Dienstverhinderungen aus Sicht der Personalverrechnung", }, "Dienstverträge - freie Dienstverträge - Werkverträge - Aushilfen - Stand 2026-01.pdf": { "slug": "dienstvertraege-werkvertraege-aushilfen-2026-01", "cluster": "bes", "stand": "2026-01", "work": "WIKU Fachbroschüre", "chapter": "Dienstverträge, freie Dienstverträge, Werkverträge, Aushilfen", "title": "Dienstverträge – freie Dienstverträge – Werkverträge – Aushilfen", }, "Fachbroschüre Dienstreise aus Sicht der Personalverrechnung_2026-01.pdf": { "slug": "dienstreise-2026-01", "cluster": "rei", "stand": "2026-01", "work": "WIKU Fachbroschüre", "chapter": "Dienstreise", "title": "Dienstreise aus Sicht der Personalverrechnung", }, "GmbH-Geschäftsführer und AG-Vorstandsmitglieder aus Sicht der Personalverrechnung - Stand 2023-07.pdf": { "slug": "gmbh-gf-ag-vorstaende-2023-07", "cluster": "gsf", "stand": "2023-07", "work": "WIKU Fachbroschüre", "chapter": "GmbH-Geschäftsführer, AG-Vorstände", "title": "GmbH-Geschäftsführer und AG-Vorstände aus Sicht der Personalverrechnung", }, "Grenzüberschreitende Arbeitnehmereinsätze aus Sicht der PV_ 2026-05.pdf": { "slug": "grenzuerschreitende-einsaetze-2026-05", "cluster": "grz", "stand": "2026-05", "work": "WIKU Fachbroschüre", "chapter": "Grenzüberschreitende Arbeitnehmereinsätze", "title": "Grenzüberschreitende Arbeitnehmereinsätze aus Sicht der Personalverrechnung", }, "Insolvenz aus Sicht der Personalverrechnung - Fachbroschüre - Stand 2024-05.pdf": { "slug": "insolvenz-2024-05", "cluster": "ins", "stand": "2024-05", "work": "WIKU Fachbroschüre", "chapter": "Insolvenz", "title": "Insolvenz aus Sicht der Personalverrechnung", }, "Lohndumping aus Sicht der Personalverrechnung - Arbeitsunterlage - Stand 2025-10.pdf": { "slug": "lohndumping-2025-10", "cluster": "lsd", "stand": "2025-10", "work": "WIKU Arbeitsunterlage", "chapter": "Lohn- und Sozialdumping", "title": "Lohndumping aus Sicht der Personalverrechnung", }, "Lohnpfändung aus Sicht der Personalverrechnung - 2026-02.pdf": { "slug": "lohnpfandung-2026-02", "cluster": "pfa", "stand": "2026-02", "work": "WIKU Fachbroschüre", "chapter": "Lohnpfändung", "title": "Lohnpfändung aus Sicht der Personalverrechnung", }, "Personalverrechnung im Güterbeförderungsgewerbe_ 2026-04.pdf": { "slug": "gueterbefoerderungsgewerbe-2026-04", "cluster": "gwe", "stand": "2026-04", "work": "WIKU Fachbroschüre", "chapter": "Güterbeförderungsgewerbe", "title": "Personalverrechnung im Güterbeförderungsgewerbe", }, "Spezialfragen zum Thema Arbeitszeit - Stand 2023-09.pdf": { "slug": "spezialfragen-arbeitszeit-2023-09", "cluster": "azg", "stand": "2023-09", "work": "WIKU Fachbroschüre", "chapter": "Arbeitszeit", "title": "Spezialfragen zum Thema Arbeitszeit (ohne Teilzeit)", }, "Spezialfragen zu Sachbezügen aus Sicht der PV_2026-08.pdf": { "slug": "spezialfragen-sachbezuege-2026-08", "cluster": "sac", "stand": "2026-08", "work": "WIKU Fachbroschüre", "chapter": "Sachbezüge", "title": "Spezialfragen zu Sachbezügen aus Sicht der Personalverrechnung", }, "Spezialfragen zu Sonderzahlungen und sonstigen Bezügen aus Sicht der PV_2026-07.pdf": { "slug": "spezialfragen-sonderzahlungen-2026-07", "cluster": "son", "stand": "2026-07", "work": "WIKU Fachbroschüre", "chapter": "Sonderzahlungen & sonstige Bezüge", "title": "Spezialfragen zu Sonderzahlungen und sonstigen Bezügen", }, "WIKU-Fachbroschüre Gastgewerbe aus Sicht der Personalverrechnung - Stand per 2025-05.pdf": { "slug": "gastgewerbe-2025-05", "cluster": "gwe", "stand": "2025-05", "work": "WIKU Fachbroschüre", "chapter": "Gastgewerbe", "title": "Gastgewerbe aus Sicht der Personalverrechnung", }, # WIKU Personal aktuell 2026 — stand = CreationDate month (see table # docstring); issues 4-5 and 8-9 are combined editions (DOPPELAUSGABE) "WIKU Personal aktuell 2026, Nr. 1.pdf": { "slug": "personal-aktuell-2026-nr-1", "cluster": "akt", "stand": "2026-01", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 1", "title": "WIKU Personal aktuell 2026, Nr. 1", }, "WIKU Personal aktuell 2026, Nr. 2.pdf": { "slug": "personal-aktuell-2026-nr-2", "cluster": "akt", "stand": "2026-02", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 2", "title": "WIKU Personal aktuell 2026, Nr. 2", }, "WIKU Personal aktuell 2026, Nr. 3.pdf": { "slug": "personal-aktuell-2026-nr-3", "cluster": "akt", "stand": "2026-02", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 3", "title": "WIKU Personal aktuell 2026, Nr. 3", }, "WIKU Personal aktuell 2026, Nr. 4-5.pdf": { "slug": "personal-aktuell-2026-nr-4-5", "cluster": "akt", "stand": "2026-03", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 4-5 (Doppelheft)", "title": "WIKU Personal aktuell 2026, Nr. 4-5 (Doppelheft)", }, "WIKU Personal aktuell 2026, Nr. 6.pdf": { "slug": "personal-aktuell-2026-nr-6", "cluster": "akt", "stand": "2026-04", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 6", "title": "WIKU Personal aktuell 2026, Nr. 6", }, "WIKU Personal aktuell 2026, Nr. 7.pdf": { "slug": "personal-aktuell-2026-nr-7", "cluster": "akt", "stand": "2026-04", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 7", "title": "WIKU Personal aktuell 2026, Nr. 7", }, "WIKU Personal aktuell 2026, Nr. 8 - 9.pdf": { "slug": "personal-aktuell-2026-nr-8-9", "cluster": "akt", "stand": "2026-05", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 8-9 (Doppelheft)", "title": "WIKU Personal aktuell 2026, Nr. 8-9 (Doppelheft)", }, "WIKU Personal aktuell 2026, Nr. 10.pdf": { "slug": "personal-aktuell-2026-nr-10", "cluster": "akt", "stand": "2026-06", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 10", "title": "WIKU Personal aktuell 2026, Nr. 10", }, "WIKU Personal aktuell 2026, Nr. 11.pdf": { "slug": "personal-aktuell-2026-nr-11", "cluster": "akt", "stand": "2026-06", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 11", "title": "WIKU Personal aktuell 2026, Nr. 11", }, "WIKU Personal aktuell 2026, Nr. 12.pdf": { "slug": "personal-aktuell-2026-nr-12", "cluster": "akt", "stand": "2026-07", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 12", "title": "WIKU Personal aktuell 2026, Nr. 12", }, "WIKU Personal aktuell 2026, Nr. 13.pdf": { "slug": "personal-aktuell-2026-nr-13", "cluster": "akt", "stand": "2026-08", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 13", "title": "WIKU Personal aktuell 2026, Nr. 13", }, "WIKU Personal aktuell 2026, Nr. 14.pdf": { "slug": "personal-aktuell-2026-nr-14", "cluster": "akt", "stand": "2026-09", "work": "WIKU Personal aktuell", "chapter": "WIKU Personal aktuell — 2026, Nr. 14", "title": "WIKU Personal aktuell 2026, Nr. 14", }, } HEADER_RE = re.compile(r"^\s*Lexis 360®\s*$") MARKER = "Lexis Briefings Personalrecht" # Breadcrumb: "Kapitel > Thema · Autor(en) · ". Dossier-style # exports merge the breadcrumb and the body into ONE line, so trailing # text is allowed and ignored by the (?P...) tail. BC_LINE_RE = re.compile( r"^\s*(?P[^>·]+?)\s*>\s*(?P.+?)\s*·\s*(?P.+?)" r"\s*·\s*(?P[A-ZÄÖÜ][a-zäöüß]+)\s+(?P\d{4})(?P.*)$" ) # Breadcrumb variant WITHOUT an author segment (batch-10 Muster exports: # "Kapitel > Thema · "). The author then defaults to the # editorial attribution in parse_meta(). BC_LINE_NOAUTHOR_RE = re.compile( r"^\s*(?P[^>·]+?)\s*>\s*(?P.+?)\s*·\s*" r"(?P[A-ZÄÖÜ][a-zäöüß]+)\s+(?P\d{4})(?P.*)$" ) STAND_RE = re.compile(r"^\d{4}-\d{2}$") ID_RE = re.compile(r"^(lb|wk)-[a-z]{3}-\d{2}$") # WIKU page-footer artefact: two short numbers separated by wide gaps # (e.g. "09 5"), i.e. page/section counters from the WIKU # typesetting — dropped in Layer 1, everything else kept (full text). WIKU_PAGE_RE = re.compile(r"^\s*\d{1,2}\s{3,}\d{1,2}\s*$") def ascii_slug(text: str) -> str: """ASCII form used for topic matching: diacritics stripped (NFKD, ü->u, ä->a), ß->ss, lowercase, non-alphanumeric -> hyphen. This matches the umlaut-dropped style of the Lexis360 export filenames.""" text = unicodedata.normalize("NFKD", text) text = "".join(c for c in text if not unicodedata.combining(c)) text = text.lower().replace("ß", "ss") text = re.sub(r"[^a-z0-9]+", "-", text).strip("-") return text def parse_pdf(pdf: Path) -> tuple[str, int, str]: """Return (text, pages, pdfinfo title).""" proc = subprocess.run( ["pdftotext", "-layout", str(pdf), "-"], capture_output=True, text=True, check=True, ) pages = 0 title = "" info = subprocess.run( ["pdfinfo", str(pdf)], capture_output=True, text=True, check=True, ) for line in info.stdout.splitlines(): if line.startswith("Pages:"): pages = int(line.split(":", 1)[1].strip()) if line.startswith("Title:"): title = line.split(":", 1)[1].strip() return proc.stdout, pages, title def clean_text(raw: str) -> str: lines = [ln for ln in raw.splitlines() if not HEADER_RE.match(ln)] text = "\n".join(lines).replace("\x0c", "") text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() + "\n" def clean_wiku_text(raw: str) -> str: """WIKU-specific cleanup: drop page-counter footer lines, keep the full text (ad blocks stay in Layer 1 — curation ignores them).""" lines = [ln for ln in raw.splitlines() if not WIKU_PAGE_RE.match(ln)] text = "\n".join(lines).replace("\x0c", "") text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() + "\n" def _match_bc(line: str): """Try the author-carrying breadcrumb regex first, then the no-author variant (batch 10).""" for rx in (BC_LINE_RE, BC_LINE_NOAUTHOR_RE): m = rx.match(line) if m: return m return None def parse_meta(text: str, pdf_title: str = "") -> dict: """Parse title + breadcrumb from the first page. Layout: a line with the work marker, the chapter title, then the breadcrumb "Kapitel > Thema · Autor · " (or the batch-10 no-author variant "Kapitel > Thema · "). Dossier-style exports merge breadcrumb and body into one line; the regex tail (?P) tolerates that. If no separate title line exists, fall back to the pdfinfo Title. """ lines = text.splitlines() marker_idx = None for i, ln in enumerate(lines[:20]): if MARKER in ln: marker_idx = i break if marker_idx is None: raise ValueError("marker 'Lexis Briefings Personalrecht' not found") title_lines: list[str] = [] m = None scan = lines[marker_idx + 1:] for pos, ln in enumerate(scan): cand = _match_bc(ln) if cand is None and ("·" in ln or " > " in ln): # dossier-style exports wrap the breadcrumb ("· Juli\n2026", # "... ·\nAutor · Juli 2026", or the topic split BEFORE the # first "·": "… > Auszahlung Normal-, Mehr- und\nÜberstunden; # Zeitguthaben · Autor · Juli 2026" — batch-8 variant). Retry # with up to two following lines joined in. joined = ln.rstrip() for nxt in scan[pos + 1:pos + 3]: joined = f"{joined} {nxt.strip()}".strip() cand = _match_bc(joined) if cand: break if cand: m = cand break if not ln.strip(): continue title_lines.append(ln.strip()) if len(title_lines) > 4: raise ValueError("breadcrumb not found near page header") if m is None: raise ValueError("breadcrumb not found") title = " ".join(t for t in title_lines if t) or pdf_title if not title: raise ValueError("no title (neither in text nor in pdfinfo)") month_no = MONTHS[m.group("month").lower()] return { "title": title, "chapter": m.group("chapter").strip(), "topic_group": m.group("topic").strip(), # batch-10 no-author variant: editorial default attribution "author": m.groupdict().get("author") or "Lexis Redaktion", "stand": f"{m.group('year')}-{month_no:02d}", "stand_human": f"{m.group('month')} {m.group('year')}", } def resolve_cluster(chapter_ascii: str, topic_ascii: str, slug: str) -> str | None: # 1. chapter-qualified pin (same topic, different chapter -> different # cluster; batch-8 lesson) pair = f"{chapter_ascii}/{topic_ascii}" if pair in CHAPTER_TOPIC_MAP: return CHAPTER_TOPIC_MAP[pair] # 2. plain topic pin if topic_ascii in TOPIC_MAP: return TOPIC_MAP[topic_ascii] # 3. keyword fallback over topic + slug for keyword, cluster in KEYWORDS: if keyword in topic_ascii or keyword in slug: return cluster return None def load_previous_ids(md_dir: Path) -> dict[str, tuple[str, int]]: """slug -> (id, batch) from the source's previous _catalog.json. IDs are frozen once assigned: a later batch must never renumber existing entries, because cross_refs and kb.json consumers rely on stable IDs. New documents continue after the highest existing number in their cluster; numbers of removed documents are retired, never reused. Each source (Lexis `lb-`, WIKU `wk-`) keeps its own catalog and therefore its own frozen ID space. """ path = md_dir / "_catalog.json" if not path.is_file(): return {} try: prev = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError: return {} out = {} for e in prev.get("entries", []): if "slug" in e and "id" in e: out[e["slug"]] = (e["id"], int(e.get("batch", 1))) return out def assign_ids(catalog: list[dict], prev: dict[str, tuple[str, int]], id_prefix: str) -> list[str]: """Freeze prev ids, assign new sequential ids (namespace `id_prefix`); returns warnings for frozen ids whose prefix no longer matches the (re)pinned cluster.""" id_warnings: list[str] = [] counters: dict[str, int] = {} for id_, _batch in prev.values(): m = ID_RE.match(id_) if m and id_[:2] == id_prefix: prefix, num = id_[3:6], int(id_[-2:]) counters[prefix] = max(counters.get(prefix, 0), num) for entry in catalog: if slug_prev := prev.get(entry["slug"]): entry["id"], entry["batch"] = slug_prev if entry["cluster"] and entry["id"][3:6] != entry["cluster"]: id_warnings.append( f"{entry['slug']}: frozen id {entry['id']} no longer matches " f"cluster '{entry['cluster']}' - retire and reassign before " f"curation") for entry in sorted(catalog, key=lambda e: (e["cluster"] or "zzz", e["slug"])): if entry.get("id") or entry["cluster"] is None: continue cluster = entry["cluster"] counters[cluster] = counters.get(cluster, 0) + 1 entry["id"] = f"{id_prefix}-{cluster}-{counters[cluster]:02d}" return id_warnings def write_layer1(entry: dict, text: str, md_dir: Path) -> None: md = md_dir / f"{entry['slug']}.md" fm = [ "---", f"id: {entry['id']}", f"batch: {entry['batch']}", f'title: "{entry["title"]}"', f'work: "{entry["work"]}"', f'chapter: "{entry["chapter"]}"', f'topic_group: "{entry["topic_group"]}"', f"cluster: {entry['cluster']}", f'author: "{entry["author"]}"', f"stand: {entry['stand']}", f'pdf: "{entry["pdf_rel"]}"', f"pages: {entry['pages']}", "---", "", ] md.write_text("\n".join(fm) + text, encoding="utf-8") def extract_lexis() -> int: src = SOURCES["lexis"] if not src["pdf_dir"].is_dir(): sys.exit(f"missing {src['pdf_dir']}") src["md_dir"].mkdir(exist_ok=True) pdfs: list[Path] = [] for sub in (None, *src.get("pdf_subdirs", ())): pdir = src["pdf_dir"] / sub if sub else src["pdf_dir"] if pdir.is_dir(): pdfs.extend(sorted(pdir.glob(src["pdf_glob"]))) catalog: list[dict] = [] texts: dict[str, str] = {} warnings: list[str] = [] seen_slugs: set[str] = set() for pdf in pdfs: slug = pdf.stem[len("Lexis360_"):] if slug in seen_slugs: warnings.append(f"{pdf.name}: slug {slug!r} collides - skipped") continue seen_slugs.add(slug) raw, pages, pdf_title = parse_pdf(pdf) text = clean_text(raw) texts[slug] = text try: meta = parse_meta(text, pdf_title) except ValueError as exc: warnings.append(f"{pdf.name}: {exc}") continue entry = { "slug": slug, "source": "lexis", "pdf_rel": pdf.relative_to(ROOT).as_posix(), "md_rel": f".lexis360/md/{slug}.md", "work": WORK_LEXIS, "pages": pages, "chars": len(text), "batch": BATCH_LEXIS, **meta, } topic_ascii = ascii_slug(entry["topic_group"]) entry["topic_group_ascii"] = topic_ascii chapter_ascii = ascii_slug(entry["chapter"]) entry["cluster"] = resolve_cluster(chapter_ascii, topic_ascii, slug) if entry["cluster"] is None: warnings.append(f"{slug}: no cluster for topic {topic_ascii!r}") catalog.append(entry) id_warnings = assign_ids(catalog, load_previous_ids(src["md_dir"]), "lb") warnings.extend(id_warnings) for entry in catalog: if entry["cluster"] is not None: write_layer1(entry, texts[entry["slug"]], src["md_dir"]) report = { "source": "lexis", "generated_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), "n_pdfs": len(pdfs), "n_entries": len(catalog), "n_unclustered": sum(1 for e in catalog if e["cluster"] is None), "entries": catalog, "warnings": warnings, } (src["md_dir"] / "_catalog.json").write_text( json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") print(f"[lexis] extracted {len(catalog)}/{len(pdfs)} " f"({report['n_unclustered']} unclustered)") topics = sorted({e["topic_group"] for e in catalog}) print(f"distinct breadcrumb topics ({len(topics)}):") for t in topics: print(f" - {t}") for w in warnings: print(f"WARNING: {w}") return 0 if not warnings else 1 def extract_wiku() -> int: """WIKU Layer 1: pin-driven extraction (no breadcrumbs — see WIKU_PINS). A PDF without a pin is a hard warning: it is NOT extracted (never guess metadata for licensed content).""" src = SOURCES["wiku"] if not src["pdf_dir"].is_dir(): sys.exit(f"missing {src['pdf_dir']}") src["md_dir"].mkdir(parents=True, exist_ok=True) pdfs = sorted(src["pdf_dir"].glob(src["pdf_glob"])) catalog: list[dict] = [] texts: dict[str, str] = {} warnings: list[str] = [] for pdf in pdfs: pin = WIKU_PINS.get(pdf.name) if pin is None: warnings.append( f"{pdf.name}: no WIKU_PINS entry (slug/cluster/stand/work/" f"chapter/title) - add the pin before extracting") continue slug = pin["slug"] raw, pages, _pdf_title = parse_pdf(pdf) text = clean_wiku_text(raw) texts[slug] = text entry = { "slug": slug, "source": "wiku", "pdf_rel": f".wiku/{pdf.name}", "md_rel": f".wiku/md/{slug}.md", "work": pin["work"], "pages": pages, "chars": len(text), "batch": BATCH_WIKU, "title": pin["title"], "chapter": pin["chapter"], "topic_group": pin["chapter"], "topic_group_ascii": ascii_slug(pin["chapter"]), "author": pin.get("author", "Wilhelm Kurzböck"), "stand": pin["stand"], "stand_human": pin["stand"], "cluster": pin["cluster"], } catalog.append(entry) id_warnings = assign_ids(catalog, load_previous_ids(src["md_dir"]), "wk") warnings.extend(id_warnings) for entry in catalog: if entry["cluster"] is not None: write_layer1(entry, texts[entry["slug"]], src["md_dir"]) report = { "source": "wiku", "generated_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), "n_pdfs": len(pdfs), "n_entries": len(catalog), "n_unclustered": sum(1 for e in catalog if e["cluster"] is None), "entries": catalog, "warnings": warnings, } (src["md_dir"] / "_catalog.json").write_text( json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") print(f"[wiku] extracted {len(catalog)}/{len(pdfs)} " f"({report['n_unclustered']} unclustered)") works = sorted({e["work"] for e in catalog}) print(f"distinct works ({len(works)}):") for w in works: print(f" - {w}") for w in warnings: print(f"WARNING: {w}") return 0 if not warnings else 1 def extract(source: str) -> int: if source == "lexis": return extract_lexis() if source == "wiku": return extract_wiku() rc = 0 for s in ("lexis", "wiku"): rc |= {"lexis": extract_lexis, "wiku": extract_wiku}[s]() return rc # --- frontmatter (controlled YAML subset) ---------------------------------- def fm_value(raw: str): raw = raw.strip() if raw.startswith('"') and raw.endswith('"'): return raw[1:-1] if raw.startswith("[") and raw.endswith("]"): inner = raw[1:-1].strip() if not inner: return [] return [v.strip().strip('"') for v in inner.split(",")] if re.fullmatch(r"-?\d+", raw): return int(raw) return raw def parse_frontmatter(text: str) -> dict: if not text.startswith("---\n"): raise ValueError("no frontmatter") end = text.index("\n---\n", 4) lines = text[4:end].splitlines() data: dict = {} current = None for ln in lines: if not ln.strip(): continue if ln.startswith(" ") and current: key, _, raw = ln.strip().partition(":") data[current][key] = fm_value(raw) elif ln.startswith("\t"): raise ValueError("tabs not allowed in frontmatter") else: key, _, raw = ln.partition(":") key = key.strip() if raw.strip() == "": data[key] = {} current = key else: data[key] = fm_value(raw) current = None return data REQUIRED = [ "id", "batch", "title", "work", "chapter", "topic", "author", "stand", "legal_bases", "tags", ] def validate(entry: dict, all_ids: set[str], source: str) -> list[str]: errors = [] for key in REQUIRED: if key not in entry: errors.append(f"{source}: missing key {key!r}") src = entry.get("source", {}) for key in ("pdf", "text"): if key not in src: errors.append(f"{source}: missing source.{key}") id_ = str(entry.get("id", "")) if not ID_RE.match(id_): errors.append(f"{source}: bad id {entry.get('id')!r}") id_ = "" if id_: id_space = id_[:2] # "lb" (Lexis) or "wk" (WIKU) work = str(entry.get("work", "")) if id_space == "lb": if work != WORK_LEXIS: errors.append( f"{source}: lb id requires work {WORK_LEXIS!r}, got {work!r}") else: if work not in WIKU_WORKS: errors.append( f"{source}: wk id requires work in " f"{list(WIKU_WORKS)}, got {work!r}") if str(entry.get("stand", "")) and not STAND_RE.match(str(entry["stand"])): errors.append(f"{source}: stand must be YYYY-MM, got {entry['stand']!r}") for ref in entry.get("cross_refs", []): if ref not in all_ids: errors.append(f"{source}: dangling cross_ref {ref!r}") max_batch = BATCH_LEXIS if id_.startswith("lb") else BATCH_WIKU try: batch_no = int(entry["batch"]) except (KeyError, TypeError, ValueError): errors.append(f"{source}: batch must be an integer 1..{max_batch}") else: if not 1 <= batch_no <= max_batch: errors.append(f"{source}: batch must be in 1..{max_batch}, got {batch_no}") if entry.get("topic") not in TOPIC_TO_PREFIX: errors.append( f"{source}: topic must be one of {sorted(TOPIC_TO_PREFIX)}, " f"got {entry.get('topic')!r}") if id_ and id_[3:6] != TOPIC_TO_PREFIX.get(str(entry.get("topic")), ""): errors.append( f"{source}: id prefix does not match topic {entry.get('topic')!r}") return errors def _entry_source(e: dict) -> str: return "wiku" if str(e.get("id", "")).startswith("wk-") else "lexis" def write_index(entries: list[dict], clusters: dict) -> None: """Generate wissensbasis/INDEX.md from curated frontmatter (derived, never hand-edited — regenerate with --registry). One shared corpus, sectioned per source (Lexis first, then WIKU), then per cluster.""" today = datetime.date.today().isoformat() by_source: dict[str, list[dict]] = {} for e in entries: by_source.setdefault(_entry_source(e), []).append(e) n_topics = len({str(e["topic"]) for e in entries}) header = ( f"# Index — Wissensbasis Personalverrechnung\n\n" f"Generiert am {today} · {len(entries)} Einträge · {n_topics} Cluster. " f"Quellen: {WORK_LEXIS} (Lexis 360, lizenzierter Export) und " "WIKU Personal (Fachbroschüren, Arbeitsunterlagen, Casebooks, " "WIKU Personal aktuell). Volltexte lokal/unversioniert " "(`.lexis360/`, `.wiku/`). Werte gelten je zum Quell-Stand " "(`stand` im Frontmatter); Widersprüche und offene Punkte sind " "mit ⚠/❓ in den Einträgen selbst dokumentiert. Schema & " "Konventionen: `README.md`.\n" ) lines = [header] source_labels = { "lexis": f"{WORK_LEXIS} (Lexis 360-Exporte)", "wiku": "WIKU Personal", } for src in ("lexis", "wiku"): src_entries = by_source.get(src, []) if not src_entries: continue lines.append(f"\n# {source_labels[src]} — {len(src_entries)} Einträge\n") by_topic: dict[str, list[dict]] = {} for e in src_entries: by_topic.setdefault(str(e["topic"]), []).append(e) for topic, cluster_entries in sorted(by_topic.items()): stands = sorted(str(e["stand"]) for e in cluster_entries) prefix = TOPIC_TO_PREFIX.get(topic) name = CLUSTERS.get(prefix, topic) lines.append( f"\n## {name} (`topic: {topic}`) — {len(cluster_entries)} " f"Einträge · Stand {stands[0]} bis {stands[-1]}\n") lines.append("| ID | Titel | Autor | Stand | Datei |") lines.append("|---|---|---|---|---|") for e in sorted(cluster_entries, key=lambda x: str(x["id"])): fname = str(e["source"]["text"]).rsplit("/", 1)[-1] lines.append( f"| {e['id']} | {e['title']} | {e['author']} | {e['stand']} " f"| {fname} |") path = CURATED_DIR.parent / "INDEX.md" path.write_text("\n".join(lines) + "\n", encoding="utf-8") print(f"INDEX.md written: {len(entries)} entries") def registry() -> int: CURATED_DIR.mkdir(parents=True, exist_ok=True) docs = sorted(CURATED_DIR.glob("*.md")) if not docs: sys.exit(f"no curated docs in {CURATED_DIR}") entries = [] errors: list[str] = [] for doc in docs: try: fm = parse_frontmatter(doc.read_text(encoding="utf-8")) except ValueError as exc: errors.append(f"{doc.name}: {exc}") continue entries.append(fm) all_ids = {str(e.get("id")) for e in entries if e.get("id")} for fm, doc in zip(entries, docs): errors.extend(validate(fm, all_ids, doc.name)) if errors: for e in errors: print(f"ERROR: {e}") sys.exit(1) entries.sort(key=lambda e: str(e.get("id"))) clusters = {} batches: dict[tuple[str, int], int] = {} works: dict[str, int] = {} for e in entries: clusters[e["topic"]] = clusters.get(e["topic"], 0) + 1 b = int(e.get("batch", 1)) space = str(e.get("id", ""))[:2] batches[(space, b)] = batches.get((space, b), 0) + 1 w = str(e.get("work", "")) works[w] = works.get(w, 0) + 1 out = { "generated_at": datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds"), "n_entries": len(entries), "sources": [ {"work": w, "n": n} for w, n in sorted(works.items()) ], "batches": [ {"source": "lexis" if space == "lb" else "wiku", "batch": b, "n": n} for (space, b), n in sorted(batches.items()) ], "layers": { "curated": "personalverrechnung/wissensbasis/dokumente", "fulltext": ".lexis360/md + .wiku/md (local, unversioned - licensed source texts)", "pdf": ".lexis360 + .wiku (local, unversioned - licensed PDF exports)", }, "schema": { "id": "lb-- (Lexis) | wk-- (WIKU)", "stand": "YYYY-MM (Stand der Quelle)", "legal_bases": "Rechtsgrundlagen as cited in the source", "cross_refs": "ids of related kb entries (lb-* and wk-* may cross-ref)", }, "clusters": [ {"slug": slug, "name": CLUSTERS.get(TOPIC_TO_PREFIX.get(slug, ""), slug), "n": n} for slug, n in sorted(clusters.items()) ], "entries": entries, } KB_JSON.parent.mkdir(parents=True, exist_ok=True) KB_JSON.write_text( json.dumps(out, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") write_index(entries, clusters) print(f"kb.json written: {len(entries)} entries, {len(clusters)} clusters") return 0 def curated_slugs_for(source: str) -> set[str]: """Layer-2 slugs of one source: Lexis = files WITHOUT the `wiku_` prefix (slug = stem), WIKU = files WITH the prefix (slug = stem minus prefix).""" if not CURATED_DIR.is_dir(): return set() slugs = set() for p in CURATED_DIR.glob("*.md"): if source == "lexis" and p.name.startswith(WIKU_SLUG_PREFIX): continue if source == "wiku" and not p.name.startswith(WIKU_SLUG_PREFIX): continue name = p.name # strip "wiku_" prefix AND the .md suffix (slug = bare stem) slugs.add(p.stem[len(WIKU_SLUG_PREFIX):] if name.startswith(WIKU_SLUG_PREFIX) else p.stem) return slugs def check(source: str) -> int: """QA: curated docs 1:1 with the source's extraction catalog, sources exist.""" problems: list[str] = [] src = SOURCES[source] catalog_path = src["md_dir"] / "_catalog.json" if not catalog_path.is_file(): sys.exit(f"run --extract --source {source} first " f"(missing {catalog_path})") catalog = json.loads(catalog_path.read_text(encoding="utf-8")) cat_slugs = {e["slug"]: e for e in catalog["entries"]} cur_slugs = curated_slugs_for(source) for slug in sorted(set(cat_slugs) - cur_slugs): problems.append(f"not curated: {slug}") for slug in sorted(cur_slugs - set(cat_slugs)): problems.append(f"curated but no source: {slug}") for slug in sorted(set(cat_slugs) & cur_slugs): text_path = ROOT / cat_slugs[slug]["md_rel"] if not text_path.is_file(): problems.append(f"layer-1 text missing: {text_path}") pdf_path = ROOT / cat_slugs[slug]["pdf_rel"] if not pdf_path.is_file(): problems.append(f"source pdf missing: {pdf_path}") for p in problems: print(f"ERROR: {p}") print(f"[{source}] check: {len(cat_slugs)} sources, {len(cur_slugs)} " f"curated, {len(problems)} problems") return 1 if problems else 0 def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--extract", action="store_true", help="extract PDFs into Layer-1 md") group.add_argument("--registry", action="store_true", help="generate kb.json from curated frontmatter " "(shared corpus)") group.add_argument("--check", action="store_true", help="QA: curated docs vs extraction catalog") parser.add_argument("--source", choices=("lexis", "wiku", "all"), default="lexis", help="source for --extract/--check " "(default: lexis; 'all' = both)") args = parser.parse_args() if args.source == "all": sources = ("lexis", "wiku") else: sources = (args.source,) if args.extract: rc = 0 for s in sources: rc |= extract(s) return rc if args.registry: return registry() rc = 0 for s in sources: rc |= check(s) return rc if __name__ == "__main__": sys.exit(main())