mirror of
http://100.103.83.12:3003/fegger/odoo-at-payroll.git
synced 2026-09-17 16:56:42 +00:00
[ADD] personalverrechnung: machine-readable KV library (all current collective agreements)
Cross-checked catalogue of all currently valid Austrian collective agreements, per the source policy: main source = the responsible employer-side chamber, kollektivvertrag.at (OEGB-Verlag) as complete cross-check baseline. - oegb/: 593 variants from kollektivvertrag.at with full consolidated texts (JSON primary format, Markdown for review/diff) plus catalog.json/.csv; fetched via the portal's JSON servlets (dashboard/structure/slices/topics), resumable - wko/: 614 current documents from the WKO KV database (1,820 branch overviews across 10 regions) with texts and catalog; match-report against the baseline: 407 pairs, 30 date disagreements flagged - chambers/chambers.json: curated registry of chamber-side sources and publication gaps (GOeD member-only, RA/Notariat not public, OCR-less scan PDFs, missing hospital-doctors KV) - kv-catalog.json/.csv: merged catalogue, 768 entries, chamber classification, main_source, cross_check status, manual_review lists - tools/fetch_kv_portal.py / fetch_wko_kv.py / build_kv_catalog.py: stdlib-only fetchers and merge step, each with a --refresh diff mode - RUNBOOK.md: update cycle for keeping the library current (annual, tied to the Wartung rhythm and KV rounds) Lohntafel PDFs are linked but not downloaded; wage tables contained in the KV texts are converted. No Odoo module changes.
This commit is contained in:
@@ -0,0 +1,511 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch the WKO Kollektivvertrags-Datenbank (wko.at) as machine-readable data.
|
||||
|
||||
The Wirtschaftskammer Österreich publishes the Kollektivverträge of its
|
||||
own sectors (Gewerbe/Handwerk, Industrie, Handel, ...) at
|
||||
https://www.wko.at/oe/kollektivvertraege. Per the project's source
|
||||
policy the WKO database is the MAIN source for WKO-negotiated KVs; the
|
||||
ÖGB portal (fetch_kv_portal.py) is the cross-check baseline.
|
||||
|
||||
Site semantics (verified 2026-09-09):
|
||||
- Per region there is a collectiveAgreementsOverview sitemap listing the
|
||||
branch pages /{region}/kollektivvertraege/{branch-slug}.
|
||||
- Each branch page lists the KV documents as <li data-gtx-years="..."
|
||||
data-gtx-archive="true|false"> entries; the Zeitraum filter on the
|
||||
site treats archive=false as "aktuell".
|
||||
- Detail pages live under /kollektivvertrag/{slug} (region-neutral) and
|
||||
carry the full text, the räumlicher/fachlicher Geltungsbereich, and
|
||||
links to Lohntafel-PDFs (linked, never downloaded - the wage tables
|
||||
that are part of the KV text are converted with the text).
|
||||
- The collectiveAgreementsDV sitemap provides a lastmod stamp per
|
||||
detail page, which drives the --refresh diff.
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
python3 personalverrechnung/tools/fetch_wko_kv.py # full fetch
|
||||
python3 personalverrechnung/tools/fetch_wko_kv.py --limit 3 # smoke test
|
||||
python3 personalverrechnung/tools/fetch_wko_kv.py --refresh
|
||||
|
||||
Outputs (curated, versioned):
|
||||
personalverrechnung/quellen/kv/wko/catalog.json
|
||||
personalverrechnung/quellen/kv/wko/texts/{slug}.json|.md
|
||||
personalverrechnung/quellen/kv/wko/match-report.json
|
||||
|
||||
Raw mirror (unversioned): .firecrawl/wko-kv/
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures as cf
|
||||
import csv
|
||||
import datetime as dt
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from fetch_kv_portal import FetchError, html_to_md, log # noqa: E402
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
RAW = REPO / ".firecrawl" / "wko-kv"
|
||||
OUT = REPO / "personalverrechnung" / "quellen" / "kv" / "wko"
|
||||
TEXTS = OUT / "texts"
|
||||
|
||||
BASE = "https://www.wko.at"
|
||||
REGIONS = ["oe", "bgld", "ktn", "noe", "ooe", "sbg", "stmk", "tirol",
|
||||
"vlbg", "wien"]
|
||||
REGION_NAMES = {"oe": "Österreich", "bgld": "Burgenland", "ktn": "Kärnten",
|
||||
"noe": "Niederösterreich", "ooe": "Oberösterreich",
|
||||
"sbg": "Salzburg", "stmk": "Steiermark", "tirol": "Tirol",
|
||||
"vlbg": "Vorarlberg", "wien": "Wien"}
|
||||
OVERVIEW_SITEMAP = BASE + "/sitemap/{region}-collectiveAgreementsOverview-sitemap-0.xml"
|
||||
DV_SITEMAP = BASE + "/sitemap/oe-collectiveAgreementsDV-sitemap-0.xml"
|
||||
DETAIL = BASE + "/kollektivvertrag/{slug}"
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "odoo-at-payroll-kv-library/1.0 (legal-source mirror; "
|
||||
"contact: repo maintainer)",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.5",
|
||||
"Accept-Language": "de-AT,de;q=0.9",
|
||||
"Accept-Encoding": "identity",
|
||||
}
|
||||
|
||||
WORKERS = 4
|
||||
RETRIES = 3
|
||||
TIMEOUT = 40
|
||||
|
||||
ENTRY_RE = re.compile(
|
||||
r'<li data-gtx-years="([^"]*)" data-gtx-archive="([^"]*)">'
|
||||
r'<a href="([^"]+)"[^>]*>(.*?)</a></li>', re.S)
|
||||
|
||||
|
||||
# ------------------------------------------------------------- transport
|
||||
|
||||
|
||||
def fetch(url: str, cache: Path | None = None, force: bool = False) -> str:
|
||||
if cache is not None and cache.exists() and not force:
|
||||
return cache.read_text(encoding="utf-8")
|
||||
last_err: Exception | None = None
|
||||
for attempt in range(RETRIES):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
|
||||
data = resp.read().decode("utf-8", "replace")
|
||||
if cache is not None:
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache.write_text(data, encoding="utf-8")
|
||||
return data
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_err = exc
|
||||
time.sleep(2 * (attempt + 1))
|
||||
raise FetchError(f"{url}: {last_err}")
|
||||
|
||||
|
||||
def fetch_json_cached(url: str, cache: Path, force: bool = False) -> dict:
|
||||
text = fetch(url, cache=cache, force=force)
|
||||
if text.lstrip().startswith("<!DOCTYPE"):
|
||||
raise FetchError(f"{url}: HTML error page instead of JSON")
|
||||
return json.loads(text)
|
||||
|
||||
|
||||
# -------------------------------------------------------------- overviews
|
||||
|
||||
|
||||
def overview_urls(region: str, force: bool = False) -> list[str]:
|
||||
cache = RAW / "sitemaps" / f"{region}-overview.xml"
|
||||
text = fetch(OVERVIEW_SITEMAP.format(region=region), cache=cache,
|
||||
force=force)
|
||||
urls = []
|
||||
for loc in re.findall(r"<loc>([^<]+)</loc>", text):
|
||||
url = html.unescape(loc).strip()
|
||||
# skip the region landing page itself (no branch selected)
|
||||
if url.rstrip("/").endswith("/kollektivvertraege"):
|
||||
continue
|
||||
urls.append(url)
|
||||
return urls
|
||||
|
||||
|
||||
def parse_overview(page: str) -> dict:
|
||||
"""Extract branch title and document entries from a branch page."""
|
||||
title = ""
|
||||
m = re.search(r"<title>(.*?)(?: - WKO)?</title>", page, re.S)
|
||||
if m:
|
||||
title = re.sub(r"\s+", " ", html.unescape(m.group(1))).strip()
|
||||
title = re.sub(r"^Kollektivverträge für\s*", "", title)
|
||||
entries = []
|
||||
for years, archive, href, inner in ENTRY_RE.findall(page):
|
||||
text = re.sub(r"<[^>]+>", " ", inner)
|
||||
text = re.sub(r"\s+", " ", html.unescape(text)).strip()
|
||||
slug = href.rstrip("/").rsplit("/kollektivvertrag/", 1)[-1]
|
||||
entries.append({
|
||||
"slug": slug,
|
||||
"href": href if href.startswith("http") else BASE + href,
|
||||
"title": text,
|
||||
"years": [y.strip() for y in years.split(",") if y.strip()],
|
||||
"archived": archive == "true",
|
||||
})
|
||||
return {"branch_title": title, "entries": entries}
|
||||
|
||||
|
||||
def fetch_overview(url: str, region: str, force: bool = False) -> dict:
|
||||
slug = url.rstrip("/").rsplit("/", 1)[-1]
|
||||
cache = RAW / "overviews" / region / f"{slug}.html"
|
||||
page = fetch(url, cache=cache, force=force)
|
||||
data = parse_overview(page)
|
||||
data["url"] = url
|
||||
data["region"] = region
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- details
|
||||
|
||||
|
||||
def dv_lastmods(force: bool = False) -> dict[str, str]:
|
||||
"""Detail-page lastmod stamps from the DV sitemap (region-neutral)."""
|
||||
cache = RAW / "sitemaps" / "oe-dv.xml"
|
||||
text = fetch(DV_SITEMAP, cache=cache, force=force)
|
||||
out = {}
|
||||
for block in re.findall(r"<url>.*?</url>", text, re.S):
|
||||
loc = re.search(r"<loc>([^<]+)</loc>", block)
|
||||
mod = re.search(r"<lastmod>([^<]+)</lastmod>", block)
|
||||
if loc and mod:
|
||||
slug = html.unescape(loc.group(1)).rstrip("/").rsplit(
|
||||
"/kollektivvertrag/", 1)[-1]
|
||||
out[slug] = mod.group(1)[:10]
|
||||
return out
|
||||
|
||||
|
||||
def parse_detail(page: str) -> dict:
|
||||
title = ""
|
||||
m = re.search(r"<h1[^>]*class=\"[^\"]*article-head-title[^\"]*\"[^>]*>"
|
||||
r"(.*?)</h1>", page, re.S)
|
||||
if m:
|
||||
title = re.sub(r"\s+", " ", html.unescape(
|
||||
re.sub(r"<[^>]+>", "", m.group(1)))).strip()
|
||||
if not title:
|
||||
m = re.search(r"<title>(.*?)(?: - WKO)?</title>", page, re.S)
|
||||
if m:
|
||||
title = re.sub(r"\s+", " ", html.unescape(m.group(1))).strip()
|
||||
|
||||
content_html = ""
|
||||
m = re.search(r'data-gtm-section="Content"(.*?)</main>', page, re.S)
|
||||
if m:
|
||||
content_html = m.group(1)
|
||||
scope = {}
|
||||
for key in ("Räumlicher Geltungsbereich", "Fachlicher Geltungsbereich"):
|
||||
sm = re.search(key + r"[^:<]*:\s*([^<\n]{2,200})", content_html)
|
||||
if sm:
|
||||
scope[key.split(" ")[0].lower()] = html.unescape(
|
||||
sm.group(1)).strip()
|
||||
archived_banner = "ARCHIVIERT" in content_html or "archiviert" in title
|
||||
pdf_links = []
|
||||
for href in re.findall(r'href="([^"]+\.pdf[^"]*)"', content_html,
|
||||
re.I):
|
||||
url = href if href.startswith("http") else BASE + href
|
||||
if url not in pdf_links:
|
||||
pdf_links.append(url)
|
||||
external: dict[str, str] = {}
|
||||
for href, label in re.findall(
|
||||
r'<a[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>',
|
||||
content_html, re.S):
|
||||
host = re.sub(r"^https?://([^/]+).*", r"\1", href)
|
||||
if "wko.at" in host:
|
||||
continue
|
||||
label = re.sub(r"\s+", " ", html.unescape(
|
||||
re.sub(r"<[^>]+>", "", label))).strip()
|
||||
external.setdefault(host, label)
|
||||
external_links = [f"{label} | {host}" for host, label in
|
||||
external.items()]
|
||||
return {
|
||||
"title": title,
|
||||
"scope": scope,
|
||||
"archived_banner": archived_banner,
|
||||
"pdf_links": pdf_links,
|
||||
"external_links": external_links,
|
||||
"content_html": content_html,
|
||||
}
|
||||
|
||||
|
||||
def fetch_detail(slug: str, force: bool = False) -> dict:
|
||||
cache = RAW / "docs" / f"{slug}.html"
|
||||
page = fetch(DETAIL.format(slug=slug), cache=cache, force=force)
|
||||
data = parse_detail(page)
|
||||
data["slug"] = slug
|
||||
data["url"] = DETAIL.format(slug=slug)
|
||||
return data
|
||||
|
||||
|
||||
# --------------------------------------------------------------- pipeline
|
||||
|
||||
|
||||
def collect(force: bool, limit: int | None, refresh: bool) -> None:
|
||||
# refresh runs always re-check the overviews/sitemaps (archive status
|
||||
# and listings must be fresh); a plain run honours --force only.
|
||||
listing_force = force or refresh
|
||||
overviews: list[dict] = []
|
||||
failures: list[str] = []
|
||||
jobs = []
|
||||
for region in REGIONS:
|
||||
for url in overview_urls(region, force=listing_force):
|
||||
jobs.append((region, url))
|
||||
if limit:
|
||||
jobs = jobs[:limit]
|
||||
log(f"overview pages to fetch: {len(jobs)}")
|
||||
with cf.ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
futs = {pool.submit(fetch_overview, url, region, listing_force):
|
||||
(region, url) for region, url in jobs}
|
||||
done = 0
|
||||
for fut in cf.as_completed(futs):
|
||||
region, url = futs[fut]
|
||||
try:
|
||||
overviews.append(fut.result())
|
||||
done += 1
|
||||
if done % 100 == 0:
|
||||
log(f" overviews done: {done}/{len(jobs)}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
failures.append(f"overview {region} {url}: {exc}")
|
||||
log(f"overviews fetched: {len(overviews)} (failed: {len(failures)})")
|
||||
|
||||
# ---- aggregate documents across regions
|
||||
docs: dict[str, dict] = {}
|
||||
for ov in overviews:
|
||||
for e in ov["entries"]:
|
||||
slug = e["slug"]
|
||||
d = docs.setdefault(slug, {
|
||||
"slug": slug, "url": e["href"], "title": e["title"],
|
||||
"regions": {}, "branch_titles": set(),
|
||||
})
|
||||
d["regions"][ov["region"]] = {
|
||||
"archived": e["archived"], "years": e["years"],
|
||||
}
|
||||
if ov["branch_title"]:
|
||||
d["branch_titles"].add(ov["branch_title"])
|
||||
if e["title"]:
|
||||
d["title"] = e["title"]
|
||||
for d in docs.values():
|
||||
d["branch_titles"] = sorted(d["branch_titles"])
|
||||
d["current_regions"] = sorted(
|
||||
r for r, v in d["regions"].items() if not v["archived"])
|
||||
d["is_current"] = bool(d["current_regions"])
|
||||
lastmods = dv_lastmods(force=listing_force)
|
||||
for d in docs.values():
|
||||
d["wko_lastmod"] = lastmods.get(d["slug"])
|
||||
log(f"unique documents listed: {len(docs)} "
|
||||
f"(current somewhere: {sum(d['is_current'] for d in docs.values())})")
|
||||
|
||||
# ---- phase 2: fetch current detail pages
|
||||
if refresh:
|
||||
prev = _previous_catalog()
|
||||
todo = [d["slug"] for d in docs.values() if d["is_current"] and (
|
||||
d["slug"] not in prev
|
||||
or prev[d["slug"]].get("wko_lastmod") != d["wko_lastmod"])]
|
||||
log(f"refresh: {len(todo)} changed/new detail pages")
|
||||
else:
|
||||
todo = [d["slug"] for d in docs.values() if d["is_current"]]
|
||||
if limit:
|
||||
todo = todo[:limit]
|
||||
details: dict[str, dict] = {}
|
||||
with cf.ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
futs = {pool.submit(fetch_detail, slug, force): slug for slug in todo}
|
||||
done = 0
|
||||
for fut in cf.as_completed(futs):
|
||||
slug = futs[fut]
|
||||
try:
|
||||
details[slug] = fut.result()
|
||||
done += 1
|
||||
if done % 50 == 0:
|
||||
log(f" details done: {done}/{len(todo)}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
failures.append(f"detail {slug}: {exc}")
|
||||
log(f"details fetched: {len(details)} (failed: {len(failures)})")
|
||||
|
||||
# ---- write curated output
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
TEXTS.mkdir(parents=True, exist_ok=True)
|
||||
rows = []
|
||||
for d in sorted(docs.values(), key=lambda x: x["slug"]):
|
||||
det = details.get(d["slug"])
|
||||
md = ""
|
||||
if det:
|
||||
content_html = det.pop("content_html", "")
|
||||
md = html_to_md(content_html)
|
||||
(TEXTS / f"{d['slug']}.json").write_text(
|
||||
json.dumps(det, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
(TEXTS / f"{d['slug']}.md").write_text(md, encoding="utf-8")
|
||||
rows.append({
|
||||
"slug": d["slug"],
|
||||
"url": d["url"],
|
||||
"title": d["title"],
|
||||
"branch_titles": d["branch_titles"],
|
||||
"is_current": d["is_current"],
|
||||
"current_regions": d["current_regions"],
|
||||
"regions": d["regions"],
|
||||
"wko_lastmod": d["wko_lastmod"],
|
||||
"scope": det["scope"] if det else None,
|
||||
"archived_banner": det["archived_banner"] if det else None,
|
||||
"pdf_links": det["pdf_links"] if det else None,
|
||||
"external_links": det["external_links"] if det else None,
|
||||
"has_text": bool(det and md.strip()),
|
||||
})
|
||||
catalog = {
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc)
|
||||
.isoformat(timespec="seconds"),
|
||||
"source": BASE + "/oe/kollektivvertraege",
|
||||
"n_docs": len(rows),
|
||||
"n_current": sum(r["is_current"] for r in rows),
|
||||
"docs": rows,
|
||||
}
|
||||
(OUT / "catalog.json").write_text(
|
||||
json.dumps(catalog, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
_write_csv(rows)
|
||||
(RAW / "failures.log").write_text("\n".join(failures) + "\n",
|
||||
encoding="utf-8")
|
||||
log(f"done: {len(rows)} docs in catalog, {len(details)} texts, "
|
||||
f"{len(failures)} failures")
|
||||
|
||||
# ---- phase 3: match against the ÖGB baseline catalog
|
||||
match_report = _match_oegb(rows)
|
||||
(OUT / "match-report.json").write_text(
|
||||
json.dumps(match_report, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
log(f"match report: {match_report['n_matched']} matched, "
|
||||
f"{match_report['n_low_confidence']} low-confidence, "
|
||||
f"{match_report['n_unmatched_wko_current']} current WKO docs "
|
||||
"without ÖGB match")
|
||||
|
||||
|
||||
def _write_csv(rows: list[dict]) -> None:
|
||||
fields = ["slug", "title", "is_current", "current_regions",
|
||||
"wko_lastmod", "has_text", "url"]
|
||||
with (OUT / "catalog.csv").open("w", newline="", encoding="utf-8") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
for r in rows:
|
||||
r2 = dict(r)
|
||||
r2["current_regions"] = ";".join(r2["current_regions"])
|
||||
writer.writerow(r2)
|
||||
|
||||
|
||||
def _previous_catalog() -> dict[str, dict]:
|
||||
path = OUT / "catalog.json"
|
||||
if not path.exists():
|
||||
return {}
|
||||
return {d["slug"]: d for d in
|
||||
json.loads(path.read_text(encoding="utf-8"))["docs"]}
|
||||
|
||||
|
||||
STOPWORDS = {
|
||||
"kollektivvertrag", "rahmenkollektivvertrag", "zusatzkollektivvertrag",
|
||||
"rahmen", "fuer", "die", "der", "das", "und", "sowie", "im",
|
||||
"in", "des", "dem", "den", "gueltig", "ab", "kv", "lohn",
|
||||
"gehaltsordnung", "lohnordnung", "gehalt", "lohntafel", "ordnung",
|
||||
"tafel", "information", "ergebnis", "abschluss",
|
||||
}
|
||||
|
||||
|
||||
def _tokens(title: str) -> set[str]:
|
||||
t = html.unescape(title or "").lower()
|
||||
t = (t.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue")
|
||||
.replace("ß", "ss"))
|
||||
words = re.findall(r"[a-z0-9]+", t)
|
||||
return {w for w in words if w not in STOPWORDS and len(w) > 1}
|
||||
|
||||
|
||||
def _fuzzy_overlap(a: set[str], b: set[str]) -> int:
|
||||
"""Count token pairs where one is a prefix of the other (min len 4),
|
||||
so German morphology (Florist/Floristen, Gutsbetrieb/-betrieben)
|
||||
does not defeat matching."""
|
||||
used: set[str] = set()
|
||||
n = 0
|
||||
for x in a:
|
||||
for y in b:
|
||||
if y in used:
|
||||
continue
|
||||
if (len(x) >= 4 and len(y) >= 4
|
||||
and (x.startswith(y) or y.startswith(x))):
|
||||
used.add(y)
|
||||
n += 1
|
||||
break
|
||||
return n
|
||||
|
||||
|
||||
def _match_oegb(wko_rows: list[dict]) -> dict:
|
||||
oegb_path = (REPO / "personalverrechnung" / "quellen" / "kv" / "oegb"
|
||||
/ "catalog.json")
|
||||
if not oegb_path.exists():
|
||||
log("no ÖGB baseline catalog yet - skipping match step "
|
||||
"(run fetch_kv_portal.py first)")
|
||||
return {"n_matched": 0, "n_low_confidence": 0,
|
||||
"n_unmatched_wko_current": 0, "matches": [],
|
||||
"note": "oegb catalog missing"}
|
||||
oegb = json.loads(oegb_path.read_text(encoding="utf-8"))["variants"]
|
||||
index = [(v, _tokens(v["title"])) for v in oegb]
|
||||
|
||||
matches = []
|
||||
matched_oegb: set[str] = set()
|
||||
low, unmatched = [], []
|
||||
for row in wko_rows:
|
||||
if not row["is_current"]:
|
||||
continue
|
||||
tw = _tokens(row["title"])
|
||||
if not tw:
|
||||
unmatched.append({"slug": row["slug"], "title": row["title"],
|
||||
"reason": "no title tokens"})
|
||||
continue
|
||||
best, best_score, best_inter, best_vid = None, 0.0, 0, None
|
||||
for v, tv in index:
|
||||
if not tv:
|
||||
continue
|
||||
inter = _fuzzy_overlap(tw, tv)
|
||||
score = inter / min(len(tw), len(tv))
|
||||
if score > best_score:
|
||||
best, best_score, best_inter, best_vid = v, score, inter, \
|
||||
v["variant_id"]
|
||||
# matched: strong multi-token overlap, or full containment of a
|
||||
# very short title (e.g. 'Floristen' in 'Floristen und
|
||||
# Blumeneinzelhaendler')
|
||||
if (best_score >= 0.75 and best_inter >= 2) or best_score >= 0.999:
|
||||
matches.append({"wko_slug": row["slug"], "wko_title": row["title"],
|
||||
"oegb_variant_id": best_vid,
|
||||
"oegb_title": best["title"], "score": best_score})
|
||||
matched_oegb.add(best_vid)
|
||||
elif best_score >= 0.5 and best_inter >= 2:
|
||||
low.append({"wko_slug": row["slug"], "wko_title": row["title"],
|
||||
"best_oegb_variant_id": best_vid,
|
||||
"best_oegb_title": best["title"] if best else None,
|
||||
"score": best_score})
|
||||
else:
|
||||
unmatched.append({"slug": row["slug"], "title": row["title"],
|
||||
"reason": f"best score {best_score:.2f}"})
|
||||
oegb_unmatched = [v["variant_id"] for v, _ in index
|
||||
if v["variant_id"] not in matched_oegb]
|
||||
return {
|
||||
"n_matched": len(matches),
|
||||
"n_low_confidence": len(low),
|
||||
"n_unmatched_wko_current": len(unmatched),
|
||||
"n_oegb_without_wko_match": len(oegb_unmatched),
|
||||
"matches": matches,
|
||||
"low_confidence": low,
|
||||
"unmatched_wko_current": unmatched,
|
||||
"oegb_without_wko_match": oegb_unmatched,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--force", action="store_true")
|
||||
ap.add_argument("--limit", type=int, default=None)
|
||||
ap.add_argument("--refresh", action="store_true",
|
||||
help="only refetch detail pages whose wko lastmod "
|
||||
"changed; overviews are always re-checked")
|
||||
args = ap.parse_args()
|
||||
collect(force=args.force, limit=args.limit, refresh=args.refresh)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user