Stufe 2: Antworttyp-Routing — Survey Map-Reduce + Rueckfrage-Regel (D13, M6)
- Planer liefert jetzt type: survey|specific. Survey-Fragen (Uebersicht ueber viele Dokumente) laufen ueber Map-Reduce: Retrieval auf survey_blocks=16 erweitert, ein Map-Call destilliert JE Block als Stichpunkte mit seiner KB-ID (MAP_SYSTEM_PROMPT), ein Reduce-Call synthetisiert die Endantwort. Grounding unveraendert: Zitier-Validierung strikt ueber die Retrieved-Union; leerer Map-Output -> Fallback auf Einzelantwort. - Systemprompt-Regel 9: haengt die Antwort wesentlich von nicht genanntem Kontext ab (Branche, Bundesland, Zeitraum), belegte allgemeine Aussage plus EINE Rueckfrage statt Verweigerung (API-first; Odoo-Chat kann die Rueckfrage als Follow-up nutzen). - Ergebnis: q-029 (WIKU-Survey, bisher hartnaeckigste Fehlverweigerung) geheilt - Teilantwort mit 5 belegten Heften; q-015 antwortet mit expliziter KV-Abhaengigkeit + Rueckfrage statt Branchen-Noise. - Eval (46 Fragen): Zitier-Praezision 97,8 %, Verweigerung 97,8 % (Gate erfuellt), erwartete Quelle 90,2 %, Latenz mean 34,2 s (Map-Reduce nur bei Survey-Fragen, ~95 s). Report lokal data/eval-qwen38-stage2.json. - Tests 57 -> 59 (Survey-Integration, Typ-Parsing).
This commit is contained in:
+57
-16
@@ -27,40 +27,51 @@ def test_should_plan_gate():
|
||||
|
||||
|
||||
def test_parse_plan_valid_and_fallback():
|
||||
subs = parse_plan(
|
||||
'Vorab: {"queries": [{"text": "mindestlohn friseur", "stand_year": "2024"}, '
|
||||
subs, qtype = parse_plan(
|
||||
'Vorab: {"type": "specific", "queries": [{"text": "mindestlohn friseur", "stand_year": "2024"}, '
|
||||
'{"text": "lohnberechnung friseur", "stand_year": null}]}',
|
||||
original="Originalfrage?",
|
||||
)
|
||||
assert [s.text for s in subs] == ["mindestlohn friseur", "lohnberechnung friseur"]
|
||||
assert subs[0].stand_year == "2024"
|
||||
assert subs[1].stand_year is None
|
||||
assert qtype == "specific"
|
||||
|
||||
# Fallbacks: kaputtes JSON, leeres Array, leere Texte
|
||||
for raw in ("kein json", '{"queries": []}', '{"queries": [{"text": ""}]}'):
|
||||
subs = parse_plan(raw, original="Originalfrage?")
|
||||
subs, qtype = parse_plan(raw, original="Originalfrage?")
|
||||
assert len(subs) == 1 and subs[0].text == "Originalfrage?"
|
||||
assert qtype == "specific"
|
||||
|
||||
# Ungueltiges Jahr -> None erzwingen, Text bleibt
|
||||
subs = parse_plan(
|
||||
'{"queries": [{"text": "x", "stand_year": "98"}]}', original="orig"
|
||||
# Ungueltiges Jahr -> None erzwingen, Text bleibt; unbekannter scope -> None
|
||||
subs, _ = parse_plan(
|
||||
'{"queries": [{"text": "x", "stand_year": "98", "scope": "xyz"}]}', original="orig"
|
||||
)
|
||||
assert subs == [SubQuery(text="x", stand_year=None)]
|
||||
assert subs == [SubQuery(text="x", stand_year=None, scope=None)]
|
||||
|
||||
|
||||
def test_parse_plan_type_survey():
|
||||
subs, qtype = parse_plan(
|
||||
'{"type": "survey", "queries": [{"text": "wiku personal aktuell 2026 neuerungen", "scope": null}]}',
|
||||
original="orig",
|
||||
)
|
||||
assert qtype == "survey"
|
||||
assert subs[0].scope is None
|
||||
|
||||
|
||||
def test_parse_plan_caps_at_three_queries():
|
||||
raw = json.dumps(
|
||||
{"queries": [{"text": f"q{i}"} for i in range(5)]}
|
||||
)
|
||||
subs = parse_plan(raw, original="orig")
|
||||
subs, _ = parse_plan(raw, original="orig")
|
||||
assert len(subs) == 3
|
||||
|
||||
|
||||
def test_plan_queries_simple_question_no_llm_call():
|
||||
client = FakeOllama(answers=[]) # darf nicht aufgerufen werden
|
||||
cfg = Config(planner_enabled=True)
|
||||
subs, planned = plan_queries("Was ist Altersteilzeit?", client, cfg)
|
||||
assert planned is False
|
||||
subs, planned, qtype = plan_queries("Was ist Altersteilzeit?", client, cfg)
|
||||
assert planned is False and qtype == "specific"
|
||||
assert subs == [SubQuery(text="Was ist Altersteilzeit?")]
|
||||
assert client.calls == 0
|
||||
|
||||
@@ -68,19 +79,19 @@ def test_plan_queries_simple_question_no_llm_call():
|
||||
def test_plan_queries_uses_planner_and_falls_back_on_error():
|
||||
cfg = Config(planner_enabled=True)
|
||||
client = FakeOllama(
|
||||
answers=['{"queries": [{"text": "atz lohnausgleich"}, {"text": "atz altersteilzeitgeld", "stand_year": null}]}']
|
||||
answers=['{"type": "specific", "queries": [{"text": "atz lohnausgleich"}, {"text": "atz altersteilzeitgeld", "stand_year": null}]}']
|
||||
)
|
||||
subs, planned = plan_queries(
|
||||
subs, planned, qtype = plan_queries(
|
||||
"Wie funktioniert der Lohnausgleich bei Altersteilzeit und was ersetzt das AMS?", client, cfg
|
||||
)
|
||||
assert planned is True and len(subs) == 2
|
||||
assert planned is True and qtype == "specific" and len(subs) == 2
|
||||
assert subs[0].text == "atz lohnausgleich"
|
||||
|
||||
# Fehler -> Originalfrage, geplant False
|
||||
failing = FakeOllama(answers=[])
|
||||
failing.chat = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("offline"))
|
||||
subs, planned = plan_queries("Wie funktioniert der Lohnausgleich 2026?", failing, cfg)
|
||||
assert planned is False
|
||||
subs, planned, qtype = plan_queries("Wie funktioniert der Lohnausgleich 2026?", failing, cfg)
|
||||
assert planned is False and qtype == "specific"
|
||||
assert subs[0].text.startswith("Wie funktioniert")
|
||||
|
||||
|
||||
@@ -95,7 +106,7 @@ def test_answer_question_planner_integration(mini_index):
|
||||
planner_max_queries=2,
|
||||
)
|
||||
client = FakeOllama(answers=[
|
||||
json.dumps({"queries": [
|
||||
json.dumps({"type": "specific", "queries": [
|
||||
{"text": "altersteilzeit lohnausgleich", "stand_year": None},
|
||||
{"text": "urlaubsanspruch", "stand_year": None},
|
||||
]}),
|
||||
@@ -112,6 +123,36 @@ def test_answer_question_planner_integration(mini_index):
|
||||
assert len(result["planned_queries"]) == 2
|
||||
|
||||
|
||||
def test_answer_question_survey_map_reduce(mini_index):
|
||||
"""Survey-Frage: 1. Planer (type=survey), 2. Map-Destillat,
|
||||
3. Reduce-Antwort. Zitier-Validierung weiterhin gegen die Union."""
|
||||
cfg = Config(
|
||||
kb_dir=mini_index.kb_dir,
|
||||
db_path=mini_index.db_path,
|
||||
embed_off=True,
|
||||
planner_enabled=True,
|
||||
survey_blocks=8,
|
||||
)
|
||||
client = FakeOllama(answers=[
|
||||
json.dumps({"type": "survey", "queries": [
|
||||
{"text": "wiku personal aktuell 2026", "stand_year": None},
|
||||
]}),
|
||||
# Map-Ausgabe: Destillat je Block mit KB-ID
|
||||
"[lb-min-01] Altersteilzeit: Lohnausgleich + ATZ-Geld.\n"
|
||||
"[lb-min-02] Urlaub: 5 Wochen je Dienstjahr.",
|
||||
# Reduce-Antwort
|
||||
"Neuerungen: ATZ-Lohnausgleich [lb-min-01]; Urlaub 5 Wochen [lb-min-02].",
|
||||
])
|
||||
result = answer_question(
|
||||
"Welche Neuerungen behandeln die Wissensbasis 2026?",
|
||||
cfg, client=client,
|
||||
)
|
||||
assert client.calls == 3 # Planer + Map + Reduce
|
||||
assert result["verified"] is True
|
||||
assert result["refused"] is False
|
||||
assert set(result["citations"]) == {"lb-min-01", "lb-min-02"}
|
||||
|
||||
|
||||
def test_search_multi_fuses_across_queries(mini_index):
|
||||
"""Multi-Query-Retrieval: Sub-Queries summieren Beitraege; Treffer aus
|
||||
beiden Themen erscheinen im Kontext."""
|
||||
|
||||
Reference in New Issue
Block a user