mirror of
http://100.103.83.12:3003/fegger/odoo-at-payroll.git
synced 2026-09-17 16:56:42 +00:00
feat(agent): harden API-first service
This commit is contained in:
+89
-74
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
@@ -66,11 +67,16 @@ class Retriever:
|
||||
raise RuntimeError(
|
||||
f"index fehlt ({self.db_path}) — zuerst 'python -m agent.cli ingest' ausführen"
|
||||
)
|
||||
self._con = sqlite3.connect(self.db_path)
|
||||
# FastAPI fuehrt synchrone Endpunkte in Worker-Threads aus. Die
|
||||
# Verbindung wird deshalb thread-uebergreifend verwendet; ein RLock
|
||||
# serialisiert die kurzen SQLite-/Matrix-Abschnitte je Retriever.
|
||||
self._lock = threading.RLock()
|
||||
self._con = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
self._con.row_factory = sqlite3.Row
|
||||
self._client = client
|
||||
self._owns_client = client is None
|
||||
self._mat: np.ndarray | None = None
|
||||
self._mat_norm: np.ndarray | None = None
|
||||
self._mat_chunk_ids: list[int] | None = None
|
||||
self._scope_cache: dict[str, set[int]] = {}
|
||||
row = self._con.execute("SELECT MIN(stand), MAX(stand) FROM chunks").fetchone()
|
||||
@@ -78,31 +84,33 @@ class Retriever:
|
||||
self._stand_max = int((row[1] or "2026-01").replace("-", ""))
|
||||
|
||||
def close(self) -> None:
|
||||
self._con.close()
|
||||
if self._owns_client and self._client is not None:
|
||||
self._client.close()
|
||||
with self._lock:
|
||||
self._con.close()
|
||||
if self._owns_client and self._client is not None:
|
||||
self._client.close()
|
||||
|
||||
# -- Index-Kennzahlen ---------------------------------------------------
|
||||
|
||||
def stats(self) -> dict:
|
||||
n_chunks = self._con.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
|
||||
n_entries = self._con.execute(
|
||||
"SELECT COUNT(DISTINCT entry_id) FROM chunks"
|
||||
).fetchone()[0]
|
||||
n_vec = self._con.execute(
|
||||
"SELECT COUNT(*) FROM vectors WHERE model = ?",
|
||||
(self.cfg.embed_model,),
|
||||
).fetchone()[0]
|
||||
meta = dict(self._con.execute("SELECT key, value FROM meta").fetchall())
|
||||
return {
|
||||
"n_entries": n_entries,
|
||||
"n_chunks": n_chunks,
|
||||
"n_vectors": n_vec,
|
||||
"dense_available": n_vec > 0 and not self.cfg.embed_off,
|
||||
"stand_min": str(self._stand_min),
|
||||
"stand_max": str(self._stand_max),
|
||||
"built_at": meta.get("built_at"),
|
||||
}
|
||||
with self._lock:
|
||||
n_chunks = self._con.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
|
||||
n_entries = self._con.execute(
|
||||
"SELECT COUNT(DISTINCT entry_id) FROM chunks"
|
||||
).fetchone()[0]
|
||||
n_vec = self._con.execute(
|
||||
"SELECT COUNT(*) FROM vectors WHERE model = ?",
|
||||
(self.cfg.embed_model,),
|
||||
).fetchone()[0]
|
||||
meta = dict(self._con.execute("SELECT key, value FROM meta").fetchall())
|
||||
return {
|
||||
"n_entries": n_entries,
|
||||
"n_chunks": n_chunks,
|
||||
"n_vectors": n_vec,
|
||||
"dense_available": n_vec > 0 and not self.cfg.embed_off,
|
||||
"stand_min": str(self._stand_min),
|
||||
"stand_max": str(self._stand_max),
|
||||
"built_at": meta.get("built_at"),
|
||||
}
|
||||
|
||||
# -- Einzelverfahren ----------------------------------------------------
|
||||
|
||||
@@ -141,7 +149,12 @@ class Retriever:
|
||||
if self.cfg.embed_off:
|
||||
return {}
|
||||
self._ensure_matrix()
|
||||
if self._mat is None or len(self._mat) == 0:
|
||||
if (
|
||||
self._mat is None
|
||||
or self._mat_norm is None
|
||||
or self._mat_chunk_ids is None
|
||||
or len(self._mat) == 0
|
||||
):
|
||||
return {}
|
||||
if self._client is None:
|
||||
self._client = OllamaClient(
|
||||
@@ -166,7 +179,7 @@ class Retriever:
|
||||
for i in order:
|
||||
if len(out) >= limit:
|
||||
break
|
||||
cid = self._mat_chunk_ids[i]
|
||||
cid = self._mat_chunk_ids[int(i)]
|
||||
if scope_ids is not None and cid not in scope_ids:
|
||||
continue
|
||||
out[cid] = float(sims[i])
|
||||
@@ -392,16 +405,17 @@ class Retriever:
|
||||
deterministisch in Direktzahlung und Alternativen zerlegt. Das gilt
|
||||
auch für den Offline-Retrieval-Eval, der keinen LLM-Planer aufruft.
|
||||
"""
|
||||
n = n_entries or self.cfg.context_blocks
|
||||
latest_year = str(self._stand_max)[:4]
|
||||
deterministic = decision_support_plan(question, default_year=latest_year)
|
||||
if deterministic:
|
||||
sub_queries, _qtype = deterministic
|
||||
return self.search_multi(sub_queries, n_entries=n)
|
||||
fused, bm_all, dn_all = self._fuse_queries(
|
||||
[SubQuery(text=question)], self.cfg.candidate_pool
|
||||
)
|
||||
return self._context_from_fused(fused, n, bm_all, dn_all)
|
||||
with self._lock:
|
||||
n = n_entries or self.cfg.context_blocks
|
||||
latest_year = str(self._stand_max)[:4]
|
||||
deterministic = decision_support_plan(question, default_year=latest_year)
|
||||
if deterministic:
|
||||
sub_queries, _qtype = deterministic
|
||||
return self.search_multi(sub_queries, n_entries=n)
|
||||
fused, bm_all, dn_all = self._fuse_queries(
|
||||
[SubQuery(text=question)], self.cfg.candidate_pool
|
||||
)
|
||||
return self._context_from_fused(fused, n, bm_all, dn_all)
|
||||
|
||||
def search_multi(
|
||||
self, sub_queries: list, n_entries: int | None = None
|
||||
@@ -412,44 +426,45 @@ class Retriever:
|
||||
seiner besten Quelle hinein — sonst dominieren Einträge, die in
|
||||
mehreren Sub-Queries mittelgut matchen). Temporal-Intent: kv-
|
||||
Einträge im gefragten Geltungsjahr erhalten temporal_boost."""
|
||||
n = n_entries or self.cfg.context_blocks
|
||||
if (
|
||||
len(sub_queries) == 1
|
||||
and sub_queries[0].scope is None
|
||||
and sub_queries[0].stand_year is None
|
||||
):
|
||||
return self.search(sub_queries[0].text, n_entries=n)
|
||||
pool = self.cfg.candidate_pool
|
||||
fused_total: dict[int, float] = {}
|
||||
bm_all: set[int] = set()
|
||||
dn_all: set[int] = set()
|
||||
per_query_entries: list[list[str]] = []
|
||||
for sq in sub_queries:
|
||||
bm, dn, fused_q = self._search_one(sq, pool)
|
||||
bm_all.update(bm)
|
||||
dn_all.update(dn)
|
||||
for cid, score in fused_q.items():
|
||||
fused_total[cid] = fused_total.get(cid, 0.0) + score
|
||||
rows_q = self._fetch_chunks(list(fused_q))
|
||||
entry_best: dict[str, float] = {}
|
||||
for cid, score in fused_q.items():
|
||||
row = rows_q.get(cid)
|
||||
if row is not None:
|
||||
eid = row["entry_id"]
|
||||
entry_best[eid] = max(entry_best.get(eid, 0.0), score)
|
||||
per_query_entries.append(
|
||||
[e for e, _ in sorted(entry_best.items(), key=lambda kv: -kv[1])]
|
||||
with self._lock:
|
||||
n = n_entries or self.cfg.context_blocks
|
||||
if (
|
||||
len(sub_queries) == 1
|
||||
and sub_queries[0].scope is None
|
||||
and sub_queries[0].stand_year is None
|
||||
):
|
||||
return self.search(sub_queries[0].text, n_entries=n)
|
||||
pool = self.cfg.candidate_pool
|
||||
fused_total: dict[int, float] = {}
|
||||
bm_all: set[int] = set()
|
||||
dn_all: set[int] = set()
|
||||
per_query_entries: list[list[str]] = []
|
||||
for sq in sub_queries:
|
||||
bm, dn, fused_q = self._search_one(sq, pool)
|
||||
bm_all.update(bm)
|
||||
dn_all.update(dn)
|
||||
for cid, score in fused_q.items():
|
||||
fused_total[cid] = fused_total.get(cid, 0.0) + score
|
||||
rows_q = self._fetch_chunks(list(fused_q))
|
||||
entry_best: dict[str, float] = {}
|
||||
for cid, score in fused_q.items():
|
||||
row = rows_q.get(cid)
|
||||
if row is not None:
|
||||
eid = row["entry_id"]
|
||||
entry_best[eid] = max(entry_best.get(eid, 0.0), score)
|
||||
per_query_entries.append(
|
||||
[e for e, _ in sorted(entry_best.items(), key=lambda kv: -kv[1])]
|
||||
)
|
||||
reserved: list[str] = []
|
||||
for entries in per_query_entries:
|
||||
taken = 0
|
||||
for eid in entries:
|
||||
if taken >= self.cfg.per_query_slots:
|
||||
break
|
||||
if eid not in reserved:
|
||||
reserved.append(eid)
|
||||
taken += 1
|
||||
years = {sq.stand_year for sq in sub_queries if sq.stand_year}
|
||||
return self._context_from_fused(
|
||||
fused_total, n, bm_all, dn_all, stand_years=years, reserved=reserved
|
||||
)
|
||||
reserved: list[str] = []
|
||||
for entries in per_query_entries:
|
||||
taken = 0
|
||||
for eid in entries:
|
||||
if taken >= self.cfg.per_query_slots:
|
||||
break
|
||||
if eid not in reserved:
|
||||
reserved.append(eid)
|
||||
taken += 1
|
||||
years = {sq.stand_year for sq in sub_queries if sq.stand_year}
|
||||
return self._context_from_fused(
|
||||
fused_total, n, bm_all, dn_all, stand_years=years, reserved=reserved
|
||||
)
|
||||
Reference in New Issue
Block a user