mirror of
http://100.103.83.12:3003/fegger/odoo-at-payroll.git
synced 2026-09-17 16:56:42 +00:00
[ADD] odoo-at-payroll: opendataloader-pdf agent skill for PDF extraction
New .agents/skills/opendataloader-pdf/ for structured PDF extraction (Markdown/JSON with bounding boxes, hybrid AI mode, scanned-PDF OCR) based on the upstream odl-pdf agent skill (Apache-2.0), with its helper scripts verify-json.py and hybrid-health.sh vendored under scripts/. Core disciplines: discover options from the installed --help (flags drift between releases), batch all inputs into one invocation (JVM per call), verify the result against intent (zero exit is not success: auto-triage skips enrichment, fallbacks drop quality, empty pipes), treat extracted content as untrusted, never commit extracted full texts of the licensed .lexis360/.wiku sources. Workflow wiring: AGENTS.md skill selection routes any PDF extraction to the skill; wissensbasis/SKILL.md cross-references it for difficult PDFs and spot-checks, with Layer-1 batch intake deliberately kept on build_lexis_kb.py (frozen IDs and --check depend on the text shape). ODL 2.5.8 installed in the user venv ~/.local/lib/python (not the Odoo .venv); all documented flags verified against the installed CLI on real .lexis360/ sources.
This commit is contained in:
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# hybrid-health.sh
|
||||
# Checks the health of a running opendataloader-pdf hybrid server.
|
||||
# Works on Windows (Git Bash), macOS, and Linux.
|
||||
# Outputs key=value pairs for machine readability.
|
||||
#
|
||||
# Vendored verbatim from the opendataloader-pdf upstream agent skill
|
||||
# (skills/odl-pdf/scripts/hybrid-health.sh), Apache-2.0.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DEFAULT_URL="http://localhost:5002"
|
||||
HYBRID_URL="${DEFAULT_URL}"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--url)
|
||||
if [[ $# -lt 2 ]]; then
|
||||
echo "Error: --url requires a value" >&2
|
||||
exit 1
|
||||
fi
|
||||
HYBRID_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--url=*)
|
||||
HYBRID_URL="${1#--url=}"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
echo "Usage: $0 [--url <url>]" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate the URL before use: reject empty or malformed values.
|
||||
# Require the form http(s)://host[:port] (optional trailing slash; no path).
|
||||
if [[ -z "${HYBRID_URL}" ]]; then
|
||||
echo "Error: --url must not be empty" >&2
|
||||
echo "Usage: $0 [--url <http(s)://host[:port]>]" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Note the excluded '@': a URL with userinfo (https://user:pass@host) is rejected
|
||||
# so credentials are never echoed back to stdout.
|
||||
if [[ ! "${HYBRID_URL}" =~ ^https?://[^[:space:]/@]+(:[0-9]+)?/?$ ]]; then
|
||||
if [[ "${HYBRID_URL}" == *@* ]]; then
|
||||
echo "Error: --url must not contain embedded credentials (userinfo '@'); pass a plain host[:port]" >&2
|
||||
else
|
||||
echo "Error: --url must be of the form http(s)://host[:port] (got: '${HYBRID_URL}')" >&2
|
||||
fi
|
||||
echo "Usage: $0 [--url <http(s)://host[:port]>]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
HEALTH_ENDPOINT="${HYBRID_URL%/}/health"
|
||||
|
||||
# Detect available HTTP client
|
||||
_http_get_status() {
|
||||
local url="$1"
|
||||
if command -v curl &>/dev/null; then
|
||||
curl --silent --output /dev/null --write-out "%{http_code}" \
|
||||
--max-time 5 --connect-timeout 3 "$url" 2>/dev/null
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget --quiet --server-response --spider --timeout=5 "$url" 2>&1 \
|
||||
| awk '/HTTP\//{print $2}' | tail -1
|
||||
else
|
||||
echo "none"
|
||||
fi
|
||||
}
|
||||
|
||||
HTTP_STATUS=$(_http_get_status "${HEALTH_ENDPOINT}" || true)
|
||||
|
||||
# No HTTP client available to probe — this is NOT "server stopped"; the check
|
||||
# could not run at all. Report a distinct state so callers don't misread it.
|
||||
if [[ "${HTTP_STATUS}" == "none" ]]; then
|
||||
echo "HYBRID_SERVER=error"
|
||||
echo "HYBRID_URL=${HYBRID_URL}"
|
||||
echo "HYBRID_STATUS=client-missing"
|
||||
echo ""
|
||||
echo "Cannot probe the hybrid server: no HTTP client (curl or wget) is available. Install one, or check the server manually."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Interpret result
|
||||
if [[ -z "${HTTP_STATUS}" || "${HTTP_STATUS}" == "000" ]]; then
|
||||
echo "HYBRID_SERVER=stopped"
|
||||
echo "HYBRID_URL=${HYBRID_URL}"
|
||||
echo "HYBRID_STATUS=none"
|
||||
echo ""
|
||||
echo "Hybrid server is not running at ${HYBRID_URL}. Start it with: opendataloader-pdf-hybrid"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The script always exits 0 (a completed health probe is not itself a failure).
|
||||
# The result is on stdout: HYBRID_SERVER=running means reachable; stopped/error
|
||||
# mean not usable. Callers must branch on that value, NOT on the exit code.
|
||||
if [[ "${HTTP_STATUS}" =~ ^2 ]]; then
|
||||
echo "HYBRID_SERVER=running"
|
||||
else
|
||||
echo "HYBRID_SERVER=error"
|
||||
fi
|
||||
|
||||
echo "HYBRID_URL=${HYBRID_URL}"
|
||||
echo "HYBRID_STATUS=${HTTP_STATUS}"
|
||||
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify-json.py — schema-tolerant summary of an opendataloader-pdf JSON output.
|
||||
|
||||
Purpose: give an agent a safe way to VERIFY extraction results (SKILL.md Stage 4)
|
||||
without hand-writing fragile jq / assuming exact key names that vary by release.
|
||||
|
||||
It parses the JSON, walks the element tree generically (any nested dict carrying a
|
||||
"type" field, under any "kids"/children key), and reports element-type counts plus
|
||||
whether text / tables / images are present. It is schema-tolerant, not fully
|
||||
agnostic: it expects ODL-style `type` and `content`/`text` field names (it does
|
||||
not assume tree location or child-key names). It does NOT decide pass/fail — the
|
||||
agent judges the summary against the user's intent.
|
||||
|
||||
Vendored verbatim from the opendataloader-pdf upstream agent skill
|
||||
(skills/odl-pdf/scripts/verify-json.py), Apache-2.0.
|
||||
|
||||
Usage:
|
||||
python verify-json.py output.json
|
||||
Exit codes:
|
||||
0 parsed successfully (summary printed)
|
||||
1 file missing, empty, or not valid JSON
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Make stdout tolerant of non-ASCII on Windows consoles (cp1252/cp949).
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
|
||||
TEXT_KEYS = ("content", "text") # tried in order; first non-empty wins
|
||||
IMAGE_TYPES = ("image", "picture", "figure")
|
||||
TABLE_TYPES = ("table",)
|
||||
|
||||
|
||||
def load(path: Path):
|
||||
if not path.exists():
|
||||
print(f"ERROR: file not found: {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8").strip()
|
||||
except UnicodeDecodeError as e:
|
||||
print(f"ERROR: not valid UTF-8 ({e}): {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not raw:
|
||||
print(f"ERROR: file is empty: {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"ERROR: not valid JSON ({e}): {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def walk(node, types, stats):
|
||||
"""Recursively find every dict that has a 'type' field; tally it."""
|
||||
if isinstance(node, dict):
|
||||
t = node.get("type")
|
||||
if isinstance(t, str):
|
||||
types[t] = types.get(t, 0) + 1
|
||||
tl = t.lower()
|
||||
if any(k in tl for k in IMAGE_TYPES):
|
||||
stats["images"] += 1
|
||||
if any(k == tl for k in TABLE_TYPES):
|
||||
stats["tables"] += 1
|
||||
for tk in TEXT_KEYS:
|
||||
v = node.get(tk)
|
||||
if isinstance(v, str) and v.strip():
|
||||
stats["text_elements"] += 1
|
||||
break
|
||||
for v in node.values():
|
||||
walk(v, types, stats)
|
||||
elif isinstance(node, list):
|
||||
for v in node:
|
||||
walk(v, types, stats)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = argv if argv is not None else sys.argv[1:]
|
||||
if len(argv) != 1:
|
||||
print("Usage: python verify-json.py <output.json>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
data = load(Path(argv[0]))
|
||||
|
||||
types = {}
|
||||
stats = {"images": 0, "tables": 0, "text_elements": 0}
|
||||
walk(data, types, stats)
|
||||
|
||||
total = sum(types.values())
|
||||
# "number of pages" key name varies; probe a few, else report unknown.
|
||||
pages = "unknown"
|
||||
if isinstance(data, dict):
|
||||
for k in ("number of pages", "number_of_pages", "pages", "page count"):
|
||||
if isinstance(data.get(k), int):
|
||||
pages = data[k]
|
||||
break
|
||||
|
||||
print("=== opendataloader-pdf JSON summary ===")
|
||||
print(f"pages: {pages}")
|
||||
print(f"typed elements: {total}")
|
||||
print(f"has_text: {stats['text_elements'] > 0} (text-bearing elements: {stats['text_elements']})")
|
||||
print(f"has_tables: {stats['tables'] > 0} (tables: {stats['tables']})")
|
||||
print(f"has_images: {stats['images'] > 0} (images/pictures: {stats['images']})")
|
||||
if types:
|
||||
print("element types:")
|
||||
for t, n in sorted(types.items(), key=lambda kv: -kv[1]):
|
||||
print(f" {t}: {n}")
|
||||
else:
|
||||
print("element types: (none found — output may be empty or an unexpected shape)")
|
||||
|
||||
print()
|
||||
print("NOTE: this is a summary, not a pass/fail. Judge it against the user's "
|
||||
"intent (SKILL.md Stage 4): e.g. no text is a FAILURE only if text was expected.")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user