Praxis CI
813bd586d6
v0.3 milestone merged to main. Mastery scoring + competency rubrics + verifiable credentials (formative-tier) shipped. 13/13 REQ-IDs covered. Next milestone: v0.4 (operator tier — cohort dashboard + auth + Postgres). ---ci--- project: praxis phase: 2 milestone: v0.3 status: complete milestone_complete: true milestone_merged_to_main: true ---/ci---
122 lines
4.0 KiB
Python
122 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""SLICE-08 TASK-08-04 — Real-LLM evidence extraction smoke test (grill Axis 7 FIX #1).
|
|
|
|
Runs ONE real session transcript through the actual deepseek-v4-flash:cloud
|
|
evidence extractor and verifies the output is valid JSON with fuzzy-matching
|
|
quotes (the extraction prompt works against the real model, not just the
|
|
scoring logic against mocked responses).
|
|
|
|
Staging-gated: this test calls a real paid LLM endpoint. It runs ONLY when the
|
|
env var `PRAXIS_RUN_REAL_LLM_TESTS=1` is set, AND requires `OLLAMA_API_KEY`.
|
|
CI must NOT set the gate env var — mocked-LLM tests stay the CI source of
|
|
truth (REQ-MAST-01 determinism is covered by the mocked tests; this script
|
|
validates the prompt+model contract against model drift).
|
|
|
|
Run:
|
|
python3 scripts/test_real_llm_evidence.py
|
|
|
|
Exit codes:
|
|
0 — SKIP (gate not set) OR PASS
|
|
1 — FAIL (gate set, real call failed or output invalid)
|
|
2 — MISCONFIG (gate set but OLLAMA_API_KEY missing)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from server.llm.ollama_cloud import OllamaCloudLLM
|
|
from server.mastery.evidence_extractor import extract_evidence
|
|
from server.mastery.rubric_loader import clear_cache, load_rubric
|
|
|
|
_REPO = Path(__file__).resolve().parent.parent
|
|
_RUBRICS_DIR = _REPO / "rubrics"
|
|
_GATE_ENV = "PRAXIS_RUN_REAL_LLM_TESTS"
|
|
|
|
_TRANSCRIPT = [
|
|
{"role": "customer", "content": "My order arrived cracked and I'm furious."},
|
|
{
|
|
"role": "learner",
|
|
"content": (
|
|
"I'm really sorry the bowl arrived cracked — that's genuinely "
|
|
"frustrating. I can refund the full amount to your original card "
|
|
"within 3 business days, or send a replacement first class tomorrow. "
|
|
"Which would you prefer?"
|
|
),
|
|
},
|
|
{"role": "customer", "content": "Just refund it."},
|
|
{
|
|
"role": "learner",
|
|
"content": (
|
|
"Of course — I've issued a full refund of $42.99 to your Visa ending "
|
|
"4421. You'll see it in 2-3 business days. Is there anything else?"
|
|
),
|
|
},
|
|
]
|
|
|
|
|
|
def _print_skip() -> None:
|
|
print(f"SKIP (set {_GATE_ENV}=1 to run)")
|
|
|
|
|
|
async def _run_real() -> int:
|
|
if not os.environ.get("OLLAMA_API_KEY", "").strip():
|
|
print(f"FAIL — {_GATE_ENV}=1 but OLLAMA_API_KEY is not set")
|
|
return 2
|
|
|
|
clear_cache()
|
|
rubric = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
llm = OllamaCloudLLM()
|
|
|
|
print("Calling deepseek-v4-flash:cloud for evidence extraction …")
|
|
result = await extract_evidence(
|
|
_TRANSCRIPT, rubric.criterion_ids(), llm, max_attempts=2
|
|
)
|
|
|
|
if result.scoring_inconclusive:
|
|
print(
|
|
f"FAIL — extraction returned scoring_inconclusive after "
|
|
f"{result.attempts} attempts; rejected quotes="
|
|
f"{result.rejected_quotes[:3]}"
|
|
)
|
|
return 1
|
|
|
|
if not result.evidence:
|
|
print(f"FAIL — extraction returned no evidence (attempts={result.attempts})")
|
|
return 1
|
|
|
|
crit_ids = {e.criterion_id for e in result.evidence}
|
|
expected = set(rubric.criterion_ids())
|
|
if not crit_ids.issubset(expected):
|
|
print(f"FAIL — unknown criterion ids: {crit_ids - expected}")
|
|
return 1
|
|
|
|
for ev in result.evidence:
|
|
if not ev.quote.strip():
|
|
print(f"FAIL — empty quote for criterion {ev.criterion_id!r}")
|
|
return 1
|
|
if not ev.signals:
|
|
print(f"FAIL — no signals for criterion {ev.criterion_id!r}")
|
|
return 1
|
|
|
|
print(f"PASS — {len(result.evidence)} evidence items extracted (attempts={result.attempts})")
|
|
for ev in result.evidence:
|
|
print(f" - {ev.criterion_id}: {len(ev.signals)} signals, quote={ev.quote[:60]!r}…")
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
if os.environ.get(_GATE_ENV, "").strip() != "1":
|
|
_print_skip()
|
|
return 0
|
|
return asyncio.run(_run_real())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main()) |