#!/usr/bin/env python3 """SLICE-08 TASK-08-01 — End-to-end P1 mastery smoke test (not a pytest). Simulates 3 sessions across 3 distinct Customer-Service scenarios → runs the mastery flow (with a mocked LLM returning canned verbatim-quote evidence) → verifies: - mastery gate opens after the 3rd passing scenario with path score >= 3.5 - theta converges upward (passes against increasing difficulty) - progress advances week-by-week as each week's gate opens - one mastery_gate_event row is recorded per session in SQLite Runnable: `python3 scripts/test_mastery_e2e.py` Exit code 0 on PASS, 1 on FAIL. Prints a PASS/FAIL summary. """ from __future__ import annotations import asyncio import json import os import sys import tempfile from pathlib import Path from typing import Any from unittest.mock import AsyncMock sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from db.store import PraxisStore, HARDCODED_LEARNER_ID from server.mastery.irt import IRTEngine, DEFAULT_THETA from server.mastery.rubric_loader import clear_cache, load_rubric from server.paths.engine import PathEngine from server.scenarios.loader import load as load_scenario from server.session_recorder import MasteryFlowDeps, SessionRecorder _REPO = Path(__file__).resolve().parent.parent _RUBRICS_DIR = _REPO / "rubrics" _SCENARIOS_DIR = _REPO / "scenarios" _PATHS_DIR = _REPO / "paths" _PATH_SLUG = "customer_service" _SCENARIO_IDS = [ "cs_refund_ca_v01", "cs_escalation_ca_v02", "cs_policy_exception_ca_v03", ] def _transcript_for(scenario_id: str) -> list[dict[str, str]]: if scenario_id == "cs_refund_ca_v01": learner_a = ( "I'm really sorry the bowl arrived cracked — that's genuinely " "frustrating. I can refund the full amount to your original card " "within 3 business days, or send a replacement first class tomorrow. " "Which would you prefer?" ) learner_b = ( "Of course — I've issued a full refund of $42.99 to your Visa ending " "4421. You'll see it in 2-3 business days. Is there anything else I " "can help with today?" ) elif scenario_id == "cs_escalation_ca_v02": learner_a = ( "I hear you — two weeks with no straight answers is genuinely " "infuriating, and you're right to push for clarity. I'm not going to " "hide behind policy. Here's what I can do right now: I'll trace the " "shipment, refund the shipping cost today, and give you a firm " "delivery date within 24 hours. Would that work?" ) learner_b = ( "Thank you for staying with me on this. I've refunded the $9.50 " "shipping charge to your card and flagged the order for immediate " "dispatch. You'll get a tracking number by email within the hour. " "Is there anything else I can do for you?" ) else: learner_a = ( "You're absolutely right — a defect appearing last week is a " "different situation from a 45-day change-of-mind. The 30-day window " "is a guideline for returns, not a hard wall for defects. I can " "offer a partial credit of 70% toward a replacement, or start a " "manufacturer warranty claim on your behalf. Which would you prefer?" ) learner_b = ( "I've issued a $30 partial credit to your original payment method " "and started the manufacturer warranty claim — they'll reach out " "within 5 business days. You'll get a confirmation email within the " "hour. Anything else I can help with today?" ) return [ {"role": "customer", "content": "I'm upset and need this resolved now."}, {"role": "learner", "content": learner_a}, {"role": "customer", "content": "Okay, go ahead with that."}, {"role": "learner", "content": learner_b}, ] def _canned_evidence(transcript: list[dict[str, str]]) -> str: t1 = transcript[1]["content"] t2 = transcript[3]["content"] return json.dumps( [ { "criterion_id": "empathy", "quote": t1, "signals": [ "named_emotion_in_own_words", "acknowledged_specific", "tone_pace_adjusted", "multiple_acknowledgement_instances", ], }, { "criterion_id": "resolution", "quote": t1, "signals": [ "concrete_method", "concrete_amount_or_channel", "concrete_next_step", "decision_tree_of_options", "matched_to_customer_preference", "confirms_acceptance", ], }, { "criterion_id": "de_escalation", "quote": t1, "signals": [ "explicit_acknowledge_reframe_offer", "cycles_acknowledge_reframe", "lowers_intensity_without_conceding_policy", ], }, { "criterion_id": "professionalism", "quote": t2, "signals": [ "plain_language", "in_role_throughout", "no_prohibited_advice", "adapts_register", "concise_for_voice", "manages_silence", ], }, ] ) class _ScriptedLLM: def __init__(self, raws: list[str]) -> None: self._iter = iter(raws) async def chat_full( self, messages: list[dict[str, str]], *, model: str | None = None, no_think: bool = False, ) -> tuple[str, dict[str, Any]]: try: raw = next(self._iter) except StopIteration as exc: raise RuntimeError("scripted LLM exhausted") from exc return raw, {"model": model or "test"} def _deps(llm: Any, scenario_id: str) -> MasteryFlowDeps: clear_cache() return MasteryFlowDeps( llm=llm, irt=IRTEngine(), path_engine=PathEngine(paths_dir=_PATHS_DIR), load_rubric=lambda: load_rubric(_PATH_SLUG, rubrics_dir=_RUBRICS_DIR), load_scenario=lambda: load_scenario(scenario_id, scenarios_dir=_SCENARIOS_DIR), load_path=lambda: PathEngine(paths_dir=_PATHS_DIR).load_path(_PATH_SLUG), ) def _fmt_pass(label: str) -> str: return f" PASS {label}" def _fmt_fail(label: str, detail: str) -> str: return f" FAIL {label} — {detail}" async def _run() -> int: failures: list[str] = [] print("=" * 70) print("SLICE-08 TASK-08-01 — End-to-end P1 mastery smoke test") print("=" * 70) with tempfile.TemporaryDirectory(prefix="praxis_e2e_") as tmp: db_path = Path(tmp) / "e2e.db" store = PraxisStore(db_path) await store.init() canned = [_canned_evidence(_transcript_for(sid)) for sid in _SCENARIO_IDS] llm = _ScriptedLLM(canned) results: list[dict[str, Any]] = [] for sid in _SCENARIO_IDS: rec = SessionRecorder(store, scenario_id=sid) await rec.start() rec.set_mastery_turns(_transcript_for(sid)) rec.set_branch_path(["accept_resolution"]) await rec.end(outcome="success", debrief_text="nicely done") res = await rec.run_mastery_flow(_deps(llm, sid)) results.append(res) # ── Check 1: every session scored (no scoring_inconclusive) ── for i, r in enumerate(results): if r["status"] != "scored": failures.append( f"session[{i}] ({_SCENARIO_IDS[i]}) status={r['status']!r} (expected 'scored')" ) # ── Check 2: every scenario passed ── for i, r in enumerate(results): if not r.get("passed"): failures.append( f"session[{i}] ({_SCENARIO_IDS[i]}) passed=False (mean={r.get('weighted_mean')})" ) # ── Check 3: theta converges upward (3 passes against increasing b) ── thetas = [r["theta"] for r in results] if not (thetas[-1] > DEFAULT_THETA and thetas[-1] >= thetas[0]): failures.append( f"theta did not converge upward: start={DEFAULT_THETA} " f"trajectory={thetas}" ) # ── Check 4: gate opens on the 3rd passing scenario ── gate_opens = [bool(r.get("gate_open")) for r in results] if not gate_opens[-1]: failures.append( f"gate did not open on 3rd passing scenario: gate_open={gate_opens}" ) # ── Check 5: gate-open path score >= 3.5 ── final_path_score = results[-1].get("weighted_mean", 0.0) progress_row = await store.get_progress(HARDCODED_LEARNER_ID, _PATH_SLUG) stored_score = float(progress_row["mastery_score"]) if progress_row else 0.0 if stored_score < 3.5: failures.append( f"stored path mastery_score {stored_score} < 3.5 (gate threshold)" ) # ── Check 6: progress advanced at least once (new_week > 1 by end) ── if progress_row is None: failures.append("no mastery_progress row persisted") else: # After 3 passing scenarios the learner should have advanced weeks. if progress_row["current_week"] < 2: failures.append( f"progress did not advance: current_week={progress_row['current_week']}" ) # ── Check 7: gate events recorded (one per scored session) ── events = await store.list_gate_events(HARDCODED_LEARNER_ID, _PATH_SLUG) if len(events) != 3: failures.append( f"expected 3 gate events, got {len(events)}" ) for ev in events: sp = json.loads(ev["scenarios_passed_json"]) rs = json.loads(ev["rubric_scores_json"]) if not isinstance(sp, list): failures.append(f"gate event {ev['id']} scenarios_passed_json not a list") if not isinstance(rs, list) or len(rs) != 4: failures.append( f"gate event {ev['id']} rubric_scores_json malformed (len={len(rs) if isinstance(rs, list) else 'NaN'})" ) # ── Summary ── print("") print(f" scenario trajectory : {_SCENARIO_IDS}") print(f" theta trajectory : {[round(t, 4) for t in thetas]}") print(f" gate-open trajectory: {gate_opens}") print(f" stored path score : {stored_score}") print( f" progress current_week: {progress_row['current_week'] if progress_row else 'N/A'}" ) print(f" gate events recorded: {len(events)}") print("") if failures: for f in failures: print(_fmt_fail("check", f)) print("") print("RESULT: FAIL") return 1 checks = [ "all 3 sessions scored", "all 3 scenarios passed", f"theta converged upward ({round(thetas[0], 3)} → {round(thetas[-1], 3)})", "gate opened on 3rd passing scenario", f"path score {stored_score} >= 3.5", "progress advanced week-by-week", "3 gate events recorded with parsable JSON evidence", ] for c in checks: print(_fmt_pass(c)) print("") print("RESULT: PASS") return 0 def main() -> int: return asyncio.run(_run()) if __name__ == "__main__": raise SystemExit(main())