#!/usr/bin/env python3 """End-to-end smoke test (TASK-05-06) — also runnable as a pytest test. Verifies the full v0.1 loop without live API keys (uses the heuristic classifier + a fake LLM for the debrief): start session → simulate 2-3 turns → trigger a branch (classifier) → end session → generate debrief → assert: - debrief non-empty - session + turns + cost logged in SQLite - latency < budget (or logged if exceeded — we log a synthetic value) Run: python scripts/e2e_smoke.py # or pytest tests/test_e2e.py """ from __future__ import annotations import asyncio import os import sys import tempfile from pathlib import Path # Make the project importable when run from the repo root. sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from db.store import PraxisStore, HARDCODED_LEARNER_ID from server.scenarios.loader import load from server.scenarios.classifier import classify_branch_sync_heuristic from server.scenarios.runtime import build_runtime from server.session_recorder import SessionRecorder from server.debrief import generate_debrief from server.guardrails.customer_service import CustomerServiceGuardrail from server.services.base import LLMProvider, LLMStreamChunk class _StubDebriefLLM(LLMProvider): """A stub LLMProvider that returns a canned debrief (no API key needed).""" name = "stub-debrief" roleplay_model = "gemma4:cloud" debrief_model = "deepseek-v4-flash:cloud" async def chat(self, messages, *, stream=True, model=None, no_think=False): yield LLMStreamChunk(content="You did well acknowledging the customer.", is_first=True) async def chat_full(self, messages, *, model=None, no_think=False): return ( "- What you did well: you acknowledged the customer's frustration and " "offered a concrete refund.\n" "- What to improve: confirm next steps explicitly.\n" "- Next step: practice the empathy-first opening.", {"output_tokens": 60, "model": model or self.debrief_model}, ) async def run_e2e(db_path: Path | str | None = None) -> dict: """Run the full e2e smoke sequence; return a result dict for assertions.""" if db_path is None: tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False) tmp.close() db_path = tmp.name store = PraxisStore(db_path) await store.init() # 1. Load the scenario. scenario = load("customer_service_refund_ca_v01") runtime = build_runtime(scenario) assert scenario.failure_mode == "escalates_unresolved", "failure_mode field present (D-009)" # 2. Start a session. recorder = SessionRecorder(store, scenario_id=scenario.id) session_id = await recorder.start() # 3. Simulate 3 turns (accept-resolution path). turns = [ {"role": "assistant", "tts_text": scenario.setup.opening_line, "latency_ms": None}, {"role": "user", "asr_text": "I'm really sorry you're frustrated. I can offer a full refund right now.", "latency_ms": 420.0}, {"role": "assistant", "tts_text": "A refund? Okay, that's something.", "latency_ms": 510.0}, {"role": "user", "asr_text": "Let me confirm the next steps for you.", "latency_ms": 380.0}, ] for t in turns: await recorder.log_turn( role=t["role"], asr_text=t.get("asr_text"), tts_text=t.get("tts_text"), latency_ms=t.get("latency_ms"), ) recorder.add_audio_minutes(1.2) # 4. Classify the branch (R7, offline — heuristic fallback, no API key). learner_turn_texts = [t["asr_text"] for t in turns if t["role"] == "user"] branch_id = classify_branch_sync_heuristic(scenario, learner_turn_texts) runtime.set_branch(branch_id) recorder.set_branch_path([branch_id]) # 5. Generate the debrief (stub LLM — no API key needed). llm = _StubDebriefLLM() guardrail = CustomerServiceGuardrail() debrief_text, _usage = await generate_debrief( llm, scenario, branch_id=branch_id, outcome=runtime.outcome, debrief_focus=runtime.debrief_focus(), learner_turns=[ {"role": t["role"], "asr_text": t.get("asr_text"), "tts_text": t.get("tts_text")} for t in turns ], guardrail=guardrail, ) recorder.add_debrief_tokens(input_tokens=150, output_tokens=60) # 6. End the session (derives cost + writes outcome + debrief + progress). breakdown = await recorder.end( outcome=runtime.outcome, tts_provider=os.environ.get("PRAXIS_TTS", "cartesia"), debrief_text=debrief_text, ) # 7. Assert DB state. sess = await store.get_session(session_id) db_turns = await store.get_turns(session_id) assert sess is not None, "session row exists" assert sess.outcome == runtime.outcome, f"outcome matches branch: {sess.outcome}" assert sess.branch_path == [branch_id], "branch path logged" assert sess.cost_estimated_cents is not None and sess.cost_estimated_cents >= 0, "cost non-null" assert sess.debrief_text == debrief_text, "debrief text persisted" assert len(db_turns) == len(turns), f"all {len(turns)} turns logged" assert breakdown.derived_cents >= 0, "cost breakdown derived" # Synthetic latency (real latency comes from the live pipeline; here we # log the max turn latency as a proxy and check against the budget). max_latency = max((t.get("latency_ms") or 0) for t in turns) budget = 600.0 within_budget = max_latency <= budget return { "session_id": session_id, "branch_id": branch_id, "outcome": sess.outcome, "turns_logged": len(db_turns), "cost_cents": sess.cost_estimated_cents, "debrief_chars": len(sess.debrief_text or ""), "max_latency_ms": max_latency, "within_budget": within_budget, "budget_ms": budget, } def main() -> int: result = asyncio.run(run_e2e()) print("\n" + "=" * 60) print("E2E SMOKE TEST — PASSED") print("=" * 60) for k, v in result.items(): print(f" {k}: {v}") print() return 0 if __name__ == "__main__": sys.exit(main())