praxis/scripts/e2e_smoke.py

#!/usr/bin/env python3
"""End-to-end smoke test (TASK-05-06) — also runnable as a pytest test.

Verifies the full v0.1 loop without live API keys (uses the heuristic
classifier + a fake LLM for the debrief):
  start session → simulate 2-3 turns → trigger a branch (classifier) →
  end session → generate debrief → assert:
    - debrief non-empty
    - session + turns + cost logged in SQLite
    - latency < budget (or logged if exceeded — we log a synthetic value)

Run:
    python scripts/e2e_smoke.py
    # or
    pytest tests/test_e2e.py
"""

from __future__ import annotations

import asyncio
import os
import sys
import tempfile
from pathlib import Path

# Make the project importable when run from the repo root.
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from db.store import PraxisStore, HARDCODED_LEARNER_ID
from server.scenarios.loader import load
from server.scenarios.classifier import classify_branch_sync_heuristic
from server.scenarios.runtime import build_runtime
from server.session_recorder import SessionRecorder
from server.debrief import generate_debrief
from server.guardrails.customer_service import CustomerServiceGuardrail
from server.services.base import LLMProvider, LLMStreamChunk


class _StubDebriefLLM(LLMProvider):
    """A stub LLMProvider that returns a canned debrief (no API key needed)."""

    name = "stub-debrief"
    roleplay_model = "gemma4:cloud"
    debrief_model = "deepseek-v4-flash:cloud"

    async def chat(self, messages, *, stream=True, model=None, no_think=False):
        yield LLMStreamChunk(content="You did well acknowledging the customer.", is_first=True)

    async def chat_full(self, messages, *, model=None, no_think=False):
        return (
            "- What you did well: you acknowledged the customer's frustration and "
            "offered a concrete refund.\n"
            "- What to improve: confirm next steps explicitly.\n"
            "- Next step: practice the empathy-first opening.",
            {"output_tokens": 60, "model": model or self.debrief_model},
        )


async def run_e2e(db_path: Path | str | None = None) -> dict:
    """Run the full e2e smoke sequence; return a result dict for assertions."""
    if db_path is None:
        tmp = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
        tmp.close()
        db_path = tmp.name

    store = PraxisStore(db_path)
    await store.init()

    # 1. Load the scenario.
    scenario = load("customer_service_refund_ca_v01")
    runtime = build_runtime(scenario)
    assert scenario.failure_mode == "escalates_unresolved", "failure_mode field present (D-009)"

    # 2. Start a session.
    recorder = SessionRecorder(store, scenario_id=scenario.id)
    session_id = await recorder.start()

    # 3. Simulate 3 turns (accept-resolution path).
    turns = [
        {"role": "assistant", "tts_text": scenario.setup.opening_line, "latency_ms": None},
        {"role": "user", "asr_text": "I'm really sorry you're frustrated. I can offer a full refund right now.", "latency_ms": 420.0},
        {"role": "assistant", "tts_text": "A refund? Okay, that's something.", "latency_ms": 510.0},
        {"role": "user", "asr_text": "Let me confirm the next steps for you.", "latency_ms": 380.0},
    ]
    for t in turns:
        await recorder.log_turn(
            role=t["role"],
            asr_text=t.get("asr_text"),
            tts_text=t.get("tts_text"),
            latency_ms=t.get("latency_ms"),
        )
    recorder.add_audio_minutes(1.2)

    # 4. Classify the branch (R7, offline — heuristic fallback, no API key).
    learner_turn_texts = [t["asr_text"] for t in turns if t["role"] == "user"]
    branch_id = classify_branch_sync_heuristic(scenario, learner_turn_texts)
    runtime.set_branch(branch_id)
    recorder.set_branch_path([branch_id])

    # 5. Generate the debrief (stub LLM — no API key needed).
    llm = _StubDebriefLLM()
    guardrail = CustomerServiceGuardrail()
    debrief_text, _usage = await generate_debrief(
        llm, scenario,
        branch_id=branch_id,
        outcome=runtime.outcome,
        debrief_focus=runtime.debrief_focus(),
        learner_turns=[
            {"role": t["role"], "asr_text": t.get("asr_text"), "tts_text": t.get("tts_text")}
            for t in turns
        ],
        guardrail=guardrail,
    )
    recorder.add_debrief_tokens(input_tokens=150, output_tokens=60)

    # 6. End the session (derives cost + writes outcome + debrief + progress).
    breakdown = await recorder.end(
        outcome=runtime.outcome,
        tts_provider=os.environ.get("PRAXIS_TTS", "cartesia"),
        debrief_text=debrief_text,
    )

    # 7. Assert DB state.
    sess = await store.get_session(session_id)
    db_turns = await store.get_turns(session_id)
    assert sess is not None, "session row exists"
    assert sess.outcome == runtime.outcome, f"outcome matches branch: {sess.outcome}"
    assert sess.branch_path == [branch_id], "branch path logged"
    assert sess.cost_estimated_cents is not None and sess.cost_estimated_cents >= 0, "cost non-null"
    assert sess.debrief_text == debrief_text, "debrief text persisted"
    assert len(db_turns) == len(turns), f"all {len(turns)} turns logged"
    assert breakdown.derived_cents >= 0, "cost breakdown derived"

    # Synthetic latency (real latency comes from the live pipeline; here we
    # log the max turn latency as a proxy and check against the budget).
    max_latency = max((t.get("latency_ms") or 0) for t in turns)
    budget = 600.0
    within_budget = max_latency <= budget

    return {
        "session_id": session_id,
        "branch_id": branch_id,
        "outcome": sess.outcome,
        "turns_logged": len(db_turns),
        "cost_cents": sess.cost_estimated_cents,
        "debrief_chars": len(sess.debrief_text or ""),
        "max_latency_ms": max_latency,
        "within_budget": within_budget,
        "budget_ms": budget,
    }


def main() -> int:
    result = asyncio.run(run_e2e())
    print("\n" + "=" * 60)
    print("E2E SMOKE TEST — PASSED")
    print("=" * 60)
    for k, v in result.items():
        print(f"  {k}: {v}")
    print()
    return 0


if __name__ == "__main__":
    sys.exit(main())