Praxis CI
ec397f2c65
v0.5 (Live Assist — on-the-job voice companion) milestone complete. 4 phases: P0 (pre-execution, v0.1.10) → P1 (assist core + guardrail, v0.1.11) → P2 (integration + tech-debt + NFR, v0.1.12) → P3 (final review + ship, v0.1.13 = milestone release). 16/16 REQs covered (3 ASSIST + 4 NFR + 9 IDEATE). 4 v0.6 backlog. 469 tests passed, 0 failed. 1 P0 fixed (guardrail processor safety). 8 P1+ flagged for v0.6. 8 v0.4 P1+ tech-debt addressed. G-049 + G-067 grill MUSTs resolved. ESCALATION-01 (PIPEDA) OPEN for human legal review before assist surface go-live. ---ci--- project: praxis phase: 3 milestone: v0.5 status: complete requirements: covered: [REQ-ASSIST-01, REQ-ASSIST-02, REQ-ASSIST-03, REQ-NFR-ASSIST-01, REQ-NFR-ASSIST-02, REQ-NFR-ASSIST-03, REQ-NFR-ASSIST-04, REQ-IDEATE-01, REQ-IDEATE-02, REQ-IDEATE-03, REQ-IDEATE-04, REQ-IDEATE-05, REQ-IDEATE-06, REQ-IDEATE-07, REQ-IDEATE-08, REQ-IDEATE-09] partial: [] ---/ci---
228 lines
8.0 KiB
Python
228 lines
8.0 KiB
Python
"""Assist cost tracking tests (TASK-11-03, REQ-IDEATE-07, C-3, D-012).
|
||
|
||
Tests:
|
||
- derive_assist_turn_cost() computes the per-turn cost (LLM + Piper TTS).
|
||
- The shift-end assist_cost_cents is the sum of per-turn costs.
|
||
- check_c3_budget() with 20 turns/shift × 20 shifts/month → within budget.
|
||
- check_c3_budget() with 100 turns/shift × 30 shifts/month → may exceed (flag=True).
|
||
- Existing derive_cost() unchanged (practice cost tests still pass).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from server.assist.budget_check import C3_TARGET_USD, check_c3_budget
|
||
from server.cost import CostBreakdown, derive_assist_turn_cost, derive_cost, load_rates
|
||
|
||
|
||
# ── derive_assist_turn_cost ─────────────────────────────────────────────────
|
||
|
||
|
||
def test_derive_assist_turn_cost_basic():
|
||
"""Per-turn cost computed from LLM tokens + Piper TTS chars."""
|
||
b = derive_assist_turn_cost(
|
||
llm_input_tokens=300,
|
||
llm_output_tokens=100,
|
||
tts_characters=400,
|
||
tts_provider="piper",
|
||
)
|
||
assert b.derived_cents >= 0
|
||
assert b.llm_input_tokens == 300
|
||
assert b.llm_output_tokens == 100
|
||
assert b.tts_characters == 400
|
||
# No debrief (D-063) + no Deepgram minutes (accounted at shift level).
|
||
assert b.debrief_input_tokens == 0
|
||
assert b.debrief_output_tokens == 0
|
||
assert b.deepgram_audio_minutes == 0.0
|
||
|
||
|
||
def test_derive_assist_turn_cost_piper_zero_tts():
|
||
"""Piper self-hosted TTS is $0 marginal cost (D-065 — Piper is assist default)."""
|
||
b = derive_assist_turn_cost(
|
||
llm_input_tokens=300,
|
||
llm_output_tokens=100,
|
||
tts_characters=10000,
|
||
tts_provider="piper",
|
||
)
|
||
# Piper rate is 0.0 per 1k chars → TTS contributes 0; only LLM cost.
|
||
# LLM: (300+100)/1000 * 0.5 = 0.2 cents → rounds to 0.
|
||
assert b.derived_cents >= 0
|
||
|
||
|
||
def test_derive_assist_turn_cost_cartesia_fallback():
|
||
"""Cartesia TTS fallback (non-default for assist — D-065 prefers Piper)."""
|
||
b = derive_assist_turn_cost(
|
||
llm_input_tokens=300,
|
||
llm_output_tokens=100,
|
||
tts_characters=1000,
|
||
tts_provider="cartesia",
|
||
)
|
||
# Cartesia rate is 3.0 per 1k chars → 1000 chars = 3.0 cents TTS.
|
||
assert b.derived_cents > 0
|
||
|
||
|
||
def test_derive_assist_turn_cost_uses_same_rates_as_derive_cost():
|
||
"""derive_assist_turn_cost uses the same load_rates() + CostBreakdown."""
|
||
rates = load_rates()
|
||
b = derive_assist_turn_cost(
|
||
llm_input_tokens=1000,
|
||
llm_output_tokens=500,
|
||
tts_characters=500,
|
||
tts_provider="piper",
|
||
rates=rates,
|
||
)
|
||
assert b.rates is rates
|
||
assert isinstance(b, CostBreakdown)
|
||
|
||
|
||
def test_derive_cost_unchanged():
|
||
"""Existing derive_cost() unchanged (practice cost tests still pass)."""
|
||
b = derive_cost(
|
||
llm_input_tokens=500,
|
||
llm_output_tokens=200,
|
||
deepgram_audio_minutes=2.0,
|
||
tts_characters=800,
|
||
debrief_input_tokens=300,
|
||
debrief_output_tokens=150,
|
||
tts_provider="cartesia",
|
||
)
|
||
assert b.derived_cents > 0
|
||
assert b.deepgram_audio_minutes == 2.0
|
||
assert b.debrief_input_tokens == 300
|
||
|
||
|
||
# ── Shift-end assist_cost_cents aggregation ─────────────────────────────────
|
||
|
||
|
||
def test_shift_end_assist_cost_is_sum_of_per_turn_costs():
|
||
"""AssistSession.assist_cost_cents is the sum of per-turn costs."""
|
||
from server.assist.session import AssistSession
|
||
from server.assist.context import AssistContext
|
||
|
||
# Construct an AssistSession without calling start() (we only test the
|
||
# cost accumulator, not the DB lifecycle).
|
||
ctx = AssistContext(
|
||
system_prompt="",
|
||
current_week=1,
|
||
scenario_tag="refund",
|
||
theta=0.0,
|
||
coaching_focus="empathy",
|
||
path_slug="customer_service",
|
||
)
|
||
session = AssistSession.__new__(AssistSession)
|
||
session.assist_cost_cents = 0
|
||
session.turn_count = 0
|
||
session.guardrail_block_count = 0
|
||
session.latency_metrics = None # not needed for this test
|
||
|
||
# Simulate 3 turns with per-turn costs.
|
||
for turn_cost in [2, 3, 1]:
|
||
session.add_assist_turn_cost(turn_cost)
|
||
|
||
assert session.assist_cost_cents == 6 # 2 + 3 + 1
|
||
|
||
|
||
# ── check_c3_budget ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_c3_budget_within_budget_typical_usage():
|
||
"""20 turns/shift × 20 shifts/month at ~$0.0005/turn → within budget.
|
||
|
||
Example from the plan: 400 turns/month at ~$0.0005/turn = ~$0.20/month —
|
||
well under the $3 C-3 target.
|
||
"""
|
||
# cost_per_turn_cents = 0.05 cents ($0.0005) — gemma4:cloud pilot rate.
|
||
result = check_c3_budget(
|
||
assist_turns_per_shift=20,
|
||
shifts_per_month=20,
|
||
cost_per_turn_cents=0.05,
|
||
)
|
||
assert result["turns_per_month"] == 400
|
||
# 400 * 0.05 / 100 = $0.20/month
|
||
assert result["monthly_assist_cost"] < 1.0
|
||
assert result["total_with_practice"] < C3_TARGET_USD
|
||
assert result["within_budget"] is True
|
||
assert result["flag"] is False
|
||
assert result["c3_target"] == C3_TARGET_USD == 3.0
|
||
|
||
|
||
def test_c3_budget_exceeds_with_high_usage():
|
||
"""100 turns/shift × 30 shifts/month at higher cost → may exceed (flag=True).
|
||
|
||
3000 turns/month at 0.15 cents/turn = $4.50/month → exceeds $3.
|
||
"""
|
||
result = check_c3_budget(
|
||
assist_turns_per_shift=100,
|
||
shifts_per_month=30,
|
||
cost_per_turn_cents=0.15,
|
||
)
|
||
assert result["turns_per_month"] == 3000
|
||
# 3000 * 0.15 / 100 = $4.50/month → over $3
|
||
assert result["monthly_assist_cost"] > C3_TARGET_USD
|
||
assert result["within_budget"] is False
|
||
assert result["flag"] is True # diagnostic flag (not enforced)
|
||
|
||
|
||
def test_c3_budget_with_practice_cost():
|
||
"""total_with_practice = assist + practice cost."""
|
||
result = check_c3_budget(
|
||
assist_turns_per_shift=20,
|
||
shifts_per_month=20,
|
||
cost_per_turn_cents=0.05,
|
||
practice_cost_per_month_usd=1.5,
|
||
)
|
||
# assist = $0.20, practice = $1.50 → total = $1.70 (within $3)
|
||
assert result["practice_cost_per_month"] == 1.5
|
||
assert result["total_with_practice"] < C3_TARGET_USD
|
||
assert result["within_budget"] is True
|
||
|
||
|
||
def test_c3_budget_with_practice_cost_exceeds():
|
||
"""Assist + practice cost exceeds $3 → flag=True (diagnostic)."""
|
||
result = check_c3_budget(
|
||
assist_turns_per_shift=50,
|
||
shifts_per_month=30,
|
||
cost_per_turn_cents=0.10,
|
||
practice_cost_per_month_usd=2.0,
|
||
)
|
||
# assist = 1500 * 0.10 / 100 = $1.50, practice = $2.00 → total = $3.50
|
||
assert result["total_with_practice"] > C3_TARGET_USD
|
||
assert result["within_budget"] is False
|
||
assert result["flag"] is True
|
||
|
||
|
||
def test_c3_budget_zero_usage():
|
||
"""0 turns → zero cost, within budget."""
|
||
result = check_c3_budget(
|
||
assist_turns_per_shift=0,
|
||
shifts_per_month=0,
|
||
cost_per_turn_cents=0.05,
|
||
)
|
||
assert result["turns_per_month"] == 0
|
||
assert result["monthly_assist_cost"] == 0.0
|
||
assert result["within_budget"] is True
|
||
assert result["flag"] is False
|
||
|
||
|
||
def test_c3_budget_is_diagnostic_not_enforced():
|
||
"""D-012: the check is diagnostic (not enforced). flag=True does not raise.
|
||
|
||
The check_c3_budget() function returns a dict with flag=True when over
|
||
budget, but does NOT raise an exception (D-012 — no enforced ceiling in
|
||
the pilot). The caller logs the flag + continues.
|
||
"""
|
||
result = check_c3_budget(
|
||
assist_turns_per_shift=1000,
|
||
shifts_per_month=30,
|
||
cost_per_turn_cents=1.0,
|
||
)
|
||
# 30000 turns * 1.0 cent / 100 = $300/month → way over $3
|
||
assert result["flag"] is True
|
||
assert result["within_budget"] is False
|
||
# No exception raised — the function returns a dict (diagnostic, not enforced).
|
||
|
||
|
||
def test_c3_target_is_3_usd():
|
||
"""C-3 target is ≤ $3/active learner/month (C-3, D-012)."""
|
||
assert C3_TARGET_USD == 3.0 |