feat(P02): complete integration + tech-debt + NFR measurement phase — v0.1.12 tagged
Phase 2 (Integration + Tech-Debt + NFR Measurement) complete. 4 slices, 2 waves, 9 tasks. 4 REQs covered. 60 new tests (469 total). 8 v0.4 P1+ tech-debt findings addressed. Verify: APPROVE_WITH_NOTES. NFR measurement (p95 latency + guardrail FP/FN), cohort aggregation assist metrics (5 new metrics, no schema change), assist cost tracking + C-3 budget check, tech-debt wave (argon2id offload, cookie-secret validation, credential enum, f-string SQL, cache persistence, zoneinfo, audit log, 429 mock). ---ci--- project: praxis phase: 2 milestone: v0.5 status: complete requirements: covered: [REQ-NFR-ASSIST-01, REQ-IDEATE-04, REQ-IDEATE-06, REQ-IDEATE-07] partial: [] ---/ci---
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
"""Assist cost tracking tests (TASK-11-03, REQ-IDEATE-07, C-3, D-012).
|
||||
|
||||
Tests:
|
||||
- derive_assist_turn_cost() computes the per-turn cost (LLM + Piper TTS).
|
||||
- The shift-end assist_cost_cents is the sum of per-turn costs.
|
||||
- check_c3_budget() with 20 turns/shift × 20 shifts/month → within budget.
|
||||
- check_c3_budget() with 100 turns/shift × 30 shifts/month → may exceed (flag=True).
|
||||
- Existing derive_cost() unchanged (practice cost tests still pass).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from server.assist.budget_check import C3_TARGET_USD, check_c3_budget
|
||||
from server.cost import CostBreakdown, derive_assist_turn_cost, derive_cost, load_rates
|
||||
|
||||
|
||||
# ── derive_assist_turn_cost ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_derive_assist_turn_cost_basic():
|
||||
"""Per-turn cost computed from LLM tokens + Piper TTS chars."""
|
||||
b = derive_assist_turn_cost(
|
||||
llm_input_tokens=300,
|
||||
llm_output_tokens=100,
|
||||
tts_characters=400,
|
||||
tts_provider="piper",
|
||||
)
|
||||
assert b.derived_cents >= 0
|
||||
assert b.llm_input_tokens == 300
|
||||
assert b.llm_output_tokens == 100
|
||||
assert b.tts_characters == 400
|
||||
# No debrief (D-063) + no Deepgram minutes (accounted at shift level).
|
||||
assert b.debrief_input_tokens == 0
|
||||
assert b.debrief_output_tokens == 0
|
||||
assert b.deepgram_audio_minutes == 0.0
|
||||
|
||||
|
||||
def test_derive_assist_turn_cost_piper_zero_tts():
|
||||
"""Piper self-hosted TTS is $0 marginal cost (D-065 — Piper is assist default)."""
|
||||
b = derive_assist_turn_cost(
|
||||
llm_input_tokens=300,
|
||||
llm_output_tokens=100,
|
||||
tts_characters=10000,
|
||||
tts_provider="piper",
|
||||
)
|
||||
# Piper rate is 0.0 per 1k chars → TTS contributes 0; only LLM cost.
|
||||
# LLM: (300+100)/1000 * 0.5 = 0.2 cents → rounds to 0.
|
||||
assert b.derived_cents >= 0
|
||||
|
||||
|
||||
def test_derive_assist_turn_cost_cartesia_fallback():
|
||||
"""Cartesia TTS fallback (non-default for assist — D-065 prefers Piper)."""
|
||||
b = derive_assist_turn_cost(
|
||||
llm_input_tokens=300,
|
||||
llm_output_tokens=100,
|
||||
tts_characters=1000,
|
||||
tts_provider="cartesia",
|
||||
)
|
||||
# Cartesia rate is 3.0 per 1k chars → 1000 chars = 3.0 cents TTS.
|
||||
assert b.derived_cents > 0
|
||||
|
||||
|
||||
def test_derive_assist_turn_cost_uses_same_rates_as_derive_cost():
|
||||
"""derive_assist_turn_cost uses the same load_rates() + CostBreakdown."""
|
||||
rates = load_rates()
|
||||
b = derive_assist_turn_cost(
|
||||
llm_input_tokens=1000,
|
||||
llm_output_tokens=500,
|
||||
tts_characters=500,
|
||||
tts_provider="piper",
|
||||
rates=rates,
|
||||
)
|
||||
assert b.rates is rates
|
||||
assert isinstance(b, CostBreakdown)
|
||||
|
||||
|
||||
def test_derive_cost_unchanged():
|
||||
"""Existing derive_cost() unchanged (practice cost tests still pass)."""
|
||||
b = derive_cost(
|
||||
llm_input_tokens=500,
|
||||
llm_output_tokens=200,
|
||||
deepgram_audio_minutes=2.0,
|
||||
tts_characters=800,
|
||||
debrief_input_tokens=300,
|
||||
debrief_output_tokens=150,
|
||||
tts_provider="cartesia",
|
||||
)
|
||||
assert b.derived_cents > 0
|
||||
assert b.deepgram_audio_minutes == 2.0
|
||||
assert b.debrief_input_tokens == 300
|
||||
|
||||
|
||||
# ── Shift-end assist_cost_cents aggregation ─────────────────────────────────
|
||||
|
||||
|
||||
def test_shift_end_assist_cost_is_sum_of_per_turn_costs():
|
||||
"""AssistSession.assist_cost_cents is the sum of per-turn costs."""
|
||||
from server.assist.session import AssistSession
|
||||
from server.assist.context import AssistContext
|
||||
|
||||
# Construct an AssistSession without calling start() (we only test the
|
||||
# cost accumulator, not the DB lifecycle).
|
||||
ctx = AssistContext(
|
||||
system_prompt="",
|
||||
current_week=1,
|
||||
scenario_tag="refund",
|
||||
theta=0.0,
|
||||
coaching_focus="empathy",
|
||||
path_slug="customer_service",
|
||||
)
|
||||
session = AssistSession.__new__(AssistSession)
|
||||
session.assist_cost_cents = 0
|
||||
session.turn_count = 0
|
||||
session.guardrail_block_count = 0
|
||||
session.latency_metrics = None # not needed for this test
|
||||
|
||||
# Simulate 3 turns with per-turn costs.
|
||||
for turn_cost in [2, 3, 1]:
|
||||
session.add_assist_turn_cost(turn_cost)
|
||||
|
||||
assert session.assist_cost_cents == 6 # 2 + 3 + 1
|
||||
|
||||
|
||||
# ── check_c3_budget ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_c3_budget_within_budget_typical_usage():
|
||||
"""20 turns/shift × 20 shifts/month at ~$0.0005/turn → within budget.
|
||||
|
||||
Example from the plan: 400 turns/month at ~$0.0005/turn = ~$0.20/month —
|
||||
well under the $3 C-3 target.
|
||||
"""
|
||||
# cost_per_turn_cents = 0.05 cents ($0.0005) — gemma4:cloud pilot rate.
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=20,
|
||||
shifts_per_month=20,
|
||||
cost_per_turn_cents=0.05,
|
||||
)
|
||||
assert result["turns_per_month"] == 400
|
||||
# 400 * 0.05 / 100 = $0.20/month
|
||||
assert result["monthly_assist_cost"] < 1.0
|
||||
assert result["total_with_practice"] < C3_TARGET_USD
|
||||
assert result["within_budget"] is True
|
||||
assert result["flag"] is False
|
||||
assert result["c3_target"] == C3_TARGET_USD == 3.0
|
||||
|
||||
|
||||
def test_c3_budget_exceeds_with_high_usage():
|
||||
"""100 turns/shift × 30 shifts/month at higher cost → may exceed (flag=True).
|
||||
|
||||
3000 turns/month at 0.15 cents/turn = $4.50/month → exceeds $3.
|
||||
"""
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=100,
|
||||
shifts_per_month=30,
|
||||
cost_per_turn_cents=0.15,
|
||||
)
|
||||
assert result["turns_per_month"] == 3000
|
||||
# 3000 * 0.15 / 100 = $4.50/month → over $3
|
||||
assert result["monthly_assist_cost"] > C3_TARGET_USD
|
||||
assert result["within_budget"] is False
|
||||
assert result["flag"] is True # diagnostic flag (not enforced)
|
||||
|
||||
|
||||
def test_c3_budget_with_practice_cost():
|
||||
"""total_with_practice = assist + practice cost."""
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=20,
|
||||
shifts_per_month=20,
|
||||
cost_per_turn_cents=0.05,
|
||||
practice_cost_per_month_usd=1.5,
|
||||
)
|
||||
# assist = $0.20, practice = $1.50 → total = $1.70 (within $3)
|
||||
assert result["practice_cost_per_month"] == 1.5
|
||||
assert result["total_with_practice"] < C3_TARGET_USD
|
||||
assert result["within_budget"] is True
|
||||
|
||||
|
||||
def test_c3_budget_with_practice_cost_exceeds():
|
||||
"""Assist + practice cost exceeds $3 → flag=True (diagnostic)."""
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=50,
|
||||
shifts_per_month=30,
|
||||
cost_per_turn_cents=0.10,
|
||||
practice_cost_per_month_usd=2.0,
|
||||
)
|
||||
# assist = 1500 * 0.10 / 100 = $1.50, practice = $2.00 → total = $3.50
|
||||
assert result["total_with_practice"] > C3_TARGET_USD
|
||||
assert result["within_budget"] is False
|
||||
assert result["flag"] is True
|
||||
|
||||
|
||||
def test_c3_budget_zero_usage():
|
||||
"""0 turns → zero cost, within budget."""
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=0,
|
||||
shifts_per_month=0,
|
||||
cost_per_turn_cents=0.05,
|
||||
)
|
||||
assert result["turns_per_month"] == 0
|
||||
assert result["monthly_assist_cost"] == 0.0
|
||||
assert result["within_budget"] is True
|
||||
assert result["flag"] is False
|
||||
|
||||
|
||||
def test_c3_budget_is_diagnostic_not_enforced():
|
||||
"""D-012: the check is diagnostic (not enforced). flag=True does not raise.
|
||||
|
||||
The check_c3_budget() function returns a dict with flag=True when over
|
||||
budget, but does NOT raise an exception (D-012 — no enforced ceiling in
|
||||
the pilot). The caller logs the flag + continues.
|
||||
"""
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=1000,
|
||||
shifts_per_month=30,
|
||||
cost_per_turn_cents=1.0,
|
||||
)
|
||||
# 30000 turns * 1.0 cent / 100 = $300/month → way over $3
|
||||
assert result["flag"] is True
|
||||
assert result["within_budget"] is False
|
||||
# No exception raised — the function returns a dict (diagnostic, not enforced).
|
||||
|
||||
|
||||
def test_c3_target_is_3_usd():
|
||||
"""C-3 target is ≤ $3/active learner/month (C-3, D-012)."""
|
||||
assert C3_TARGET_USD == 3.0
|
||||
+278
-1
@@ -89,6 +89,65 @@ def test_cookie_secret_unset_generates_random(monkeypatch):
|
||||
assert len(kw["secret_key"]) >= 32
|
||||
|
||||
|
||||
def test_cookie_secret_short_logs_warning_accepted(monkeypatch, caplog):
|
||||
"""TASK-12-02 (P1+ #3): a secret <32 bytes logs a WARNING but is accepted.
|
||||
|
||||
A short non-empty secret (e.g., 'x') weakens the HMAC signature. The
|
||||
secret is still accepted (backward compat — pilot); post-pilot this
|
||||
should be a hard error. The WARNING is logged with remediation guidance.
|
||||
"""
|
||||
from loguru import logger as _logger
|
||||
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "short-secret") # 11 bytes < 32
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
|
||||
# Capture loguru warnings.
|
||||
msgs: list[str] = []
|
||||
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
|
||||
try:
|
||||
kw = get_session_middleware_kwargs()
|
||||
finally:
|
||||
_logger.remove(sink_id)
|
||||
# The short secret is accepted (backward compat — no hard error in pilot).
|
||||
assert kw["secret_key"] == "short-secret"
|
||||
# A WARNING about the short secret was logged.
|
||||
assert any("<32 bytes" in m for m in msgs), \
|
||||
"short PRAXIS_COOKIE_SECRET should log a <32 bytes WARNING"
|
||||
|
||||
|
||||
def test_cookie_secret_32_bytes_no_warning(monkeypatch, caplog):
|
||||
"""TASK-12-02: a secret >=32 bytes logs no <32 bytes warning."""
|
||||
from loguru import logger as _logger
|
||||
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "x" * 32) # exactly 32 bytes
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
|
||||
msgs: list[str] = []
|
||||
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
|
||||
try:
|
||||
kw = get_session_middleware_kwargs()
|
||||
finally:
|
||||
_logger.remove(sink_id)
|
||||
assert kw["secret_key"] == "x" * 32
|
||||
# No <32 bytes warning (the secret is exactly 32 bytes).
|
||||
assert not any("<32 bytes" in m for m in msgs), \
|
||||
"32-byte secret should NOT log a <32 bytes warning"
|
||||
|
||||
|
||||
def test_cookie_secret_long_no_warning(monkeypatch):
|
||||
"""TASK-12-02: a secret >32 bytes logs no warning."""
|
||||
from loguru import logger as _logger
|
||||
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "x" * 64) # 64 bytes
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
|
||||
msgs: list[str] = []
|
||||
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
|
||||
try:
|
||||
kw = get_session_middleware_kwargs()
|
||||
finally:
|
||||
_logger.remove(sink_id)
|
||||
assert kw["secret_key"] == "x" * 64
|
||||
assert not any("<32 bytes" in m for m in msgs)
|
||||
|
||||
|
||||
# ── current_operator dependency ─────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -307,4 +366,222 @@ def test_rate_limit_login_decorator():
|
||||
|
||||
|
||||
def test_limiter_is_in_memory():
|
||||
assert getattr(limiter, "_storage_uri", "memory://") == "memory://" or limiter._storage is not None
|
||||
assert getattr(limiter, "_storage_uri", "memory://") == "memory://" or limiter._storage is not None
|
||||
|
||||
|
||||
# ── TASK-12-04 (P1+ #1/#2/#5): argon2id offload + 429 mock test + audit log ──
|
||||
|
||||
|
||||
def test_login_argon2id_offloaded_to_thread():
|
||||
"""TASK-12-04 (P1+ #1): verify_password is offloaded to asyncio.to_thread.
|
||||
|
||||
The login handler should call verify_password via asyncio.to_thread (not
|
||||
directly) so the ~100-300ms argon2id hashing does not block the event loop.
|
||||
We verify by patching asyncio.to_thread to record the call.
|
||||
"""
|
||||
import asyncio as _asyncio
|
||||
|
||||
op = {
|
||||
"id": "44444444-4444-4444-4444-444444444444",
|
||||
"username": "erin",
|
||||
"display_name": "Erin",
|
||||
"role": "operator",
|
||||
"is_active": True,
|
||||
"password_hash": hash_password("pw"),
|
||||
}
|
||||
store = _mock_store(operator_row=op)
|
||||
store.get_operator_by_username = AsyncMock(return_value=op)
|
||||
store.update_last_login = AsyncMock()
|
||||
store.pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.execute = AsyncMock()
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
store.pool.acquire = MagicMock(return_value=cm)
|
||||
|
||||
to_thread_calls: list = []
|
||||
real_to_thread = _asyncio.to_thread
|
||||
|
||||
async def _spy_to_thread(func, *args, **kwargs):
|
||||
to_thread_calls.append((func, args, kwargs))
|
||||
return await real_to_thread(func, *args, **kwargs)
|
||||
|
||||
import server.auth.routes as _routes_mod
|
||||
orig = _routes_mod.asyncio.to_thread
|
||||
_routes_mod.asyncio.to_thread = _spy_to_thread
|
||||
try:
|
||||
app = _make_app_with_store(store)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/api/operator/login", json={"username": "erin", "password": "pw"})
|
||||
assert r.status_code == 200
|
||||
finally:
|
||||
_routes_mod.asyncio.to_thread = orig
|
||||
|
||||
# verify_password should have been called via asyncio.to_thread.
|
||||
assert to_thread_calls, "login should offload verify_password to asyncio.to_thread"
|
||||
func = to_thread_calls[0][0]
|
||||
assert func.__name__ == "verify_password", (
|
||||
f"expected verify_password offloaded, got {func.__name__}"
|
||||
)
|
||||
|
||||
|
||||
def test_login_rehash_offloaded_to_thread():
|
||||
"""TASK-12-04 (P1+ #1): hash_password (rehash) is also offloaded to thread."""
|
||||
from argon2 import PasswordHasher
|
||||
|
||||
weak_hasher = PasswordHasher(time_cost=1, memory_cost=8, parallelism=1)
|
||||
op = {
|
||||
"id": "55555555-5555-5555-5555-555555555555",
|
||||
"username": "frank",
|
||||
"display_name": "Frank",
|
||||
"role": "operator",
|
||||
"is_active": True,
|
||||
"password_hash": weak_hasher.hash("pw"),
|
||||
}
|
||||
store = _mock_store(operator_row=op)
|
||||
store.get_operator_by_username = AsyncMock(return_value=op)
|
||||
store.update_last_login = AsyncMock()
|
||||
store.pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.execute = AsyncMock()
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
store.pool.acquire = MagicMock(return_value=cm)
|
||||
|
||||
import asyncio as _asyncio
|
||||
import server.auth.routes as _routes_mod
|
||||
|
||||
to_thread_calls: list = []
|
||||
real_to_thread = _routes_mod.asyncio.to_thread
|
||||
|
||||
async def _spy_to_thread(func, *args, **kwargs):
|
||||
to_thread_calls.append((func, args, kwargs))
|
||||
return await real_to_thread(func, *args, **kwargs)
|
||||
|
||||
_routes_mod.asyncio.to_thread = _spy_to_thread
|
||||
try:
|
||||
app = _make_app_with_store(store)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/api/operator/login", json={"username": "frank", "password": "pw"})
|
||||
assert r.status_code == 200
|
||||
finally:
|
||||
_routes_mod.asyncio.to_thread = real_to_thread
|
||||
|
||||
# Both verify_password + hash_password should be offloaded.
|
||||
func_names = [c[0].__name__ for c in to_thread_calls]
|
||||
assert "verify_password" in func_names
|
||||
assert "hash_password" in func_names, "rehash should offload hash_password to thread"
|
||||
|
||||
|
||||
def test_login_rate_limit_429_after_5_attempts():
|
||||
"""TASK-12-04 (P1+ #2): mock-based 429 test — 6th login attempt → 429.
|
||||
|
||||
The full 6th-attempt→429 path is in the PG-requiring integration test; this
|
||||
adds a mock-based test for CI coverage without Postgres. slowapi's in-memory
|
||||
limiter tracks per-IP; 5/minute → 6th attempt gets 429.
|
||||
"""
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from slowapi.middleware import SlowAPIMiddleware
|
||||
from slowapi import _rate_limit_exceeded_handler
|
||||
|
||||
op = {
|
||||
"id": "66666666-6666-6666-6666-666666666666",
|
||||
"username": "grace",
|
||||
"display_name": "Grace",
|
||||
"role": "operator",
|
||||
"is_active": True,
|
||||
"password_hash": hash_password("pw"),
|
||||
}
|
||||
store = _mock_store(operator_row=op)
|
||||
store.get_operator_by_username = AsyncMock(return_value=op)
|
||||
store.update_last_login = AsyncMock()
|
||||
store.pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.execute = AsyncMock()
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
store.pool.acquire = MagicMock(return_value=cm)
|
||||
|
||||
app = _make_app_with_store(store)
|
||||
app.state.limiter = limiter
|
||||
app.add_middleware(SlowAPIMiddleware)
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
with TestClient(app) as client:
|
||||
# 5 attempts should succeed (or 401 for wrong password — both count).
|
||||
statuses: list[int] = []
|
||||
for _ in range(5):
|
||||
r = client.post(
|
||||
"/api/operator/login", json={"username": "grace", "password": "pw"}
|
||||
)
|
||||
statuses.append(r.status_code)
|
||||
# The 5 attempts should not be 429 (within the 5/minute limit).
|
||||
assert all(s != 429 for s in statuses), f"first 5 should not be 429: {statuses}"
|
||||
# 6th attempt → 429 (rate limit exceeded).
|
||||
r6 = client.post(
|
||||
"/api/operator/login", json={"username": "grace", "password": "pw"}
|
||||
)
|
||||
assert r6.status_code == 429, (
|
||||
f"6th login attempt should be rate-limited (429), got {r6.status_code}"
|
||||
)
|
||||
|
||||
|
||||
def test_credential_revocation_logs_audit_event():
|
||||
"""TASK-12-04 (P1+ #5): credential revocation logs operator + cred_id.
|
||||
|
||||
The revoke_credential endpoint should log an application-level audit event
|
||||
(no audit_log table — the log is sufficient for pilot per D-056).
|
||||
"""
|
||||
import logging as _logging
|
||||
|
||||
from server.operator.credentials import router as creds_router
|
||||
|
||||
op = {
|
||||
"id": "77777777-7777-7777-7777-777777777777",
|
||||
"username": "heidi",
|
||||
"display_name": "Heidi",
|
||||
"role": "operator",
|
||||
}
|
||||
store = MagicMock()
|
||||
store.get_credential = AsyncMock(return_value={"id": "cred-xyz", "status": "active"})
|
||||
store.set_credential_status = AsyncMock()
|
||||
|
||||
app = FastAPI()
|
||||
app.state.pg_store = store
|
||||
app.add_middleware(SessionMiddleware, secret_key="test-secret-1234567890abcdef")
|
||||
app.include_router(creds_router)
|
||||
# Stub auth.
|
||||
from server.auth.dependencies import current_operator
|
||||
from server.auth.models import Operator
|
||||
|
||||
async def _stub_op():
|
||||
return Operator(id=op["id"], username=op["username"],
|
||||
display_name=op["display_name"], role=op["role"])
|
||||
app.dependency_overrides[current_operator] = _stub_op
|
||||
|
||||
# Capture the audit log.
|
||||
cred_log = _logging.getLogger("server.operator.credentials")
|
||||
records: list[_logging.LogRecord] = []
|
||||
handler = _logging.Handler()
|
||||
handler.emit = records.append # type: ignore[method-assign]
|
||||
cred_log.addHandler(handler)
|
||||
cred_log.setLevel(_logging.INFO)
|
||||
try:
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/api/operator/credentials/cred-xyz/revoke")
|
||||
assert r.status_code == 200
|
||||
assert r.json()["status"] == "revoked"
|
||||
finally:
|
||||
cred_log.removeHandler(handler)
|
||||
|
||||
# The audit log should contain the operator id + cred_id.
|
||||
audit_msgs = [r.getMessage() for r in records if r.levelno >= _logging.INFO]
|
||||
assert any("credential revoked" in m for m in audit_msgs), \
|
||||
f"revocation should log 'credential revoked': {audit_msgs}"
|
||||
assert any("cred-xyz" in m for m in audit_msgs), \
|
||||
f"audit log should contain cred_id: {audit_msgs}"
|
||||
assert any(op["id"] in m for m in audit_msgs), \
|
||||
f"audit log should contain operator id: {audit_msgs}"
|
||||
@@ -0,0 +1,416 @@
|
||||
"""Cohort assist aggregation tests (TASK-10-03, D-062, D-063, REQ-NFR-ASSIST-04).
|
||||
|
||||
Tests the _aggregate_assist branch in server/cohort/aggregator.py with a
|
||||
mocked PgStore (no Postgres required). Verifies:
|
||||
- _aggregate_assist() upserts the 5 core assist metrics + p95 latency.
|
||||
- k-anon suppression: <10 distinct learners → suppressed.
|
||||
- Idempotent upsert: same session_outcome twice → same aggregate.
|
||||
- assist_guardrail_block_rate = blocks / turns.
|
||||
- The practice branch (_aggregate_practice) is unchanged (backward compat).
|
||||
- The dashboard endpoints return assist rows (cohort + failure-patterns).
|
||||
|
||||
D-063 (binding): assist does NOT update mastery. The _aggregate_assist branch
|
||||
computes NO mastery metrics (no rubric scores, no gate_open_rate).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from server.cohort.aggregator import (
|
||||
K_ANON_THRESHOLD,
|
||||
_aggregate_assist,
|
||||
_aggregate_practice,
|
||||
aggregate_session,
|
||||
)
|
||||
from server.cohort.hook import on_session_end
|
||||
|
||||
|
||||
def _mock_pg_store():
|
||||
store = MagicMock()
|
||||
store.upsert_cohort_aggregate = AsyncMock()
|
||||
return store
|
||||
|
||||
|
||||
def _assist_outcome(
|
||||
learner_ref: str,
|
||||
path: str = "customer_service",
|
||||
turn_count: int = 20,
|
||||
blocks: int = 2,
|
||||
p95_latency_ms: float | None = 580.0,
|
||||
cost_cents: int = 20,
|
||||
outcome: str = "completed",
|
||||
) -> dict:
|
||||
return {
|
||||
"learner_ref": learner_ref,
|
||||
"path": path,
|
||||
"scenario_id": f"assist:refund",
|
||||
"outcome": outcome,
|
||||
"session_type": "assist",
|
||||
"rubric_scores": [], # D-063: no rubric scores for assist
|
||||
"failure_mode": None,
|
||||
"branch_path": [],
|
||||
"assist_turn_count": turn_count,
|
||||
"guardrail_blocks": blocks,
|
||||
"assist_p95_latency_ms": p95_latency_ms,
|
||||
"assist_p50_latency_ms": 500.0,
|
||||
"assist_p99_latency_ms": 620.0,
|
||||
"assist_within_pilot": True,
|
||||
"assist_cost_cents": cost_cents,
|
||||
"timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _practice_outcome(learner_ref: str, path: str = "customer_service") -> dict:
|
||||
return {
|
||||
"learner_ref": learner_ref,
|
||||
"path": path,
|
||||
"scenario_id": f"{path}_v01",
|
||||
"outcome": "pass",
|
||||
"session_type": "practice",
|
||||
"rubric_scores": [{"criterion_id": "empathy", "score": 4.0}],
|
||||
"failure_mode": None,
|
||||
"branch_path": ["accept"],
|
||||
"timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
# ── Assist metrics upserted ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_assist_upserts_5_core_metrics():
|
||||
"""_aggregate_assist upserts the 5 core assist metrics (REQ-NFR-ASSIST-04)."""
|
||||
store = _mock_pg_store()
|
||||
await _aggregate_assist(store, _assist_outcome("learner-1"))
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
assert "assist_shifts_count" in metrics
|
||||
assert "assist_active_learners_count" in metrics
|
||||
assert "assist_turns_count" in metrics
|
||||
assert "assist_avg_turns_per_shift" in metrics
|
||||
assert "assist_guardrail_block_rate" in metrics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_assist_upserts_p95_latency():
|
||||
"""_aggregate_assist upserts assist_p95_latency_ms (D-072, TASK-09-01)."""
|
||||
store = _mock_pg_store()
|
||||
await _aggregate_assist(store, _assist_outcome("learner-1", p95_latency_ms=580.0))
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
assert "assist_p95_latency_ms" in metrics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_assist_upserts_avg_cost():
|
||||
"""_aggregate_assist upserts assist_avg_cost_per_shift (TASK-11-01)."""
|
||||
store = _mock_pg_store()
|
||||
await _aggregate_assist(store, _assist_outcome("learner-1", cost_cents=25))
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
assert "assist_avg_cost_per_shift" in metrics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_assist_no_mastery_metrics():
|
||||
"""D-063: _aggregate_assist computes NO mastery metrics."""
|
||||
store = _mock_pg_store()
|
||||
await _aggregate_assist(store, _assist_outcome("learner-1"))
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
# No mastery metrics should be present.
|
||||
assert "gate_open_rate" not in metrics
|
||||
assert "median_mastery_score" not in metrics
|
||||
assert not any(m.startswith("rubric_criterion_mean:") for m in metrics)
|
||||
# No practice metrics either (assist is a separate branch).
|
||||
assert "sessions_count" not in metrics
|
||||
|
||||
|
||||
# ── k-anonymity suppression ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assist_9_learners_suppressed():
|
||||
"""<10 distinct learners → all assist cells suppressed."""
|
||||
store = _mock_pg_store()
|
||||
for i in range(9):
|
||||
await _aggregate_assist(store, _assist_outcome(f"learner-{i}"))
|
||||
suppressed = [c for c in store.upsert_cohort_aggregate.call_args_list if c.args[6] is True]
|
||||
non_suppressed = [c for c in store.upsert_cohort_aggregate.call_args_list if c.args[6] is False]
|
||||
assert suppressed, "assist cells should be suppressed with <10 learners"
|
||||
assert not non_suppressed, "no assist cell should be non-suppressed with 9 learners"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assist_10_learners_not_suppressed():
|
||||
"""≥10 distinct learners → assist cells not suppressed."""
|
||||
store = _mock_pg_store()
|
||||
for i in range(10):
|
||||
await _aggregate_assist(store, _assist_outcome(f"learner-{i}"))
|
||||
non_suppressed = [c for c in store.upsert_cohort_aggregate.call_args_list if c.args[6] is False]
|
||||
assert non_suppressed, "assist cells should NOT be suppressed at 10 learners"
|
||||
for c in non_suppressed:
|
||||
assert c.args[4] is not None, "non-suppressed cell value must not be None"
|
||||
|
||||
|
||||
# ── Idempotent upsert ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assist_idempotent_same_outcome_twice():
|
||||
"""Re-running with the same outcome produces consistent upserts (idempotent)."""
|
||||
store = _mock_pg_store()
|
||||
outcome = _assist_outcome("learner-x")
|
||||
await _aggregate_assist(store, outcome)
|
||||
first_call_count = store.upsert_cohort_aggregate.call_count
|
||||
await _aggregate_assist(store, outcome)
|
||||
second_call_count = store.upsert_cohort_aggregate.call_count
|
||||
# Both runs produce upsert calls (the DB ON CONFLICT makes them idempotent).
|
||||
assert second_call_count >= first_call_count
|
||||
assert store.upsert_cohort_aggregate.called
|
||||
|
||||
|
||||
# ── assist_guardrail_block_rate = blocks / turns ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assist_guardrail_block_rate_computed():
|
||||
"""assist_guardrail_block_rate = blocks / turns (safety signal)."""
|
||||
store = _mock_pg_store()
|
||||
# 10 learners so the cell is not suppressed (we can read the value).
|
||||
for i in range(10):
|
||||
await _aggregate_assist(store, _assist_outcome(f"learner-{i}", turn_count=20, blocks=2))
|
||||
block_rate_cells = [
|
||||
c for c in store.upsert_cohort_aggregate.call_args_list
|
||||
if c.args[1] == "assist_guardrail_block_rate" and c.args[6] is False
|
||||
]
|
||||
assert block_rate_cells, "should have a non-suppressed assist_guardrail_block_rate cell"
|
||||
# The running mean of per-shift block rates (2/20 = 0.1) → ~0.1.
|
||||
rate = block_rate_cells[-1].args[4]
|
||||
assert rate is not None
|
||||
assert 0.05 <= rate <= 0.15 # ~0.1 with running-mean drift
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assist_zero_turns_block_rate_is_zero():
|
||||
"""0 turns → block_rate = 0.0 (no division by zero)."""
|
||||
store = _mock_pg_store()
|
||||
for i in range(10):
|
||||
await _aggregate_assist(store, _assist_outcome(f"learner-{i}", turn_count=0, blocks=0))
|
||||
block_rate_cells = [
|
||||
c for c in store.upsert_cohort_aggregate.call_args_list
|
||||
if c.args[1] == "assist_guardrail_block_rate" and c.args[6] is False
|
||||
]
|
||||
assert block_rate_cells
|
||||
rate = block_rate_cells[-1].args[4]
|
||||
assert rate == 0.0
|
||||
|
||||
|
||||
# ── Practice branch unchanged (backward compat) ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_session_dispatches_to_practice():
|
||||
"""aggregate_session with session_type='practice' → _aggregate_practice."""
|
||||
store = _mock_pg_store()
|
||||
await aggregate_session(store, _practice_outcome("learner-1"))
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
# Practice metrics should be present.
|
||||
assert "sessions_count" in metrics
|
||||
assert "active_learners_count" in metrics
|
||||
# Assist metrics should NOT be present (practice branch).
|
||||
assert "assist_shifts_count" not in metrics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_session_dispatches_to_assist():
|
||||
"""aggregate_session with session_type='assist' → _aggregate_assist."""
|
||||
store = _mock_pg_store()
|
||||
await aggregate_session(store, _assist_outcome("learner-1"))
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
assert "assist_shifts_count" in metrics
|
||||
assert "sessions_count" not in metrics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aggregate_session_default_is_practice():
|
||||
"""aggregate_session with no session_type → practice (backward compat)."""
|
||||
store = _mock_pg_store()
|
||||
outcome = _practice_outcome("learner-1")
|
||||
outcome.pop("session_type") # omit session_type → default practice
|
||||
await aggregate_session(store, outcome)
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
assert "sessions_count" in metrics
|
||||
assert "assist_shifts_count" not in metrics
|
||||
|
||||
|
||||
# ── No PII in assist upsert calls ───────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assist_no_pii_in_upsert_calls():
|
||||
"""No raw learner_ref leaks into assist aggregate cell args (D-031)."""
|
||||
store = _mock_pg_store()
|
||||
await _aggregate_assist(store, _assist_outcome("learner-sensitive-id-1234"))
|
||||
for c in store.upsert_cohort_aggregate.call_args_list:
|
||||
for arg in c.args:
|
||||
assert "learner-sensitive-id-1234" not in str(arg), \
|
||||
"raw learner_ref must not leak into assist aggregate cell args"
|
||||
assert isinstance(c.args[5], int) # cell_count is an int
|
||||
|
||||
|
||||
# ── Hook dispatches assist correctly ───────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hook_dispatches_assist_session():
|
||||
"""on_session_end with session_type='assist' → _aggregate_assist (no error)."""
|
||||
store = _mock_pg_store()
|
||||
await on_session_end(store, _assist_outcome("learner-1"))
|
||||
assert store.upsert_cohort_aggregate.called
|
||||
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
|
||||
assert "assist_shifts_count" in metrics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hook_assist_no_postgres_is_noop():
|
||||
"""on_session_end with no Postgres → no-op (assist hook)."""
|
||||
await on_session_end(None, _assist_outcome("learner-1"))
|
||||
|
||||
|
||||
# ── Dashboard endpoints return assist rows ──────────────────────────────────
|
||||
|
||||
|
||||
def _make_app_with_assist_rows(rows: list[dict]):
|
||||
"""Build a minimal FastAPI app with the cohort + failure-patterns routers
|
||||
+ a mocked pg_store returning `rows`."""
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from server.operator.cohort import router as cohort_router
|
||||
from server.operator.failure_patterns import router as failure_router
|
||||
|
||||
app = FastAPI()
|
||||
pg_store = MagicMock()
|
||||
pg_store.pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.fetch = AsyncMock(return_value=rows)
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
pg_store.pool.acquire = MagicMock(return_value=cm)
|
||||
app.state.pg_store = pg_store
|
||||
# Bypass auth for these tests by stubbing current_operator.
|
||||
from server.auth.dependencies import current_operator
|
||||
from server.auth.models import Operator
|
||||
|
||||
async def _stub_op():
|
||||
return Operator(id="op-1", username="tester", display_name="T", role="operator")
|
||||
app.dependency_overrides[current_operator] = _stub_op
|
||||
app.include_router(cohort_router)
|
||||
app.include_router(failure_router)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _assist_metric_row(metric: str, value: float, suppressed: bool = False) -> dict:
|
||||
return {
|
||||
"path": "customer_service",
|
||||
"metric": metric,
|
||||
"window_start": _dt.date.today() - _dt.timedelta(days=6),
|
||||
"window_end": _dt.date.today(),
|
||||
"value": value if not suppressed else None,
|
||||
"cell_count": 12,
|
||||
"cell_suppressed": suppressed,
|
||||
"updated_at": _dt.datetime.now(_dt.timezone.utc),
|
||||
}
|
||||
|
||||
|
||||
def test_cohort_endpoint_returns_assist_rows():
|
||||
"""GET /api/operator/cohort returns assist_shifts_count + assist_turns_count."""
|
||||
rows = [
|
||||
_assist_metric_row("sessions_count", 15.0),
|
||||
_assist_metric_row("active_learners_count", 12.0),
|
||||
_assist_metric_row("assist_shifts_count", 8.0),
|
||||
_assist_metric_row("assist_turns_count", 160.0),
|
||||
]
|
||||
client = _make_app_with_assist_rows(rows)
|
||||
r = client.get("/api/operator/cohort")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
metrics = {c["metric"] for v in data["views"] for c in v["metrics"]}
|
||||
assert "assist_shifts_count" in metrics
|
||||
assert "assist_turns_count" in metrics
|
||||
assert "sessions_count" in metrics # practice still present
|
||||
|
||||
|
||||
def test_failure_patterns_endpoint_returns_guardrail_block_rate():
|
||||
"""GET /api/operator/failure-patterns returns assist_guardrail_block_rate."""
|
||||
rows = [
|
||||
_assist_metric_row("failure_mode:missed_apology", 3.0),
|
||||
_assist_metric_row("branch:escalate", 5.0),
|
||||
_assist_metric_row("assist_guardrail_block_rate", 0.08),
|
||||
]
|
||||
client = _make_app_with_assist_rows(rows)
|
||||
r = client.get("/api/operator/failure-patterns")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
metrics = {c["metric"] for v in data["views"] for c in v["metrics"]}
|
||||
assert "assist_guardrail_block_rate" in metrics
|
||||
assert "failure_mode:missed_apology" in metrics # practice failure patterns still present
|
||||
|
||||
|
||||
def test_mastery_endpoint_excludes_assist_metrics():
|
||||
"""D-063: GET /api/operator/mastery does NOT return assist metrics."""
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from server.auth.dependencies import current_operator
|
||||
from server.auth.models import Operator
|
||||
from server.operator.mastery import router as mastery_router
|
||||
|
||||
rows = [
|
||||
_assist_metric_row("gate_open_rate", 0.5),
|
||||
_assist_metric_row("median_mastery_score", 3.8),
|
||||
_assist_metric_row("assist_shifts_count", 8.0), # should be EXCLUDED
|
||||
_assist_metric_row("assist_guardrail_block_rate", 0.08), # EXCLUDED
|
||||
]
|
||||
app = FastAPI()
|
||||
pg_store = MagicMock()
|
||||
pg_store.pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.fetch = AsyncMock(return_value=rows)
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
pg_store.pool.acquire = MagicMock(return_value=cm)
|
||||
app.state.pg_store = pg_store
|
||||
|
||||
async def _stub_op():
|
||||
return Operator(id="op-1", username="tester", display_name="T", role="operator")
|
||||
app.dependency_overrides[current_operator] = _stub_op
|
||||
app.include_router(mastery_router)
|
||||
client = TestClient(app)
|
||||
r = client.get("/api/operator/mastery")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
metrics = {c["metric"] for v in data["views"] for c in v["metrics"]}
|
||||
assert "gate_open_rate" in metrics
|
||||
assert "median_mastery_score" in metrics
|
||||
# D-063: assist metrics must NOT appear in the mastery view.
|
||||
assert "assist_shifts_count" not in metrics
|
||||
assert "assist_guardrail_block_rate" not in metrics
|
||||
|
||||
|
||||
def test_cohort_endpoint_suppressed_assist_cells():
|
||||
"""Suppressed assist cells have value=null + cell_suppressed=true (k-anon)."""
|
||||
rows = [
|
||||
_assist_metric_row("assist_shifts_count", 0.0, suppressed=True),
|
||||
_assist_metric_row("assist_turns_count", 0.0, suppressed=True),
|
||||
]
|
||||
client = _make_app_with_assist_rows(rows)
|
||||
r = client.get("/api/operator/cohort")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
for v in data["views"]:
|
||||
for c in v["metrics"]:
|
||||
if c["metric"] in ("assist_shifts_count", "assist_turns_count"):
|
||||
assert c["cell_suppressed"] is True
|
||||
assert c["value"] is None
|
||||
@@ -44,6 +44,71 @@ def test_seconds_until_next_03_ct_exactly_03_rolls_to_tomorrow():
|
||||
assert secs >= 86390 # ~24h
|
||||
|
||||
|
||||
# ── TASK-12-04 (P1+ #6): zoneinfo DST-aware scheduler ───────────────────────
|
||||
|
||||
|
||||
def test_nightly_scheduler_uses_zoneinfo_america_winnipeg():
|
||||
"""TASK-12-04 (P1+ #6): CT is zoneinfo.ZoneInfo('America/Winnipeg') (DST-aware).
|
||||
|
||||
The v0.4 fixed UTC-5 offset is replaced with ZoneInfo("America/Winnipeg")
|
||||
which correctly handles CST (UTC-6) in winter + CDT (UTC-5) in summer.
|
||||
"""
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
assert isinstance(CT, ZoneInfo), f"CT should be a ZoneInfo, got {type(CT)}"
|
||||
assert str(CT) == "America/Winnipeg", f"CT should be America/Winnipeg, got {CT}"
|
||||
|
||||
|
||||
def test_nightly_scheduler_dst_summer_cdt():
|
||||
"""TASK-12-04 (P1+ #6): summer (August) → CDT (UTC-5).
|
||||
|
||||
In August 2026, America/Winnipeg is on CDT (UTC-5). A 01:00 local time
|
||||
should be 06:00 UTC. The scheduler computes seconds until 03:00 local.
|
||||
"""
|
||||
# 2026-08-04 is summer → CDT (UTC-5).
|
||||
now_local = _dt.datetime(2026, 8, 4, 1, 0, tzinfo=CT)
|
||||
# 01:00 CDT = 06:00 UTC.
|
||||
assert now_local.utcoffset() == _dt.timedelta(hours=-5), (
|
||||
f"August should be CDT (UTC-5), got offset {now_local.utcoffset()}"
|
||||
)
|
||||
secs = seconds_until_next_03_ct(now_local)
|
||||
# 01:00 → 03:00 = 2h = 7200s.
|
||||
assert 7190 <= secs <= 7200
|
||||
|
||||
|
||||
def test_nightly_scheduler_dst_winter_cst():
|
||||
"""TASK-12-04 (P1+ #6): winter (January) → CST (UTC-6).
|
||||
|
||||
In January 2027, America/Winnipeg is on CST (UTC-6). A 01:00 local time
|
||||
should be 07:00 UTC. The v0.4 fixed UTC-5 offset would have been wrong
|
||||
by 1h in winter; the ZoneInfo correctly handles the DST transition.
|
||||
"""
|
||||
# 2027-01-15 is winter → CST (UTC-6).
|
||||
now_local = _dt.datetime(2027, 1, 15, 1, 0, tzinfo=CT)
|
||||
assert now_local.utcoffset() == _dt.timedelta(hours=-6), (
|
||||
f"January should be CST (UTC-6), got offset {now_local.utcoffset()}"
|
||||
)
|
||||
secs = seconds_until_next_03_ct(now_local)
|
||||
# 01:00 → 03:00 = 2h = 7200s.
|
||||
assert 7190 <= secs <= 7200
|
||||
|
||||
|
||||
def test_nightly_scheduler_dst_transition_spring_2027():
|
||||
"""TASK-12-04 (P1+ #6): DST spring forward — 2027-03-14 02:00 → 03:00 CDT.
|
||||
|
||||
On 2027-03-14, DST springs forward at 02:00 local (CST → CDT). The ZoneInfo
|
||||
correctly handles the transition (the 02:00 hour is skipped). The scheduler
|
||||
should still compute a valid seconds-until-03:00.
|
||||
"""
|
||||
# 2027-03-14 01:00 CST (before spring forward) → 03:00 CDT is 1h later
|
||||
# (the 02:00 hour is skipped → 01:59 CST → 03:00 CDT).
|
||||
now_local = _dt.datetime(2027, 3, 14, 1, 0, tzinfo=CT)
|
||||
secs = seconds_until_next_03_ct(now_local)
|
||||
# 01:00 CST → 03:00 CDT is 1h (the 02:00 hour is skipped).
|
||||
# The exact value depends on the DST transition; assert it's ≤ 2h.
|
||||
assert 0 < secs <= 7200, f"spring-forward seconds should be <= 2h, got {secs}"
|
||||
|
||||
|
||||
# ── Reconciliation recomputes all windows ──────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Mock-based tests for set_credential_status enum + f-string SQL fix
|
||||
(TASK-12-03, P1+ #4/#8 from v0.4 REVIEW).
|
||||
|
||||
These tests do NOT require Postgres (they use a mock asyncpg pool). They
|
||||
verify:
|
||||
- 'revoked' uses a parameterized query with revoked_at=now() (no f-string).
|
||||
- 'active' clears revoked_at=NULL (re-activation).
|
||||
- Invalid status → ValueError (enum validation — P1+ #4).
|
||||
- No f-string interpolation in the SQL (P1+ #8 code smell fix).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from db.pg_store import PgStore
|
||||
|
||||
|
||||
def _mock_pool_with_conn():
|
||||
"""Build a mock asyncpg pool + conn that records execute() calls."""
|
||||
pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.execute = AsyncMock()
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
pool.acquire = MagicMock(return_value=cm)
|
||||
return pool, conn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_credential_status_revoked_uses_parameterized_query():
|
||||
"""TASK-12-03 (P1+ #8): 'revoked' uses a parameterized query (no f-string)."""
|
||||
pool, conn = _mock_pool_with_conn()
|
||||
store = PgStore(pool)
|
||||
await store.set_credential_status("cred-1", "revoked")
|
||||
# Exactly one execute call.
|
||||
assert conn.execute.await_count == 1
|
||||
sql, status_arg, cred_arg = conn.execute.await_args.args
|
||||
# No f-string interpolation — the SQL is a literal with $1, $2.
|
||||
assert "revoked_at = now()" in sql
|
||||
assert "$1" in sql and "$2" in sql
|
||||
assert status_arg == "revoked"
|
||||
assert cred_arg == "cred-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_credential_status_active_clears_revoked_at():
|
||||
"""TASK-12-03: 'active' clears revoked_at=NULL (re-activation)."""
|
||||
pool, conn = _mock_pool_with_conn()
|
||||
store = PgStore(pool)
|
||||
await store.set_credential_status("cred-1", "active")
|
||||
assert conn.execute.await_count == 1
|
||||
sql, status_arg, cred_arg = conn.execute.await_args.args
|
||||
assert "revoked_at = NULL" in sql
|
||||
assert status_arg == "active"
|
||||
assert cred_arg == "cred-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_credential_status_invalid_raises_value_error():
|
||||
"""TASK-12-03 (P1+ #4): invalid status → ValueError (enum validation)."""
|
||||
pool, conn = _mock_pool_with_conn()
|
||||
store = PgStore(pool)
|
||||
for bad_status in ("pending", "suspended", "deleted", "", "REVOKED", "active "):
|
||||
with pytest.raises(ValueError, match="Invalid credential status"):
|
||||
await store.set_credential_status("cred-1", bad_status)
|
||||
# No execute call should have been made (validation happens before the query).
|
||||
assert conn.execute.await_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_credential_status_no_fstring_in_sql():
|
||||
"""TASK-12-03 (P1+ #8): no f-string interpolation in the SQL (code smell fix).
|
||||
|
||||
The SQL must be a literal string (no f-string {extra} interpolation). The
|
||||
status + cred_id are bound parameters ($1, $2), not interpolated.
|
||||
"""
|
||||
pool, conn = _mock_pool_with_conn()
|
||||
store = PgStore(pool)
|
||||
await store.set_credential_status("cred-1", "revoked")
|
||||
sql = conn.execute.await_args.args[0]
|
||||
# The SQL must NOT contain an f-string-interpolated extra clause. The old
|
||||
# code had f"UPDATE ... SET status = $1{extra} WHERE id = $2" where extra
|
||||
# was ', revoked_at = now()' or ''. The new code has two explicit queries.
|
||||
# Verify the SQL is a literal (no {extra}-style interpolation artifacts).
|
||||
assert "{extra}" not in sql
|
||||
assert "UPDATE issued_credentials SET status = $1, revoked_at = now()" in sql
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_credential_status_revoked_then_active():
|
||||
"""TASK-12-03: revoke then re-activate (active clears revoked_at)."""
|
||||
pool, conn = _mock_pool_with_conn()
|
||||
store = PgStore(pool)
|
||||
# Revoke.
|
||||
await store.set_credential_status("cred-1", "revoked")
|
||||
revoke_sql = conn.execute.await_args.args[0]
|
||||
assert "revoked_at = now()" in revoke_sql
|
||||
# Re-activate (active clears revoked_at).
|
||||
conn.execute.reset_mock()
|
||||
await store.set_credential_status("cred-1", "active")
|
||||
active_sql = conn.execute.await_args.args[0]
|
||||
assert "revoked_at = NULL" in active_sql
|
||||
@@ -0,0 +1,290 @@
|
||||
"""NFR measurement tests (TASK-09-03, REQ-NFR-ASSIST-01, REQ-IDEATE-04, D-072).
|
||||
|
||||
Tests the measurement infrastructure (NOT the actual latency — that's a Phase-1
|
||||
live measurement, not a CI test):
|
||||
- AssistLatencyMetrics: p95/p50/p99 computed correctly from mock records.
|
||||
D-072: within_target = (p95 < 600), within_pilot = (p95 <= 650).
|
||||
- GuardrailMetrics: false_positive_rate on the tuning corpus, false_negative_rate
|
||||
on the direct-answer corpus, nightly_trend on mock turns.
|
||||
|
||||
D-072 binding: the pilot tolerance is ≤ 650ms. The target is < 600ms (C-8). The
|
||||
test ASSERTS that the measurement infrastructure works (percentiles + flags),
|
||||
not that the actual latency is under budget.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as _dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from server.assist.guardrail_metrics import GuardrailMetrics
|
||||
from server.assist.latency_metrics import (
|
||||
PILOT_TOLERANCE_MS,
|
||||
TARGET_MS,
|
||||
AssistLatencyMetrics,
|
||||
)
|
||||
from server.latency import LatencyRecord
|
||||
|
||||
|
||||
# ── AssistLatencyMetrics ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _record(e2e_ms: float) -> LatencyRecord:
|
||||
"""Build a LatencyRecord with a specific e2e_asr_to_tts_ms value."""
|
||||
# e2e = tts_first_audio_ms - transcript_ready_ms. Use a non-zero base
|
||||
# because LatencyRecord.e2e_asr_to_tts_ms guards on truthiness (0.0 is falsy).
|
||||
base = 100.0
|
||||
return LatencyRecord(
|
||||
transcript_ready_ms=base,
|
||||
tts_first_audio_ms=base + e2e_ms,
|
||||
)
|
||||
|
||||
|
||||
def test_latency_empty_returns_none():
|
||||
m = AssistLatencyMetrics()
|
||||
assert m.p50() is None
|
||||
assert m.p95() is None
|
||||
assert m.p99() is None
|
||||
s = m.summary()
|
||||
assert s["count"] == 0
|
||||
assert s["p95"] is None
|
||||
assert s["within_target"] is False # no records → not within target
|
||||
assert s["within_pilot"] is False
|
||||
|
||||
|
||||
def test_latency_p95_p50_p99_computed():
|
||||
"""100 mock records: some <600ms, some 600-650ms, some >650ms.
|
||||
|
||||
Verifies p50/p95/p99 are computed correctly + the within_target/within_pilot
|
||||
flags reflect the p95 against the D-072 thresholds.
|
||||
"""
|
||||
m = AssistLatencyMetrics()
|
||||
# 80 records < 600ms (within target), 15 records 600-650ms (within pilot),
|
||||
# 5 records > 650ms (over pilot tolerance).
|
||||
for i in range(80):
|
||||
m.record(_record(500.0 + i)) # 500..579ms
|
||||
for i in range(15):
|
||||
m.record(_record(610.0 + i)) # 610..624ms
|
||||
for i in range(5):
|
||||
m.record(_record(700.0 + i)) # 700..704ms
|
||||
|
||||
s = m.summary()
|
||||
assert s["count"] == 100
|
||||
assert s["p50"] is not None
|
||||
assert s["p95"] is not None
|
||||
assert s["p99"] is not None
|
||||
# p50 should be in the < 600ms range (median of the 80 < 600ms records).
|
||||
assert s["p50"] < 600.0
|
||||
# p95: nearest-rank index = ceil(0.95 * 100) - 1 = 94 (0-indexed) → the 95th
|
||||
# sorted value. 80 records are 500..579, 15 are 610..624, 5 are 700..704.
|
||||
# Sorted: [500..579 (80), 610..624 (15), 700..704 (5)]. Index 94 → 610..624
|
||||
# range (index 80..94 = the 610..624 set; index 94 = 624.0).
|
||||
assert 610.0 <= s["p95"] <= 625.0
|
||||
# p99: index = ceil(0.99 * 100) - 1 = 98 → the 99th sorted value (700..704).
|
||||
assert s["p99"] >= 700.0
|
||||
# D-072: within_target = (p95 < 600). p95 is ~624 → not within target.
|
||||
assert s["within_target"] is False
|
||||
# D-072: within_pilot = (p95 <= 650). p95 is ~624 → within pilot.
|
||||
assert s["within_pilot"] is True
|
||||
# D-072 thresholds documented in the summary.
|
||||
assert s["target_ms"] == TARGET_MS == 600
|
||||
assert s["pilot_tolerance_ms"] == PILOT_TOLERANCE_MS == 650
|
||||
|
||||
|
||||
def test_latency_within_target_when_p95_under_600():
|
||||
"""All records < 600ms → within_target=True, within_pilot=True."""
|
||||
m = AssistLatencyMetrics()
|
||||
for i in range(20):
|
||||
m.record(_record(400.0 + i)) # 400..419ms
|
||||
s = m.summary()
|
||||
assert s["p95"] < 600.0
|
||||
assert s["within_target"] is True
|
||||
assert s["within_pilot"] is True
|
||||
|
||||
|
||||
def test_latency_over_pilot_when_p95_over_650():
|
||||
"""All records > 650ms → within_target=False, within_pilot=False."""
|
||||
m = AssistLatencyMetrics()
|
||||
for i in range(20):
|
||||
m.record(_record(700.0 + i)) # 700..719ms
|
||||
s = m.summary()
|
||||
assert s["p95"] > 650.0
|
||||
assert s["within_target"] is False
|
||||
assert s["within_pilot"] is False
|
||||
|
||||
|
||||
def test_latency_pilot_boundary_exactly_650():
|
||||
"""D-072 boundary: p95 == 650 → within_pilot=True (≤ is inclusive)."""
|
||||
m = AssistLatencyMetrics()
|
||||
# 20 records all exactly 650ms → p95 = 650.0
|
||||
for _ in range(20):
|
||||
m.record(_record(650.0))
|
||||
s = m.summary()
|
||||
assert s["p95"] == 650.0
|
||||
assert s["within_pilot"] is True # ≤ 650 (inclusive)
|
||||
assert s["within_target"] is False # < 600 (strict)
|
||||
|
||||
|
||||
def test_latency_d072_thresholds_documented():
|
||||
"""D-072: the pilot tolerance (≤650ms) + target (<600ms) are documented."""
|
||||
assert TARGET_MS == 600
|
||||
assert PILOT_TOLERANCE_MS == 650
|
||||
assert PILOT_TOLERANCE_MS > TARGET_MS # pilot tolerance is more lenient
|
||||
|
||||
|
||||
# ── GuardrailMetrics ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_fp_rate_on_coaching_corpus():
|
||||
"""FP rate on the tuning corpus < 5% (REQ-IDEATE-04 target)."""
|
||||
gm = GuardrailMetrics()
|
||||
rate, mis, total = await gm.false_positive_rate()
|
||||
print(f"\n[nfr] guardrail FP rate: {rate:.1%} ({mis}/{total})")
|
||||
assert rate < 0.05, (
|
||||
f"guardrail FP rate {rate:.1%} exceeds 5% target — the regex is "
|
||||
f"over-matching coaching responses. {mis}/{total} blocked."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_fn_rate_on_direct_corpus():
|
||||
"""FN rate on the direct-answer corpus < 5% (REQ-IDEATE-04 target)."""
|
||||
gm = GuardrailMetrics()
|
||||
rate, mis, total = await gm.false_negative_rate()
|
||||
print(f"\n[nfr] guardrail FN rate: {rate:.1%} ({mis}/{total})")
|
||||
assert rate < 0.05, (
|
||||
f"guardrail FN rate {rate:.1%} exceeds 5% target — the regex is "
|
||||
f"under-matching direct answers. {mis}/{total} allowed."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_adversarial_fn_measured():
|
||||
"""Adversarial FN rate measured + reported (G-067 — ≤ 20% pilot threshold).
|
||||
|
||||
This test does NOT assert the 5% target (the adversarial set is the
|
||||
residual-risk set, not the tuning target). It asserts the measurement
|
||||
infrastructure works + the rate is within the G-067 pilot threshold (≤ 20%).
|
||||
"""
|
||||
gm = GuardrailMetrics()
|
||||
rate, mis, total = await gm.adversarial_false_negative_rate()
|
||||
print(f"\n[nfr] guardrail adversarial FN rate: {rate:.1%} ({mis}/{total})")
|
||||
# G-067: ≤ 20% pilot threshold (the binding contract from GRILL-v0.5).
|
||||
assert rate <= 0.20, (
|
||||
f"adversarial FN rate {rate:.1%} exceeds G-067 ≤20% threshold — "
|
||||
f"re-tune the regex or escalate. {mis}/{total} slipped through."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_nightly_trend_on_mock_turns(tmp_path: Path):
|
||||
"""nightly_trend() samples 24h of assist turns + reports fn_candidates.
|
||||
|
||||
Seeds a temp SQLite store with assist turns (some coaching, some with
|
||||
direct-answer heuristic patterns) + verifies the nightly trend detects
|
||||
fn_candidates.
|
||||
"""
|
||||
from db.migrate import apply_migrations
|
||||
from db.store import PraxisStore
|
||||
|
||||
db = tmp_path / "test_nfr_nightly.db"
|
||||
apply_migrations(db)
|
||||
store = PraxisStore(db)
|
||||
await store.init()
|
||||
|
||||
# Seed an assist session + turns.
|
||||
session_id = await store.start_session_typed(
|
||||
"learner-1", "assist:refund", session_type="assist"
|
||||
)
|
||||
# Turn 1: a coaching response (allowed, no fn_candidate).
|
||||
await store.log_turn_with_verdict(
|
||||
session_id, 0, role="assistant",
|
||||
asr_text="customer wants refund",
|
||||
tts_text="What do you think the customer needs right now?",
|
||||
latency_ms=580.0,
|
||||
guardrail_verdict_json=json.dumps({"allowed": True, "category": "coaching"}),
|
||||
)
|
||||
# Turn 2: a direct-answer response that slipped past the guardrail
|
||||
# (allowed=True in the verdict, but the heuristic catches it).
|
||||
await store.log_turn_with_verdict(
|
||||
session_id, 1, role="assistant",
|
||||
asr_text="what should I say",
|
||||
tts_text="You should say: I'm sorry, here's a refund.",
|
||||
latency_ms=590.0,
|
||||
guardrail_verdict_json=json.dumps({"allowed": True, "category": "coaching"}),
|
||||
)
|
||||
# Turn 3: a blocked response (guardrail caught it).
|
||||
await store.log_turn_with_verdict(
|
||||
session_id, 2, role="assistant",
|
||||
asr_text="help me",
|
||||
tts_text="Tell the customer: we will issue a full refund now.",
|
||||
latency_ms=570.0,
|
||||
guardrail_verdict_json=json.dumps({"allowed": False, "category": "blocked_direct_script"}),
|
||||
)
|
||||
|
||||
gm = GuardrailMetrics()
|
||||
trend = await gm.nightly_trend(store)
|
||||
|
||||
assert trend["total_turns"] == 3
|
||||
assert trend["blocked"] == 1
|
||||
# Turn 2 should be flagged as an fn_candidate. The guardrail re-check may
|
||||
# catch it as a regression (it now blocks what it previously allowed) OR
|
||||
# the heuristic may catch it as a direct-answer pattern. Either way, it
|
||||
# must appear in fn_candidates.
|
||||
assert len(trend["fn_candidates"]) >= 1
|
||||
seqs = [c.get("turn_seq") for c in trend["fn_candidates"]]
|
||||
assert 1 in seqs, "turn 2 (direct-answer that slipped past) must be flagged"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_nightly_trend_empty_store(tmp_path: Path):
|
||||
"""nightly_trend() on an empty store returns zeros + no fn_candidates."""
|
||||
from db.migrate import apply_migrations
|
||||
from db.store import PraxisStore
|
||||
|
||||
db = tmp_path / "test_nfr_nightly_empty.db"
|
||||
apply_migrations(db)
|
||||
store = PraxisStore(db)
|
||||
await store.init()
|
||||
|
||||
gm = GuardrailMetrics()
|
||||
trend = await gm.nightly_trend(store)
|
||||
assert trend["total_turns"] == 0
|
||||
assert trend["blocked"] == 0
|
||||
assert trend["fn_candidates"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_nightly_trend_excludes_practice_turns(tmp_path: Path):
|
||||
"""nightly_trend() only samples assist turns (not practice turns)."""
|
||||
from db.migrate import apply_migrations
|
||||
from db.store import PraxisStore
|
||||
|
||||
db = tmp_path / "test_nfr_nightly_practice.db"
|
||||
apply_migrations(db)
|
||||
store = PraxisStore(db)
|
||||
await store.init()
|
||||
|
||||
# Seed a practice session (NOT assist) with a turn.
|
||||
practice_id = await store.start_session_typed(
|
||||
"learner-1", "cs_refund_ca_v01", session_type="practice"
|
||||
)
|
||||
await store.log_turn_with_verdict(
|
||||
practice_id, 0, role="assistant",
|
||||
asr_text="hello",
|
||||
tts_text="You should say sorry.",
|
||||
latency_ms=500.0,
|
||||
guardrail_verdict_json=json.dumps({"allowed": True, "category": "coaching"}),
|
||||
)
|
||||
|
||||
gm = GuardrailMetrics()
|
||||
trend = await gm.nightly_trend(store)
|
||||
# Practice turns must NOT appear in the assist nightly trend.
|
||||
assert trend["total_turns"] == 0
|
||||
assert trend["fn_candidates"] == []
|
||||
@@ -0,0 +1,353 @@
|
||||
"""P2 integration test — assist aggregation → endpoint → cost → NFR (TASK-12-05).
|
||||
|
||||
Requires Postgres (skips if PRAXIS_PG_DSN not set). End-to-end P2 integration:
|
||||
1. Seed 12 mock assist shifts (12 distinct learners — above k-anon threshold).
|
||||
2. Run the aggregation hook for each → cohort_aggregates populated with assist metrics.
|
||||
3. GET /api/operator/cohort (with auth cookie) → returns assist volume (non-suppressed).
|
||||
4. GET /api/operator/failure-patterns → returns assist_guardrail_block_rate.
|
||||
5. Seed 5 more assist shifts from 5 NEW distinct learners for a different path →
|
||||
GET /api/operator/cohort for that path → suppressed cells (5 < 10).
|
||||
6. Verify assist_p95_latency_ms is in the aggregates.
|
||||
7. Verify assist_cost_cents is in the session_outcome.
|
||||
8. Verify the C-3 budget check runs at shift-end.
|
||||
9. Verify the tech-debt fixes: aggregation cache survives restart (mock),
|
||||
cookie-secret warning, credential status enum, argon2id offloaded.
|
||||
|
||||
G-038 differencing-attack e2e: k-anon threshold enforced (12 not suppressed,
|
||||
5 suppressed). No per-learner data in any response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as _dt
|
||||
import os
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("PRAXIS_PG_DSN"),
|
||||
reason="PRAXIS_PG_DSN not set — P2 assist integration tests skipped.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def pg_pool():
|
||||
import asyncpg
|
||||
|
||||
pool = await asyncpg.create_pool(
|
||||
dsn=os.environ["PRAXIS_PG_DSN"], min_size=1, max_size=5, command_timeout=10,
|
||||
)
|
||||
try:
|
||||
yield pool
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def pg_store(pg_pool):
|
||||
from db.pg_migrate import apply_pg_migrations
|
||||
from db.pg_store import PgStore
|
||||
|
||||
await apply_pg_migrations(pg_pool)
|
||||
# Clean cohort_aggregates + operators for an isolated run.
|
||||
async with pg_pool.acquire() as conn:
|
||||
await conn.execute("TRUNCATE cohort_aggregates, operators, issued_credentials")
|
||||
return PgStore(pg_pool)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def authed_client(pg_store):
|
||||
"""A TestClient with auth + the operator routers wired to pg_store."""
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from server.auth.dependencies import current_operator
|
||||
from server.auth.models import Operator
|
||||
from server.operator.cohort import router as cohort_router
|
||||
from server.operator.failure_patterns import router as failure_router
|
||||
from server.operator.mastery import router as mastery_router
|
||||
|
||||
app = FastAPI()
|
||||
app.state.pg_store = pg_store
|
||||
app.add_middleware(SessionMiddleware, secret_key="test-secret-1234567890abcdef1234567890")
|
||||
app.include_router(cohort_router)
|
||||
app.include_router(failure_router)
|
||||
app.include_router(mastery_router)
|
||||
|
||||
# Stub auth — every request is operator "integration-tester".
|
||||
async def _stub_op():
|
||||
return Operator(id="op-1", username="tester", display_name="T", role="operator")
|
||||
app.dependency_overrides[current_operator] = _stub_op
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def _assist_outcome(
|
||||
learner_ref: str,
|
||||
path: str = "customer_service",
|
||||
turn_count: int = 20,
|
||||
blocks: int = 2,
|
||||
p95_latency_ms: float = 580.0,
|
||||
cost_cents: int = 20,
|
||||
) -> dict:
|
||||
return {
|
||||
"learner_ref": learner_ref,
|
||||
"path": path,
|
||||
"scenario_id": "assist:refund",
|
||||
"outcome": "completed",
|
||||
"session_type": "assist",
|
||||
"rubric_scores": [],
|
||||
"failure_mode": None,
|
||||
"branch_path": [],
|
||||
"assist_turn_count": turn_count,
|
||||
"guardrail_blocks": blocks,
|
||||
"assist_p95_latency_ms": p95_latency_ms,
|
||||
"assist_p50_latency_ms": 500.0,
|
||||
"assist_p99_latency_ms": 620.0,
|
||||
"assist_within_pilot": True,
|
||||
"assist_cost_cents": cost_cents,
|
||||
"timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_assist_aggregation_k_anon_threshold(pg_store, authed_client):
|
||||
"""1-4: 12 assist shifts (12 learners) → non-suppressed; 5 → suppressed."""
|
||||
from server.cohort.aggregator import aggregate_session
|
||||
|
||||
# 1. Seed 12 assist shifts for 'customer_service' (12 distinct learners).
|
||||
for i in range(12):
|
||||
await aggregate_session(pg_store, _assist_outcome(f"learner-{i}"))
|
||||
|
||||
# 2. Verify cohort_aggregates has assist metrics.
|
||||
async with pg_store.pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT metric, value, cell_count, cell_suppressed "
|
||||
"FROM cohort_aggregates WHERE path = 'customer_service' "
|
||||
"AND metric LIKE 'assist_%'"
|
||||
)
|
||||
metrics = {r["metric"]: r for r in rows}
|
||||
assert "assist_shifts_count" in metrics
|
||||
assert "assist_turns_count" in metrics
|
||||
assert "assist_active_learners_count" in metrics
|
||||
assert "assist_guardrail_block_rate" in metrics
|
||||
# 12 learners → not suppressed.
|
||||
assert metrics["assist_active_learners_count"]["cell_suppressed"] is False
|
||||
assert metrics["assist_active_learners_count"]["value"] == 12.0
|
||||
|
||||
# 3. GET /api/operator/cohort → returns assist volume (non-suppressed).
|
||||
r = authed_client.get("/api/operator/cohort")
|
||||
assert r.status_code == 200
|
||||
cohort_metrics = {
|
||||
c["metric"]: c for v in r.json()["views"] if v["path"] == "customer_service"
|
||||
for c in v["metrics"]
|
||||
}
|
||||
assert "assist_shifts_count" in cohort_metrics
|
||||
assert cohort_metrics["assist_shifts_count"]["cell_suppressed"] is False
|
||||
|
||||
# 4. GET /api/operator/failure-patterns → returns assist_guardrail_block_rate.
|
||||
r = authed_client.get("/api/operator/failure-patterns")
|
||||
assert r.status_code == 200
|
||||
fp_metrics = {
|
||||
c["metric"]: c for v in r.json()["views"] if v["path"] == "customer_service"
|
||||
for c in v["metrics"]
|
||||
}
|
||||
assert "assist_guardrail_block_rate" in fp_metrics
|
||||
|
||||
# 5. Seed 5 assist shifts for a DIFFERENT path (5 NEW learners) → suppressed.
|
||||
for i in range(5):
|
||||
await aggregate_session(pg_store, _assist_outcome(f"new-learner-{i}", path="retail_sales"))
|
||||
|
||||
r = authed_client.get("/api/operator/cohort")
|
||||
retail_metrics = {
|
||||
c["metric"]: c for v in r.json()["views"] if v["path"] == "retail_sales"
|
||||
for c in v["metrics"]
|
||||
}
|
||||
assert "assist_shifts_count" in retail_metrics
|
||||
# 5 < 10 → suppressed.
|
||||
assert retail_metrics["assist_shifts_count"]["cell_suppressed"] is True
|
||||
assert retail_metrics["assist_shifts_count"]["value"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_assist_p95_latency_in_aggregates(pg_store):
|
||||
"""6: assist_p95_latency_ms is in the aggregates (D-072)."""
|
||||
from server.cohort.aggregator import aggregate_session
|
||||
|
||||
for i in range(12):
|
||||
await aggregate_session(pg_store, _assist_outcome(f"learner-{i}", p95_latency_ms=580.0))
|
||||
|
||||
async with pg_store.pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT value, cell_suppressed FROM cohort_aggregates "
|
||||
"WHERE path = 'customer_service' AND metric = 'assist_p95_latency_ms'"
|
||||
)
|
||||
assert row is not None
|
||||
assert row["cell_suppressed"] is False
|
||||
assert row["value"] is not None
|
||||
# The running mean of per-shift p95 (580.0) → ~580.
|
||||
assert 570.0 <= float(row["value"]) <= 590.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_assist_cost_cents_in_session_outcome():
|
||||
"""7: assist_cost_cents is in the session_outcome (TASK-11-01)."""
|
||||
from server.assist.session import AssistSession
|
||||
from server.assist.context import AssistContext
|
||||
|
||||
ctx = AssistContext(
|
||||
system_prompt="", current_week=1, scenario_tag="refund",
|
||||
theta=0.0, coaching_focus="empathy", path_slug="customer_service",
|
||||
)
|
||||
session = AssistSession.__new__(AssistSession)
|
||||
session.assist_cost_cents = 0
|
||||
session.turn_count = 3
|
||||
session.guardrail_block_count = 0
|
||||
session.latency_metrics = MagicMock()
|
||||
session.latency_metrics.summary = MagicMock(return_value={
|
||||
"p50": 500.0, "p95": 580.0, "p99": 620.0, "count": 3,
|
||||
"target_ms": 600, "pilot_tolerance_ms": 650,
|
||||
"within_target": True, "within_pilot": True,
|
||||
})
|
||||
session.context = ctx
|
||||
session.learner_id = "learner-1"
|
||||
session.session_id = "test-session"
|
||||
|
||||
# Add 3 turns of cost.
|
||||
session.add_assist_turn_cost(5)
|
||||
session.add_assist_turn_cost(10)
|
||||
session.add_assist_turn_cost(3)
|
||||
|
||||
outcome = session._build_session_outcome("completed")
|
||||
assert outcome["assist_cost_cents"] == 18 # 5 + 10 + 3
|
||||
assert outcome["session_type"] == "assist"
|
||||
assert outcome["assist_p95_latency_ms"] == 580.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_c3_budget_check_runs():
|
||||
"""8: the C-3 budget check runs + reports within_budget (TASK-11-02)."""
|
||||
from server.assist.budget_check import C3_TARGET_USD, check_c3_budget
|
||||
|
||||
# 20 turns/shift × 20 shifts/month at 0.05 cents/turn → $0.20/month.
|
||||
result = check_c3_budget(
|
||||
assist_turns_per_shift=20,
|
||||
shifts_per_month=20,
|
||||
cost_per_turn_cents=0.05,
|
||||
)
|
||||
assert result["within_budget"] is True
|
||||
assert result["total_with_practice"] <= C3_TARGET_USD
|
||||
assert result["c3_target"] == 3.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_techdebt_aggregation_cache_survives_restart(pg_store, tmp_path):
|
||||
"""9a: aggregation cache survives a restart (TASK-12-01, P1+ #7)."""
|
||||
from server.cohort.aggregator import aggregate_session
|
||||
from server.cohort.learner_cache import (
|
||||
_clear_learner_cache,
|
||||
_count_distinct_learners,
|
||||
_load_learner_cache,
|
||||
)
|
||||
|
||||
# Point the cache to a temp file.
|
||||
pg_store.cohort_cache_db_path = str(tmp_path / "cache.db")
|
||||
|
||||
# Seed 10 learners.
|
||||
for i in range(10):
|
||||
await aggregate_session(pg_store, _assist_outcome(f"learner-{i}"))
|
||||
|
||||
# The persisted cache should have 10 distinct learners for this path.
|
||||
window_start = (_dt.datetime.now(_dt.timezone.utc).date() - _dt.timedelta(days=6))
|
||||
count = await _count_distinct_learners(pg_store, "customer_service", window_start)
|
||||
assert count == 10
|
||||
|
||||
# Simulate a restart: clear the in-memory cache + reload from SQLite.
|
||||
if hasattr(pg_store, "_agg_cache"):
|
||||
del pg_store._agg_cache
|
||||
loaded = await _load_learner_cache(pg_store)
|
||||
key = ("customer_service", "__learners__", window_start)
|
||||
assert key in loaded
|
||||
assert len(loaded[key]) == 10 # survived the "restart"
|
||||
|
||||
# Clear the cache (nightly reconciliation).
|
||||
await _clear_learner_cache(pg_store)
|
||||
count_after_clear = await _count_distinct_learners(pg_store, "customer_service", window_start)
|
||||
assert count_after_clear == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_techdebt_cookie_secret_warning(monkeypatch):
|
||||
"""9b: cookie-secret <32 bytes logs a WARNING (TASK-12-02, P1+ #3)."""
|
||||
from loguru import logger as _logger
|
||||
from server.auth.cookies import get_session_middleware_kwargs
|
||||
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "short") # 5 bytes < 32
|
||||
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
|
||||
msgs: list[str] = []
|
||||
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
|
||||
try:
|
||||
kw = get_session_middleware_kwargs()
|
||||
finally:
|
||||
_logger.remove(sink_id)
|
||||
assert kw["secret_key"] == "short" # accepted (backward compat)
|
||||
assert any("<32 bytes" in m for m in msgs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_techdebt_credential_status_enum():
|
||||
"""9c: set_credential_status enum validation (TASK-12-03, P1+ #4)."""
|
||||
from db.pg_store import PgStore
|
||||
|
||||
pool = MagicMock()
|
||||
conn = MagicMock()
|
||||
conn.execute = AsyncMock()
|
||||
cm = MagicMock()
|
||||
cm.__aenter__ = AsyncMock(return_value=conn)
|
||||
cm.__aexit__ = AsyncMock(return_value=None)
|
||||
pool.acquire = MagicMock(return_value=cm)
|
||||
|
||||
store = PgStore(pool)
|
||||
# Invalid status → ValueError.
|
||||
with pytest.raises(ValueError, match="Invalid credential status"):
|
||||
await store.set_credential_status("cred-1", "deleted")
|
||||
# Valid statuses work.
|
||||
await store.set_credential_status("cred-1", "revoked")
|
||||
await store.set_credential_status("cred-1", "active")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_techdebt_argon2id_offloaded():
|
||||
"""9d: argon2id verify_password offloaded to asyncio.to_thread (P1+ #1)."""
|
||||
import asyncio as _asyncio
|
||||
import server.auth.routes as _routes_mod
|
||||
from server.auth.passwords import hash_password
|
||||
|
||||
# The login handler should use asyncio.to_thread for verify_password.
|
||||
# Verify the module imports asyncio + the handler references to_thread.
|
||||
assert hasattr(_routes_mod, "asyncio")
|
||||
assert _asyncio.to_thread is _routes_mod.asyncio.to_thread
|
||||
|
||||
# Functional check: verify_password is callable via to_thread.
|
||||
h = hash_password("pw")
|
||||
result = await _asyncio.to_thread(_routes_mod.verify_password, h, "pw")
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_p2_no_per_learner_data_in_responses(pg_store, authed_client):
|
||||
"""No per-learner data in any dashboard response (D-031, G-038)."""
|
||||
from server.cohort.aggregator import aggregate_session
|
||||
|
||||
for i in range(12):
|
||||
await aggregate_session(pg_store, _assist_outcome(f"learner-sensitive-{i}"))
|
||||
|
||||
for endpoint in ("/api/operator/cohort", "/api/operator/failure-patterns", "/api/operator/mastery"):
|
||||
r = authed_client.get(endpoint)
|
||||
assert r.status_code == 200
|
||||
# No learner ref should appear in the response.
|
||||
text = r.text
|
||||
assert "learner-sensitive-" not in text, \
|
||||
f"per-learner data leaked in {endpoint} response"
|
||||
Reference in New Issue
Block a user