feat(P02): complete integration + tech-debt + NFR measurement phase — v0.1.12 tagged

Phase 2 (Integration + Tech-Debt + NFR Measurement) complete.
4 slices, 2 waves, 9 tasks. 4 REQs covered. 60 new tests (469 total).
8 v0.4 P1+ tech-debt findings addressed. Verify: APPROVE_WITH_NOTES.

NFR measurement (p95 latency + guardrail FP/FN), cohort aggregation
assist metrics (5 new metrics, no schema change), assist cost tracking
+ C-3 budget check, tech-debt wave (argon2id offload, cookie-secret
validation, credential enum, f-string SQL, cache persistence, zoneinfo,
audit log, 429 mock).

---ci---
project: praxis
phase: 2
milestone: v0.5
status: complete
requirements:
  covered: [REQ-NFR-ASSIST-01, REQ-IDEATE-04, REQ-IDEATE-06, REQ-IDEATE-07]
  partial: []
---/ci---
This commit is contained in:
Praxis CI
2026-08-04 22:11:51 +00:00
parent 38b97ee751
commit bdcf793db2
23 changed files with 3364 additions and 30 deletions
+228
View File
@@ -0,0 +1,228 @@
"""Assist cost tracking tests (TASK-11-03, REQ-IDEATE-07, C-3, D-012).
Tests:
- derive_assist_turn_cost() computes the per-turn cost (LLM + Piper TTS).
- The shift-end assist_cost_cents is the sum of per-turn costs.
- check_c3_budget() with 20 turns/shift × 20 shifts/month → within budget.
- check_c3_budget() with 100 turns/shift × 30 shifts/month → may exceed (flag=True).
- Existing derive_cost() unchanged (practice cost tests still pass).
"""
from __future__ import annotations
import pytest
from server.assist.budget_check import C3_TARGET_USD, check_c3_budget
from server.cost import CostBreakdown, derive_assist_turn_cost, derive_cost, load_rates
# ── derive_assist_turn_cost ─────────────────────────────────────────────────
def test_derive_assist_turn_cost_basic():
"""Per-turn cost computed from LLM tokens + Piper TTS chars."""
b = derive_assist_turn_cost(
llm_input_tokens=300,
llm_output_tokens=100,
tts_characters=400,
tts_provider="piper",
)
assert b.derived_cents >= 0
assert b.llm_input_tokens == 300
assert b.llm_output_tokens == 100
assert b.tts_characters == 400
# No debrief (D-063) + no Deepgram minutes (accounted at shift level).
assert b.debrief_input_tokens == 0
assert b.debrief_output_tokens == 0
assert b.deepgram_audio_minutes == 0.0
def test_derive_assist_turn_cost_piper_zero_tts():
"""Piper self-hosted TTS is $0 marginal cost (D-065 — Piper is assist default)."""
b = derive_assist_turn_cost(
llm_input_tokens=300,
llm_output_tokens=100,
tts_characters=10000,
tts_provider="piper",
)
# Piper rate is 0.0 per 1k chars → TTS contributes 0; only LLM cost.
# LLM: (300+100)/1000 * 0.5 = 0.2 cents → rounds to 0.
assert b.derived_cents >= 0
def test_derive_assist_turn_cost_cartesia_fallback():
"""Cartesia TTS fallback (non-default for assist — D-065 prefers Piper)."""
b = derive_assist_turn_cost(
llm_input_tokens=300,
llm_output_tokens=100,
tts_characters=1000,
tts_provider="cartesia",
)
# Cartesia rate is 3.0 per 1k chars → 1000 chars = 3.0 cents TTS.
assert b.derived_cents > 0
def test_derive_assist_turn_cost_uses_same_rates_as_derive_cost():
"""derive_assist_turn_cost uses the same load_rates() + CostBreakdown."""
rates = load_rates()
b = derive_assist_turn_cost(
llm_input_tokens=1000,
llm_output_tokens=500,
tts_characters=500,
tts_provider="piper",
rates=rates,
)
assert b.rates is rates
assert isinstance(b, CostBreakdown)
def test_derive_cost_unchanged():
"""Existing derive_cost() unchanged (practice cost tests still pass)."""
b = derive_cost(
llm_input_tokens=500,
llm_output_tokens=200,
deepgram_audio_minutes=2.0,
tts_characters=800,
debrief_input_tokens=300,
debrief_output_tokens=150,
tts_provider="cartesia",
)
assert b.derived_cents > 0
assert b.deepgram_audio_minutes == 2.0
assert b.debrief_input_tokens == 300
# ── Shift-end assist_cost_cents aggregation ─────────────────────────────────
def test_shift_end_assist_cost_is_sum_of_per_turn_costs():
"""AssistSession.assist_cost_cents is the sum of per-turn costs."""
from server.assist.session import AssistSession
from server.assist.context import AssistContext
# Construct an AssistSession without calling start() (we only test the
# cost accumulator, not the DB lifecycle).
ctx = AssistContext(
system_prompt="",
current_week=1,
scenario_tag="refund",
theta=0.0,
coaching_focus="empathy",
path_slug="customer_service",
)
session = AssistSession.__new__(AssistSession)
session.assist_cost_cents = 0
session.turn_count = 0
session.guardrail_block_count = 0
session.latency_metrics = None # not needed for this test
# Simulate 3 turns with per-turn costs.
for turn_cost in [2, 3, 1]:
session.add_assist_turn_cost(turn_cost)
assert session.assist_cost_cents == 6 # 2 + 3 + 1
# ── check_c3_budget ────────────────────────────────────────────────────────
def test_c3_budget_within_budget_typical_usage():
"""20 turns/shift × 20 shifts/month at ~$0.0005/turn → within budget.
Example from the plan: 400 turns/month at ~$0.0005/turn = ~$0.20/month —
well under the $3 C-3 target.
"""
# cost_per_turn_cents = 0.05 cents ($0.0005) — gemma4:cloud pilot rate.
result = check_c3_budget(
assist_turns_per_shift=20,
shifts_per_month=20,
cost_per_turn_cents=0.05,
)
assert result["turns_per_month"] == 400
# 400 * 0.05 / 100 = $0.20/month
assert result["monthly_assist_cost"] < 1.0
assert result["total_with_practice"] < C3_TARGET_USD
assert result["within_budget"] is True
assert result["flag"] is False
assert result["c3_target"] == C3_TARGET_USD == 3.0
def test_c3_budget_exceeds_with_high_usage():
"""100 turns/shift × 30 shifts/month at higher cost → may exceed (flag=True).
3000 turns/month at 0.15 cents/turn = $4.50/month → exceeds $3.
"""
result = check_c3_budget(
assist_turns_per_shift=100,
shifts_per_month=30,
cost_per_turn_cents=0.15,
)
assert result["turns_per_month"] == 3000
# 3000 * 0.15 / 100 = $4.50/month → over $3
assert result["monthly_assist_cost"] > C3_TARGET_USD
assert result["within_budget"] is False
assert result["flag"] is True # diagnostic flag (not enforced)
def test_c3_budget_with_practice_cost():
"""total_with_practice = assist + practice cost."""
result = check_c3_budget(
assist_turns_per_shift=20,
shifts_per_month=20,
cost_per_turn_cents=0.05,
practice_cost_per_month_usd=1.5,
)
# assist = $0.20, practice = $1.50 → total = $1.70 (within $3)
assert result["practice_cost_per_month"] == 1.5
assert result["total_with_practice"] < C3_TARGET_USD
assert result["within_budget"] is True
def test_c3_budget_with_practice_cost_exceeds():
"""Assist + practice cost exceeds $3 → flag=True (diagnostic)."""
result = check_c3_budget(
assist_turns_per_shift=50,
shifts_per_month=30,
cost_per_turn_cents=0.10,
practice_cost_per_month_usd=2.0,
)
# assist = 1500 * 0.10 / 100 = $1.50, practice = $2.00 → total = $3.50
assert result["total_with_practice"] > C3_TARGET_USD
assert result["within_budget"] is False
assert result["flag"] is True
def test_c3_budget_zero_usage():
"""0 turns → zero cost, within budget."""
result = check_c3_budget(
assist_turns_per_shift=0,
shifts_per_month=0,
cost_per_turn_cents=0.05,
)
assert result["turns_per_month"] == 0
assert result["monthly_assist_cost"] == 0.0
assert result["within_budget"] is True
assert result["flag"] is False
def test_c3_budget_is_diagnostic_not_enforced():
"""D-012: the check is diagnostic (not enforced). flag=True does not raise.
The check_c3_budget() function returns a dict with flag=True when over
budget, but does NOT raise an exception (D-012 — no enforced ceiling in
the pilot). The caller logs the flag + continues.
"""
result = check_c3_budget(
assist_turns_per_shift=1000,
shifts_per_month=30,
cost_per_turn_cents=1.0,
)
# 30000 turns * 1.0 cent / 100 = $300/month → way over $3
assert result["flag"] is True
assert result["within_budget"] is False
# No exception raised — the function returns a dict (diagnostic, not enforced).
def test_c3_target_is_3_usd():
"""C-3 target is ≤ $3/active learner/month (C-3, D-012)."""
assert C3_TARGET_USD == 3.0
+278 -1
View File
@@ -89,6 +89,65 @@ def test_cookie_secret_unset_generates_random(monkeypatch):
assert len(kw["secret_key"]) >= 32
def test_cookie_secret_short_logs_warning_accepted(monkeypatch, caplog):
"""TASK-12-02 (P1+ #3): a secret <32 bytes logs a WARNING but is accepted.
A short non-empty secret (e.g., 'x') weakens the HMAC signature. The
secret is still accepted (backward compat — pilot); post-pilot this
should be a hard error. The WARNING is logged with remediation guidance.
"""
from loguru import logger as _logger
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "short-secret") # 11 bytes < 32
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
# Capture loguru warnings.
msgs: list[str] = []
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
try:
kw = get_session_middleware_kwargs()
finally:
_logger.remove(sink_id)
# The short secret is accepted (backward compat — no hard error in pilot).
assert kw["secret_key"] == "short-secret"
# A WARNING about the short secret was logged.
assert any("<32 bytes" in m for m in msgs), \
"short PRAXIS_COOKIE_SECRET should log a <32 bytes WARNING"
def test_cookie_secret_32_bytes_no_warning(monkeypatch, caplog):
"""TASK-12-02: a secret >=32 bytes logs no <32 bytes warning."""
from loguru import logger as _logger
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "x" * 32) # exactly 32 bytes
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
msgs: list[str] = []
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
try:
kw = get_session_middleware_kwargs()
finally:
_logger.remove(sink_id)
assert kw["secret_key"] == "x" * 32
# No <32 bytes warning (the secret is exactly 32 bytes).
assert not any("<32 bytes" in m for m in msgs), \
"32-byte secret should NOT log a <32 bytes warning"
def test_cookie_secret_long_no_warning(monkeypatch):
"""TASK-12-02: a secret >32 bytes logs no warning."""
from loguru import logger as _logger
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "x" * 64) # 64 bytes
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
msgs: list[str] = []
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
try:
kw = get_session_middleware_kwargs()
finally:
_logger.remove(sink_id)
assert kw["secret_key"] == "x" * 64
assert not any("<32 bytes" in m for m in msgs)
# ── current_operator dependency ─────────────────────────────────────────────
@@ -307,4 +366,222 @@ def test_rate_limit_login_decorator():
def test_limiter_is_in_memory():
assert getattr(limiter, "_storage_uri", "memory://") == "memory://" or limiter._storage is not None
assert getattr(limiter, "_storage_uri", "memory://") == "memory://" or limiter._storage is not None
# ── TASK-12-04 (P1+ #1/#2/#5): argon2id offload + 429 mock test + audit log ──
def test_login_argon2id_offloaded_to_thread():
"""TASK-12-04 (P1+ #1): verify_password is offloaded to asyncio.to_thread.
The login handler should call verify_password via asyncio.to_thread (not
directly) so the ~100-300ms argon2id hashing does not block the event loop.
We verify by patching asyncio.to_thread to record the call.
"""
import asyncio as _asyncio
op = {
"id": "44444444-4444-4444-4444-444444444444",
"username": "erin",
"display_name": "Erin",
"role": "operator",
"is_active": True,
"password_hash": hash_password("pw"),
}
store = _mock_store(operator_row=op)
store.get_operator_by_username = AsyncMock(return_value=op)
store.update_last_login = AsyncMock()
store.pool = MagicMock()
conn = MagicMock()
conn.execute = AsyncMock()
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
store.pool.acquire = MagicMock(return_value=cm)
to_thread_calls: list = []
real_to_thread = _asyncio.to_thread
async def _spy_to_thread(func, *args, **kwargs):
to_thread_calls.append((func, args, kwargs))
return await real_to_thread(func, *args, **kwargs)
import server.auth.routes as _routes_mod
orig = _routes_mod.asyncio.to_thread
_routes_mod.asyncio.to_thread = _spy_to_thread
try:
app = _make_app_with_store(store)
with TestClient(app) as client:
r = client.post("/api/operator/login", json={"username": "erin", "password": "pw"})
assert r.status_code == 200
finally:
_routes_mod.asyncio.to_thread = orig
# verify_password should have been called via asyncio.to_thread.
assert to_thread_calls, "login should offload verify_password to asyncio.to_thread"
func = to_thread_calls[0][0]
assert func.__name__ == "verify_password", (
f"expected verify_password offloaded, got {func.__name__}"
)
def test_login_rehash_offloaded_to_thread():
"""TASK-12-04 (P1+ #1): hash_password (rehash) is also offloaded to thread."""
from argon2 import PasswordHasher
weak_hasher = PasswordHasher(time_cost=1, memory_cost=8, parallelism=1)
op = {
"id": "55555555-5555-5555-5555-555555555555",
"username": "frank",
"display_name": "Frank",
"role": "operator",
"is_active": True,
"password_hash": weak_hasher.hash("pw"),
}
store = _mock_store(operator_row=op)
store.get_operator_by_username = AsyncMock(return_value=op)
store.update_last_login = AsyncMock()
store.pool = MagicMock()
conn = MagicMock()
conn.execute = AsyncMock()
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
store.pool.acquire = MagicMock(return_value=cm)
import asyncio as _asyncio
import server.auth.routes as _routes_mod
to_thread_calls: list = []
real_to_thread = _routes_mod.asyncio.to_thread
async def _spy_to_thread(func, *args, **kwargs):
to_thread_calls.append((func, args, kwargs))
return await real_to_thread(func, *args, **kwargs)
_routes_mod.asyncio.to_thread = _spy_to_thread
try:
app = _make_app_with_store(store)
with TestClient(app) as client:
r = client.post("/api/operator/login", json={"username": "frank", "password": "pw"})
assert r.status_code == 200
finally:
_routes_mod.asyncio.to_thread = real_to_thread
# Both verify_password + hash_password should be offloaded.
func_names = [c[0].__name__ for c in to_thread_calls]
assert "verify_password" in func_names
assert "hash_password" in func_names, "rehash should offload hash_password to thread"
def test_login_rate_limit_429_after_5_attempts():
"""TASK-12-04 (P1+ #2): mock-based 429 test — 6th login attempt → 429.
The full 6th-attempt→429 path is in the PG-requiring integration test; this
adds a mock-based test for CI coverage without Postgres. slowapi's in-memory
limiter tracks per-IP; 5/minute → 6th attempt gets 429.
"""
from slowapi.errors import RateLimitExceeded
from slowapi.middleware import SlowAPIMiddleware
from slowapi import _rate_limit_exceeded_handler
op = {
"id": "66666666-6666-6666-6666-666666666666",
"username": "grace",
"display_name": "Grace",
"role": "operator",
"is_active": True,
"password_hash": hash_password("pw"),
}
store = _mock_store(operator_row=op)
store.get_operator_by_username = AsyncMock(return_value=op)
store.update_last_login = AsyncMock()
store.pool = MagicMock()
conn = MagicMock()
conn.execute = AsyncMock()
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
store.pool.acquire = MagicMock(return_value=cm)
app = _make_app_with_store(store)
app.state.limiter = limiter
app.add_middleware(SlowAPIMiddleware)
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
with TestClient(app) as client:
# 5 attempts should succeed (or 401 for wrong password — both count).
statuses: list[int] = []
for _ in range(5):
r = client.post(
"/api/operator/login", json={"username": "grace", "password": "pw"}
)
statuses.append(r.status_code)
# The 5 attempts should not be 429 (within the 5/minute limit).
assert all(s != 429 for s in statuses), f"first 5 should not be 429: {statuses}"
# 6th attempt → 429 (rate limit exceeded).
r6 = client.post(
"/api/operator/login", json={"username": "grace", "password": "pw"}
)
assert r6.status_code == 429, (
f"6th login attempt should be rate-limited (429), got {r6.status_code}"
)
def test_credential_revocation_logs_audit_event():
"""TASK-12-04 (P1+ #5): credential revocation logs operator + cred_id.
The revoke_credential endpoint should log an application-level audit event
(no audit_log table — the log is sufficient for pilot per D-056).
"""
import logging as _logging
from server.operator.credentials import router as creds_router
op = {
"id": "77777777-7777-7777-7777-777777777777",
"username": "heidi",
"display_name": "Heidi",
"role": "operator",
}
store = MagicMock()
store.get_credential = AsyncMock(return_value={"id": "cred-xyz", "status": "active"})
store.set_credential_status = AsyncMock()
app = FastAPI()
app.state.pg_store = store
app.add_middleware(SessionMiddleware, secret_key="test-secret-1234567890abcdef")
app.include_router(creds_router)
# Stub auth.
from server.auth.dependencies import current_operator
from server.auth.models import Operator
async def _stub_op():
return Operator(id=op["id"], username=op["username"],
display_name=op["display_name"], role=op["role"])
app.dependency_overrides[current_operator] = _stub_op
# Capture the audit log.
cred_log = _logging.getLogger("server.operator.credentials")
records: list[_logging.LogRecord] = []
handler = _logging.Handler()
handler.emit = records.append # type: ignore[method-assign]
cred_log.addHandler(handler)
cred_log.setLevel(_logging.INFO)
try:
with TestClient(app) as client:
r = client.post("/api/operator/credentials/cred-xyz/revoke")
assert r.status_code == 200
assert r.json()["status"] == "revoked"
finally:
cred_log.removeHandler(handler)
# The audit log should contain the operator id + cred_id.
audit_msgs = [r.getMessage() for r in records if r.levelno >= _logging.INFO]
assert any("credential revoked" in m for m in audit_msgs), \
f"revocation should log 'credential revoked': {audit_msgs}"
assert any("cred-xyz" in m for m in audit_msgs), \
f"audit log should contain cred_id: {audit_msgs}"
assert any(op["id"] in m for m in audit_msgs), \
f"audit log should contain operator id: {audit_msgs}"
+416
View File
@@ -0,0 +1,416 @@
"""Cohort assist aggregation tests (TASK-10-03, D-062, D-063, REQ-NFR-ASSIST-04).
Tests the _aggregate_assist branch in server/cohort/aggregator.py with a
mocked PgStore (no Postgres required). Verifies:
- _aggregate_assist() upserts the 5 core assist metrics + p95 latency.
- k-anon suppression: <10 distinct learners → suppressed.
- Idempotent upsert: same session_outcome twice → same aggregate.
- assist_guardrail_block_rate = blocks / turns.
- The practice branch (_aggregate_practice) is unchanged (backward compat).
- The dashboard endpoints return assist rows (cohort + failure-patterns).
D-063 (binding): assist does NOT update mastery. The _aggregate_assist branch
computes NO mastery metrics (no rubric scores, no gate_open_rate).
"""
from __future__ import annotations
import datetime as _dt
from unittest.mock import AsyncMock, MagicMock
import pytest
from server.cohort.aggregator import (
K_ANON_THRESHOLD,
_aggregate_assist,
_aggregate_practice,
aggregate_session,
)
from server.cohort.hook import on_session_end
def _mock_pg_store():
store = MagicMock()
store.upsert_cohort_aggregate = AsyncMock()
return store
def _assist_outcome(
learner_ref: str,
path: str = "customer_service",
turn_count: int = 20,
blocks: int = 2,
p95_latency_ms: float | None = 580.0,
cost_cents: int = 20,
outcome: str = "completed",
) -> dict:
return {
"learner_ref": learner_ref,
"path": path,
"scenario_id": f"assist:refund",
"outcome": outcome,
"session_type": "assist",
"rubric_scores": [], # D-063: no rubric scores for assist
"failure_mode": None,
"branch_path": [],
"assist_turn_count": turn_count,
"guardrail_blocks": blocks,
"assist_p95_latency_ms": p95_latency_ms,
"assist_p50_latency_ms": 500.0,
"assist_p99_latency_ms": 620.0,
"assist_within_pilot": True,
"assist_cost_cents": cost_cents,
"timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(),
}
def _practice_outcome(learner_ref: str, path: str = "customer_service") -> dict:
return {
"learner_ref": learner_ref,
"path": path,
"scenario_id": f"{path}_v01",
"outcome": "pass",
"session_type": "practice",
"rubric_scores": [{"criterion_id": "empathy", "score": 4.0}],
"failure_mode": None,
"branch_path": ["accept"],
"timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(),
}
# ── Assist metrics upserted ─────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_aggregate_assist_upserts_5_core_metrics():
"""_aggregate_assist upserts the 5 core assist metrics (REQ-NFR-ASSIST-04)."""
store = _mock_pg_store()
await _aggregate_assist(store, _assist_outcome("learner-1"))
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
assert "assist_shifts_count" in metrics
assert "assist_active_learners_count" in metrics
assert "assist_turns_count" in metrics
assert "assist_avg_turns_per_shift" in metrics
assert "assist_guardrail_block_rate" in metrics
@pytest.mark.asyncio
async def test_aggregate_assist_upserts_p95_latency():
"""_aggregate_assist upserts assist_p95_latency_ms (D-072, TASK-09-01)."""
store = _mock_pg_store()
await _aggregate_assist(store, _assist_outcome("learner-1", p95_latency_ms=580.0))
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
assert "assist_p95_latency_ms" in metrics
@pytest.mark.asyncio
async def test_aggregate_assist_upserts_avg_cost():
"""_aggregate_assist upserts assist_avg_cost_per_shift (TASK-11-01)."""
store = _mock_pg_store()
await _aggregate_assist(store, _assist_outcome("learner-1", cost_cents=25))
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
assert "assist_avg_cost_per_shift" in metrics
@pytest.mark.asyncio
async def test_aggregate_assist_no_mastery_metrics():
"""D-063: _aggregate_assist computes NO mastery metrics."""
store = _mock_pg_store()
await _aggregate_assist(store, _assist_outcome("learner-1"))
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
# No mastery metrics should be present.
assert "gate_open_rate" not in metrics
assert "median_mastery_score" not in metrics
assert not any(m.startswith("rubric_criterion_mean:") for m in metrics)
# No practice metrics either (assist is a separate branch).
assert "sessions_count" not in metrics
# ── k-anonymity suppression ──────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_assist_9_learners_suppressed():
"""<10 distinct learners → all assist cells suppressed."""
store = _mock_pg_store()
for i in range(9):
await _aggregate_assist(store, _assist_outcome(f"learner-{i}"))
suppressed = [c for c in store.upsert_cohort_aggregate.call_args_list if c.args[6] is True]
non_suppressed = [c for c in store.upsert_cohort_aggregate.call_args_list if c.args[6] is False]
assert suppressed, "assist cells should be suppressed with <10 learners"
assert not non_suppressed, "no assist cell should be non-suppressed with 9 learners"
@pytest.mark.asyncio
async def test_assist_10_learners_not_suppressed():
"""≥10 distinct learners → assist cells not suppressed."""
store = _mock_pg_store()
for i in range(10):
await _aggregate_assist(store, _assist_outcome(f"learner-{i}"))
non_suppressed = [c for c in store.upsert_cohort_aggregate.call_args_list if c.args[6] is False]
assert non_suppressed, "assist cells should NOT be suppressed at 10 learners"
for c in non_suppressed:
assert c.args[4] is not None, "non-suppressed cell value must not be None"
# ── Idempotent upsert ───────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_assist_idempotent_same_outcome_twice():
"""Re-running with the same outcome produces consistent upserts (idempotent)."""
store = _mock_pg_store()
outcome = _assist_outcome("learner-x")
await _aggregate_assist(store, outcome)
first_call_count = store.upsert_cohort_aggregate.call_count
await _aggregate_assist(store, outcome)
second_call_count = store.upsert_cohort_aggregate.call_count
# Both runs produce upsert calls (the DB ON CONFLICT makes them idempotent).
assert second_call_count >= first_call_count
assert store.upsert_cohort_aggregate.called
# ── assist_guardrail_block_rate = blocks / turns ─────────────────────────────
@pytest.mark.asyncio
async def test_assist_guardrail_block_rate_computed():
"""assist_guardrail_block_rate = blocks / turns (safety signal)."""
store = _mock_pg_store()
# 10 learners so the cell is not suppressed (we can read the value).
for i in range(10):
await _aggregate_assist(store, _assist_outcome(f"learner-{i}", turn_count=20, blocks=2))
block_rate_cells = [
c for c in store.upsert_cohort_aggregate.call_args_list
if c.args[1] == "assist_guardrail_block_rate" and c.args[6] is False
]
assert block_rate_cells, "should have a non-suppressed assist_guardrail_block_rate cell"
# The running mean of per-shift block rates (2/20 = 0.1) → ~0.1.
rate = block_rate_cells[-1].args[4]
assert rate is not None
assert 0.05 <= rate <= 0.15 # ~0.1 with running-mean drift
@pytest.mark.asyncio
async def test_assist_zero_turns_block_rate_is_zero():
"""0 turns → block_rate = 0.0 (no division by zero)."""
store = _mock_pg_store()
for i in range(10):
await _aggregate_assist(store, _assist_outcome(f"learner-{i}", turn_count=0, blocks=0))
block_rate_cells = [
c for c in store.upsert_cohort_aggregate.call_args_list
if c.args[1] == "assist_guardrail_block_rate" and c.args[6] is False
]
assert block_rate_cells
rate = block_rate_cells[-1].args[4]
assert rate == 0.0
# ── Practice branch unchanged (backward compat) ─────────────────────────────
@pytest.mark.asyncio
async def test_aggregate_session_dispatches_to_practice():
"""aggregate_session with session_type='practice' → _aggregate_practice."""
store = _mock_pg_store()
await aggregate_session(store, _practice_outcome("learner-1"))
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
# Practice metrics should be present.
assert "sessions_count" in metrics
assert "active_learners_count" in metrics
# Assist metrics should NOT be present (practice branch).
assert "assist_shifts_count" not in metrics
@pytest.mark.asyncio
async def test_aggregate_session_dispatches_to_assist():
"""aggregate_session with session_type='assist' → _aggregate_assist."""
store = _mock_pg_store()
await aggregate_session(store, _assist_outcome("learner-1"))
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
assert "assist_shifts_count" in metrics
assert "sessions_count" not in metrics
@pytest.mark.asyncio
async def test_aggregate_session_default_is_practice():
"""aggregate_session with no session_type → practice (backward compat)."""
store = _mock_pg_store()
outcome = _practice_outcome("learner-1")
outcome.pop("session_type") # omit session_type → default practice
await aggregate_session(store, outcome)
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
assert "sessions_count" in metrics
assert "assist_shifts_count" not in metrics
# ── No PII in assist upsert calls ───────────────────────────────────────────
@pytest.mark.asyncio
async def test_assist_no_pii_in_upsert_calls():
"""No raw learner_ref leaks into assist aggregate cell args (D-031)."""
store = _mock_pg_store()
await _aggregate_assist(store, _assist_outcome("learner-sensitive-id-1234"))
for c in store.upsert_cohort_aggregate.call_args_list:
for arg in c.args:
assert "learner-sensitive-id-1234" not in str(arg), \
"raw learner_ref must not leak into assist aggregate cell args"
assert isinstance(c.args[5], int) # cell_count is an int
# ── Hook dispatches assist correctly ───────────────────────────────────────
@pytest.mark.asyncio
async def test_hook_dispatches_assist_session():
"""on_session_end with session_type='assist' → _aggregate_assist (no error)."""
store = _mock_pg_store()
await on_session_end(store, _assist_outcome("learner-1"))
assert store.upsert_cohort_aggregate.called
metrics = {c.args[1] for c in store.upsert_cohort_aggregate.call_args_list}
assert "assist_shifts_count" in metrics
@pytest.mark.asyncio
async def test_hook_assist_no_postgres_is_noop():
"""on_session_end with no Postgres → no-op (assist hook)."""
await on_session_end(None, _assist_outcome("learner-1"))
# ── Dashboard endpoints return assist rows ──────────────────────────────────
def _make_app_with_assist_rows(rows: list[dict]):
"""Build a minimal FastAPI app with the cohort + failure-patterns routers
+ a mocked pg_store returning `rows`."""
from fastapi import FastAPI
from fastapi.testclient import TestClient
from server.operator.cohort import router as cohort_router
from server.operator.failure_patterns import router as failure_router
app = FastAPI()
pg_store = MagicMock()
pg_store.pool = MagicMock()
conn = MagicMock()
conn.fetch = AsyncMock(return_value=rows)
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
pg_store.pool.acquire = MagicMock(return_value=cm)
app.state.pg_store = pg_store
# Bypass auth for these tests by stubbing current_operator.
from server.auth.dependencies import current_operator
from server.auth.models import Operator
async def _stub_op():
return Operator(id="op-1", username="tester", display_name="T", role="operator")
app.dependency_overrides[current_operator] = _stub_op
app.include_router(cohort_router)
app.include_router(failure_router)
return TestClient(app)
def _assist_metric_row(metric: str, value: float, suppressed: bool = False) -> dict:
return {
"path": "customer_service",
"metric": metric,
"window_start": _dt.date.today() - _dt.timedelta(days=6),
"window_end": _dt.date.today(),
"value": value if not suppressed else None,
"cell_count": 12,
"cell_suppressed": suppressed,
"updated_at": _dt.datetime.now(_dt.timezone.utc),
}
def test_cohort_endpoint_returns_assist_rows():
"""GET /api/operator/cohort returns assist_shifts_count + assist_turns_count."""
rows = [
_assist_metric_row("sessions_count", 15.0),
_assist_metric_row("active_learners_count", 12.0),
_assist_metric_row("assist_shifts_count", 8.0),
_assist_metric_row("assist_turns_count", 160.0),
]
client = _make_app_with_assist_rows(rows)
r = client.get("/api/operator/cohort")
assert r.status_code == 200
data = r.json()
metrics = {c["metric"] for v in data["views"] for c in v["metrics"]}
assert "assist_shifts_count" in metrics
assert "assist_turns_count" in metrics
assert "sessions_count" in metrics # practice still present
def test_failure_patterns_endpoint_returns_guardrail_block_rate():
"""GET /api/operator/failure-patterns returns assist_guardrail_block_rate."""
rows = [
_assist_metric_row("failure_mode:missed_apology", 3.0),
_assist_metric_row("branch:escalate", 5.0),
_assist_metric_row("assist_guardrail_block_rate", 0.08),
]
client = _make_app_with_assist_rows(rows)
r = client.get("/api/operator/failure-patterns")
assert r.status_code == 200
data = r.json()
metrics = {c["metric"] for v in data["views"] for c in v["metrics"]}
assert "assist_guardrail_block_rate" in metrics
assert "failure_mode:missed_apology" in metrics # practice failure patterns still present
def test_mastery_endpoint_excludes_assist_metrics():
"""D-063: GET /api/operator/mastery does NOT return assist metrics."""
from fastapi import FastAPI
from fastapi.testclient import TestClient
from server.auth.dependencies import current_operator
from server.auth.models import Operator
from server.operator.mastery import router as mastery_router
rows = [
_assist_metric_row("gate_open_rate", 0.5),
_assist_metric_row("median_mastery_score", 3.8),
_assist_metric_row("assist_shifts_count", 8.0), # should be EXCLUDED
_assist_metric_row("assist_guardrail_block_rate", 0.08), # EXCLUDED
]
app = FastAPI()
pg_store = MagicMock()
pg_store.pool = MagicMock()
conn = MagicMock()
conn.fetch = AsyncMock(return_value=rows)
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
pg_store.pool.acquire = MagicMock(return_value=cm)
app.state.pg_store = pg_store
async def _stub_op():
return Operator(id="op-1", username="tester", display_name="T", role="operator")
app.dependency_overrides[current_operator] = _stub_op
app.include_router(mastery_router)
client = TestClient(app)
r = client.get("/api/operator/mastery")
assert r.status_code == 200
data = r.json()
metrics = {c["metric"] for v in data["views"] for c in v["metrics"]}
assert "gate_open_rate" in metrics
assert "median_mastery_score" in metrics
# D-063: assist metrics must NOT appear in the mastery view.
assert "assist_shifts_count" not in metrics
assert "assist_guardrail_block_rate" not in metrics
def test_cohort_endpoint_suppressed_assist_cells():
"""Suppressed assist cells have value=null + cell_suppressed=true (k-anon)."""
rows = [
_assist_metric_row("assist_shifts_count", 0.0, suppressed=True),
_assist_metric_row("assist_turns_count", 0.0, suppressed=True),
]
client = _make_app_with_assist_rows(rows)
r = client.get("/api/operator/cohort")
assert r.status_code == 200
data = r.json()
for v in data["views"]:
for c in v["metrics"]:
if c["metric"] in ("assist_shifts_count", "assist_turns_count"):
assert c["cell_suppressed"] is True
assert c["value"] is None
+65
View File
@@ -44,6 +44,71 @@ def test_seconds_until_next_03_ct_exactly_03_rolls_to_tomorrow():
assert secs >= 86390 # ~24h
# ── TASK-12-04 (P1+ #6): zoneinfo DST-aware scheduler ───────────────────────
def test_nightly_scheduler_uses_zoneinfo_america_winnipeg():
"""TASK-12-04 (P1+ #6): CT is zoneinfo.ZoneInfo('America/Winnipeg') (DST-aware).
The v0.4 fixed UTC-5 offset is replaced with ZoneInfo("America/Winnipeg")
which correctly handles CST (UTC-6) in winter + CDT (UTC-5) in summer.
"""
from zoneinfo import ZoneInfo
assert isinstance(CT, ZoneInfo), f"CT should be a ZoneInfo, got {type(CT)}"
assert str(CT) == "America/Winnipeg", f"CT should be America/Winnipeg, got {CT}"
def test_nightly_scheduler_dst_summer_cdt():
"""TASK-12-04 (P1+ #6): summer (August) → CDT (UTC-5).
In August 2026, America/Winnipeg is on CDT (UTC-5). A 01:00 local time
should be 06:00 UTC. The scheduler computes seconds until 03:00 local.
"""
# 2026-08-04 is summer → CDT (UTC-5).
now_local = _dt.datetime(2026, 8, 4, 1, 0, tzinfo=CT)
# 01:00 CDT = 06:00 UTC.
assert now_local.utcoffset() == _dt.timedelta(hours=-5), (
f"August should be CDT (UTC-5), got offset {now_local.utcoffset()}"
)
secs = seconds_until_next_03_ct(now_local)
# 01:00 → 03:00 = 2h = 7200s.
assert 7190 <= secs <= 7200
def test_nightly_scheduler_dst_winter_cst():
"""TASK-12-04 (P1+ #6): winter (January) → CST (UTC-6).
In January 2027, America/Winnipeg is on CST (UTC-6). A 01:00 local time
should be 07:00 UTC. The v0.4 fixed UTC-5 offset would have been wrong
by 1h in winter; the ZoneInfo correctly handles the DST transition.
"""
# 2027-01-15 is winter → CST (UTC-6).
now_local = _dt.datetime(2027, 1, 15, 1, 0, tzinfo=CT)
assert now_local.utcoffset() == _dt.timedelta(hours=-6), (
f"January should be CST (UTC-6), got offset {now_local.utcoffset()}"
)
secs = seconds_until_next_03_ct(now_local)
# 01:00 → 03:00 = 2h = 7200s.
assert 7190 <= secs <= 7200
def test_nightly_scheduler_dst_transition_spring_2027():
"""TASK-12-04 (P1+ #6): DST spring forward — 2027-03-14 02:00 → 03:00 CDT.
On 2027-03-14, DST springs forward at 02:00 local (CST → CDT). The ZoneInfo
correctly handles the transition (the 02:00 hour is skipped). The scheduler
should still compute a valid seconds-until-03:00.
"""
# 2027-03-14 01:00 CST (before spring forward) → 03:00 CDT is 1h later
# (the 02:00 hour is skipped → 01:59 CST → 03:00 CDT).
now_local = _dt.datetime(2027, 3, 14, 1, 0, tzinfo=CT)
secs = seconds_until_next_03_ct(now_local)
# 01:00 CST → 03:00 CDT is 1h (the 02:00 hour is skipped).
# The exact value depends on the DST transition; assert it's ≤ 2h.
assert 0 < secs <= 7200, f"spring-forward seconds should be <= 2h, got {secs}"
# ── Reconciliation recomputes all windows ──────────────────────────────────
+106
View File
@@ -0,0 +1,106 @@
"""Mock-based tests for set_credential_status enum + f-string SQL fix
(TASK-12-03, P1+ #4/#8 from v0.4 REVIEW).
These tests do NOT require Postgres (they use a mock asyncpg pool). They
verify:
- 'revoked' uses a parameterized query with revoked_at=now() (no f-string).
- 'active' clears revoked_at=NULL (re-activation).
- Invalid status → ValueError (enum validation — P1+ #4).
- No f-string interpolation in the SQL (P1+ #8 code smell fix).
"""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
import pytest
from db.pg_store import PgStore
def _mock_pool_with_conn():
"""Build a mock asyncpg pool + conn that records execute() calls."""
pool = MagicMock()
conn = MagicMock()
conn.execute = AsyncMock()
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
pool.acquire = MagicMock(return_value=cm)
return pool, conn
@pytest.mark.asyncio
async def test_set_credential_status_revoked_uses_parameterized_query():
"""TASK-12-03 (P1+ #8): 'revoked' uses a parameterized query (no f-string)."""
pool, conn = _mock_pool_with_conn()
store = PgStore(pool)
await store.set_credential_status("cred-1", "revoked")
# Exactly one execute call.
assert conn.execute.await_count == 1
sql, status_arg, cred_arg = conn.execute.await_args.args
# No f-string interpolation — the SQL is a literal with $1, $2.
assert "revoked_at = now()" in sql
assert "$1" in sql and "$2" in sql
assert status_arg == "revoked"
assert cred_arg == "cred-1"
@pytest.mark.asyncio
async def test_set_credential_status_active_clears_revoked_at():
"""TASK-12-03: 'active' clears revoked_at=NULL (re-activation)."""
pool, conn = _mock_pool_with_conn()
store = PgStore(pool)
await store.set_credential_status("cred-1", "active")
assert conn.execute.await_count == 1
sql, status_arg, cred_arg = conn.execute.await_args.args
assert "revoked_at = NULL" in sql
assert status_arg == "active"
assert cred_arg == "cred-1"
@pytest.mark.asyncio
async def test_set_credential_status_invalid_raises_value_error():
"""TASK-12-03 (P1+ #4): invalid status → ValueError (enum validation)."""
pool, conn = _mock_pool_with_conn()
store = PgStore(pool)
for bad_status in ("pending", "suspended", "deleted", "", "REVOKED", "active "):
with pytest.raises(ValueError, match="Invalid credential status"):
await store.set_credential_status("cred-1", bad_status)
# No execute call should have been made (validation happens before the query).
assert conn.execute.await_count == 0
@pytest.mark.asyncio
async def test_set_credential_status_no_fstring_in_sql():
"""TASK-12-03 (P1+ #8): no f-string interpolation in the SQL (code smell fix).
The SQL must be a literal string (no f-string {extra} interpolation). The
status + cred_id are bound parameters ($1, $2), not interpolated.
"""
pool, conn = _mock_pool_with_conn()
store = PgStore(pool)
await store.set_credential_status("cred-1", "revoked")
sql = conn.execute.await_args.args[0]
# The SQL must NOT contain an f-string-interpolated extra clause. The old
# code had f"UPDATE ... SET status = $1{extra} WHERE id = $2" where extra
# was ', revoked_at = now()' or ''. The new code has two explicit queries.
# Verify the SQL is a literal (no {extra}-style interpolation artifacts).
assert "{extra}" not in sql
assert "UPDATE issued_credentials SET status = $1, revoked_at = now()" in sql
@pytest.mark.asyncio
async def test_set_credential_status_revoked_then_active():
"""TASK-12-03: revoke then re-activate (active clears revoked_at)."""
pool, conn = _mock_pool_with_conn()
store = PgStore(pool)
# Revoke.
await store.set_credential_status("cred-1", "revoked")
revoke_sql = conn.execute.await_args.args[0]
assert "revoked_at = now()" in revoke_sql
# Re-activate (active clears revoked_at).
conn.execute.reset_mock()
await store.set_credential_status("cred-1", "active")
active_sql = conn.execute.await_args.args[0]
assert "revoked_at = NULL" in active_sql
+290
View File
@@ -0,0 +1,290 @@
"""NFR measurement tests (TASK-09-03, REQ-NFR-ASSIST-01, REQ-IDEATE-04, D-072).
Tests the measurement infrastructure (NOT the actual latency — that's a Phase-1
live measurement, not a CI test):
- AssistLatencyMetrics: p95/p50/p99 computed correctly from mock records.
D-072: within_target = (p95 < 600), within_pilot = (p95 <= 650).
- GuardrailMetrics: false_positive_rate on the tuning corpus, false_negative_rate
on the direct-answer corpus, nightly_trend on mock turns.
D-072 binding: the pilot tolerance is ≤ 650ms. The target is < 600ms (C-8). The
test ASSERTS that the measurement infrastructure works (percentiles + flags),
not that the actual latency is under budget.
"""
from __future__ import annotations
import asyncio
import datetime as _dt
import json
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from server.assist.guardrail_metrics import GuardrailMetrics
from server.assist.latency_metrics import (
PILOT_TOLERANCE_MS,
TARGET_MS,
AssistLatencyMetrics,
)
from server.latency import LatencyRecord
# ── AssistLatencyMetrics ────────────────────────────────────────────────────
def _record(e2e_ms: float) -> LatencyRecord:
"""Build a LatencyRecord with a specific e2e_asr_to_tts_ms value."""
# e2e = tts_first_audio_ms - transcript_ready_ms. Use a non-zero base
# because LatencyRecord.e2e_asr_to_tts_ms guards on truthiness (0.0 is falsy).
base = 100.0
return LatencyRecord(
transcript_ready_ms=base,
tts_first_audio_ms=base + e2e_ms,
)
def test_latency_empty_returns_none():
m = AssistLatencyMetrics()
assert m.p50() is None
assert m.p95() is None
assert m.p99() is None
s = m.summary()
assert s["count"] == 0
assert s["p95"] is None
assert s["within_target"] is False # no records → not within target
assert s["within_pilot"] is False
def test_latency_p95_p50_p99_computed():
"""100 mock records: some <600ms, some 600-650ms, some >650ms.
Verifies p50/p95/p99 are computed correctly + the within_target/within_pilot
flags reflect the p95 against the D-072 thresholds.
"""
m = AssistLatencyMetrics()
# 80 records < 600ms (within target), 15 records 600-650ms (within pilot),
# 5 records > 650ms (over pilot tolerance).
for i in range(80):
m.record(_record(500.0 + i)) # 500..579ms
for i in range(15):
m.record(_record(610.0 + i)) # 610..624ms
for i in range(5):
m.record(_record(700.0 + i)) # 700..704ms
s = m.summary()
assert s["count"] == 100
assert s["p50"] is not None
assert s["p95"] is not None
assert s["p99"] is not None
# p50 should be in the < 600ms range (median of the 80 < 600ms records).
assert s["p50"] < 600.0
# p95: nearest-rank index = ceil(0.95 * 100) - 1 = 94 (0-indexed) → the 95th
# sorted value. 80 records are 500..579, 15 are 610..624, 5 are 700..704.
# Sorted: [500..579 (80), 610..624 (15), 700..704 (5)]. Index 94 → 610..624
# range (index 80..94 = the 610..624 set; index 94 = 624.0).
assert 610.0 <= s["p95"] <= 625.0
# p99: index = ceil(0.99 * 100) - 1 = 98 → the 99th sorted value (700..704).
assert s["p99"] >= 700.0
# D-072: within_target = (p95 < 600). p95 is ~624 → not within target.
assert s["within_target"] is False
# D-072: within_pilot = (p95 <= 650). p95 is ~624 → within pilot.
assert s["within_pilot"] is True
# D-072 thresholds documented in the summary.
assert s["target_ms"] == TARGET_MS == 600
assert s["pilot_tolerance_ms"] == PILOT_TOLERANCE_MS == 650
def test_latency_within_target_when_p95_under_600():
"""All records < 600ms → within_target=True, within_pilot=True."""
m = AssistLatencyMetrics()
for i in range(20):
m.record(_record(400.0 + i)) # 400..419ms
s = m.summary()
assert s["p95"] < 600.0
assert s["within_target"] is True
assert s["within_pilot"] is True
def test_latency_over_pilot_when_p95_over_650():
"""All records > 650ms → within_target=False, within_pilot=False."""
m = AssistLatencyMetrics()
for i in range(20):
m.record(_record(700.0 + i)) # 700..719ms
s = m.summary()
assert s["p95"] > 650.0
assert s["within_target"] is False
assert s["within_pilot"] is False
def test_latency_pilot_boundary_exactly_650():
"""D-072 boundary: p95 == 650 → within_pilot=True (≤ is inclusive)."""
m = AssistLatencyMetrics()
# 20 records all exactly 650ms → p95 = 650.0
for _ in range(20):
m.record(_record(650.0))
s = m.summary()
assert s["p95"] == 650.0
assert s["within_pilot"] is True # ≤ 650 (inclusive)
assert s["within_target"] is False # < 600 (strict)
def test_latency_d072_thresholds_documented():
"""D-072: the pilot tolerance (≤650ms) + target (<600ms) are documented."""
assert TARGET_MS == 600
assert PILOT_TOLERANCE_MS == 650
assert PILOT_TOLERANCE_MS > TARGET_MS # pilot tolerance is more lenient
# ── GuardrailMetrics ────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_guardrail_fp_rate_on_coaching_corpus():
"""FP rate on the tuning corpus < 5% (REQ-IDEATE-04 target)."""
gm = GuardrailMetrics()
rate, mis, total = await gm.false_positive_rate()
print(f"\n[nfr] guardrail FP rate: {rate:.1%} ({mis}/{total})")
assert rate < 0.05, (
f"guardrail FP rate {rate:.1%} exceeds 5% target — the regex is "
f"over-matching coaching responses. {mis}/{total} blocked."
)
@pytest.mark.asyncio
async def test_guardrail_fn_rate_on_direct_corpus():
"""FN rate on the direct-answer corpus < 5% (REQ-IDEATE-04 target)."""
gm = GuardrailMetrics()
rate, mis, total = await gm.false_negative_rate()
print(f"\n[nfr] guardrail FN rate: {rate:.1%} ({mis}/{total})")
assert rate < 0.05, (
f"guardrail FN rate {rate:.1%} exceeds 5% target — the regex is "
f"under-matching direct answers. {mis}/{total} allowed."
)
@pytest.mark.asyncio
async def test_guardrail_adversarial_fn_measured():
"""Adversarial FN rate measured + reported (G-067 — ≤ 20% pilot threshold).
This test does NOT assert the 5% target (the adversarial set is the
residual-risk set, not the tuning target). It asserts the measurement
infrastructure works + the rate is within the G-067 pilot threshold (≤ 20%).
"""
gm = GuardrailMetrics()
rate, mis, total = await gm.adversarial_false_negative_rate()
print(f"\n[nfr] guardrail adversarial FN rate: {rate:.1%} ({mis}/{total})")
# G-067: ≤ 20% pilot threshold (the binding contract from GRILL-v0.5).
assert rate <= 0.20, (
f"adversarial FN rate {rate:.1%} exceeds G-067 ≤20% threshold — "
f"re-tune the regex or escalate. {mis}/{total} slipped through."
)
@pytest.mark.asyncio
async def test_guardrail_nightly_trend_on_mock_turns(tmp_path: Path):
"""nightly_trend() samples 24h of assist turns + reports fn_candidates.
Seeds a temp SQLite store with assist turns (some coaching, some with
direct-answer heuristic patterns) + verifies the nightly trend detects
fn_candidates.
"""
from db.migrate import apply_migrations
from db.store import PraxisStore
db = tmp_path / "test_nfr_nightly.db"
apply_migrations(db)
store = PraxisStore(db)
await store.init()
# Seed an assist session + turns.
session_id = await store.start_session_typed(
"learner-1", "assist:refund", session_type="assist"
)
# Turn 1: a coaching response (allowed, no fn_candidate).
await store.log_turn_with_verdict(
session_id, 0, role="assistant",
asr_text="customer wants refund",
tts_text="What do you think the customer needs right now?",
latency_ms=580.0,
guardrail_verdict_json=json.dumps({"allowed": True, "category": "coaching"}),
)
# Turn 2: a direct-answer response that slipped past the guardrail
# (allowed=True in the verdict, but the heuristic catches it).
await store.log_turn_with_verdict(
session_id, 1, role="assistant",
asr_text="what should I say",
tts_text="You should say: I'm sorry, here's a refund.",
latency_ms=590.0,
guardrail_verdict_json=json.dumps({"allowed": True, "category": "coaching"}),
)
# Turn 3: a blocked response (guardrail caught it).
await store.log_turn_with_verdict(
session_id, 2, role="assistant",
asr_text="help me",
tts_text="Tell the customer: we will issue a full refund now.",
latency_ms=570.0,
guardrail_verdict_json=json.dumps({"allowed": False, "category": "blocked_direct_script"}),
)
gm = GuardrailMetrics()
trend = await gm.nightly_trend(store)
assert trend["total_turns"] == 3
assert trend["blocked"] == 1
# Turn 2 should be flagged as an fn_candidate. The guardrail re-check may
# catch it as a regression (it now blocks what it previously allowed) OR
# the heuristic may catch it as a direct-answer pattern. Either way, it
# must appear in fn_candidates.
assert len(trend["fn_candidates"]) >= 1
seqs = [c.get("turn_seq") for c in trend["fn_candidates"]]
assert 1 in seqs, "turn 2 (direct-answer that slipped past) must be flagged"
@pytest.mark.asyncio
async def test_guardrail_nightly_trend_empty_store(tmp_path: Path):
"""nightly_trend() on an empty store returns zeros + no fn_candidates."""
from db.migrate import apply_migrations
from db.store import PraxisStore
db = tmp_path / "test_nfr_nightly_empty.db"
apply_migrations(db)
store = PraxisStore(db)
await store.init()
gm = GuardrailMetrics()
trend = await gm.nightly_trend(store)
assert trend["total_turns"] == 0
assert trend["blocked"] == 0
assert trend["fn_candidates"] == []
@pytest.mark.asyncio
async def test_guardrail_nightly_trend_excludes_practice_turns(tmp_path: Path):
"""nightly_trend() only samples assist turns (not practice turns)."""
from db.migrate import apply_migrations
from db.store import PraxisStore
db = tmp_path / "test_nfr_nightly_practice.db"
apply_migrations(db)
store = PraxisStore(db)
await store.init()
# Seed a practice session (NOT assist) with a turn.
practice_id = await store.start_session_typed(
"learner-1", "cs_refund_ca_v01", session_type="practice"
)
await store.log_turn_with_verdict(
practice_id, 0, role="assistant",
asr_text="hello",
tts_text="You should say sorry.",
latency_ms=500.0,
guardrail_verdict_json=json.dumps({"allowed": True, "category": "coaching"}),
)
gm = GuardrailMetrics()
trend = await gm.nightly_trend(store)
# Practice turns must NOT appear in the assist nightly trend.
assert trend["total_turns"] == 0
assert trend["fn_candidates"] == []
+353
View File
@@ -0,0 +1,353 @@
"""P2 integration test — assist aggregation → endpoint → cost → NFR (TASK-12-05).
Requires Postgres (skips if PRAXIS_PG_DSN not set). End-to-end P2 integration:
1. Seed 12 mock assist shifts (12 distinct learners — above k-anon threshold).
2. Run the aggregation hook for each → cohort_aggregates populated with assist metrics.
3. GET /api/operator/cohort (with auth cookie) → returns assist volume (non-suppressed).
4. GET /api/operator/failure-patterns → returns assist_guardrail_block_rate.
5. Seed 5 more assist shifts from 5 NEW distinct learners for a different path →
GET /api/operator/cohort for that path → suppressed cells (5 < 10).
6. Verify assist_p95_latency_ms is in the aggregates.
7. Verify assist_cost_cents is in the session_outcome.
8. Verify the C-3 budget check runs at shift-end.
9. Verify the tech-debt fixes: aggregation cache survives restart (mock),
cookie-secret warning, credential status enum, argon2id offloaded.
G-038 differencing-attack e2e: k-anon threshold enforced (12 not suppressed,
5 suppressed). No per-learner data in any response.
"""
from __future__ import annotations
import asyncio
import datetime as _dt
import os
from unittest.mock import AsyncMock, MagicMock
import pytest
pytestmark = pytest.mark.skipif(
not os.environ.get("PRAXIS_PG_DSN"),
reason="PRAXIS_PG_DSN not set — P2 assist integration tests skipped.",
)
@pytest.fixture
async def pg_pool():
import asyncpg
pool = await asyncpg.create_pool(
dsn=os.environ["PRAXIS_PG_DSN"], min_size=1, max_size=5, command_timeout=10,
)
try:
yield pool
finally:
await pool.close()
@pytest.fixture
async def pg_store(pg_pool):
from db.pg_migrate import apply_pg_migrations
from db.pg_store import PgStore
await apply_pg_migrations(pg_pool)
# Clean cohort_aggregates + operators for an isolated run.
async with pg_pool.acquire() as conn:
await conn.execute("TRUNCATE cohort_aggregates, operators, issued_credentials")
return PgStore(pg_pool)
@pytest.fixture
async def authed_client(pg_store):
"""A TestClient with auth + the operator routers wired to pg_store."""
from fastapi import FastAPI
from fastapi.testclient import TestClient
from starlette.middleware.sessions import SessionMiddleware
from server.auth.dependencies import current_operator
from server.auth.models import Operator
from server.operator.cohort import router as cohort_router
from server.operator.failure_patterns import router as failure_router
from server.operator.mastery import router as mastery_router
app = FastAPI()
app.state.pg_store = pg_store
app.add_middleware(SessionMiddleware, secret_key="test-secret-1234567890abcdef1234567890")
app.include_router(cohort_router)
app.include_router(failure_router)
app.include_router(mastery_router)
# Stub auth — every request is operator "integration-tester".
async def _stub_op():
return Operator(id="op-1", username="tester", display_name="T", role="operator")
app.dependency_overrides[current_operator] = _stub_op
return TestClient(app)
def _assist_outcome(
learner_ref: str,
path: str = "customer_service",
turn_count: int = 20,
blocks: int = 2,
p95_latency_ms: float = 580.0,
cost_cents: int = 20,
) -> dict:
return {
"learner_ref": learner_ref,
"path": path,
"scenario_id": "assist:refund",
"outcome": "completed",
"session_type": "assist",
"rubric_scores": [],
"failure_mode": None,
"branch_path": [],
"assist_turn_count": turn_count,
"guardrail_blocks": blocks,
"assist_p95_latency_ms": p95_latency_ms,
"assist_p50_latency_ms": 500.0,
"assist_p99_latency_ms": 620.0,
"assist_within_pilot": True,
"assist_cost_cents": cost_cents,
"timestamp": _dt.datetime.now(_dt.timezone.utc).isoformat(),
}
@pytest.mark.asyncio
async def test_p2_assist_aggregation_k_anon_threshold(pg_store, authed_client):
"""1-4: 12 assist shifts (12 learners) → non-suppressed; 5 → suppressed."""
from server.cohort.aggregator import aggregate_session
# 1. Seed 12 assist shifts for 'customer_service' (12 distinct learners).
for i in range(12):
await aggregate_session(pg_store, _assist_outcome(f"learner-{i}"))
# 2. Verify cohort_aggregates has assist metrics.
async with pg_store.pool.acquire() as conn:
rows = await conn.fetch(
"SELECT metric, value, cell_count, cell_suppressed "
"FROM cohort_aggregates WHERE path = 'customer_service' "
"AND metric LIKE 'assist_%'"
)
metrics = {r["metric"]: r for r in rows}
assert "assist_shifts_count" in metrics
assert "assist_turns_count" in metrics
assert "assist_active_learners_count" in metrics
assert "assist_guardrail_block_rate" in metrics
# 12 learners → not suppressed.
assert metrics["assist_active_learners_count"]["cell_suppressed"] is False
assert metrics["assist_active_learners_count"]["value"] == 12.0
# 3. GET /api/operator/cohort → returns assist volume (non-suppressed).
r = authed_client.get("/api/operator/cohort")
assert r.status_code == 200
cohort_metrics = {
c["metric"]: c for v in r.json()["views"] if v["path"] == "customer_service"
for c in v["metrics"]
}
assert "assist_shifts_count" in cohort_metrics
assert cohort_metrics["assist_shifts_count"]["cell_suppressed"] is False
# 4. GET /api/operator/failure-patterns → returns assist_guardrail_block_rate.
r = authed_client.get("/api/operator/failure-patterns")
assert r.status_code == 200
fp_metrics = {
c["metric"]: c for v in r.json()["views"] if v["path"] == "customer_service"
for c in v["metrics"]
}
assert "assist_guardrail_block_rate" in fp_metrics
# 5. Seed 5 assist shifts for a DIFFERENT path (5 NEW learners) → suppressed.
for i in range(5):
await aggregate_session(pg_store, _assist_outcome(f"new-learner-{i}", path="retail_sales"))
r = authed_client.get("/api/operator/cohort")
retail_metrics = {
c["metric"]: c for v in r.json()["views"] if v["path"] == "retail_sales"
for c in v["metrics"]
}
assert "assist_shifts_count" in retail_metrics
# 5 < 10 → suppressed.
assert retail_metrics["assist_shifts_count"]["cell_suppressed"] is True
assert retail_metrics["assist_shifts_count"]["value"] is None
@pytest.mark.asyncio
async def test_p2_assist_p95_latency_in_aggregates(pg_store):
"""6: assist_p95_latency_ms is in the aggregates (D-072)."""
from server.cohort.aggregator import aggregate_session
for i in range(12):
await aggregate_session(pg_store, _assist_outcome(f"learner-{i}", p95_latency_ms=580.0))
async with pg_store.pool.acquire() as conn:
row = await conn.fetchrow(
"SELECT value, cell_suppressed FROM cohort_aggregates "
"WHERE path = 'customer_service' AND metric = 'assist_p95_latency_ms'"
)
assert row is not None
assert row["cell_suppressed"] is False
assert row["value"] is not None
# The running mean of per-shift p95 (580.0) → ~580.
assert 570.0 <= float(row["value"]) <= 590.0
@pytest.mark.asyncio
async def test_p2_assist_cost_cents_in_session_outcome():
"""7: assist_cost_cents is in the session_outcome (TASK-11-01)."""
from server.assist.session import AssistSession
from server.assist.context import AssistContext
ctx = AssistContext(
system_prompt="", current_week=1, scenario_tag="refund",
theta=0.0, coaching_focus="empathy", path_slug="customer_service",
)
session = AssistSession.__new__(AssistSession)
session.assist_cost_cents = 0
session.turn_count = 3
session.guardrail_block_count = 0
session.latency_metrics = MagicMock()
session.latency_metrics.summary = MagicMock(return_value={
"p50": 500.0, "p95": 580.0, "p99": 620.0, "count": 3,
"target_ms": 600, "pilot_tolerance_ms": 650,
"within_target": True, "within_pilot": True,
})
session.context = ctx
session.learner_id = "learner-1"
session.session_id = "test-session"
# Add 3 turns of cost.
session.add_assist_turn_cost(5)
session.add_assist_turn_cost(10)
session.add_assist_turn_cost(3)
outcome = session._build_session_outcome("completed")
assert outcome["assist_cost_cents"] == 18 # 5 + 10 + 3
assert outcome["session_type"] == "assist"
assert outcome["assist_p95_latency_ms"] == 580.0
@pytest.mark.asyncio
async def test_p2_c3_budget_check_runs():
"""8: the C-3 budget check runs + reports within_budget (TASK-11-02)."""
from server.assist.budget_check import C3_TARGET_USD, check_c3_budget
# 20 turns/shift × 20 shifts/month at 0.05 cents/turn → $0.20/month.
result = check_c3_budget(
assist_turns_per_shift=20,
shifts_per_month=20,
cost_per_turn_cents=0.05,
)
assert result["within_budget"] is True
assert result["total_with_practice"] <= C3_TARGET_USD
assert result["c3_target"] == 3.0
@pytest.mark.asyncio
async def test_p2_techdebt_aggregation_cache_survives_restart(pg_store, tmp_path):
"""9a: aggregation cache survives a restart (TASK-12-01, P1+ #7)."""
from server.cohort.aggregator import aggregate_session
from server.cohort.learner_cache import (
_clear_learner_cache,
_count_distinct_learners,
_load_learner_cache,
)
# Point the cache to a temp file.
pg_store.cohort_cache_db_path = str(tmp_path / "cache.db")
# Seed 10 learners.
for i in range(10):
await aggregate_session(pg_store, _assist_outcome(f"learner-{i}"))
# The persisted cache should have 10 distinct learners for this path.
window_start = (_dt.datetime.now(_dt.timezone.utc).date() - _dt.timedelta(days=6))
count = await _count_distinct_learners(pg_store, "customer_service", window_start)
assert count == 10
# Simulate a restart: clear the in-memory cache + reload from SQLite.
if hasattr(pg_store, "_agg_cache"):
del pg_store._agg_cache
loaded = await _load_learner_cache(pg_store)
key = ("customer_service", "__learners__", window_start)
assert key in loaded
assert len(loaded[key]) == 10 # survived the "restart"
# Clear the cache (nightly reconciliation).
await _clear_learner_cache(pg_store)
count_after_clear = await _count_distinct_learners(pg_store, "customer_service", window_start)
assert count_after_clear == 0
@pytest.mark.asyncio
async def test_p2_techdebt_cookie_secret_warning(monkeypatch):
"""9b: cookie-secret <32 bytes logs a WARNING (TASK-12-02, P1+ #3)."""
from loguru import logger as _logger
from server.auth.cookies import get_session_middleware_kwargs
monkeypatch.setenv("PRAXIS_COOKIE_SECRET", "short") # 5 bytes < 32
monkeypatch.setenv("PRAXIS_COOKIE_SECURE", "true")
msgs: list[str] = []
sink_id = _logger.add(lambda m: msgs.append(str(m)), level="WARNING")
try:
kw = get_session_middleware_kwargs()
finally:
_logger.remove(sink_id)
assert kw["secret_key"] == "short" # accepted (backward compat)
assert any("<32 bytes" in m for m in msgs)
@pytest.mark.asyncio
async def test_p2_techdebt_credential_status_enum():
"""9c: set_credential_status enum validation (TASK-12-03, P1+ #4)."""
from db.pg_store import PgStore
pool = MagicMock()
conn = MagicMock()
conn.execute = AsyncMock()
cm = MagicMock()
cm.__aenter__ = AsyncMock(return_value=conn)
cm.__aexit__ = AsyncMock(return_value=None)
pool.acquire = MagicMock(return_value=cm)
store = PgStore(pool)
# Invalid status → ValueError.
with pytest.raises(ValueError, match="Invalid credential status"):
await store.set_credential_status("cred-1", "deleted")
# Valid statuses work.
await store.set_credential_status("cred-1", "revoked")
await store.set_credential_status("cred-1", "active")
@pytest.mark.asyncio
async def test_p2_techdebt_argon2id_offloaded():
"""9d: argon2id verify_password offloaded to asyncio.to_thread (P1+ #1)."""
import asyncio as _asyncio
import server.auth.routes as _routes_mod
from server.auth.passwords import hash_password
# The login handler should use asyncio.to_thread for verify_password.
# Verify the module imports asyncio + the handler references to_thread.
assert hasattr(_routes_mod, "asyncio")
assert _asyncio.to_thread is _routes_mod.asyncio.to_thread
# Functional check: verify_password is callable via to_thread.
h = hash_password("pw")
result = await _asyncio.to_thread(_routes_mod.verify_password, h, "pw")
assert result is True
@pytest.mark.asyncio
async def test_p2_no_per_learner_data_in_responses(pg_store, authed_client):
"""No per-learner data in any dashboard response (D-031, G-038)."""
from server.cohort.aggregator import aggregate_session
for i in range(12):
await aggregate_session(pg_store, _assist_outcome(f"learner-sensitive-{i}"))
for endpoint in ("/api/operator/cohort", "/api/operator/failure-patterns", "/api/operator/mastery"):
r = authed_client.get(endpoint)
assert r.status_code == 200
# No learner ref should appear in the response.
text = r.text
assert "learner-sensitive-" not in text, \
f"per-learner data leaked in {endpoint} response"