"""SLICE-07 TASK-07-04 — IRT selection integration (next-scenario recommendation). Verifies that `library.select_for_theta` + `irt.select_scenario` pick the right scenario for a given (theta, path) pair. Tests both cold-start (observations < 5 → difficulty-based) and warm-start (>= 5 → theta-based) selection paths against the real scenario library + index. """ from __future__ import annotations import math from pathlib import Path from unittest.mock import MagicMock import pytest from server.mastery.irt import ( COLD_START_MIN_OBSERVATIONS, DEFAULT_THETA, IRTEngine, ) from server.scenarios.library import ScenarioLibrary from server.scenarios.schema import Scenario _SCENARIOS_DIR = Path(__file__).resolve().parent.parent / "scenarios" def _library() -> ScenarioLibrary: return ScenarioLibrary(scenarios_dir=_SCENARIOS_DIR) def _logit(p: float) -> float: return math.log(p / (1.0 - p)) # ── warm-start: delegates to library.select_for_theta ───────────────────────── def test_warm_start_selects_scenario_near_target_p(): library = _library() theta = 1.0 target_p = 0.7 selected = IRTEngine.select_scenario( theta=theta, library=library, path="customer_service", target_p=target_p, observations=COLD_START_MIN_OBSERVATIONS, ) assert selected is not None assert isinstance(selected, Scenario) # The selected scenario's difficulty should be the closest to theta - logit(p). entries = library.list_by_path("customer_service") target_b = theta - _logit(target_p) best_id = min(entries, key=lambda e: abs(float(e.difficulty) - target_b)).id assert selected.id == best_id def test_warm_start_low_theta_picks_easiest(): library = _library() selected = IRTEngine.select_scenario( theta=-3.0, library=library, path="customer_service", target_p=0.7, observations=10, ) assert selected is not None entries = library.list_by_path("customer_service") easiest = min(entries, key=lambda e: e.difficulty) assert selected.id == easiest.id def test_warm_start_high_theta_picks_hardest(): library = _library() selected = IRTEngine.select_scenario( theta=10.0, library=library, path="customer_service", target_p=0.7, observations=10, ) assert selected is not None entries = library.list_by_path("customer_service") hardest = max(entries, key=lambda e: e.difficulty) assert selected.id == hardest.id def test_warm_start_target_p_half_uses_theta_directly(): library = _library() theta = 3.0 selected = IRTEngine.select_scenario( theta=theta, library=library, path="customer_service", target_p=0.5, observations=COLD_START_MIN_OBSERVATIONS, ) assert selected is not None # logit(0.5) == 0 → target_b == theta. entries = library.list_by_path("customer_service") best_id = min(entries, key=lambda e: abs(float(e.difficulty) - theta)).id assert selected.id == best_id # ── cold-start: difficulty-based fallback (observations < 5) ────────────────── def test_cold_start_uses_difficulty_not_theta_based_selection(): library = _library() theta = 2.0 target_p = 0.7 cold = IRTEngine.select_scenario( theta=theta, library=library, path="customer_service", target_p=target_p, observations=COLD_START_MIN_OBSERVATIONS - 1, ) # Cold-start target difficulty = clamp(round(theta + logit(target_p)), 1, 5). target_difficulty = max(1, min(5, round(theta + _logit(target_p)))) entries = library.list_by_path("customer_service") expected = min(entries, key=lambda e: abs(e.difficulty - target_difficulty)) assert cold is not None assert cold.id == expected.id def test_cold_start_boundary_observations_just_below_threshold(): library = _library() selected = IRTEngine.select_scenario( theta=0.0, library=library, path="customer_service", target_p=0.7, observations=COLD_START_MIN_OBSERVATIONS - 1, ) assert selected is not None # At theta=0 + logit(0.7) ≈ 0.847 → round → 1 → easiest scenario. entries = library.list_by_path("customer_service") easiest = min(entries, key=lambda e: e.difficulty) assert selected.id == easiest.id def test_cold_start_at_threshold_switches_to_warm(): """At exactly COLD_START_MIN_OBSERVATIONS, warm-start takes over.""" library = _library() theta = 1.5 selected_warm = IRTEngine.select_scenario( theta=theta, library=library, path="customer_service", target_p=0.7, observations=COLD_START_MIN_OBSERVATIONS, ) # Compare against the warm-start selection directly. expected = library.select_for_theta(theta, "customer_service", target_p=0.7) assert selected_warm is not None assert expected is not None assert selected_warm.id == expected.id def test_cold_start_clamps_high_theta_to_hardest(): library = _library() selected = IRTEngine.select_scenario( theta=10.0, library=library, path="customer_service", target_p=0.7, observations=0, ) assert selected is not None entries = library.list_by_path("customer_service") hardest = max(entries, key=lambda e: e.difficulty) assert selected.id == hardest.id def test_cold_start_clamps_low_theta_to_easiest(): library = _library() selected = IRTEngine.select_scenario( theta=-10.0, library=library, path="customer_service", target_p=0.7, observations=2, ) assert selected is not None entries = library.list_by_path("customer_service") easiest = min(entries, key=lambda e: e.difficulty) assert selected.id == easiest.id # ── empty-path guard ─────────────────────────────────────────────────────────── def test_select_returns_none_for_unknown_path_warm_start(): library = _library() selected = IRTEngine.select_scenario( theta=1.0, library=library, path="nonexistent_path", target_p=0.7, observations=10, ) assert selected is None def test_select_returns_none_for_unknown_path_cold_start(): library = _library() selected = IRTEngine.select_scenario( theta=1.0, library=library, path="nonexistent_path", target_p=0.7, observations=0, ) assert selected is None # ── library.select_for_theta direct contract ────────────────────────────────── def test_library_select_for_theta_targets_predicted_p(): library = _library() theta = 0.0 target_p = 0.7 selected = library.select_for_theta(theta, "customer_service", target_p=target_p) assert selected is not None # Predicted P for the selected scenario's difficulty should be the closest # to target_p among all scenarios in the path. entries = library.list_by_path("customer_service") predicted = { e.id: IRTEngine.P_success(theta, float(e.difficulty)) for e in entries } closest = min(predicted, key=lambda sid: abs(predicted[sid] - target_p)) assert selected.id == closest def test_library_select_for_theta_is_deterministic(): library = _library() a = library.select_for_theta(1.2, "customer_service", target_p=0.7) b = library.select_for_theta(1.2, "customer_service", target_p=0.7) assert a is not None and b is not None assert a.id == b.id