praxis/tests/test_irt_selection_integration.py

"""SLICE-07 TASK-07-04 — IRT selection integration (next-scenario recommendation).

Verifies that `library.select_for_theta` + `irt.select_scenario` pick the right
scenario for a given (theta, path) pair. Tests both cold-start
(observations < 5 → difficulty-based) and warm-start (>= 5 → theta-based)
selection paths against the real scenario library + index.
"""

from __future__ import annotations

import math
from pathlib import Path
from unittest.mock import MagicMock

import pytest

from server.mastery.irt import (
    COLD_START_MIN_OBSERVATIONS,
    DEFAULT_THETA,
    IRTEngine,
)
from server.scenarios.library import ScenarioLibrary
from server.scenarios.schema import Scenario

_SCENARIOS_DIR = Path(__file__).resolve().parent.parent / "scenarios"


def _library() -> ScenarioLibrary:
    return ScenarioLibrary(scenarios_dir=_SCENARIOS_DIR)


def _logit(p: float) -> float:
    return math.log(p / (1.0 - p))


# ── warm-start: delegates to library.select_for_theta ─────────────────────────


def test_warm_start_selects_scenario_near_target_p():
    library = _library()
    theta = 1.0
    target_p = 0.7
    selected = IRTEngine.select_scenario(
        theta=theta,
        library=library,
        path="customer_service",
        target_p=target_p,
        observations=COLD_START_MIN_OBSERVATIONS,
    )
    assert selected is not None
    assert isinstance(selected, Scenario)
    # The selected scenario's difficulty should be the closest to theta - logit(p).
    entries = library.list_by_path("customer_service")
    target_b = theta - _logit(target_p)
    best_id = min(entries, key=lambda e: abs(float(e.difficulty) - target_b)).id
    assert selected.id == best_id


def test_warm_start_low_theta_picks_easiest():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=-3.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=10,
    )
    assert selected is not None
    entries = library.list_by_path("customer_service")
    easiest = min(entries, key=lambda e: e.difficulty)
    assert selected.id == easiest.id


def test_warm_start_high_theta_picks_hardest():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=10.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=10,
    )
    assert selected is not None
    entries = library.list_by_path("customer_service")
    hardest = max(entries, key=lambda e: e.difficulty)
    assert selected.id == hardest.id


def test_warm_start_target_p_half_uses_theta_directly():
    library = _library()
    theta = 3.0
    selected = IRTEngine.select_scenario(
        theta=theta,
        library=library,
        path="customer_service",
        target_p=0.5,
        observations=COLD_START_MIN_OBSERVATIONS,
    )
    assert selected is not None
    # logit(0.5) == 0 → target_b == theta.
    entries = library.list_by_path("customer_service")
    best_id = min(entries, key=lambda e: abs(float(e.difficulty) - theta)).id
    assert selected.id == best_id


# ── cold-start: difficulty-based fallback (observations < 5) ──────────────────


def test_cold_start_uses_difficulty_not_theta_based_selection():
    library = _library()
    theta = 2.0
    target_p = 0.7
    cold = IRTEngine.select_scenario(
        theta=theta,
        library=library,
        path="customer_service",
        target_p=target_p,
        observations=COLD_START_MIN_OBSERVATIONS - 1,
    )
    # Cold-start target difficulty = clamp(round(theta + logit(target_p)), 1, 5).
    target_difficulty = max(1, min(5, round(theta + _logit(target_p))))
    entries = library.list_by_path("customer_service")
    expected = min(entries, key=lambda e: abs(e.difficulty - target_difficulty))
    assert cold is not None
    assert cold.id == expected.id


def test_cold_start_boundary_observations_just_below_threshold():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=0.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=COLD_START_MIN_OBSERVATIONS - 1,
    )
    assert selected is not None
    # At theta=0 + logit(0.7) ≈ 0.847 → round → 1 → easiest scenario.
    entries = library.list_by_path("customer_service")
    easiest = min(entries, key=lambda e: e.difficulty)
    assert selected.id == easiest.id


def test_cold_start_at_threshold_switches_to_warm():
    """At exactly COLD_START_MIN_OBSERVATIONS, warm-start takes over."""
    library = _library()
    theta = 1.5
    selected_warm = IRTEngine.select_scenario(
        theta=theta,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=COLD_START_MIN_OBSERVATIONS,
    )
    # Compare against the warm-start selection directly.
    expected = library.select_for_theta(theta, "customer_service", target_p=0.7)
    assert selected_warm is not None
    assert expected is not None
    assert selected_warm.id == expected.id


def test_cold_start_clamps_high_theta_to_hardest():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=10.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=0,
    )
    assert selected is not None
    entries = library.list_by_path("customer_service")
    hardest = max(entries, key=lambda e: e.difficulty)
    assert selected.id == hardest.id


def test_cold_start_clamps_low_theta_to_easiest():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=-10.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=2,
    )
    assert selected is not None
    entries = library.list_by_path("customer_service")
    easiest = min(entries, key=lambda e: e.difficulty)
    assert selected.id == easiest.id


# ── empty-path guard ───────────────────────────────────────────────────────────


def test_select_returns_none_for_unknown_path_warm_start():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=1.0,
        library=library,
        path="nonexistent_path",
        target_p=0.7,
        observations=10,
    )
    assert selected is None


def test_select_returns_none_for_unknown_path_cold_start():
    library = _library()
    selected = IRTEngine.select_scenario(
        theta=1.0,
        library=library,
        path="nonexistent_path",
        target_p=0.7,
        observations=0,
    )
    assert selected is None


# ── library.select_for_theta direct contract ──────────────────────────────────


def test_library_select_for_theta_targets_predicted_p():
    library = _library()
    theta = 0.0
    target_p = 0.7
    selected = library.select_for_theta(theta, "customer_service", target_p=target_p)
    assert selected is not None
    # Predicted P for the selected scenario's difficulty should be the closest
    # to target_p among all scenarios in the path.
    entries = library.list_by_path("customer_service")
    predicted = {
        e.id: IRTEngine.P_success(theta, float(e.difficulty)) for e in entries
    }
    closest = min(predicted, key=lambda sid: abs(predicted[sid] - target_p))
    assert selected.id == closest


def test_library_select_for_theta_is_deterministic():
    library = _library()
    a = library.select_for_theta(1.2, "customer_service", target_p=0.7)
    b = library.select_for_theta(1.2, "customer_service", target_p=0.7)
    assert a is not None and b is not None
    assert a.id == b.id