praxis/tests/test_irt.py

"""Unit tests for the IRT engine (SLICE-04, TASK-04-03)."""

from __future__ import annotations

from pathlib import Path
from unittest.mock import MagicMock

import pytest

from server.mastery.irt import (
    COLD_START_MIN_OBSERVATIONS,
    IRTEngine,
)
from server.scenarios.schema import Scenario


def test_p_success_theta_equals_b_is_half():
    assert IRTEngine.P_success(0.0, 0.0) == pytest.approx(0.5)
    assert IRTEngine.P_success(2.5, 2.5) == pytest.approx(0.5)


def test_p_success_theta_above_b_above_half():
    assert IRTEngine.P_success(1.0, 0.0) > 0.5
    assert IRTEngine.P_success(3.0, 1.0) > 0.5
    assert IRTEngine.P_success(0.0, -1.0) > 0.5


def test_p_success_theta_below_b_below_half():
    assert IRTEngine.P_success(0.0, 1.0) < 0.5
    assert IRTEngine.P_success(-2.0, 0.0) < 0.5


def test_p_success_in_range():
    for theta in [-3.0, -1.0, 0.0, 1.0, 3.0]:
        for b in [-2.0, 0.0, 2.0]:
            p = IRTEngine.P_success(theta, b)
            assert 0.0 < p < 1.0


def test_update_theta_success_increases():
    theta, sigma_sq = 0.0, 1.0
    b = 0.0
    for _ in range(10):
        theta, sigma_sq = IRTEngine.update_theta(theta, sigma_sq, 1.0, b)
    assert theta > 0.0


def test_update_theta_failure_decreases():
    theta, sigma_sq = 0.0, 1.0
    b = 0.0
    for _ in range(10):
        theta, sigma_sq = IRTEngine.update_theta(theta, sigma_sq, 0.0, b)
    assert theta < 0.0


def test_update_theta_sigma_sq_shrages_each_observation():
    theta, sigma_sq = 0.0, 1.0
    b = 0.5
    prev = sigma_sq
    for _ in range(10):
        theta, sigma_sq = IRTEngine.update_theta(theta, sigma_sq, 1.0, b)
        assert sigma_sq < prev
        prev = sigma_sq


def test_select_scenario_cold_start_uses_difficulty():
    library = MagicMock()
    entries = [
        MagicMock(id="easy", difficulty=1),
        MagicMock(id="mid", difficulty=3),
        MagicMock(id="hard", difficulty=5),
    ]
    library.list_by_path.return_value = entries
    library.get.side_effect = lambda sid: MagicMock(id=sid)

    selected = IRTEngine.select_scenario(
        theta=2.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=0,
    )
    assert selected is not None
    library.list_by_path.assert_called_once_with("customer_service")
    library.get.assert_called_once()
    chosen_id = library.get.call_args.args[0]
    assert chosen_id == "mid"


def test_select_scenario_cold_start_clamps_to_range():
    library = MagicMock()
    entries = [
        MagicMock(id="easy", difficulty=1),
        MagicMock(id="mid", difficulty=3),
        MagicMock(id="hard", difficulty=5),
    ]
    library.list_by_path.return_value = entries
    library.get.side_effect = lambda sid: MagicMock(id=sid)

    selected = IRTEngine.select_scenario(
        theta=10.0,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=2,
    )
    assert selected is not None
    chosen_id = library.get.call_args.args[0]
    assert chosen_id == "hard"


def test_select_scenario_cold_start_threshold_boundary():
    library = MagicMock()
    library.list_by_path.return_value = [MagicMock(id="only", difficulty=3)]
    library.get.side_effect = lambda sid: MagicMock(id=sid)

    IRTEngine.select_scenario(
        theta=0.5,
        library=library,
        path="customer_service",
        observations=COLD_START_MIN_OBSERVATIONS - 1,
    )
    library.list_by_path.assert_called_once()
    library.get.assert_called_once()


def test_select_scenario_warm_start_delegates_to_library():
    library = MagicMock()
    expected = MagicMock(spec=Scenario)
    library.select_for_theta.return_value = expected

    selected = IRTEngine.select_scenario(
        theta=1.2,
        library=library,
        path="customer_service",
        target_p=0.7,
        observations=COLD_START_MIN_OBSERVATIONS,
    )
    assert selected is expected
    library.select_for_theta.assert_called_once_with(1.2, "customer_service", target_p=0.7)
    library.list_by_path.assert_not_called()


def test_select_scenario_cold_start_empty_library_returns_none():
    library = MagicMock()
    library.list_by_path.return_value = []

    selected = IRTEngine.select_scenario(
        theta=0.0,
        library=library,
        path="customer_service",
        observations=0,
    )
    assert selected is None


def test_select_scenario_warm_start_delegates_target_p():
    library = MagicMock()
    expected = MagicMock(spec=Scenario)
    library.select_for_theta.return_value = expected

    IRTEngine.select_scenario(
        theta=0.8,
        library=library,
        path="customer_service",
        target_p=0.5,
        observations=10,
    )
    library.select_for_theta.assert_called_once_with(0.8, "customer_service", target_p=0.5)


def test_update_theta_converges_to_b_with_sampled_outcomes():
    import random

    rng = random.Random(0)
    b = 2.0
    final_thetas = []
    for _ in range(50):
        theta, sigma_sq = 0.0, 1.0
        for _ in range(100):
            p_true = IRTEngine.P_success(b, b)
            outcome = 1.0 if rng.random() < p_true else 0.0
            theta, sigma_sq = IRTEngine.update_theta(theta, sigma_sq, outcome, b)
        final_thetas.append(theta)
    mean_theta = sum(final_thetas) / len(final_thetas)
    assert mean_theta > 0.0
    assert abs(mean_theta - b) < 1.0