praxis/tests/test_scenario_library_content.py

"""Scenario library content validation tests (SLICE-06, TASK-06-03).

Verifies the 6 Customer Service scenarios authored in SLICE-06:
  - all 6 load via the Pydantic schema (no validation errors)
  - rubric_criteria reference only valid criterion ids from rubrics/customer_service.yaml
  - each rubric criterion is exercised by >= MIN_COVERAGE (2) scenarios (check_coverage)
  - version is valid semver (1.0.0)
  - scenarios/index.yaml is in sync with the scenario files (ids + versions match)
  - path.validate_scenarios_exist(library) passes for paths/customer_service.yaml
"""

from __future__ import annotations

from pathlib import Path

import pytest

from server.mastery.rubric_loader import load_rubric
from server.paths.engine import PathEngine
from server.scenarios.library import ScenarioLibrary
from server.scenarios.loader import load
from server.scenarios.schema import Scenario

_REPO_SCENARIOS_DIR = Path(__file__).resolve().parent.parent / "scenarios"

EXPECTED_SCENARIO_IDS = [
    "cs_refund_ca_v01",
    "cs_escalation_ca_v02",
    "cs_policy_exception_ca_v03",
    "cs_multi_issue_ca_v04",
    "cs_recovery_ca_v05",
    "cs_mastery_demonstration_ca_v06",
]

VALID_CRITERION_IDS = {"empathy", "resolution", "de_escalation", "professionalism"}


@pytest.fixture(scope="module")
def library() -> ScenarioLibrary:
    lib = ScenarioLibrary()
    lib.load()
    return lib


@pytest.fixture(scope="module")
def rubric():
    return load_rubric("customer_service")


def test_all_six_scenarios_load_via_schema():
    for sid in EXPECTED_SCENARIO_IDS:
        s = load(sid)
        assert isinstance(s, Scenario)
        assert s.id == sid


def test_each_scenario_rubric_criteria_reference_valid_ids(rubric):
    valid = set(rubric.criterion_ids())
    assert valid == VALID_CRITERION_IDS
    for sid in EXPECTED_SCENARIO_IDS:
        s = load(sid)
        assert s.rubric_criteria, f"scenario {sid} has no rubric_criteria"
        for m in s.rubric_criteria:
            assert m.criterion_id in valid, (
                f"scenario {sid} references unknown criterion {m.criterion_id!r}"
            )


def test_each_scenario_covers_all_four_criteria():
    for sid in EXPECTED_SCENARIO_IDS:
        s = load(sid)
        ids = set(s.rubric_criterion_ids())
        assert ids == VALID_CRITERION_IDS, (
            f"scenario {sid} rubric criteria {ids} != {VALID_CRITERION_IDS}"
        )


def test_min_coverage_per_criterion_satisfied(library):
    counts = library.check_coverage("customer_service")
    assert counts, "check_coverage returned empty counts"
    for cid in VALID_CRITERION_IDS:
        assert cid in counts, f"criterion {cid!r} not covered by any scenario"
        assert counts[cid] >= ScenarioLibrary.MIN_COVERAGE, (
            f"criterion {cid!r} covered by {counts[cid]} scenarios "
            f"< MIN_COVERAGE={ScenarioLibrary.MIN_COVERAGE}"
        )


def test_each_scenario_has_valid_semver():
    for sid in EXPECTED_SCENARIO_IDS:
        s = load(sid)
        assert s.version == "1.0.0", f"scenario {sid} version={s.version!r}"


def test_irt_target_p_defaults():
    for sid in EXPECTED_SCENARIO_IDS:
        s = load(sid)
        if sid == "cs_mastery_demonstration_ca_v06":
            assert s.irt_target_p == 0.5, (
                f"mastery-gate scenario {sid} should have irt_target_p=0.5 (D-035)"
            )
        else:
            assert s.irt_target_p == 0.7, (
                f"practice scenario {sid} should have irt_target_p=0.7"
            )


def test_index_in_sync_with_files(library):
    entries = library.entries()
    index_ids = {e.id for e in entries}
    for sid in EXPECTED_SCENARIO_IDS:
        assert sid in index_ids, f"scenario {sid} missing from index.yaml"
    for e in entries:
        s = library.get(e.id)
        assert s.id == e.id, f"id mismatch: index={e.id!r} yaml={s.id!r}"
        assert s.version == e.version, (
            f"version mismatch for {e.id}: index={e.version!r} yaml={s.version!r}"
        )
        assert s.difficulty == e.difficulty, (
            f"difficulty mismatch for {e.id}: index={e.difficulty} yaml={s.difficulty}"
        )
        assert set(s.rubric_criterion_ids()) == set(e.rubric_criteria), (
            f"rubric_criteria mismatch for {e.id}: "
            f"index={e.rubric_criteria} yaml={s.rubric_criterion_ids()}"
        )


def test_path_validate_scenarios_exist_passes(library):
    engine = PathEngine()
    path = engine.load_path("customer_service")
    referenced = engine.validate_scenarios_exist(path, library)
    assert set(referenced) == set(EXPECTED_SCENARIO_IDS)


def test_scenario_file_paths_resolve(library):
    for e in library.entries():
        p = _REPO_SCENARIOS_DIR / e.path
        assert p.exists(), f"index path {e.path!r} does not resolve to a file"


def test_failure_modes_match_expected():
    expected = {
        "cs_refund_ca_v01": "escalates_unresolved",
        "cs_escalation_ca_v02": "escalates_unresolved",
        "cs_policy_exception_ca_v03": "policy_rigid",
        "cs_multi_issue_ca_v04": "multi_issue_drop",
        "cs_recovery_ca_v05": "recovery_missed",
        "cs_mastery_demonstration_ca_v06": "none",
    }
    for sid, fm in expected.items():
        s = load(sid)
        assert s.failure_mode == fm, f"scenario {sid} failure_mode={s.failure_mode!r} != {fm!r}"


def test_difficulty_progression_one_to_five():
    expected = {
        "cs_refund_ca_v01": 1,
        "cs_escalation_ca_v02": 2,
        "cs_policy_exception_ca_v03": 3,
        "cs_multi_issue_ca_v04": 3,
        "cs_recovery_ca_v05": 4,
        "cs_mastery_demonstration_ca_v06": 5,
    }
    for sid, d in expected.items():
        s = load(sid)
        assert s.difficulty == d, f"scenario {sid} difficulty={s.difficulty} != {d}"


def test_index_author_and_provenance(library):
    for e in library.entries():
        assert e.author == "expert", f"scenario {e.id} author={e.author!r} != 'expert'"
        assert e.generated_from is None, (
            f"expert scenario {e.id} should have no generated_from, got {e.generated_from!r}"
        )


def test_v01_scenario_still_loads_from_subdirectory():
    s = load("cs_refund_ca_v01")
    assert s.id == "cs_refund_ca_v01"
    assert s.rubric_criteria, "v01 extended scenario must have rubric_criteria"
    assert s.branch_by_id("accept_resolution") is not None
    assert s.branch_by_id("escalate") is not None