"""Scenario library content validation tests (SLICE-06, TASK-06-03). Verifies the 6 Customer Service scenarios authored in SLICE-06: - all 6 load via the Pydantic schema (no validation errors) - rubric_criteria reference only valid criterion ids from rubrics/customer_service.yaml - each rubric criterion is exercised by >= MIN_COVERAGE (2) scenarios (check_coverage) - version is valid semver (1.0.0) - scenarios/index.yaml is in sync with the scenario files (ids + versions match) - path.validate_scenarios_exist(library) passes for paths/customer_service.yaml """ from __future__ import annotations from pathlib import Path import pytest from server.mastery.rubric_loader import load_rubric from server.paths.engine import PathEngine from server.scenarios.library import ScenarioLibrary from server.scenarios.loader import load from server.scenarios.schema import Scenario _REPO_SCENARIOS_DIR = Path(__file__).resolve().parent.parent / "scenarios" EXPECTED_SCENARIO_IDS = [ "cs_refund_ca_v01", "cs_escalation_ca_v02", "cs_policy_exception_ca_v03", "cs_multi_issue_ca_v04", "cs_recovery_ca_v05", "cs_mastery_demonstration_ca_v06", ] VALID_CRITERION_IDS = {"empathy", "resolution", "de_escalation", "professionalism"} @pytest.fixture(scope="module") def library() -> ScenarioLibrary: lib = ScenarioLibrary() lib.load() return lib @pytest.fixture(scope="module") def rubric(): return load_rubric("customer_service") def test_all_six_scenarios_load_via_schema(): for sid in EXPECTED_SCENARIO_IDS: s = load(sid) assert isinstance(s, Scenario) assert s.id == sid def test_each_scenario_rubric_criteria_reference_valid_ids(rubric): valid = set(rubric.criterion_ids()) assert valid == VALID_CRITERION_IDS for sid in EXPECTED_SCENARIO_IDS: s = load(sid) assert s.rubric_criteria, f"scenario {sid} has no rubric_criteria" for m in s.rubric_criteria: assert m.criterion_id in valid, ( f"scenario {sid} references unknown criterion {m.criterion_id!r}" ) def test_each_scenario_covers_all_four_criteria(): for sid in EXPECTED_SCENARIO_IDS: s = load(sid) ids = set(s.rubric_criterion_ids()) assert ids == VALID_CRITERION_IDS, ( f"scenario {sid} rubric criteria {ids} != {VALID_CRITERION_IDS}" ) def test_min_coverage_per_criterion_satisfied(library): counts = library.check_coverage("customer_service") assert counts, "check_coverage returned empty counts" for cid in VALID_CRITERION_IDS: assert cid in counts, f"criterion {cid!r} not covered by any scenario" assert counts[cid] >= ScenarioLibrary.MIN_COVERAGE, ( f"criterion {cid!r} covered by {counts[cid]} scenarios " f"< MIN_COVERAGE={ScenarioLibrary.MIN_COVERAGE}" ) def test_each_scenario_has_valid_semver(): for sid in EXPECTED_SCENARIO_IDS: s = load(sid) assert s.version == "1.0.0", f"scenario {sid} version={s.version!r}" def test_irt_target_p_defaults(): for sid in EXPECTED_SCENARIO_IDS: s = load(sid) if sid == "cs_mastery_demonstration_ca_v06": assert s.irt_target_p == 0.5, ( f"mastery-gate scenario {sid} should have irt_target_p=0.5 (D-035)" ) else: assert s.irt_target_p == 0.7, ( f"practice scenario {sid} should have irt_target_p=0.7" ) def test_index_in_sync_with_files(library): entries = library.entries() index_ids = {e.id for e in entries} for sid in EXPECTED_SCENARIO_IDS: assert sid in index_ids, f"scenario {sid} missing from index.yaml" for e in entries: s = library.get(e.id) assert s.id == e.id, f"id mismatch: index={e.id!r} yaml={s.id!r}" assert s.version == e.version, ( f"version mismatch for {e.id}: index={e.version!r} yaml={s.version!r}" ) assert s.difficulty == e.difficulty, ( f"difficulty mismatch for {e.id}: index={e.difficulty} yaml={s.difficulty}" ) assert set(s.rubric_criterion_ids()) == set(e.rubric_criteria), ( f"rubric_criteria mismatch for {e.id}: " f"index={e.rubric_criteria} yaml={s.rubric_criterion_ids()}" ) def test_path_validate_scenarios_exist_passes(library): engine = PathEngine() path = engine.load_path("customer_service") referenced = engine.validate_scenarios_exist(path, library) assert set(referenced) == set(EXPECTED_SCENARIO_IDS) def test_scenario_file_paths_resolve(library): for e in library.entries(): p = _REPO_SCENARIOS_DIR / e.path assert p.exists(), f"index path {e.path!r} does not resolve to a file" def test_failure_modes_match_expected(): expected = { "cs_refund_ca_v01": "escalates_unresolved", "cs_escalation_ca_v02": "escalates_unresolved", "cs_policy_exception_ca_v03": "policy_rigid", "cs_multi_issue_ca_v04": "multi_issue_drop", "cs_recovery_ca_v05": "recovery_missed", "cs_mastery_demonstration_ca_v06": "none", } for sid, fm in expected.items(): s = load(sid) assert s.failure_mode == fm, f"scenario {sid} failure_mode={s.failure_mode!r} != {fm!r}" def test_difficulty_progression_one_to_five(): expected = { "cs_refund_ca_v01": 1, "cs_escalation_ca_v02": 2, "cs_policy_exception_ca_v03": 3, "cs_multi_issue_ca_v04": 3, "cs_recovery_ca_v05": 4, "cs_mastery_demonstration_ca_v06": 5, } for sid, d in expected.items(): s = load(sid) assert s.difficulty == d, f"scenario {sid} difficulty={s.difficulty} != {d}" def test_index_author_and_provenance(library): for e in library.entries(): assert e.author == "expert", f"scenario {e.id} author={e.author!r} != 'expert'" assert e.generated_from is None, ( f"expert scenario {e.id} should have no generated_from, got {e.generated_from!r}" ) def test_v01_scenario_still_loads_from_subdirectory(): s = load("cs_refund_ca_v01") assert s.id == "cs_refund_ca_v01" assert s.rubric_criteria, "v01 extended scenario must have rubric_criteria" assert s.branch_by_id("accept_resolution") is not None assert s.branch_by_id("escalate") is not None