Praxis CI
813bd586d6
v0.3 milestone merged to main. Mastery scoring + competency rubrics + verifiable credentials (formative-tier) shipped. 13/13 REQ-IDs covered. Next milestone: v0.4 (operator tier — cohort dashboard + auth + Postgres). ---ci--- project: praxis phase: 2 milestone: v0.3 status: complete milestone_complete: true milestone_merged_to_main: true ---/ci---
238 lines
7.6 KiB
Python
238 lines
7.6 KiB
Python
"""Unit tests for the rubric schema + loader (SLICE-01: TASK-01-04)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from server.mastery.rubric_loader import clear_cache, load_rubric
|
|
from server.mastery.rubric_schema import Rubric, RubricCriterion, RubricLevel, ValidationError
|
|
|
|
_RUBRICS_DIR = Path(__file__).resolve().parent.parent / "rubrics"
|
|
|
|
|
|
def _valid_rubric_dict() -> dict:
|
|
return {
|
|
"id": "customer_service",
|
|
"skill": "customer_service",
|
|
"description": "CS rubric for refund/complaint",
|
|
"criteria": [
|
|
{
|
|
"id": "empathy",
|
|
"name": "Empathy",
|
|
"weight": 0.35,
|
|
"conjunctive_floor": None,
|
|
"levels": [
|
|
{"level": i, "label": f"L{i}", "anchor": f"anchor {i}", "signals": [f"s{i}"]}
|
|
for i in range(1, 6)
|
|
],
|
|
},
|
|
{
|
|
"id": "resolution",
|
|
"name": "Resolution",
|
|
"weight": 0.30,
|
|
"levels": [
|
|
{"level": i, "label": f"L{i}", "anchor": f"anchor {i}", "signals": [f"s{i}"]}
|
|
for i in range(1, 6)
|
|
],
|
|
},
|
|
{
|
|
"id": "de_escalation",
|
|
"name": "De-escalation",
|
|
"weight": 0.20,
|
|
"levels": [
|
|
{"level": i, "label": f"L{i}", "anchor": f"anchor {i}", "signals": [f"s{i}"]}
|
|
for i in range(1, 6)
|
|
],
|
|
},
|
|
{
|
|
"id": "professionalism",
|
|
"name": "Professionalism",
|
|
"weight": 0.15,
|
|
"conjunctive_floor": 2,
|
|
"levels": [
|
|
{"level": i, "label": f"L{i}", "anchor": f"anchor {i}", "signals": [f"s{i}"]}
|
|
for i in range(1, 6)
|
|
],
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def test_valid_rubric_parses():
|
|
r = Rubric.model_validate(_valid_rubric_dict())
|
|
assert r.id == "customer_service"
|
|
assert r.skill == "customer_service"
|
|
assert len(r.criteria) == 4
|
|
assert r.criterion_ids() == ["empathy", "resolution", "de_escalation", "professionalism"]
|
|
|
|
|
|
def test_weights_sum_to_one():
|
|
r = Rubric.model_validate(_valid_rubric_dict())
|
|
total = sum(c.weight for c in r.criteria)
|
|
assert abs(total - 1.0) < 1e-6
|
|
|
|
|
|
def test_reject_invalid_weights():
|
|
bad = _valid_rubric_dict()
|
|
bad["criteria"][0]["weight"] = 0.50 # now sums to 1.15
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_reject_weights_not_summing_to_one_low():
|
|
bad = _valid_rubric_dict()
|
|
bad["criteria"][0]["weight"] = 0.10 # now sums to 0.75
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_reject_missing_levels():
|
|
bad = _valid_rubric_dict()
|
|
bad["criteria"][0]["levels"] = bad["criteria"][0]["levels"][:4] # only 4 levels
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_reject_too_many_levels():
|
|
bad = copy.deepcopy(_valid_rubric_dict())
|
|
bad["criteria"][0]["levels"].append(
|
|
{"level": 6, "label": "L6", "anchor": "anchor 6", "signals": ["s6"]}
|
|
)
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_reject_non_sequential_levels():
|
|
bad = copy.deepcopy(_valid_rubric_dict())
|
|
bad["criteria"][0]["levels"] = [
|
|
{"level": i, "label": f"L{i}", "anchor": f"anchor {i}", "signals": [f"s{i}"]}
|
|
for i in [1, 2, 3, 4, 6] # skips 5, includes 6
|
|
]
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_reject_duplicate_criterion_ids():
|
|
bad = copy.deepcopy(_valid_rubric_dict())
|
|
bad["criteria"][1]["id"] = "empathy" # duplicate
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_reject_empty_signals():
|
|
bad = copy.deepcopy(_valid_rubric_dict())
|
|
bad["criteria"][0]["levels"][0]["signals"] = []
|
|
with pytest.raises(ValidationError):
|
|
Rubric.model_validate(bad)
|
|
|
|
|
|
def test_criterion_lookup_by_id():
|
|
r = Rubric.model_validate(_valid_rubric_dict())
|
|
c = r.criterion_by_id("empathy")
|
|
assert c is not None
|
|
assert c.id == "empathy"
|
|
assert c.weight == 0.35
|
|
assert r.criterion_by_id("nonexistent") is None
|
|
|
|
|
|
def test_level_lookup_by_value():
|
|
c = RubricCriterion.model_validate(_valid_rubric_dict()["criteria"][0])
|
|
lvl3 = c.level_by_value(3)
|
|
assert lvl3 is not None
|
|
assert lvl3.level == 3
|
|
assert c.level_by_value(99) is None
|
|
|
|
|
|
def test_conjunctive_floor_field():
|
|
r = Rubric.model_validate(_valid_rubric_dict())
|
|
assert r.criterion_by_id("professionalism").conjunctive_floor == 2
|
|
assert r.criterion_by_id("empathy").conjunctive_floor is None
|
|
|
|
|
|
def test_archetype_weights_override():
|
|
d = _valid_rubric_dict()
|
|
d["archetype_weights"] = {
|
|
"complaint": {
|
|
"empathy": 0.40,
|
|
"resolution": 0.25,
|
|
"de_escalation": 0.20,
|
|
"professionalism": 0.15,
|
|
}
|
|
}
|
|
r = Rubric.model_validate(d)
|
|
base = r.weights_for_archetype(None)
|
|
assert base["empathy"] == 0.35
|
|
complaint = r.weights_for_archetype("complaint")
|
|
assert complaint["empathy"] == 0.40
|
|
assert complaint["resolution"] == 0.25
|
|
|
|
|
|
def test_load_customer_service_rubric_yaml():
|
|
clear_cache()
|
|
r = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
assert r.id == "customer_service"
|
|
assert r.skill == "customer_service"
|
|
assert len(r.criteria) == 4
|
|
assert {c.id for c in r.criteria} == {"empathy", "resolution", "de_escalation", "professionalism"}
|
|
assert r.criterion_by_id("professionalism").conjunctive_floor == 2
|
|
assert r.archetype_weights is not None
|
|
assert "refund" in r.archetype_weights
|
|
assert "complaint" in r.archetype_weights
|
|
|
|
|
|
def test_load_rubric_caches():
|
|
clear_cache()
|
|
r1 = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
r2 = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
assert r1 is r2
|
|
|
|
|
|
def test_load_rubric_missing_file_raises():
|
|
clear_cache()
|
|
with pytest.raises(FileNotFoundError):
|
|
load_rubric("does_not_exist", rubrics_dir=_RUBRICS_DIR)
|
|
|
|
|
|
def test_loaded_rubric_yaml_weights_sum_to_one():
|
|
clear_cache()
|
|
r = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
total = sum(c.weight for c in r.criteria)
|
|
assert abs(total - 1.0) < 1e-6
|
|
|
|
|
|
def test_loaded_rubric_has_five_levels_per_criterion():
|
|
clear_cache()
|
|
r = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
for c in r.criteria:
|
|
assert len(c.levels) == 5
|
|
assert sorted(lvl.level for lvl in c.levels) == [1, 2, 3, 4, 5]
|
|
|
|
|
|
def test_loaded_rubric_levels_have_signals():
|
|
clear_cache()
|
|
r = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
for c in r.criteria:
|
|
for lvl in c.levels:
|
|
assert len(lvl.signals) >= 1
|
|
assert all(isinstance(s, str) and s for s in lvl.signals)
|
|
|
|
|
|
def test_rubric_level_model_validation():
|
|
lvl = RubricLevel(level=3, label="Competent", anchor="...", signals=["a", "b"])
|
|
assert lvl.level == 3
|
|
with pytest.raises(ValidationError):
|
|
RubricLevel(level=0, label="x", anchor="x", signals=["a"])
|
|
with pytest.raises(ValidationError):
|
|
RubricLevel(level=6, label="x", anchor="x", signals=["a"])
|
|
|
|
|
|
def test_loaded_rubric_escalated_weights_present():
|
|
clear_cache()
|
|
r = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
|
|
assert r.escalated_weights is not None
|
|
assert abs(sum(r.escalated_weights.values()) - 1.0) < 1e-6
|
|
assert r.escalated_weights["de_escalation"] == 0.40 |