Praxis CI
813bd586d6
v0.3 milestone merged to main. Mastery scoring + competency rubrics + verifiable credentials (formative-tier) shipped. 13/13 REQ-IDs covered. Next milestone: v0.4 (operator tier — cohort dashboard + auth + Postgres). ---ci--- project: praxis phase: 2 milestone: v0.3 status: complete milestone_complete: true milestone_merged_to_main: true ---/ci---
67 lines
2.1 KiB
Python
67 lines
2.1 KiB
Python
"""Rule-based rubric scorer — deterministic (SLICE-03 TASK-03-02, REQ-NFR-MAST-01).
|
|
|
|
No LLM. Maps evidence signals to rubric level anchors: for each criterion, pick
|
|
the highest level whose `signals[]` are all present in the matched evidence,
|
|
fallback to level 1 if no level matches. The output is reproducible given the
|
|
same (evidence, rubric) pair.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from server.mastery.evidence_extractor import Evidence
|
|
from server.mastery.rubric_schema import Rubric, RubricCriterion
|
|
|
|
|
|
class CriterionScore(BaseModel):
|
|
criterion_id: str
|
|
level: int = Field(ge=1, le=5)
|
|
weight: float
|
|
evidence_quote: str = ""
|
|
matched_signals: list[str] = Field(default_factory=list)
|
|
|
|
|
|
def _evidence_for(evidence: list[Evidence], criterion_id: str) -> Evidence | None:
|
|
for ev in evidence:
|
|
if ev.criterion_id == criterion_id:
|
|
return ev
|
|
return None
|
|
|
|
|
|
def _level_for_criterion(criterion: RubricCriterion, ev: Evidence | None) -> tuple[int, list[str]]:
|
|
if ev is None or not ev.signals:
|
|
return 1, []
|
|
ev_signals = set(ev.signals)
|
|
best_level = 1
|
|
best_signals: list[str] = []
|
|
for lvl in sorted(criterion.levels, key=lambda l: l.level):
|
|
if all(s in ev_signals for s in lvl.signals):
|
|
best_level = lvl.level
|
|
best_signals = list(lvl.signals)
|
|
return best_level, best_signals
|
|
|
|
|
|
def score(evidence: list[Evidence], rubric: Rubric) -> list[CriterionScore]:
|
|
"""Score evidence against the rubric — deterministic, no LLM.
|
|
|
|
Returns one CriterionScore per rubric criterion, in rubric order. Criteria
|
|
with no matching evidence get level 1 (the "Fail" anchor).
|
|
"""
|
|
out: list[CriterionScore] = []
|
|
for c in rubric.criteria:
|
|
ev = _evidence_for(evidence, c.id)
|
|
level, matched = _level_for_criterion(c, ev)
|
|
out.append(
|
|
CriterionScore(
|
|
criterion_id=c.id,
|
|
level=level,
|
|
weight=c.weight,
|
|
evidence_quote=ev.quote if ev else "",
|
|
matched_signals=matched,
|
|
)
|
|
)
|
|
return out
|
|
|
|
|
|
__all__ = ["CriterionScore", "score"] |