praxis/tests/test_evidence_extractor_integration.py

"""SLICE-03 TASK-03-05 — evidence extractor integration test (mocked LLM)."""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock

import pytest

from server.mastery.evidence_extractor import Evidence, extract_evidence
from server.mastery.mastery_score import compute_scenario_score
from server.mastery.rubric_loader import clear_cache, load_rubric
from server.mastery.rubric_scorer import score

_RUBRICS_DIR = Path(__file__).resolve().parent.parent / "rubrics"


def _turns() -> list[dict]:
    return [
        {"role": "customer", "content": "My order arrived cracked and I'm furious."},
        {
            "role": "learner",
            "content": (
                "I'm really sorry the bowl arrived cracked — that's genuinely "
                "frustrating. I can refund the full amount to your original card "
                "within 3 business days, or send a replacement first class tomorrow. "
                "Which would you prefer?"
            ),
        },
        {"role": "customer", "content": "Just refund it."},
        {
            "role": "learner",
            "content": (
                "Of course — I've issued a full refund of $42.99 to your Visa ending "
                "4421. You'll see it in 2-3 business days. Is there anything else?"
            ),
        },
    ]


def _canned_good() -> str:
    t1 = _turns()[1]["content"]
    t2 = _turns()[3]["content"]
    return json.dumps(
        [
            {"criterion_id": "empathy", "quote": t1, "signals": ["named_emotion_in_own_words", "acknowledged_specific"]},
            {"criterion_id": "resolution", "quote": t1, "signals": ["concrete_method", "concrete_amount_or_channel", "concrete_next_step"]},
            {"criterion_id": "de_escalation", "quote": t1, "signals": ["explicit_acknowledge_reframe_offer"]},
            {"criterion_id": "professionalism", "quote": t2, "signals": ["plain_language", "in_role_throughout", "no_prohibited_advice"]},
        ]
    )


def _canned_bad() -> str:
    return json.dumps(
        [
            {"criterion_id": "empathy", "quote": "I apologize for the inconvenience, dear customer.", "signals": ["named_emotion_in_own_words"]},
            {"criterion_id": "resolution", "quote": "I will issue a refund shortly.", "signals": ["concrete_method"]},
        ]
    )


def _canned_malformed() -> str:
    return "not json at all {["


def _make_llm(raws: list[str]) -> AsyncMock:
    llm = AsyncMock()
    llm.chat_full = AsyncMock(side_effect=[(r, {"model": "test"}) for r in raws])
    return llm


def _rubric():
    clear_cache()
    return load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)


@pytest.mark.asyncio
async def test_end_to_end_extraction_to_scoring_deterministic():
    rubric = _rubric()
    llm = _make_llm([_canned_good(), _canned_good()])
    res1 = await extract_evidence(_turns(), rubric.criterion_ids(), llm)
    res2 = await extract_evidence(_turns(), rubric.criterion_ids(), llm)
    assert not res1.scoring_inconclusive and not res2.scoring_inconclusive

    cs1 = score(res1.evidence, rubric)
    cs2 = score(res2.evidence, rubric)
    assert [s.model_dump() for s in cs1] == [s.model_dump() for s in cs2]

    ss = compute_scenario_score(cs1, rubric)
    assert ss.passed is True
    assert ss.weighted_mean >= 3.0


@pytest.mark.asyncio
async def test_json_schema_validation_rejects_malformed_then_recovers():
    rubric = _rubric()
    llm = _make_llm([_canned_malformed(), _canned_good()])
    res = await extract_evidence(_turns(), rubric.criterion_ids(), llm)
    assert not res.scoring_inconclusive
    assert res.attempts == 2
    assert {e.criterion_id for e in res.evidence} == {"empathy", "resolution", "de_escalation", "professionalism"}


@pytest.mark.asyncio
async def test_unknown_criterion_id_rejected():
    rubric = _rubric()
    raw = json.dumps(
        [{"criterion_id": "nope", "quote": _turns()[1]["content"], "signals": ["x"]}]
    )
    llm = _make_llm([raw, _canned_good()])
    res = await extract_evidence(_turns(), rubric.criterion_ids(), llm)
    assert not res.scoring_inconclusive
    assert all(e.criterion_id != "nope" for e in res.evidence)


@pytest.mark.asyncio
async def test_inconclusive_when_bad_quotes_twice():
    rubric = _rubric()
    llm = _make_llm([_canned_bad(), _canned_bad(), _canned_bad()])
    res = await extract_evidence(_turns(), rubric.criterion_ids(), llm, max_attempts=2)
    assert res.scoring_inconclusive is True
    assert res.evidence == []
    assert res.attempts == 3


@pytest.mark.asyncio
async def test_inconclusive_result_does_not_score_to_zero_scenario():
    rubric = _rubric()
    llm = _make_llm([_canned_bad(), _canned_bad(), _canned_bad()])
    res = await extract_evidence(_turns(), rubric.criterion_ids(), llm, max_attempts=2)
    assert res.scoring_inconclusive
    # callers must NOT compute a scenario score from inconclusive evidence;
    # verify that scoring empty evidence yields a level-1 fail, which the
    # session_recorder MUST skip (the contract is: inconclusive → no score).
    empty_scores = score(res.evidence, rubric)
    ss = compute_scenario_score(empty_scores, rubric)
    assert ss.passed is False
    # The integration contract: scoring_inconclusive short-circuits upstream
    # before compute_scenario_score is ever called. This test documents that
    # empty-evidence scoring is NOT what inconclusive means — inconclusive is
    # a distinct branch that yields no scenario score at all.


@pytest.mark.asyncio
async def test_quote_fuzzy_match_against_transcript():
    rubric = _rubric()
    t1 = _turns()[1]["content"]
    near = t1.replace("—", "-").rstrip(".")
    raw = json.dumps(
        [
            {"criterion_id": "empathy", "quote": near, "signals": ["named_emotion_in_own_words", "acknowledged_specific"]},
            {"criterion_id": "resolution", "quote": near, "signals": ["concrete_method", "concrete_amount_or_channel", "concrete_next_step"]},
            {"criterion_id": "de_escalation", "quote": near, "signals": ["explicit_acknowledge_reframe_offer"]},
            {"criterion_id": "professionalism", "quote": _turns()[3]["content"], "signals": ["plain_language", "in_role_throughout", "no_prohibited_advice"]},
        ]
    )
    llm = _make_llm([raw])
    res = await extract_evidence(_turns(), rubric.criterion_ids(), llm)
    assert not res.scoring_inconclusive
    assert res.attempts == 1