praxis/server/mastery/evidence_extractor.py

"""Evidence extractor — LLM-extract-then-verify (SLICE-03 TASK-03-01).

Off-voice-path: called after the session ends. Calls deepseek-v4-flash:cloud
to pull verbatim-quote evidence per rubric criterion, then fuzzy-matches each
quote against the transcript (R-MAST-02). Hallucinated quotes are rejected and
re-extracted (max 2 attempts). On final failure the scenario is marked
`scoring_inconclusive=True` — it does NOT silently fail to zero and does NOT
penalize the learner (grill Axis 4 MUST #3).
"""

from __future__ import annotations

import json
import logging
from difflib import SequenceMatcher
from typing import Any

from pydantic import BaseModel, Field, ValidationError

from server.services.base import LLMProvider

log = logging.getLogger(__name__)

_QUOTE_MATCH_THRESHOLD = 0.85
_MAX_REEXTRACTION_ATTEMPTS = 2
_EXTRACTION_MODEL = "deepseek-v4-flash:cloud"


class Evidence(BaseModel):
    criterion_id: str
    quote: str
    signals: list[str] = Field(default_factory=list)


class ExtractionResult(BaseModel):
    evidence: list[Evidence] = Field(default_factory=list)
    scoring_inconclusive: bool = False
    attempts: int = 0
    rejected_quotes: list[str] = Field(default_factory=list)


def _transcript_text(turns: list[dict]) -> str:
    parts: list[str] = []
    for t in turns:
        role = t.get("role", "")
        content = t.get("content", "") or t.get("text", "")
        if content:
            parts.append(f"{role}: {content}")
    return "\n".join(parts)


def _fuzzy_contains(haystack: str, quote: str) -> bool:
    if not quote.strip():
        return False
    if quote in haystack:
        return True
    qlen = len(quote)
    if qlen >= len(haystack):
        return SequenceMatcher(None, quote, haystack).ratio() >= _QUOTE_MATCH_THRESHOLD
    best = 0.0
    window = qlen + max(20, qlen // 4)
    step = max(1, qlen // 4)
    i = 0
    while i <= len(haystack) - qlen:
        end = min(len(haystack), i + window)
        r = SequenceMatcher(None, quote, haystack[i:end]).ratio()
        if r > best:
            best = r
            if best >= _QUOTE_MATCH_THRESHOLD:
                return True
        i += step
    return best >= _QUOTE_MATCH_THRESHOLD


def _build_prompt(turns: list[dict], rubric_criteria: list[str]) -> list[dict[str, str]]:
    transcript = _transcript_text(turns)
    crit_block = "\n".join(f"- {c}" for c in rubric_criteria)
    system = (
        "You are an evidence extraction engine for a customer-service coaching rubric. "
        "For each rubric criterion, find the single most representative verbatim quote "
        "from the learner's utterances in the transcript, plus the observable behavior "
        "signal tags that apply. Quotes MUST be copied verbatim from the learner's "
        "spoken turns — do not paraphrase, do not invent."
    )
    user = (
        f"Rubric criteria:\n{crit_block}\n\n"
        f"Transcript:\n{transcript}\n\n"
        "Return ONLY a JSON array. Each element: "
        '{"criterion_id": <string>, "quote": <verbatim learner quote>, '
        '"signals": [<string>, ...]}. '
        "Omit a criterion if no evidence is present. No prose, no markdown fences."
    )
    return [{"role": "system", "content": system}, {"role": "user", "content": user}]


def _parse_evidence_json(raw: str, allowed_criteria: list[str]) -> list[Evidence]:
    text = raw.strip()
    if text.startswith("```"):
        text = text.strip("`")
        if text.lower().startswith("json"):
            text = text[4:]
        text = text.strip()
    try:
        data = json.loads(text)
    except json.JSONDecodeError as exc:
        raise ValueError(f"evidence JSON parse failed: {exc}") from exc
    if not isinstance(data, list):
        raise ValueError("evidence JSON must be a list")
    allowed = set(allowed_criteria)
    out: list[Evidence] = []
    for item in data:
        try:
            ev = Evidence.model_validate(item)
        except ValidationError as exc:
            raise ValueError(f"evidence item schema invalid: {exc}") from exc
        if ev.criterion_id not in allowed:
            raise ValueError(f"unknown criterion_id: {ev.criterion_id}")
        out.append(ev)
    return out


async def extract_evidence(
    turns: list[dict],
    rubric_criteria: list[str],
    llm: LLMProvider,
    *,
    model: str | None = None,
    max_attempts: int = _MAX_REEXTRACTION_ATTEMPTS,
) -> ExtractionResult:
    """Extract verbatim-quote evidence per criterion via LLM + fuzzy verification.

    Args:
        turns: session transcript turns (each dict has role + content/text).
        rubric_criteria: criterion ids to extract evidence for.
        llm: LLMProvider whose chat_full returns the model's response.
        model: override the extraction model (default deepseek-v4-flash:cloud).
        max_attempts: max re-extraction attempts after the initial call (default 2).

    Returns:
        ExtractionResult — either with `.evidence` populated, or with
        `.scoring_inconclusive=True` if quotes could not be verified after the
        retry budget (grill Axis 4 MUST #3 — never silently fail to zero).
    """
    mdl = model or _EXTRACTION_MODEL
    transcript_text = _transcript_text(turns)
    rejected: list[str] = []
    attempts = 0

    for attempt in range(max_attempts + 1):
        attempts = attempt + 1
        messages = _build_prompt(turns, rubric_criteria)
        if attempt > 0 and rejected:
            messages.append(
                {
                    "role": "user",
                    "content": (
                        "The following quotes were NOT found verbatim in the transcript "
                        "and must be replaced with exact learner utterances:\n- "
                        + "\n- ".join(rejected[-6:])
                        + "\n\nRe-emit the full JSON array with corrected verbatim quotes."
                    ),
                }
            )

        try:
            raw, _usage = await llm.chat_full(messages, model=mdl, no_think=True)
        except Exception as exc:
            log.warning("evidence extraction LLM call failed (attempt %d): %s", attempts, exc)
            continue

        try:
            candidates = _parse_evidence_json(raw, rubric_criteria)
        except ValueError as exc:
            log.warning("evidence JSON invalid (attempt %d): %s", attempts, exc)
            continue

        verified: list[Evidence] = []
        bad: list[str] = []
        for ev in candidates:
            if _fuzzy_contains(transcript_text, ev.quote):
                verified.append(ev)
            else:
                bad.append(ev.quote)

        if not bad and verified:
            return ExtractionResult(evidence=verified, attempts=attempts, rejected_quotes=rejected)
        rejected.extend(bad)
        if not verified and not bad:
            continue

    log.error(
        "evidence extraction scoring_inconclusive after %d attempts; rejected=%r",
        attempts,
        rejected,
    )
    return ExtractionResult(
        evidence=[],
        scoring_inconclusive=True,
        attempts=attempts,
        rejected_quotes=rejected,
    )


__all__ = ["Evidence", "ExtractionResult", "extract_evidence"]