"""Evidence extractor — LLM-extract-then-verify (SLICE-03 TASK-03-01). Off-voice-path: called after the session ends. Calls deepseek-v4-flash:cloud to pull verbatim-quote evidence per rubric criterion, then fuzzy-matches each quote against the transcript (R-MAST-02). Hallucinated quotes are rejected and re-extracted (max 2 attempts). On final failure the scenario is marked `scoring_inconclusive=True` — it does NOT silently fail to zero and does NOT penalize the learner (grill Axis 4 MUST #3). """ from __future__ import annotations import json import logging from difflib import SequenceMatcher from typing import Any from pydantic import BaseModel, Field, ValidationError from server.services.base import LLMProvider log = logging.getLogger(__name__) _QUOTE_MATCH_THRESHOLD = 0.85 _MAX_REEXTRACTION_ATTEMPTS = 2 _EXTRACTION_MODEL = "deepseek-v4-flash:cloud" class Evidence(BaseModel): criterion_id: str quote: str signals: list[str] = Field(default_factory=list) class ExtractionResult(BaseModel): evidence: list[Evidence] = Field(default_factory=list) scoring_inconclusive: bool = False attempts: int = 0 rejected_quotes: list[str] = Field(default_factory=list) def _transcript_text(turns: list[dict]) -> str: parts: list[str] = [] for t in turns: role = t.get("role", "") content = t.get("content", "") or t.get("text", "") if content: parts.append(f"{role}: {content}") return "\n".join(parts) def _fuzzy_contains(haystack: str, quote: str) -> bool: if not quote.strip(): return False if quote in haystack: return True qlen = len(quote) if qlen >= len(haystack): return SequenceMatcher(None, quote, haystack).ratio() >= _QUOTE_MATCH_THRESHOLD best = 0.0 window = qlen + max(20, qlen // 4) step = max(1, qlen // 4) i = 0 while i <= len(haystack) - qlen: end = min(len(haystack), i + window) r = SequenceMatcher(None, quote, haystack[i:end]).ratio() if r > best: best = r if best >= _QUOTE_MATCH_THRESHOLD: return True i += step return best >= _QUOTE_MATCH_THRESHOLD def _build_prompt(turns: list[dict], rubric_criteria: list[str]) -> list[dict[str, str]]: transcript = _transcript_text(turns) crit_block = "\n".join(f"- {c}" for c in rubric_criteria) system = ( "You are an evidence extraction engine for a customer-service coaching rubric. " "For each rubric criterion, find the single most representative verbatim quote " "from the learner's utterances in the transcript, plus the observable behavior " "signal tags that apply. Quotes MUST be copied verbatim from the learner's " "spoken turns — do not paraphrase, do not invent." ) user = ( f"Rubric criteria:\n{crit_block}\n\n" f"Transcript:\n{transcript}\n\n" "Return ONLY a JSON array. Each element: " '{"criterion_id": , "quote": , ' '"signals": [, ...]}. ' "Omit a criterion if no evidence is present. No prose, no markdown fences." ) return [{"role": "system", "content": system}, {"role": "user", "content": user}] def _parse_evidence_json(raw: str, allowed_criteria: list[str]) -> list[Evidence]: text = raw.strip() if text.startswith("```"): text = text.strip("`") if text.lower().startswith("json"): text = text[4:] text = text.strip() try: data = json.loads(text) except json.JSONDecodeError as exc: raise ValueError(f"evidence JSON parse failed: {exc}") from exc if not isinstance(data, list): raise ValueError("evidence JSON must be a list") allowed = set(allowed_criteria) out: list[Evidence] = [] for item in data: try: ev = Evidence.model_validate(item) except ValidationError as exc: raise ValueError(f"evidence item schema invalid: {exc}") from exc if ev.criterion_id not in allowed: raise ValueError(f"unknown criterion_id: {ev.criterion_id}") out.append(ev) return out async def extract_evidence( turns: list[dict], rubric_criteria: list[str], llm: LLMProvider, *, model: str | None = None, max_attempts: int = _MAX_REEXTRACTION_ATTEMPTS, ) -> ExtractionResult: """Extract verbatim-quote evidence per criterion via LLM + fuzzy verification. Args: turns: session transcript turns (each dict has role + content/text). rubric_criteria: criterion ids to extract evidence for. llm: LLMProvider whose chat_full returns the model's response. model: override the extraction model (default deepseek-v4-flash:cloud). max_attempts: max re-extraction attempts after the initial call (default 2). Returns: ExtractionResult — either with `.evidence` populated, or with `.scoring_inconclusive=True` if quotes could not be verified after the retry budget (grill Axis 4 MUST #3 — never silently fail to zero). """ mdl = model or _EXTRACTION_MODEL transcript_text = _transcript_text(turns) rejected: list[str] = [] attempts = 0 for attempt in range(max_attempts + 1): attempts = attempt + 1 messages = _build_prompt(turns, rubric_criteria) if attempt > 0 and rejected: messages.append( { "role": "user", "content": ( "The following quotes were NOT found verbatim in the transcript " "and must be replaced with exact learner utterances:\n- " + "\n- ".join(rejected[-6:]) + "\n\nRe-emit the full JSON array with corrected verbatim quotes." ), } ) try: raw, _usage = await llm.chat_full(messages, model=mdl, no_think=True) except Exception as exc: log.warning("evidence extraction LLM call failed (attempt %d): %s", attempts, exc) continue try: candidates = _parse_evidence_json(raw, rubric_criteria) except ValueError as exc: log.warning("evidence JSON invalid (attempt %d): %s", attempts, exc) continue verified: list[Evidence] = [] bad: list[str] = [] for ev in candidates: if _fuzzy_contains(transcript_text, ev.quote): verified.append(ev) else: bad.append(ev.quote) if not bad and verified: return ExtractionResult(evidence=verified, attempts=attempts, rejected_quotes=rejected) rejected.extend(bad) if not verified and not bad: continue log.error( "evidence extraction scoring_inconclusive after %d attempts; rejected=%r", attempts, rejected, ) return ExtractionResult( evidence=[], scoring_inconclusive=True, attempts=attempts, rejected_quotes=rejected, ) __all__ = ["Evidence", "ExtractionResult", "extract_evidence"]