#!/usr/bin/env python3
"""SLICE-08 TASK-08-04 — Real-LLM evidence extraction smoke test (grill Axis 7 FIX #1).

Runs ONE real session transcript through the actual deepseek-v4-flash:cloud
evidence extractor and verifies the output is valid JSON with fuzzy-matching
quotes (the extraction prompt works against the real model, not just the
scoring logic against mocked responses).

Staging-gated: this test calls a real paid LLM endpoint. It runs ONLY when the
env var `PRAXIS_RUN_REAL_LLM_TESTS=1` is set, AND requires `OLLAMA_API_KEY`.
CI must NOT set the gate env var — mocked-LLM tests stay the CI source of
truth (REQ-MAST-01 determinism is covered by the mocked tests; this script
validates the prompt+model contract against model drift).

Run:
    python3 scripts/test_real_llm_evidence.py

Exit codes:
  0 — SKIP (gate not set) OR PASS
  1 — FAIL (gate set, real call failed or output invalid)
  2 — MISCONFIG (gate set but OLLAMA_API_KEY missing)
"""

from __future__ import annotations

import asyncio
import json
import os
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from server.llm.ollama_cloud import OllamaCloudLLM
from server.mastery.evidence_extractor import extract_evidence
from server.mastery.rubric_loader import clear_cache, load_rubric

_REPO = Path(__file__).resolve().parent.parent
_RUBRICS_DIR = _REPO / "rubrics"
_GATE_ENV = "PRAXIS_RUN_REAL_LLM_TESTS"

_TRANSCRIPT = [
    {"role": "customer", "content": "My order arrived cracked and I'm furious."},
    {
        "role": "learner",
        "content": (
            "I'm really sorry the bowl arrived cracked — that's genuinely "
            "frustrating. I can refund the full amount to your original card "
            "within 3 business days, or send a replacement first class tomorrow. "
            "Which would you prefer?"
        ),
    },
    {"role": "customer", "content": "Just refund it."},
    {
        "role": "learner",
        "content": (
            "Of course — I've issued a full refund of $42.99 to your Visa ending "
            "4421. You'll see it in 2-3 business days. Is there anything else?"
        ),
    },
]


def _print_skip() -> None:
    print(f"SKIP (set {_GATE_ENV}=1 to run)")


async def _run_real() -> int:
    if not os.environ.get("OLLAMA_API_KEY", "").strip():
        print(f"FAIL — {_GATE_ENV}=1 but OLLAMA_API_KEY is not set")
        return 2

    clear_cache()
    rubric = load_rubric("customer_service", rubrics_dir=_RUBRICS_DIR)
    llm = OllamaCloudLLM()

    print("Calling deepseek-v4-flash:cloud for evidence extraction …")
    result = await extract_evidence(
        _TRANSCRIPT, rubric.criterion_ids(), llm, max_attempts=2
    )

    if result.scoring_inconclusive:
        print(
            f"FAIL — extraction returned scoring_inconclusive after "
            f"{result.attempts} attempts; rejected quotes="
            f"{result.rejected_quotes[:3]}"
        )
        return 1

    if not result.evidence:
        print(f"FAIL — extraction returned no evidence (attempts={result.attempts})")
        return 1

    crit_ids = {e.criterion_id for e in result.evidence}
    expected = set(rubric.criterion_ids())
    if not crit_ids.issubset(expected):
        print(f"FAIL — unknown criterion ids: {crit_ids - expected}")
        return 1

    for ev in result.evidence:
        if not ev.quote.strip():
            print(f"FAIL — empty quote for criterion {ev.criterion_id!r}")
            return 1
        if not ev.signals:
            print(f"FAIL — no signals for criterion {ev.criterion_id!r}")
            return 1

    print(f"PASS — {len(result.evidence)} evidence items extracted (attempts={result.attempts})")
    for ev in result.evidence:
        print(f"  - {ev.criterion_id}: {len(ev.signals)} signals, quote={ev.quote[:60]!r}…")
    return 0


def main() -> int:
    if os.environ.get(_GATE_ENV, "").strip() != "1":
        _print_skip()
        return 0
    return asyncio.run(_run_real())


if __name__ == "__main__":
    raise SystemExit(main())