praxis/tests/test_assist_pipeline.py

"""Tests for build_assist_pipeline + LiveAssistGuardrailProcessor (TASK-05-03, REQ-IDEATE-02).

Verifies:
  - The pipeline structure is correct (Piper TTS default, guardrail processor
    between llm and tts, no opening line).
  - The LLM context is the ≤150-token assist prompt.
  - The LiveAssistGuardrailProcessor passes allowed text through.
  - The processor blocks direct-answer text → CANNED_FALLBACK.
  - The processor retries on a retry-eligible block.
  - The processor does NOT retry on false-authority (hard violation).
  - The verdict is logged to the session.

These tests mock the WebRTC connection + transport so no live keys are needed.
The pipeline structure is verified by inspecting the Pipeline's processors list.
"""

from __future__ import annotations

import asyncio
import os
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from server.assist.context import AssistContext, COACHING_INSTRUCTION
from server.assist.guardrail_processor import LiveAssistGuardrailProcessor
from server.guardrails.live_assist import (
    CANNED_FALLBACK,
    LiveAssistGuardrail,
)
from server.services.base import GuardrailContext


def _make_context() -> AssistContext:
    return AssistContext(
        system_prompt=f"{COACHING_INSTRUCTION}\n\nWeek 1, damaged-product refund.\n\nBe brief.",
        current_week=1,
        scenario_tag="damaged-product refund",
        theta=0.0,
        coaching_focus="empathy",
        path_slug="customer_service",
    )


def test_assist_system_prompt_under_word_budget():
    """D-066: the assist system prompt is ≤200 words (≈150 tokens)."""
    ctx = _make_context()
    assert len(ctx.system_prompt.split()) <= 200


def test_build_assist_pipeline_structure():
    """TASK-05-01: build_assist_pipeline returns a valid pipeline with the right structure.

    Mocks the WebRTC connection + services so no live keys are needed. Verifies
    the pipeline contains the guardrail processor + uses Piper TTS by default.
    """
    # Mock the Pipecat services + aggregators + runner so no live keys/event loop needed.
    with patch("server.pipeline._build_transport") as mock_transport, \
         patch("server.pipeline._build_stt") as mock_stt, \
         patch("server.pipeline._build_llm") as mock_llm, \
         patch("server.assist.pipeline._build_tts_piper") as mock_tts, \
         patch("pipecat.processors.aggregators.llm_response_universal.LLMContextAggregator") as mock_agg, \
         patch("pipecat.pipeline.runner.PipelineRunner") as mock_runner_cls:
        mock_transport.return_value = MagicMock(name="transport")
        mock_stt.return_value = MagicMock(name="stt")
        mock_llm.return_value = MagicMock(name="llm")
        mock_tts.return_value = MagicMock(name="piper_tts")
        mock_agg.return_value = MagicMock(name="aggregator")
        mock_runner_cls.return_value = MagicMock(name="runner")

        from server.assist.pipeline import build_assist_pipeline

        ctx = _make_context()
        webrtc_conn = MagicMock(name="webrtc_connection")
        pipeline, task, runner, transport = build_assist_pipeline(
            webrtc_conn, context=ctx
        )
        # The pipeline has processors; verify the guardrail processor is present.
        processors = list(pipeline.processors)
        assert any(isinstance(p, LiveAssistGuardrailProcessor) for p in processors), (
            "LiveAssistGuardrailProcessor must be in the pipeline (D-060 layer 2)"
        )
        # Piper TTS was used (D-065 default).
        mock_tts.assert_called_once()
        # No opening line is played (assist is invoked mid-shift).


def test_build_assist_pipeline_uses_cartesia_when_env_set():
    """TASK-05-01: PRAXIS_ASSIST_TTS=cartesia falls back to Cartesia (for testing)."""
    with patch.dict(os.environ, {"PRAXIS_ASSIST_TTS": "cartesia"}), \
         patch("server.pipeline._build_transport") as mock_transport, \
         patch("server.pipeline._build_stt") as mock_stt, \
         patch("server.pipeline._build_llm") as mock_llm, \
         patch("server.pipeline._build_tts") as mock_cartesia, \
         patch("pipecat.processors.aggregators.llm_response_universal.LLMContextAggregator") as mock_agg, \
         patch("pipecat.pipeline.runner.PipelineRunner") as mock_runner_cls:
        mock_transport.return_value = MagicMock()
        mock_stt.return_value = MagicMock()
        mock_llm.return_value = MagicMock()
        mock_cartesia.return_value = MagicMock(name="cartesia_tts")
        mock_agg.return_value = MagicMock(name="aggregator")
        mock_runner_cls.return_value = MagicMock(name="runner")

        from server.assist.pipeline import build_assist_pipeline

        ctx = _make_context()
        pipeline, task, runner, transport = build_assist_pipeline(
            MagicMock(), context=ctx
        )
        mock_cartesia.assert_called_once()


# ── LiveAssistGuardrailProcessor behavior ────────────────────────────────────


def _make_processor(session=None, llm_context=None) -> LiveAssistGuardrailProcessor:
    """Build a processor with a mock frame pusher for isolated testing."""
    proc = LiveAssistGuardrailProcessor(
        guardrail=LiveAssistGuardrail(),
        session=session,
        llm_context=llm_context,
    )
    proc.push_frame = AsyncMock()
    return proc


def test_processor_passes_allowed_text_through():
    """Allowed coaching text → buffered, then pushed to TTS as one frame (REQ-ASSIST-03).

    The guardrail MUST complete its check before any text reaches TTS. This means
    text is buffered (not streamed) + pushed as a single TextFrame on
    LLMFullResponseEndFrame after the guardrail allows it. This is the safety-critical
    behavior: a blocked response never reaches TTS.
    """
    proc = _make_processor()

    async def _run():
        from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame

        # Simulate LLM text chunks.
        await proc.process_frame(TextFrame(text="What do you think "), direction=1)
        await proc.process_frame(TextFrame(text="the customer needs?"), direction=1)
        # End of LLM response.
        end_frame = LLMFullResponseEndFrame()
        await proc.process_frame(end_frame, direction=1)

    asyncio.run(_run())
    # The buffered text was pushed as a single TextFrame (not streamed chunk-by-chunk).
    pushed_texts = [
        call.args[0].text for call in proc.push_frame.await_args_list
        if hasattr(call.args[0], "text")
    ]
    assert "What do you think the customer needs?" in pushed_texts
    # The LLMFullResponseEndFrame was also pushed (to signal TTS the response is done).
    assert proc.push_frame.await_count >= 2  # 1 buffered text + 1 end frame


def test_processor_blocks_direct_answer():
    """Direct-answer text → CANNED_FALLBACK emitted (no pass-through of the blocked text)."""
    proc = _make_processor()

    async def _run():
        from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame

        await proc.process_frame(TextFrame(text="You should say sorry."), direction=1)
        end_frame = LLMFullResponseEndFrame()
        await proc.process_frame(end_frame, direction=1)

    asyncio.run(_run())
    # A TextFrame with CANNED_FALLBACK was pushed.
    pushed_texts = [
        call.args[0].text for call in proc.push_frame.await_args_list
        if hasattr(call.args[0], "text")
    ]
    assert CANNED_FALLBACK in pushed_texts


def test_processor_retries_on_retry_eligible_block():
    """Retry-eligible block (direct-answer) → inject RETRY_INSTRUCTION + retry."""
    from pipecat.processors.aggregators.llm_context import LLMContext

    llm_context = LLMContext()
    proc = _make_processor(llm_context=llm_context)
    messages_before = len(llm_context.get_messages())

    async def _run():
        from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame

        await proc.process_frame(TextFrame(text="You should say sorry."), direction=1)
        end_frame = LLMFullResponseEndFrame()
        await proc.process_frame(end_frame, direction=1)

    asyncio.run(_run())
    # The RETRY_INSTRUCTION was injected into the context (G-049 validated).
    messages_after = len(llm_context.get_messages())
    assert messages_after == messages_before + 1
    injected = llm_context.get_messages()[-1]
    assert "coaching question" in (injected.get("content") or "").lower()
    # The retry flag is set (no second retry).
    assert proc._retry_used is True


def test_processor_no_retry_on_false_authority():
    """Hard violation (false-authority) → CANNED_FALLBACK immediately, no retry."""
    from pipecat.processors.aggregators.llm_context import LLMContext

    llm_context = LLMContext()
    proc = _make_processor(llm_context=llm_context)
    messages_before = len(llm_context.get_messages())

    async def _run():
        from pipecat.frames.frames import LLMFullResponseEndFrame, TextFrame

        await proc.process_frame(TextFrame(text="I am your manager."), direction=1)
        end_frame = LLMFullResponseEndFrame()
        await proc.process_frame(end_frame, direction=1)

    asyncio.run(_run())
    # No retry message was injected (hard violation).
    messages_after = len(llm_context.get_messages())
    assert messages_after == messages_before
    # CANNED_FALLBACK was emitted.
    pushed_texts = [
        call.args[0].text for call in proc.push_frame.await_args_list
        if hasattr(call.args[0], "text")
    ]
    assert CANNED_FALLBACK in pushed_texts


def test_processor_logs_verdict_to_session():
    """The verdict is logged to the session (D-060 layer 3)."""
    session = MagicMock()
    session.log_assist_turn_partial = AsyncMock(return_value=0)
    session.log_assist_turn_complete = AsyncMock()
    session.guardrail_block_count = 0
    proc = _make_processor(session=session)

    async def _run():
        from pipecat.frames.frames import (
            LLMFullResponseEndFrame,
            TextFrame,
            TranscriptionFrame,
        )

        # ASR transcript (partial turn write — REQ-IDEATE-09).
        await proc.process_frame(
            TranscriptionFrame(text="Customer wants refund", user_id="u", timestamp=""),
            direction=1,
        )
        # LLM response (direct answer → blocked).
        await proc.process_frame(TextFrame(text="You should say sorry."), direction=1)
        await proc.process_frame(LLMFullResponseEndFrame(), direction=1)

    asyncio.run(_run())
    # The partial turn was written (REQ-IDEATE-09).
    session.log_assist_turn_partial.assert_awaited_once_with("Customer wants refund")
    # The complete turn was written with the verdict.
    session.log_assist_turn_complete.assert_awaited_once()
    # The verdict passed to log_assist_turn_complete has allowed=False (block).
    complete_call = session.log_assist_turn_complete.await_args
    verdict_arg = complete_call.kwargs.get("guardrail_verdict") or complete_call.args[2]
    assert verdict_arg["allowed"] is False
    # (The real AssistSession.log_assist_turn_complete increments guardrail_block_count
    # when the verdict has allowed=False — verified in test_p1_guardrail_e2e.py.)


def test_processor_incremental_audit_log_partial_turn():
    """REQ-IDEATE-09: a partial turn (ASR only) is written before the LLM response."""
    session = MagicMock()
    session.log_assist_turn_partial = AsyncMock(return_value=0)
    session.log_assist_turn_complete = AsyncMock()
    session.guardrail_block_count = 0
    proc = _make_processor(session=session)

    async def _run_partial_only():
        from pipecat.frames.frames import TranscriptionFrame

        # ASR transcript arrives but the LLM never responds (simulated abrupt termination).
        await proc.process_frame(
            TranscriptionFrame(text="Customer is upset", user_id="u", timestamp=""),
            direction=1,
        )

    asyncio.run(_run_partial_only())
    # The partial turn was written even though the LLM never responded.
    session.log_assist_turn_partial.assert_awaited_once_with("Customer is upset")
    session.log_assist_turn_complete.assert_not_awaited()