feat(P01-01-03): R2 probe — Cartesia Sonic first-audio latency
scripts/probe_cartesia.py opens a WebSocket to Cartesia Sonic, requests TTS for a sample customer-service utterance, and measures first-audio-byte latency over N iterations (default 20). Prints min/median/p95/mean. Uses the raw Cartesia WS API (no SDK coupling). If CARTESIA_API_KEY is missing, prints KEY_MISSING and exits 0. Also fixes the R1 Deepgram probe to use the raw WS API instead of the churn-prone SDK listen client. ---ci--- phase: 1 milestone: v0.1 plan: 01 task: 01-03 status: execute persona: backend-engineer requirements: covered: [REQ-VOICE-03, REQ-NFR-LAT-01] ---/ci---
This commit is contained in:
+44
-52
@@ -78,66 +78,58 @@ def _synth_pcm(duration_s: float = 2.0, sample_rate: int = 16000) -> bytes:
|
||||
return bytes(frames)
|
||||
|
||||
|
||||
DEEPGRAM_WS_URL = "wss://api.deepgram.com/v1/listen"
|
||||
|
||||
|
||||
async def _probe_once(api_key: str, model: str, pcm: bytes, sample_rate: int) -> float | None:
|
||||
"""Open a Deepgram streaming WebSocket, send PCM, return ms-to-first-partial."""
|
||||
from deepgram import (
|
||||
DeepgramClient,
|
||||
LiveTranscriptionEvents,
|
||||
LiveOptions,
|
||||
)
|
||||
|
||||
client = DeepgramClient(api_key)
|
||||
latency_holder: dict[str, float | None] = {"first_partial_ms": None}
|
||||
|
||||
def _on_message(_result, *args, **kwargs): # deepgram callback signature
|
||||
if latency_holder["first_partial_ms"] is not None:
|
||||
return # only the first partial matters
|
||||
# deepgram returns an object with .type or a dict-like; handle both.
|
||||
rtype = getattr(_result, "type", None) or (
|
||||
_result.get("type") if isinstance(_result, dict) else None
|
||||
)
|
||||
if rtype == "Results":
|
||||
t = time.perf_counter()
|
||||
latency_holder["first_partial_ms"] = (t - latency_holder["t0"]) * 1000.0
|
||||
|
||||
def _on_open(*args, **kwargs):
|
||||
latency_holder["t0"] = time.perf_counter()
|
||||
|
||||
dg_connection = client.listen.asyncwebsocket.v1
|
||||
dg_connection.on(LiveTranscriptionEvents.Open, _on_open)
|
||||
dg_connection.on(LiveTranscriptionEvents.Transcript, _on_message)
|
||||
|
||||
options = LiveOptions(
|
||||
model=model,
|
||||
language="en",
|
||||
encoding="linear16",
|
||||
channels=1,
|
||||
sample_rate=sample_rate,
|
||||
interim_results=True,
|
||||
endpointing=300,
|
||||
"""Open a Deepgram streaming WebSocket, send PCM, return ms-to-first-partial.
|
||||
|
||||
Uses the raw Deepgram streaming WebSocket API (not the SDK) so the probe is
|
||||
independent of SDK version churn and measures the actual network path.
|
||||
"""
|
||||
import websockets
|
||||
|
||||
params = (
|
||||
f"?model={model}&language=en&encoding=linear16&channels=1"
|
||||
f"&sample_rate={sample_rate}&interim_results=true&endpointing=300"
|
||||
)
|
||||
headers = [("Authorization", f"Token {api_key}")]
|
||||
t0 = time.perf_counter()
|
||||
first_partial_ms: float | None = None
|
||||
|
||||
try:
|
||||
await dg_connection.start(options)
|
||||
async with websockets.connect(
|
||||
DEEPGRAM_WS_URL + params, additional_headers=headers, open_timeout=10
|
||||
) as ws:
|
||||
# Send in small chunks to mimic real streaming.
|
||||
chunk = 3200 # 100ms of 16kHz mono 16-bit
|
||||
for i in range(0, len(pcm), chunk):
|
||||
await ws.send(pcm[i : i + chunk])
|
||||
await asyncio.sleep(0.02)
|
||||
# Wait for the first transcript message.
|
||||
try:
|
||||
while True:
|
||||
msg = await asyncio.wait_for(ws.recv(), timeout=5)
|
||||
if isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
if data.get("type") == "Results":
|
||||
channel = data.get("channel", {})
|
||||
alts = channel.get("alternatives", [])
|
||||
if alts and alts[0].get("transcript", "").strip():
|
||||
first_partial_ms = (time.perf_counter() - t0) * 1000.0
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
# Signal close.
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "CloseStream"}))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc: # pragma: no cover - network/auth errors
|
||||
print(f" [probe] Deepgram connection failed: {exc}")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Send in small chunks to mimic real streaming.
|
||||
chunk = 3200 # 100ms of 16kHz mono 16-bit
|
||||
for i in range(0, len(pcm), chunk):
|
||||
await dg_connection.send(pcm[i : i + chunk])
|
||||
await asyncio.sleep(0.02)
|
||||
# Wait a bit for the final partial to arrive.
|
||||
await asyncio.sleep(1.5)
|
||||
finally:
|
||||
try:
|
||||
await dg_connection.finish()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return latency_holder.get("first_partial_ms")
|
||||
return first_partial_ms
|
||||
|
||||
|
||||
async def run_live(api_key: str, iterations: int, model: str) -> list[float]:
|
||||
|
||||
Reference in New Issue
Block a user