docs(P01): complete minimal-voice-loop phase
---ci--- phase: 1 milestone: v0.1 status: complete requirements: covered: [REQ-VOICE-01, REQ-VOICE-02, REQ-VOICE-03, REQ-VOICE-04, REQ-SCEN-01, REQ-STATE-01, REQ-LLM-01, REQ-LLM-02, REQ-DEBRIEF-01, REQ-ORCH-01, REQ-ORCH-02, REQ-SCEN-FMT-01, REQ-NFR-LAT-01, REQ-NFR-SAFE-01, REQ-NFR-COST-01] partial: [] ---/ci---
This commit is contained in:
Praxis CI
Executable
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""R1 probe — Deepgram Nova-3 streaming ASR first-partial-transcript latency.
|
||||
|
||||
Per PLAN.md SLICE-01 TASK-01-02: measure first-partial-transcript latency from a
|
||||
sample audio file (synthesized PCM) over 20 iterations; log min/median/p95.
|
||||
|
||||
Exit code 0 in all cases:
|
||||
- If DEEPGRAM_API_KEY is missing, print a clear KEY_MISSING banner and exit 0
|
||||
(the probe infrastructure is the deliverable; live numbers come when keys
|
||||
are provisioned).
|
||||
- If the key is present, run the live probe and print a latency table.
|
||||
|
||||
Usage:
|
||||
python scripts/probe_deepgram.py [--iterations N] [--model nova-3]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# Make the project importable when run from the repo root.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
except ImportError: # pragma: no cover - dotenv is a declared dep
|
||||
pass
|
||||
|
||||
|
||||
def _banner(msg: str) -> None:
|
||||
print("\n" + "=" * 72)
|
||||
print(msg)
|
||||
print("=" * 72 + "\n")
|
||||
|
||||
|
||||
def _require_key() -> str | None:
|
||||
"""Return the Deepgram API key or None (with a printed banner if missing)."""
|
||||
key = os.environ.get("DEEPGRAM_API_KEY", "").strip()
|
||||
if not key:
|
||||
_banner(
|
||||
"KEY_MISSING — DEEPGRAM_API_KEY not set.\n"
|
||||
" Cannot run live Deepgram probe. Probe infrastructure is built\n"
|
||||
" and ready; live measurements are pending API key provisioning.\n"
|
||||
" Set DEEPGRAM_API_KEY in .env (see .env.example) and re-run."
|
||||
)
|
||||
return None
|
||||
return key
|
||||
|
||||
|
||||
def _synth_pcm(duration_s: float = 2.0, sample_rate: int = 16000) -> bytes:
|
||||
"""Synthesize a short mono 16-bit PCM buffer (silence + a low tone).
|
||||
|
||||
Deepgram needs real audio frames; we generate a recognizable signal so the
|
||||
streaming endpoint returns a partial. The exact transcript content is not
|
||||
the point — the *latency to first partial* is.
|
||||
"""
|
||||
import math
|
||||
import struct
|
||||
|
||||
n = int(duration_s * sample_rate)
|
||||
frames = bytearray()
|
||||
for i in range(n):
|
||||
# 220 Hz tone for the first 1.5s, then silence — a clearly voiced segment.
|
||||
if i < int(1.5 * sample_rate):
|
||||
sample = int(16000 * math.sin(2 * math.pi * 220 * i / sample_rate))
|
||||
else:
|
||||
sample = 0
|
||||
frames += struct.pack("<h", sample)
|
||||
return bytes(frames)
|
||||
|
||||
|
||||
DEEPGRAM_WS_URL = "wss://api.deepgram.com/v1/listen"
|
||||
|
||||
|
||||
async def _probe_once(api_key: str, model: str, pcm: bytes, sample_rate: int) -> float | None:
|
||||
"""Open a Deepgram streaming WebSocket, send PCM, return ms-to-first-partial.
|
||||
|
||||
Uses the raw Deepgram streaming WebSocket API (not the SDK) so the probe is
|
||||
independent of SDK version churn and measures the actual network path.
|
||||
"""
|
||||
import websockets
|
||||
|
||||
params = (
|
||||
f"?model={model}&language=en&encoding=linear16&channels=1"
|
||||
f"&sample_rate={sample_rate}&interim_results=true&endpointing=300"
|
||||
)
|
||||
headers = [("Authorization", f"Token {api_key}")]
|
||||
t0 = time.perf_counter()
|
||||
first_partial_ms: float | None = None
|
||||
|
||||
try:
|
||||
async with websockets.connect(
|
||||
DEEPGRAM_WS_URL + params, additional_headers=headers, open_timeout=10
|
||||
) as ws:
|
||||
# Send in small chunks to mimic real streaming.
|
||||
chunk = 3200 # 100ms of 16kHz mono 16-bit
|
||||
for i in range(0, len(pcm), chunk):
|
||||
await ws.send(pcm[i : i + chunk])
|
||||
await asyncio.sleep(0.02)
|
||||
# Wait for the first transcript message.
|
||||
try:
|
||||
while True:
|
||||
msg = await asyncio.wait_for(ws.recv(), timeout=5)
|
||||
if isinstance(msg, str):
|
||||
data = json.loads(msg)
|
||||
if data.get("type") == "Results":
|
||||
channel = data.get("channel", {})
|
||||
alts = channel.get("alternatives", [])
|
||||
if alts and alts[0].get("transcript", "").strip():
|
||||
first_partial_ms = (time.perf_counter() - t0) * 1000.0
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
# Signal close.
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "CloseStream"}))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc: # pragma: no cover - network/auth errors
|
||||
print(f" [probe] Deepgram connection failed: {exc}")
|
||||
return None
|
||||
|
||||
return first_partial_ms
|
||||
|
||||
|
||||
async def run_live(api_key: str, iterations: int, model: str) -> list[float]:
|
||||
sample_rate = 16000
|
||||
pcm = _synth_pcm(duration_s=2.0, sample_rate=sample_rate)
|
||||
samples: list[float] = []
|
||||
print(f" Running {iterations} Deepgram Nova-3 iterations (model={model})...")
|
||||
for i in range(iterations):
|
||||
ms = await _probe_once(api_key, model, pcm, sample_rate)
|
||||
if ms is not None:
|
||||
samples.append(ms)
|
||||
print(f" [{i + 1:2d}/{iterations}] first-partial: {ms:6.1f} ms")
|
||||
else:
|
||||
print(f" [{i + 1:2d}/{iterations}] no partial received (skipped)")
|
||||
await asyncio.sleep(0.3)
|
||||
return samples
|
||||
|
||||
|
||||
def _summarize(samples: list[float], label: str) -> dict:
|
||||
if not samples:
|
||||
print(f"\n {label}: no samples collected.\n")
|
||||
return {"label": label, "n": 0}
|
||||
s = sorted(samples)
|
||||
p95 = s[int(0.95 * (len(s) - 1))]
|
||||
row = {
|
||||
"label": label,
|
||||
"n": len(s),
|
||||
"min_ms": round(min(s), 1),
|
||||
"median_ms": round(statistics.median(s), 1),
|
||||
"p95_ms": round(p95, 1),
|
||||
"mean_ms": round(statistics.mean(s), 1),
|
||||
}
|
||||
print(
|
||||
f" {label}: n={row['n']} min={row['min_ms']:.1f} "
|
||||
f"median={row['median_ms']:.1f} p95={row['p95_ms']:.1f} "
|
||||
f"mean={row['mean_ms']:.1f} (ms)"
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
async def amain() -> int:
|
||||
parser = argparse.ArgumentParser(description="R1 Deepgram Nova-3 latency probe")
|
||||
parser.add_argument("--iterations", type=int, default=20)
|
||||
parser.add_argument("--model", default=os.environ.get("DEEPGRAM_MODEL", "nova-3"))
|
||||
parser.add_argument("--out", default=None, help="optional JSON results path")
|
||||
args = parser.parse_args()
|
||||
|
||||
_banner("R1 PROBE — Deepgram Nova-3 first-partial-transcript latency")
|
||||
api_key = _require_key()
|
||||
if api_key is None:
|
||||
return 0
|
||||
|
||||
samples = await run_live(api_key, args.iterations, args.model)
|
||||
summary = _summarize(samples, "deepgram_nova3_first_partial")
|
||||
print()
|
||||
if args.out:
|
||||
Path(args.out).write_text(json.dumps(summary, indent=2))
|
||||
print(f" Wrote {args.out}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return asyncio.run(amain())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user