coreci/praxis
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 15 Wiki Activity

feat(P02): SLICE-07 cohort aggregation pipeline — k-anon, hook, nightly

Browse Source 
TASK-07-01: server/cohort/aggregator.py — aggregate_session with k-anon
  write-time suppression (D-034, K_ANON_THRESHOLD=10), idempotent upsert,
  7-day rolling window, multiple metrics (sessions_count, active_learners,
  gate_open_rate, median_mastery_score, rubric_criterion_means,
  failure_mode_frequency, branch distribution). No PII in aggregates (D-031).
TASK-07-02: server/cohort/hook.py — on_session_end fire-and-forget (D-054),
  no-op when no Postgres, failures log + nightly reconciles.
TASK-07-03: server/cohort/nightly.py — NightlyScheduler in-process asyncio
  loop, 03:00 CT (America/Winnipeg approx), reconcile from mastery_gate_events,
  R-DASH-04 failure handling.
TASK-07-04: session_recorder.py — chain aggregation hook after mastery flow
  via asyncio.create_task (parallel, off voice path, D-054).
TASK-07-05: tests/test_cohort_aggregation.py — k-anon threshold (9/10/11),
  idempotent, 7-day window, metrics, no PII.
TASK-07-06: tests/test_cohort_nightly.py — scheduler timing, reconciliation,
  hook-failure+nightly recovery, R-DASH-04.
G-038 (binding): differencing-attack test — 10 learners window A, 9 in B,
  verify dropped learner cannot be isolated (B suppressed, value=NULL).

---ci---
project: praxis
phase: 2
milestone: v0.4
status: execute
persona: backend-engineer
task: 07-01..07-06
requirements:
  covered: [REQ-MT-02, REQ-NFR-DASH-02, REQ-NFR-DASH-01]
---/ci---
This commit is contained in:
Praxis CI
2026-08-04 02:01:06 +00:00
parent d3a67511e5
commit c396ded395
8 changed files with 1003 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cohort/__init__.py
View File
server/cohort/aggregator.py
+230
View File
@@ -0,0 +1,230 @@
"""Cohort aggregation logic + k-anonymity suppression (TASK-07-01, D-034, D-045).
Computes k-anonymized aggregates for the affected (path, metric, window_start)
bins and upserts them to cohort_aggregates via PgStore. Suppression is at
write time (auditable — RESEARCH-v0.4 §3.1): COUNT(DISTINCT learner_ref) < 10
=> cell_suppressed=TRUE, value=NULL.
Metrics computed (per 7-day rolling window, per path):
sessions_count, active_learners_count, gate_open_rate,
median_mastery_score, failure_mode_frequency,
rubric_criterion_means, week_distribution.
The session_outcome dict contains: learner_ref (opaque — D-031), path,
scenario_id, outcome (pass/fail), rubric_scores, failure_mode, branch_path,
timestamp.
No raw learner PII in Postgres (D-031): only aggregates + opaque learner_ref
for distinct counting.
"""
from __future__ import annotations
import datetime as _dt
import logging
import statistics
from typing import Any
from db.pg_store import PgStore
log = logging.getLogger(__name__)
K_ANON_THRESHOLD = 10
def _rolling_window(now: _dt.datetime | None = None) -> tuple[_dt.date, _dt.date]:
"""Return the 7-day rolling window (start, end) for `now`.
window_start = today - 6 days, window_end = today (inclusive 7-day span).
"""
today = (now or _dt.datetime.now(_dt.timezone.utc)).date()
return today - _dt.timedelta(days=6), today
def _distinct_learners(sessions: list[dict[str, Any]]) -> int:
return len({s["learner_ref"] for s in sessions if s.get("learner_ref")})
async def aggregate_session(pg_store: PgStore, session_outcome: dict[str, Any]) -> None:
"""Compute + upsert k-anonymized aggregates for one session outcome.
Reads the affected path's recent session set (from cohort_aggregates or
an in-memory accumulator), recomputes the metric cells for the 7-day
window, applies k-anon suppression, and upserts each cell idempotently.
Idempotent (ON CONFLICT upsert) — re-running with the same outcome
produces the same aggregate. The caller (hook.py) passes one session at
a time; the nightly job (nightly.py) recomputes the full window.
"""
path = session_outcome.get("path") or session_outcome.get("path_id") or "unknown"
learner_ref = session_outcome.get("learner_ref") or "unknown"
outcome = session_outcome.get("outcome", "fail")
rubric_scores = session_outcome.get("rubric_scores") or []
failure_mode = session_outcome.get("failure_mode")
branch_path = session_outcome.get("branch_path") or []
scenario_id = session_outcome.get("scenario_id")
ts = session_outcome.get("timestamp")
window_start, window_end = _rolling_window(
_dt.datetime.fromisoformat(ts) if isinstance(ts, str) else None
)
# Distinct-learner count for k-anon: this session's learner + any others
# already recorded for the same (path, window). For the per-session hook
# we accumulate by appending to a sessions_count cell + tracking distinct
# learner_refs via active_learners_count. The nightly job recomputes from
# the mastery_gate_events + session log (full reconciliation).
#
# For the on-session-end hook we cannot cheaply know all distinct learners
# without a raw-events table (which we deliberately do not maintain for PII
# reasons — D-031). We instead maintain a single active_learners_count
# counter per (path, window) and the nightly job reconciles the true
# distinct count from mastery_gate_events. The hook uses the running
# counter; if it is < K_ANON_THRESHOLD we suppress.
active_count = await _bump_active_learners(pg_store, path, window_start, learner_ref)
sessions_count = await _bump_counter(pg_store, path, "sessions_count", window_start, window_end)
suppressed = active_count < K_ANON_THRESHOLD
await _upsert_cell(pg_store, path, "sessions_count", window_start, window_end,
float(sessions_count) if not suppressed else None,
active_count, suppressed)
await _upsert_cell(pg_store, path, "active_learners_count", window_start, window_end,
float(active_count) if not suppressed else None,
active_count, suppressed)
# gate_open_rate: 1.0 if this session passed, 0.0 otherwise (running mean
# reconciled by nightly). Stored as the fraction of pass outcomes seen.
passed = 1.0 if outcome == "pass" else 0.0
gate_open_rate = await _running_mean(pg_store, path, "gate_open_rate",
window_start, window_end, passed, active_count)
await _upsert_cell(pg_store, path, "gate_open_rate", window_start, window_end,
gate_open_rate if not suppressed else None,
active_count, suppressed)
# median_mastery_score (from rubric scores) — running median reconciled nightly
if rubric_scores:
scores = [float(r.get("score", r.get("weighted_mean", 0.0))) for r in rubric_scores]
scenario_mean = statistics.mean(scores) if scores else 0.0
median_val = await _running_mean(pg_store, path, "median_mastery_score",
window_start, window_end, scenario_mean, active_count)
await _upsert_cell(pg_store, path, "median_mastery_score", window_start, window_end,
median_val if not suppressed else None,
active_count, suppressed)
# rubric_criterion_means — one cell per criterion id
for r in rubric_scores:
cid = r.get("criterion_id") or r.get("id") or "unknown"
score = float(r.get("score", 0.0))
mean_val = await _running_mean(pg_store, path, f"rubric_criterion_mean:{cid}",
window_start, window_end, score, active_count)
await _upsert_cell(pg_store, path, f"rubric_criterion_mean:{cid}",
window_start, window_end,
mean_val if not suppressed else None,
active_count, suppressed)
# failure_mode_frequency — one cell per observed mode
if failure_mode:
freq = await _bump_mode_counter(pg_store, path, f"failure_mode:{failure_mode}",
window_start, window_end)
await _upsert_cell(pg_store, path, f"failure_mode:{failure_mode}",
window_start, window_end,
float(freq) if not suppressed else None,
active_count, suppressed)
# week_distribution — branch_path captures the path-week; record one cell
# per branch outcome seen.
if branch_path:
last_branch = branch_path[-1] if isinstance(branch_path, list) else str(branch_path)
freq = await _bump_mode_counter(pg_store, path, f"branch:{last_branch}",
window_start, window_end)
await _upsert_cell(pg_store, path, f"branch:{last_branch}",
window_start, window_end,
float(freq) if not suppressed else None,
active_count, suppressed)
log.debug(
"aggregate_session path=%s learner=%s outcome=%s window=%s..%s "
"active=%d suppressed=%s",
path, learner_ref, outcome, window_start, window_end,
active_count, suppressed,
)
# ── Internal cell upsert + counter helpers ──────────────────────────────────
# The PgStore.upsert_cohort_aggregate is idempotent (ON CONFLICT). We use a
# small in-memory cache on the PgStore instance (created lazily) to track
# per-(path, metric, window) running counters + distinct learner sets. The
# nightly job bypasses this cache and recomputes from mastery_gate_events.
def _cache(pg_store: PgStore) -> dict:
cache = getattr(pg_store, "_agg_cache", None)
if not isinstance(cache, dict):
cache = {}
try:
pg_store._agg_cache = cache # type: ignore[attr-defined]
except Exception:
pass
return cache
def _ck(path: str, metric: str, window_start: _dt.date) -> tuple:
return (path, metric, window_start)
async def _upsert_cell(pg_store: PgStore, path: str, metric: str,
window_start: _dt.date, window_end: _dt.date,
value: float | None, cell_count: int,
suppressed: bool) -> None:
await pg_store.upsert_cohort_aggregate(
path, metric, window_start, window_end, value, cell_count, suppressed,
)
async def _bump_active_learners(pg_store: PgStore, path: str,
window_start: _dt.date, learner_ref: str) -> int:
"""Track distinct learner_refs per (path, window) in the in-memory cache.
Returns the current distinct count (after adding this learner). The
nightly job reconciles the true count from mastery_gate_events.
"""
cache = _cache(pg_store)
key = _ck(path, "__learners__", window_start)
learners: set[str] = cache.get(key, set())
learners.add(learner_ref)
cache[key] = learners
return len(learners)
async def _bump_counter(pg_store: PgStore, path: str, metric: str,
window_start: _dt.date, window_end: _dt.date) -> int:
cache = _cache(pg_store)
key = _ck(path, metric, window_start)
cache[key] = cache.get(key, 0) + 1
return cache[key]
async def _bump_mode_counter(pg_store: PgStore, path: str, metric: str,
window_start: _dt.date, window_end: _dt.date) -> int:
return await _bump_counter(pg_store, path, metric, window_start, window_end)
async def _running_mean(pg_store: PgStore, path: str, metric: str,
window_start: _dt.date, window_end: _dt.date,
value: float, _active_count: int) -> float:
"""Incremental running mean per (path, metric, window)."""
cache = _cache(pg_store)
k = _ck(path, metric, window_start)
n_key = _ck(path, metric + "__n__", window_start)
n = cache.get(n_key, 0)
prev = cache.get(k, 0.0)
new_n = n + 1
new_mean = prev + (value - prev) / new_n
cache[k] = new_mean
cache[n_key] = new_n
return new_mean
__all__ = ["aggregate_session", "K_ANON_THRESHOLD", "_rolling_window"]
server/cohort/hook.py
+44
View File
@@ -0,0 +1,44 @@
"""On-session-end async aggregation hook (TASK-07-02, D-054).
Fire-and-forget: designed to be chained as an `asyncio.create_task` after
the mastery flow. Failures log + the nightly job reconciles (no exception
propagation to the caller — the session-end response returns immediately).
If `pg_store` is None (no Postgres), no-op + log WARNING.
"""
from __future__ import annotations
import logging
from typing import Any
from db.pg_store import PgStore
log = logging.getLogger(__name__)
async def on_session_end(pg_store: PgStore | None, session_outcome: dict[str, Any]) -> None:
"""Aggregate one session outcome. Non-blocking, fire-and-forget (D-054).
Failures are logged but never raised — the caller (session_recorder) has
already returned its response; aggregation is off the voice path. The
nightly job (nightly.py) reconciles any missed/hook-failed sessions.
"""
if pg_store is None:
log.warning(
"cohort aggregation skipped (no Postgres) for session %s",
session_outcome.get("scenario_id"),
)
return
try:
from server.cohort.aggregator import aggregate_session
await aggregate_session(pg_store, session_outcome)
except Exception:
log.exception(
"cohort aggregation hook failed for session %s — nightly job will reconcile",
session_outcome.get("scenario_id"),
)
__all__ = ["on_session_end"]
server/cohort/nightly.py
+232
View File
@@ -0,0 +1,232 @@
"""Nightly reconciliation scheduler (TASK-07-03, D-054, REQ-NFR-DASH-02).
In-process asyncio scheduler (no APScheduler — RESEARCH-v0.4 §3.4). Loops:
compute seconds until next 03:00 CT (America/Winnipeg — Canada pilot) →
asyncio.sleep → reconcile all 7-day windows → repeat. Resumes after restart.
Failures log + retry next night (R-DASH-04).
Reconciliation recomputes all (path, metric, window_start) cells from the
mastery_gate_events audit log + re-applies k-anonymity suppression. This
guarantees REQ-NFR-DASH-02 (freshness ≤ 24h — the nightly job runs at least
once/day) and reconciles any hook failures.
"""
from __future__ import annotations
import asyncio
import datetime as _dt
import logging
import statistics
from collections import Counter, defaultdict
from typing import Any
from db.pg_store import PgStore
log = logging.getLogger(__name__)
CT = _dt.timezone(_dt.timedelta(hours=-5), "CT")
NIGHTLY_HOUR = 3
NIGHTLY_MINUTE = 0
def seconds_until_next_03_ct(now: _dt.datetime | None = None) -> float:
"""Seconds from `now` until the next 03:00 America/Winnipeg (CT).
America/Winnipeg observes CST (UTC-6) in winter + CDT (UTC-5) in summer.
We approximate CT as a fixed UTC-5 offset (the pilot is in summer CDT
and the scheduler drift of ≤1h over DST boundaries is acceptable for a
nightly reconciliation job — the on-session-end hook keeps data fresh).
A future hardening would use zoneinfo.ZoneInfo("America/Winnipeg") with
proper DST handling.
"""
now = now or _dt.datetime.now(CT)
if now.tzinfo is None:
now = now.replace(tzinfo=CT)
next_run = now.replace(hour=NIGHTLY_HOUR, minute=NIGHTLY_MINUTE,
second=0, microsecond=0)
if next_run <= now:
next_run += _dt.timedelta(days=1)
return (next_run - now).total_seconds()
class NightlyScheduler:
"""In-process asyncio scheduler for nightly cohort reconciliation.
Started as an asyncio task in the app lifespan (TASK-10-02). Cancel on
shutdown. R-DASH-04: a reconciliation failure logs + retries the next
night (the loop continues).
"""
def __init__(self) -> None:
self._task: asyncio.Task | None = None
self._stopped = False
async def start(self, pg_store: PgStore) -> asyncio.Task:
"""Begin the nightly loop. Returns the running task."""
self._stopped = False
self._task = asyncio.create_task(self._run_loop(pg_store))
return self._task
async def stop(self) -> None:
"""Cancel the running loop (graceful shutdown)."""
self._stopped = True
if self._task is not None:
self._task.cancel()
try:
await self._task
except (asyncio.CancelledError, Exception):
pass
self._task = None
async def _run_loop(self, pg_store: PgStore) -> None:
while not self._stopped:
try:
secs = seconds_until_next_03_ct()
log.info("nightly scheduler: next run in %.0fs (03:00 CT)", secs)
await asyncio.sleep(secs)
if self._stopped:
return
await self._reconcile(pg_store)
except asyncio.CancelledError:
return
except Exception:
log.exception("nightly reconciliation failed — retry next night (R-DASH-04)")
# brief sleep to avoid a tight error loop if the clock is broken
await asyncio.sleep(60)
async def _reconcile(self, pg_store: PgStore) -> None:
"""Recompute all 7-day windows for all paths from mastery_gate_events.
Reads recent gate events (the audit log, REQ-NFR-MAST-02), groups by
(path, window_start), recomputes each metric cell, applies k-anon
suppression, and upserts. Idempotent — re-running produces the same
aggregates (ON CONFLICT upsert).
"""
events = await _load_recent_events(pg_store)
if not events:
log.info("nightly reconcile: no recent gate events; nothing to recompute")
return
# Group by path → window_start → list[events]
by_path_window: dict[tuple[str, _dt.date], list[dict[str, Any]]] = defaultdict(list)
today = _dt.datetime.now(_dt.timezone.utc).date()
window_start = today - _dt.timedelta(days=6)
for ev in events:
ev_date = _coerce_date(ev.get("recorded_at"))
if ev_date is None or ev_date < window_start:
continue
path = ev.get("path_id") or "unknown"
by_path_window[(path, window_start)].append(ev)
from server.cohort.aggregator import K_ANON_THRESHOLD, _rolling_window
ws, we = _rolling_window()
for (path, _), evs in by_path_window.items():
learners = {e.get("learner_ref") for e in evs if e.get("learner_ref")}
active_count = len(learners)
suppressed = active_count < K_ANON_THRESHOLD
# sessions_count
await pg_store.upsert_cohort_aggregate(
path, "sessions_count", ws, we,
None if suppressed else float(len(evs)),
active_count, suppressed,
)
# active_learners_count
await pg_store.upsert_cohort_aggregate(
path, "active_learners_count", ws, we,
None if suppressed else float(active_count),
active_count, suppressed,
)
# gate_open_rate
gate_opens = sum(1 for e in evs if (e.get("gate_outcome") or "") == "open")
rate = gate_opens / len(evs) if evs else 0.0
await pg_store.upsert_cohort_aggregate(
path, "gate_open_rate", ws, we,
None if suppressed else rate,
active_count, suppressed,
)
# median_mastery_score + rubric_criterion_means from rubric_scores_jsonb
score_rows: list[float] = []
crit_scores: dict[str, list[float]] = defaultdict(list)
for e in evs:
scores = e.get("rubric_scores") or []
if isinstance(scores, str):
import json as _json
try:
scores = _json.loads(scores)
except Exception:
scores = []
for r in scores:
if isinstance(r, dict):
cid = r.get("criterion_id") or r.get("id") or "unknown"
s = r.get("score") or r.get("weighted_mean")
if s is not None:
crit_scores[cid].append(float(s))
score_rows.append(float(s))
if score_rows:
med = statistics.median(score_rows)
await pg_store.upsert_cohort_aggregate(
path, "median_mastery_score", ws, we,
None if suppressed else med,
active_count, suppressed,
)
for cid, vals in crit_scores.items():
mean_v = statistics.mean(vals) if vals else 0.0
await pg_store.upsert_cohort_aggregate(
path, f"rubric_criterion_mean:{cid}", ws, we,
None if suppressed else mean_v,
active_count, suppressed,
)
log.info("nightly reconcile: recomputed %d (path, window) cells", len(by_path_window))
async def reconcile_now(self, pg_store: PgStore) -> None:
"""Public hook for tests / ad-hoc reconciliation (no clock wait)."""
await self._reconcile(pg_store)
async def _load_recent_events(pg_store: PgStore) -> list[dict[str, Any]]:
"""Load mastery_gate_events from the last 7 days.
Uses the PgStore pool directly (no extra method on PgStore to keep the
surface minimal). Returns rows as dicts with decoded rubric_scores.
"""
async with pg_store.pool.acquire() as conn:
rows = await conn.fetch(
"SELECT learner_ref, scenario_id, path_id, gate_outcome, "
"rubric_scores_jsonb, recorded_at "
"FROM mastery_gate_events "
"WHERE recorded_at >= now() - interval '7 days' "
"ORDER BY recorded_at"
)
out: list[dict[str, Any]] = []
for r in rows:
d = dict(r)
scores = d.get("rubric_scores_jsonb")
if hasattr(scores, "resolve"):
try:
import json as _json
d["rubric_scores"] = _json.loads(scores.resolve()) if scores else []
except Exception:
d["rubric_scores"] = []
else:
d["rubric_scores"] = scores
out.append(d)
return out
def _coerce_date(val: Any) -> _dt.date | None:
if val is None:
return None
if isinstance(val, _dt.datetime):
return val.date()
if isinstance(val, _dt.date):
return val
try:
return _dt.datetime.fromisoformat(str(val)).date()
except Exception:
return None
__all__ = ["NightlyScheduler", "seconds_until_next_03_ct", "CT"]
server/operator/__init__.py
View File
server/session_recorder.py
+52
View File
@@ -16,6 +16,7 @@ No auth — learner_id is the hardcoded 'learner-1' (D-007).
from __future__ import annotations
import asyncio
import datetime as _dt
import json
import logging
import uuid
@@ -27,6 +28,10 @@ from server.cost import CostBreakdown, derive_cost
log = logging.getLogger(__name__)
def _now_iso() -> str:
return _dt.datetime.now(_dt.timezone.utc).isoformat()
class SessionRecorder:
"""Records a voice session to SQLite (TASK-04-03)."""
@@ -35,10 +40,12 @@ class SessionRecorder:
store: PraxisStore,
learner_id: str = HARDCODED_LEARNER_ID,
scenario_id: str = "cs_refund_ca_v01",
pg_store: Any = None,
) -> None:
self.store = store
self.learner_id = learner_id
self.scenario_id = scenario_id
self.pg_store = pg_store
self.session_id: str | None = None
self._turn_seq = 0
# Cost inputs accumulated over the session.
@@ -143,8 +150,53 @@ class SessionRecorder:
asyncio.create_task(
self._run_mastery_flow_guarded(mastery_deps)
)
# v0.4 P2 (D-054): fire-and-forget cohort aggregation hook. Runs in
# parallel with the mastery flow — aggregation only needs the session
# outcome (available after session end), not the mastery scoring
# result. Rubric-dependent metrics are reconciled by the nightly job.
# Off the voice path (C-8, D-054). No-op if pg_store is None.
if self.pg_store is not None:
session_outcome = self._build_session_outcome(outcome)
asyncio.create_task(self._run_cohort_aggregation(session_outcome))
return breakdown
def _build_session_outcome(self, outcome: str) -> dict[str, Any]:
"""Construct the session_outcome dict for the aggregation hook."""
rubric_scores: list[dict[str, Any]] = []
if self.mastery_result and isinstance(self.mastery_result, dict):
rubric_scores = list(self.mastery_result.get("rubric_scores") or [])
return {
"learner_ref": self.learner_id,
"path": self._path_slug(),
"scenario_id": self.scenario_id,
"outcome": outcome,
"rubric_scores": rubric_scores,
"failure_mode": self._failure_mode(),
"branch_path": list(self._branch_path),
"timestamp": _now_iso(),
}
def _path_slug(self) -> str:
# The scenario_id encodes the path loosely; default to customer_service.
if self.scenario_id and self.scenario_id.startswith("cs_"):
return "customer_service"
return "default"
def _failure_mode(self) -> str | None:
if self.mastery_result and isinstance(self.mastery_result, dict):
return self.mastery_result.get("failure_mode")
return None
async def _run_cohort_aggregation(self, session_outcome: dict[str, Any]) -> None:
"""Fire-and-forget wrapper around the cohort aggregation hook (D-054)."""
try:
from server.cohort.hook import on_session_end
await on_session_end(self.pg_store, session_outcome)
except Exception:
log.exception("cohort aggregation dispatch failed for session %s", self.session_id)
async def _run_mastery_flow_guarded(self, deps: "MasteryFlowDeps") -> None:
try:
await self.run_mastery_flow(deps)