feat(P03 W2): outcome backfill (REQ-317) + escalation_reason (REQ-318)
REQ-317: core/metrics/outcome_backfill.py backfills fact_decision.outcome pending -> succeeded/failed after run.completed/run.failed; idempotent + terminal (does not overwrite a non-pending outcome); wired into the collector. The Post-Pilot AI Decision Accuracy denominator is now grounded (fact_decision.outcome is not stuck pending). REQ-318: ai.decision.made on a block band carries escalation_reason: 'confidence' (the only value in v1.26 — a block is always confidence- driven; future milestones may add 'policy'). Persisted into fact_run by the collector. The Post-Pilot Human Escalation Frequency denominator is now grounded. ---ci--- project: acdl phase: 3 milestone: v1.26 status: execute wave: W2 ---
This commit is contained in:
@@ -27,6 +27,27 @@ def _iso8601_now():
|
||||
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _backfill_outcome(decision_id, outcome):
|
||||
"""Transition fact_decision.outcome pending -> outcome (REQ-317).
|
||||
|
||||
Best-effort: logs a warning and skips if decision_id is missing or the
|
||||
backfill raises. Never raises — the run is already completing/failing
|
||||
and the manifest write is the source of truth for the run outcome.
|
||||
"""
|
||||
if not decision_id:
|
||||
# A run that failed before ai.decision.made was emitted has no
|
||||
# decision to backfill (e.g. a schema-validation failure). Skip
|
||||
# silently rather than pollute stderr on every clean run.
|
||||
return None
|
||||
try:
|
||||
from core.metrics import outcome_backfill
|
||||
return outcome_backfill.backfill(decision_id, outcome)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
print(f"[run_manifest] outcome backfill skipped for {decision_id}: {exc}",
|
||||
file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def _run_id():
|
||||
return f"run-{int(time.time())}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
@@ -44,7 +65,7 @@ def start_run(contract_id, environment, stages=None):
|
||||
return run_id
|
||||
|
||||
|
||||
def complete_run(run_id, contract_id, environment, stages, exit_code, confidence=None, hitl=None, policy=None, cost_estimate_usd=None, decision_id=None):
|
||||
def complete_run(run_id, contract_id, environment, stages, exit_code, confidence=None, hitl=None, policy=None, cost_estimate_usd=None, decision_id=None, escalation_reason=None):
|
||||
"""Emit nova.run.completed + write the per-run manifest JSON.
|
||||
|
||||
Args:
|
||||
@@ -58,6 +79,10 @@ def complete_run(run_id, contract_id, environment, stages, exit_code, confidence
|
||||
policy: optional {passed, failed, skipped}
|
||||
cost_estimate_usd: optional float
|
||||
decision_id: optional string (links to the Decision Ledger)
|
||||
escalation_reason: optional string (REQ-318) — "confidence" when
|
||||
the ai.decision.made band was block; absent/None otherwise.
|
||||
Persisted into the manifest so the collector can write it
|
||||
into fact_run (Post-Pilot Human Escalation Frequency denom).
|
||||
"""
|
||||
started_at = stages[0].get("started_at", _iso8601_now()) if stages else _iso8601_now()
|
||||
completed_at = _iso8601_now()
|
||||
@@ -83,6 +108,8 @@ def complete_run(run_id, contract_id, environment, stages, exit_code, confidence
|
||||
manifest["cost_estimate_usd"] = cost_estimate_usd
|
||||
if decision_id:
|
||||
manifest["decision_id"] = decision_id
|
||||
if escalation_reason:
|
||||
manifest["escalation_reason"] = escalation_reason
|
||||
|
||||
os.makedirs(_RUNS_DIR, exist_ok=True)
|
||||
manifest_path = os.path.join(_RUNS_DIR, f"{run_id}.json")
|
||||
@@ -92,6 +119,14 @@ def complete_run(run_id, contract_id, environment, stages, exit_code, confidence
|
||||
event_type = "nova.run.completed" if exit_code == 0 else "nova.run.failed"
|
||||
emit(event_type, run_id, environment, manifest, contract_id=contract_id)
|
||||
|
||||
# REQ-317: backfill fact_decision.outcome pending -> succeeded/failed
|
||||
# after the run completes. The decision_id links the run to the
|
||||
# Decision Ledger entry written by ai.decision.made. Best-effort: a
|
||||
# run that failed before ai.decision.made was emitted has no
|
||||
# decision_id and the backfill is a no-op (the run outcome is still
|
||||
# captured in the manifest above).
|
||||
backfill_result = _backfill_outcome(decision_id, outcome)
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user