feat(P3): submission-readiness input contract — schema + validator + docs + tests (REQ-217..220)
REQ-217: schemas/submission-readiness.schema.json (JSON Schema draft 2020-12) defines acceptable-to-start as a superset gate above contract.schema.json: contractId, environment, tags (5 Nova tags D-054), policyPreconditions, profile (developer|agentic), appSource (repo+ref), per-env mandatory (W3.E: qa→e2eSuite+loadTest, prod→runbook+dashboard+oncall, dr→drDrillRef), agentic markers (naturalLanguageIntent+confidenceAtSubmission+agentTrace). REQ-218: core/submission_readiness.py validator with check_readiness() + ReadinessResult (structured pass/fail + reason codes). Wired as contract_ingestor.py --check-readiness (D-133). Reason codes: MISSING_TAGS, ENV_MISSING_MANDATORY, AGENTIC_MISSING_INTENT, MISSING_APP_SOURCE, POLICY_PRECONDITION_MISSING. Never raises — all failures are reason codes. REQ-219: docs/submission-readiness.md (good + rejected examples + reason-code catalog + compliance-standard equivalence). REQ-220: tests/test_submission_readiness.py — 16 tests, all pass. Covers: good-pass, good-agentic-pass, missing-tags, empty-tag, qa-missing-e2e, prod-missing-runbook, dr-missing-drdrill, prod-all-pass, agentic-missing-all, agentic-missing-one, missing-appsource, appsource-missing-ref, empty-policy, result-structure. ---ci--- project: acdl phase: 3 milestone: v1.18 status: execute requirements: covered: [REQ-217, REQ-218, REQ-219, REQ-220] partial: [] ---/ci---
This commit is contained in:
Jon Chery
@@ -0,0 +1,189 @@
|
||||
"""tests/test_submission_readiness.py — REQ-220.
|
||||
|
||||
Covers: good contract passes; missing tags fail with MISSING_TAGS;
|
||||
env-missing-mandatory fails with ENV_MISSING_MANDATORY:<env>:<field>;
|
||||
agentic profile missing intent fails with AGENTIC_MISSING_INTENT;
|
||||
missing appSource fails with MISSING_APP_SOURCE.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from core.submission_readiness import check_readiness, ReadinessResult
|
||||
|
||||
GOOD_TAGS = {
|
||||
"nova:owner": "consumer-repo",
|
||||
"nova:contract": "uuid-1234",
|
||||
"nova:environment": "dev",
|
||||
"nova:cost-center": "nova-default",
|
||||
"nova:ref": "CHG0678912",
|
||||
}
|
||||
|
||||
|
||||
def _base(**overrides):
|
||||
submission = {
|
||||
"contractId": "uuid-1234",
|
||||
"id": "webapi",
|
||||
"name": "Customer Web API",
|
||||
"environment": "dev",
|
||||
"tags": dict(GOOD_TAGS),
|
||||
"policyPreconditions": {"public-ingress": False, "encryption_enabled": True},
|
||||
"profile": "developer",
|
||||
"appSource": {"repo": "consumer/repo", "ref": "main"},
|
||||
"infrastructure": {
|
||||
"static-assets": {"inputs": {"bucket_name": "webapi-assets"}}
|
||||
},
|
||||
}
|
||||
submission.update(overrides)
|
||||
return submission
|
||||
|
||||
|
||||
class TestGoodContract(unittest.TestCase):
|
||||
def test_good_contract_passes(self):
|
||||
result = check_readiness(_base())
|
||||
self.assertTrue(result.ready, f"Expected ready, got: {result.reason_codes}")
|
||||
self.assertEqual(result.contract_id, "uuid-1234")
|
||||
|
||||
def test_good_agentic_contract_passes(self):
|
||||
submission = _base(
|
||||
profile="agentic",
|
||||
naturalLanguageIntent="A web API for customer data",
|
||||
confidenceAtSubmission=0.85,
|
||||
agentTrace="LLM generated contract from issue #42",
|
||||
)
|
||||
result = check_readiness(submission)
|
||||
self.assertTrue(result.ready, f"Expected ready, got: {result.reason_codes}")
|
||||
|
||||
|
||||
class TestMissingTags(unittest.TestCase):
|
||||
def test_missing_tags_fail(self):
|
||||
submission = _base()
|
||||
submission["tags"] = {"nova:owner": "consumer-repo"}
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
codes = " ".join(result.reason_codes)
|
||||
self.assertIn("MISSING_TAGS", codes)
|
||||
self.assertIn("nova:contract", codes)
|
||||
self.assertIn("nova:environment", codes)
|
||||
self.assertIn("nova:cost-center", codes)
|
||||
self.assertIn("nova:ref", codes)
|
||||
|
||||
def test_empty_tag_value_fails(self):
|
||||
submission = _base()
|
||||
submission["tags"]["nova:owner"] = ""
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
self.assertTrue(any("MISSING_TAGS" in c for c in result.reason_codes))
|
||||
|
||||
|
||||
class TestEnvMissingMandatory(unittest.TestCase):
|
||||
def test_qa_missing_e2e_suite_fails(self):
|
||||
submission = _base(environment="qa")
|
||||
submission["tags"]["nova:environment"] = "qa"
|
||||
# No validation.e2eSuite
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
codes = " ".join(result.reason_codes)
|
||||
self.assertIn("ENV_MISSING_MANDATORY:qa:validation.e2eSuite", codes)
|
||||
|
||||
def test_prod_missing_runbook_fails(self):
|
||||
submission = _base(environment="prod")
|
||||
submission["tags"]["nova:environment"] = "prod"
|
||||
# No runbook/dashboard/oncall
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
codes = " ".join(result.reason_codes)
|
||||
self.assertIn("ENV_MISSING_MANDATORY:prod:runbook", codes)
|
||||
self.assertIn("ENV_MISSING_MANDATORY:prod:dashboard", codes)
|
||||
self.assertIn("ENV_MISSING_MANDATORY:prod:oncall", codes)
|
||||
|
||||
def test_dr_missing_drdrillref_fails(self):
|
||||
submission = _base(environment="dr")
|
||||
submission["tags"]["nova:environment"] = "dr"
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
codes = " ".join(result.reason_codes)
|
||||
self.assertIn("ENV_MISSING_MANDATORY:dr:drDrillRef", codes)
|
||||
|
||||
def test_prod_with_all_mandatory_passes(self):
|
||||
submission = _base(
|
||||
environment="prod",
|
||||
runbook="docs/runbooks/webapi.md",
|
||||
dashboard="https://grafana/nova/webapi",
|
||||
oncall="oncall@company.com",
|
||||
)
|
||||
submission["tags"]["nova:environment"] = "prod"
|
||||
result = check_readiness(submission)
|
||||
self.assertTrue(result.ready, f"Expected ready, got: {result.reason_codes}")
|
||||
|
||||
|
||||
class TestAgenticMissingIntent(unittest.TestCase):
|
||||
def test_agentic_missing_all_markers_fails(self):
|
||||
submission = _base(profile="agentic")
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
codes = " ".join(result.reason_codes)
|
||||
self.assertIn("AGENTIC_MISSING_INTENT:naturalLanguageIntent", codes)
|
||||
self.assertIn("AGENTIC_MISSING_INTENT:confidenceAtSubmission", codes)
|
||||
self.assertIn("AGENTIC_MISSING_INTENT:agentTrace", codes)
|
||||
|
||||
def test_agentic_missing_one_marker_fails(self):
|
||||
submission = _base(
|
||||
profile="agentic",
|
||||
naturalLanguageIntent="A web API",
|
||||
confidenceAtSubmission=0.85,
|
||||
# agentTrace missing
|
||||
)
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
self.assertTrue(any("agentTrace" in c for c in result.reason_codes))
|
||||
|
||||
|
||||
class TestMissingAppSource(unittest.TestCase):
|
||||
def test_missing_appsource_fails(self):
|
||||
submission = _base()
|
||||
del submission["appSource"]
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
self.assertTrue(any("MISSING_APP_SOURCE" in c for c in result.reason_codes))
|
||||
|
||||
def test_appsource_missing_ref_fails(self):
|
||||
submission = _base()
|
||||
submission["appSource"] = {"repo": "consumer/repo"}
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
self.assertTrue(any("MISSING_APP_SOURCE" in c for c in result.reason_codes))
|
||||
|
||||
|
||||
class TestPolicyPreconditionMissing(unittest.TestCase):
|
||||
def test_empty_policy_fails(self):
|
||||
submission = _base()
|
||||
submission["policyPreconditions"] = {}
|
||||
result = check_readiness(submission)
|
||||
self.assertFalse(result.ready)
|
||||
self.assertTrue(any("POLICY_PRECONDITION_MISSING" in c for c in result.reason_codes))
|
||||
|
||||
|
||||
class TestReadinessResultStructure(unittest.TestCase):
|
||||
def test_result_to_dict(self):
|
||||
result = check_readiness(_base())
|
||||
d = result.to_dict()
|
||||
self.assertIn("ready", d)
|
||||
self.assertIn("reason_codes", d)
|
||||
self.assertIn("contractId", d)
|
||||
|
||||
def test_result_str_ready(self):
|
||||
result = check_readiness(_base())
|
||||
self.assertIn("READY", str(result))
|
||||
|
||||
def test_result_str_not_ready(self):
|
||||
submission = _base()
|
||||
del submission["appSource"]
|
||||
result = check_readiness(submission)
|
||||
self.assertIn("NOT READY", str(result))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user