Jon Chery
8bcf7296d5
REQ-223: mcp/atelier/server.py plugin-registry MCP server (stdio, D-135). NovaAtelierServer wraps MCPServer (SDK v2, D-137) if installed; degrades to _ToolRegistry fallback if SDK absent (testable in CI without SDK). plugins/principles.py (lookup_principle, list_domains, matrix_lookup) + plugins/validation.py (validate_against_principles — agentic validation beyond Wiz/Checkmarx/Mend). 4 tools, 2 plugins. REQ-224: mcp/atelier/vendor/ pinned Atelier v0.3.6 (D-136) — core/ first-principles, domains/security/first-principles, review/agent-checklist, matrix/principles-matrix. vendor/VERSION.md + scripts/update_atelier_vendor.sh for intentional upgrades. mcp/atelier/README.md (tools, architecture, running, vendoring, extensibility, transport). REQ-225: tests/test_atelier_mcp.py — 16 tests, all pass. Covers: plugin discovery (both loaded), 4 tools registered, lookup_security_P4 (+P1, unknown domain/principle), list_domains (19, security-relevant, ui-ux-not), matrix_lookup (security 10 P-rules, unknown), validation (good-passes, bad-secret-fails, bad-swallowed-error-fails, bad-obfuscated-names-fails, result-structure). ---ci--- project: acdl phase: 5 milestone: v1.18 status: execute requirements: covered: [REQ-223, REQ-224, REQ-225] partial: [] ---/ci---
78 lines
4.5 KiB
Python
78 lines
4.5 KiB
Python
"""mcp/atelier/plugins/validation.py — agentic validation against Atelier principles.
|
|
|
|
Implements 1 MCP tool (REQ-223):
|
|
- atelier.validate_against_principles(snippet, domains) → pass/fail per
|
|
checklist item with the principle citation. This is the agentic
|
|
validation BEYOND deterministic scanners (Wiz/Checkmarx/Mend) — it
|
|
catches correctness/clarity/simplicity/observability gaps that
|
|
deterministic tools cannot.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
# Condensed checklist: core C1-C8 + security domain. Each item is a
|
|
# (check_id, description, heuristic_pattern, principle_citation).
|
|
_CHECKLIST = [
|
|
# C1 Correctness
|
|
{"id": "C1.1", "desc": "Does the code do what the task asked, completely?", "heuristic": r"TODO|FIXME|pass\s*$", "principle": "C1 Correctness", "neg": True},
|
|
{"id": "C1.2", "desc": "Does it handle failure cases? (errors, timeouts)", "heuristic": r"except\s*:?\s*pass", "principle": "C1 Correctness", "neg": True},
|
|
{"id": "C1.3", "desc": "Is there a test that would fail if the code were wrong?", "heuristic": r"def test_|describe\(", "principle": "C1 Correctness", "neg": False, "optional": True},
|
|
# C2 Clarity
|
|
{"id": "C2.1", "desc": "Are names intent-revealing? (no 'data', 'temp', 'x')", "heuristic": r"\b(data|temp|x|foo|bar|doStuff)\b", "principle": "C2 Clarity", "neg": True},
|
|
# C3 Simplicity
|
|
{"id": "C3.1", "desc": "Is there dead code? (unreachable branches)", "heuristic": r"return\s+\w+\s*$.*return", "principle": "C3 Simplicity", "neg": True, "multiline": True},
|
|
# C7 Observability
|
|
{"id": "C7.1", "desc": "Are there logs for significant events?", "heuristic": r"log(ger|ging)?|print\(|console\.", "principle": "C7 Observability", "neg": False, "optional": True},
|
|
{"id": "C7.2", "desc": "Are there secrets in logs?", "heuristic": r"password|secret|token|api_key", "principle": "C7 Observability + Security P4", "neg": True},
|
|
# Security
|
|
{"id": "SEC.1", "desc": "No secrets in code/logs/URLs", "heuristic": r"(password|secret|token|api_key)\s*=\s*['\"]", "principle": "Security P4 Secrets Never Exposed", "neg": True},
|
|
{"id": "SEC.2", "desc": "Input validated at the boundary", "heuristic": r"validate|schema|assert", "principle": "Security P1 Boundary Validation", "neg": False, "optional": True},
|
|
{"id": "SEC.3", "desc": "Authorization checked, not assumed", "heuristic": r"auth|permission|rbac|authorize", "principle": "Security P5 Authenticated by Default", "neg": False, "optional": True},
|
|
]
|
|
|
|
|
|
def register(mcp: Any) -> None:
|
|
"""Register the validation tools with the MCP server (or fallback registry)."""
|
|
|
|
@mcp.tool()
|
|
def atelier_validate_against_principles(snippet: str, domains: list[str] | None = None) -> dict[str, Any]:
|
|
"""Validate a code/diff snippet against Atelier principles.
|
|
|
|
Runs the agent-checklist items against the snippet and returns
|
|
pass/fail per item with the principle citation. This is the
|
|
agentic validation BEYOND deterministic scanners (Wiz/Checkmarx/
|
|
Mend) — it catches correctness/clarity/simplicity/observability
|
|
gaps that deterministic tools cannot.
|
|
|
|
Args:
|
|
snippet: The code or diff text to validate.
|
|
domains: Optional list of domains to include (default: core + security).
|
|
"""
|
|
results: list[dict[str, Any]] = []
|
|
for check in _CHECKLIST:
|
|
pattern = check["heuristic"]
|
|
flags = re.DOTALL if check.get("multiline") else 0
|
|
found = bool(re.search(pattern, snippet, flags))
|
|
# neg=True means finding the pattern is a FAIL; neg=False means finding is a PASS
|
|
if check.get("neg"):
|
|
status = "FAIL" if found else "PASS"
|
|
else:
|
|
if check.get("optional"):
|
|
status = "PASS" if found else "WARN"
|
|
else:
|
|
status = "PASS" if found else "WARN"
|
|
results.append({
|
|
"check_id": check["id"],
|
|
"description": check["desc"],
|
|
"status": status,
|
|
"principle": check["principle"],
|
|
})
|
|
all_pass = all(r["status"] == "PASS" for r in results)
|
|
return {
|
|
"overall": "PASS" if all_pass else "FAIL",
|
|
"results": results,
|
|
"domains_checked": domains or ["core", "security"],
|
|
"note": "Agentic validation beyond Wiz/Checkmarx/Mend — catches correctness, clarity, simplicity, observability gaps.",
|
|
} |