Jon Chery
b33431c1a6
---ci---
project: ci
phase: 4
milestone: v0.5
status: complete
decisions:
- id: D-028
decision: Phase 4 Verification Intelligence complete
rationale: All INTEL requirements covered; 31 suites, 355 tests
confidence: 0.95
alternatives: []
requirements:
covered: [INTEL-01, INTEL-02, INTEL-03]
---/ci---
389 lines
12 KiB
TypeScript
389 lines
12 KiB
TypeScript
import * as fs from "node:fs";
|
|
import * as path from "node:path";
|
|
import { execSync } from "node:child_process";
|
|
import { VerificationLayer, VerificationResult, VerificationCheck } from "./types.js";
|
|
|
|
const TEST_FRAMEWORK_PATTERNS = [
|
|
{ name: "Jest", pattern: /jest\.config\.(js|ts|mjs|cjs)|"jest":\s*\{|describe\s*\(|it\s*\(|test\s*\(/ },
|
|
{ name: "Mocha", pattern: /mocha|describe\s*\(|it\s*\(/ },
|
|
{ name: "Vitest", pattern: /vitest\.config\.(ts|js)|from\s+['"]vitest['"]/ },
|
|
];
|
|
|
|
const MUST_HAVE_KEYWORDS = [
|
|
"must", "shall", "required", "needs to", "has to", "will",
|
|
"should", "critical", "essential", "mandatory", "necessary",
|
|
];
|
|
|
|
export class BehavioralVerification extends VerificationLayer {
|
|
readonly layer = 2;
|
|
readonly name = "Behavioral";
|
|
|
|
async verify(projectPath: string, phase: number): Promise<VerificationResult> {
|
|
const start = Date.now();
|
|
const checks: VerificationCheck[] = [];
|
|
|
|
checks.push(this.checkTestFramework(projectPath));
|
|
checks.push(this.checkTestFiles(projectPath));
|
|
checks.push(this.checkSpecificationRequirements(projectPath));
|
|
checks.push(this.checkPlanMustHaves(projectPath, phase));
|
|
checks.push(this.checkCodeHasExports(projectPath));
|
|
checks.push(this.checkRequirementTestCoverage(projectPath));
|
|
|
|
const passed = checks.every((c) => c.status !== "fail");
|
|
return {
|
|
layer: this.layer,
|
|
name: this.name,
|
|
passed,
|
|
checks,
|
|
summary: `${checks.filter((c) => c.status === "pass").length}/${checks.length} checks passed`,
|
|
duration_ms: Date.now() - start,
|
|
};
|
|
}
|
|
|
|
private checkTestFramework(projectPath: string): VerificationCheck {
|
|
const packageJsonPath = path.join(projectPath, "package.json");
|
|
if (!fs.existsSync(packageJsonPath)) {
|
|
return this.check("Test framework detected", "skipped", "No package.json found");
|
|
}
|
|
|
|
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
|
|
const devDeps = Object.keys(packageJson.devDependencies || {});
|
|
const deps = Object.keys(packageJson.dependencies || {});
|
|
const allDeps = [...devDeps, ...deps];
|
|
|
|
const testDeps = allDeps.filter((d) =>
|
|
["jest", "mocha", "vitest", "jasmine", "ava", "tape"].includes(d)
|
|
);
|
|
|
|
if (testDeps.length > 0) {
|
|
return this.check(
|
|
"Test framework detected",
|
|
"pass",
|
|
`Found test framework(s): ${testDeps.join(", ")}`
|
|
);
|
|
}
|
|
|
|
const configFiles = ["jest.config.js", "jest.config.ts", "vitest.config.ts", "vitest.config.js", ".mocharc.yml", ".mocharc.json"];
|
|
const foundConfig = configFiles.filter((f) => fs.existsSync(path.join(projectPath, f)));
|
|
|
|
if (foundConfig.length > 0) {
|
|
return this.check(
|
|
"Test framework detected",
|
|
"pass",
|
|
`Found test config: ${foundConfig.join(", ")}`
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Test framework detected",
|
|
"warning",
|
|
"No test framework found in dependencies or config files"
|
|
);
|
|
}
|
|
|
|
private checkTestFiles(projectPath: string): VerificationCheck {
|
|
const testDirs = ["src", "test", "tests", "__tests__"];
|
|
const testFiles: string[] = [];
|
|
|
|
for (const dir of testDirs) {
|
|
const fullPath = path.join(projectPath, dir);
|
|
if (fs.existsSync(fullPath)) {
|
|
testFiles.push(...this.findTestFiles(fullPath, projectPath));
|
|
}
|
|
}
|
|
|
|
if (testFiles.length === 0) {
|
|
return this.check(
|
|
"Test files exist",
|
|
"warning",
|
|
"No test files found. Behavioral verification cannot run without tests."
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Test files exist",
|
|
"pass",
|
|
`Found ${testFiles.length} test file(s)`
|
|
);
|
|
}
|
|
|
|
private checkSpecificationRequirements(projectPath: string): VerificationCheck {
|
|
const reqPath = path.join(projectPath, ".ci", "REQUIREMENTS.md");
|
|
const projectPath_md = path.join(projectPath, ".ci", "PROJECT.md");
|
|
|
|
const specPath = reqPath;
|
|
if (!fs.existsSync(specPath)) {
|
|
const altPath = projectPath_md;
|
|
if (!fs.existsSync(altPath)) {
|
|
return this.check(
|
|
"Specification requirements traceable",
|
|
"skipped",
|
|
"No REQUIREMENTS.md or PROJECT.md found"
|
|
);
|
|
}
|
|
return this.checkFromProjectMd(altPath);
|
|
}
|
|
|
|
const content = fs.readFileSync(specPath, "utf-8");
|
|
const requirements = content
|
|
.split("\n")
|
|
.filter((line) => /^\|.*\|.*\|.*\|/.test(line) && !line.includes("REQ-ID") && !line.includes("---"))
|
|
.map((line) => {
|
|
const cols = line.split("|").map((c) => c.trim()).filter(Boolean);
|
|
return cols.length >= 2 ? cols[1] : "";
|
|
})
|
|
.filter(Boolean);
|
|
|
|
if (requirements.length === 0) {
|
|
const listRequirements = content
|
|
.split("\n")
|
|
.filter((line) => line.trim().startsWith("- "))
|
|
.map((line) => line.trim().slice(2));
|
|
if (listRequirements.length === 0) {
|
|
return this.check(
|
|
"Specification requirements traceable",
|
|
"warning",
|
|
"No requirements found in REQUIREMENTS.md"
|
|
);
|
|
}
|
|
return this.check(
|
|
"Specification requirements traceable",
|
|
"pass",
|
|
`Found ${listRequirements.length} requirement(s)`
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Specification requirements traceable",
|
|
"pass",
|
|
`Found ${requirements.length} requirement(s) in REQUIREMENTS.md`
|
|
);
|
|
}
|
|
|
|
private checkFromProjectMd(specPath: string): VerificationCheck {
|
|
const content = fs.readFileSync(specPath, "utf-8");
|
|
const requirements = content
|
|
.split("\n")
|
|
.filter((line) => line.trim().startsWith("- "))
|
|
.map((line) => line.trim().slice(2));
|
|
|
|
const mustHaves = requirements.filter((r) =>
|
|
MUST_HAVE_KEYWORDS.some((kw) => r.toLowerCase().includes(kw))
|
|
);
|
|
|
|
if (mustHaves.length === 0 && requirements.length === 0) {
|
|
return this.check(
|
|
"Specification requirements traceable",
|
|
"warning",
|
|
"No requirements found in PROJECT.md"
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Specification requirements traceable",
|
|
"pass",
|
|
`Found ${requirements.length} requirement(s), ${mustHaves.length} must-have(s)`
|
|
);
|
|
}
|
|
|
|
private checkPlanMustHaves(projectPath: string, phase: number): VerificationCheck {
|
|
const roadmapPath = path.join(
|
|
projectPath,
|
|
".ci",
|
|
"ROADMAP.md"
|
|
);
|
|
|
|
if (!fs.existsSync(roadmapPath)) {
|
|
return this.check(
|
|
"Plan must-haves covered",
|
|
"skipped",
|
|
"No ROADMAP.md found — run 'ci init' first"
|
|
);
|
|
}
|
|
|
|
const content = fs.readFileSync(roadmapPath, "utf-8");
|
|
const hasMustHaves = content.toLowerCase().includes("must");
|
|
const hasPhases = content.includes("Phase") || content.includes("phase");
|
|
|
|
if (!hasPhases && !hasMustHaves) {
|
|
return this.check(
|
|
"Plan must-haves covered",
|
|
"warning",
|
|
"ROADMAP.md has no phases or must-have items"
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Plan must-haves covered",
|
|
"pass",
|
|
"ROADMAP.md contains phase definitions"
|
|
);
|
|
}
|
|
|
|
private checkRequirementTestCoverage(projectPath: string): VerificationCheck {
|
|
const isGitRepo = fs.existsSync(path.join(projectPath, ".git"));
|
|
if (!isGitRepo) {
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"skipped",
|
|
"Not a git repository — cannot check requirement coverage from commit history"
|
|
);
|
|
}
|
|
|
|
try {
|
|
const raw = execSync(
|
|
`git log --all --max-count=100 --format="%B%x01"`,
|
|
{ cwd: projectPath, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], timeout: 5000 }
|
|
);
|
|
|
|
const coveredReqs = new Set<string>();
|
|
const ciBlockRegex = /---ci---[\s\S]*?---\/ci---/g;
|
|
const entries = raw.split("\x01").filter(Boolean);
|
|
|
|
for (const entry of entries) {
|
|
let match;
|
|
while ((match = ciBlockRegex.exec(entry)) !== null) {
|
|
const reqMatch = match[0].match(/covered:\s*\[([^\]]*)\]/);
|
|
if (reqMatch) {
|
|
const reqs = reqMatch[1].split(",").map((r: string) => r.trim().replace(/['"]/g, "")).filter(Boolean);
|
|
for (const req of reqs) coveredReqs.add(req);
|
|
}
|
|
}
|
|
ciBlockRegex.lastIndex = 0;
|
|
}
|
|
|
|
const reqPath = path.join(projectPath, ".ci", "REQUIREMENTS.md");
|
|
if (!fs.existsSync(reqPath)) {
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"skipped",
|
|
"No REQUIREMENTS.md found to check coverage against"
|
|
);
|
|
}
|
|
|
|
const content = fs.readFileSync(reqPath, "utf-8");
|
|
const allReqs = content
|
|
.split("\n")
|
|
.filter((line) => /^\|.*\|.*\|.*\|/.test(line) && !line.includes("REQ-ID") && !line.includes("---"))
|
|
.map((line) => {
|
|
const cols = line.split("|").map((c) => c.trim()).filter(Boolean);
|
|
return cols.length >= 1 ? cols[0] : "";
|
|
})
|
|
.filter(Boolean);
|
|
|
|
if (allReqs.length === 0) {
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"skipped",
|
|
"No requirements with REQ-IDs found in REQUIREMENTS.md"
|
|
);
|
|
}
|
|
|
|
const covered = allReqs.filter((r) => coveredReqs.has(r));
|
|
const coveragePct = Math.round((covered.length / allReqs.length) * 100);
|
|
|
|
if (coveragePct >= 80) {
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"pass",
|
|
`${covered.length}/${allReqs.length} requirements covered (${coveragePct}%)`
|
|
);
|
|
}
|
|
|
|
if (coveragePct >= 50) {
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"warning",
|
|
`${covered.length}/${allReqs.length} requirements covered (${coveragePct}%) — target ≥80%`
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"warning",
|
|
`${covered.length}/${allReqs.length} requirements covered (${coveragePct}%) — significant gaps`
|
|
);
|
|
} catch {
|
|
return this.check(
|
|
"Requirement test coverage via git log",
|
|
"skipped",
|
|
"Could not read git log for requirement coverage"
|
|
);
|
|
}
|
|
}
|
|
|
|
private checkCodeHasExports(projectPath: string): VerificationCheck {
|
|
const srcDir = path.join(projectPath, "src");
|
|
if (!fs.existsSync(srcDir)) {
|
|
return this.check("Source code has exports", "skipped", "No src/ directory found");
|
|
}
|
|
|
|
const tsFiles = this.collectTsFiles(srcDir);
|
|
const filesWithoutExports: string[] = [];
|
|
|
|
for (const file of tsFiles) {
|
|
const content = fs.readFileSync(file, "utf-8");
|
|
const hasExport = /\bexport\s+/.test(content);
|
|
if (!hasExport && content.trim().length > 0) {
|
|
filesWithoutExports.push(path.relative(projectPath, file));
|
|
}
|
|
}
|
|
|
|
if (filesWithoutExports.length === 0) {
|
|
return this.check(
|
|
"Source code has exports",
|
|
"pass",
|
|
`All ${tsFiles.length} source files have exports`
|
|
);
|
|
}
|
|
|
|
if (filesWithoutExports.length > tsFiles.length * 0.5) {
|
|
return this.check(
|
|
"Source code has exports",
|
|
"warning",
|
|
`${filesWithoutExports.length}/${tsFiles.length} files have no exports`
|
|
);
|
|
}
|
|
|
|
return this.check(
|
|
"Source code has exports",
|
|
"pass",
|
|
`Most files export symbols (${tsFiles.length - filesWithoutExports.length}/${tsFiles.length})`
|
|
);
|
|
}
|
|
|
|
private collectTsFiles(dir: string): string[] {
|
|
const files: string[] = [];
|
|
if (!fs.existsSync(dir)) return files;
|
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(dir, entry.name);
|
|
if (entry.isDirectory() && entry.name !== "node_modules") {
|
|
files.push(...this.collectTsFiles(fullPath));
|
|
} else if (entry.name.endsWith(".ts") && !entry.name.endsWith(".d.ts") && !entry.name.endsWith(".test.ts")) {
|
|
files.push(fullPath);
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
private findTestFiles(dir: string, projectPath: string): string[] {
|
|
const files: string[] = [];
|
|
if (!fs.existsSync(dir)) return files;
|
|
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const fullPath = path.join(dir, entry.name);
|
|
if (entry.isDirectory() && entry.name !== "node_modules") {
|
|
files.push(...this.findTestFiles(fullPath, projectPath));
|
|
} else if (
|
|
entry.name.endsWith(".test.ts") ||
|
|
entry.name.endsWith(".test.js") ||
|
|
entry.name.endsWith(".spec.ts") ||
|
|
entry.name.endsWith(".spec.js")
|
|
) {
|
|
files.push(path.relative(projectPath, fullPath));
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
} |