Jon Chery
d58fd0bdde
---ci---
phase: 3
milestone: v0.10
status: execute
decisions:
- id: D-084
decision: Dual integration: standalone ciagent ideate + --ideate flag on run
confidence: 0.90
requirements:
covered:
- IDEATE-07
- IDEATE-08
- IDEATE-15
---/ci---
- IDEATE-07: External signal collection (npm audit, dependency staleness) tested
- IDEATE-08: Cascade impact analysis (--affected) tested
- IDEATE-15: --ideate flag on ciagent run inserts IDEATE stage between RESEARCH and PLAN
- Tests for runAffected, runExternal, runCrossProject
- 541 tests passing
386 lines
17 KiB
TypeScript
386 lines
17 KiB
TypeScript
import * as fs from "node:fs";
|
|
import * as path from "node:path";
|
|
import * as os from "node:os";
|
|
import { IdeationAgent } from "../agents/ideation-agent.js";
|
|
import { IdeationEngine, resetIdeaCounter } from "../core/ideation.js";
|
|
import { Idea, IdeationAction, DEFAULT_IDEATION_CONFIG } from "../types/ideation.js";
|
|
|
|
describe("IdeationAgent", () => {
|
|
let tempDir: string;
|
|
|
|
beforeEach(() => {
|
|
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "ciagent-ideation-test-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("agent name is ideation-agent", () => {
|
|
const agent = new IdeationAgent();
|
|
expect(agent.name).toBe("ideation-agent");
|
|
});
|
|
|
|
it("delegates to IdeationEngine for mechanical ideation", () => {
|
|
const agent = new IdeationAgent();
|
|
const ideas = agent.mechanicalIdeate(tempDir);
|
|
expect(Array.isArray(ideas)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("IdeationEngine", () => {
|
|
let tempDir: string;
|
|
|
|
beforeEach(() => {
|
|
resetIdeaCounter();
|
|
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "ciagent-engine-test-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("generates ideas from uncovered requirements", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "REQUIREMENTS.md"),
|
|
"# Requirements\n\n## v1 Requirements\n\n### Core\n\n- **REQ-01**: First requirement\n- **REQ-02**: Second requirement\n\n## Traceability\n\n| Requirement | Phase | Status |\n|-------------|-------|--------|\n| REQ-01 | Phase 1 | pending |\n| REQ-02 | Phase 1 | pending |\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "PROJECT.md"),
|
|
"# Test Project\n\n## What This Is\n\nA test project.\n\n## Requirements\n\n### Validated\n\n- REQ-01: First\n\n### Active\n\n- [ ] REQ-02: Second\n\n## Constraints\n\n- Must work\n\n## Key Decisions\n\n| Decision | Rationale | Outcome |\n|----------|-----------|--------|\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ROADMAP.md"),
|
|
"# Roadmap\n\n## Overview\n\nTest roadmap.\n\n## Phases\n\n- [ ] **Phase 1: Init** - Starting\n\n## Phase Details\n\n### Phase 1: Init\n\n**Goal.**: Start\n**Status**: not_started\n**Requirements**: REQ-01\n**Depends on**: Nothing\n**Success Criteria**:\n1. Project initialized\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ARCHITECTURE.md"),
|
|
"# Architecture\n\n## Overview\n\nTest architecture.\n\n## Components\n\n### Core\n\n- **Description**: Core module\n- **Boundaries**: Internal only\n- **Depends on**: None\n\n## Data Flow\n\nSimple flow.\n\n## Build Order\n\n1. Core\n"
|
|
);
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const ideas = engine.runMechanical(["coverage"]);
|
|
const reqIdeas = ideas.filter((i) => i.source === "uncovered_requirement");
|
|
expect(reqIdeas.length).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
it("detects architecture drift when documented components are missing", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ARCHITECTURE.md"),
|
|
"# Architecture\n\n## Overview\n\nTest.\n\n## Components\n\n### NonExistentModule\n\n- **Description**: A module that does not exist\n- **Boundaries**: None\n- **Depends on**: None\n\n## Data Flow\n\nFlow.\n\n## Build Order\n\n1. Core\n"
|
|
);
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const ideas = engine.runMechanical(["architecture"]);
|
|
const driftIdeas = ideas.filter((i) => i.source === "architecture_drift");
|
|
expect(driftIdeas.length).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
it("detects spec ambiguity or spec missing", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "PROJECT.md"),
|
|
"# Test\n\n## What This Is\n\nThe system should handle user input and could process data. It might also log events.\n\n## Requirements\n\n### Validated\n\n\n### Active\n\n- [ ] The system should handle errors\n- [ ] Users could configure settings\n- [ ] It might send notifications\n\n## Constraints\n\n- Must work\n\n## Key Decisions\n\n| Decision | Rationale | Outcome |\n|----------|-----------|--------|\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "REQUIREMENTS.md"),
|
|
"# Requirements\n\n## v1\n\n- REQ-01: Test\n\n## Traceability\n\n| Requirement | Phase | Status |\n|-------------|-------|--------|\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ROADMAP.md"),
|
|
"# Roadmap\n\n## Overview\n\nTest\n\n## Phases\n\n\n## Phase Details\n\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ARCHITECTURE.md"),
|
|
"# Architecture\n\n## Overview\n\nTest\n\n## Components\n\n## Data Flow\n\nTest\n\n## Build Order\n\n1. Test\n"
|
|
);
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const ideas = engine.runMechanical(["spec"]);
|
|
const specIdeas = ideas.filter((i) => i.source === "spec_ambiguity" || i.source === "spec_missing" || i.source === "spec_contradiction");
|
|
expect(specIdeas.length).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
it("returns empty ideas when no project files exist", () => {
|
|
const engine = new IdeationEngine(tempDir);
|
|
const ideas = engine.runMechanical();
|
|
expect(Array.isArray(ideas)).toBe(true);
|
|
});
|
|
|
|
it("formats ideas as readable text", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(path.join(ciagentDir, "PROJECT.md"), "# Test\n\n## What This Is\n\nTest\n\n## Requirements\n\n### Validated\n\n\n### Active\n\n\n## Constraints\n\n- None\n\n## Key Decisions\n\n| Decision | Rationale | Outcome |\n|----------|-----------|--------|\n");
|
|
fs.writeFileSync(path.join(ciagentDir, "REQUIREMENTS.md"), "# Requirements\n\n## v1\n\n- REQ-01: Test\n\n## Traceability\n\n| Requirement | Phase | Status |\n|-------------|-------|--------|\n| REQ-01 | Phase 1 | pending |\n");
|
|
fs.writeFileSync(path.join(ciagentDir, "ROADMAP.md"), "# Roadmap\n\n## Overview\n\nTest\n\n## Phases\n\n\n## Phase Details\n\n");
|
|
fs.writeFileSync(path.join(ciagentDir, "ARCHITECTURE.md"), "# Architecture\n\n## Overview\n\nTest\n\n## Components\n\n## Data Flow\n\nTest\n\n## Build Order\n\n1. Test\n");
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const formatted = engine.formatIdeas(engine.runMechanical());
|
|
expect(typeof formatted).toBe("string");
|
|
});
|
|
|
|
it("formats ideas as JSON", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(path.join(ciagentDir, "PROJECT.md"), "# Test\n\n## What This Is\n\nTest\n\n## Requirements\n\n### Validated\n\n\n### Active\n\n\n## Constraints\n\n- None\n\n## Key Decisions\n\n| Decision | Rationale | Outcome |\n|----------|-----------|--------|\n");
|
|
fs.writeFileSync(path.join(ciagentDir, "REQUIREMENTS.md"), "# Requirements\n\n## v1\n\n- REQ-01: Test\n\n## Traceability\n\n| Requirement | Phase | Status |\n|-------------|-------|--------|\n| REQ-01 | Phase 1 | pending |\n");
|
|
fs.writeFileSync(path.join(ciagentDir, "ROADMAP.md"), "# Roadmap\n\n## Overview\n\nTest\n\n## Phases\n\n\n## Phase Details\n\n");
|
|
fs.writeFileSync(path.join(ciagentDir, "ARCHITECTURE.md"), "# Architecture\n\n## Overview\n\nTest\n\n## Components\n\n## Data Flow\n\nTest\n\n## Build Order\n\n1. Test\n");
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const result = engine.formatIdeasJson(engine.runMechanical());
|
|
expect(result).toHaveProperty("ideas");
|
|
expect(result).toHaveProperty("summary");
|
|
expect(result).toHaveProperty("project");
|
|
});
|
|
|
|
describe("acceptIdea", () => {
|
|
let acceptDir: string;
|
|
|
|
beforeEach(() => {
|
|
resetIdeaCounter();
|
|
acceptDir = fs.mkdtempSync(path.join(os.tmpdir(), "ciagent-accept-test-"));
|
|
const ciagentDir = path.join(acceptDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "REQUIREMENTS.md"),
|
|
"# Requirements\n\n## v1 Requirements\n\n### Core\n\n- **CORE-01**: Test core requirement\n\n## Traceability\n\n| Requirement | Phase | Status |\n|-------------|-------|--------|\n| CORE-01 | Phase 1 | pending |\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ROADMAP.md"),
|
|
"# Roadmap\n\n## Overview\n\nTest roadmap.\n\n## Phases\n\n- [x] **Phase 1: Init** - Starting\n\n## Phase Details\n\n### Phase 1: Init\n\n**Goal.**: Start\n**Status**: complete\n**Requirements**: CORE-01\n**Depends on**: Nothing\n**Success Criteria**:\n1. Project initialized\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "PROJECT.md"),
|
|
"# Test\n\n## What This Is\n\nTest\n\n## Requirements\n\n### Validated\n\n\n### Active\n\n\n## Constraints\n\n- None\n\n## Key Decisions\n\n| Decision | Rationale | Outcome |\n|----------|-----------|--------|\n"
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ARCHITECTURE.md"),
|
|
"# Architecture\n\n## Overview\n\nTest\n\n## Components\n\n## Data Flow\n\nTest\n\n## Build Order\n\n1. Test\n"
|
|
);
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(acceptDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("accepts an idea and updates REQUIREMENTS.md and ROADMAP.md", () => {
|
|
const engine = new IdeationEngine(acceptDir);
|
|
const idea: Idea = {
|
|
id: "IDEATE-01",
|
|
source: "uncovered_requirement",
|
|
category: "coverage",
|
|
title: "Add rate limiting to cloud backends",
|
|
rationale: "No rate limiting REQ exists for cloud backends.",
|
|
confidence: 0.92,
|
|
actions: ["add_requirement", "update_roadmap"],
|
|
tier: "mechanical",
|
|
};
|
|
|
|
const result = engine.acceptIdea(idea);
|
|
|
|
expect(result.addedToRequirements).toBe(true);
|
|
expect(result.addedToRoadmap).toBe(true);
|
|
expect(result.reqId).toBe("IDEATE-01");
|
|
});
|
|
|
|
it("acceptIdeas accepts multiple ideas", () => {
|
|
const engine = new IdeationEngine(acceptDir);
|
|
const ideas: Idea[] = [
|
|
{
|
|
id: "IDEATE-01",
|
|
source: "uncovered_requirement",
|
|
category: "coverage",
|
|
title: "Add rate limiting",
|
|
rationale: "No rate limiting.",
|
|
confidence: 0.9,
|
|
actions: ["add_requirement"],
|
|
tier: "mechanical",
|
|
},
|
|
{
|
|
id: "IDEATE-02",
|
|
source: "architecture_drift",
|
|
category: "architecture",
|
|
title: "Fix architecture drift",
|
|
rationale: "Component documented but missing.",
|
|
confidence: 0.8,
|
|
actions: ["update_architecture"],
|
|
tier: "mechanical",
|
|
},
|
|
];
|
|
|
|
const { accepted, results } = engine.acceptIdeas(ideas);
|
|
expect(accepted.length).toBe(2);
|
|
expect(results.length).toBe(2);
|
|
expect(results.every((r) => r.addedToRequirements || r.addedToRoadmap)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("Phase 2: Backend-enriched and chaos", () => {
|
|
let tempDir: string;
|
|
|
|
beforeEach(() => {
|
|
resetIdeaCounter();
|
|
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "ciagent-p2-test-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("runBackendEnriched prioritizes mechanical findings", () => {
|
|
const engine = new IdeationEngine(tempDir);
|
|
const mechanicalIdeas: Idea[] = [
|
|
{
|
|
id: "IDEATE-01",
|
|
source: "uncovered_requirement",
|
|
category: "coverage",
|
|
title: "Missing test",
|
|
rationale: "No test file",
|
|
confidence: 0.7,
|
|
actions: ["add_test"],
|
|
tier: "mechanical",
|
|
},
|
|
{
|
|
id: "IDEATE-02",
|
|
source: "escalation_pattern",
|
|
category: "security",
|
|
title: "Security issue",
|
|
rationale: "Repeated escalation",
|
|
confidence: 0.8,
|
|
actions: ["add_security_pattern"],
|
|
tier: "mechanical",
|
|
},
|
|
];
|
|
|
|
const enriched = engine.runBackendEnriched(mechanicalIdeas);
|
|
expect(enriched.length).toBeGreaterThanOrEqual(2);
|
|
const prioritizedIdeas = enriched.filter((i) => i.source === "uncovered_requirement" || i.source === "escalation_pattern");
|
|
expect(prioritizedIdeas.length).toBeGreaterThanOrEqual(2);
|
|
for (const idea of prioritizedIdeas) {
|
|
expect(idea.tier).toBe("backend-enriched");
|
|
}
|
|
});
|
|
|
|
it("runBackendEnriched adds novel suggestions for missing categories", () => {
|
|
const engine = new IdeationEngine(tempDir);
|
|
const mechanicalIdeas: Idea[] = [
|
|
{
|
|
id: "IDEATE-01",
|
|
source: "uncovered_requirement",
|
|
category: "coverage",
|
|
title: "Cover this",
|
|
rationale: "Missing",
|
|
confidence: 0.7,
|
|
actions: ["add_test"],
|
|
tier: "mechanical",
|
|
},
|
|
];
|
|
|
|
const enriched = engine.runBackendEnriched(mechanicalIdeas);
|
|
const novelIdeas = enriched.filter((i) => i.source === "improvement_pattern");
|
|
expect(novelIdeas.length).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
it("generateChaosScenarios uses default scenarios when enabled", () => {
|
|
const engine = new IdeationEngine(tempDir);
|
|
const chaosIdeas = engine.generateChaosScenarios();
|
|
|
|
expect(chaosIdeas.length).toBe(3);
|
|
expect(chaosIdeas.every((i) => i.source === "chaos_scenario")).toBe(true);
|
|
expect(chaosIdeas.every((i) => i.category === "chaos")).toBe(true);
|
|
expect(chaosIdeas.every((i) => i.tier === "backend-enriched")).toBe(true);
|
|
expect(chaosIdeas.every((i) => i.confidence >= 0.5)).toBe(true);
|
|
const titles = chaosIdeas.map((i) => i.title);
|
|
expect(titles.some((t) => t.includes("backend"))).toBe(true);
|
|
expect(titles.some((t) => t.includes("requirement"))).toBe(true);
|
|
expect(titles.some((t) => t.includes("coverage"))).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe("Phase 3: External signals and cascade impact", () => {
|
|
let tempDir: string;
|
|
|
|
beforeEach(() => {
|
|
resetIdeaCounter();
|
|
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "ciagent-p3-test-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(tempDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("runAffected detects cascade from architecture.md", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "ARCHITECTURE.md"),
|
|
"# Architecture\n\n## Overview\n\nTest.\n\n## Components\n\n### CLI\n\n- **Description**: Command line interface\n- **Boundaries**: User-facing only\n- **Depends on**: Core\n\n### Core\n\n- **Description**: Core engine\n- **Boundaries**: Internal only\n- **Depends on**: None\n\n## Data Flow\n\nSimple.\n\n## Build Order\n\n1. CLI\n2. Core\n"
|
|
);
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const ideas = engine.runAffected();
|
|
expect(Array.isArray(ideas)).toBe(true);
|
|
});
|
|
|
|
it("runExternal handles missing npm gracefully", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [], active_project: "default" })
|
|
);
|
|
|
|
const engine = new IdeationEngine(tempDir);
|
|
const ideas = engine.runExternal();
|
|
expect(Array.isArray(ideas)).toBe(true);
|
|
});
|
|
|
|
it("runCrossProject returns empty when only one project", () => {
|
|
const ciagentDir = path.join(tempDir, ".ciagent");
|
|
fs.mkdirSync(ciagentDir, { recursive: true });
|
|
fs.writeFileSync(
|
|
path.join(ciagentDir, "config.json"),
|
|
JSON.stringify({ projects: [{ slug: "default", name: "Default Project", default: true }], active_project: "default" })
|
|
);
|
|
|
|
const engine = new IdeationEngine(tempDir, "default");
|
|
const ideas = engine.runCrossProject();
|
|
expect(ideas).toEqual([]);
|
|
});
|
|
});
|
|
}); |