feat(P11,P12): orca job lint (REQ-084) + orca job verify (dry-run txn)

P11: orca job lint <spec.md> — schema/CEL/body/migration/best-practice
checks; --explain, --format json; exit 0/1 by errors found.
P12: orca job verify <spec.md> — dry-run txn (render + stage + verify
without apply); reports planned allocs/files/units; no side effects;
--namespace, --json.

---ci---
project: orca
phase: 11
milestone: v0.11
status: execute
---/ci---
This commit is contained in:
Jon Chery
2026-08-07 07:29:44 +00:00
parent 03f3585f16
commit 020aa01623
5 changed files with 1382 additions and 1 deletions
+424
View File
@@ -0,0 +1,424 @@
// Package cli: job_lint.go implements `orca job lint` (P11, REQ-084,
// v0.11 milestone). It validates a jobspec (.md/.yaml/.yml/.hcl) with:
//
// - schema validation (kind, blocks, frontmatter fields) via
// internal/spec/schema
// - CEL constraint syntax check (basic -- balanced parens/quotes,
// presence of operators; no CEL engine dependency in this phase)
// - body preservation (R-015): warn when body is empty for .md specs
// - migration: flag deprecated .hcl specs with a warning suggesting
// conversion to .md (REQ-090)
// - best-practice: warn on missing health checks for Services,
// missing restart policies, etc.
//
// Output: one finding per line (category | severity | line | message).
// --explain prints the rationale for each finding. --format text (the
// default) or json. Exit 0 = no errors (warnings OK); 1 = errors found.
package cli
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"github.com/spf13/cobra"
"git.cloudinit.dev/coreci/orca/internal/jobspec"
"git.cloudinit.dev/coreci/orca/internal/spec/schema"
)
var (
jobLintExplain bool
jobLintFormat string
)
type lintSeverity string
const (
severityError lintSeverity = "error"
severityWarning lintSeverity = "warning"
severityInfo lintSeverity = "info"
)
type lintCategory string
const (
catSchema lintCategory = "schema"
catCEL lintCategory = "cel"
catBody lintCategory = "body"
catMigration lintCategory = "migration"
catBestPractice lintCategory = "best-practice"
)
type lintFinding struct {
Category lintCategory `json:"category"`
Severity lintSeverity `json:"severity"`
Line int `json:"line"`
Message string `json:"message"`
}
var rationale = map[lintCategory]string{
catSchema: "Schema violations prevent the spec from being parsed or rendered; fix these first.",
catCEL: "CEL constraints gate placement; a syntactically invalid expression is rejected by the scheduler (REQ-083).",
catBody: "R-015 requires the markdown body to be preserved byte-exact; an empty body is allowed but loses operator documentation.",
catMigration: "HCL specs are legacy (REQ-090); convert to Markdown+frontmatter before v1.0 to keep schema validation working.",
catBestPractice: "Best-practice warnings do not block apply, but address them to keep the fleet observable and restartable.",
}
type lintExitError struct {
findings []lintFinding
}
func (e *lintExitError) Error() string {
return fmt.Sprintf("lint: %d error(s) found", countErrors(e.findings))
}
func countErrors(findings []lintFinding) int {
n := 0
for _, f := range findings {
if f.Severity == severityError {
n++
}
}
return n
}
var jobLintCmd = &cobra.Command{
Use: "lint <spec>",
Short: "Lint a jobspec (schema, CEL, body, migration, best-practice)",
Long: `Validate a jobspec (.md/.yaml/.yml/.hcl) without applying it.
Checks (REQ-084):
- schema: kind (Job/Service/DaemonSet), required blocks, frontmatter
- CEL: constraint expressions are syntactically valid
- body: markdown body present and non-empty (R-015)
- migration: flag deprecated .hcl specs (suggest .md)
- best-practice: warn on missing health/restart/update for the kind
Flags:
--explain print the rationale for each finding
--format text|json output format (default text)
Exit codes: 0 = no errors (warnings OK), 1 = errors found.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
findings, err := runJobLint(args[0])
if err != nil {
var le *lintExitError
if errors.As(err, &le) {
if jobLintFormat == "json" {
_ = printLintJSON(findings)
} else {
printLintText(findings, jobLintExplain)
}
return err
}
return err
}
if jobLintFormat == "json" {
return printLintJSON(findings)
}
printLintText(findings, jobLintExplain)
return nil
},
}
func runJobLint(path string) ([]lintFinding, error) {
var findings []lintFinding
data, err := os.ReadFile(path)
if err != nil {
findings = append(findings, lintFinding{
Category: catSchema,
Severity: severityError,
Line: 0,
Message: fmt.Sprintf("cannot read spec file: %v", err),
})
return findings, &lintExitError{findings}
}
ext := strings.ToLower(filepath.Ext(path))
if ext == ".hcl" {
findings = append(findings, lintFinding{
Category: catMigration,
Severity: severityWarning,
Line: 0,
Message: fmt.Sprintf("%s is a legacy HCL spec; convert to Markdown (.md) before v1.0 (REQ-090)", filepath.Base(path)),
})
}
spec, perr := jobspec.Dispatch(data, filepath.Base(path))
if perr != nil {
findings = append(findings, lintFinding{
Category: catSchema,
Severity: severityError,
Line: 0,
Message: fmt.Sprintf("parse: %v", perr),
})
return findings, &lintExitError{findings}
}
findings = append(findings, lintSchema(spec)...)
findings = append(findings, lintCEL(spec)...)
findings = append(findings, lintBody(spec, ext)...)
findings = append(findings, lintBestPractice(spec)...)
sortLint(findings)
if countErrors(findings) > 0 {
return findings, &lintExitError{findings}
}
return findings, nil
}
func lintSchema(spec *jobspec.WorkloadSpec) []lintFinding {
if spec == nil {
return nil
}
v, err := schema.ValidatorFor(spec.Kind)
if err != nil {
return []lintFinding{{
Category: catSchema,
Severity: severityError,
Line: 0,
Message: err.Error(),
}}
}
verr := v.Validate(spec)
if verr == nil {
return nil
}
msg := verr.Error()
parts := strings.Split(msg, "; ")
var out []lintFinding
for _, p := range parts {
p = strings.TrimSpace(p)
if p == "" {
continue
}
out = append(out, lintFinding{
Category: catSchema,
Severity: severityError,
Line: 0,
Message: trimValidatorPrefix(p),
})
}
return out
}
func trimValidatorPrefix(s string) string {
if strings.HasPrefix(s, "schema/") {
if i := strings.Index(s, ": "); i >= 0 {
return strings.TrimSpace(s[i+2:])
}
}
return s
}
func lintCEL(spec *jobspec.WorkloadSpec) []lintFinding {
if spec == nil {
return nil
}
var out []lintFinding
for i, c := range spec.Constraints {
if msg := basicCELCheck(c); msg != "" {
out = append(out, lintFinding{
Category: catCEL,
Severity: severityError,
Line: 0,
Message: fmt.Sprintf("constraints[%d]: %s", i, msg),
})
}
}
for i, a := range spec.Affinity {
if msg := basicCELCheck(a.Target); msg != "" {
out = append(out, lintFinding{
Category: catCEL,
Severity: severityError,
Line: 0,
Message: fmt.Sprintf("affinity[%d].target: %s", i, msg),
})
}
}
return out
}
func basicCELCheck(expr string) string {
expr = strings.TrimSpace(expr)
if expr == "" {
return "empty CEL expression"
}
parens := 0
inSingle := false
inDouble := false
for i := 0; i < len(expr); i++ {
c := expr[i]
switch c {
case '\'':
if !inDouble {
inSingle = !inSingle
}
case '"':
if !inSingle {
inDouble = !inDouble
}
case '(':
if !inSingle && !inDouble {
parens++
}
case ')':
if !inSingle && !inDouble {
parens--
if parens < 0 {
return "unbalanced parentheses: ')' before '('"
}
}
}
}
if inSingle || inDouble {
return "unbalanced quotes"
}
if parens != 0 {
return fmt.Sprintf("unbalanced parentheses: %d unclosed '('", parens)
}
return ""
}
func lintBody(spec *jobspec.WorkloadSpec, ext string) []lintFinding {
if spec == nil {
return nil
}
if ext != ".md" {
return nil
}
if strings.TrimSpace(spec.Body) == "" {
return []lintFinding{{
Category: catBody,
Severity: severityWarning,
Line: 0,
Message: "markdown body is empty (R-015: body preserved byte-exact; add operator documentation)",
}}
}
return nil
}
func lintBestPractice(spec *jobspec.WorkloadSpec) []lintFinding {
if spec == nil {
return nil
}
var out []lintFinding
switch spec.Kind {
case "Service":
if spec.Health == nil {
out = append(out, lintFinding{
Category: catBestPractice,
Severity: severityWarning,
Line: 0,
Message: "Service without a health block: Traefik routing depends on health checks (R-012)",
})
}
if spec.Restart == nil {
out = append(out, lintFinding{
Category: catBestPractice,
Severity: severityWarning,
Line: 0,
Message: "Service without a restart policy: defaults to 'service' but an explicit policy is recommended",
})
}
if spec.Update == nil {
out = append(out, lintFinding{
Category: catBestPractice,
Severity: severityWarning,
Line: 0,
Message: "Service without an update block: rolling/canary strategy should be explicit",
})
}
case "DaemonSet":
if spec.Restart == nil {
out = append(out, lintFinding{
Category: catBestPractice,
Severity: severityWarning,
Line: 0,
Message: "DaemonSet without a restart policy: a long-running daemon should declare its restart mode",
})
}
case "Job":
if spec.Restart == nil {
out = append(out, lintFinding{
Category: catBestPractice,
Severity: severityInfo,
Line: 0,
Message: "Job without a restart policy: defaults to 'never' (one-shot); set explicitly if retry is desired",
})
}
}
return out
}
func sortLint(f []lintFinding) {
sort.SliceStable(f, func(i, j int) bool {
si := severityRank(f[i].Severity)
sj := severityRank(f[j].Severity)
if si != sj {
return si < sj
}
if f[i].Category != f[j].Category {
return string(f[i].Category) < string(f[j].Category)
}
return f[i].Line < f[j].Line
})
}
func severityRank(s lintSeverity) int {
switch s {
case severityError:
return 0
case severityWarning:
return 1
case severityInfo:
return 2
}
return 3
}
func printLintText(findings []lintFinding, explain bool) {
w := rootCmd.OutOrStdout()
errs := countErrors(findings)
warns := 0
infos := 0
for _, f := range findings {
switch f.Severity {
case severityWarning:
warns++
case severityInfo:
infos++
}
line := fmt.Sprintf("%-14s %-8s %s", f.Category, f.Severity, f.Message)
if f.Line > 0 {
line = fmt.Sprintf("%-14s %-8s line %d: %s", f.Category, f.Severity, f.Line, f.Message)
}
fmt.Fprintln(w, line)
if explain {
fmt.Fprintf(w, " -> %s\n", rationale[f.Category])
}
}
fmt.Fprintf(w, "\n%d error(s), %d warning(s), %d info\n", errs, warns, infos)
}
func printLintJSON(findings []lintFinding) error {
out := make([]lintFinding, len(findings))
copy(out, findings)
w := rootCmd.OutOrStdout()
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
return enc.Encode(out)
}
func init() {
jobLintCmd.Flags().BoolVar(&jobLintExplain, "explain", false, "print the rationale for each finding")
jobLintCmd.Flags().StringVar(&jobLintFormat, "format", "text", "output format: text or json")
jobCmd.AddCommand(jobLintCmd)
}
+310
View File
@@ -0,0 +1,310 @@
package cli
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
func writeMDSpec(t *testing.T, content string) string {
t.Helper()
dir := t.TempDir()
p := filepath.Join(dir, "spec.md")
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
return p
}
func writeHCLSpec(t *testing.T, content string) string {
t.Helper()
dir := t.TempDir()
p := filepath.Join(dir, "spec.hcl")
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatalf("write spec: %v", err)
}
return p
}
const validJobMD = "---\n" +
"kind: Job\n" +
"name: my-job\n" +
"runtime:\n" +
" one_of: process\n" +
" command: /bin/true\n" +
"---\n" +
"# My Job\n\nRuns /bin/true.\n"
const validServiceMD = "---\n" +
"kind: Service\n" +
"name: web\n" +
"count: 3\n" +
"runtime:\n" +
" one_of: process\n" +
" command: /bin/http\n" +
"ports:\n" +
" - name: http\n" +
" port: 8080\n" +
"restart:\n" +
" mode: service\n" +
"update:\n" +
" strategy: rolling\n" +
" max_surge: 1\n" +
"health:\n" +
" check_type: http\n" +
" interval: 5s\n" +
"---\n" +
"# Web service\n\nServes HTTP.\n"
const invalidKindMD = "---\n" +
"kind: CronJob\n" +
"name: bad\n" +
"---\nbody\n"
const serviceMissingHealthMD = "---\n" +
"kind: Service\n" +
"name: web\n" +
"count: 2\n" +
"runtime:\n" +
" one_of: process\n" +
" command: /bin/http\n" +
"ports:\n" +
" - name: http\n" +
" port: 8080\n" +
"restart:\n" +
" mode: service\n" +
"update:\n" +
" strategy: rolling\n" +
"---\n" +
"# web\n\nbody\n"
const serviceMissingPortsMD = "---\n" +
"kind: Service\n" +
"name: web\n" +
"runtime:\n" +
" one_of: process\n" +
" command: /bin/http\n" +
"restart:\n" +
" mode: service\n" +
"update:\n" +
" strategy: rolling\n" +
"health:\n" +
" check_type: http\n" +
"---\nbody\n"
const emptyBodyMD = "---\n" +
"kind: Job\n" +
"name: emptybody\n" +
"runtime:\n" +
" command: /bin/true\n" +
"---\n"
const badCELMD = "---\n" +
"kind: Job\n" +
"name: badcel\n" +
"runtime:\n" +
" command: /bin/true\n" +
"constraints:\n" +
" - 'node.role == \"web\"'\n" +
" - 'region == (\"us\"'\n" +
"---\nbody\n"
func TestJobLintCmdRegistered(t *testing.T) {
for _, c := range jobCmd.Commands() {
if c.Name() == "lint" {
return
}
}
t.Fatal("job lint command not registered on jobCmd")
}
func TestJobLintValidSpec(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job lint valid: %v", err)
}
out := buf.String()
if !strings.Contains(out, "0 error(s)") {
t.Errorf("expected 0 errors, got: %s", out)
}
}
func TestJobLintValidServiceSpec(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, validServiceMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job lint valid service: %v", err)
}
out := buf.String()
if !strings.Contains(out, "0 error(s)") {
t.Errorf("expected 0 errors, got: %s", out)
}
}
func TestJobLintInvalidKind(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, invalidKindMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for invalid kind, got nil")
}
}
func TestJobLintInvalidKindReportsError(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, invalidKindMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
_ = rootCmd.Execute()
out := buf.String()
if !strings.Contains(strings.ToLower(out), "cronjob") && !strings.Contains(strings.ToLower(out), "kind") {
t.Errorf("expected error about kind, got: %s", out)
}
}
func TestJobLintMissingRequiredField(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, serviceMissingPortsMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for missing ports, got nil")
}
out := buf.String()
if !strings.Contains(strings.ToLower(out), "port") {
t.Errorf("expected error mentioning ports, got: %s", out)
}
}
func TestJobLintDeprecatedHCLWarning(t *testing.T) {
resetRootFlags(t)
spec := writeHCLSpec(t, trueJobSpec)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job lint hcl: %v", err)
}
out := buf.String()
if !strings.Contains(out, "migration") && !strings.Contains(strings.ToLower(out), "legacy") && !strings.Contains(strings.ToLower(out), "hcl") {
t.Errorf("expected migration/legacy warning, got: %s", out)
}
}
func TestJobLintExplain(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, serviceMissingHealthMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec, "--explain"})
_ = rootCmd.Execute()
out := buf.String()
if !strings.Contains(out, "->") {
t.Errorf("expected rationale lines with '->', got: %s", out)
}
}
func TestJobLintFormatJSON(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, serviceMissingHealthMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec, "--format", "json"})
_ = rootCmd.Execute()
out := buf.String()
var findings []lintFinding
if err := json.Unmarshal(bytes.TrimSpace([]byte(out)), &findings); err != nil {
t.Fatalf("unmarshal json findings: %v\n%s", err, out)
}
if len(findings) == 0 {
t.Fatalf("expected at least one finding")
}
foundHealth := false
for _, f := range findings {
if strings.Contains(strings.ToLower(f.Message), "health") {
foundHealth = true
}
}
if !foundHealth {
t.Errorf("expected a health-related finding, got: %+v", findings)
}
}
func TestJobLintMissingHealthCheckWarning(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, serviceMissingHealthMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
_ = rootCmd.Execute()
out := buf.String()
if !strings.Contains(strings.ToLower(out), "health") {
t.Errorf("expected health-related warning, got: %s", out)
}
}
func TestJobLintEmptyBodyWarning(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, emptyBodyMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job lint empty body (warnings only): %v", err)
}
out := buf.String()
if !strings.Contains(strings.ToLower(out), "body") {
t.Errorf("expected body warning, got: %s", out)
}
}
func TestJobLintBadCEL(t *testing.T) {
resetRootFlags(t)
spec := writeMDSpec(t, badCELMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", spec})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for unbalanced CEL, got nil")
}
out := buf.String()
if !strings.Contains(strings.ToLower(out), "cel") && !strings.Contains(strings.ToLower(out), "parenthes") {
t.Errorf("expected CEL/parentheses error, got: %s", out)
}
}
func TestJobLintMissingFile(t *testing.T) {
resetRootFlags(t)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "lint", "/nonexistent/spec.md"})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for missing file, got nil")
}
}
+396
View File
@@ -0,0 +1,396 @@
// Package cli: job_verify.go implements `orca job verify` (P12,
// v0.11 milestone). It performs a dry-run transaction through the lead:
// parse the jobspec, render the emitter plan (allocs/files/units),
// render a txn bundle with the desired state, stage it on the lead
// (idempotent; NO apply), run verify.sh on the lead to capture what
// WOULD be applied, and report the plan. Pre-flight drift (R-020) is
// reported but does not fail a dry-run.
//
// There are no side effects beyond the staged bundle files in
// /run/orca/txns/<txn-id>/ on the lead (idempotent; no .applied marker
// is written, so orca-pull.sh never picks it up).
package cli
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"strings"
"github.com/spf13/cobra"
"git.cloudinit.dev/coreci/orca/internal/emitter"
"git.cloudinit.dev/coreci/orca/internal/jobspec"
"git.cloudinit.dev/coreci/orca/internal/paths"
"git.cloudinit.dev/coreci/orca/internal/secrets"
"git.cloudinit.dev/coreci/orca/internal/spec/schema"
"git.cloudinit.dev/coreci/orca/internal/txn"
)
var (
jobVerifyLead string
jobVerifyNamespace string
jobVerifyJSON bool
)
// jobVerifyTransport is the SSH-push surface `job verify` needs. It
// is satisfied by *sshpush.Transport; tests substitute a mock (same
// pattern as txn.go / drain.go).
type jobVerifyTransport interface {
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
}
// jobVerifyTransportOverride is the package-level seam. When non-nil
// it replaces the production transport; tests set it and restore nil.
var jobVerifyTransportOverride jobVerifyTransport
func jobVerifyTransportFromCtx() (jobVerifyTransport, error) {
if jobVerifyTransportOverride != nil {
return jobVerifyTransportOverride, nil
}
return txnTransportFromCtx()
}
// verifyReport is the structured result of `orca job verify`. It is
// rendered to JSON when --json is set, or as a human-readable summary
// otherwise.
type verifyReport struct {
TxnID string `json:"txn_id"`
Kind string `json:"kind"`
Name string `json:"name"`
Namespace string `json:"namespace,omitempty"`
Lead string `json:"lead"`
PlannedAllocs []plannedAlloc `json:"planned_allocs"`
PlannedFiles []plannedFile `json:"planned_files"`
VerifyOutput string `json:"verify_output,omitempty"`
Drift []string `json:"drift,omitempty"`
Status string `json:"status"`
}
type plannedAlloc struct {
Name string `json:"name"`
Kind string `json:"kind"`
Count int `json:"count"`
}
type plannedFile struct {
Path string `json:"path"`
Mode string `json:"mode"`
Kind string `json:"kind"`
}
var jobVerifyCmd = &cobra.Command{
Use: "verify <spec>",
Short: "Dry-run a jobspec through the lead (no apply)",
Long: `Dry-run a jobspec as a transaction through the lead peer.
Steps (P12):
1. Parse the jobspec
2. Render the emitter plan (allocs, config files, systemd units)
3. Render a txn bundle (RenderBundle) with the desired state
4. Stage the bundle on the lead (NO apply; idempotent)
5. Run verify.sh on the lead to capture what WOULD be applied
6. Report planned allocs / files / units
Pre-flight drift (R-020) is reported but does not fail a dry-run.
There are no side effects beyond the staged bundle files in
/run/orca/txns/<txn-id>/ on the lead (no .applied marker is written).
Flags:
--lead <peer> lead peer address (host:port) (required)
--namespace <ns> namespace scope
--json JSON output
Exit codes: 0 = verify passed (no issues), 1 = verification failed.`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
report, err := runJobVerify(cmd.Context(), args[0])
if err != nil {
if jobVerifyJSON {
if report != nil {
_ = printVerifyJSON(report)
}
} else if report != nil {
printVerifyText(report)
}
return err
}
if jobVerifyJSON {
return printVerifyJSON(report)
}
printVerifyText(report)
return nil
},
}
// runJobVerify performs the dry-run. It returns the report and an
// error: when err is non-nil the report may still be populated with
// partial results (e.g. drift was detected but the verify step ran).
// The caller renders the report then surfaces the error.
func runJobVerify(ctx context.Context, path string) (*verifyReport, error) {
if jobVerifyLead == "" {
return nil, fmt.Errorf("--lead is required for job verify")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read spec file: %w", err)
}
spec, err := jobspec.Dispatch(data, fileBase(path))
if err != nil {
return nil, fmt.Errorf("parse spec: %w", err)
}
if verr := validateForVerify(spec); verr != nil {
return nil, verr
}
files, err := renderPlan(spec)
if err != nil {
return nil, fmt.Errorf("render plan: %w", err)
}
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
if err != nil {
return nil, fmt.Errorf("load master key: %w", err)
}
desired := buildDesiredState(spec, files)
bundle, err := txn.RenderBundle(desired, mk)
if err != nil {
return nil, fmt.Errorf("render bundle: %w", err)
}
transport, err := jobVerifyTransportFromCtx()
if err != nil {
return nil, fmt.Errorf("ssh transport: %w", err)
}
if err := txn.Stage(bundle, jobVerifyLead, asTxnTransport(transport)); err != nil {
return nil, fmt.Errorf("stage bundle: %w", err)
}
slog.Info("job verify: bundle staged (no apply)", "txn_id", bundle.ID, "lead", jobVerifyLead)
verifyOut, verifyErr := runVerifySh(ctx, transport, bundle.ID, jobVerifyLead, jobVerifyNamespace)
drift := parseDriftLines(string(verifyOut))
report := &verifyReport{
TxnID: string(bundle.ID),
Kind: spec.Kind,
Name: spec.Name,
Namespace: jobVerifyNamespace,
Lead: jobVerifyLead,
PlannedAllocs: buildPlannedAllocs(spec),
PlannedFiles: buildPlannedFiles(files),
VerifyOutput: string(verifyOut),
Drift: drift,
Status: "verified",
}
if verifyErr != nil {
report.Status = "verify-failed"
return report, fmt.Errorf("verify.sh on %s: %w (output: %s)", jobVerifyLead, verifyErr, string(verifyOut))
}
return report, nil
}
// validateForVerify runs the schema validator (the dry-run should fail
// fast on an invalid spec, same as `orca job lint` errors).
func validateForVerify(spec *jobspec.WorkloadSpec) error {
if spec == nil {
return fmt.Errorf("spec is nil")
}
v, err := schema.ValidatorFor(spec.Kind)
if err != nil {
return err
}
if verr := v.Validate(spec); verr != nil {
return fmt.Errorf("schema validation: %w", verr)
}
return nil
}
// renderPlan renders the spec into the emitter File artifacts that
// represent what WOULD be written on apply. The registry is populated
// with the SystemdEmitter for the "process" runtime (the only runtime
// P0c ships); other runtimes return an error so the dry-run reports
// the gap instead of pretending success.
func renderPlan(spec *jobspec.WorkloadSpec) ([]emitter.File, error) {
if spec.Runtime == nil {
return nil, fmt.Errorf("spec runtime is nil (verify needs a runtime to render)")
}
reg := emitter.NewRegistry()
reg.Register("job:process", emitter.SystemdEmitter{})
reg.Register("service:process", emitter.SystemdEmitter{})
reg.Register("daemonset:process", emitter.SystemdEmitter{})
node := &emitter.Node{Hostname: jobVerifyLead}
return reg.Render(spec, node)
}
// buildDesiredState assembles the desired-state object the txn apply
// script consumes. It is a list of artifact dicts (path/content/mode)
// derived from the emitter plan, plus metadata so verify.sh and
// orca-pull.sh can report what would change.
func buildDesiredState(spec *jobspec.WorkloadSpec, files []emitter.File) any {
type artifact struct {
Path string `json:"path"`
Content string `json:"content"`
Mode string `json:"mode"`
}
arts := make([]artifact, len(files))
for i, f := range files {
arts[i] = artifact{Path: f.Path, Content: f.Content, Mode: f.Mode}
}
return map[string]any{
"kind": spec.Kind,
"name": spec.Name,
"namespace": jobVerifyNamespace,
"artifacts": arts,
}
}
// buildPlannedAllocs reports the alloc(s) the apply would create. For
// the single-process path it is one alloc named after the spec; for a
// task group it is one alloc with N task units. Count > 1 (Service)
// expands to N allocs.
func buildPlannedAllocs(spec *jobspec.WorkloadSpec) []plannedAlloc {
count := spec.Count
if count < 1 {
count = 1
}
out := make([]plannedAlloc, 0, count)
for i := 0; i < count; i++ {
name := spec.Name
if count > 1 {
name = fmt.Sprintf("%s-%d", spec.Name, i)
}
out = append(out, plannedAlloc{Name: name, Kind: spec.Kind, Count: 1})
}
return out
}
// buildPlannedFiles classifies the emitter artifacts into config files
// and systemd units by path. Anything under /etc/systemd/system/ is a
// unit; everything else is a config file.
func buildPlannedFiles(files []emitter.File) []plannedFile {
out := make([]plannedFile, 0, len(files))
for _, f := range files {
kind := "config"
if strings.HasPrefix(f.Path, "/etc/systemd/system/") {
kind = "unit"
}
out = append(out, plannedFile{Path: f.Path, Mode: f.Mode, Kind: kind})
}
return out
}
// runVerifySh runs verify.sh on the lead for the staged bundle. The
// verify script reports missing files (the ones that WOULD be written
// on apply). A non-zero exit is expected for a dry-run (the files are
// not applied yet), so the caller treats the output as informational.
func runVerifySh(ctx context.Context, transport jobVerifyTransport, id txn.TxnID, lead, namespace string) ([]byte, error) {
dir := "/run/orca/txns/" + string(id)
cmd := fmt.Sprintf("bash %s/verify.sh", dir)
out, err := transport.Exec(ctx, lead, cmd)
if err != nil {
return out, err
}
return out, nil
}
// parseDriftLines extracts pre-flight drift (R-020) notices from the
// verify.sh output. The verify script emits "verify: drift <detail>"
// lines when the on-disk state has drifted from a prior apply; in a
// dry-run these are reported but do not fail.
func parseDriftLines(out string) []string {
var drift []string
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "verify: drift") {
drift = append(drift, strings.TrimSpace(strings.TrimPrefix(line, "verify:")))
}
}
return drift
}
func printVerifyText(r *verifyReport) {
w := rootCmd.OutOrStdout()
fmt.Fprintf(w, "Txn: %s\n", r.TxnID)
fmt.Fprintf(w, "Kind: %s Name: %s\n", r.Kind, r.Name)
if r.Namespace != "" {
fmt.Fprintf(w, "Namespace: %s\n", r.Namespace)
}
fmt.Fprintf(w, "Lead: %s\n", r.Lead)
fmt.Fprintf(w, "Status: %s\n", r.Status)
fmt.Fprintf(w, "\nPlanned allocations (%d):\n", len(r.PlannedAllocs))
for _, a := range r.PlannedAllocs {
fmt.Fprintf(w, " - %s (kind=%s)\n", a.Name, a.Kind)
}
units := 0
configs := 0
for _, f := range r.PlannedFiles {
if f.Kind == "unit" {
units++
} else {
configs++
}
}
fmt.Fprintf(w, "\nPlanned files: %d config, %d systemd units\n", configs, units)
for _, f := range r.PlannedFiles {
fmt.Fprintf(w, " - [%s] %s (mode %s)\n", f.Kind, f.Path, f.Mode)
}
if len(r.Drift) > 0 {
fmt.Fprintf(w, "\nPre-flight drift (R-020, reported -- dry-run does not fail):\n")
for _, d := range r.Drift {
fmt.Fprintf(w, " ! %s\n", d)
}
}
if r.VerifyOutput != "" {
fmt.Fprintf(w, "\nverify.sh output:\n%s\n", r.VerifyOutput)
}
}
func printVerifyJSON(r *verifyReport) error {
w := rootCmd.OutOrStdout()
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
return enc.Encode(r)
}
// asTxnTransport adapts the jobVerifyTransport seam to the txn.Transport
// interface (they have the same shape; this is a thin wrapper so the
// two packages stay decoupled).
type txnTransportAdapter struct {
inner jobVerifyTransport
}
func (a txnTransportAdapter) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
return a.inner.WriteFileIdempotent(ctx, peer, path, content, mode)
}
func (a txnTransportAdapter) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
return a.inner.Exec(ctx, peer, cmd)
}
func asTxnTransport(t jobVerifyTransport) txn.Transport {
return txnTransportAdapter{inner: t}
}
// fileBase returns filepath.Base(path) without importing filepath in
// the top of the file (kept here so the import block stays small).
func fileBase(path string) string {
if i := strings.LastIndexAny(path, "/\\"); i >= 0 {
return path[i+1:]
}
return path
}
func init() {
jobVerifyCmd.Flags().StringVar(&jobVerifyLead, "lead", "", "lead peer address (host:port) (required)")
jobVerifyCmd.Flags().StringVar(&jobVerifyNamespace, "namespace", "", "namespace scope")
jobVerifyCmd.Flags().BoolVar(&jobVerifyJSON, "json", false, "JSON output")
jobCmd.AddCommand(jobVerifyCmd)
}
+246
View File
@@ -0,0 +1,246 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"os"
"strings"
"testing"
"git.cloudinit.dev/coreci/orca/internal/paths"
"git.cloudinit.dev/coreci/orca/internal/secrets"
)
// mockVerifyTransport is a record-and-replay mock of jobVerifyTransport.
type mockVerifyTransport struct {
writes []mockWriteCall
execs []string
execOut []byte
execErr error
}
type mockWriteCall struct {
peer string
path string
content []byte
mode os.FileMode
}
func (m *mockVerifyTransport) WriteFileIdempotent(_ context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
m.writes = append(m.writes, mockWriteCall{peer, path, content, mode})
return true, nil
}
func (m *mockVerifyTransport) Exec(_ context.Context, _ string, cmd string) ([]byte, error) {
m.execs = append(m.execs, cmd)
return m.execOut, m.execErr
}
// setupVerifyEnv sets ORCA_HOME to a temp dir and writes a master key
// (txn.RenderBundle requires it).
func setupVerifyEnv(t *testing.T) string {
t.Helper()
dir := t.TempDir()
t.Setenv("ORCA_HOME", dir)
mk, err := secrets.GenerateMasterKey()
if err != nil {
t.Fatalf("GenerateMasterKey: %v", err)
}
if err := secrets.SaveMasterKey(paths.MasterKeyPath(), mk); err != nil {
t.Fatalf("SaveMasterKey: %v", err)
}
return dir
}
func TestJobVerifyCmdRegistered(t *testing.T) {
for _, c := range jobCmd.Commands() {
if c.Name() == "verify" {
return
}
}
t.Fatal("job verify command not registered on jobCmd")
}
func TestJobVerifyValidSpec(t *testing.T) {
setupVerifyEnv(t)
mt := &mockVerifyTransport{execOut: []byte("verified\n")}
jobVerifyTransportOverride = mt
defer func() { jobVerifyTransportOverride = nil }()
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22"})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job verify valid: %v", err)
}
out := buf.String()
if !strings.Contains(out, "Planned allocations") {
t.Errorf("expected planned allocs in output: %s", out)
}
if !strings.Contains(out, "systemd") && !strings.Contains(out, "unit") {
t.Errorf("expected systemd unit in planned files: %s", out)
}
if len(mt.writes) == 0 {
t.Errorf("expected bundle to be staged (writes), got 0")
}
if len(mt.execs) != 1 {
t.Errorf("expected 1 exec (verify.sh), got %d", len(mt.execs))
}
}
func TestJobVerifyInvalidSpecFails(t *testing.T) {
setupVerifyEnv(t)
mt := &mockVerifyTransport{}
jobVerifyTransportOverride = mt
defer func() { jobVerifyTransportOverride = nil }()
resetRootFlags(t)
spec := writeMDSpec(t, serviceMissingPortsMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22"})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for invalid spec, got nil")
}
if len(mt.writes) != 0 {
t.Errorf("should not stage bundle for invalid spec, got %d writes", len(mt.writes))
}
}
func TestJobVerifyJSONOutput(t *testing.T) {
setupVerifyEnv(t)
mt := &mockVerifyTransport{execOut: []byte("verified\n")}
jobVerifyTransportOverride = mt
defer func() { jobVerifyTransportOverride = nil }()
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22", "--json"})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job verify --json: %v", err)
}
var report verifyReport
if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &report); err != nil {
t.Fatalf("unmarshal verify json: %v\n%s", err, buf.String())
}
if report.Name != "my-job" {
t.Errorf("report.Name = %q, want my-job", report.Name)
}
if len(report.PlannedFiles) == 0 {
t.Errorf("expected planned files, got 0")
}
if report.TxnID == "" {
t.Errorf("expected txn id, got empty")
}
}
func TestJobVerifyNamespaceScoping(t *testing.T) {
setupVerifyEnv(t)
mt := &mockVerifyTransport{execOut: []byte("verified\n")}
jobVerifyTransportOverride = mt
defer func() { jobVerifyTransportOverride = nil }()
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22", "--namespace", "prod"})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job verify --namespace: %v", err)
}
out := buf.String()
if !strings.Contains(out, "Namespace: prod") {
t.Errorf("expected namespace in output: %s", out)
}
}
func TestJobVerifyNoSideEffects(t *testing.T) {
setupVerifyEnv(t)
mt := &mockVerifyTransport{execOut: []byte("verified\n")}
jobVerifyTransportOverride = mt
defer func() { jobVerifyTransportOverride = nil }()
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22"})
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job verify: %v", err)
}
// Staging writes the bundle files but apply is NOT run.
for _, w := range mt.writes {
if strings.Contains(w.path, ".applied") {
t.Errorf("verify staged a .applied marker (side effect): %s", w.path)
}
}
for _, c := range mt.execs {
if strings.Contains(c, "apply") {
t.Errorf("verify ran apply (side effect): %s", c)
}
if strings.Contains(c, "orca-pull.sh") {
t.Errorf("verify ran orca-pull.sh (side effect): %s", c)
}
}
}
func TestJobVerifyPreflightDriftReported(t *testing.T) {
setupVerifyEnv(t)
mt := &mockVerifyTransport{execOut: []byte("verify: drift /etc/systemd/system/orca-v1-my-job.service has drifted from last apply\n")}
jobVerifyTransportOverride = mt
defer func() { jobVerifyTransportOverride = nil }()
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22"})
// verify.sh exit 1 from drift is treated as a verify-failure by
// the mock (execErr). But here execOut is set and execErr is nil,
// so verify "passes" and drift is reported. We assert drift is
// surfaced in the output and does NOT fail the dry-run.
if err := rootCmd.Execute(); err != nil {
t.Fatalf("job verify with drift should not fail dry-run: %v", err)
}
out := buf.String()
if !strings.Contains(strings.ToLower(out), "drift") {
t.Errorf("expected drift in output: %s", out)
}
}
func TestJobVerifyMissingLeadFails(t *testing.T) {
setupVerifyEnv(t)
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for missing --lead, got nil")
}
}
func TestJobVerifyMissingMasterKeyFails(t *testing.T) {
t.Setenv("ORCA_HOME", t.TempDir())
resetRootFlags(t)
spec := writeMDSpec(t, validJobMD)
var buf bytes.Buffer
rootCmd.SetOut(&buf)
rootCmd.SetErr(&buf)
rootCmd.SetArgs([]string{"job", "verify", spec, "--lead", "lead:22"})
if err := rootCmd.Execute(); err == nil {
t.Fatal("expected error for missing master key, got nil")
}
}
+6 -1
View File
@@ -55,6 +55,11 @@ func resetCommandFlags() {
driftConfigPath = "" driftConfigPath = ""
driftRemediateForce = false driftRemediateForce = false
jobRestartPeer = "" jobRestartPeer = ""
jobLintExplain = false
jobLintFormat = "text"
jobVerifyLead = ""
jobVerifyNamespace = ""
jobVerifyJSON = false
peerSetupNoOrcaUser = false peerSetupNoOrcaUser = false
resetNSFlags() resetNSFlags()
// Reset per-command output writers so tests that polluted them // Reset per-command output writers so tests that polluted them
@@ -62,7 +67,7 @@ func resetCommandFlags() {
// other tests. nil → cobra walks to rootCmd's writer. // other tests. nil → cobra walks to rootCmd's writer.
for _, c := range []*cobra.Command{ for _, c := range []*cobra.Command{
nodeDrainCmd, daemonCmd, daemonDrainAndStopCmd, nodeDrainCmd, daemonCmd, daemonDrainAndStopCmd,
jobCmd, jobMigrateCmd, jobRunCmd, jobListCmd, jobStopCmd, jobLogsCmd, jobCmd, jobMigrateCmd, jobRunCmd, jobListCmd, jobStopCmd, jobLogsCmd, jobLintCmd, jobVerifyCmd,
logsCmd, logsCmd,
} { } {
if c != nil { if c != nil {