Jon Chery
020aa01623
P11: orca job lint <spec.md> — schema/CEL/body/migration/best-practice checks; --explain, --format json; exit 0/1 by errors found. P12: orca job verify <spec.md> — dry-run txn (render + stage + verify without apply); reports planned allocs/files/units; no side effects; --namespace, --json. ---ci--- project: orca phase: 11 milestone: v0.11 status: execute ---/ci---
397 lines
13 KiB
Go
397 lines
13 KiB
Go
// Package cli: job_verify.go implements `orca job verify` (P12,
|
|
// v0.11 milestone). It performs a dry-run transaction through the lead:
|
|
// parse the jobspec, render the emitter plan (allocs/files/units),
|
|
// render a txn bundle with the desired state, stage it on the lead
|
|
// (idempotent; NO apply), run verify.sh on the lead to capture what
|
|
// WOULD be applied, and report the plan. Pre-flight drift (R-020) is
|
|
// reported but does not fail a dry-run.
|
|
//
|
|
// There are no side effects beyond the staged bundle files in
|
|
// /run/orca/txns/<txn-id>/ on the lead (idempotent; no .applied marker
|
|
// is written, so orca-pull.sh never picks it up).
|
|
package cli
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
|
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
|
"git.cloudinit.dev/coreci/orca/internal/paths"
|
|
"git.cloudinit.dev/coreci/orca/internal/secrets"
|
|
"git.cloudinit.dev/coreci/orca/internal/spec/schema"
|
|
"git.cloudinit.dev/coreci/orca/internal/txn"
|
|
)
|
|
|
|
var (
|
|
jobVerifyLead string
|
|
jobVerifyNamespace string
|
|
jobVerifyJSON bool
|
|
)
|
|
|
|
// jobVerifyTransport is the SSH-push surface `job verify` needs. It
|
|
// is satisfied by *sshpush.Transport; tests substitute a mock (same
|
|
// pattern as txn.go / drain.go).
|
|
type jobVerifyTransport interface {
|
|
WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error)
|
|
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
|
}
|
|
|
|
// jobVerifyTransportOverride is the package-level seam. When non-nil
|
|
// it replaces the production transport; tests set it and restore nil.
|
|
var jobVerifyTransportOverride jobVerifyTransport
|
|
|
|
func jobVerifyTransportFromCtx() (jobVerifyTransport, error) {
|
|
if jobVerifyTransportOverride != nil {
|
|
return jobVerifyTransportOverride, nil
|
|
}
|
|
return txnTransportFromCtx()
|
|
}
|
|
|
|
// verifyReport is the structured result of `orca job verify`. It is
|
|
// rendered to JSON when --json is set, or as a human-readable summary
|
|
// otherwise.
|
|
type verifyReport struct {
|
|
TxnID string `json:"txn_id"`
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name"`
|
|
Namespace string `json:"namespace,omitempty"`
|
|
Lead string `json:"lead"`
|
|
PlannedAllocs []plannedAlloc `json:"planned_allocs"`
|
|
PlannedFiles []plannedFile `json:"planned_files"`
|
|
VerifyOutput string `json:"verify_output,omitempty"`
|
|
Drift []string `json:"drift,omitempty"`
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
type plannedAlloc struct {
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
Count int `json:"count"`
|
|
}
|
|
|
|
type plannedFile struct {
|
|
Path string `json:"path"`
|
|
Mode string `json:"mode"`
|
|
Kind string `json:"kind"`
|
|
}
|
|
|
|
var jobVerifyCmd = &cobra.Command{
|
|
Use: "verify <spec>",
|
|
Short: "Dry-run a jobspec through the lead (no apply)",
|
|
Long: `Dry-run a jobspec as a transaction through the lead peer.
|
|
|
|
Steps (P12):
|
|
1. Parse the jobspec
|
|
2. Render the emitter plan (allocs, config files, systemd units)
|
|
3. Render a txn bundle (RenderBundle) with the desired state
|
|
4. Stage the bundle on the lead (NO apply; idempotent)
|
|
5. Run verify.sh on the lead to capture what WOULD be applied
|
|
6. Report planned allocs / files / units
|
|
|
|
Pre-flight drift (R-020) is reported but does not fail a dry-run.
|
|
There are no side effects beyond the staged bundle files in
|
|
/run/orca/txns/<txn-id>/ on the lead (no .applied marker is written).
|
|
|
|
Flags:
|
|
--lead <peer> lead peer address (host:port) (required)
|
|
--namespace <ns> namespace scope
|
|
--json JSON output
|
|
|
|
Exit codes: 0 = verify passed (no issues), 1 = verification failed.`,
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
report, err := runJobVerify(cmd.Context(), args[0])
|
|
if err != nil {
|
|
if jobVerifyJSON {
|
|
if report != nil {
|
|
_ = printVerifyJSON(report)
|
|
}
|
|
} else if report != nil {
|
|
printVerifyText(report)
|
|
}
|
|
return err
|
|
}
|
|
if jobVerifyJSON {
|
|
return printVerifyJSON(report)
|
|
}
|
|
printVerifyText(report)
|
|
return nil
|
|
},
|
|
}
|
|
|
|
// runJobVerify performs the dry-run. It returns the report and an
|
|
// error: when err is non-nil the report may still be populated with
|
|
// partial results (e.g. drift was detected but the verify step ran).
|
|
// The caller renders the report then surfaces the error.
|
|
func runJobVerify(ctx context.Context, path string) (*verifyReport, error) {
|
|
if jobVerifyLead == "" {
|
|
return nil, fmt.Errorf("--lead is required for job verify")
|
|
}
|
|
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read spec file: %w", err)
|
|
}
|
|
spec, err := jobspec.Dispatch(data, fileBase(path))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("parse spec: %w", err)
|
|
}
|
|
|
|
if verr := validateForVerify(spec); verr != nil {
|
|
return nil, verr
|
|
}
|
|
|
|
files, err := renderPlan(spec)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("render plan: %w", err)
|
|
}
|
|
|
|
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("load master key: %w", err)
|
|
}
|
|
|
|
desired := buildDesiredState(spec, files)
|
|
bundle, err := txn.RenderBundle(desired, mk)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("render bundle: %w", err)
|
|
}
|
|
|
|
transport, err := jobVerifyTransportFromCtx()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ssh transport: %w", err)
|
|
}
|
|
|
|
if err := txn.Stage(bundle, jobVerifyLead, asTxnTransport(transport)); err != nil {
|
|
return nil, fmt.Errorf("stage bundle: %w", err)
|
|
}
|
|
slog.Info("job verify: bundle staged (no apply)", "txn_id", bundle.ID, "lead", jobVerifyLead)
|
|
|
|
verifyOut, verifyErr := runVerifySh(ctx, transport, bundle.ID, jobVerifyLead, jobVerifyNamespace)
|
|
drift := parseDriftLines(string(verifyOut))
|
|
|
|
report := &verifyReport{
|
|
TxnID: string(bundle.ID),
|
|
Kind: spec.Kind,
|
|
Name: spec.Name,
|
|
Namespace: jobVerifyNamespace,
|
|
Lead: jobVerifyLead,
|
|
PlannedAllocs: buildPlannedAllocs(spec),
|
|
PlannedFiles: buildPlannedFiles(files),
|
|
VerifyOutput: string(verifyOut),
|
|
Drift: drift,
|
|
Status: "verified",
|
|
}
|
|
|
|
if verifyErr != nil {
|
|
report.Status = "verify-failed"
|
|
return report, fmt.Errorf("verify.sh on %s: %w (output: %s)", jobVerifyLead, verifyErr, string(verifyOut))
|
|
}
|
|
return report, nil
|
|
}
|
|
|
|
// validateForVerify runs the schema validator (the dry-run should fail
|
|
// fast on an invalid spec, same as `orca job lint` errors).
|
|
func validateForVerify(spec *jobspec.WorkloadSpec) error {
|
|
if spec == nil {
|
|
return fmt.Errorf("spec is nil")
|
|
}
|
|
v, err := schema.ValidatorFor(spec.Kind)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if verr := v.Validate(spec); verr != nil {
|
|
return fmt.Errorf("schema validation: %w", verr)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// renderPlan renders the spec into the emitter File artifacts that
|
|
// represent what WOULD be written on apply. The registry is populated
|
|
// with the SystemdEmitter for the "process" runtime (the only runtime
|
|
// P0c ships); other runtimes return an error so the dry-run reports
|
|
// the gap instead of pretending success.
|
|
func renderPlan(spec *jobspec.WorkloadSpec) ([]emitter.File, error) {
|
|
if spec.Runtime == nil {
|
|
return nil, fmt.Errorf("spec runtime is nil (verify needs a runtime to render)")
|
|
}
|
|
reg := emitter.NewRegistry()
|
|
reg.Register("job:process", emitter.SystemdEmitter{})
|
|
reg.Register("service:process", emitter.SystemdEmitter{})
|
|
reg.Register("daemonset:process", emitter.SystemdEmitter{})
|
|
node := &emitter.Node{Hostname: jobVerifyLead}
|
|
return reg.Render(spec, node)
|
|
}
|
|
|
|
// buildDesiredState assembles the desired-state object the txn apply
|
|
// script consumes. It is a list of artifact dicts (path/content/mode)
|
|
// derived from the emitter plan, plus metadata so verify.sh and
|
|
// orca-pull.sh can report what would change.
|
|
func buildDesiredState(spec *jobspec.WorkloadSpec, files []emitter.File) any {
|
|
type artifact struct {
|
|
Path string `json:"path"`
|
|
Content string `json:"content"`
|
|
Mode string `json:"mode"`
|
|
}
|
|
arts := make([]artifact, len(files))
|
|
for i, f := range files {
|
|
arts[i] = artifact{Path: f.Path, Content: f.Content, Mode: f.Mode}
|
|
}
|
|
return map[string]any{
|
|
"kind": spec.Kind,
|
|
"name": spec.Name,
|
|
"namespace": jobVerifyNamespace,
|
|
"artifacts": arts,
|
|
}
|
|
}
|
|
|
|
// buildPlannedAllocs reports the alloc(s) the apply would create. For
|
|
// the single-process path it is one alloc named after the spec; for a
|
|
// task group it is one alloc with N task units. Count > 1 (Service)
|
|
// expands to N allocs.
|
|
func buildPlannedAllocs(spec *jobspec.WorkloadSpec) []plannedAlloc {
|
|
count := spec.Count
|
|
if count < 1 {
|
|
count = 1
|
|
}
|
|
out := make([]plannedAlloc, 0, count)
|
|
for i := 0; i < count; i++ {
|
|
name := spec.Name
|
|
if count > 1 {
|
|
name = fmt.Sprintf("%s-%d", spec.Name, i)
|
|
}
|
|
out = append(out, plannedAlloc{Name: name, Kind: spec.Kind, Count: 1})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// buildPlannedFiles classifies the emitter artifacts into config files
|
|
// and systemd units by path. Anything under /etc/systemd/system/ is a
|
|
// unit; everything else is a config file.
|
|
func buildPlannedFiles(files []emitter.File) []plannedFile {
|
|
out := make([]plannedFile, 0, len(files))
|
|
for _, f := range files {
|
|
kind := "config"
|
|
if strings.HasPrefix(f.Path, "/etc/systemd/system/") {
|
|
kind = "unit"
|
|
}
|
|
out = append(out, plannedFile{Path: f.Path, Mode: f.Mode, Kind: kind})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// runVerifySh runs verify.sh on the lead for the staged bundle. The
|
|
// verify script reports missing files (the ones that WOULD be written
|
|
// on apply). A non-zero exit is expected for a dry-run (the files are
|
|
// not applied yet), so the caller treats the output as informational.
|
|
func runVerifySh(ctx context.Context, transport jobVerifyTransport, id txn.TxnID, lead, namespace string) ([]byte, error) {
|
|
dir := "/run/orca/txns/" + string(id)
|
|
cmd := fmt.Sprintf("bash %s/verify.sh", dir)
|
|
out, err := transport.Exec(ctx, lead, cmd)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// parseDriftLines extracts pre-flight drift (R-020) notices from the
|
|
// verify.sh output. The verify script emits "verify: drift <detail>"
|
|
// lines when the on-disk state has drifted from a prior apply; in a
|
|
// dry-run these are reported but do not fail.
|
|
func parseDriftLines(out string) []string {
|
|
var drift []string
|
|
for _, line := range strings.Split(out, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if strings.HasPrefix(line, "verify: drift") {
|
|
drift = append(drift, strings.TrimSpace(strings.TrimPrefix(line, "verify:")))
|
|
}
|
|
}
|
|
return drift
|
|
}
|
|
|
|
func printVerifyText(r *verifyReport) {
|
|
w := rootCmd.OutOrStdout()
|
|
fmt.Fprintf(w, "Txn: %s\n", r.TxnID)
|
|
fmt.Fprintf(w, "Kind: %s Name: %s\n", r.Kind, r.Name)
|
|
if r.Namespace != "" {
|
|
fmt.Fprintf(w, "Namespace: %s\n", r.Namespace)
|
|
}
|
|
fmt.Fprintf(w, "Lead: %s\n", r.Lead)
|
|
fmt.Fprintf(w, "Status: %s\n", r.Status)
|
|
fmt.Fprintf(w, "\nPlanned allocations (%d):\n", len(r.PlannedAllocs))
|
|
for _, a := range r.PlannedAllocs {
|
|
fmt.Fprintf(w, " - %s (kind=%s)\n", a.Name, a.Kind)
|
|
}
|
|
units := 0
|
|
configs := 0
|
|
for _, f := range r.PlannedFiles {
|
|
if f.Kind == "unit" {
|
|
units++
|
|
} else {
|
|
configs++
|
|
}
|
|
}
|
|
fmt.Fprintf(w, "\nPlanned files: %d config, %d systemd units\n", configs, units)
|
|
for _, f := range r.PlannedFiles {
|
|
fmt.Fprintf(w, " - [%s] %s (mode %s)\n", f.Kind, f.Path, f.Mode)
|
|
}
|
|
if len(r.Drift) > 0 {
|
|
fmt.Fprintf(w, "\nPre-flight drift (R-020, reported -- dry-run does not fail):\n")
|
|
for _, d := range r.Drift {
|
|
fmt.Fprintf(w, " ! %s\n", d)
|
|
}
|
|
}
|
|
if r.VerifyOutput != "" {
|
|
fmt.Fprintf(w, "\nverify.sh output:\n%s\n", r.VerifyOutput)
|
|
}
|
|
}
|
|
|
|
func printVerifyJSON(r *verifyReport) error {
|
|
w := rootCmd.OutOrStdout()
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
return enc.Encode(r)
|
|
}
|
|
|
|
// asTxnTransport adapts the jobVerifyTransport seam to the txn.Transport
|
|
// interface (they have the same shape; this is a thin wrapper so the
|
|
// two packages stay decoupled).
|
|
type txnTransportAdapter struct {
|
|
inner jobVerifyTransport
|
|
}
|
|
|
|
func (a txnTransportAdapter) WriteFileIdempotent(ctx context.Context, peer string, path string, content []byte, mode os.FileMode) (bool, error) {
|
|
return a.inner.WriteFileIdempotent(ctx, peer, path, content, mode)
|
|
}
|
|
|
|
func (a txnTransportAdapter) Exec(ctx context.Context, peer string, cmd string) ([]byte, error) {
|
|
return a.inner.Exec(ctx, peer, cmd)
|
|
}
|
|
|
|
func asTxnTransport(t jobVerifyTransport) txn.Transport {
|
|
return txnTransportAdapter{inner: t}
|
|
}
|
|
|
|
// fileBase returns filepath.Base(path) without importing filepath in
|
|
// the top of the file (kept here so the import block stays small).
|
|
func fileBase(path string) string {
|
|
if i := strings.LastIndexAny(path, "/\\"); i >= 0 {
|
|
return path[i+1:]
|
|
}
|
|
return path
|
|
}
|
|
|
|
func init() {
|
|
jobVerifyCmd.Flags().StringVar(&jobVerifyLead, "lead", "", "lead peer address (host:port) (required)")
|
|
jobVerifyCmd.Flags().StringVar(&jobVerifyNamespace, "namespace", "", "namespace scope")
|
|
jobVerifyCmd.Flags().BoolVar(&jobVerifyJSON, "json", false, "JSON output")
|
|
jobCmd.AddCommand(jobVerifyCmd)
|
|
}
|