package cli import ( "bytes" "context" "encoding/json" "os" "path/filepath" "strings" "sync" "testing" "time" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/model" "git.cloudinit.dev/coreci/orca/internal/store" ) func writeJobSpec(t *testing.T, content string) string { t.Helper() dir := t.TempDir() p := filepath.Join(dir, "spec.hcl") if err := os.WriteFile(p, []byte(content), 0o644); err != nil { t.Fatalf("write spec: %v", err) } return p } const trueJobSpec = `job "true" {} task "t" { command = "/bin/true" } ` const falseJobSpec = `job "false" {} task "t" { command = "/bin/false" } ` func TestJobRunComplete(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() spec := writeJobSpec(t, trueJobSpec) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "run", spec}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job run: %v", err) } if !strings.Contains(buf.String(), "Job complete") { t.Errorf("job run output unexpected: %s", buf.String()) } } func TestJobRunCompleteJSON(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() spec := writeJobSpec(t, trueJobSpec) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "run", spec, "--json"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job run --json: %v", err) } var result map[string]any if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &result); err != nil { t.Fatalf("unmarshal job run json: %v\n%s", err, buf.String()) } if result["status"] != "complete" { t.Errorf("job run --json status = %v, want complete", result["status"]) } } func TestJobRunFailed(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() spec := writeJobSpec(t, falseJobSpec) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "run", spec}) if err := rootCmd.Execute(); err == nil { t.Fatal("expected error for failing job, got nil") } } func TestJobRunFailedJSON(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() spec := writeJobSpec(t, falseJobSpec) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "run", spec, "--json"}) if err := rootCmd.Execute(); err == nil { t.Fatal("expected error for failing job --json, got nil") } if !strings.Contains(buf.String(), "failed") { t.Errorf("job run --json failed output unexpected: %s", buf.String()) } } func TestJobRunMissingSpecFile(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "run", "/nonexistent/spec.hcl"}) if err := rootCmd.Execute(); err == nil { t.Fatal("expected error for missing spec file, got nil") } } func TestJobListEmpty(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "list"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job list: %v", err) } if !strings.Contains(buf.String(), "No jobs") { t.Errorf("job list empty output unexpected: %s", buf.String()) } } func TestJobListJSONEmpty(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "list", "--json"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job list --json: %v", err) } var jobs []map[string]any if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &jobs); err != nil { t.Fatalf("unmarshal job list json: %v\n%s", err, buf.String()) } if len(jobs) != 0 { t.Errorf("job list --json empty = %d jobs, want 0", len(jobs)) } } func TestJobListAfterRun(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() spec := writeJobSpec(t, trueJobSpec) resetRootFlags(t) rootCmd.SetArgs([]string{"job", "run", spec}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job run: %v", err) } resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "list"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job list: %v", err) } out := buf.String() if !strings.Contains(out, "true") { t.Errorf("job list missing job name: %s", out) } } func TestJobStop(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() jobID := seedJob(t, "stopper", model.JobStatusRunning) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop", jobID}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job stop: %v", err) } if !strings.Contains(buf.String(), "Job stopped") { t.Errorf("job stop output unexpected: %s", buf.String()) } } func TestJobStopJSON(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() jobID := seedJob(t, "jsonstopper", model.JobStatusRunning) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop", jobID, "--json"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job stop --json: %v", err) } var result map[string]any if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &result); err != nil { t.Fatalf("unmarshal job stop json: %v\n%s", err, buf.String()) } if result["status"] != "stopped" { t.Errorf("job stop --json status = %v, want stopped", result["status"]) } } func TestJobStopNotFound(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop", "nonexistent-id"}) if err := rootCmd.Execute(); err == nil { t.Fatal("expected error for job stop not found, got nil") } } func TestJobStopMissingID(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop"}) if err := rootCmd.Execute(); err == nil { t.Fatal("expected error for job stop without id, got nil") } } func TestJobLogsEmpty(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() jobID := seedJob(t, "logger", model.JobStatusComplete) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "logs", jobID}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job logs: %v", err) } if !strings.Contains(buf.String(), "No tasks") { t.Errorf("job logs empty output unexpected: %s", buf.String()) } } func TestJobLogsJSONEmpty(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() jobID := seedJob(t, "jsonlogger", model.JobStatusComplete) resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "logs", jobID, "--json"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job logs --json: %v", err) } var tasks []map[string]any if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &tasks); err != nil { t.Fatalf("unmarshal job logs json: %v\n%s", err, buf.String()) } if len(tasks) != 0 { t.Errorf("job logs --json empty = %d tasks, want 0", len(tasks)) } } func TestJobLogsMissingID(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() resetRootFlags(t) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "logs"}) if err := rootCmd.Execute(); err == nil { t.Fatal("expected error for job logs without id, got nil") } } func seedJob(t *testing.T, name string, status model.JobStatus) string { t.Helper() db, err := store.Open(certpaths.DBPath()) if err != nil { t.Fatalf("open db: %v", err) } defer db.Close() repo := store.NewJobRepo(db) j := &model.Job{ ID: "job-" + name, Name: name, Spec: "spec.hcl", Status: status, } if err := repo.Insert(t.Context(), j); err != nil { t.Fatalf("insert job: %v", err) } return j.ID } // mockJobStopExec is a record-and-replay SSH execer for `orca job stop` // tests (same pattern as mockDrainExec / mockLogsExec). type mockJobStopExec struct { mu sync.Mutex responses []jobStopMockResp calls []jobStopMockCall } type jobStopMockResp struct { match string out string exit int } type jobStopMockCall struct { peer string cmd string } func (m *mockJobStopExec) Exec(_ context.Context, peer, cmd string) ([]byte, error) { m.mu.Lock() defer m.mu.Unlock() m.calls = append(m.calls, jobStopMockCall{peer: peer, cmd: cmd}) for _, r := range m.responses { if r.match == "" || strings.Contains(cmd, r.match) { return []byte(r.out), nil } } return []byte(""), nil } func (m *mockJobStopExec) callsFor(match string) []jobStopMockCall { m.mu.Lock() defer m.mu.Unlock() var out []jobStopMockCall for _, c := range m.calls { if strings.Contains(c.cmd, match) { out = append(out, c) } } return out } // TestJobStopSSH verifies that `orca job stop` sends a real // 'systemctl stop' via SSH to the target node when the job has a // recorded allocation history (REQ-158, P09 T6). func TestJobStopSSH(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() if err := runInit(discardWriter{}); err != nil { t.Fatalf("init: %v", err) } // Seed a node and a job, then record an alloc_history entry // linking the job to the node. nodeID := seedNode(t, "worker-1", "worker-1:8443") jobID := seedJob(t, "webapp", model.JobStatusRunning) db, err := store.Open(certpaths.DBPath()) if err != nil { t.Fatalf("open db: %v", err) } defer db.Close() hist := store.NewAllocHistoryRepo(db) ctx := context.Background() if err := hist.EnsureSchema(ctx); err != nil { t.Fatalf("ensure schema: %v", err) } if err := hist.Record(ctx, store.AllocHistoryEntry{ AllocID: "default/webapp-0", JobID: jobID, NodeID: nodeID, Namespace: "default", ToState: "created", Timestamp: time.Now().UTC(), }); err != nil { t.Fatalf("record alloc history: %v", err) } // Wire the mock SSH transport (must be after resetRootFlags so // resetCommandFlags doesn't nil it out). resetRootFlags(t) mock := &mockJobStopExec{} prev := jobStopTransportOverride jobStopTransportOverride = mock t.Cleanup(func() { jobStopTransportOverride = prev }) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop", jobID}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job stop: %v", err) } // Verify systemctl stop was called via SSH. stopCalls := mock.callsFor("systemctl stop") if len(stopCalls) == 0 { t.Fatalf("expected systemctl stop SSH call, got %d calls: %v", len(mock.calls), mock.calls) } if !strings.Contains(stopCalls[0].cmd, "orca-alloc-webapp-*") { t.Errorf("expected 'orca-alloc-webapp-*' in cmd, got: %s", stopCalls[0].cmd) } if !strings.Contains(stopCalls[0].peer, "worker-1") { t.Errorf("expected peer to contain 'worker-1', got: %s", stopCalls[0].peer) } // Verify the DB status was updated. repo := store.NewJobRepo(db) job, err := repo.Get(ctx, jobID) if err != nil { t.Fatalf("get job: %v", err) } if job.Status != model.JobStatusStopped { t.Errorf("job status = %v, want stopped", job.Status) } } // TestJobStopSSHPeerOverride verifies that --peer bypasses the // alloc_history lookup and uses the given peer directly (REQ-158). func TestJobStopSSHPeerOverride(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() if err := runInit(discardWriter{}); err != nil { t.Fatalf("init: %v", err) } jobID := seedJob(t, "webapp2", model.JobStatusRunning) resetRootFlags(t) mock := &mockJobStopExec{} prev := jobStopTransportOverride jobStopTransportOverride = mock t.Cleanup(func() { jobStopTransportOverride = prev }) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop", jobID, "--peer", "10.0.0.5:22"}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job stop: %v", err) } stopCalls := mock.callsFor("systemctl stop") if len(stopCalls) == 0 { t.Fatalf("expected systemctl stop SSH call, got %d calls", len(mock.calls)) } if stopCalls[0].peer != "10.0.0.5:22" { t.Errorf("peer = %s, want 10.0.0.5:22", stopCalls[0].peer) } } // TestJobStopNoNodeFallback verifies that when no node is found in // alloc_history, the job is still stopped in the DB (soft stop // fallback) without attempting SSH (REQ-158). func TestJobStopNoNodeFallback(t *testing.T) { _, cleanup := initTestEnv(t) defer cleanup() if err := runInit(discardWriter{}); err != nil { t.Fatalf("init: %v", err) } jobID := seedJob(t, "localjob", model.JobStatusRunning) resetRootFlags(t) mock := &mockJobStopExec{} prev := jobStopTransportOverride jobStopTransportOverride = mock t.Cleanup(func() { jobStopTransportOverride = prev }) var buf bytes.Buffer rootCmd.SetOut(&buf) rootCmd.SetErr(&buf) rootCmd.SetArgs([]string{"job", "stop", jobID}) if err := rootCmd.Execute(); err != nil { t.Fatalf("job stop: %v", err) } // No SSH calls should have been made (no node found). if len(mock.calls) > 0 { t.Errorf("expected 0 SSH calls, got %d: %v", len(mock.calls), mock.calls) } // Verify the output mentions "DB only". if !strings.Contains(buf.String(), "DB only") { t.Errorf("output should mention 'DB only', got: %s", buf.String()) } }