orca/internal/cli/job_test.go

package cli

import (
	"bytes"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/model"
	"git.cloudinit.dev/coreci/orca/internal/store"
)

func writeJobSpec(t *testing.T, content string) string {
	t.Helper()
	dir := t.TempDir()
	p := filepath.Join(dir, "spec.hcl")
	if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
		t.Fatalf("write spec: %v", err)
	}
	return p
}

const trueJobSpec = `job "true" {}
task "t" {
  command = "/bin/true"
}
`

const falseJobSpec = `job "false" {}
task "t" {
  command = "/bin/false"
}
`

func TestJobRunComplete(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	spec := writeJobSpec(t, trueJobSpec)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "run", spec})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job run: %v", err)
	}
	if !strings.Contains(buf.String(), "Job complete") {
		t.Errorf("job run output unexpected: %s", buf.String())
	}
}

func TestJobRunCompleteJSON(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	spec := writeJobSpec(t, trueJobSpec)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "run", spec, "--json"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job run --json: %v", err)
	}
	var result map[string]any
	if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &result); err != nil {
		t.Fatalf("unmarshal job run json: %v\n%s", err, buf.String())
	}
	if result["status"] != "complete" {
		t.Errorf("job run --json status = %v, want complete", result["status"])
	}
}

func TestJobRunFailed(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	spec := writeJobSpec(t, falseJobSpec)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "run", spec})
	if err := rootCmd.Execute(); err == nil {
		t.Fatal("expected error for failing job, got nil")
	}
}

func TestJobRunFailedJSON(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	spec := writeJobSpec(t, falseJobSpec)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "run", spec, "--json"})
	if err := rootCmd.Execute(); err == nil {
		t.Fatal("expected error for failing job --json, got nil")
	}
	if !strings.Contains(buf.String(), "failed") {
		t.Errorf("job run --json failed output unexpected: %s", buf.String())
	}
}

func TestJobRunMissingSpecFile(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "run", "/nonexistent/spec.hcl"})
	if err := rootCmd.Execute(); err == nil {
		t.Fatal("expected error for missing spec file, got nil")
	}
}

func TestJobListEmpty(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "list"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job list: %v", err)
	}
	if !strings.Contains(buf.String(), "No jobs") {
		t.Errorf("job list empty output unexpected: %s", buf.String())
	}
}

func TestJobListJSONEmpty(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "list", "--json"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job list --json: %v", err)
	}
	var jobs []map[string]any
	if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &jobs); err != nil {
		t.Fatalf("unmarshal job list json: %v\n%s", err, buf.String())
	}
	if len(jobs) != 0 {
		t.Errorf("job list --json empty = %d jobs, want 0", len(jobs))
	}
}

func TestJobListAfterRun(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	spec := writeJobSpec(t, trueJobSpec)
	resetRootFlags(t)
	rootCmd.SetArgs([]string{"job", "run", spec})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job run: %v", err)
	}
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "list"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job list: %v", err)
	}
	out := buf.String()
	if !strings.Contains(out, "true") {
		t.Errorf("job list missing job name: %s", out)
	}
}

func TestJobStop(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	jobID := seedJob(t, "stopper", model.JobStatusRunning)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop", jobID})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job stop: %v", err)
	}
	if !strings.Contains(buf.String(), "Job stopped") {
		t.Errorf("job stop output unexpected: %s", buf.String())
	}
}

func TestJobStopJSON(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	jobID := seedJob(t, "jsonstopper", model.JobStatusRunning)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop", jobID, "--json"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job stop --json: %v", err)
	}
	var result map[string]any
	if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &result); err != nil {
		t.Fatalf("unmarshal job stop json: %v\n%s", err, buf.String())
	}
	if result["status"] != "stopped" {
		t.Errorf("job stop --json status = %v, want stopped", result["status"])
	}
}

func TestJobStopNotFound(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop", "nonexistent-id"})
	if err := rootCmd.Execute(); err == nil {
		t.Fatal("expected error for job stop not found, got nil")
	}
}

func TestJobStopMissingID(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop"})
	if err := rootCmd.Execute(); err == nil {
		t.Fatal("expected error for job stop without id, got nil")
	}
}

func TestJobLogsEmpty(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	jobID := seedJob(t, "logger", model.JobStatusComplete)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "logs", jobID})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job logs: %v", err)
	}
	if !strings.Contains(buf.String(), "No tasks") {
		t.Errorf("job logs empty output unexpected: %s", buf.String())
	}
}

func TestJobLogsJSONEmpty(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	jobID := seedJob(t, "jsonlogger", model.JobStatusComplete)
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "logs", jobID, "--json"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job logs --json: %v", err)
	}
	var tasks []map[string]any
	if err := json.Unmarshal(bytes.TrimSpace(buf.Bytes()), &tasks); err != nil {
		t.Fatalf("unmarshal job logs json: %v\n%s", err, buf.String())
	}
	if len(tasks) != 0 {
		t.Errorf("job logs --json empty = %d tasks, want 0", len(tasks))
	}
}

func TestJobLogsMissingID(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	resetRootFlags(t)
	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "logs"})
	if err := rootCmd.Execute(); err == nil {
		t.Fatal("expected error for job logs without id, got nil")
	}
}

func seedJob(t *testing.T, name string, status model.JobStatus) string {
	t.Helper()
	db, err := store.Open(certpaths.DBPath())
	if err != nil {
		t.Fatalf("open db: %v", err)
	}
	defer db.Close()
	repo := store.NewJobRepo(db)
	j := &model.Job{
		ID:     "job-" + name,
		Name:   name,
		Spec:   "spec.hcl",
		Status: status,
	}
	if err := repo.Insert(t.Context(), j); err != nil {
		t.Fatalf("insert job: %v", err)
	}
	return j.ID
}


// mockJobStopExec is a record-and-replay SSH execer for `orca job stop`
// tests (same pattern as mockDrainExec / mockLogsExec).
type mockJobStopExec struct {
	mu        sync.Mutex
	responses []jobStopMockResp
	calls     []jobStopMockCall
}

type jobStopMockResp struct {
	match string
	out   string
	exit  int
}

type jobStopMockCall struct {
	peer string
	cmd  string
}

func (m *mockJobStopExec) Exec(_ context.Context, peer, cmd string) ([]byte, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.calls = append(m.calls, jobStopMockCall{peer: peer, cmd: cmd})
	for _, r := range m.responses {
		if r.match == "" || strings.Contains(cmd, r.match) {
			return []byte(r.out), nil
		}
	}
	return []byte(""), nil
}

func (m *mockJobStopExec) callsFor(match string) []jobStopMockCall {
	m.mu.Lock()
	defer m.mu.Unlock()
	var out []jobStopMockCall
	for _, c := range m.calls {
		if strings.Contains(c.cmd, match) {
			out = append(out, c)
		}
	}
	return out
}

// TestJobStopSSH verifies that `orca job stop` sends a real
// 'systemctl stop' via SSH to the target node when the job has a
// recorded allocation history (REQ-158, P09 T6).
func TestJobStopSSH(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	if err := runInit(discardWriter{}); err != nil {
		t.Fatalf("init: %v", err)
	}

	// Seed a node and a job, then record an alloc_history entry
	// linking the job to the node.
	nodeID := seedNode(t, "worker-1", "worker-1:8443")
	jobID := seedJob(t, "webapp", model.JobStatusRunning)

	db, err := store.Open(certpaths.DBPath())
	if err != nil {
		t.Fatalf("open db: %v", err)
	}
	defer db.Close()
	hist := store.NewAllocHistoryRepo(db)
	ctx := context.Background()
	if err := hist.EnsureSchema(ctx); err != nil {
		t.Fatalf("ensure schema: %v", err)
	}
	if err := hist.Record(ctx, store.AllocHistoryEntry{
		AllocID:   "default/webapp-0",
		JobID:     jobID,
		NodeID:    nodeID,
		Namespace: "default",
		ToState:   "created",
		Timestamp: time.Now().UTC(),
	}); err != nil {
		t.Fatalf("record alloc history: %v", err)
	}

	// Wire the mock SSH transport (must be after resetRootFlags so
	// resetCommandFlags doesn't nil it out).
	resetRootFlags(t)
	mock := &mockJobStopExec{}
	prev := jobStopTransportOverride
	jobStopTransportOverride = mock
	t.Cleanup(func() { jobStopTransportOverride = prev })

	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop", jobID})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job stop: %v", err)
	}

	// Verify systemctl stop was called via SSH.
	stopCalls := mock.callsFor("systemctl stop")
	if len(stopCalls) == 0 {
		t.Fatalf("expected systemctl stop SSH call, got %d calls: %v", len(mock.calls), mock.calls)
	}
	if !strings.Contains(stopCalls[0].cmd, "orca-alloc-webapp-*") {
		t.Errorf("expected 'orca-alloc-webapp-*' in cmd, got: %s", stopCalls[0].cmd)
	}
	if !strings.Contains(stopCalls[0].peer, "worker-1") {
		t.Errorf("expected peer to contain 'worker-1', got: %s", stopCalls[0].peer)
	}

	// Verify the DB status was updated.
	repo := store.NewJobRepo(db)
	job, err := repo.Get(ctx, jobID)
	if err != nil {
		t.Fatalf("get job: %v", err)
	}
	if job.Status != model.JobStatusStopped {
		t.Errorf("job status = %v, want stopped", job.Status)
	}
}

// TestJobStopSSHPeerOverride verifies that --peer bypasses the
// alloc_history lookup and uses the given peer directly (REQ-158).
func TestJobStopSSHPeerOverride(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	if err := runInit(discardWriter{}); err != nil {
		t.Fatalf("init: %v", err)
	}

	jobID := seedJob(t, "webapp2", model.JobStatusRunning)

	resetRootFlags(t)
	mock := &mockJobStopExec{}
	prev := jobStopTransportOverride
	jobStopTransportOverride = mock
	t.Cleanup(func() { jobStopTransportOverride = prev })

	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop", jobID, "--peer", "10.0.0.5:22"})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job stop: %v", err)
	}

	stopCalls := mock.callsFor("systemctl stop")
	if len(stopCalls) == 0 {
		t.Fatalf("expected systemctl stop SSH call, got %d calls", len(mock.calls))
	}
	if stopCalls[0].peer != "10.0.0.5:22" {
		t.Errorf("peer = %s, want 10.0.0.5:22", stopCalls[0].peer)
	}
}

// TestJobStopNoNodeFallback verifies that when no node is found in
// alloc_history, the job is still stopped in the DB (soft stop
// fallback) without attempting SSH (REQ-158).
func TestJobStopNoNodeFallback(t *testing.T) {
	_, cleanup := initTestEnv(t)
	defer cleanup()
	if err := runInit(discardWriter{}); err != nil {
		t.Fatalf("init: %v", err)
	}

	jobID := seedJob(t, "localjob", model.JobStatusRunning)

	resetRootFlags(t)
	mock := &mockJobStopExec{}
	prev := jobStopTransportOverride
	jobStopTransportOverride = mock
	t.Cleanup(func() { jobStopTransportOverride = prev })

	var buf bytes.Buffer
	rootCmd.SetOut(&buf)
	rootCmd.SetErr(&buf)
	rootCmd.SetArgs([]string{"job", "stop", jobID})
	if err := rootCmd.Execute(); err != nil {
		t.Fatalf("job stop: %v", err)
	}

	// No SSH calls should have been made (no node found).
	if len(mock.calls) > 0 {
		t.Errorf("expected 0 SSH calls, got %d: %v", len(mock.calls), mock.calls)
	}

	// Verify the output mentions "DB only".
	if !strings.Contains(buf.String(), "DB only") {
		t.Errorf("output should mention 'DB only', got: %s", buf.String())
	}
}