orca/internal/engine/executor.go

package engine

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"os/exec"
	"sync"
	"time"

	"github.com/google/uuid"

	"git.cloudinit.dev/coreci/orca/internal/model"
	"git.cloudinit.dev/coreci/orca/internal/store"
)

type Executor struct {
	jobs  *store.JobRepo
	tasks *store.TaskRepo
	log   *slog.Logger
	mu    sync.Mutex
}

func NewExecutor(jobs *store.JobRepo, tasks *store.TaskRepo, log *slog.Logger) *Executor {
	if log == nil {
		log = slog.Default()
	}
	return &Executor{jobs: jobs, tasks: tasks, log: log}
}

// Submit is the dispatch-friendly entry point (v0.2 P02). It parses
// the spec bytes as a minimal TaskSpec and runs a single task under
// a fresh job. Returns the job ID. This is intentionally simpler
// than the v0.1 Run() entry point — the cross-node dispatch wire
// format is a flat task (one process), not a multi-task job.
//
// The spec format is a JSON object with at least:
//
//	{ "name": "...", "command": "...", "args": [...], "env": [...] }
//
// All fields except command are optional.
func (e *Executor) Submit(ctx context.Context, specBytes []byte) (string, error) {
	type wireSpec struct {
		Name    string   `json:"name"`
		Command string   `json:"command"`
		Args    []string `json:"args"`
		Env     []string `json:"env"`
	}
	var ws wireSpec
	if err := json.Unmarshal(specBytes, &ws); err != nil {
		return "", fmt.Errorf("Executor.Submit: parse: %w", err)
	}
	if ws.Command == "" {
		return "", errors.New("Executor.Submit: spec.command is required")
	}
	if ws.Name == "" {
		ws.Name = "dispatched"
	}
	job := &model.Job{
		ID:     uuid.NewString(),
		Spec:   string(specBytes),
		Status: model.JobStatusPending,
	}
	ts := TaskSpec{
		Name:    ws.Name,
		Command: ws.Command,
		Args:    ws.Args,
		Env:     ws.Env,
	}
	if err := e.Run(ctx, job, []TaskSpec{ts}); err != nil {
		return job.ID, err
	}
	return job.ID, nil
}

// Status returns the current state of a job for the Status dispatch
// endpoint. The returned string is one of: "pending", "running",
// "complete", "failed", "stopped". Maps to model.JobStatus* values.
func (e *Executor) Status(ctx context.Context, jobID string) (string, error) {
	if e.jobs == nil {
		return "", errors.New("Executor.Status: nil job repo")
	}
	j, err := e.jobs.Get(ctx, jobID)
	if err != nil {
		return "", err
	}
	return string(j.Status), nil
}

type TaskSpec struct {
	Name    string
	Command string
	Args    []string
	Env     []string
}

func (e *Executor) Run(ctx context.Context, job *model.Job, specs []TaskSpec) error {
	// REQ-156 / P07 T6: the mutex previously guarded the ENTIRE job
	// (insert + status transitions + task execution + wait). That
	// serialized unrelated jobs against each other and held the lock
	// across long-running child processes, blocking concurrent
	// Submit/Status/Run callers. The mutex is now scoped ONLY to the
	// DB inserts/updates (the part that must be serialized against
	// the single-writer SQLite connection pool — see store.Open
	// SetMaxOpenConns(1)). The task goroutines spawned below do not
	// hold e.mu; they share the per-job failure counter via a local
	// sync.Mutex.

	// Insert the job + flip to Running under the lock (serializes
	// the DB writes; the underlying SQLite busy_timeout(5000) +
	// SetMaxOpenConns(1) handles contention).
	e.mu.Lock()
	if err := e.jobs.Insert(ctx, job); err != nil {
		e.mu.Unlock()
		return err
	}
	if err := e.jobs.UpdateStatus(ctx, job.ID, model.JobStatusRunning, 0); err != nil {
		e.mu.Unlock()
		return err
	}
	e.mu.Unlock()

	// Task execution runs WITHOUT e.mu — concurrent jobs (and
	// concurrent Submit/Status callers) are no longer blocked by a
	// long-running child process.
	var (
		wg          sync.WaitGroup
		failedCount int
		mu          sync.Mutex
	)
	for _, ts := range specs {
		wg.Add(1)
		go func(ts TaskSpec) {
			defer wg.Done()
			if err := e.runOne(ctx, job, ts); err != nil {
				mu.Lock()
				failedCount++
				e.log.Error("task failed",
					slog.String("job_id", job.ID),
					slog.String("task", ts.Name),
					slog.String("error", err.Error()))
				mu.Unlock()
			}
		}(ts)
	}
	wg.Wait()

	// Final status transition under the lock (the DB write is the
	// only thing that needs serialization).
	e.mu.Lock()
	defer e.mu.Unlock()
	if failedCount > 0 {
		if err := e.jobs.UpdateStatus(ctx, job.ID, model.JobStatusFailed, 1); err != nil {
			return err
		}
		return fmt.Errorf("%d/%d tasks failed", failedCount, len(specs))
	}
	if err := e.jobs.UpdateStatus(ctx, job.ID, model.JobStatusComplete, 0); err != nil {
		return err
	}
	return nil
}

func (e *Executor) runOne(ctx context.Context, job *model.Job, ts TaskSpec) error {
	task := &model.Task{
		ID:      uuid.NewString(),
		JobID:   job.ID,
		Command: ts.Command,
		Args:    ts.Args,
		Env:     ts.Env,
		Status:  model.TaskStatusPending,
	}
	if err := e.tasks.Insert(ctx, task); err != nil {
		return err
	}

	cmd := exec.CommandContext(ctx, ts.Command, ts.Args...)
	cmd.Env = append(cmd.Environ(), ts.Env...)
	// WaitDelay (Go 1.25+) bounds the time spent waiting on a child process
	// that fails to exit after the context is canceled.
	cmd.WaitDelay = 5 * time.Second

	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	if err := cmd.Start(); err != nil {
		_ = e.tasks.UpdateKilled(ctx, task.ID)
		return fmt.Errorf("start: %w", err)
	}

	if err := e.tasks.UpdateRunning(ctx, task.ID, cmd.Process.Pid); err != nil {
		e.log.Warn("update running failed", slog.String("error", err.Error()))
	}

	e.log.Info("task started",
		slog.String("job_id", job.ID),
		slog.String("task", ts.Name),
		slog.Int("pid", cmd.Process.Pid))

	done := make(chan error, 1)
	go func() { done <- cmd.Wait() }()

	select {
	case err := <-done:
		exitCode := 0
		if err != nil {
			if ee, ok := err.(*exec.ExitError); ok {
				exitCode = ee.ExitCode()
			} else {
				exitCode = 1
			}
		}
		_ = e.tasks.UpdateDone(ctx, task.ID, exitCode, stdout.String(), stderr.String())
		if err != nil {
			return err
		}
		return nil
	case <-ctx.Done():
		// WaitDelay (set above) gives the process a grace period to exit
		// cleanly before being killed.
		_ = e.tasks.UpdateKilled(ctx, task.ID)
		return ctx.Err()
	}
}