cf3d98eb2b
R-022: orca job run now deploys to remote nodes via scheduler -> emitter -> SSH-push. Local exec fallback only when no remote nodes registered. jobspec parser (REQ-152): - schedule: and timeout: now parsed (were silently dropped) - DaemonSet Count no longer defaults to 1 (was breaking DaemonSet) - restart: policy translated to systemd Restart=/StartLimitBurst - job lint: advisory warnings for cron/health/update/affinity (honest) scheduler wiring (REQ-151, C-44): - new internal/cli/job_dispatch.go: dispatchDecision + deployRemote - scheduler.Schedule evaluates constraints/capacity/affinity - --target overrides scheduler (manual pinning) - local fallback only when len(ready non-localhost nodes)==0 - C-44: SSH-push failure returns error (no silent local fallback) - systemd-analyze verify on rendered unit before deploy Tests: 22 new test functions covering scheduler, parser, C-44, local fallback, target override, systemd-analyze skip, restart directives. ---ci--- project: orca phase: 3 milestone: v0.13 status: complete requirements: covered: [151, 152] ---/ci---
297 lines
11 KiB
Go
297 lines
11 KiB
Go
package emitter
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
|
)
|
|
|
|
// SystemdEmitter is the Emitter implementation for the "process"
|
|
// runtime. It renders the systemd unit file for the workload,
|
|
// including the lifecycle hooks (P04) and the R-007 socket plumbing
|
|
// (P08).
|
|
//
|
|
// Lifecycle hooks map to systemd semantics (PRD §10.1):
|
|
//
|
|
// - lifecycle.pre_stop → ExecStop= (the command run on stop; systemd
|
|
// runs ExecStop, then kills the main process after the deadline).
|
|
// - lifecycle.post_start → ExecStartPost= (runs after the main
|
|
// process starts).
|
|
//
|
|
// systemd has no ExecStartPre equivalent for a "pre_start" hook; the
|
|
// spec does not define pre_start (only pre_stop and post_start per
|
|
// PRD §10.1), so no mapping is needed.
|
|
//
|
|
// The unit name carries the `orca-v1-` prefix per the dual-write
|
|
// window (REQ-090) so the v0.9 SSH-push path does not collide with the
|
|
// v0.8 daemon's `orca-<job>.service` units during the migration
|
|
// window.
|
|
//
|
|
// Later phases extend this emitter:
|
|
//
|
|
// - v0.10-P03: secrets via EnvironmentFile= + LoadCredential=
|
|
type SystemdEmitter struct{}
|
|
|
|
// unitNamePrefix is the v0.9 SSH-push unit-name prefix. The v0.8
|
|
// daemon uses `orca-<job>.service`; the v0.9 path uses
|
|
// `orca-v1-<spec.Name>.service` so the two never overlap (REQ-090,
|
|
// I-C-006). The prefix is load-bearing — do not change it without
|
|
// updating the dual-write window contract.
|
|
const unitNamePrefix = "orca-v1-"
|
|
|
|
// Render renders the systemd unit file for a process-runtime workload.
|
|
//
|
|
// When the spec has no Tasks (the single-process case, the historical
|
|
// shape), the unit name is
|
|
// /etc/systemd/system/<unitNamePrefix><spec.Name>.service and the
|
|
// content is a [Service] block with ExecStart, optional ExecStartPost
|
|
// (lifecycle.post_start), optional ExecStop (lifecycle.pre_stop), and
|
|
// the R-007 socket-plumbing lines (RuntimeDirectory=, optional
|
|
// TCP-bind ExecStartPre). Mode is 0644.
|
|
//
|
|
// When the spec has a task group (P06, spec.Tasks non-empty), the
|
|
// alloc is multi-process and Render emits one systemd unit per task
|
|
// (`orca-v1-alloc-<alloc-id>-<task-name>.service`) plus a single
|
|
// grouping target unit (`orca-v1-alloc-<alloc-id>.target`) that
|
|
// starts/stops all tasks together. Each per-task unit carries
|
|
// `PartOf=orca-v1-alloc-<alloc-id>.target` and is
|
|
// `WantedBy=multi-user.target` so the task starts at boot. Tasks
|
|
// that omit their own runtime inherit the top-level spec.Runtime as
|
|
// the per-group default.
|
|
//
|
|
// The rendered shape (single-process) is:
|
|
//
|
|
// [Service]
|
|
// ExecStart=<runtime command>
|
|
// ExecStartPost=<post_start command 1>
|
|
// ExecStartPost=<post_start command 2>
|
|
// ExecStop=<pre_stop command 1>
|
|
// ExecStop=<pre_stop command 2>
|
|
// RuntimeDirectory=orca/alloc-<alloc-id>
|
|
// # socket: /run/orca/alloc-<alloc-id>/port-<name>.sock
|
|
// ExecStartPre=/bin/echo orca: bind 127.0.0.1 port <name> (tcp, R-007 opt-in)
|
|
//
|
|
// Returns an error if the spec is nil, the spec is missing its name,
|
|
// the runtime block is nil, or the runtime command is empty (a
|
|
// workload with no command has nothing to ExecStart). For task groups,
|
|
// returns an error if any task has no resolvable runtime command.
|
|
func (SystemdEmitter) Render(spec *jobspec.WorkloadSpec, node *Node) ([]File, error) {
|
|
if spec == nil {
|
|
return nil, errors.New("emitter/systemd: spec is nil")
|
|
}
|
|
if strings.TrimSpace(spec.Name) == "" {
|
|
return nil, errors.New("emitter/systemd: spec name is empty")
|
|
}
|
|
if len(spec.Tasks) > 0 {
|
|
return renderTaskGroup(spec, node)
|
|
}
|
|
if spec.Runtime == nil {
|
|
return nil, errors.New("emitter/systemd: runtime block is nil")
|
|
}
|
|
if strings.TrimSpace(spec.Runtime.Command) == "" {
|
|
return nil, errors.New("emitter/systemd: runtime command is empty")
|
|
}
|
|
path := fmt.Sprintf("/etc/systemd/system/%s%s.service", unitNamePrefix, spec.Name)
|
|
content := renderSystemdUnit(spec)
|
|
return []File{{Path: path, Content: content, Mode: "0644"}}, nil
|
|
}
|
|
|
|
// renderTaskGroup renders one systemd unit per task plus the grouping
|
|
// target unit. Each task's runtime falls back to the top-level
|
|
// spec.Runtime when the task omits its own. Tasks with no resolvable
|
|
// command (no task.Command, no task.Runtime.Command, no top-level
|
|
// Runtime) return an error.
|
|
func renderTaskGroup(spec *jobspec.WorkloadSpec, node *Node) ([]File, error) {
|
|
allocID := spec.Name
|
|
targetUnit := fmt.Sprintf("%salloc-%s.target", unitNamePrefix, allocID)
|
|
targetPath := fmt.Sprintf("/etc/systemd/system/%s", targetUnit)
|
|
var files []File
|
|
for _, task := range spec.Tasks {
|
|
rt := taskRuntime(spec, &task)
|
|
if rt == nil {
|
|
return nil, fmt.Errorf("emitter/systemd: task %q has no runtime (set tasks[].runtime or top-level runtime)", task.Name)
|
|
}
|
|
cmd := taskCommand(spec, &task, rt)
|
|
if strings.TrimSpace(cmd) == "" {
|
|
return nil, fmt.Errorf("emitter/systemd: task %q command is empty", task.Name)
|
|
}
|
|
unitName := fmt.Sprintf("%salloc-%s-%s.service", unitNamePrefix, allocID, task.Name)
|
|
path := fmt.Sprintf("/etc/systemd/system/%s", unitName)
|
|
content := renderTaskUnit(spec, &task, rt, cmd, targetUnit)
|
|
files = append(files, File{Path: path, Content: content, Mode: "0644"})
|
|
}
|
|
files = append(files, File{
|
|
Path: targetPath,
|
|
Content: renderTargetUnit(targetUnit, spec, allocID),
|
|
Mode: "0644",
|
|
})
|
|
return files, nil
|
|
}
|
|
|
|
// taskRuntime returns the effective runtime for a task: the task's own
|
|
// runtime when set, otherwise the top-level spec.Runtime (the per-group
|
|
// default). Returns nil when neither is set.
|
|
func taskRuntime(spec *jobspec.WorkloadSpec, task *jobspec.TaskGroupTask) *jobspec.RuntimeBlock {
|
|
if task.Runtime != nil {
|
|
return task.Runtime
|
|
}
|
|
return spec.Runtime
|
|
}
|
|
|
|
// taskCommand returns the ExecStart command for a task. A task-level
|
|
// Command takes precedence; otherwise the task's runtime command is
|
|
// used; otherwise the top-level runtime command is used. Returns an
|
|
// empty string when none is set.
|
|
func taskCommand(spec *jobspec.WorkloadSpec, task *jobspec.TaskGroupTask, rt *jobspec.RuntimeBlock) string {
|
|
if strings.TrimSpace(task.Command) != "" {
|
|
return task.Command
|
|
}
|
|
if rt != nil && strings.TrimSpace(rt.Command) != "" {
|
|
return rt.Command
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// renderTaskUnit renders a single per-task systemd [Unit]+[Service]
|
|
// block. The unit is `PartOf=` the alloc target and
|
|
// `WantedBy=multi-user.target` so it starts at boot and stops with the
|
|
// group. The [Service] block carries the task's ExecStart and the
|
|
// socket-plumbing lines derived from the spec's ports.
|
|
func renderTaskUnit(spec *jobspec.WorkloadSpec, task *jobspec.TaskGroupTask, rt *jobspec.RuntimeBlock, cmd, targetUnit string) string {
|
|
var b strings.Builder
|
|
b.WriteString("[Unit]\n")
|
|
b.WriteString(fmt.Sprintf("Description=orca alloc task %s\n", task.Name))
|
|
b.WriteString(fmt.Sprintf("PartOf=%s\n", targetUnit))
|
|
b.WriteString("\n[Service]\n")
|
|
b.WriteString(fmt.Sprintf("ExecStart=%s\n", cmd))
|
|
for _, line := range renderRestartDirectives(spec) {
|
|
b.WriteString(line)
|
|
b.WriteString("\n")
|
|
}
|
|
for _, line := range (SocketEmitter{}).RenderSocketLines(spec) {
|
|
b.WriteString(line)
|
|
b.WriteString("\n")
|
|
}
|
|
b.WriteString("\n[Install]\n")
|
|
b.WriteString("WantedBy=multi-user.target\n")
|
|
return b.String()
|
|
}
|
|
|
|
// renderTargetUnit renders the grouping target unit
|
|
// (`orca-v1-alloc-<alloc-id>.target`) that starts/stops all tasks
|
|
// together. The [Unit] block lists every per-task unit under Wants=
|
|
// so `systemctl start <target>` brings them all up, and
|
|
// `systemctl stop <target>` tears them down (PartOf= propagates stop).
|
|
func renderTargetUnit(targetUnit string, spec *jobspec.WorkloadSpec, allocID string) string {
|
|
var b strings.Builder
|
|
b.WriteString("[Unit]\n")
|
|
b.WriteString(fmt.Sprintf("Description=orca alloc %s task group\n", allocID))
|
|
for _, task := range spec.Tasks {
|
|
b.WriteString(fmt.Sprintf("Wants=%salloc-%s-%s.service\n", unitNamePrefix, allocID, task.Name))
|
|
}
|
|
b.WriteString("\n[Install]\n")
|
|
b.WriteString("WantedBy=multi-user.target\n")
|
|
return b.String()
|
|
}
|
|
|
|
// renderSystemdUnit renders the full [Service] block for the spec,
|
|
// including ExecStart, lifecycle hooks (ExecStartPost, ExecStop), and
|
|
// the R-007 socket-plumbing lines (RuntimeDirectory=, optional
|
|
// TCP-bind ExecStartPre). The output is a single string with a
|
|
// trailing newline per line.
|
|
func renderSystemdUnit(spec *jobspec.WorkloadSpec) string {
|
|
var b strings.Builder
|
|
b.WriteString("[Service]\n")
|
|
b.WriteString(fmt.Sprintf("ExecStart=%s\n", spec.Runtime.Command))
|
|
// Restart policy (REQ-152/T3): translate spec.Restart into the
|
|
// systemd Restart= / StartLimitBurst= / StartLimitIntervalSec=
|
|
// (or RestartSec=) directives. See renderRestartDirectives.
|
|
for _, line := range renderRestartDirectives(spec) {
|
|
b.WriteString(line)
|
|
b.WriteString("\n")
|
|
}
|
|
// Lifecycle: post_start → ExecStartPost (runs after start).
|
|
for _, cmd := range lifecyclePostStart(spec) {
|
|
b.WriteString(fmt.Sprintf("ExecStartPost=%s\n", cmd))
|
|
}
|
|
// Lifecycle: pre_stop → ExecStop (runs before the process is killed).
|
|
for _, cmd := range lifecyclePreStop(spec) {
|
|
b.WriteString(fmt.Sprintf("ExecStop=%s\n", cmd))
|
|
}
|
|
// R-007 socket plumbing: RuntimeDirectory= per port + optional
|
|
// TCP-bind ExecStartPre.
|
|
for _, line := range (SocketEmitter{}).RenderSocketLines(spec) {
|
|
b.WriteString(line)
|
|
b.WriteString("\n")
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// lifecyclePostStart returns the post_start lifecycle commands for
|
|
// the spec, or nil when the spec has no lifecycle block or no
|
|
// post_start commands.
|
|
func lifecyclePostStart(spec *jobspec.WorkloadSpec) []string {
|
|
if spec.Lifecycle == nil {
|
|
return nil
|
|
}
|
|
return spec.Lifecycle.PostStart
|
|
}
|
|
|
|
// lifecyclePreStop returns the pre_stop lifecycle commands for the
|
|
// spec, or nil when the spec has no lifecycle block or no pre_stop
|
|
// commands.
|
|
func lifecyclePreStop(spec *jobspec.WorkloadSpec) []string {
|
|
if spec.Lifecycle == nil {
|
|
return nil
|
|
}
|
|
return spec.Lifecycle.PreStop
|
|
}
|
|
|
|
// renderRestartDirectives translates the spec.Restart block into the
|
|
// systemd [Service]/[Unit] restart directives (REQ-152/T3):
|
|
//
|
|
// - never → Restart=no (explicit; omitted when Restart is nil)
|
|
// - on-failure → Restart=on-failure + StartLimitBurst=<MaxRetries>
|
|
// - service → Restart=always + StartLimitBurst=<MaxRetries> (when
|
|
// MaxRetries > 0)
|
|
//
|
|
// The delay (a duration string like "5s") maps to StartLimitIntervalSec=
|
|
// when set; for the on-failure/service modes a non-empty delay also
|
|
// emits RestartSec=<delay> so systemd backs off between restart attempts.
|
|
// A nil Restart block produces no directives (the caller's default
|
|
// applies — for a [Service] with no Restart= that is Restart=no).
|
|
func renderRestartDirectives(spec *jobspec.WorkloadSpec) []string {
|
|
if spec.Restart == nil {
|
|
return nil
|
|
}
|
|
var out []string
|
|
switch spec.Restart.Mode {
|
|
case "never", "":
|
|
out = append(out, "Restart=no")
|
|
case "on-failure":
|
|
out = append(out, "Restart=on-failure")
|
|
if spec.Restart.MaxRetries > 0 {
|
|
out = append(out, fmt.Sprintf("StartLimitBurst=%d", spec.Restart.MaxRetries))
|
|
}
|
|
case "service":
|
|
out = append(out, "Restart=always")
|
|
if spec.Restart.MaxRetries > 0 {
|
|
out = append(out, fmt.Sprintf("StartLimitBurst=%d", spec.Restart.MaxRetries))
|
|
}
|
|
default:
|
|
// Unknown mode: emit Restart=no so the unit is explicit and
|
|
// systemd-analyze verify does not reject an unknown value.
|
|
out = append(out, "Restart=no")
|
|
}
|
|
if spec.Restart.Delay != "" {
|
|
switch spec.Restart.Mode {
|
|
case "on-failure", "service":
|
|
out = append(out, fmt.Sprintf("RestartSec=%s", spec.Restart.Delay))
|
|
out = append(out, fmt.Sprintf("StartLimitIntervalSec=%s", spec.Restart.Delay))
|
|
}
|
|
}
|
|
return out
|
|
}
|