docs(milestone): complete scheduling-streaming (v0.3)

---ci---
project: orca
phase: 3
milestone: v0.3
status: complete
requirements:
  covered: [REQ-022, REQ-030, REQ-032]
  partial: []
---/ci---

v0.3 milestone merged to main. Includes all v0.2 work (P08-P10) that
was previously on the milestone branch but not yet merged to main, plus
the v0.3 completion work (iter.Seq streaming + doctor network/db).

v0.2 phases included: P08 (mTLS), P09 (scheduling), P10 (security scan).
v0.3 phases: P0 (pre-execution), P1 (iter.Seq streaming), P2 (doctor),
P3 (final review+ship).

Total: 40 requirements, all complete. No new go.mod dependencies.
Full test suite passes under -race. gofmt + go vet clean.
This commit is contained in:
Jon Chery
2026-08-01 20:06:47 +00:00
parent f503404dda
commit df58bc25a3
52 changed files with 5488 additions and 198 deletions
+211
View File
@@ -0,0 +1,211 @@
// Package engine — dispatcher.go implements the cross-node job
// dispatch logic (v0.2 P02). The dispatcher is the bridge between
// the local "should I run this?" decision (scheduler.PickNode) and
// the remote "please run this" call (transport.DispatchClient).
//
// Flow:
//
// 1. Receive a job spec (HCL bytes from the CLI).
// 2. Parse the spec into a JobSpec (cpu/mem/disk).
// 3. Check local capacity. If it fits, run locally via the local
// executor. If not, pick a peer and dispatch.
// 4. Return the job ID and the node that actually accepted it.
package engine
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"sync"
"git.cloudinit.dev/coreci/orca/internal/store"
"git.cloudinit.dev/coreci/orca/internal/transport"
)
// Dispatcher is the public surface; constructed via NewDispatcher.
type Dispatcher struct {
log *slog.Logger
capacity *store.CapacityRepo
peers *PeerRegistry
executor LocalExecutor
dedupe *transport.IdempotencyStore
mu sync.Mutex
}
// LocalExecutor is the contract the dispatcher uses to run jobs on
// the local node. The engine.Executor satisfies this.
type LocalExecutor interface {
Submit(ctx context.Context, specBytes []byte) (jobID string, err error)
Status(ctx context.Context, jobID string) (state string, err error)
}
// NewDispatcher builds a Dispatcher.
func NewDispatcher(log *slog.Logger, capacity *store.CapacityRepo, peers *PeerRegistry, exec LocalExecutor) *Dispatcher {
if log == nil {
log = slog.Default()
}
return &Dispatcher{
log: log,
capacity: capacity,
peers: peers,
executor: exec,
dedupe: transport.NewIdempotencyStore(),
}
}
// Dedupe exposes the in-memory dedupe store for testing.
func (d *Dispatcher) Dedupe() *transport.IdempotencyStore { return d.dedupe }
// Submit runs the spec locally if it fits, otherwise dispatches to a
// peer. Returns the (jobID, chosenNodeID) pair. If `target` is
// non-empty, it overrides bin-packing.
func (d *Dispatcher) Submit(ctx context.Context, target string, specBytes []byte, idempotencyKey string) (jobID, nodeID string, err error) {
if len(specBytes) == 0 {
return "", "", errors.New("Dispatcher.Submit: empty spec")
}
if idempotencyKey != "" {
if jid, ok := d.dedupe.Get(idempotencyKey); ok {
return jid, "self", nil
}
}
parsed, err := parseInlineSpec(specBytes)
if err != nil {
return "", "", fmt.Errorf("Dispatcher.Submit: parse spec: %w", err)
}
// 1. Explicit target: dispatch there.
if target != "" {
return d.dispatchTo(ctx, target, specBytes, idempotencyKey)
}
// 2. Check local capacity.
if d.capacity != nil {
local, err := d.capacity.Get(ctx, "self")
if err == nil && parsed.Fits(local) {
jid, lerr := d.executor.Submit(ctx, specBytes)
if lerr != nil {
return "", "", fmt.Errorf("Dispatcher.Submit: local: %w", lerr)
}
if idempotencyKey != "" {
d.dedupe.Put(idempotencyKey, jid)
}
d.log.Info("dispatch.local",
slog.String("event", "dispatch.local"),
slog.String("job_id", jid),
slog.String("node_id", "self"),
)
return jid, "self", nil
}
}
// 3. Pick a peer.
if d.peers == nil {
return "", "", errors.New("Dispatcher.Submit: no local capacity and no peer registry")
}
peers, err := d.peers.All(ctx)
if err != nil {
return "", "", fmt.Errorf("Dispatcher.Submit: list peers: %w", err)
}
if len(peers) == 0 {
return "", "", errors.New("Dispatcher.Submit: no peers registered")
}
var caps []*store.NodeCapacity
for _, p := range peers {
caps = append(caps, p.Capacity)
}
best, _, err := PickNode(parsed, caps)
if err != nil {
return "", "", fmt.Errorf("Dispatcher.Submit: %w", err)
}
var chosen *Peer
for _, p := range peers {
if p.NodeID == best.NodeID {
chosen = p
break
}
}
if chosen == nil {
return "", "", fmt.Errorf("Dispatcher.Submit: chosen node %s has no peer record", best.NodeID)
}
return d.dispatchToPeer(ctx, chosen, specBytes, idempotencyKey)
}
// dispatchTo sends a Submit to a specific node id (looked up in the peer registry).
func (d *Dispatcher) dispatchTo(ctx context.Context, targetNode string, specBytes []byte, idempotencyKey string) (string, string, error) {
if d.peers == nil {
return "", "", errors.New("dispatchTo: no peer registry")
}
peers, err := d.peers.All(ctx)
if err != nil {
return "", "", fmt.Errorf("dispatchTo: list peers: %w", err)
}
for _, p := range peers {
if p.NodeID == targetNode {
return d.dispatchToPeer(ctx, p, specBytes, idempotencyKey)
}
}
return "", "", fmt.Errorf("dispatchTo: target node %q not found in peer registry", targetNode)
}
// dispatchToPeer opens an mTLS client and calls Submit on the peer.
func (d *Dispatcher) dispatchToPeer(ctx context.Context, p *Peer, specBytes []byte, idempotencyKey string) (string, string, error) {
if p.CAPath == "" || p.ServerName == "" {
return "", "", fmt.Errorf("dispatchToPeer: peer %s missing CA or server name", p.NodeID)
}
client, err := transport.NewDispatchClient(p.CAPath, p.ServerName, "https://"+p.Address)
if err != nil {
return "", "", fmt.Errorf("dispatchToPeer: %w", err)
}
resp, err := client.Submit(ctx, specBytes, idempotencyKey)
if err != nil {
return "", "", fmt.Errorf("dispatchToPeer: %w", err)
}
if idempotencyKey != "" {
d.dedupe.Put(idempotencyKey, resp.JobID)
}
d.log.Info("dispatch.peer",
slog.String("event", "dispatch.peer"),
slog.String("job_id", resp.JobID),
slog.String("node_id", p.NodeID),
)
return resp.JobID, p.NodeID, nil
}
// LocalSubmit / LocalStatus satisfy the transport.Dispatcher
// interface (the server-side counterpart of DispatchClient).
func (d *Dispatcher) LocalSubmit(ctx context.Context, specBytes []byte) (string, error) {
if d.executor == nil {
return "", errors.New("Dispatcher.LocalSubmit: no local executor")
}
return d.executor.Submit(ctx, specBytes)
}
func (d *Dispatcher) LocalStatus(ctx context.Context, jobID string) (string, error) {
if d.executor == nil {
return "", errors.New("Dispatcher.LocalStatus: no local executor")
}
return d.executor.Status(ctx, jobID)
}
// parseInlineSpec parses a minimal JSON spec with cpu_millicores,
// memory_mib, disk_mib fields. The CLI uses this as the wire format
// for cross-node dispatch; full HCL parsing is in internal/jobspec.
func parseInlineSpec(b []byte) (JobSpec, error) {
type wire struct {
CPUMillicores int64 `json:"cpu_millicores"`
MemoryMiB int64 `json:"memory_mib"`
DiskMiB int64 `json:"disk_mib"`
}
var w wire
if err := json.Unmarshal(b, &w); err != nil {
return JobSpec{}, fmt.Errorf("parseInlineSpec: %w", err)
}
return JobSpec{
CPUMillicores: w.CPUMillicores,
MemoryMiB: w.MemoryMiB,
DiskMiB: w.DiskMiB,
}, nil
}
+61
View File
@@ -3,6 +3,8 @@ package engine
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os/exec"
@@ -29,6 +31,65 @@ func NewExecutor(jobs *store.JobRepo, tasks *store.TaskRepo, log *slog.Logger) *
return &Executor{jobs: jobs, tasks: tasks, log: log}
}
// Submit is the dispatch-friendly entry point (v0.2 P02). It parses
// the spec bytes as a minimal TaskSpec and runs a single task under
// a fresh job. Returns the job ID. This is intentionally simpler
// than the v0.1 Run() entry point — the cross-node dispatch wire
// format is a flat task (one process), not a multi-task job.
//
// The spec format is a JSON object with at least:
//
// { "name": "...", "command": "...", "args": [...], "env": [...] }
//
// All fields except command are optional.
func (e *Executor) Submit(ctx context.Context, specBytes []byte) (string, error) {
type wireSpec struct {
Name string `json:"name"`
Command string `json:"command"`
Args []string `json:"args"`
Env []string `json:"env"`
}
var ws wireSpec
if err := json.Unmarshal(specBytes, &ws); err != nil {
return "", fmt.Errorf("Executor.Submit: parse: %w", err)
}
if ws.Command == "" {
return "", errors.New("Executor.Submit: spec.command is required")
}
if ws.Name == "" {
ws.Name = "dispatched"
}
job := &model.Job{
ID: uuid.NewString(),
Spec: string(specBytes),
Status: model.JobStatusPending,
}
ts := TaskSpec{
Name: ws.Name,
Command: ws.Command,
Args: ws.Args,
Env: ws.Env,
}
if err := e.Run(ctx, job, []TaskSpec{ts}); err != nil {
return job.ID, err
}
return job.ID, nil
}
// Status returns the current state of a job for the Status dispatch
// endpoint. The returned string is one of: "pending", "running",
// "complete", "failed", "stopped". Maps to model.JobStatus* values.
func (e *Executor) Status(ctx context.Context, jobID string) (string, error) {
if e.jobs == nil {
return "", errors.New("Executor.Status: nil job repo")
}
j, err := e.jobs.Get(ctx, jobID)
if err != nil {
return "", err
}
return string(j.Status), nil
}
type TaskSpec struct {
Name string
Command string
+106
View File
@@ -0,0 +1,106 @@
// Package engine — peer.go implements the peer registry for multi-node
// scheduling (v0.2 P02). A peer is a remote orca node reachable over
// mTLS. The registry is in-memory plus optionally SQLite-persisted;
// for P02 the in-memory map is the source of truth and persistence
// is best-effort.
package engine
import (
"context"
"fmt"
"sort"
"sync"
"time"
"git.cloudinit.dev/coreci/orca/internal/store"
)
// Peer is a remote orca node reachable over mTLS.
type Peer struct {
NodeID string
Address string // host:port (the peer's daemon listener)
ServerName string // expected SAN on the peer's cert
CAPath string // path to the CA cert this peer validates against
LastSeen time.Time
Capacity *store.NodeCapacity
}
// PeerRegistry tracks known peers. Methods are safe for concurrent
// use; the underlying map is guarded by a sync.RWMutex.
type PeerRegistry struct {
mu sync.RWMutex
peers map[string]*Peer
// optional persistence (not required for P02; can be added later)
persist PeerPersister
}
// PeerPersister is an optional callback for persisting peer records.
// P02 doesn't use it; it's here for the P03 audit log integration.
type PeerPersister interface {
SavePeer(ctx context.Context, p *Peer) error
}
// NewPeerRegistry returns an empty registry.
func NewPeerRegistry() *PeerRegistry {
return &PeerRegistry{peers: make(map[string]*Peer)}
}
// Add inserts or updates a peer record.
func (r *PeerRegistry) Add(p *Peer) error {
if p == nil {
return fmt.Errorf("PeerRegistry.Add: nil peer")
}
if p.NodeID == "" {
return fmt.Errorf("PeerRegistry.Add: NodeID is required")
}
r.mu.Lock()
r.peers[p.NodeID] = p
r.mu.Unlock()
return nil
}
// Remove deletes a peer by ID. Returns true if a peer was removed.
func (r *PeerRegistry) Remove(nodeID string) bool {
r.mu.Lock()
defer r.mu.Unlock()
_, ok := r.peers[nodeID]
if ok {
delete(r.peers, nodeID)
}
return ok
}
// Get returns the peer with the given ID, or nil.
func (r *PeerRegistry) Get(nodeID string) *Peer {
r.mu.RLock()
defer r.mu.RUnlock()
return r.peers[nodeID]
}
// All returns a snapshot of all peers, sorted by NodeID for determinism.
func (r *PeerRegistry) All(_ context.Context) ([]*Peer, error) {
r.mu.RLock()
out := make([]*Peer, 0, len(r.peers))
for _, p := range r.peers {
out = append(out, p)
}
r.mu.RUnlock()
sort.Slice(out, func(i, j int) bool { return out[i].NodeID < out[j].NodeID })
return out, nil
}
// Len returns the number of registered peers.
func (r *PeerRegistry) Len() int {
r.mu.RLock()
defer r.mu.RUnlock()
return len(r.peers)
}
// UpdateLastSeen bumps the LastSeen timestamp on a peer.
func (r *PeerRegistry) UpdateLastSeen(nodeID string) {
r.mu.Lock()
if p, ok := r.peers[nodeID]; ok {
p.LastSeen = time.Now().UTC()
}
r.mu.Unlock()
}
+117
View File
@@ -0,0 +1,117 @@
// Package engine — scheduler.go implements best-fit bin-packing for
// the multi-node scheduler (v0.2 P02, REQ-028). The scheduler
// receives a JobSpec, looks at the local NodeCapacity, and either
// runs locally or falls through to a remote peer via the dispatcher.
//
// The bin-pack scoring is intentionally simple: pick the node with
// the most free capacity (cpu_millicores + memory_mib weighted 1:1
// after normalization). This is deterministic and easy to test.
package engine
import (
"context"
"fmt"
"sort"
"git.cloudinit.dev/coreci/orca/internal/model"
"git.cloudinit.dev/coreci/orca/internal/store"
)
// JobSpec is a minimal projection of the spec needed for scheduling
// decisions. The full spec parsing is in internal/jobspec; this is
// just enough to ask "does this fit?" and "where should it go?".
type JobSpec struct {
CPUMillicores int64
MemoryMiB int64
DiskMiB int64
}
// Fits reports whether the local node has enough free capacity to
// run the spec. Capacity accounting is conservative: a job is allowed
// to run only if cpu + memory + disk are all >= the spec.
func (s JobSpec) Fits(c *store.NodeCapacity) bool {
if c == nil {
return false
}
return c.CPUMillicores >= s.CPUMillicores &&
c.MemoryMiB >= s.MemoryMiB &&
c.DiskMiB >= s.DiskMiB
}
// Score returns a sortable score for bin-packing; higher = more free
// capacity. Weighted roughly toward CPU (which is usually the
// constraint) but normalized so the test isn't fragile.
func (s JobSpec) Score(c *store.NodeCapacity) int64 {
if c == nil {
return -1
}
// Use 1:1 weighting in normalized units (millicores vs MiB) to
// keep the score monotonic. This isn't physically meaningful
// (mixing units) but it gives a stable ordering for tests.
freeCPU := c.CPUMillicores - s.CPUMillicores
freeMem := c.MemoryMiB - s.MemoryMiB
if freeCPU < 0 || freeMem < 0 {
return -1
}
return freeCPU + freeMem
}
// PickNode selects the best-fit node from a slice of capacities.
// Returns the chosen *store.NodeCapacity and its index, or an error
// if none can fit. Ties are broken by NodeID (lexicographic) for
// determinism.
func PickNode(spec JobSpec, capacities []*store.NodeCapacity) (*store.NodeCapacity, int, error) {
if len(capacities) == 0 {
return nil, -1, fmt.Errorf("PickNode: no nodes available")
}
type scored struct {
c *store.NodeCapacity
idx int
score int64
}
var fits []scored
for i, c := range capacities {
if !spec.Fits(c) {
continue
}
fits = append(fits, scored{c: c, idx: i, score: spec.Score(c)})
}
if len(fits) == 0 {
return nil, -1, fmt.Errorf("PickNode: no node can fit the spec (cpu=%d mem=%d disk=%d)",
spec.CPUMillicores, spec.MemoryMiB, spec.DiskMiB)
}
sort.SliceStable(fits, func(i, j int) bool {
if fits[i].score != fits[j].score {
return fits[i].score > fits[j].score
}
return fits[i].c.NodeID < fits[j].c.NodeID
})
return fits[0].c, fits[0].idx, nil
}
// LocalNode is a minimal abstraction of the local node for the
// scheduler. The concrete implementation reads from the
// store.CapacityRepo.
type LocalNode interface {
Capacity(ctx context.Context) (*store.NodeCapacity, error)
}
// memLocalNode returns capacity from a fixed *store.NodeCapacity.
// Useful for tests; production code wraps CapacityRepo.
type memLocalNode struct{ c *store.NodeCapacity }
// MemLocalNode returns a LocalNode backed by a fixed capacity. Test-only.
func MemLocalNode(c *store.NodeCapacity) LocalNode {
return &memLocalNode{c: c}
}
func (m *memLocalNode) Capacity(_ context.Context) (*store.NodeCapacity, error) {
if m.c == nil {
return nil, store.ErrNotFound
}
return m.c, nil
}
// ensure model import compiles even if unused above (placeholder for
// future scheduler fields that take *model.Node).
var _ = model.NodeStateReady
+66
View File
@@ -0,0 +1,66 @@
package engine
import (
"testing"
"git.cloudinit.dev/coreci/orca/internal/store"
)
func TestPickNodeBestFit(t *testing.T) {
caps := []*store.NodeCapacity{
{NodeID: "node-b", CPUMillicores: 1000, MemoryMiB: 1024, DiskMiB: 1024},
{NodeID: "node-a", CPUMillicores: 4000, MemoryMiB: 4096, DiskMiB: 4096},
{NodeID: "node-c", CPUMillicores: 500, MemoryMiB: 512, DiskMiB: 512},
}
spec := JobSpec{CPUMillicores: 1000, MemoryMiB: 1024, DiskMiB: 1024}
got, idx, err := PickNode(spec, caps)
if err != nil {
t.Fatalf("PickNode: %v", err)
}
if got.NodeID != "node-a" {
t.Errorf("PickNode: got %s, want node-a (most free capacity)", got.NodeID)
}
if idx != 1 {
t.Errorf("PickNode: got idx %d, want 1", idx)
}
}
func TestPickNodeNoFit(t *testing.T) {
caps := []*store.NodeCapacity{
{NodeID: "node-a", CPUMillicores: 100, MemoryMiB: 100, DiskMiB: 100},
}
spec := JobSpec{CPUMillicores: 1000, MemoryMiB: 1024, DiskMiB: 1024}
_, _, err := PickNode(spec, caps)
if err == nil {
t.Fatal("expected PickNode to fail when no node can fit")
}
}
func TestPickNodeTieDeterministic(t *testing.T) {
// Two nodes with identical free capacity. Tie broken by NodeID
// (lexicographic) for determinism.
caps := []*store.NodeCapacity{
{NodeID: "node-z", CPUMillicores: 4000, MemoryMiB: 4096, DiskMiB: 4096},
{NodeID: "node-a", CPUMillicores: 4000, MemoryMiB: 4096, DiskMiB: 4096},
}
spec := JobSpec{CPUMillicores: 1000, MemoryMiB: 1024, DiskMiB: 1024}
got, _, err := PickNode(spec, caps)
if err != nil {
t.Fatalf("PickNode: %v", err)
}
if got.NodeID != "node-a" {
t.Errorf("PickNode tie-break: got %s, want node-a (lexicographic)", got.NodeID)
}
}
func TestJobSpecFits(t *testing.T) {
spec := JobSpec{CPUMillicores: 1000, MemoryMiB: 1024, DiskMiB: 1024}
c := &store.NodeCapacity{CPUMillicores: 2000, MemoryMiB: 2048, DiskMiB: 2048}
if !spec.Fits(c) {
t.Error("Fits: should fit")
}
c.CPUMillicores = 500
if spec.Fits(c) {
t.Error("Fits: should not fit (CPU too low)")
}
}