docs(milestone): complete scheduling-streaming (v0.3)

---ci---
project: orca
phase: 3
milestone: v0.3
status: complete
requirements:
  covered: [REQ-022, REQ-030, REQ-032]
  partial: []
---/ci---

v0.3 milestone merged to main. Includes all v0.2 work (P08-P10) that
was previously on the milestone branch but not yet merged to main, plus
the v0.3 completion work (iter.Seq streaming + doctor network/db).

v0.2 phases included: P08 (mTLS), P09 (scheduling), P10 (security scan).
v0.3 phases: P0 (pre-execution), P1 (iter.Seq streaming), P2 (doctor),
P3 (final review+ship).

Total: 40 requirements, all complete. No new go.mod dependencies.
Full test suite passes under -race. gofmt + go vet clean.
This commit is contained in:
Jon Chery
2026-08-01 20:06:47 +00:00
parent f503404dda
commit df58bc25a3
52 changed files with 5488 additions and 198 deletions
+124
View File
@@ -0,0 +1,124 @@
// Package store — capacity_repo.go implements persistence for NodeCapacity
// declarations (v0.2 P02). Capacity is declared per node via
// `orca node capacity --set` (or from `~/.orca/node.hcl` at join time).
// The dispatcher reads capacity rows to bin-pack jobs across nodes.
package store
import (
"context"
"database/sql"
"errors"
"fmt"
"time"
)
// NodeCapacity is the per-node resource declaration consumed by the
// scheduler. Units:
// - CPUMillicores: 1000 = 1 vCPU
// - MemoryMiB: mebibytes of RAM
// - DiskMiB: mebibytes of scratch disk
type NodeCapacity struct {
NodeID string
CPUMillicores int64
MemoryMiB int64
DiskMiB int64
UpdatedAt time.Time
}
// CapacityRepo is the persistence layer for NodeCapacity rows.
type CapacityRepo struct {
db *sql.DB
}
// NewCapacityRepo returns a CapacityRepo backed by the given DB.
func NewCapacityRepo(db *sql.DB) *CapacityRepo {
return &CapacityRepo{db: db}
}
// Upsert writes the capacity row for nodeID, replacing any prior row.
// The UpdatedAt column is set to time.Now().UTC() unless the caller
// supplied a non-zero value.
func (r *CapacityRepo) Upsert(ctx context.Context, c *NodeCapacity) error {
if c == nil {
return errors.New("CapacityRepo.Upsert: nil capacity")
}
if c.NodeID == "" {
return errors.New("CapacityRepo.Upsert: NodeID is required")
}
if c.UpdatedAt.IsZero() {
c.UpdatedAt = time.Now().UTC()
}
_, err := r.db.ExecContext(ctx, `
INSERT INTO node_capacity (node_id, cpu_millicores, memory_mib, disk_mib, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(node_id) DO UPDATE SET
cpu_millicores = excluded.cpu_millicores,
memory_mib = excluded.memory_mib,
disk_mib = excluded.disk_mib,
updated_at = excluded.updated_at
`, c.NodeID, c.CPUMillicores, c.MemoryMiB, c.DiskMiB, c.UpdatedAt)
if err != nil {
return fmt.Errorf("CapacityRepo.Upsert: %w", err)
}
return nil
}
// Get returns the capacity for nodeID or ErrNotFound.
func (r *CapacityRepo) Get(ctx context.Context, nodeID string) (*NodeCapacity, error) {
if nodeID == "" {
return nil, errors.New("CapacityRepo.Get: nodeID is required")
}
row := r.db.QueryRowContext(ctx, `
SELECT node_id, cpu_millicores, memory_mib, disk_mib, updated_at
FROM node_capacity WHERE node_id = ?
`, nodeID)
var c NodeCapacity
if err := row.Scan(&c.NodeID, &c.CPUMillicores, &c.MemoryMiB, &c.DiskMiB, &c.UpdatedAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("CapacityRepo.Get: %w", err)
}
return &c, nil
}
// List returns all capacity rows ordered by node_id.
func (r *CapacityRepo) List(ctx context.Context) ([]*NodeCapacity, error) {
rows, err := r.db.QueryContext(ctx, `
SELECT node_id, cpu_millicores, memory_mib, disk_mib, updated_at
FROM node_capacity ORDER BY node_id
`)
if err != nil {
return nil, fmt.Errorf("CapacityRepo.List: %w", err)
}
defer rows.Close()
var out []*NodeCapacity
for rows.Next() {
var c NodeCapacity
if err := rows.Scan(&c.NodeID, &c.CPUMillicores, &c.MemoryMiB, &c.DiskMiB, &c.UpdatedAt); err != nil {
return nil, fmt.Errorf("CapacityRepo.List: scan: %w", err)
}
out = append(out, &c)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("CapacityRepo.List: rows: %w", err)
}
return out, nil
}
// Delete removes the capacity row for nodeID. Returns ErrNotFound if
// the row doesn't exist.
func (r *CapacityRepo) Delete(ctx context.Context, nodeID string) error {
res, err := r.db.ExecContext(ctx, `DELETE FROM node_capacity WHERE node_id = ?`, nodeID)
if err != nil {
return fmt.Errorf("CapacityRepo.Delete: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("CapacityRepo.Delete: rows: %w", err)
}
if n == 0 {
return ErrNotFound
}
return nil
}
+74
View File
@@ -0,0 +1,74 @@
package store
import (
"context"
"path/filepath"
"testing"
)
func TestCapacityRepoUpsertGetList(t *testing.T) {
dir := t.TempDir()
db, err := Open(filepath.Join(dir, "test.db"))
if err != nil {
t.Fatalf("Open: %v", err)
}
defer db.Close()
repo := NewCapacityRepo(db)
ctx := context.Background()
// Empty initially.
if _, err := repo.Get(ctx, "self"); err == nil {
t.Error("expected ErrNotFound on empty store")
}
rows, err := repo.List(ctx)
if err != nil {
t.Fatalf("List: %v", err)
}
if len(rows) != 0 {
t.Errorf("List: got %d rows, want 0", len(rows))
}
// Insert.
c1 := &NodeCapacity{NodeID: "self", CPUMillicores: 4000, MemoryMiB: 4096, DiskMiB: 4096}
if err := repo.Upsert(ctx, c1); err != nil {
t.Fatalf("Upsert: %v", err)
}
got, err := repo.Get(ctx, "self")
if err != nil {
t.Fatalf("Get: %v", err)
}
if got.CPUMillicores != 4000 || got.MemoryMiB != 4096 || got.DiskMiB != 4096 {
t.Errorf("Get: got %+v, want cpu=4000 mem=4096 disk=4096", got)
}
// Update (overwrite).
c2 := &NodeCapacity{NodeID: "self", CPUMillicores: 8000, MemoryMiB: 8192, DiskMiB: 8192}
if err := repo.Upsert(ctx, c2); err != nil {
t.Fatalf("Upsert(update): %v", err)
}
got, _ = repo.Get(ctx, "self")
if got.CPUMillicores != 8000 {
t.Errorf("Update: cpu=%d, want 8000", got.CPUMillicores)
}
// Add a second node.
c3 := &NodeCapacity{NodeID: "peer-1", CPUMillicores: 2000, MemoryMiB: 2048, DiskMiB: 2048}
if err := repo.Upsert(ctx, c3); err != nil {
t.Fatalf("Upsert(peer-1): %v", err)
}
rows, _ = repo.List(ctx)
if len(rows) != 2 {
t.Errorf("List: got %d rows, want 2", len(rows))
}
// Delete.
if err := repo.Delete(ctx, "peer-1"); err != nil {
t.Fatalf("Delete: %v", err)
}
if _, err := repo.Get(ctx, "peer-1"); err == nil {
t.Error("expected ErrNotFound after Delete")
}
if err := repo.Delete(ctx, "missing"); err == nil {
t.Error("expected ErrNotFound on Delete of missing row")
}
}
+54
View File
@@ -6,11 +6,19 @@ import (
"encoding/json"
"errors"
"fmt"
"iter"
"log/slog"
"time"
"git.cloudinit.dev/coreci/orca/internal/model"
)
// watchInterval is the poll cadence used by JobRepo.Watch and NodeRepo.Watch.
// It is an unexported package var (default 1s) so tests can override it to a
// small value for deterministic assertions (D-043). Do not change it from
// production code paths.
var watchInterval = 1 * time.Second
type JobRepo struct {
db *sql.DB
}
@@ -59,6 +67,52 @@ func (r *JobRepo) List(ctx context.Context) ([]*model.Job, error) {
return jobs, rows.Err()
}
// Watch yields the full snapshot of jobs on a watchInterval ticker until ctx
// is cancelled or the consumer stops pulling (yield returns false). It does
// not spawn a goroutine; the polling loop runs inline in the caller's
// goroutine via the range-over-func pull protocol (D-032).
//
// Each tick re-runs the List query and yields one []*model.Job snapshot
// containing ALL rows for that tick (G-001). The first yield happens
// immediately before the first ticker wait, so the consumer sees the initial
// state with no watchInterval delay (G-002). Transient query/scan errors are
// logged via slog.Default().Warn and the loop continues to the next tick
// rather than terminating the stream (D-034 lite). The ticker is stopped and
// rows are closed on every exit path (ctx.Done, yield==false, scan error).
func (r *JobRepo) Watch(ctx context.Context) iter.Seq[[]*model.Job] {
return func(yield func([]*model.Job) bool) {
ticker := time.NewTicker(watchInterval)
defer ticker.Stop()
for {
rows, err := r.db.QueryContext(ctx,
`SELECT id, name, spec, status, exit_code, created_at, started_at, ended_at FROM jobs ORDER BY created_at DESC`)
if err != nil {
slog.Default().Warn("watch jobs: query failed", "error", err)
// fall through to the select to wait for the next tick
} else {
snapshot := make([]*model.Job, 0)
for rows.Next() {
j, scanErr := scanJob(rows)
if scanErr != nil {
slog.Default().Warn("watch jobs: scan failed", "error", scanErr)
continue
}
snapshot = append(snapshot, j)
}
rows.Close()
if !yield(snapshot) {
return // consumer stopped pulling
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
}
func (r *JobRepo) UpdateStatus(ctx context.Context, id string, status model.JobStatus, exitCode int) error {
now := time.Now().UTC()
var startedAt, endedAt *time.Time
+201
View File
@@ -0,0 +1,201 @@
package store
import (
"context"
"path/filepath"
"testing"
"time"
"git.cloudinit.dev/coreci/orca/internal/model"
)
func openJobTestDB(t *testing.T) (*JobRepo, func()) {
t.Helper()
path := filepath.Join(t.TempDir(), "test.db")
db, err := Open(path)
if err != nil {
t.Fatalf("open db: %v", err)
}
return NewJobRepo(db), func() { _ = db.Close() }
}
func insertJob(t *testing.T, repo *JobRepo, ctx context.Context, id, name string) {
t.Helper()
if err := repo.Insert(ctx, &model.Job{
ID: id,
Name: name,
Spec: "test",
Status: model.JobStatusPending,
}); err != nil {
t.Fatalf("insert job %s: %v", id, err)
}
}
// withFastWatch sets watchInterval to a small value for deterministic tests and
// restores the default (1s) on cleanup.
func withFastWatch(t *testing.T, d time.Duration) {
t.Helper()
prev := watchInterval
watchInterval = d
t.Cleanup(func() { watchInterval = prev })
}
// TestJobRepoWatch_YieldsSnapshots verifies each yield is a complete tick
// snapshot (G-001): the first snapshot contains only the first job, and a
// later snapshot contains both jobs after a second insert.
func TestJobRepoWatch_YieldsSnapshots(t *testing.T) {
withFastWatch(t, 10*time.Millisecond)
repo, cleanup := openJobTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
insertJob(t, repo, ctx, "job-1", "alpha")
var snapshots [][]*model.Job
done := make(chan struct{})
go func() {
defer close(done)
for snap := range repo.Watch(ctx) {
snapshots = append(snapshots, snap)
if len(snapshots) >= 40 {
cancel()
return
}
}
}()
// Insert a second job after a short delay so a later tick observes it.
// Use a background context — the watch ctx may be cancelled by the
// goroutine above once it collects enough snapshots.
time.Sleep(100 * time.Millisecond)
insertJob(t, repo, context.Background(), "job-2", "beta")
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("watch did not complete within 2s")
}
if len(snapshots) == 0 {
t.Fatal("expected at least one snapshot, got none")
}
// First snapshot must contain only the first job (G-001).
if len(snapshots[0]) != 1 || snapshots[0][0].ID != "job-1" {
t.Errorf("first snapshot = %+v, want only job-1", snapshots[0])
}
// At least one later snapshot must contain both jobs.
foundBoth := false
for _, snap := range snapshots[1:] {
ids := make(map[string]bool, len(snap))
for _, j := range snap {
ids[j.ID] = true
}
if ids["job-1"] && ids["job-2"] {
foundBoth = true
break
}
}
if !foundBoth {
t.Errorf("no snapshot contained both jobs; snapshots=%v", snapshots)
}
}
// TestJobRepoWatch_ImmediateFirstYield verifies G-002: the first snapshot
// arrives before the first ticker wait, i.e. well under the watchInterval.
func TestJobRepoWatch_ImmediateFirstYield(t *testing.T) {
withFastWatch(t, 200*time.Millisecond)
repo, cleanup := openJobTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
insertJob(t, repo, ctx, "job-immediate", "first")
start := time.Now()
var firstSnap []*model.Job
got := make(chan struct{})
go func() {
for snap := range repo.Watch(ctx) {
firstSnap = snap
close(got)
cancel()
return
}
}()
select {
case <-got:
case <-time.After(100 * time.Millisecond):
t.Fatal("first yield took >100ms; expected immediate (G-002)")
}
elapsed := time.Since(start)
if elapsed > 100*time.Millisecond {
t.Errorf("first yield took %v; expected immediate (G-002)", elapsed)
}
if len(firstSnap) != 1 || firstSnap[0].ID != "job-immediate" {
t.Errorf("first snapshot = %+v, want job-immediate", firstSnap)
}
}
// TestJobRepoWatch_StopsOnConsumerBreak verifies the yield==false path: the
// range loop returns promptly when the consumer breaks after the first yield.
func TestJobRepoWatch_StopsOnConsumerBreak(t *testing.T) {
withFastWatch(t, 10*time.Millisecond)
repo, cleanup := openJobTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
insertJob(t, repo, ctx, "job-break", "break")
done := make(chan struct{})
go func() {
defer close(done)
for range repo.Watch(ctx) {
break // stop pulling immediately after the first snapshot
}
}()
select {
case <-done:
// success: range returned
case <-time.After(500 * time.Millisecond):
t.Fatal("watch did not stop on consumer break within 500ms")
}
}
// TestJobRepoWatch_StopsOnCtxCancel verifies the loop exits promptly after
// ctx is cancelled.
func TestJobRepoWatch_StopsOnCtxCancel(t *testing.T) {
withFastWatch(t, 10*time.Millisecond)
repo, cleanup := openJobTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
insertJob(t, repo, ctx, "job-cancel", "cancel")
done := make(chan struct{})
go func() {
defer close(done)
for range repo.Watch(ctx) {
// drain until cancelled
}
}()
// Let at least one tick land, then cancel.
time.Sleep(20 * time.Millisecond)
cancel()
select {
case <-done:
// success
case <-time.After(500 * time.Millisecond):
t.Fatal("watch did not stop on ctx cancel within 500ms")
}
}
+15
View File
@@ -12,6 +12,21 @@ import (
//go:embed migrations/*.sql
var migrationsFS embed.FS
// MigrationVersion returns the name of the highest applied migration
// (e.g. "0005_node_capacity.sql"). Returns ("", nil) if no migrations
// have been applied (fresh or empty database).
func MigrationVersion(ctx context.Context, db *sql.DB) (string, error) {
var name string
err := db.QueryRowContext(ctx, `SELECT name FROM schema_migrations ORDER BY name DESC LIMIT 1`).Scan(&name)
if err == sql.ErrNoRows {
return "", nil
}
if err != nil {
return "", fmt.Errorf("query migration version: %w", err)
}
return name, nil
}
func migrate(db *sql.DB) error {
entries, err := migrationsFS.ReadDir("migrations")
if err != nil {
+37
View File
@@ -0,0 +1,37 @@
package store
import (
"context"
"path/filepath"
"testing"
)
func TestMigrationVersion(t *testing.T) {
dir := t.TempDir()
db, err := Open(filepath.Join(dir, "test.db"))
if err != nil {
t.Fatalf("open db: %v", err)
}
defer db.Close()
ctx := context.Background()
version, err := MigrationVersion(ctx, db)
if err != nil {
t.Fatalf("migration version: %v", err)
}
if version != "0005_node_capacity.sql" {
t.Errorf("MigrationVersion = %q, want 0005_node_capacity.sql", version)
}
// Empty the migrations table → should return ("", nil).
if _, err := db.ExecContext(ctx, "DELETE FROM schema_migrations"); err != nil {
t.Fatalf("clear migrations: %v", err)
}
version, err = MigrationVersion(ctx, db)
if err != nil {
t.Fatalf("migration version after clear: %v", err)
}
if version != "" {
t.Errorf("MigrationVersion after clear = %q, want empty", version)
}
}
@@ -0,0 +1,12 @@
-- Node capacity declaration for multi-node scheduling (v0.2 P02).
-- Loaded from `~/.orca/node.hcl` at `orca node join` and updated via
-- `orca node capacity --set`. Read by the dispatcher for bin-packing.
CREATE TABLE IF NOT EXISTS node_capacity (
node_id TEXT PRIMARY KEY,
cpu_millicores INTEGER NOT NULL,
memory_mib INTEGER NOT NULL,
disk_mib INTEGER NOT NULL,
updated_at DATETIME NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_capacity_updated ON node_capacity(updated_at);
+36
View File
@@ -6,6 +6,8 @@ import (
"encoding/json"
"errors"
"fmt"
"iter"
"log/slog"
"time"
"git.cloudinit.dev/coreci/orca/internal/model"
@@ -69,6 +71,40 @@ func (r *NodeRepo) List(ctx context.Context) ([]*model.Node, error) {
return nodes, rows.Err()
}
func (r *NodeRepo) Watch(ctx context.Context) iter.Seq[[]*model.Node] {
return func(yield func([]*model.Node) bool) {
ticker := time.NewTicker(watchInterval)
defer ticker.Stop()
for {
rows, err := r.db.QueryContext(ctx,
`SELECT id, name, address, state, joined_at, last_seen, metadata FROM nodes ORDER BY joined_at ASC`)
if err != nil {
slog.Default().Warn("watch nodes: query failed", "error", err)
// fall through to the select to wait for the next tick
} else {
snapshot := make([]*model.Node, 0)
for rows.Next() {
n, scanErr := scanNode(rows)
if scanErr != nil {
slog.Default().Warn("watch nodes: scan failed", "error", scanErr)
continue
}
snapshot = append(snapshot, n)
}
rows.Close()
if !yield(snapshot) {
return // consumer stopped pulling
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
}
func (r *NodeRepo) UpdateState(ctx context.Context, id string, state model.NodeState) error {
res, err := r.db.ExecContext(ctx,
`UPDATE nodes SET state = ?, last_seen = ? WHERE id = ?`,
+156
View File
@@ -100,3 +100,159 @@ func TestNodeRepo_Delete(t *testing.T) {
t.Errorf("expected ErrNotFound, got %v", err)
}
}
func insertNode(t *testing.T, repo *NodeRepo, ctx context.Context, id, name string) {
t.Helper()
if err := repo.Insert(ctx, &model.Node{
ID: id,
Name: name,
Address: "addr",
State: model.NodeStateReady,
JoinedAt: time.Now().UTC(),
LastSeen: time.Now().UTC(),
}); err != nil {
t.Fatalf("insert node %s: %v", id, err)
}
}
func TestNodeRepoWatch_YieldsSnapshots(t *testing.T) {
withFastWatch(t, 10*time.Millisecond)
repo, cleanup := openTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
insertNode(t, repo, ctx, "node-1", "alpha")
var snapshots [][]*model.Node
done := make(chan struct{})
go func() {
defer close(done)
for snap := range repo.Watch(ctx) {
snapshots = append(snapshots, snap)
if len(snapshots) >= 40 {
cancel()
return
}
}
}()
time.Sleep(100 * time.Millisecond)
insertNode(t, repo, context.Background(), "node-2", "beta")
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("watch did not complete within 2s")
}
if len(snapshots) == 0 {
t.Fatal("expected at least one snapshot, got none")
}
if len(snapshots[0]) != 1 || snapshots[0][0].ID != "node-1" {
t.Errorf("first snapshot = %+v, want only node-1", snapshots[0])
}
foundBoth := false
for _, snap := range snapshots[1:] {
ids := make(map[string]bool, len(snap))
for _, n := range snap {
ids[n.ID] = true
}
if ids["node-1"] && ids["node-2"] {
foundBoth = true
break
}
}
if !foundBoth {
t.Errorf("no snapshot contained both nodes; snapshots=%v", snapshots)
}
}
func TestNodeRepoWatch_ImmediateFirstYield(t *testing.T) {
withFastWatch(t, 200*time.Millisecond)
repo, cleanup := openTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
insertNode(t, repo, ctx, "node-immediate", "first")
start := time.Now()
var firstSnap []*model.Node
got := make(chan struct{})
go func() {
for snap := range repo.Watch(ctx) {
firstSnap = snap
close(got)
cancel()
return
}
}()
select {
case <-got:
case <-time.After(100 * time.Millisecond):
t.Fatal("first yield took >100ms; expected immediate (G-002)")
}
elapsed := time.Since(start)
if elapsed > 100*time.Millisecond {
t.Errorf("first yield took %v; expected immediate (G-002)", elapsed)
}
if len(firstSnap) != 1 || firstSnap[0].ID != "node-immediate" {
t.Errorf("first snapshot = %+v, want node-immediate", firstSnap)
}
}
func TestNodeRepoWatch_StopsOnConsumerBreak(t *testing.T) {
withFastWatch(t, 10*time.Millisecond)
repo, cleanup := openTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
insertNode(t, repo, ctx, "node-break", "break")
done := make(chan struct{})
go func() {
defer close(done)
for range repo.Watch(ctx) {
break
}
}()
select {
case <-done:
case <-time.After(500 * time.Millisecond):
t.Fatal("watch did not stop on consumer break within 500ms")
}
}
func TestNodeRepoWatch_StopsOnCtxCancel(t *testing.T) {
withFastWatch(t, 10*time.Millisecond)
repo, cleanup := openTestDB(t)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
insertNode(t, repo, ctx, "node-cancel", "cancel")
done := make(chan struct{})
go func() {
defer close(done)
for range repo.Watch(ctx) {
}
}()
time.Sleep(20 * time.Millisecond)
cancel()
select {
case <-done:
case <-time.After(500 * time.Millisecond):
t.Fatal("watch did not stop on ctx cancel within 500ms")
}
}