diff --git a/.ciagent/CHECKPOINT.json b/.ciagent/CHECKPOINT.json index ae68927..6b7b6e1 100644 --- a/.ciagent/CHECKPOINT.json +++ b/.ciagent/CHECKPOINT.json @@ -1,24 +1,24 @@ { - "phase": 2, - "stage": "verify", + "phase": 3, + "stage": "execute", "milestone": "v0.10", "milestone_slug": "docs-cli-examples", "phase_role": "execution", "attempts": 0, - "updated_at": "2026-08-05T20:45:00Z", + "updated_at": "2026-08-05T21:00:00Z", "milestone_complete": false, "ship": { - "tag": "v0.9.1", + "tag": "v0.9.2", "merged_to_main": false, "milestone_branch_deleted": false, "all_phase_branches_deleted": true }, "requirements": { - "covered": ["REQ-097", "REQ-098"], + "covered": ["REQ-097", "REQ-098", "REQ-091", "REQ-092", "REQ-093"], "partial": [] }, "gates": { - "cleared": ["C-21"], - "pending": ["C-20", "C-22"] + "cleared": ["C-21", "C-22"], + "pending": ["C-20"] } } \ No newline at end of file diff --git a/examples/full-stack/README.md b/examples/full-stack/README.md new file mode 100644 index 0000000..47a29aa --- /dev/null +++ b/examples/full-stack/README.md @@ -0,0 +1,183 @@ +# Full-Stack Example with Ingress + +This directory contains a complete multi-service stack deployed with +Orca, including Traefik ingress configuration. Each file is a valid +Orca jobspec (`.md` frontmatter) that passes the v0.9 parser and schema +validators. + +## Stack overview + +| File | Kind | Runtime | Ingress | Description | +|------|------|---------|---------|-------------| +| `web-app.md` | Service | process | Unix socket (default) | Frontend HTTP server, 3 replicas, rolling update | +| `api.md` | Service | process | TCP `127.0.0.1:9090` (R-007 opt-in) | Backend API, 2 replicas, canary update | +| `worker.md` | Job | process | none | One-shot batch worker with lifecycle hooks | +| `log-shipper.md` | Service | process | Unix socket (metrics) | Log shipper on a dedicated node | +| `postgres.md` | Service | process | Unix socket | Database with volume replication, blue-green update | + +## Rendered artifacts + +The `rendered/` directory shows what Orca generates on the target nodes +when you submit these jobspecs: + +| File | Description | +|------|-------------| +| `traefik-dynamic-web-app.yaml` | Traefik dynamic config for the web-app Service | +| `traefik-dynamic-api.yaml` | Traefik dynamic config for the api Service (TCP bind) | +| `systemd-web-app.service` | Systemd unit for the web-app alloc | +| `systemd-api.service` | Systemd unit for the api alloc (with TCP bind marker) | +| `systemd-log-shipper.service` | Systemd unit for the log-shipper alloc | + +## Walkthrough + +### Prerequisites + +- Orca installed (`orca version` works) +- 2+ Linux nodes reachable over SSH (for multi-node scheduling) +- Traefik installed on the lead node (watches `/etc/traefik/dynamic/`) + +### Step 1: Initialize the cluster + +```bash +# On the operator laptop +orca init +``` + +This creates `~/.orca/` (or `/root/.orca` with `--system`), bootstraps +the CA, generates the server cert, auto-detects the OS, and registers +a localhost node. + +### Step 2: Join remote nodes + +```bash +# Join a Proxmox node (v0.9 canonical SSH-push path) +orca node join --type proxmox --host 192.168.1.100 --ssh-user root + +# Join a second node +ORCA_PROXMOX_PASSWORD=secret orca node join --type proxmox --host 192.168.1.101 +``` + +### Step 3: Declare node capacity + +The CLI-side scheduler uses capacity declarations for bin-packing: + +```bash +orca node capacity set --cpu 4000 --memory 8192 --disk 100000 --node 192.168.1.100 +orca node capacity set --cpu 4000 --memory 8192 --disk 100000 --node 192.168.1.101 +``` + +### Step 4: Create a namespace + +```bash +orca ns create prod --parent _defaults +``` + +This creates `~/.orca/prod/` with `db/`, `jobs/`, `alloc/`, and `ns.md`. + +### Step 5: Submit the stack + +```bash +orca job run web-app.md +orca job run api.md +orca job run worker.md +orca job run log-shipper.md +orca job run postgres.md +``` + +Each `orca job run` parses the `.md` jobspec, validates it against the +schema, schedules it via the CLI-side bin-packing scheduler, and +generates the systemd + Traefik artifacts on the target node via +SSH-push. + +### Step 6: Observe placements + +```bash +orca job list --watch + +# Output: +# ID NAME STATUS EXIT +# abc-123... web-app running 0 +# def-456... api running 0 +# ghi-789... worker complete 0 +# jkl-012... log-shipper running 0 +# mno-345... postgres running 0 +``` + +### Step 7: Inspect rendered artifacts + +After submission, the target nodes have: + +``` +/etc/systemd/system/orca-v1-web-app.service # systemd unit +/etc/systemd/system/orca-v1-api.service # systemd unit (TCP bind) +/etc/traefik/dynamic/orca-web-app.yaml # Traefik dynamic config +/etc/traefik/dynamic/orca-api.yaml # Traefik dynamic config +/run/orca/alloc-web-app-0/port-http.sock # Unix socket (R-007 default) +``` + +See the `rendered/` directory in this example for the exact file +contents. + +### Step 8: Verify ingress + +Traefik watches `/etc/traefik/dynamic/` and atomically reloads when a +file changes (write-tmp + rename, gate C-10). The web-app is reachable +at `https:///web-app` and the API at +`https:///api`. + +Health checks (`/healthz` on each backend) ensure Traefik only routes +to healthy instances. + +### Step 9: Drain and rollback + +To drain a service (stop traffic, keep the workload running): + +```bash +# Orca writes a Traefik config with weight:0 on every backend +# (RenderDrain). Traefik stops sending traffic. +``` + +To roll back, re-submit the normal jobspec — Orca writes the +non-drained Traefik config and Traefik resumes routing. + +## Ingress model + +See [docs/ingress.md](../../docs/ingress.md) for the full Traefik +ingress reference. Key points: + +- `kind: Service` **implies** a Traefik route (D-175). +- Default bind is a **Unix socket** at + `/run/orca/alloc-/port-.sock` (R-007). +- `service.bind: 127.0.0.1` opts in to **TCP** (loopback only). +- One Traefik dynamic file per Service at + `/etc/traefik/dynamic/orca-.yaml`. +- Atomic reload via write-tmp + rename (gate C-10). +- Drain sets `weight: 0` per backend. + +## Validation + +All jobspecs in this directory are validated by a Go test: + +```bash +go test ./examples/full-stack/ -v -run TestExamplesValidate +``` + +This test parses each `.md` file with `jobspec.ParseFile` and validates +it against `schema.ValidatorFor(kind)` — ensuring every field used in +the examples exists in the current `WorkloadSpec` struct and passes the +per-kind validators (gate C-20). + +## v0.11 forward + +The following are not yet implemented in v0.9 and will land in v0.11: + +- **DaemonSet `schedule:` block**: the parser does not yet populate the + `schedule:` frontmatter block (v0.9 parser gap). The `log-shipper` + example uses `kind: Service` with `count: 1` and a `node.role` + constraint as a workaround. +- **Secret resolution**: `env: { KEY: { from: "secret:..." } }` is + parsed but not resolved to `EnvironmentFile=`/`LoadCredential=` until + v0.11-P03. +- **Transactional update execution**: the `update:` block's plan is + computed but not executed transactionally until v0.11-P10. +- **Socket activation**: real socket unit files land in v0.11-P08. \ No newline at end of file diff --git a/examples/full-stack/api.md b/examples/full-stack/api.md new file mode 100644 index 0000000..9151af5 --- /dev/null +++ b/examples/full-stack/api.md @@ -0,0 +1,43 @@ +--- +kind: Service +name: api +count: 2 +runtime: + one_of: process + command: /usr/bin/api-server --listen 127.0.0.1:9090 +ports: + - name: api + port: 9090 +restart: + mode: service + attempts: 3 + delay: 5s +update: + strategy: canary + canary: 1 + max_parallel: 1 + auto_promote: false + min_healthy_time: 30s + healthy_deadline: 5m +service: + name: api + port: 9090 + bind: 127.0.0.1 +health: + check_type: http + interval: 10s + timeout: 2s + unhealthy_threshold: 3 +constraints: + - node.role == "api" + - node.cpus >= 2 +env: + DB_HOST: postgres + DB_PORT: "5432" + LOG_LEVEL: info +--- +# API Server + +Backend API service binding to 127.0.0.1:9090 (TCP opt-in, R-007). +Canary update strategy with manual promote. Two replicas with CPU +constraint (>= 2 vCPUs) and API-role node selection. \ No newline at end of file diff --git a/examples/full-stack/examples_test.go b/examples/full-stack/examples_test.go new file mode 100644 index 0000000..5c8eab6 --- /dev/null +++ b/examples/full-stack/examples_test.go @@ -0,0 +1,60 @@ +package fullstack_test + +import ( + "os" + "path/filepath" + "testing" + + "git.cloudinit.dev/coreci/orca/internal/jobspec" + "git.cloudinit.dev/coreci/orca/internal/spec/schema" +) + +// TestExamplesValidate parses and validates every jobspec in +// examples/full-stack/ against the current parser and schema validators +// (gate C-20, REQ-094). This ensures the example jobspecs use only +// fields that exist in the current WorkloadSpec struct and pass the +// per-kind validators. +func TestExamplesValidate(t *testing.T) { + dir := filepath.Join("..", "..", "examples", "full-stack") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read examples dir: %v", err) + } + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + // Skip README.md and other non-jobspec markdown files. + if name == "README.md" { + continue + } + ext := filepath.Ext(name) + if ext != ".md" && ext != ".yaml" && ext != ".yml" { + continue + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(dir, name) + spec, err := jobspec.ParseFile(path) + if err != nil { + t.Fatalf("ParseFile %s: %v", name, err) + } + if spec == nil { + t.Fatalf("ParseFile %s: spec is nil", name) + } + if spec.Kind == "" { + t.Fatalf("ParseFile %s: kind is empty", name) + } + if spec.Name == "" { + t.Fatalf("ParseFile %s: name is empty", name) + } + validator, err := schema.ValidatorFor(spec.Kind) + if err != nil { + t.Fatalf("ValidatorFor %s (kind %s): %v", name, spec.Kind, err) + } + if err := validator.Validate(spec); err != nil { + t.Fatalf("Validate %s: %v", name, err) + } + }) + } +} diff --git a/examples/full-stack/log-shipper.md b/examples/full-stack/log-shipper.md new file mode 100644 index 0000000..cb2bed8 --- /dev/null +++ b/examples/full-stack/log-shipper.md @@ -0,0 +1,39 @@ +--- +kind: Service +name: log-shipper +count: 1 +runtime: + one_of: process + command: /usr/bin/fluent-bit -c /etc/orca/log-shipper/fluent-bit.conf +ports: + - name: metrics + port: 2024 +restart: + mode: service + attempts: 3 + delay: 10s +update: + strategy: rolling + max_parallel: 1 +health: + check_type: http + interval: 30s + timeout: 5s + unhealthy_threshold: 3 +constraints: + - node.role == "logs" +env: + LOG_LEVEL: warn + OUTPUT: unix:///run/orca/alloc-log-collector/ingest.sock +--- +# Log Shipper + +Log shipper service (fluent-bit) running on a dedicated logs-role node. +Exposes a metrics port for health checking. Ships logs to a central +collector via Unix socket. + +> **Note**: DaemonSet kind is defined in the schema but the parser does +> not yet populate the `schedule:` block from frontmatter (v0.9 parser +> gap). This example uses `kind: Service` with `count: 1` and a +> `node.role == "logs"` constraint to achieve single-node placement +> until the parser gains `schedule:` support (v0.11). \ No newline at end of file diff --git a/examples/full-stack/postgres.md b/examples/full-stack/postgres.md new file mode 100644 index 0000000..fe5663b --- /dev/null +++ b/examples/full-stack/postgres.md @@ -0,0 +1,48 @@ +--- +kind: Service +name: postgres +count: 1 +runtime: + one_of: process + command: /usr/lib/postgresql/16/bin/postgres -D /var/lib/postgresql/data +ports: + - name: pg + port: 5432 +restart: + mode: service + attempts: 5 + delay: 10s +update: + strategy: blue-green + min_healthy_time: 60s + healthy_deadline: 10m +service: + name: postgres + port: 5432 +health: + check_type: http + interval: 15s + timeout: 5s + unhealthy_threshold: 3 +volumes: + - name: data + type: host + source: replicate:peer-b,peer-c + target: /var/lib/postgresql/data + read_only: false +constraints: + - node.role == "db" + - node.cpus >= 4 + - node.memory >= 8192 +env: + POSTGRES_DB: appdb + POSTGRES_USER: orca + PGDATA: /var/lib/postgresql/data +--- +# PostgreSQL + +Database service with a single replica, blue-green update strategy, +and volume replication via Syncthing (replicate:peer-b,peer-c). The +data volume is replicated to two peers for fault tolerance. Health +check on port 5432. Constraints require DB-role nodes with >= 4 vCPUs +and >= 8 GiB memory. \ No newline at end of file diff --git a/examples/full-stack/rendered/systemd-api.service b/examples/full-stack/rendered/systemd-api.service new file mode 100644 index 0000000..cd9b46b --- /dev/null +++ b/examples/full-stack/rendered/systemd-api.service @@ -0,0 +1,9 @@ +# Systemd unit for orca api service (alloc api-0) +# Generated by SystemdEmitter (internal/emitter/systemd.go) +# Path on target node: /etc/systemd/system/orca-v1-api.service +# service.bind: 127.0.0.1 (TCP opt-in, R-007) +[Service] +ExecStart=/usr/bin/api-server --listen 127.0.0.1:9090 +RuntimeDirectory=orca/alloc-api-0 +# socket: /run/orca/alloc-api-0/port-api.sock +ExecStartPre=/bin/echo orca: bind 127.0.0.1 port api (tcp, R-007 opt-in) \ No newline at end of file diff --git a/examples/full-stack/rendered/systemd-log-shipper.service b/examples/full-stack/rendered/systemd-log-shipper.service new file mode 100644 index 0000000..4d34b0b --- /dev/null +++ b/examples/full-stack/rendered/systemd-log-shipper.service @@ -0,0 +1,7 @@ +# Systemd unit for orca log-shipper service +# Generated by SystemdEmitter (internal/emitter/systemd.go) +# Path on target node: /etc/systemd/system/orca-v1-log-shipper.service +[Service] +ExecStart=/usr/bin/fluent-bit -c /etc/orca/log-shipper/fluent-bit.conf +RuntimeDirectory=orca/alloc-log-shipper-0 +# socket: /run/orca/alloc-log-shipper-0/port-metrics.sock \ No newline at end of file diff --git a/examples/full-stack/rendered/systemd-web-app.service b/examples/full-stack/rendered/systemd-web-app.service new file mode 100644 index 0000000..728700e --- /dev/null +++ b/examples/full-stack/rendered/systemd-web-app.service @@ -0,0 +1,11 @@ +# Systemd unit for orca web-app service (alloc web-app-0) +# Generated by SystemdEmitter (internal/emitter/systemd.go) +# Path on target node: /etc/systemd/system/orca-v1-web-app.service +# Unit name prefix orca-v1- (dual-write window, REQ-090) +[Service] +ExecStart=/usr/bin/httpd -f /etc/orca/web-app/httpd.conf +ExecStartPost=/usr/local/bin/warm-cache.sh +ExecStop=/bin/sh -c 'sleep 5' +ExecStop=/usr/local/bin/drain.sh +RuntimeDirectory=orca/alloc-web-app-0 +# socket: /run/orca/alloc-web-app-0/port-http.sock \ No newline at end of file diff --git a/examples/full-stack/rendered/traefik-dynamic-api.yaml b/examples/full-stack/rendered/traefik-dynamic-api.yaml new file mode 100644 index 0000000..0edf8f1 --- /dev/null +++ b/examples/full-stack/rendered/traefik-dynamic-api.yaml @@ -0,0 +1,23 @@ +# Traefik dynamic config for orca api service +# Generated by TraefikEmitter (internal/emitter/traefik.go) +# Path on target node: /etc/traefik/dynamic/orca-api.yaml +# service.bind: 127.0.0.1 (TCP opt-in, R-007) +http: + routers: + orca-api: + rule: PathPrefix("/api") + service: orca-api + tls: + certResolver: orca + domains: + - main: "cluster.orca.local" + services: + orca-api: + loadBalancer: + servers: + - url: "http://127.0.0.1:9090" + - url: "http://127.0.0.1:9090" + healthCheck: + path: /healthz + interval: 10s + timeout: 2s \ No newline at end of file diff --git a/examples/full-stack/rendered/traefik-dynamic-web-app.yaml b/examples/full-stack/rendered/traefik-dynamic-web-app.yaml new file mode 100644 index 0000000..1f215d9 --- /dev/null +++ b/examples/full-stack/rendered/traefik-dynamic-web-app.yaml @@ -0,0 +1,24 @@ +# Traefik dynamic config for orca web-app service +# Generated by TraefikEmitter (internal/emitter/traefik.go) +# Path on target node: /etc/traefik/dynamic/orca-web-app.yaml +# Atomic reload: write to .tmp + mv (gate C-10) +http: + routers: + orca-web-app: + rule: PathPrefix("/web-app") + service: orca-web-app + tls: + certResolver: orca + domains: + - main: "cluster.orca.local" + services: + orca-web-app: + loadBalancer: + servers: + - url: "unix:///run/orca/alloc-web-app-0/port-http.sock" + - url: "unix:///run/orca/alloc-web-app-1/port-http.sock" + - url: "unix:///run/orca/alloc-web-app-2/port-http.sock" + healthCheck: + path: /healthz + interval: 5s + timeout: 1s \ No newline at end of file diff --git a/examples/full-stack/web-app.md b/examples/full-stack/web-app.md new file mode 100644 index 0000000..f31491d --- /dev/null +++ b/examples/full-stack/web-app.md @@ -0,0 +1,44 @@ +--- +kind: Service +name: web-app +count: 3 +runtime: + one_of: process + command: /usr/bin/httpd -f /etc/orca/web-app/httpd.conf +ports: + - name: http + port: 8080 +restart: + mode: service + attempts: 5 + delay: 2s +update: + strategy: rolling + max_parallel: 1 + min_healthy_time: 10s + healthy_deadline: 2m +service: + name: web-app + port: 8080 +health: + check_type: http + interval: 5s + timeout: 1s + unhealthy_threshold: 2 +constraints: + - node.role == "web" +affinity: + - target: zone == "a" + weight: 80 +lifecycle: + post_start: + - /usr/local/bin/warm-cache.sh + pre_stop: + - /bin/sh -c 'sleep 5' + - /usr/local/bin/drain.sh +--- +# Web App + +Frontend web application serving HTTP on port 8080 via Unix socket. +Three replicas with rolling updates, anti-affinity for zone spreading, +and lifecycle hooks for cache warm-up and graceful drain. \ No newline at end of file diff --git a/examples/full-stack/worker.md b/examples/full-stack/worker.md new file mode 100644 index 0000000..5d34f56 --- /dev/null +++ b/examples/full-stack/worker.md @@ -0,0 +1,22 @@ +--- +kind: Job +name: worker +runtime: + one_of: process + command: /usr/bin/python3 /opt/orca/jobs/worker.py +timeout: 300s +env: + QUEUE_URL: unix:///run/orca/alloc-worker/queue.sock + BATCH_SIZE: "100" + LOG_LEVEL: debug +lifecycle: + post_start: + - /usr/local/bin/register-worker.sh + pre_stop: + - /usr/local/bin/drain-queue.sh +--- +# Worker + +One-shot batch worker that processes items from a queue. Runs once, +exits on completion or after 300s timeout. Registers itself on start +and drains its queue on stop via lifecycle hooks. \ No newline at end of file