Compare commits
36 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 61c97c847c | |||
| 6f04b22df0 | |||
| eadf2cc2c5 | |||
| 93ac4bda66 | |||
| 454040fdd1 | |||
| c95bd73e42 | |||
| ecdba833d9 | |||
| 6e65eadaa5 | |||
| 1b7aac71f6 | |||
| ea42a17474 | |||
| 5013209e31 | |||
| dea472f443 | |||
| dccdb746ea | |||
| 080919fde6 | |||
| 3551b37ac0 | |||
| 65e50e465b | |||
| 0e7ee4f324 | |||
| b925fda3aa | |||
| 9853aee589 | |||
| 5e0b899f1a | |||
| 0424f8ce02 | |||
| 5600531bd7 | |||
| d324939699 | |||
| 00efe25ce4 | |||
| 1ad6780df1 | |||
| 7dc7980d74 | |||
| 790109ea24 | |||
| 64cbbd543e | |||
| 16440a89f2 | |||
| a6ceb13491 | |||
| c0ba30824e | |||
| 52e17aefbf | |||
| b6dd86fdf3 | |||
| ed91d68fbf | |||
| 531b36924c | |||
| 3a3ea74d76 |
@@ -872,3 +872,70 @@ scheduler.Schedule(spec, nodes) → emitter.Render(unit) → sshpush.Deploy(targ
|
|||||||
- IPv6 `net.JoinHostPort` in all SSH dial paths.
|
- IPv6 `net.JoinHostPort` in all SSH dial paths.
|
||||||
- Explicit timeouts on all SSH commands.
|
- Explicit timeouts on all SSH commands.
|
||||||
- Root SIGINT/SIGTERM handler for clean exit on non-watch commands.
|
- Root SIGINT/SIGTERM handler for clean exit on non-watch commands.
|
||||||
|
|
||||||
|
## v0.14 Deltas — Ingress Bootstrap Completeness (R-024)
|
||||||
|
|
||||||
|
### R-024: Traefik as Podman Container
|
||||||
|
|
||||||
|
Traefik runs exclusively as a podman container, deployed from the
|
||||||
|
custom `orca-traefik` image (published per release via `Dockerfile.traefik`
|
||||||
|
+ `scripts/release.sh` + `.coreci.yml container-publish-traefik`).
|
||||||
|
|
||||||
|
The v0.13 binary+systemd install (`internal/traefik/install.go`) is
|
||||||
|
replaced by an idempotent podman container reconciler
|
||||||
|
(`EnsureTraefikContainerLocal`/`Remote`). The container runs with
|
||||||
|
`--network host`, `--restart=unless-stopped`, and volume mounts for
|
||||||
|
`traefik.yml` (static config), `dynamic` (dynamic config), and
|
||||||
|
`step-ca-root.crt` (future mTLS). No SELinux `:Z` flag.
|
||||||
|
|
||||||
|
### Three Ingress Topologies
|
||||||
|
|
||||||
|
1. **Linux** (`orca init` / `orca node join --type linux`):
|
||||||
|
host → nft DNAT → podman traefik (host network).
|
||||||
|
`internal/ingress/bootstrap.go` → `BootstrapLocalIngress` /
|
||||||
|
`BootstrapRemoteIngress`.
|
||||||
|
|
||||||
|
2. **Proxmox Native** (`--ingress-mode native`, default):
|
||||||
|
PVE host → nft DNAT (target = LXC bridge IP) → LXC
|
||||||
|
(`--features nesting=1,keyctl=1,fuse=1`) → podman traefik.
|
||||||
|
`internal/proxmox/bootstrap.go` → `provisionNativeIngressLXC`.
|
||||||
|
|
||||||
|
3. **Proxmox Floating-IP** (`--ingress-mode floating-ip`):
|
||||||
|
LXC owns the floating IP (`net0 bridge=vmbr0,hwaddr=<mac>,
|
||||||
|
ip=<floating-ip>/<prefix>,gw=<gateway>`) → nft inside LXC →
|
||||||
|
podman traefik. The ingress LXC is registered as a `linux` node
|
||||||
|
(name=`ingress`) so `orca job run` pushes traefik dynamic config.
|
||||||
|
`internal/proxmox/ingress_lxc.go` → `ProvisionIngressLXC`.
|
||||||
|
|
||||||
|
### nft Emitter Changes
|
||||||
|
|
||||||
|
`internal/emitter/nft.go`:
|
||||||
|
- `DNATTarget` field (C-51: validated via `net.ParseIP`). Default
|
||||||
|
`127.0.0.1`; proxmox native uses LXC bridge IP.
|
||||||
|
- `EnableSNAT` field + postrouting masquerade chain: `ip saddr
|
||||||
|
127.0.0.0/8 oifname != "lo" masquerade` (research Topic 1).
|
||||||
|
- Input/forward chain priority shifted from `filter` (=0) to `-10`
|
||||||
|
(research Topic 2: pve-firewall coexistence — avoids same-priority
|
||||||
|
undefined evaluation order).
|
||||||
|
|
||||||
|
### TLS Model
|
||||||
|
|
||||||
|
v0.14 drops `certResolver: orca` from the dynamic config (traefik v3.3
|
||||||
|
only supports `acme`/`tailscale` resolvers, not CA-file-based). The
|
||||||
|
dynamic config emits `tls: {}` (traefik default cert). Real mTLS via
|
||||||
|
`tls.certificates` + `tls.options.default.clientAuth.caFiles` is
|
||||||
|
deferred to v0.15 (grill G-003, confidence 0.55 < 0.60).
|
||||||
|
|
||||||
|
### Migration 0009
|
||||||
|
|
||||||
|
`ALTER TABLE nodes ADD COLUMN ingress_mode TEXT NOT NULL DEFAULT '';`
|
||||||
|
Values: `""` (legacy), `"native"`, `"floating-ip"`. `IngressMode` field
|
||||||
|
on `model.Node`.
|
||||||
|
|
||||||
|
### New CLI
|
||||||
|
|
||||||
|
- `orca doctor ingress` — verifies podman container running, nft
|
||||||
|
DNAT+SNAT, dynamic dir, step-ca root CA.
|
||||||
|
- `--ingress-mode` flag on `orca node join --type proxmox`.
|
||||||
|
- `--floating-ip`, `--gateway`, `--mac`, `--net-prefix` flags for
|
||||||
|
floating-IP mode.
|
||||||
|
|||||||
+11
-15
@@ -1,21 +1,17 @@
|
|||||||
{
|
{
|
||||||
"phase": 1,
|
"phase": 2,
|
||||||
"stage": "complete",
|
"stage": "complete",
|
||||||
"milestone": "v0.13",
|
"milestone": "v0.15",
|
||||||
"milestone_slug": "production-hardening-2",
|
"milestone_slug": "ci-release-pipeline",
|
||||||
"phase_role": "execution",
|
"phase_role": "final",
|
||||||
"attempts": 0,
|
"attempts": 0,
|
||||||
"updated_at": "2026-08-07T19:05:00Z",
|
"updated_at": "2026-08-10T21:05:00Z",
|
||||||
"milestone_complete": false,
|
"milestone_complete": true,
|
||||||
"previous_milestone": "v0.12",
|
"previous_milestone": "v0.14",
|
||||||
"phase_count": 14,
|
"phases_shipped": ["P0","P1","P2"],
|
||||||
"phases_shipped": ["P0", "P1"],
|
"tags_shipped": ["v0.14.0","v0.14.1"],
|
||||||
"tags_shipped": ["v0.12.0", "v0.12.1"],
|
|
||||||
"requirements": {
|
"requirements": {
|
||||||
"covered": [149],
|
"covered": [180,181,182],
|
||||||
"partial": []
|
"partial": []
|
||||||
},
|
}
|
||||||
"binding_conditions": ["C-39","C-40","C-41","C-42","C-43","C-44","C-45","C-46","C-47","C-48","C-49"],
|
|
||||||
"load_bearing_rule": "R-022",
|
|
||||||
"next_milestone": "v1.0"
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
# CLARIFY v0.14: Ingress Bootstrap Completeness
|
||||||
|
|
||||||
|
**Autonomy**: full (config `autonomy.level = "full"`)
|
||||||
|
**Budget**: 10 questions (used: 5 interactive, 5 auto-resolved)
|
||||||
|
**Result**: all ambiguities resolved; 9 decisions (D-255..D-263) recorded in PROJECT.md
|
||||||
|
|
||||||
|
## Interactive Questions (resolved with operator)
|
||||||
|
|
||||||
|
### Q1: iptables vs nft emitter
|
||||||
|
**Ambiguity**: Codebase is nft-only (D-218). Operator said "iptables used for SNAT/DNAT".
|
||||||
|
**Resolution**: D-255 (partial) — Extend the existing nft emitter (`internal/emitter/nft.go`) with SNAT/MASQUERADE postrouting rules. "iptables" in the operator's description is read as "kernel NAT" — nft is the modern backend. No literal iptables emitter.
|
||||||
|
**Confidence**: 0.95
|
||||||
|
|
||||||
|
### Q2: ingress LXC lifecycle
|
||||||
|
**Ambiguity**: Should the floating-IP LXC be registered as an orca-managed node or left as a one-shot external proxy?
|
||||||
|
**Resolution**: D-259 — Register as a managed `linux` node (name=`ingress`, addr=`<floating-ip>:8443`). `orca job run` pushes traefik dynamic config to it like any linux node. This makes routing-to-other-hosts work.
|
||||||
|
**Confidence**: 0.92
|
||||||
|
|
||||||
|
### Q3: Floating-IP network topology
|
||||||
|
**Ambiguity**: Where does SNAT run and how does the LXC get the public IP?
|
||||||
|
**Resolution**: LXC owns floating IP on eth0 (`net0 bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway>`). nft runs inside the LXC. Matches Hetzner/OVH floating-IP-with-MAC model.
|
||||||
|
**Confidence**: 0.90
|
||||||
|
|
||||||
|
### Q4: TLS cert resolver in the image
|
||||||
|
**Ambiguity**: The emitted static config references `certResolver: orca` but never declares the `certificatesResolvers.orca` block. v0.11 claimed P10 would wire step-ca but it was never implemented.
|
||||||
|
**Resolution**: D-257 — Bake a `certificatesResolvers.orca.tls: { }` block pointing at a mounted `/etc/orca/step-ca-root.crt` CA file. No ACME (offline-first, R-001). If the file is absent at start, traefik logs a warning and serves plain HTTP (graceful degradation). The orca bootstrap pushes the cluster root CA into the container volume.
|
||||||
|
**Confidence**: 0.90
|
||||||
|
|
||||||
|
### Q5: Container networking
|
||||||
|
**Ambiguity**: nft DNAT targets 127.0.0.1:8080/8443 on the host. How does the podman container receive that traffic?
|
||||||
|
**Resolution**: D-256 — `--network host` so traefik binds 127.0.0.1:8080/8443 directly on the host (or LXC) loopback. No container port publishing. Simplest, matches the hybrid R-017 model.
|
||||||
|
**Confidence**: 0.92
|
||||||
|
|
||||||
|
### Q6: Dynamic config volume strategy
|
||||||
|
**Ambiguity**: How does the podman traefik container consume dynamic config? Today `orca job run` writes `/etc/traefik/dynamic/orca-<svc>.yaml` on the host.
|
||||||
|
**Resolution**: D-258 — Mount `/etc/traefik/dynamic` from host (`-v /etc/traefik/dynamic:/etc/traefik/dynamic:Z`). Zero changes to the existing `deployRemote` WriteFile path. The image's file provider watches `/etc/traefik/dynamic` as today.
|
||||||
|
**Confidence**: 0.95
|
||||||
|
|
||||||
|
## Auto-resolved questions (full autonomy, no operator interaction)
|
||||||
|
|
||||||
|
### Q7: Floating-IP mode — does the PVE host also register as a node?
|
||||||
|
**Ambiguity**: In floating-IP mode, only the ingress LXC handles traffic. Does the PVE host also get registered?
|
||||||
|
**Resolution**: D-259 — Yes. The PVE host registers as a `proxmox` node (for `pct`/`qm` workload dispatch). The ingress LXC registers as a `linux` node (for traefik dynamic config pushes). Both are in the registry.
|
||||||
|
**Confidence**: 0.92
|
||||||
|
|
||||||
|
### Q8: `--ingress-mode` persistence
|
||||||
|
**Ambiguity**: Should `--ingress-mode` be stored on the node record so `doctor ingress` knows which check path to run?
|
||||||
|
**Resolution**: D-260 — Yes. Add `IngressMode` field to `model.Node` + a schema migration (0007). Values: `""` (legacy/default for linux/localhost), `"native"`, `"floating-ip"`.
|
||||||
|
**Confidence**: 0.90
|
||||||
|
|
||||||
|
### Q9: MAC generation when `--mac` omitted
|
||||||
|
**Ambiguity**: In floating-IP mode, if `--mac` is not provided, should orca generate one or require it?
|
||||||
|
**Resolution**: D-261 — Interactive mode: generate a random locally-administered MAC (`02:XX:XX:XX:XX:XX`) and print it for operator confirmation. `--json` mode: require `--mac` explicitly (no silent generation — non-interactive means explicit inputs).
|
||||||
|
**Confidence**: 0.88
|
||||||
|
|
||||||
|
### Q10: Proxmox native nft DNAT target
|
||||||
|
**Ambiguity**: In native mode, traefik runs inside an LXC. LXC has its own network namespace. nft DNAT to `127.0.0.1:8443` on the PVE host would NOT reach a container inside an LXC (different loopback). What's the DNAT target?
|
||||||
|
**Resolution**: D-262 — The nft DNAT target is parameterized via `NftClusterConfig.DNATTarget` (default `127.0.0.1:8443`). For proxmox native mode, the DNAT target is the LXC's bridge IP (`<lxc-ip>:8443`). The LXC gets a DHCP/static bridge IP; orca discovers it after `pct start` via `pct list` or `pct inspect`.
|
||||||
|
**Confidence**: 0.90
|
||||||
|
|
||||||
|
## Additional decisions (derived from constraints, no ambiguity)
|
||||||
|
|
||||||
|
### D-263: LXC podman requirements
|
||||||
|
Ubuntu 24.04 LXC template does not have podman preinstalled. Bootstrap must:
|
||||||
|
1. `pct create` with `--features nesting=1,keyctl=1` (required for podman in unprivileged LXC)
|
||||||
|
2. After LXC start: `apt-get update && apt-get install -y podman nftables` inside the LXC
|
||||||
|
3. Then `podman pull orca-traefik:<tag>` + `podman run ...`
|
||||||
|
|
||||||
|
This adds ~30-60s to the join time. Documented in `docs/uat.md`.
|
||||||
|
|
||||||
|
### Image tag strategy
|
||||||
|
The `orca-traefik` image uses the same version tag as the orca release (`v0.13.x` line). The podman reconciler resolves the tag from `internal/cli.version`. In dev builds (version="dev"), it falls back to `latest`.
|
||||||
|
|
||||||
|
### Registry auth
|
||||||
|
The `orca-traefik` image is in the same registry/org as `orca` (`git.cloudinit.dev/coreci/`). Pulls are anonymous (REQ-045, repo is public). No `podman login` needed on workers.
|
||||||
|
|
||||||
|
## Requirements impact
|
||||||
|
|
||||||
|
No new requirements beyond REQ-171..REQ-179 (already in REQUIREMENTS.md). The clarify stage confirmed scope and resolved all implementation ambiguities. The 9 decisions (D-255..D-263) are recorded in PROJECT.md.
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# CLARIFY + RESEARCH + PLAN v0.15: CI Release Pipeline Fix
|
||||||
|
|
||||||
|
## Decisions
|
||||||
|
|
||||||
|
| ID | Decision | Rationale | Confidence |
|
||||||
|
|----|----------|-----------|------------|
|
||||||
|
| D-264 | Secret name = `PAT_TOKEN` (not `GITEA_PAT`) | Gitea reserves `GITEA_` prefix for built-in secrets | 1.0 (validated) |
|
||||||
|
| D-265 | Use `tea actions secrets create` CLI | Operator instruction: no API | 1.0 (validated) |
|
||||||
|
| D-266 | Container publishing in Gitea Actions, not CoreCI | CoreCI's podman executor appends `sh -c` which conflicts with kaniko's `/kaniko/executor` entrypoint. Gitea Actions `container:` supports `options: --entrypoint` | 0.95 |
|
||||||
|
| D-267 | kaniko `executor:debug` image | Includes `/bin/sh`; Gitea Actions can override entrypoint to `/bin/sh` then run kaniko via shell | 0.90 |
|
||||||
|
| D-268 | `coreci run` for validate/build/test/release (tarball); Gitea Actions for container publishing | Clean separation: CoreCI owns the pipeline, Gitea Actions owns the trigger + container publish | 0.95 |
|
||||||
|
|
||||||
|
## Research: CoreCI podman executor entrypoint issue
|
||||||
|
|
||||||
|
CoreCI's `internal/runner/podman_executor.go:48-51`:
|
||||||
|
```go
|
||||||
|
args = append(args, image) // e.g. gcr.io/kaniko-project/executor:debug
|
||||||
|
if job.Invoke != "" {
|
||||||
|
args = append(args, "sh", "-c", job.Invoke)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
This produces: `podman run ... <image> sh -c "<commands>"`
|
||||||
|
With kaniko:debug (entrypoint `/kaniko/executor`), the actual command is:
|
||||||
|
`/kaniko/executor sh -c "<commands>"` — kaniko fails (sh is not a kaniko flag).
|
||||||
|
|
||||||
|
**Conclusion**: kaniko cannot be used as a CoreCI step image. Container
|
||||||
|
publishing must move to the Gitea Actions workflow, which supports
|
||||||
|
`container: options: --entrypoint /bin/sh` to override the entrypoint.
|
||||||
|
|
||||||
|
## Plan
|
||||||
|
|
||||||
|
### Phase 1 (only execution phase)
|
||||||
|
|
||||||
|
**Files to create/modify:**
|
||||||
|
|
||||||
|
1. `.gitea/workflows/release.yml` — Gitea Actions workflow:
|
||||||
|
- `on: push: tags: ['v*']`
|
||||||
|
- Job 1 `ci`: checkout + install Go + install coreci + `coreci run`
|
||||||
|
(executes validate/build/test/release from .coreci.yml)
|
||||||
|
- Job 2 `container-orca`: checkout + kaniko build+push orca image
|
||||||
|
(needs job 1; uses `container: gcr.io/kaniko-project/executor:debug`
|
||||||
|
with `options: --entrypoint /bin/sh`)
|
||||||
|
- Job 3 `container-traefik`: checkout + kaniko build+push orca-traefik image
|
||||||
|
(needs job 1; same kaniko approach)
|
||||||
|
|
||||||
|
2. `.coreci.yml` — remove `container-publish` and `container-publish-traefik`
|
||||||
|
steps (they now live in the Gitea Actions workflow). Keep the
|
||||||
|
`gitea-release` step (tarball + Gitea release).
|
||||||
|
|
||||||
|
3. `scripts/trigger_coreci.sh` — add tag ref handling (or document that
|
||||||
|
Gitea Actions is the trigger; the hook is for branch-push CI only).
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
# GRILL v0.14: Ingress Bootstrap Completeness — Red-Team Review
|
||||||
|
|
||||||
|
**Date**: 2026-08-10
|
||||||
|
**Reviewer**: ci-griller (adversarial)
|
||||||
|
**Subject**: PLAN_v0.14.md (9 phases, P0–P8)
|
||||||
|
**Confidence in plan as-written**: **0.45 — RETHINK** (was Proceed-eligible until the mTLS scope expansion was chosen)
|
||||||
|
**Verdict**: **RETHINK** — the plan is technically grounded in strong research but contains (a) one outright correctness defect that will break execution, (b) two requirements/plan contradictions that ship broken config, (c) one scope expansion chosen during this grill that adds a 10th phase the plan does not contain, and (d) one regression of a Completed requirement. The research is the strongest artifact; the plan diverges from it in load-bearing ways.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## How this grill was conducted
|
||||||
|
|
||||||
|
Every plan claim was checked against the actual codebase (`internal/traefik/install.go`, `internal/emitter/nft.go`, `internal/emitter/traefik.go`, `internal/cli/init.go`, `internal/cli/upgrade.go`, `internal/cli/doctor_nft.go`, `internal/proxmox/bootstrap.go`, `internal/linux/bootstrap.go`, `internal/store/migrations/`, `internal/certpaths/`, `internal/sshpush/`, `scripts/release.sh`, `.coreci.yml`, `Dockerfile`, git tags, git log). Findings cite file:line. Four binding questions were asked interactively; answers recorded as G-001..G-004 below.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Per-Axis Findings
|
||||||
|
|
||||||
|
### Axis 1 — Feasibility
|
||||||
|
|
||||||
|
**Finding F1.1 (BLOCKER — migration number collision).** PLAN P5 T3 (line 194) specifies "Schema migration 0007: `ALTER TABLE nodes ADD COLUMN ingress_mode TEXT DEFAULT ''`". Migration 0007 **already exists** — `internal/store/migrations/0007_certs_serial_unique.sql` (added v0.7, P1-001). The current head is `0008_audit_tamper_evidence.sql`. The migrator runs files in lexical order and records applied versions in `schema_migrations`. Reusing 0007 will either (a) silently no-op on DBs that already recorded 0007, leaving `ingress_mode` un-added, or (b) break the migration ledger. This is a guaranteed execution-time defect.
|
||||||
|
**Evidence**: `internal/store/migrations/0007_certs_serial_unique.sql`, `internal/store/migrations/0008_audit_tamper_evidence.sql`, `internal/cli/init_test.go:84` (test asserts head = 0008).
|
||||||
|
**Resolved by**: G-001 (use `0009_ingress_mode.sql`).
|
||||||
|
|
||||||
|
**Finding F1.2 (BLOCKER — `certpaths.CAPath()` does not exist).** PLAN P3 T5 step 2 (line 142) says "Push cluster root CA … from `certpaths.CAPath()` if exists". The function `certpaths.CAPath()` does not exist. The real API is `certpaths.CACertPath()` (`internal/certpaths/certpaths.go:35`) and `certpaths.CAKeyPath()` (`:39`). P4 T1 step 2 repeats the same phantom reference. This will not compile.
|
||||||
|
**Evidence**: `internal/certpaths/certpaths.go:35` (`func CACertPath() string`); grep for `CAPath` returns zero matches in `internal/certpaths/`.
|
||||||
|
|
||||||
|
**Finding F1.3 (no `install_test.go` to rewrite).** PLAN P2 T10 (line 112) says "rewrite `internal/traefik/install_test.go`". That file does not exist — `internal/traefik/` contains only `install.go` (glob confirms). The task is "create", not "rewrite". Minor, but signals the plan was written against an imagined codebase shape, not the real one.
|
||||||
|
**Evidence**: `glob internal/traefik/*.go` → only `install.go`.
|
||||||
|
|
||||||
|
**Finding F1.4 (good — fake-SSH harness exists and is adequate).** PLAN P7 T8 claims a "hermetic fake-SSH harness" can assert `pct create` + `podman run` inside LXC. Verified: `internal/proxmox/ssh_session_test.go:23` (`fakeSSHServer`) and `internal/sshpush/transport_test.go:27` implement an in-process SSH server with `runCommand(cmd)` pattern-matching. It can assert the right commands are *sent* (e.g. `pct create … --features nesting=1,keyctl=1,fuse=1`). It cannot actually create an LXC or run podman — but the plan only claims command assertion, which is achievable. The harness is real and reusable.
|
||||||
|
|
||||||
|
**Finding F1.5 (good — callsites verified).** PLAN P2 T3/T4/T5 reference `init.go:254-266`, `linux/bootstrap.go:160-172`, `proxmox/bootstrap.go:250-255`. Verified: `init.go:256` calls `installTraefikLocal()`; `linux/bootstrap.go:170` calls `traefik.InstallRemote("", sshExecFn)`; `proxmox/bootstrap.go:253` calls `traefik.InstallRemote("", runRemote)`. The line numbers are accurate within a few lines. The plan was written against the real callsites.
|
||||||
|
|
||||||
|
### Axis 2 — Scope
|
||||||
|
|
||||||
|
**Finding F2.1 (BLOCKER — mTLS scope expansion chosen, no phase exists).** During this grill (G-003) the operator chose "Wire real mTLS now" over the plan's `tls: {}` default-cert approach. This requires step-ca to mint server certs into `/etc/traefik/dynamic` + dynamic `tls.certificates` + `tls.options.default.clientAuth.caFiles`. **Step-ca server-cert minting was deferred since v0.11 and is not implemented.** The 9-phase plan contains no such phase. The operator then chose (G-004-adjacent) to add a step-ca cert minting phase, growing v0.14 from 9 to 10 phases. The plan as written does not reflect this. Until a P-step for cert minting is added, the plan is incomplete relative to the chosen direction.
|
||||||
|
**Impact**: v0.14 cannot ship real mTLS on its current 9 phases. Either add the phase (10 phases, more risk) or revert to `tls: {}` and defer mTLS to v0.15.
|
||||||
|
|
||||||
|
**Finding F2.2 (regression — `traefik-on-public-ip` opt-out lost).** REQ-100 (Complete, v0.11) established the `--public-binding=traefik-on-public-ip` opt-out: traefik binds `:443`/`:80` directly instead of `127.0.0.1:8443`/`8080` + nft DNAT. The existing `RenderTraefikStaticConfig` (`internal/emitter/traefik.go:273`) implements this via `TraefikStaticOpts.PublicBinding`. The v0.14 plan's baked image (`docker/orca-traefik/traefik.yml`, PLAN lines 33-37) hard-codes `127.0.0.1:8443`/`8080` — there is no opt-out path in the image. Baking the static config into the image freezes out the opt-out mode that v0.11 shipped. This is a regression of a Completed requirement.
|
||||||
|
**Evidence**: `internal/emitter/traefik.go:255-266` (`publicWebSecure`/`publicWeb` switch on `PublicBinding`); `internal/emitter/traefik.go:282-300` (rendered static config); REQUIREMENTS REQ-100 line 218.
|
||||||
|
|
||||||
|
**Finding F2.3 (9 phases is borderline; 10 is too many).** Even before the mTLS expansion, 9 phases for an "ingress bootstrap completeness" milestone is heavy. P5 (proxmox native) and P6 (floating-IP) are the two most complex (LXC creation + podman-in-LXC + apt-get + nft-inside-LXC). They could potentially be merged into one phase with two code paths, since they share `ProvisionIngressLXC` plumbing. With the mTLS expansion, 10 phases is too many for a single milestone — split v0.14 into v0.14a (linux ingress + mTLS) and v0.14b (proxmox ingress).
|
||||||
|
|
||||||
|
**Finding F2.4 (good — research-validated decisions are sound).** The 7 research topics (nft postrouting, pve-firewall priority, LXC features, traefik Dockerfile, pct create syntax, podman restart, SELinux) are well-sourced and the recommended approaches are technically correct. This is the strongest artifact in the v0.14 dossier.
|
||||||
|
|
||||||
|
### Axis 3 — Dependencies
|
||||||
|
|
||||||
|
**Finding F3.1 (good — ordering is correct).** P1 (image) → P2 (reconciler) → P3 (nft+init) → P4 (remote linux) → P5 (proxmox native) → P6 (floating-IP) → P7 (doctor+tests) → P8 (ship). Each phase references the prior phase's output (P3 T6 calls `EnsureTraefikContainerLocal` from P2; P5 T4 step 10 calls `EnsureTraefikContainerRemote` from P2). No phase can ship before its prerequisite.
|
||||||
|
|
||||||
|
**Finding F3.2 (hidden dependency — P2 T8 depends on P1 T1).** P2 T8 (drop `certResolver: orca`) edits `internal/emitter/traefik.go`. P1 T1 bakes the static config. The dynamic config (`traefik.go`) and static config (`docker/orca-traefik/traefik.yml`) must be consistent: if the static config has no `certificatesResolvers.orca` (P1 T1 correctly omits it) but the dynamic config still references `certResolver: orca` (until P2 T8), traefik logs a warning on every reload. The plan orders P1 before P2, so there is a window (P1 shipped, P2 not yet) where the published image + the live dynamic config are inconsistent. This is acceptable only if P1 and P2 ship in quick succession; if P1 stalls, the image is published with a known TLS-config mismatch.
|
||||||
|
|
||||||
|
### Axis 4 — Security
|
||||||
|
|
||||||
|
**Finding F4.1 (good — nft injection guard already present).** `internal/emitter/nft.go:104-130` (`partitionTrustedProbes`) validates every `TrustedProbes` entry as IP/CIDR before rendering (F9 guard). The v0.14 plan adds `DNATTarget` (P3 T1) — this is a NEW string field rendered directly into `dnat to <DNATTarget>:8443`. **The plan does not specify validation of `DNATTarget`.** If `DNATTarget` is user-controllable (via `--floating-ip` or cluster config), an unvalidated value is an nft-syntax injection vector. The existing F9 guard covers `TrustedProbes`; `DNATTarget` needs the same treatment.
|
||||||
|
**Binding**: C-51 (see below).
|
||||||
|
|
||||||
|
**Finding F4.2 (container escape surface — LXC nesting).** `--features nesting=1,keyctl=1,fuse=1` (P5/P6) is the documented requirement for podman-in-LXC, but nesting exposes host procfs/sysfs to the guest (Proxmox docs, RESEARCH Topic 3). This is an accepted tradeoff for container-in-container, but the plan does not document the threat-model acceptance. For a "production hardening" lineage, the ingress LXC is now a privileged-ish surface (nesting+keyctl) running a podman container pulling an image from a registry. The supply chain is: `git.cloudinit.dev/coreci/orca-traefik:<version>` (P1). If the registry is compromised or the tag is re-pushed, the ingress LXC runs attacker code at the host-LXC boundary. The plan has no image-signing/verification step (`podman pull --cert-dir` or cosign).
|
||||||
|
**Binding**: C-52 (see below).
|
||||||
|
|
||||||
|
**Finding F4.3 (nft rules injection via re-apply).** P3 T4 changes `flush table` to `delete table`. RESEARCH Topic 1 establishes that `delete table` on a missing table errors, and the fix is to pre-create the table (`nft add table inet orca-ingress 2>/dev/null || true`) before `nft -f`. P3 T5 step 4 does this. Good. But the re-apply path (P3 T5 step 5, P4 T1 step 5) runs `nft -f` which replaces the whole table — if a concurrent process (pve-firewall, operator) adds rules to `orca-ingress` between the pre-create and the `nft -f`, they are wiped. This is by design (orca owns the table) but should be documented as "orca is the sole owner of `table inet orca-ingress`".
|
||||||
|
|
||||||
|
### Axis 5 — Operational
|
||||||
|
|
||||||
|
**Finding F5.1 (BLOCKER — no upgrade path from v0.13 binary+systemd to v0.14 podman).** P2 T6 (line 104) says "update legacy cutover to pull new image + recreate container instead of sed-ing traefik.yml". But `internal/cli/upgrade.go:244-407` implements a Traefik `:443` → `127.0.0.1:8443` cutover that does `systemctl restart traefik` (line 382) — it restarts the **systemd service**, not a container. P2 removes systemd unit generation. There is **no phase** that: (a) detects the legacy `orca-traefik.service`, (b) stops+disables it, (c) removes `/usr/local/bin/traefik` + `/etc/systemd/system/orca-traefik.service`, (d) then runs `EnsureTraefikContainerLocal`. Without this, upgrading a live v0.13 cluster leaves a dead systemd unit AND a new podman container both trying to bind `127.0.0.1:8443` → port conflict, traefik down.
|
||||||
|
**Resolved by**: G-004 (P2 T6 must remove legacy unit + binary, idempotent, tested).
|
||||||
|
|
||||||
|
**Finding F5.2 (podman/podman-restart.service unavailable).** P2 T2 enables `podman-restart.service`. RESEARCH Topic 6 establishes this is not enabled by default on Ubuntu 24.04 and must be enabled. But the plan does not specify what happens if `podman` is not installed on the target host (a fresh linux node join, P4). `EnsureTraefikContainerLocal`/`Remote` calls `podman pull`/`podman run` — if `podman` is absent, this fails. The plan has no "install podman first" step for the linux topology (P3/P4). For proxmox (P5/P6) the plan installs podman via `apt-get install -y podman conmon crun fuse-overlayfs` inside the LXC (T4 step 4 / T1 step 6). But for linux nodes (P3/P4), there is no podman-install step. This is an under-specification.
|
||||||
|
**Binding**: C-50 (see below).
|
||||||
|
|
||||||
|
**Finding F5.3 (good — `--network host` is correct).** RESEARCH Topic 3 confirms `--network host` inside an LXC binds the LXC's netns, so traefik binds `127.0.0.1:8080/8443` on the LXC loopback and nft on the PVE host DNATs to the LXC IP. This is sound. The `DNATTarget` parameterization (D-262) correctly distinguishes `127.0.0.1` (linux/localhost) from `<lxc-ip>` (proxmox native).
|
||||||
|
|
||||||
|
**Finding F5.4 (reboot persistence chain has a gap).** RESEARCH Topic 6 establishes the chain: Proxmox boot → `--onboot 1` starts LXC → LXC systemd starts → `podman-restart.service` restarts container. P5 T4 step 6 and P6 T1 step 8 enable `podman-restart.service` inside the LXC. But `--onboot 1` starts the LXC **after** the Proxmox host's network is up — if the floating IP is on a bridge that depends on a physical link that's slow to come up, the LXC may start before the bridge is ready, and the floating-IP `eth0` config may fail. The plan does not address LXC-start ordering relative to bridge readiness.
|
||||||
|
|
||||||
|
### Axis 6 — Testing
|
||||||
|
|
||||||
|
**Finding F6.1 (good — harness is real).** `fakeSSHServer` (Axis 1 F1.4) can assert `pct create` with the right `--features` and `net0` args, `apt-get install podman`, `podman run` with `--network host`. P7 T8's claims are achievable.
|
||||||
|
|
||||||
|
**Finding F6.2 (gap — no test for the v0.13→v0.14 upgrade).** G-004 requires the upgrade path to be tested. The plan's P2 Wave 4 (T10) tests the reconciler but not the legacy-removal path. There is no test that: (a) simulates a host with `orca-traefik.service` present, (b) runs the upgrade, (c) asserts the unit is stopped+disabled+removed, (d) asserts the podman container is running. This must be added to P2 T10.
|
||||||
|
|
||||||
|
**Finding F6.3 (gap — no test for DNATTarget validation).** F4.1 identifies `DNATTarget` as an injection vector. P3 T8 (nft_test) asserts `DNATTarget` substitution but does not specify a test for invalid `DNATTarget` values (e.g. `1.2.3.4:8443; flush ruleset`). The existing F9 guard test pattern should be extended.
|
||||||
|
|
||||||
|
**Finding F6.4 (good — doctor_nft extension is incremental).** P3 T7 extends `doctor_nft.go`. The existing file (`internal/cli/doctor_nft.go`) is well-structured with `nftCheckResult` lines; adding postrouting/masquerade/DNATTarget assertions is straightforward.
|
||||||
|
|
||||||
|
### Axis 7 — Performance
|
||||||
|
|
||||||
|
**Finding F7.1 (apt-get install podman on every proxmox join — 30-60s).** CLARIFY D-263 acknowledges "~30-60s to the join time". P5 T4 step 4 and P6 T1 step 6 run `apt-get update && apt-get install -y podman conmon crun fuse-overlayfs nftables` inside the LXC on every join. This is acceptable for a one-time bootstrap but painful if re-run. The reconciler must be idempotent (skip if podman already installed). The plan does not specify an idempotency check for the apt-get step.
|
||||||
|
**Binding**: C-53 (see below).
|
||||||
|
|
||||||
|
**Finding F7.2 (image pull latency).** `podman pull orca-traefik:<tag>` (P2 T1 step 3, P5/P6) pulls from `git.cloudinit.dev/coreci/`. On a proxmox host behind a slow link, this can take 10-30s for a ~150MB traefik image. The plan has no pull-timeout. If the registry is unreachable (offline-first is R-001!), the pull fails and traefik never starts. **R-001 (offline-first) is violated**: the plan depends on a registry pull at bootstrap time. There is no "pre-pull" or "bundle image into the LXC template" fallback.
|
||||||
|
**Binding**: C-54 (see below) — this is a tension with R-001 that the plan does not acknowledge.
|
||||||
|
|
||||||
|
**Finding F7.3 (nft re-apply disruption).** `nft -f` replaces the table atomically (single transaction). Existing connections are NOT disrupted (conntrack holds them). New connections during the apply window (<1ms) may be dropped. This is acceptable. No finding.
|
||||||
|
|
||||||
|
### Axis 8 — Cost
|
||||||
|
|
||||||
|
**Finding F8.1 (2 images per release — sustainable).** P1 adds `orca-traefik` image alongside `orca`. `.coreci.yml` gets a `container-publish-traefik` step (P1 T5). `scripts/release.sh` gets a second docker block (P1 T4). The traefik image is small (~150MB, Alpine-based). Registry storage: 2 images × N releases. At v0.13.x cadence (8 tags), that's 16 image-tags per milestone. Sustainable for a private Gitea registry. No finding.
|
||||||
|
|
||||||
|
**Finding F8.2 (good — release.sh extension is minimal).** P1 T4 adds ~8 lines to `scripts/release.sh` after line 212. Verified the insertion point (line 213 is end of existing docker block). Clean.
|
||||||
|
|
||||||
|
### Axis 9 — Completeness (3 topologies)
|
||||||
|
|
||||||
|
**Finding F9.1 (linux topology — covered by P3+P4).** `orca init` (P3) bootstraps nft+podman on the lead; `orca node join --type linux` (P4) does it remotely. Complete.
|
||||||
|
|
||||||
|
**Finding F9.2 (proxmox-native — covered by P5, but LXC IP discovery is hand-wavy).** P5 T4 step 9 says "Discover LXC IP via `pct config <vmid>` (parse `net0` line) or `pct exec <vmid> -- hostname -I`". But in native mode the LXC is created **without** a static IP (P5 T4 step 2 has no `ip=` in the `pct create` — unlike P6 which has `ip=<floating-ip>/<prefix>`). So the LXC gets a DHCP/bridge IP that is not known at create time. P5 step 9 discovers it after `pct start`. But step 10 then re-applies nft with the discovered IP. This is a two-phase apply: first apply with default `127.0.0.1` (wrong for native), then re-apply with LXC IP. The plan does not specify what happens to traffic between the first and second apply (it DNATs to 127.0.0.1:8443 on the PVE host where nothing listens → connections refused). There is a window of ingress downtime during native-mode bootstrap.
|
||||||
|
**Binding**: C-55 (see below).
|
||||||
|
|
||||||
|
**Finding F9.3 (floating-IP — covered by P6, but MAC uniqueness is untested).** P6 T4 generates a random `02:XX:XX:XX:XX:XX` MAC in interactive mode. The plan does not check for MAC collision on the bridge. RESEARCH Topic 5 pitfall 2: "hwaddr must be unique on the bridge". A random 02: prefix has 46 bits of entropy — collision is unlikely on a single bridge but not impossible across a multi-node cluster.
|
||||||
|
**Binding**: C-56 (see below).
|
||||||
|
|
||||||
|
**Finding F9.4 (no `localhost`/lead topology with podman).** P3 bootstraps the lead via `EnsureTraefikContainerLocal`. But `orca init` runs on the lead — if the lead has no podman installed (fresh host), `EnsureTraefikContainerLocal` fails. Same as F5.2 but for the lead. The plan assumes podman is present on the lead. No install step.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Binding Conditions (gates that MUST be met before a phase ships)
|
||||||
|
|
||||||
|
| ID | Gate | Phase | Severity |
|
||||||
|
|----|------|-------|----------|
|
||||||
|
| **C-50** | P2 must not break existing `orca init` on a host without podman installed — either install podman as part of `BootstrapLocalIngress` (P3) or emit a clear error with install instructions. Same for `orca node join --type linux` (P4). | P2/P3/P4 | BLOCKER |
|
||||||
|
| **C-51** | `NftClusterConfig.DNATTarget` must be validated as `net.ParseIP` or `ip:port` before rendering. Unvalidated values are an nft-syntax injection vector (same F9 guard as TrustedProbes). Test required. | P3 | BLOCKER |
|
||||||
|
| **C-52** | The `orca-traefik` image supply chain must be documented: registry is public (anonymous pull per REQ-045), no image signing in v0.14. Document the threat-model acceptance: a compromised registry = attacker code in the ingress LXC. Add `podman image trust` or cosign verification as a v0.15 hardening item. | P1/P8 | High |
|
||||||
|
| **C-53** | The `apt-get install podman` step inside the LXC (P5 T4 step 4, P6 T1 step 6) must be idempotent: check `command -v podman` first, skip if present. Re-running join on an existing LXC must not re-run apt-get. | P5/P6 | High |
|
||||||
|
| **C-54** | R-001 (offline-first) tension: `podman pull` at bootstrap requires registry reachability. Either (a) document that ingress bootstrap requires online access (exception to R-001), or (b) pre-bundle the `orca-traefik` image into the LXC template / load from a local archive. The plan must acknowledge this tension explicitly. | P2/P5/P6 | BLOCKER |
|
||||||
|
| **C-55** | P5 native-mode bootstrap must not create a window of ingress downtime. The first nft apply must use the LXC IP (discovered after `pct start` but before the first nft apply), OR the plan must accept and document the downtime window. Two-phase apply (default → LXC IP) is a transient outage. | P5 | High |
|
||||||
|
| **C-56** | P6 MAC generation must check for collision against existing nodes' MACs in the cluster registry. Reject or regenerate on collision. | P6 | Medium |
|
||||||
|
| **C-57** | P2 T6 must implement the v0.13→v0.14 upgrade: detect `orca-traefik.service`, stop+disable, remove `/usr/local/bin/traefik` + unit file, then `EnsureTraefikContainerLocal`. Idempotent. Tested with a simulated-legacy-host test (F6.2). | P2 | BLOCKER |
|
||||||
|
| **C-58** | The baked `docker/orca-traefik/traefik.yml` must not regress REQ-100's `traefik-on-public-ip` opt-out. Either (a) bake both configs and select via env/flag, or (b) document that the opt-out is dropped in v0.14 and update REQ-100, or (c) mount the static config from host (not baked) so `RenderTraefikStaticConfig` still works. | P1 | High |
|
||||||
|
| **C-59** | Migration for `ingress_mode` must be `0009_ingress_mode.sql`, NOT 0007 (already taken by certs_serial_unique). | P5 | BLOCKER |
|
||||||
|
| **C-60** | `certpaths.CAPath()` references in P3 T5 and P4 T1 must be corrected to `certpaths.CACertPath()`. | P3/P4 | BLOCKER (compile) |
|
||||||
|
| **C-61** | REQ-172 must be amended: `--restart=always` → `--restart=unless-stopped` (per RESEARCH Topic 6 + PLAN), and `:Z` → omitted (per G-002). The requirement text contradicts the plan and research. | P2 | High |
|
||||||
|
| **C-62** | If mTLS is in-scope for v0.14 (per G-003), a new phase must be added implementing step-ca server-cert minting into `/etc/traefik/dynamic` + dynamic `tls.certificates` + `tls.options.default.clientAuth.caFiles`. The plan currently has no such phase (P2 T8 emits `tls: {}`). | NEW PHASE | BLOCKER |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase Challenges (specific challenges a phase must overcome)
|
||||||
|
|
||||||
|
| ID | Challenge | Phase |
|
||||||
|
|----|-----------|-------|
|
||||||
|
| **PC-01** | P1 must bake a static config that doesn't regress the `traefik-on-public-ip` opt-out (REQ-100). Baking freezes the config; the opt-out needs a runtime switch. | P1 |
|
||||||
|
| **PC-02** | P2 must handle the 3-way TLS contradiction (CLARIFY D-257 vs RESEARCH Topic 4 vs PLAN T8). Per G-003, real mTLS is chosen — P2 alone cannot deliver it; a new phase is needed. | P2 + new |
|
||||||
|
| **PC-03** | P2 T6 must remove the legacy systemd unit + binary without breaking a running v0.13 cluster. The existing `upgrade.go` cutover logic (lines 244-407) must be rewritten to stop+disable+remove the unit, not `systemctl restart traefik`. | P2 |
|
||||||
|
| **PC-04** | P3 must install podman on the lead if absent (C-50). The plan assumes podman is present. `BootstrapLocalIngress` must either install it or fail with a clear message. | P3 |
|
||||||
|
| **PC-05** | P3 T5 must use `certpaths.CACertPath()` not the phantom `certpaths.CAPath()`. | P3 |
|
||||||
|
| **PC-06** | P3 T1 (`DNATTarget`) must validate input (C-51). New string field rendered into nft ruleset — injection risk. | P3 |
|
||||||
|
| **PC-07** | P3 T4 (first-apply flush-table fix) must work across nft versions. RESEARCH establishes `delete table` on missing table is version-dependent. The pre-create approach (`nft add table … 2>/dev/null \|\| true` before `nft -f`) is robust; the plan uses it (T5 step 4). Verify on Proxmox kernel. | P3 |
|
||||||
|
| **PC-08** | P5 native-mode has a two-phase nft apply (default 127.0.0.1 → LXC IP) that creates a transient outage window. Must be eliminated or documented (C-55). | P5 |
|
||||||
|
| **PC-09** | P5/P6 `apt-get install podman` inside LXC takes 30-60s and must be idempotent (C-53). Re-join must not re-install. | P5/P6 |
|
||||||
|
| **PC-10** | P5/P6 `podman pull` requires registry reachability, violating R-001 (offline-first). Must be acknowledged or mitigated (C-54). | P5/P6 |
|
||||||
|
| **PC-11** | P5/P6 LXC reboot chain (Proxmox boot → `--onboot 1` → LXC systemd → `podman-restart.service`) has a gap: LXC may start before bridge is ready. Floating-IP `eth0` config may fail. | P5/P6 |
|
||||||
|
| **PC-12** | P6 MAC generation must check for collision (C-56). | P6 |
|
||||||
|
| **PC-13** | P7 T8 integration test must include the v0.13→v0.14 upgrade path (F6.2): simulated legacy host → upgrade → assert unit removed + podman running. | P7 |
|
||||||
|
| **PC-14** | If the new mTLS phase is added (G-003), it must mint server certs into `/etc/traefik/dynamic` atomically (C-10 protocol) and traefik must reload them via the file provider watch. Step-ca minting was deferred since v0.11 — this is net-new work, not a completion. | new phase |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Binding Decisions (from interactive grill)
|
||||||
|
|
||||||
|
| ID | Decision | Rationale | Confidence | Alternatives rejected |
|
||||||
|
|----|----------|-----------|------------|---------------------|
|
||||||
|
| **G-001** | Migration for `ingress_mode` is `0009_ingress_mode.sql` | 0007 is already `certs_serial_unique`; 0008 is `audit_tamper_evidence`. Reusing 0007 breaks the migrator. | 0.95 | Renumber existing (breaks deployed DBs); accept collision (guaranteed defect) |
|
||||||
|
| **G-002** | Omit `:Z` flag on volume mounts; use `:ro` on both | RESEARCH Topic 7: `:Z` relabels host dirs to private container label, blocks host-side orca writes on SELinux. No-op on Ubuntu/Proxmox but a latent footgun. PLAN P2 T1 + RESEARCH agree; CLARIFY D-258 + REQ-172 are wrong. | 0.90 | Keep `:Z` (wrong on future SELinux); `:z` shared (unnecessary) |
|
||||||
|
| **G-003** | Wire real mTLS now (dynamic `tls.certificates` + `clientAuth.caFiles`) | Operator chose this over `tls: {}` default. Requires step-ca server-cert minting which is not implemented (deferred since v0.11). | 0.55 | `tls: {}` now (plan's approach, defers mTLS); keep `certResolver: orca` (broken — key doesn't exist in traefik v3.3) |
|
||||||
|
| **G-004** | P2 T6 must remove legacy systemd unit + binary on upgrade from v0.13 | Without it, upgrading a live v0.13 cluster leaves a dead systemd unit + a new podman container both binding 127.0.0.1:8443 → port conflict. Idempotent + tested. | 0.90 | Leave legacy unit (port conflict); fresh-installs only (unacceptable) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Escalations (unresolved, confidence < 0.60)
|
||||||
|
|
||||||
|
| ID | Escalation | Confidence | Reason |
|
||||||
|
|----|------------|------------|--------|
|
||||||
|
| **E-001** | G-003 (real mTLS now) creates a scope expansion that the 9-phase plan does not contain. The operator chose to add a step-ca cert minting phase (growing v0.14 to 10 phases) but the plan has not been updated to reflect this. Until the new phase is specified (scope, tasks, tests), v0.14's mTLS direction is **undetermined**. The plan as written ships `tls: {}` (no mTLS), which contradicts G-003. | 0.55 | The choice is made but the plan does not reflect it. This is a plan-spec gap, not a technical unknown. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Meta — Closing Review
|
||||||
|
|
||||||
|
**What the auditor would flag**:
|
||||||
|
1. The plan ships a known-broken TLS config. RESEARCH Topic 4 calls `certResolver: orca` "the biggest v0.14 finding" and says it does not exist in traefik v3.3. CLARIFY D-257 and REQ-171 still specify it. The plan (P2 T8) drops it but emits `tls: {}` (no real TLS). G-003 chose real mTLS, which the plan doesn't contain. **Three artifacts disagree on TLS.**
|
||||||
|
2. The plan regresses REQ-100 (`traefik-on-public-ip` opt-out) by baking the static config.
|
||||||
|
3. The plan has no upgrade path from v0.13 (binary+systemd) to v0.14 (podman). G-004 binds the fix but the plan must be updated.
|
||||||
|
4. The plan references a phantom function (`certpaths.CAPath()`) and a phantom migration number (0007). Both will fail at compile/execution time.
|
||||||
|
5. The plan's offline-first claim (R-001) is violated by `podman pull` at bootstrap (C-54).
|
||||||
|
|
||||||
|
**What the project is NOT doing that it should**:
|
||||||
|
- Image supply-chain verification (no cosign, no `podman image trust`).
|
||||||
|
- LXC-start ordering relative to bridge readiness (reboot persistence gap).
|
||||||
|
- Idempotency check for `apt-get install podman` inside LXC.
|
||||||
|
- MAC collision check on the bridge.
|
||||||
|
- A test for the v0.13→v0.14 upgrade path.
|
||||||
|
|
||||||
|
**Simplest 80%-value version**: Ship P1 (image) + P2 (reconciler, with legacy-removal) + P3 (nft+init) + P4 (linux remote) + P7 (doctor+tests). Defer P5 (proxmox native) and P6 (floating-IP) to v0.15. This delivers the linux topology (the most common) + the podman migration + nft completeness, and avoids the two most complex phases (LXC+podman-in-LXC). If G-003 (real mTLS) holds, add the mTLS phase to v0.14a. Proxmox ingress becomes v0.14b/v0.15.
|
||||||
|
|
||||||
|
**What must be true for v0.14 to succeed in 90 days**:
|
||||||
|
1. The 3-way TLS contradiction is resolved in the plan (not just in this grill). Today: unresolved.
|
||||||
|
2. The migration number is 0009. Today: plan says 0007 (wrong).
|
||||||
|
3. The upgrade path from v0.13 is specified and tested. Today: not specified.
|
||||||
|
4. The `certpaths.CAPath()` phantom is fixed. Today: not fixed.
|
||||||
|
5. The offline-first tension (podman pull) is acknowledged. Today: not acknowledged.
|
||||||
|
6. If mTLS is in-scope, the new phase is written. Today: no such phase.
|
||||||
|
|
||||||
|
**Confidence**: 0.45 that the plan as-written can ship v0.14 without rework. The research is strong; the plan diverges from it and from the codebase in load-bearing ways. The mTLS scope expansion (G-003) makes it worse unless the plan is updated.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Verdict
|
||||||
|
|
||||||
|
**RETHINK** (confidence 0.45).
|
||||||
|
|
||||||
|
The plan must be revised to:
|
||||||
|
1. Fix the migration number → 0009 (C-59, G-001).
|
||||||
|
2. Fix `certpaths.CAPath()` → `certpaths.CACertPath()` (C-60).
|
||||||
|
3. Add the v0.13→v0.14 upgrade path to P2 T6 (C-57, G-004).
|
||||||
|
4. Resolve the TLS model: either add a new mTLS phase (G-003) or revert to `tls: {}` and defer mTLS to v0.15. The plan cannot ship `certResolver: orca` (broken) and cannot ship `tls: {}` if G-003 holds.
|
||||||
|
5. Amend REQ-172: `--restart=unless-stopped` (not `always`), omit `:Z` (C-61, G-002).
|
||||||
|
6. Address the offline-first tension (C-54) or document the exception.
|
||||||
|
7. Address the REQ-100 regression (C-58): bake-vs-mount the static config.
|
||||||
|
8. Add `DNATTarget` validation (C-51).
|
||||||
|
9. Add podman-install step for linux topology (C-50).
|
||||||
|
10. If mTLS is in-scope, write the new phase (C-62, E-001).
|
||||||
|
|
||||||
|
Once these are addressed, the plan is feasible. The research foundation is solid; the plan just needs to actually follow it.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*This grill is recorded in `.ciagent/GRILL_v0.14.md`. Escalations are visible via `ciagent audit`. Binding decisions (G-001..G-004) should be promoted to PROJECT.md via a follow-up clarify or explicitly by the operator. The grill surfaces; it does not rewrite.*
|
||||||
+29
-23
@@ -10,41 +10,47 @@ deactivated:
|
|||||||
- network-engineer
|
- network-engineer
|
||||||
- devops-engineer
|
- devops-engineer
|
||||||
phase_specific:
|
phase_specific:
|
||||||
- uat-engineer (P12 only)
|
- release-engineer (P1 only — Dockerfile.traefik + release pipeline)
|
||||||
reason: |
|
reason: |
|
||||||
Orca v0.13 is a production-hardening milestone. The active roster is
|
Orca v0.14 is the ingress bootstrap completeness milestone. The active
|
||||||
trimmed to the four personas that own the hardening work:
|
roster owns the podman-traefik container + nft SNAT/DNAT + proxmox LXC
|
||||||
- lead-developer: coordinates phase decomposition, owns scheduler
|
ingress work:
|
||||||
wiring (R-022) and jobspec parser fixes (P03)
|
- lead-developer: coordinates phase decomposition, owns podman traefik
|
||||||
- backend-engineer: owns ACL enforcement wiring (R-023), injection
|
reconciler (P2), nft emitter extension (P3), init/bootstrap wiring (P3,P4)
|
||||||
hardening (P02), transport/SSH safety (P08), concurrency (P07)
|
- backend-engineer: owns Dockerfile.traefik + release pipeline (P1),
|
||||||
- data-engineer: owns SQLite busy_timeout, audit chain race fix,
|
proxmox native ingress mode (P5), floating-IP LXC provisioning (P6),
|
||||||
migration safety, DB retention (P05, P07, P09)
|
doctor ingress (P7)
|
||||||
- security-engineer: owns toolchain vulns (P01), seal/audit CLI
|
- data-engineer: owns IngressMode schema migration (P5), node record
|
||||||
(P05), auth init-idp (P06), key zeroing, WebAuthn reg auth (P04)
|
updates for floating-IP LXC registration (P6)
|
||||||
|
- security-engineer: owns nft priority collision fix (pve-firewall
|
||||||
|
coexistence), step-ca root CA push to nodes, TLS model change
|
||||||
|
(certResolver → dynamic tls.certificates)
|
||||||
|
|
||||||
network-engineer and devops-engineer are deactivated — their territory
|
network-engineer and devops-engineer are deactivated — their territory
|
||||||
(nft ruleset, collector scripts) is covered by backend-engineer in
|
(nft ruleset, container deployment) is covered by backend-engineer +
|
||||||
this milestone. cli-engineer and frontend-engineer remain deactivated
|
lead-developer in this milestone. cli-engineer and frontend-engineer
|
||||||
(no CLI framework or UI work).
|
remain deactivated (no CLI framework or UI work).
|
||||||
|
|
||||||
uat-engineer is phase-specific for P12 (UAT plan + signoff script).
|
release-engineer is phase-specific for P1 (Dockerfile.traefik +
|
||||||
|
release.sh + .coreci.yml container-publish-traefik step).
|
||||||
|
|
||||||
Territory enforcement is warn mode (config.json
|
Territory enforcement is warn mode (config.json
|
||||||
personas.territory_enforcement=warn). Cross-territory fixes (e.g. a
|
personas.territory_enforcement=warn). Cross-territory fixes (e.g. a
|
||||||
fix that touches both daemon handlers and SQLite) are allowed with a
|
fix that touches both nft emitter and proxmox bootstrap) are allowed
|
||||||
warning.
|
with a warning.
|
||||||
|
|
||||||
Framework alignment (from go.mod):
|
Framework alignment (from go.mod):
|
||||||
- lead-developer: cobra
|
- lead-developer: cobra
|
||||||
- backend-engineer: cobra, connectrpc
|
- backend-engineer: cobra, podman (CLI), pct (CLI via SSH)
|
||||||
- data-engineer: modernc/sqlite
|
- data-engineer: modernc/sqlite
|
||||||
- security-engineer: go-webauthn, go-jose, x/crypto
|
- security-engineer: nft, step-ca, TLS
|
||||||
- uat-engineer: bash, bats
|
- release-engineer: docker, .coreci.yml
|
||||||
|
|
||||||
Constraint alignment:
|
Constraint alignment:
|
||||||
- All personas: offline-first, no-redundant-implementations
|
- All personas: offline-first, no-redundant-implementations
|
||||||
- backend-engineer: API-first, error-handling, security-first
|
- backend-engineer: API-first, error-handling, security-first,
|
||||||
|
container-first (R-024)
|
||||||
- data-engineer: schema-first, migration-safe, local-storage-only
|
- data-engineer: schema-first, migration-safe, local-storage-only
|
||||||
- security-engineer: deny-by-default, zero-trust, no-passwords (R-021)
|
- security-engineer: deny-by-default, zero-trust, no-passwords (R-021),
|
||||||
- uat-engineer: idempotent, read-only, claim-coverage
|
pve-firewall-coexistence
|
||||||
|
- release-engineer: per-release-tagging, registry-auth
|
||||||
@@ -0,0 +1,340 @@
|
|||||||
|
# PLAN v0.14: Ingress Bootstrap Completeness
|
||||||
|
|
||||||
|
**Status**: active. 9 phases (P0 + P1..P7 + P8 final). Each phase ships a
|
||||||
|
patch tag on the v0.13.x line. This plan references requirement IDs from
|
||||||
|
REQUIREMENTS.md and follows the vertical-slice integrity rule (each phase
|
||||||
|
is independently shippable).
|
||||||
|
|
||||||
|
**Research-validated decisions** (from RESEARCH_v0.14.md + GRILL_v0.14.md):
|
||||||
|
|
||||||
|
- nft postrouting: `ip saddr 127.0.0.0/8 oifname != "lo" masquerade`
|
||||||
|
- nft first-apply: pre-create table (`nft add table inet orca-ingress 2>/dev/null || true`) before `nft -f`
|
||||||
|
- pve-firewall: shift orca input/forward chains to `priority -10` (before pve-firewall's 0)
|
||||||
|
- LXC features: `nesting=1,keyctl=1,fuse=1` (fuse=1 for fuse-overlayfs)
|
||||||
|
- traefik TLS: **drop `certResolver: orca`** — does not exist in v3.3; emit `tls: {}` for v0.14 (real mTLS via dynamic `tls.certificates` + `clientAuth.caFiles` deferred to v0.15 — grill G-003 confidence 0.55 < 0.60 threshold, auto-resolved to defer)
|
||||||
|
- podman restart: `--restart=unless-stopped` + enable `podman-restart.service`
|
||||||
|
- volumes: omit `:Z` flag, use `:ro` on both mounts
|
||||||
|
- traefik image: `FROM traefik:v3.3.0`, `ENTRYPOINT ["/traefik"]` inherited, `CMD ["--configFile=/etc/traefik/traefik.yml"]`
|
||||||
|
- `NftClusterConfig.DNATTarget`: default `127.0.0.1:8443`/`:8080`; proxmox-native = `<lxc-ip>:8443`/`:8080`
|
||||||
|
- Migration: `0009_ingress_mode.sql` (NOT 0007 — already taken by certs_serial_unique)
|
||||||
|
- CA path: `certpaths.CACertPath()` (NOT `certpaths.CAPath()` — does not exist)
|
||||||
|
- Upgrade path: P2 must detect+remove legacy `orca-traefik.service` + `/usr/local/bin/traefik` before starting podman container (C-57)
|
||||||
|
- Podman install: `BootstrapLocalIngress` and `BootstrapRemoteIngress` must install podman if absent (C-50)
|
||||||
|
- Offline-first: `podman pull` requires registry reachability — documented exception to R-001 for ingress bootstrap (C-54)
|
||||||
|
- Static config: mount from host (not baked) to preserve `traefik-on-public-ip` opt-out (C-58)
|
||||||
|
- DNATTarget validation: `net.ParseIP` or `ip:port` parse before render (C-51)
|
||||||
|
- apt-get idempotency: `command -v podman` check before install (C-53)
|
||||||
|
- MAC collision: check against existing nodes' MACs (C-56)
|
||||||
|
- Native-mode nft: first apply uses LXC IP (not default 127.0.0.1) — discover LXC IP before first nft apply (C-55)
|
||||||
|
|
||||||
|
## Phase 0: Pre-execution (this phase)
|
||||||
|
|
||||||
|
**Status**: complete. SPECIFY → CLARIFY → RESEARCH → PLAN → GRILL → SHIP.
|
||||||
|
Ships as `v0.13.0`.
|
||||||
|
|
||||||
|
## Phase 1: `orca-traefik` container image + release pipeline (REQ-171)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.1` | **Type**: feat | **Persona**: release-engineer (phase-specific) + backend-engineer
|
||||||
|
|
||||||
|
### Wave 1 (image)
|
||||||
|
- **T1**: Create `docker/orca-traefik/traefik.yml` — the **default** static config baked into the image (used when no host-side override is mounted):
|
||||||
|
```yaml
|
||||||
|
entryPoints:
|
||||||
|
websecure:
|
||||||
|
address: "127.0.0.1:8443"
|
||||||
|
web:
|
||||||
|
address: "127.0.0.1:8080"
|
||||||
|
traefik:
|
||||||
|
address: "127.0.0.1:8081"
|
||||||
|
providers:
|
||||||
|
file:
|
||||||
|
directory: "/etc/traefik/dynamic"
|
||||||
|
watch: true
|
||||||
|
log:
|
||||||
|
level: INFO
|
||||||
|
format: json
|
||||||
|
accessLog:
|
||||||
|
format: json
|
||||||
|
```
|
||||||
|
No `certificatesResolvers` (research finding: does not exist for CA-based; TLS is via dynamic config).
|
||||||
|
**C-58**: The baked config is a default. The podman run command also mounts a host-side `/etc/traefik/traefik.yml` if it exists (overriding the baked one), preserving the `traefik-on-public-ip` opt-out (REQ-100). The reconciler renders the static config via `emitter.RenderTraefikStaticConfig` to `/etc/traefik/traefik.yml` on the host, then mounts it `-v /etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro`. This way `PublicBinding` opt-out still works.
|
||||||
|
- **T2**: Create `Dockerfile.traefik` at repo root:
|
||||||
|
```dockerfile
|
||||||
|
FROM traefik:v3.3.0
|
||||||
|
LABEL org.opencontainers.image.title="orca-traefik"
|
||||||
|
LABEL org.opencontainers.image.source="https://git.cloudinit.dev/coreci/orca"
|
||||||
|
COPY docker/orca-traefik/traefik.yml /etc/traefik/traefik.yml
|
||||||
|
CMD ["--configFile=/etc/traefik/traefik.yml"]
|
||||||
|
```
|
||||||
|
(ENTRYPOINT inherited as `["/traefik"]` from base image.)
|
||||||
|
- **T3**: Create placeholder `docker/orca-traefik/step-ca-root.crt` (empty file) — real CA is volume-mounted at runtime. If absent, traefik starts without TLS termination (graceful).
|
||||||
|
|
||||||
|
### Wave 2 (release pipeline)
|
||||||
|
- **T4**: `scripts/release.sh` — add a second docker block after the existing one (~line 212):
|
||||||
|
```bash
|
||||||
|
# Build + push orca-traefik image
|
||||||
|
TRAEFIK_IMAGE="${CONTAINER_REGISTRY}/${CONTAINER_OWNER}/orca-traefik"
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
docker build -f Dockerfile.traefik -t "${TRAEFIK_IMAGE}:${VERSION}" -t "${TRAEFIK_IMAGE}:latest" .
|
||||||
|
docker push "${TRAEFIK_IMAGE}:${VERSION}"
|
||||||
|
docker push "${TRAEFIK_IMAGE}:latest"
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
- **T5**: `.coreci.yml` — add `container-publish-traefik` step mirroring `container-publish` with `CONTAINER_IMAGE=orca-traefik` + `DOCKERFILE=Dockerfile.traefik`.
|
||||||
|
|
||||||
|
### Wave 3 (tests)
|
||||||
|
- **T6**: Verify `docker build -f Dockerfile.traefik .` succeeds and the resulting image starts traefik with `--configFile=/etc/traefik/traefik.yml` (can test with `docker run --rm orca-traefik --version`).
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `Dockerfile.traefik` builds successfully
|
||||||
|
- [ ] Image starts traefik with the baked static config
|
||||||
|
- [ ] `release.sh` publishes `orca-traefik:<version>` + `:latest`
|
||||||
|
- [ ] `.coreci.yml` has `container-publish-traefik` step
|
||||||
|
|
||||||
|
## Phase 2: Podman traefik reconciler (REQ-172)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.2` | **Type**: feat | **Persona**: lead-developer
|
||||||
|
|
||||||
|
### Wave 1 (reconciler)
|
||||||
|
- **T1**: Rewrite `internal/traefik/install.go` — replace binary+systemd install with podman container reconciler:
|
||||||
|
- `EnsureTraefikContainer(ctx, execFn, image, tag)` — idempotent:
|
||||||
|
1. `podman inspect orca-traefik` → if running, no-op; if stopped, `podman start orca-traefik`; if absent, go to step 2
|
||||||
|
2. `mkdir -p /etc/traefik/dynamic /etc/orca`
|
||||||
|
3. `podman pull <image>:<tag>`
|
||||||
|
4. `podman run -d --name orca-traefik --restart=unless-stopped --network host -v /etc/traefik/dynamic:/etc/traefik/dynamic:ro -v /etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro <image>:<tag>`
|
||||||
|
- `EnsureTraefikContainerLocal(ctx, image, tag)` — uses `exec.CommandContext("podman", ...)` locally
|
||||||
|
- `EnsureTraefikContainerRemote(ctx, execFn, image, tag)` — uses SSH exec function
|
||||||
|
- Image/tag resolution: `git.cloudinit.dev/coreci/orca-traefik:<version>` where version = `internal/cli.version` (or `latest` if dev)
|
||||||
|
- **Remove** systemd unit generation + `systemctl enable`
|
||||||
|
- **T2**: Add `podman-restart.service` enable step: `systemctl enable --now podman-restart.service` (research finding: needed for reboot persistence)
|
||||||
|
- **T2a**: **C-50**: `EnsureTraefikContainerLocal`/`Remote` must check `command -v podman` first. If absent: on localhost, attempt `apt-get install -y podman` (or fail with clear install instructions if no apt). On remote, `apt-get install -y podman conmon crun fuse-overlayfs` via SSH. Non-fatal warn if podman unavailable (offline host) — traefik won't start but `orca init` succeeds (same tolerance as v0.13).
|
||||||
|
|
||||||
|
### Wave 2 (callsite updates + v0.13 upgrade path)
|
||||||
|
- **T3**: `internal/cli/init.go:254-266` — replace `installTraefikLocal()` with `EnsureTraefikContainerLocal`
|
||||||
|
- **T4**: `internal/linux/bootstrap.go:160-172` — replace `traefik.InstallRemote` with `EnsureTraefikContainerRemote`
|
||||||
|
- **T5**: `internal/proxmox/bootstrap.go:250-255` — replace `traefik.InstallRemote` with `EnsureTraefikContainerRemote` (for native mode; floating-IP calls it inside the LXC in P6)
|
||||||
|
- **T6**: **C-57 (v0.13→v0.14 upgrade path)**: `internal/cli/upgrade.go` — rewrite the Traefik cutover to:
|
||||||
|
1. Detect legacy `orca-traefik.service`: `systemctl is-active orca-traefik.service`
|
||||||
|
2. If active: `systemctl stop orca-traefik.service && systemctl disable orca-traefik.service`
|
||||||
|
3. Remove `/etc/systemd/system/orca-traefik.service` + `/usr/local/bin/traefik` (if exists)
|
||||||
|
4. `systemctl daemon-reload`
|
||||||
|
5. Render static config via `emitter.RenderTraefikStaticConfig` to `/etc/traefik/traefik.yml`
|
||||||
|
6. `EnsureTraefikContainerLocal` (pull + run podman container)
|
||||||
|
7. Idempotent: if no legacy unit, skip steps 1-4
|
||||||
|
- **T7**: `internal/cli/traefik_install.go` — update CLI wrapper
|
||||||
|
|
||||||
|
### Wave 3 (TLS model fix — research finding)
|
||||||
|
- **T8**: `internal/emitter/traefik.go` — drop `certResolver: orca` from the dynamic config router TLS stanza (line ~185-188). Replace with `tls: {}` (empty TLS stanza — traefik uses its default cert). Document that real mTLS via `tls.certificates` + `tls.options.default.clientAuth.caFiles` will be wired when step-ca mints certs into the dynamic dir (post-v0.14 or v1.x).
|
||||||
|
- **T9**: Update `internal/emitter/traefik_test.go` — remove assertion for `certResolver: orca`, add assertion for `tls: {}` presence.
|
||||||
|
|
||||||
|
### Wave 4 (tests)
|
||||||
|
- **T10**: Create `internal/traefik/install_test.go` (new file — F1.3: does not exist today) — assert `podman run` is invoked (not `curl|tar`), `--restart=unless-stopped --network host` present, volume mounts present, `podman-restart.service` enabled.
|
||||||
|
- **T10a**: **C-57/F6.2**: Add v0.13→v0.14 upgrade test: simulate a host with `orca-traefik.service` present (fake), run upgrade, assert unit stopped+disabled+removed, podman container running.
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `orca init` → `podman inspect orca-traefik` shows running
|
||||||
|
- [ ] `podman logs orca-traefik` shows traefik started with baked config
|
||||||
|
- [ ] No systemd `orca-traefik.service` generated
|
||||||
|
- [ ] `--restart=unless-stopped` + `podman-restart.service` enabled
|
||||||
|
- [ ] `certResolver: orca` removed from dynamic config
|
||||||
|
|
||||||
|
## Phase 3: nft SNAT+DNAT + `orca init` ingress bootstrap (REQ-173)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.3` | **Type**: feat | **Persona**: lead-developer + security-engineer
|
||||||
|
|
||||||
|
### Wave 1 (nft emitter extension)
|
||||||
|
- **T1**: `internal/emitter/nft.go` — add `DNATTarget` field to `NftClusterConfig` (default `127.0.0.1`). Render DNAT rules as `dnat to <DNATTarget>:8443` / `dnat to <DNATTarget>:8080`. **C-51**: Validate `DNATTarget` with `net.ParseIP` before rendering. Reject invalid values with error (same F9 injection guard pattern as `partitionTrustedProbes`).
|
||||||
|
- **T2**: `internal/emitter/nft.go` — add `EnableSNAT bool` (default true) + `postrouting` chain:
|
||||||
|
```nft
|
||||||
|
chain postrouting {
|
||||||
|
type nat hook postrouting priority 100; policy accept;
|
||||||
|
ip saddr 127.0.0.0/8 oifname != "lo" masquerade
|
||||||
|
}
|
||||||
|
```
|
||||||
|
Only when `EnableSNAT == true`.
|
||||||
|
- **T3**: `internal/emitter/nft.go` — shift `input` and `forward` chain priorities from `filter` (=0) to `-10` (research finding: avoids pve-firewall same-priority undefined order).
|
||||||
|
- **T4**: `internal/emitter/nft.go` — fix first-apply flush-table bug: change `flush table inet orca-ingress` to `delete table inet orca-ingress` (nft ≥1.0 treats delete-of-missing as warning in `-f` mode). If that's version-unsafe, the apply step (T7) pre-creates the table.
|
||||||
|
|
||||||
|
### Wave 2 (ingress bootstrap)
|
||||||
|
- **T5**: New `internal/ingress/bootstrap.go`:
|
||||||
|
- `BootstrapLocalIngress(ctx)`:
|
||||||
|
1. `mkdir -p /etc/traefik/dynamic /etc/orca`
|
||||||
|
2. **C-60**: Push cluster root CA to `/etc/orca/step-ca-root.crt` from `certpaths.CACertPath()` (if exists, else empty placeholder)
|
||||||
|
3. Render static config via `emitter.RenderTraefikStaticConfig` to `/etc/traefik/traefik.yml` (preserves `traefik-on-public-ip` opt-out — C-58)
|
||||||
|
4. Render `orca.nft` via `NftEmitter.RenderNftConfig` + write to `/etc/nftables.d/orca.nft`
|
||||||
|
5. Pre-create table: `nft add table inet orca-ingress 2>/dev/null || true`
|
||||||
|
6. Apply: `nft -f /etc/nftables.d/orca.nft`
|
||||||
|
7. **C-50**: Ensure podman installed (check `command -v podman`, install if absent)
|
||||||
|
8. `EnsureTraefikContainerLocal` (from P2) — mounts `/etc/traefik/traefik.yml:ro` + `/etc/traefik/dynamic:ro` + `/etc/orca/step-ca-root.crt:ro`
|
||||||
|
- Each step non-fatal warn (offline host tolerance)
|
||||||
|
- **T6**: Wire into `internal/cli/init.go` after `EnsureTraefikContainerLocal` (Step 4e, replacing the old traefik install step).
|
||||||
|
|
||||||
|
### Wave 3 (doctor nft update)
|
||||||
|
- **T7**: `internal/cli/doctor_nft.go` — extend assertions: postrouting masquerade present, DNAT target matches `NftClusterConfig.DNATTarget`.
|
||||||
|
|
||||||
|
### Wave 4 (tests)
|
||||||
|
- **T8**: `internal/emitter/nft_test.go` — assert postrouting chain present when `EnableSNAT=true`, absent when false. Assert `DNATTarget` substitution. Assert priority `-10` on input/forward.
|
||||||
|
- **T9**: Integration test: `orca init` → `nft list table inet orca-ingress` shows DNAT + postrouting; `podman inspect orca-traefik` running.
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `orca init` → nft table has DNAT + postrouting masquerade
|
||||||
|
- [ ] nft input/forward chains at priority -10
|
||||||
|
- [ ] First-apply doesn't error (table pre-created or delete-table idiom)
|
||||||
|
- [ ] `/etc/orca/step-ca-root.crt` exists (real CA or placeholder)
|
||||||
|
- [ ] `podman inspect orca-traefik` running
|
||||||
|
|
||||||
|
## Phase 4: `orca node join --type linux` remote ingress bootstrap (REQ-174)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.4` | **Type**: feat | **Persona**: lead-developer
|
||||||
|
|
||||||
|
### Wave 1 (remote ingress)
|
||||||
|
- **T1**: `internal/ingress/bootstrap.go` — add `BootstrapRemoteIngress(ctx, execFn)`:
|
||||||
|
1. `mkdir -p /etc/traefik/dynamic /etc/orca` (remote)
|
||||||
|
2. **C-60**: Push step-ca root CA to remote `/etc/orca/step-ca-root.crt` from `certpaths.CACertPath()` via `WriteFile`
|
||||||
|
3. Render `orca.nft` + write to remote `/etc/nftables.d/orca.nft` via `WriteFile`
|
||||||
|
4. `nft add table inet orca-ingress 2>/dev/null || true` (remote)
|
||||||
|
5. `nft -f /etc/nftables.d/orca.nft` (remote)
|
||||||
|
6. `systemctl enable --now podman-restart.service` (remote)
|
||||||
|
7. `EnsureTraefikContainerRemote` (from P2)
|
||||||
|
- **T2**: Wire into `internal/linux/bootstrap.go` after the traefik container reconciler step.
|
||||||
|
- **T3**: Extend `linux.Result` with `IngressOK bool` for reporting.
|
||||||
|
|
||||||
|
### Wave 2 (tests)
|
||||||
|
- **T4**: Fake-SSH test: assert remote `nft -f` + `podman run` + `WriteFile` for step-ca CA invoked.
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `orca node join --type linux --host <ip>` → remote has podman traefik running + nft applied + step-ca CA mounted
|
||||||
|
- [ ] `doctor ingress --peer <linux-node>` passes
|
||||||
|
|
||||||
|
## Phase 5: Proxmox native ingress mode (REQ-175)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.5` | **Type**: feat | **Persona**: backend-engineer + data-engineer
|
||||||
|
|
||||||
|
### Wave 1 (flags + schema)
|
||||||
|
- **T1**: Add flags to `node join`: `--ingress-mode` (values: `native` default, `floating-ip`), `--floating-ip`, `--gateway`, `--mac`, `--net-prefix` (default `24`).
|
||||||
|
- **T2**: Add `IngressMode` field to `model.Node` (string: `""`, `"native"`, `"floating-ip"`).
|
||||||
|
- **T3**: **C-59**: Schema migration `0009_ingress_mode.sql` (NOT 0007 — already taken): `ALTER TABLE nodes ADD COLUMN ingress_mode TEXT DEFAULT '';`
|
||||||
|
|
||||||
|
### Wave 2 (native mode bootstrap)
|
||||||
|
- **T4**: In `proxmox.BootstrapProxmox`, when `IngressMode == "native"`:
|
||||||
|
1. On the PVE host: render+apply nft with `DNATTarget = <lxc-bridge-ip>` (the traefik LXC's IP, discovered after `pct start`)
|
||||||
|
2. Create unprivileged LXC with `--features nesting=1,keyctl=1,fuse=1` (research finding: fuse=1 for fuse-overlayfs). `pct create <vmid> local:vztmpl/<template> --hostname orca-traefik --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --onboot 1 --memory 2048 --swap 0 --rootfs local:8`
|
||||||
|
3. `pct start <vmid>`
|
||||||
|
3a. **C-55**: Discover LXC IP via `pct config <vmid>` (parse `net0` line for `ip=`) or `pct exec <vmid> -- hostname -I` BEFORE the first nft apply. The nft DNAT target is set to the LXC IP from the start — no two-phase apply, no downtime window.
|
||||||
|
4. Inside the LXC: **C-53**: `command -v podman >/dev/null 2>&1 || (apt-get update && apt-get install -y podman conmon crun fuse-overlayfs nftables)` — idempotent, skip if podman already installed
|
||||||
|
5. Configure podman storage (`/etc/containers/storage.conf`): `mount_program = "/usr/bin/fuse-overlayfs"` (fallback: `driver = "vfs"`)
|
||||||
|
6. `systemctl enable --now podman-restart.service` (inside LXC)
|
||||||
|
7. Push step-ca root CA into LXC
|
||||||
|
8. `EnsureTraefikContainerRemote` (podman pull + run inside LXC with `--network host`)
|
||||||
|
9. **C-55**: Render+apply nft on PVE host with `DNATTarget = <lxc-ip>` (discovered in step 3a) — single apply, no downtime window
|
||||||
|
|
||||||
|
### Wave 3 (registration)
|
||||||
|
- **T5**: Register PVE host as `proxmox` node with `IngressMode: "native"`.
|
||||||
|
|
||||||
|
### Wave 4 (tests)
|
||||||
|
- **T6**: Fake-SSH test: assert `pct create` with `--features nesting=1,keyctl=1,fuse=1`, `apt-get install podman`, `podman run` inside LXC, nft DNAT target = LXC IP.
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `orca node join --type proxmox --host <ip>` (native) → LXC created with nesting+keyctl+fuse
|
||||||
|
- [ ] Podman + orca-traefik running inside LXC
|
||||||
|
- [ ] PVE host nft DNATs to LXC IP
|
||||||
|
- [ ] `IngressMode: "native"` on node record
|
||||||
|
- [ ] Schema migration 0007 applied
|
||||||
|
|
||||||
|
## Phase 6: Proxmox floating-IP LXC "ingress" (REQ-176)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.6` | **Type**: feat | **Persona**: backend-engineer
|
||||||
|
|
||||||
|
### Wave 1 (LXC provisioning)
|
||||||
|
- **T1**: New `internal/proxmox/ingress_lxc.go` — `ProvisionIngressLXC(ctx, opts)`:
|
||||||
|
1. `pveam download local <template>` (idempotent, already in bootstrap)
|
||||||
|
2. Deterministic VMID for "ingress" (hash of host+"ingress")
|
||||||
|
3. `pct create <vmid> local:vztmpl/<template> --hostname ingress --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --net0 name=eth0,bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway> --onboot 1 --memory 2048 --swap 0 --rootfs local:8`
|
||||||
|
4. `pct start <vmid>`
|
||||||
|
5. Wait for LXC network (retry SSH to `<floating-ip>` with backoff, 60s timeout)
|
||||||
|
6. Inside the LXC: **C-53**: `command -v podman >/dev/null 2>&1 || (apt-get update && apt-get install -y podman conmon crun fuse-overlayfs nftables)` — idempotent
|
||||||
|
7. Configure podman storage (fuse-overlayfs / vfs fallback)
|
||||||
|
8. `systemctl enable --now podman-restart.service`
|
||||||
|
9. Push step-ca root CA into LXC
|
||||||
|
10. Render+apply nft INSIDE the LXC (DNAT `:443`→`127.0.0.1:8443`, `:80`→`127.0.0.1:8080` + postrouting masquerade) — `DNATTarget = 127.0.0.1` here because traefik runs with `--network host` inside the LXC
|
||||||
|
11. `EnsureTraefikContainerRemote` (podman pull + run inside LXC with `--network host`)
|
||||||
|
12. Push orca SSH pubkey into LXC for future `job run` traefik dynamic-config pushes
|
||||||
|
|
||||||
|
### Wave 2 (registration)
|
||||||
|
- **T2**: Register LXC as managed node: `registry.Join` with `Kind: "linux"`, `Name: "ingress"`, `Address: "<floating-ip>:8443"`, `OS: "linux"`, `IngressMode: "floating-ip"`.
|
||||||
|
- **T3**: Also register PVE host as `proxmox` node (for workload dispatch).
|
||||||
|
|
||||||
|
### Wave 3 (interactive prompting)
|
||||||
|
- **T4**: Interactive prompting in `joinProxmox` (node.go): when `--ingress-mode` empty and `!jsonOutput`:
|
||||||
|
- Prompt "Ingress mode [native/floating-ip] (default native): "
|
||||||
|
- If `floating-ip`: prompt for floating IP (validate `net.ParseIP`), gateway (validate `net.ParseIP`), MAC (validate `net.ParseMAC`; generate `02:XX:XX:XX:XX:XX` random if empty + confirm; **C-56**: check MAC against existing nodes' MACs in cluster registry, regenerate on collision), net-prefix (default 24, validate 8-32)
|
||||||
|
- In `--json` mode: require `--mac` explicitly if `--ingress-mode floating-ip` (no silent generation)
|
||||||
|
|
||||||
|
### Wave 4 (routing)
|
||||||
|
- **T5**: `joinProxmox` in `node.go` routes: native → P5 path; floating-ip → `ProvisionIngressLXC` + register LXC + register PVE host.
|
||||||
|
|
||||||
|
### Wave 5 (tests)
|
||||||
|
- **T6**: Fake-SSH test: assert `pct create` with `net0 name=eth0,bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway>`, `--features nesting=1,keyctl=1,fuse=1`; LXC registered as `linux` node named `ingress` at `<floating-ip>:8443`; PVE host registered as `proxmox`.
|
||||||
|
- **T7**: Interactive prompt test: stdin simulation → mode selection + param entry + validation.
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `orca node join --type proxmox --host <ip> --ingress-mode floating-ip --floating-ip 203.0.113.10 --gateway 203.0.113.1 --mac 02:01:02:03:04:05` → LXC `ingress` created
|
||||||
|
- [ ] LXC has podman traefik running + nft applied inside LXC
|
||||||
|
- [ ] Node `ingress` registered as `linux` at `203.0.113.10:8443`
|
||||||
|
- [ ] PVE host registered as `proxmox`
|
||||||
|
- [ ] Interactive prompt works when flags absent + not `--json`
|
||||||
|
- [ ] IP/MAC/gateway validation rejects invalid input
|
||||||
|
|
||||||
|
## Phase 7: `doctor ingress` + docs + integration tests (REQ-177, REQ-178, REQ-179)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.7` | **Type**: feat+docs+test | **Persona**: backend-engineer + lead-developer
|
||||||
|
|
||||||
|
### Wave 1 (doctor ingress)
|
||||||
|
- **T1**: `internal/cli/doctor_ingress.go` — `orca doctor ingress [--peer <name>]`:
|
||||||
|
1. `podman inspect orca-traefik` → running?
|
||||||
|
2. nft DNAT+SNAT applied (reuse `doctor_nft` logic)
|
||||||
|
3. `/etc/traefik/dynamic` exists
|
||||||
|
4. step-ca root CA mounted (`podman inspect` volume check or file-exists check)
|
||||||
|
5. For proxmox-native: checks the LXC (via `pct exec`)
|
||||||
|
6. For floating-ip: checks the ingress LXC over SSH
|
||||||
|
7. Uses SSH-push for remote peers
|
||||||
|
|
||||||
|
### Wave 2 (UAT assertions)
|
||||||
|
- **T2**: `scripts/uat-signoff.sh` — add assertions: `40 ingress_podman_traefik`, `41 ingress_nft_dnat_snat`, `42 ingress_linux_worker`, `43 ingress_proxmox_native_lxc` or `43 ingress_floating_ip_lxc` (depending on topology).
|
||||||
|
|
||||||
|
### Wave 3 (docs)
|
||||||
|
- **T3**: `docs/cli.md` — document `--ingress-mode`, `--floating-ip`, `--gateway`, `--mac`, `--net-prefix` flags + `orca doctor ingress`.
|
||||||
|
- **T4**: `docs/uat.md` — add floating-IP topology variant; update native topology to assert ingress bootstrap.
|
||||||
|
- **T5**: `docs/ingress.md` — podman-traefik image section: `Dockerfile.traefik`, volume mounts, TLS model (dynamic `tls.certificates`, not certResolver), `--network host` rationale.
|
||||||
|
- **T6**: `docs/docker.md` — `orca-traefik` image: build, publish, pull.
|
||||||
|
- **T7**: `.ciagent/ARCHITECTURE.md` — R-024 + ingress bootstrap section (3 topologies, nft+podman stack on each).
|
||||||
|
|
||||||
|
### Wave 4 (integration tests)
|
||||||
|
- **T8**: `tests/ingress_bootstrap_test.go` — hermetic fake-SSH harness:
|
||||||
|
- init → podman traefik running + nft applied
|
||||||
|
- linux join → remote podman + nft + step-ca CA push
|
||||||
|
- proxmox native → LXC created with `nesting=1,keyctl=1,fuse=1` + podman traefik + nft DNAT to LXC IP
|
||||||
|
- floating-ip → `pct create` with correct `net0` args + LXC registered as `linux` node
|
||||||
|
- release.sh builds orca-traefik image (Dockerfile.traefik parses)
|
||||||
|
|
||||||
|
### Wave 5 (verify)
|
||||||
|
- **T9**: `make build && make test && make lint && make verify-docs` all pass.
|
||||||
|
|
||||||
|
### Must-haves
|
||||||
|
- [ ] `orca doctor ingress` exits 0 on a properly bootstrapped node
|
||||||
|
- [ ] UAT signoff script includes new assertions
|
||||||
|
- [ ] `make verify-docs` passes
|
||||||
|
- [ ] Integration tests pass in CI `validate`
|
||||||
|
- [ ] ARCHITECTURE.md ingress section matches shipped code
|
||||||
|
|
||||||
|
## Phase 8: Final review + ship + audit (milestone release)
|
||||||
|
|
||||||
|
**Tag**: `v0.13.8` = **v0.14 milestone release** | **Type**: docs+review
|
||||||
|
|
||||||
|
- Multi-persona code review across all phases
|
||||||
|
- Audit: `verify-reqs`, git-log ↔ `.ciagent/` reconstruction
|
||||||
|
- Merge `phase/08` → `milestone/v0.14` → `main`
|
||||||
|
- Tag `v0.13.8` + release with milestone summary
|
||||||
|
- Build + publish both container images (`orca` + `orca-traefik`)
|
||||||
|
- Delete all milestone branches (tags preserve history)
|
||||||
|
- Mark all REQ-171..179 as complete in REQUIREMENTS.md + ROADMAP.md
|
||||||
@@ -707,6 +707,52 @@ v0.13 closes all critical/high/medium findings (15 new requirements,
|
|||||||
14 phases) and delivers the **UAT plan + signoff script** that gates
|
14 phases) and delivers the **UAT plan + signoff script** that gates
|
||||||
the v1.0.0 cut.
|
the v1.0.0 cut.
|
||||||
|
|
||||||
|
### v0.14 Milestone: Ingress Bootstrap Completeness
|
||||||
|
|
||||||
|
**Scope**: ensure that linux & proxmox types are properly bootstrapped
|
||||||
|
with traefik during cluster init or node join. All cluster endpoints are
|
||||||
|
provisioned as sockets (R-007); routing between jobs and services
|
||||||
|
depends on traefik being present and properly configured. v0.13 shipped
|
||||||
|
traefik binary + systemd unit + empty dynamic dir but never wrote the
|
||||||
|
static config nor applied nft rules — `orca-traefik.service` fails on a
|
||||||
|
fresh `orca init` and `orca doctor nft` FAILs. v0.14 replaces the
|
||||||
|
binary+systemd model with a **podman container** running a custom
|
||||||
|
`orca-traefik` image, and completes the nft SNAT+DNAT ingress stack on
|
||||||
|
every node type.
|
||||||
|
|
||||||
|
**New load-bearing rule**: **R-024** — Traefik runs exclusively as a
|
||||||
|
podman container, deployed from the `orca-traefik` image published per
|
||||||
|
release. Every orca-managed ingress surface bootstraps: nft DNAT
|
||||||
|
(`:443→127.0.0.1:8443`, `:80→127.0.0.1:8080`) + SNAT/MASQUERADE
|
||||||
|
postrouting + `podman run -d --restart=always --network host -v
|
||||||
|
/etc/traefik/dynamic:/etc/traefik/dynamic:Z -v
|
||||||
|
/etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro
|
||||||
|
git.cloudinit.dev/coreci/orca-traefik:<tag>`. No node joins without a
|
||||||
|
functional podman-traefik ingress data plane.
|
||||||
|
|
||||||
|
**Three topologies** (per operator constraints):
|
||||||
|
1. **Linux**: host → nft → `podman run orca-traefik` (host network)
|
||||||
|
2. **Proxmox Native**: host → nft → LXC (nesting=1) → `podman run orca-traefik`
|
||||||
|
3. **Proxmox Floating-IP**: LXC (owns floating IP) → nft (inside LXC) → `podman run orca-traefik`
|
||||||
|
|
||||||
|
**Milestone type**: feature (multiple `feat` phases). Tags on v0.13.x
|
||||||
|
patch line: `v0.13.0` (P0) ... `v0.13.8` (P8 final = v0.14 milestone
|
||||||
|
release). 9 phases, 9 net-new requirements (REQ-171..REQ-179).
|
||||||
|
|
||||||
|
### v0.14 Decisions (D-series, full autonomy)
|
||||||
|
|
||||||
|
| ID | Question | Decision | Rationale | Confidence |
|
||||||
|
|----|----------|----------|-----------|------------|
|
||||||
|
| D-255 | Traefik deployment model? | **Podman container from custom `orca-traefik` image** | Operator constraint: traefik always deployed as a container. Replaces v0.13 binary+systemd. Image bakes static config. | 0.95 |
|
||||||
|
| D-256 | Container network mode? | **`--network host`** | Binds 127.0.0.1:8080/8443 directly on host/LXC loopback; nft DNAT targets that. No port publishing complexity. | 0.92 |
|
||||||
|
| D-257 | TLS cert resolver in image? | **No certResolver; `tls: {}` for v0.14, real mTLS deferred to v0.15** | Grill G-003 (confidence 0.55 < 0.60) auto-resolved to defer. Traefik v3.3 `certificatesResolvers` only supports acme/tailscale, not CA-file. Drop `certResolver: orca` (broken). Emit `tls: {}` in dynamic config. Real mTLS via dynamic `tls.certificates` + `tls.options.default.clientAuth.caFiles` lands in v0.15 when step-ca mints server certs. | 0.90 |
|
||||||
|
| D-258 | Dynamic config volume? | **Mount `/etc/traefik/dynamic` from host** | Zero changes to existing `deployRemote` WriteFile path (`job_dispatch.go:243`). File provider watches it. | 0.95 |
|
||||||
|
| D-259 | Floating-IP mode: register PVE host too? | **Yes — PVE host as `proxmox` + ingress LXC as `linux`** | PVE host needed in registry for `pct`/`qm` workload dispatch. Both register. | 0.92 |
|
||||||
|
| D-260 | `--ingress-mode` persistence? | **Store `IngressMode` on `model.Node`** | `doctor ingress` needs to know which check path to run. Schema migration. | 0.90 |
|
||||||
|
| D-261 | MAC generation when `--mac` omitted? | **Generate random `02:XX:...` in interactive mode; require `--mac` in `--json` mode** | Interactive: generate + confirm. Non-interactive: explicit required (no silent generation). | 0.88 |
|
||||||
|
| D-262 | Proxmox native nft DNAT target? | **LXC bridge IP (not 127.0.0.1)** | LXC has its own network namespace; 127.0.0.1 on PVE host ≠ LXC loopback. `NftClusterConfig.DNATTarget` field (default `127.0.0.1:8443`; native mode = `<lxc-ip>:8443`). | 0.90 |
|
||||||
|
| D-263 | LXC podman requirements? | **`--features nesting=1,keyctl=1` + `apt-get install podman`** | Ubuntu 24.04 LXC template has no podman preinstalled. Nesting+keyctl required for podman in unprivileged LXC. | 0.88 |
|
||||||
|
|
||||||
### v0.13 Decisions (D-series, full autonomy)
|
### v0.13 Decisions (D-series, full autonomy)
|
||||||
|
|
||||||
| ID | Question | Decision | Rationale | Confidence |
|
| ID | Question | Decision | Rationale | Confidence |
|
||||||
|
|||||||
+128
-48
@@ -254,7 +254,7 @@ operator decision Q2=C.
|
|||||||
|
|
||||||
## v0.12 Milestone Summary — Security Hardening (Zero-Trust Identity)
|
## v0.12 Milestone Summary — Security Hardening (Zero-Trust Identity)
|
||||||
|
|
||||||
**Status**: in progress (Phase 0). 30 net-new requirements (REQ-119..REQ-148)
|
**Status**: complete (shipped as v0.11.x tags; milestone release v0.11.28). 30 net-new requirements (REQ-119..REQ-148)
|
||||||
derived from the v0.12 threat-model review (25 findings F1..F25) and the
|
derived from the v0.12 threat-model review (25 findings F1..F25) and the
|
||||||
zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
||||||
(P0 + P01..P27 + P28 final) and RESEARCH_v0.12.md for the full threat model.
|
(P0 + P01..P27 + P28 final) and RESEARCH_v0.12.md for the full threat model.
|
||||||
@@ -263,41 +263,41 @@ zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
|||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-119 | Command injection fix in `internal/runtime/podman.go` & `wasm.go`: shell-quote `cmdStr` via `shellQuote` in SSH exec interpolation (`podman.go:57`, `wasm.go:39`); add injection regression tests (bats + Go) covering `;`, `\|`, `$()`, backticks, newline injection (F3) | High | **v0.12 P01** | pending |
|
| REQ-119 | Command injection fix in `internal/runtime/podman.go` & `wasm.go`: shell-quote `cmdStr` via `shellQuote` in SSH exec interpolation (`podman.go:57`, `wasm.go:39`); add injection regression tests (bats + Go) covering `;`, `\|`, `$()`, backticks, newline injection (F3) | High | **v0.12 P01** | complete |
|
||||||
| REQ-120 | Namespace path traversal fix: `validateNamespaceName` in `internal/ns/` rejects `..`, `/`, leading `-`, null bytes, control chars in `ns create`/`ns inherit`/`ns set-constraint`; add fuzz test (F4) | High | **v0.12 P02** | pending |
|
| REQ-120 | Namespace path traversal fix: `validateNamespaceName` in `internal/ns/` rejects `..`, `/`, leading `-`, null bytes, control chars in `ns create`/`ns inherit`/`ns set-constraint`; add fuzz test (F4) | High | **v0.12 P02** | complete |
|
||||||
| REQ-121 | Txn apply path allowlist: `apply.sh` python heredoc validates every `path` in `desired-state.json` against a prefix allowlist (`/etc/orca/`, `/etc/traefik/orca*`, `/etc/systemd/system/orca-*`, `/etc/nftables.d/orca*`, `/etc/syncthing/orca*`); rejects otherwise; HMAC-signed manifest unchanged (F5) | High | **v0.12 P03** | pending |
|
| REQ-121 | Txn apply path allowlist: `apply.sh` python heredoc validates every `path` in `desired-state.json` against a prefix allowlist (`/etc/orca/`, `/etc/traefik/orca*`, `/etc/systemd/system/orca-*`, `/etc/nftables.d/orca*`, `/etc/syncthing/orca*`); rejects otherwise; HMAC-signed manifest unchanged (F5) | High | **v0.12 P03** | complete |
|
||||||
|
|
||||||
### Wave B — Zero-trust identity
|
### Wave B — Zero-trust identity
|
||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-122 | ACL enforcement wiring: `acl.Check` invoked in daemon handlers (read/write/admin by route) and SSH-push applier (validates `ORCA_OIDC_TOKEN` env var against JWKS before applying any txn); deny-by-default enforced; actor recorded in audit (F1, foundational for REQ-145) | High | **v0.12 P06** | pending |
|
| REQ-122 | ACL enforcement wiring: `acl.Check` invoked in daemon handlers (read/write/admin by route) and SSH-push applier (validates `ORCA_OIDC_TOKEN` env var against JWKS before applying any txn); deny-by-default enforced; actor recorded in audit (F1, foundational for REQ-145) | High | **v0.12 P06** | complete |
|
||||||
| REQ-123 | Daemon auth hardening: mandatory mTLS (remove plaintext mode entirely); OIDC bearer accepted as second factor on human-facing endpoints; `MaxBytesReader` body limits; pprof loopback-only by default, refuse non-loopback without `--pprof-allow-public` confirmation (F6, F24) | High | **v0.12 P09** | pending |
|
| REQ-123 | Daemon auth hardening: mandatory mTLS (remove plaintext mode entirely); OIDC bearer accepted as second factor on human-facing endpoints; `MaxBytesReader` body limits; pprof loopback-only by default, refuse non-loopback without `--pprof-allow-public` confirmation (F6, F24) | High | **v0.12 P09** | complete |
|
||||||
| REQ-124 | HTTP request body size limits: `http.MaxBytesReader` on all JSON-decoding handlers; `MaxHeaderBytes` set; rejects oversized bodies (F24) | Medium | **v0.12 P09** | pending |
|
| REQ-124 | HTTP request body size limits: `http.MaxBytesReader` on all JSON-decoding handlers; `MaxHeaderBytes` set; rejects oversized bodies (F24) | Medium | **v0.12 P09** | complete |
|
||||||
| REQ-125 | Audit log tamper-evidence: hash-chained entries (`prev_hash = sha256(prev_row \|\| payload)`), HMAC-SHA256 under master key on the chain head; `orca doctor audit` verifies the chain; append-only enforcement via SQLite trigger blocking UPDATE/DELETE; actor field carries OIDC `sub` or SPIFFE SVID (F2) | High | **v0.12 P10** | pending |
|
| REQ-125 | Audit log tamper-evidence: hash-chained entries (`prev_hash = sha256(prev_row \|\| payload)`), HMAC-SHA256 under master key on the chain head; `orca doctor audit` verifies the chain; append-only enforcement via SQLite trigger blocking UPDATE/DELETE; actor field carries OIDC `sub` or SPIFFE SVID (F2) | High | **v0.12 P10** | complete |
|
||||||
| REQ-126 | SVID chain validation: `VerifySVID` validates the full cert chain against the CA pool, not just the URI SAN; reject certs signed by unknown CAs even with correct URI (F9) | High | **v0.12 P11** | pending |
|
| REQ-126 | SVID chain validation: `VerifySVID` validates the full cert chain against the CA pool, not just the URI SAN; reject certs signed by unknown CAs even with correct URI (F9) | High | **v0.12 P11** | complete |
|
||||||
| REQ-127 | Backup symlink validation: `Restore` rejects `Linkname` that's absolute, contains `..`, or points outside `ORCA_HOME`; add regression test with crafted tarball (F7) | High | **v0.12 P12** | pending |
|
| REQ-127 | Backup symlink validation: `Restore` rejects `Linkname` that's absolute, contains `..`, or points outside `ORCA_HOME`; add regression test with crafted tarball (F7) | High | **v0.12 P12** | complete |
|
||||||
| REQ-128 | step-ca /tmp hardening: `step ca certificate` writes to 0600 temp under `ClusterDir()/step-tmp/` (or `TMPDIR` override), not world-readable `/tmp`; cleanup in `defer` (F10) | High | **v0.12 P13** | pending |
|
| REQ-128 | step-ca /tmp hardening: `step ca certificate` writes to 0600 temp under `ClusterDir()/step-tmp/` (or `TMPDIR` override), not world-readable `/tmp`; cleanup in `defer` (F10) | High | **v0.12 P13** | complete |
|
||||||
| REQ-129 | Master key rotation: `orca secrets rotate-master` re-encrypts all namespace secrets under a new master key; new master key re-sealed to OIDC as part of the same operation; `--dry-run` + atomic + automatic rollback to old sealed key on any ns failure; no passphrase (R-021) (F12) | High | **v0.12 P14** | pending |
|
| REQ-129 | Master key rotation: `orca secrets rotate-master` re-encrypts all namespace secrets under a new master key; new master key re-sealed to OIDC as part of the same operation; `--dry-run` + atomic + automatic rollback to old sealed key on any ns failure; no passphrase (R-021) (F12) | High | **v0.12 P14** | complete |
|
||||||
| REQ-130 | File-mode audit expansion: `EnforceFileModes` extended to SSH key, master key (sealed blob), server cert/key, known_hosts; `orca doctor modes` checks all; startup refuses to run on violation (F13) | Medium | **v0.12 P15** | pending |
|
| REQ-130 | File-mode audit expansion: `EnforceFileModes` extended to SSH key, master key (sealed blob), server cert/key, known_hosts; `orca doctor modes` checks all; startup refuses to run on violation (F13) | Medium | **v0.12 P15** | complete |
|
||||||
| REQ-131 | aggregate.sh JSON injection fix + drift-gate parse fix: replace `printf` interpolation with `jq`-based JSON construction (or Go-side aggregator emitting JSON); fix `orca-pull.sh` R-020 parsing to use `jq` instead of grep (F11, F18) | High | **v0.12 P16** | pending |
|
| REQ-131 | aggregate.sh JSON injection fix + drift-gate parse fix: replace `printf` interpolation with `jq`-based JSON construction (or Go-side aggregator emitting JSON); fix `orca-pull.sh` R-020 parsing to use `jq` instead of grep (F11, F18) | High | **v0.12 P16** | complete |
|
||||||
| REQ-132 | install.sh checksum+GPG verification: release.sh publishes `SHA256SUMS` + `SHA256SUMS.asc` (GPG-signed) alongside tarball; install.sh verifies before `tar -xzf`; fail closed on mismatch (F14) | High | **v0.12 P17** | pending |
|
| REQ-132 | install.sh checksum+GPG verification: release.sh publishes `SHA256SUMS` + `SHA256SUMS.asc` (GPG-signed) alongside tarball; install.sh verifies before `tar -xzf`; fail closed on mismatch (F14) | High | **v0.12 P17** | complete |
|
||||||
| REQ-133 | nftables ruleset hardening: add conntrack bounds (`ct state established,related accept`), input default-deny on orca chain, drop invalid packets; `orca doctor nft` audits live ruleset against emitted one (F21) | Medium | **v0.12 P18** | pending |
|
| REQ-133 | nftables ruleset hardening: add conntrack bounds (`ct state established,related accept`), input default-deny on orca chain, drop invalid packets; `orca doctor nft` audits live ruleset against emitted one (F21) | Medium | **v0.12 P18** | complete |
|
||||||
| REQ-134 | sudoers hardening: add NOEXEC to `apt-get`/`dpkg` (or remove if unused); `orca doctor proxmox` audits sudoers file against expected allowlist (F22) | Medium | **v0.12 P19** | pending |
|
| REQ-134 | sudoers hardening: add NOEXEC to `apt-get`/`dpkg` (or remove if unused); `orca doctor proxmox` audits sudoers file against expected allowlist (F22) | Medium | **v0.12 P19** | complete |
|
||||||
| REQ-135 | System user consistency: Proxmox bootstrap creates `nologin` system user (`-r -s /usr/sbin/nologin`), matching peer-setup; `orca doctor` flags inconsistency on existing peers; `orca upgrade` migrates (F23) | Medium | **v0.12 P20** | pending |
|
| REQ-135 | System user consistency: Proxmox bootstrap creates `nologin` system user (`-r -s /usr/sbin/nologin`), matching peer-setup; `orca doctor` flags inconsistency on existing peers; `orca upgrade` migrates (F23) | Medium | **v0.12 P20** | complete |
|
||||||
| REQ-136 | SQLite file-mode + at-rest encryption: `store.Open` sets DB file mode 0600; optional `--encrypt-db` (CGO-free fallback per C-31: file-mode 0600 + documented threat if SQLCipher needs CGO); no CGO (F8) | High | **v0.12 P21** | pending |
|
| REQ-136 | SQLite file-mode + at-rest encryption: `store.Open` sets DB file mode 0600; optional `--encrypt-db` (CGO-free fallback per C-31: file-mode 0600 + documented threat if SQLCipher needs CGO); no CGO (F8) | High | **v0.12 P21** | complete |
|
||||||
| REQ-137 | Migration safety: `copyFile` -> atomic temp+rename; `migrateDBSchema` runs in transaction with `foreign_keys(ON)`; pre-migration backup step (uses `internal/backup`); document manual rollback; v0.11->v0.12 identity migration: `orca upgrade` refuses clusters using `--password`/bare-tokens without `--accept-identity-migration` (F19, C-34) | High | **v0.12 P22** | pending |
|
| REQ-137 | Migration safety: `copyFile` -> atomic temp+rename; `migrateDBSchema` runs in transaction with `foreign_keys(ON)`; pre-migration backup step (uses `internal/backup`); document manual rollback; v0.11->v0.12 identity migration: `orca upgrade` refuses clusters using `--password`/bare-tokens without `--accept-identity-migration` (F19, C-34) | High | **v0.12 P22** | complete |
|
||||||
| REQ-138 | Legacy CA/mTLS/daemon + step-ca password-provisioner deletion: remove `internal/security/ca.go` legacy CA, `internal/transport/mtls.go` deprecated path, daemon plaintext mode; migrate `orca init`/`orca cert *` to step-ca exclusively; `certpaths` (v0.8 layout) removed; delete step-ca `--password-file` provisioner (replaced by OIDC provisioner); **gate: P06/P08/P09/P11 all shipped** (F16) | High | **v0.12 P23** | pending |
|
| REQ-138 | Legacy CA/mTLS/daemon + step-ca password-provisioner deletion: remove `internal/security/ca.go` legacy CA, `internal/transport/mtls.go` deprecated path, daemon plaintext mode; migrate `orca init`/`orca cert *` to step-ca exclusively; `certpaths` (v0.8 layout) removed; delete step-ca `--password-file` provisioner (replaced by OIDC provisioner); **gate: P06/P08/P09/P11 all shipped** (F16) | High | **v0.12 P23** | complete |
|
||||||
| REQ-139 | known_hosts tightening + transport hardening: `Flock` tightens pre-existing looser perms to 0600; `classifyDialErr` switched from substring to typed errors; add SSH-exec rate limiting (token bucket per peer) (F15, F25) | Medium | **v0.12 P24** | pending |
|
| REQ-139 | known_hosts tightening + transport hardening: `Flock` tightens pre-existing looser perms to 0600; `classifyDialErr` switched from substring to typed errors; add SSH-exec rate limiting (token bucket per peer) (F15, F25) | Medium | **v0.12 P24** | complete |
|
||||||
| REQ-140 | Drift event authentication: drift events signed with per-peer HMAC key (derived from master key); aggregator rejects unsigned/forged events; `orca-drift-notify.sh` reads key from 0600 file owned by `orca` (F18) | Medium | **v0.12 P25** | pending |
|
| REQ-140 | Drift event authentication: drift events signed with per-peer HMAC key (derived from master key); aggregator rejects unsigned/forged events; `orca-drift-notify.sh` reads key from 0600 file owned by `orca` (F18) | Medium | **v0.12 P25** | complete |
|
||||||
| REQ-141 | Security integration test suite: hermetic harness exercising injection, traversal, symlink, drift-forgery, audit-tamper, daemon-auth-negative, OIDC mock-IdP flow, ACL-with-OIDC-claims negative tests, unseal/seal, WebAuthn virtual-authenticator ceremony, password-removal regression (assert `--password` is rejected); gates in `.coreci.yml` `validate` (C-33) | High | **v0.12 P26** | pending |
|
| REQ-141 | Security integration test suite: hermetic harness exercising injection, traversal, symlink, drift-forgery, audit-tamper, daemon-auth-negative, OIDC mock-IdP flow, ACL-with-OIDC-claims negative tests, unseal/seal, WebAuthn virtual-authenticator ceremony, password-removal regression (assert `--password` is rejected); gates in `.coreci.yml` `validate` (C-33) | High | **v0.12 P26** | complete |
|
||||||
| REQ-142 | Zero-trust + OIDC + WebAuthn + threat-model docs: `docs/threat-model.md` (STRIDE + zero-trust model + OIDC data-flow), `docs/oidc.md` (configure your IdP, Dex offline quickstart, claim-to-namespace mapping), `docs/webauthn.md` (passkey registration, RP ID, secure context), `docs/security-runbook.md` (unseal/seal, master key rotation, incident response, sudoers audit, nft audit); README security section names "no orca credentials" as an invariant | Medium | **v0.12 P27** | pending |
|
| REQ-142 | Zero-trust + OIDC + WebAuthn + threat-model docs: `docs/threat-model.md` (STRIDE + zero-trust model + OIDC data-flow), `docs/oidc.md` (configure your IdP, Dex offline quickstart, claim-to-namespace mapping), `docs/webauthn.md` (passkey registration, RP ID, secure context), `docs/security-runbook.md` (unseal/seal, master key rotation, incident response, sudoers audit, nft audit); README security section names "no orca credentials" as an invariant | Medium | **v0.12 P27** | complete |
|
||||||
| REQ-143 | Final review + ship + audit: multi-persona review across all phases, `ciagent-audit` reconstruction test, milestone merge to main, tag `v0.11.29` (= v0.12 milestone release per feature-milestone rule) | High | **v0.12 P28** | pending |
|
| REQ-143 | Final review + ship + audit: multi-persona review across all phases, `ciagent-audit` reconstruction test, milestone merge to main, tag `v0.11.29` (= v0.12 milestone release per feature-milestone rule) | High | **v0.12 P28** | complete |
|
||||||
| REQ-144 | OIDC client + bundled Dex: `orca auth login`/`logout`/`status`/`init-idp`; OIDC config block (`oidc.issuer`, `client_id`, `client_secret`, `scopes`); bundled Dex systemd unit + Traefik route on the lead; BYO external IdP override via `oidc.issuer` repoint; JWKS caching + refresh; token storage at `~/.orca/credentials.json` (0600); `--oidc` flag on commands requiring identity; browser auth-code + PKCE + local loopback redirect; headless device-code fallback (D-238..D-247) | High | **v0.12 P04** | pending |
|
| REQ-144 | OIDC client + bundled Dex: `orca auth login`/`logout`/`status`/`init-idp`; OIDC config block (`oidc.issuer`, `client_id`, `client_secret`, `scopes`); bundled Dex systemd unit + Traefik route on the lead; BYO external IdP override via `oidc.issuer` repoint; JWKS caching + refresh; token storage at `~/.orca/credentials.json` (0600); `--oidc` flag on commands requiring identity; browser auth-code + PKCE + local loopback redirect; headless device-code fallback (D-238..D-247) | High | **v0.12 P04** | complete |
|
||||||
| REQ-145 | ACL rewrite to OIDC claims: remove `KindToken` entirely; `KindSpiffe` stays for machine identity; new `KindOidc` maps `sub`+`groups` -> namespace permissions; `acl.Check` takes OIDC claims struct; deny-by-default enforced in daemon + SSH-push applier; `acl.json` mode tightened to 0600 (F1) | High | **v0.12 P06** | pending |
|
| REQ-145 | ACL rewrite to OIDC claims: remove `KindToken` entirely; `KindSpiffe` stays for machine identity; new `KindOidc` maps `sub`+`groups` -> namespace permissions; `acl.Check` takes OIDC claims struct; deny-by-default enforced in daemon + SSH-push applier; `acl.json` mode tightened to 0600 (F1) | High | **v0.12 P06** | complete |
|
||||||
| REQ-146 | Remove all password/token paths (breaking): delete `--password`/`$ORCA_PROXMOX_PASSWORD` from Proxmox join (replace with pre-staged-key-only or `step ssh` OIDC cert exchange); delete step-ca `--password-file` provisioner (migrate to OIDC provisioner); delete any bare-token CLI paths; documented in migration guide (R-021, C-34) | High | **v0.12 P07** | pending |
|
| REQ-146 | Remove all password/token paths (breaking): delete `--password`/`$ORCA_PROXMOX_PASSWORD` from Proxmox join (replace with pre-staged-key-only or `step ssh` OIDC cert exchange); delete step-ca `--password-file` provisioner (migrate to OIDC provisioner); delete any bare-token CLI paths; documented in migration guide (R-021, C-34) | High | **v0.12 P07** | complete |
|
||||||
| REQ-147 | Master key seal-to-OIDC + Shamir recovery: master key encrypted with key derived from OIDC token exchange at unseal; `orca cluster unseal`/`seal`; sealed blob at `ClusterDir()/master.key.sealed` (0600); raw key never on disk; Shamir 3-of-5 shards printed at seal time; recovery via `--recovery` + 3 shards; mTLS-only offline path derives seal key from cluster CA (D-241, C-35) | High | **v0.12 P08** | pending |
|
| REQ-147 | Master key seal-to-OIDC + Shamir recovery: master key encrypted with key derived from OIDC token exchange at unseal; `orca cluster unseal`/`seal`; sealed blob at `ClusterDir()/master.key.sealed` (0600); raw key never on disk; Shamir 3-of-5 shards printed at seal time; recovery via `--recovery` + 3 shards; mTLS-only offline path derives seal key from cluster CA (D-241, C-35) | High | **v0.12 P08** | complete |
|
||||||
| REQ-148 | WebAuthn connector for Dex (passkeys): `orca-webauthn-connector` (~300 LoC Go, `go-webauthn`); register/login ceremonies at `/orca/webauthn/{register,login}` behind Traefik; `orca auth register` browser flow; passkey storage SQLite `ClusterDir()/webauthn-credentials.db` (0600, public keys only); RP ID = cluster Traefik domain; secure context via step-ca cert; headless device-code fallback; virtual-authenticator integration tests (D-240, D-243, D-244, C-38) | High | **v0.12 P05** | pending |
|
| REQ-148 | WebAuthn connector for Dex (passkeys): `orca-webauthn-connector` (~300 LoC Go, `go-webauthn`); register/login ceremonies at `/orca/webauthn/{register,login}` behind Traefik; `orca auth register` browser flow; passkey storage SQLite `ClusterDir()/webauthn-credentials.db` (0600, public keys only); RP ID = cluster Traefik domain; secure context via step-ca cert; headless device-code fallback; virtual-authenticator integration tests (D-240, D-243, D-244, C-38) | High | **v0.12 P05** | complete |
|
||||||
|
|
||||||
### Scope notes (v0.12)
|
### Scope notes (v0.12)
|
||||||
|
|
||||||
@@ -311,7 +311,7 @@ zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
|||||||
|
|
||||||
## Milestone v0.13: Production Hardening Round 2 + UAT Plan
|
## Milestone v0.13: Production Hardening Round 2 + UAT Plan
|
||||||
|
|
||||||
**Status**: in progress (2026-08-07). v0.12 (Security Hardening) is
|
**Status**: complete (2026-08-10). v0.12 (Security Hardening) is
|
||||||
COMPLETE; v0.13 is the final hardening round before the v1.0.0
|
COMPLETE; v0.13 is the final hardening round before the v1.0.0
|
||||||
production-ready tag. v1.0.0 is gated on the UAT signoff script
|
production-ready tag. v1.0.0 is gated on the UAT signoff script
|
||||||
(`scripts/uat-signoff.sh`) delivered by this milestone.
|
(`scripts/uat-signoff.sh`) delivered by this milestone.
|
||||||
@@ -320,41 +320,41 @@ production-ready tag. v1.0.0 is gated on the UAT signoff script
|
|||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-149 | Go toolchain bump to 1.25.12+ (closes 24 stdlib vulns: archive/tar GO-2025-4014/GO-2026-4869, crypto/tls GO-2026-5856/GO-2025-4008, crypto/x509 GO-2026-5037/4947/4946/GO-2025-4175/4155/4013, net/http GO-2026-4918/GO-2025-4012, net/url GO-2026-4601/4341/GO-2025-4010, encoding/pem GO-2025-4009, os GO-2026-4602); `govulncheck -show verbose` triage of 6 imported third-party vulns; bump deps with reachable traces | High | **v0.13 P01** | pending |
|
| REQ-149 | Go toolchain bump to 1.25.12+ (closes 24 stdlib vulns: archive/tar GO-2025-4014/GO-2026-4869, crypto/tls GO-2026-5856/GO-2025-4008, crypto/x509 GO-2026-5037/4947/4946/GO-2025-4175/4155/4013, net/http GO-2026-4918/GO-2025-4012, net/url GO-2026-4601/4341/GO-2025-4010, encoding/pem GO-2025-4009, os GO-2026-4602); `govulncheck -show verbose` triage of 6 imported third-party vulns; bump deps with reachable traces | High | **v0.13 P01** | complete |
|
||||||
| REQ-150 | Input validation & injection hardening: (a) `orca logs --job` validate against `^[A-Za-z0-9_-]+$`, use `shellQuote` not `%q` (critical: backtick RCE via SSH fanout); (b) pprof `isLoopback(":6060")` treat empty host as non-loopback/bind-all, reject unless explicit public-allow flag wired; remove phantom `--pprof-allow-public` references, make loopback-only a hard invariant; (c) backup restore tar-slip fix: use `filepath.Rel(target, dest)` containment check instead of `HasPrefix(name, "..")`; (d) `orca txn rollback` validate txn ID against `^T-[0-9a-f]{16}$`; (e) `orca nft diff --against` validate txn ID before `filepath.Join`; (f) `drain stopAlloc` validate `allocID` against `^[A-Za-z0-9_-]+$` before `systemctl stop`; (g) `cluster_compat` `shellQuote(first)` for peer dir name; (h) `runtime/podman.go` use `shellQuote(image)` not `%q`; (i) nft `TrustedProbes` validate each entry with `net.ParseIP`/`net.ParseCIDR`; (j) sudoers: validate `--proxmox-user`/`--proxmox-role` against `^[a-z_][a-z0-9_-]{0,31}$`; write to fixed `/etc/sudoers.d/orca`; `shellQuote` all pveum/useradd; `validateSudoers` check the actual file written; (k) `nft country block add` validate `^[A-Z]{2}$` | Critical | **v0.13 P02** | pending |
|
| REQ-150 | Input validation & injection hardening: (a) `orca logs --job` validate against `^[A-Za-z0-9_-]+$`, use `shellQuote` not `%q` (critical: backtick RCE via SSH fanout); (b) pprof `isLoopback(":6060")` treat empty host as non-loopback/bind-all, reject unless explicit public-allow flag wired; remove phantom `--pprof-allow-public` references, make loopback-only a hard invariant; (c) backup restore tar-slip fix: use `filepath.Rel(target, dest)` containment check instead of `HasPrefix(name, "..")`; (d) `orca txn rollback` validate txn ID against `^T-[0-9a-f]{16}$`; (e) `orca nft diff --against` validate txn ID before `filepath.Join`; (f) `drain stopAlloc` validate `allocID` against `^[A-Za-z0-9_-]+$` before `systemctl stop`; (g) `cluster_compat` `shellQuote(first)` for peer dir name; (h) `runtime/podman.go` use `shellQuote(image)` not `%q`; (i) nft `TrustedProbes` validate each entry with `net.ParseIP`/`net.ParseCIDR`; (j) sudoers: validate `--proxmox-user`/`--proxmox-role` against `^[a-z_][a-z0-9_-]{0,31}$`; write to fixed `/etc/sudoers.d/orca`; `shellQuote` all pveum/useradd; `validateSudoers` check the actual file written; (k) `nft country block add` validate `^[A-Z]{2}$` | Critical | **v0.13 P02** | complete |
|
||||||
|
|
||||||
### Wave B — Scheduler wiring & jobspec parser (architectural)
|
### Wave B — Scheduler wiring & jobspec parser (architectural)
|
||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-151 | Scheduler/deployment wiring: wire `internal/scheduler.Schedule()` into `orca job run` — replace local `exec.CommandContext` path with: evaluate constraints/capacity/affinity via scheduler → render systemd units via `internal/emitter` → SSH-push to target via `internal/sshpush`; `--target` overrides scheduler selection; capacity enforced (reject job if no node fits); CEL constraints evaluated; affinity weighted scoring; `systemd-analyze verify` on rendered unit before deploy; `job run` without `--target` uses scheduler bin-packing across registered nodes | Critical | **v0.13 P03** | pending |
|
| REQ-151 | Scheduler/deployment wiring: wire `internal/scheduler.Schedule()` into `orca job run` — replace local `exec.CommandContext` path with: evaluate constraints/capacity/affinity via scheduler → render systemd units via `internal/emitter` → SSH-push to target via `internal/sshpush`; `--target` overrides scheduler selection; capacity enforced (reject job if no node fits); CEL constraints evaluated; affinity weighted scoring; `systemd-analyze verify` on rendered unit before deploy; `job run` without `--target` uses scheduler bin-packing across registered nodes | Critical | **v0.13 P03** | complete |
|
||||||
| REQ-152 | jobspec parser fixes: add `case "schedule":` and `case "timeout":` to top-level switch in `internal/jobspec/markdown.go` (currently silently dropped); fix DaemonSet — parser must not default Count to 1 for DaemonSet (validator rejects Count!=0); DaemonSet schedule block actually parsed and stored; `timeout:` on Jobs parsed and enforced (kill after duration); `restart:` policy translated to systemd `Restart=`/`StartLimitBurst` in emitter; add `job lint` warnings for advisory-only fields (cron, health, update, affinity) with honest "not enforced in this version" message | Critical | **v0.13 P03** | pending |
|
| REQ-152 | jobspec parser fixes: add `case "schedule":` and `case "timeout":` to top-level switch in `internal/jobspec/markdown.go` (currently silently dropped); fix DaemonSet — parser must not default Count to 1 for DaemonSet (validator rejects Count!=0); DaemonSet schedule block actually parsed and stored; `timeout:` on Jobs parsed and enforced (kill after duration); `restart:` policy translated to systemd `Restart=`/`StartLimitBurst` in emitter; add `job lint` warnings for advisory-only fields (cron, health, update, affinity) with honest "not enforced in this version" message | Critical | **v0.13 P03** | complete |
|
||||||
|
|
||||||
### Wave C — Zero-trust enforcement wiring
|
### Wave C — Zero-trust enforcement wiring
|
||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-153 | ACL enforcement + WebAuthn registration auth: (a) wire `acl.Check` into all 5 daemon handlers (`dispatch`/`jobs`/`nodes`/`tasks`/`health`) — extract OIDC sub/SPIFFE SVID from mTLS peer cert, check against ACL for namespace+verb, deny-by-default; (b) wire `acl.Check` into sshpush applier + txn apply path (validate `ORCA_OIDC_TOKEN` bearer against JWKS); (c) thread OIDC sub/SVID into audit `actor` field (replaces "cli"/"daemon"); (d) fix `acl.json` mode 0644→0600; (e) fix WebAuthn unauthenticated registration — `/orca/webauthn/register` requires existing authenticated session or admin bootstrap token; do not allow overwriting existing credentials without re-auth; (f) add flock on `acl.json` for concurrent grant/revoke | Critical | **v0.13 P04** | pending |
|
| REQ-153 | ACL enforcement + WebAuthn registration auth: (a) wire `acl.Check` into all 5 daemon handlers (`dispatch`/`jobs`/`nodes`/`tasks`/`health`) — extract OIDC sub/SPIFFE SVID from mTLS peer cert, check against ACL for namespace+verb, deny-by-default; (b) wire `acl.Check` into sshpush applier + txn apply path (validate `ORCA_OIDC_TOKEN` bearer against JWKS); (c) thread OIDC sub/SVID into audit `actor` field (replaces "cli"/"daemon"); (d) fix `acl.json` mode 0644→0600; (e) fix WebAuthn unauthenticated registration — `/orca/webauthn/register` requires existing authenticated session or admin bootstrap token; do not allow overwriting existing credentials without re-auth; (f) add flock on `acl.json` for concurrent grant/revoke | Critical | **v0.13 P04** | complete |
|
||||||
| REQ-154 | Seal/audit CLI + chain race + key zeroing: (a) implement `orca cluster seal`/`unseal` (OIDC token exchange→unwrap master key→zeroed on shutdown; Shamir 3-of-5 shards printed at seal time; sealed blob at `ClusterDir()/master.key.sealed` 0600); (b) implement `orca doctor audit` (invokes `AuditRepo.VerifyChain`); (c) implement `orca doctor modes` (invokes `EnforceFileModes` across ORCA_HOME); (d) fix audit hash-chain race — `Append` uses `BEGIN IMMEDIATE` transaction; (e) fix `secrets rotate-master` to actually re-seal to OIDC; (f) zero master key / namespace keys / SVID private keys after use (defense-in-depth against pprof heap extraction) | High | **v0.13 P05** | pending |
|
| REQ-154 | Seal/audit CLI + chain race + key zeroing: (a) implement `orca cluster seal`/`unseal` (OIDC token exchange→unwrap master key→zeroed on shutdown; Shamir 3-of-5 shards printed at seal time; sealed blob at `ClusterDir()/master.key.sealed` 0600); (b) implement `orca doctor audit` (invokes `AuditRepo.VerifyChain`); (c) implement `orca doctor modes` (invokes `EnforceFileModes` across ORCA_HOME); (d) fix audit hash-chain race — `Append` uses `BEGIN IMMEDIATE` transaction; (e) fix `secrets rotate-master` to actually re-seal to OIDC; (f) zero master key / namespace keys / SVID private keys after use (defense-in-depth against pprof heap extraction) | High | **v0.13 P05** | complete |
|
||||||
| REQ-155 | auth init-idp real + auth register: (a) implement `orca auth init-idp` — render Dex systemd unit + config template + Traefik dynamic route from `internal/webauthn/` connector at `https://<cluster>/orca/webauthn/{register,login}`; RP ID = cluster Traefik domain (C-38); HTTPS secure context via step-ca cert; atomic deploy with rollback; (b) implement `orca auth register` (browser flow to WebAuthn registration endpoint); (c) `loadOIDCConfig` config-file loading (`oidc.issuer` in config, not flags-only); (d) `orca doctor oidc` health check | High | **v0.13 P06** | pending |
|
| REQ-155 | auth init-idp real + auth register: (a) implement `orca auth init-idp` — render Dex systemd unit + config template + Traefik dynamic route from `internal/webauthn/` connector at `https://<cluster>/orca/webauthn/{register,login}`; RP ID = cluster Traefik domain (C-38); HTTPS secure context via step-ca cert; atomic deploy with rollback; (b) implement `orca auth register` (browser flow to WebAuthn registration endpoint); (c) `loadOIDCConfig` config-file loading (`oidc.issuer` in config, not flags-only); (d) `orca doctor oidc` health check | High | **v0.13 P06** | complete |
|
||||||
|
|
||||||
### Wave D — Concurrency, transport, migration safety
|
### Wave D — Concurrency, transport, migration safety
|
||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-156 | Concurrency safety: (a) SQLite `busy_timeout(5000)` + `SetMaxOpenConns(1)` on all DSNs (store, cache, recovery, webauthn); (b) secrets file flock (concurrent `secrets set` on same ns no longer loses data); (c) upgrade lock file (refuse concurrent `orca upgrade`); (d) backup lock file; (e) cache invalidation by write commands (`node join`/`leave`, `ns create`/`delete`, `job run`/`stop` invalidate relevant cache class — read-after-write consistency); (f) `Executor.Run` mutex scope fix (hold only for DB inserts, not whole job duration); (g) `ns create` atomic dir+ns.md write; (h) `writeCurrentLead` atomic write; (i) consolidate 3 divergent `writeAtomic` impls onto `security.WriteAtomic`; (j) WebAuthn session stores guarded with `sync.Mutex` | High | **v0.13 P07** | pending |
|
| REQ-156 | Concurrency safety: (a) SQLite `busy_timeout(5000)` + `SetMaxOpenConns(1)` on all DSNs (store, cache, recovery, webauthn); (b) secrets file flock (concurrent `secrets set` on same ns no longer loses data); (c) upgrade lock file (refuse concurrent `orca upgrade`); (d) backup lock file; (e) cache invalidation by write commands (`node join`/`leave`, `ns create`/`delete`, `job run`/`stop` invalidate relevant cache class — read-after-write consistency); (f) `Executor.Run` mutex scope fix (hold only for DB inserts, not whole job duration); (g) `ns create` atomic dir+ns.md write; (h) `writeCurrentLead` atomic write; (i) consolidate 3 divergent `writeAtomic` impls onto `security.WriteAtomic`; (j) WebAuthn session stores guarded with `sync.Mutex` | High | **v0.13 P07** | complete |
|
||||||
| REQ-157 | Transport & SSH safety: (a) replace substring matching in `transport.IsTransient` AND `sshpush.isTransient` with typed sentinels (`errors.Is`); (b) `rotateSSHKeys` 2-phase atomic swap (stage new key on all peers → atomic swap → verify → cleanup old); (c) `known_hosts` flock field actually read by `dial()` (TOFU callback uses new field, not v0.8 `certpaths.KnownHostsPath()`); (d) IPv6 `net.JoinHostPort` in proxmox SSH dial + drain `splitHostPort`; (e) explicit timeouts for all SSH commands (peer-setup, drift remediate/ack, txn rollback, job restart — use `context.WithTimeout`); (f) `verifyCutover` use `security.ClientTLSConfig` with orca CA pool; (g) OIDC callback server `ReadHeaderTimeout: 5s`; (h) root SIGINT/SIGTERM handler for non-watch commands (clean SSH session + temp file cleanup) | High | **v0.13 P08** | pending |
|
| REQ-157 | Transport & SSH safety: (a) replace substring matching in `transport.IsTransient` AND `sshpush.isTransient` with typed sentinels (`errors.Is`); (b) `rotateSSHKeys` 2-phase atomic swap (stage new key on all peers → atomic swap → verify → cleanup old); (c) `known_hosts` flock field actually read by `dial()` (TOFU callback uses new field, not v0.8 `certpaths.KnownHostsPath()`); (d) IPv6 `net.JoinHostPort` in proxmox SSH dial + drain `splitHostPort`; (e) explicit timeouts for all SSH commands (peer-setup, drift remediate/ack, txn rollback, job restart — use `context.WithTimeout`); (f) `verifyCutover` use `security.ClientTLSConfig` with orca CA pool; (g) OIDC callback server `ReadHeaderTimeout: 5s`; (h) root SIGINT/SIGTERM handler for non-watch commands (clean SSH session + temp file cleanup) | High | **v0.13 P08** | complete |
|
||||||
| REQ-158 | Migration & operational safety: (a) migration transaction + torn-write fix — `migrateDBSchema` wraps ALTER TABLE in transaction; crash after `os.Rename` but before schema fixup is recoverable; (b) `job stop` real `systemctl stop` via SSH (matches `job restart` pattern; honest semantics); (c) DB retention/compaction for `jobs`/`tasks`/`audit_log` tables (retention policy + `orca doctor db` compaction check); (d) `orca logs --lines` cap + `--since` upper bound (prevent OOM from unbounded journalctl output); (e) cache DB mode 0600 (matches `store.Open`); (f) `upgrade.go` cutover backup-file + atomic-rename (replace direct `sed -i`) | High | **v0.13 P09** | pending |
|
| REQ-158 | Migration & operational safety: (a) migration transaction + torn-write fix — `migrateDBSchema` wraps ALTER TABLE in transaction; crash after `os.Rename` but before schema fixup is recoverable; (b) `job stop` real `systemctl stop` via SSH (matches `job restart` pattern; honest semantics); (c) DB retention/compaction for `jobs`/`tasks`/`audit_log` tables (retention policy + `orca doctor db` compaction check); (d) `orca logs --lines` cap + `--since` upper bound (prevent OOM from unbounded journalctl output); (e) cache DB mode 0600 (matches `store.Open`); (f) `upgrade.go` cutover backup-file + atomic-rename (replace direct `sed -i`) | High | **v0.13 P09** | complete |
|
||||||
|
|
||||||
### Wave E — Observability, docs, UAT
|
### Wave E — Observability, docs, UAT
|
||||||
|
|
||||||
| ID | Requirement | Priority | Phase | Status |
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|----|-------------|----------|-------|--------|
|
|----|-------------|----------|-------|--------|
|
||||||
| REQ-159 | Observability expansion: metrics add `orca_jobs_by_state` histogram, `orca_drift_events_total` counter, `orca_ssh_errors_total` counter, `orca_txn_apply_total`/`orca_txn_rollback_total` counters, `orca_acl_denials_total` counter, `orca_audit_chain_head` gauge; new `docs/metrics.md` with Prometheus scrape config; security headers middleware on daemon (`X-Content-Type-Options`, `X-Frame-Options`) | Medium | **v0.13 P10** | pending |
|
| REQ-159 | Observability expansion: metrics add `orca_jobs_by_state` histogram, `orca_drift_events_total` counter, `orca_ssh_errors_total` counter, `orca_txn_apply_total`/`orca_txn_rollback_total` counters, `orca_acl_denials_total` counter, `orca_audit_chain_head` gauge; new `docs/metrics.md` with Prometheus scrape config; security headers middleware on daemon (`X-Content-Type-Options`, `X-Frame-Options`) | Medium | **v0.13 P10** | complete |
|
||||||
| REQ-160 | Doc drift round 2: (a) README — update status banner (v0.12+v0.13 complete), latest tag, subcommand table (add `auth`/`nft`/`peer-setup`/`secrets rotate-master`), correct "mTLS by default" claim (SSH-push is canonical, mTLS deprecated), add missing docs to table; (b) `docs/cli.md` — complete rewrite covering all ~40 subcommands; (c) CHANGELOG regen; (d) help text fixes (`job run` HCL→markdown, `job stop` daemon→SSH-push); (e) `docs/webauthn.md` add `auth register`; (f) `docs/namespace.md` add `inherit`/`set-constraint`; (g) `docs/install.md`+`docker.md` update version refs; (h) `docs/security-runbook.md` match P05 reality; (i) fix `verify-reqs` bold-format regex (currently bypasses v0.12); (j) fix ROADMAP/REQUIREMENTS v0.12 status hygiene; (k) `docs/security-scanning.md` gosec.json; (l) `internal/proxmox/bootstrap.go` comments (password→key auth); (m) deprecate `orca status` stub; (n) `make verify-docs` target (cli.md ↔ `orca --help` consistency) | High | **v0.13 P11** | pending |
|
| REQ-160 | Doc drift round 2: (a) README — update status banner (v0.12+v0.13 complete), latest tag, subcommand table (add `auth`/`nft`/`peer-setup`/`secrets rotate-master`), correct "mTLS by default" claim (SSH-push is canonical, mTLS deprecated), add missing docs to table; (b) `docs/cli.md` — complete rewrite covering all ~40 subcommands; (c) CHANGELOG regen; (d) help text fixes (`job run` HCL→markdown, `job stop` daemon→SSH-push); (e) `docs/webauthn.md` add `auth register`; (f) `docs/namespace.md` add `inherit`/`set-constraint`; (g) `docs/install.md`+`docker.md` update version refs; (h) `docs/security-runbook.md` match P05 reality; (i) fix `verify-reqs` bold-format regex (currently bypasses v0.12); (j) fix ROADMAP/REQUIREMENTS v0.12 status hygiene; (k) `docs/security-scanning.md` gosec.json; (l) `internal/proxmox/bootstrap.go` comments (password→key auth); (m) deprecate `orca status` stub; (n) `make verify-docs` target (cli.md ↔ `orca --help` consistency) | High | **v0.13 P11** | complete |
|
||||||
| REQ-161 | `--type linux` SSH-join: implement `NodeKindLinux` path (reserved at `model/node.go:29`); new `internal/linux/bootstrap.go` mirroring Proxmox pattern — orca pubkey deploy → `orca` system user → drift-events dir → no PVE role; key-auth only (R-021); `orca node join --type linux --host <ip> --ssh-user root --ssh-key <path>`; `peer-setup.go` kept as documented fallback | High | **v0.13 P12** | pending |
|
| REQ-161 | `--type linux` SSH-join: implement `NodeKindLinux` path (reserved at `model/node.go:29`); new `internal/linux/bootstrap.go` mirroring Proxmox pattern — orca pubkey deploy → `orca` system user → drift-events dir → no PVE role; key-auth only (R-021); `orca node join --type linux --host <ip> --ssh-user root --ssh-key <path>`; `peer-setup.go` kept as documented fallback | High | **v0.13 P12** | complete |
|
||||||
| REQ-162 | UAT plan: `docs/uat.md` — 3-host topology (lead Ubuntu 22.04 + pve01 Proxmox VE 8/9 + worker01 Ubuntu 22.04); step-by-step with exact commands (bootstrap→onboard Proxmox→onboard Ubuntu worker→capacity→namespace→deploy full stack→migrate between hosts→exercise every claim); claim matrix mapping ~35 feature claims to UAT steps; signoff procedure (run `scripts/uat-signoff.sh`, paste output) | Critical | **v0.13 P12** | pending |
|
| REQ-162 | UAT plan: `docs/uat.md` — 3-host topology (lead Ubuntu 22.04 + pve01 Proxmox VE 8/9 + worker01 Ubuntu 22.04); step-by-step with exact commands (bootstrap→onboard Proxmox→onboard Ubuntu worker→capacity→namespace→deploy full stack→migrate between hosts→exercise every claim); claim matrix mapping ~35 feature claims to UAT steps; signoff procedure (run `scripts/uat-signoff.sh`, paste output) | Critical | **v0.13 P12** | complete |
|
||||||
| REQ-163 | UAT signoff script: `scripts/uat-signoff.sh` — idempotent, `set -euo pipefail`, ~35 named assertions covering all feature claims; read + non-mutating only (doctor, list, --dry-run); exit 0 iff all pass; `scripts/uat-smoke.sh` — pure-CLI subset for CI `validate` (version, acl file mode, doctor modes, no-password grep, metrics shape); tests for both scripts | Critical | **v0.13 P12** | pending |
|
| REQ-163 | UAT signoff script: `scripts/uat-signoff.sh` — idempotent, `set -euo pipefail`, ~35 named assertions covering all feature claims; read + non-mutating only (doctor, list, --dry-run); exit 0 iff all pass; `scripts/uat-smoke.sh` — pure-CLI subset for CI `validate` (version, acl file mode, doctor modes, no-password grep, metrics shape); tests for both scripts | Critical | **v0.13 P12** | complete |
|
||||||
|
|
||||||
### Scope notes (v0.13)
|
### Scope notes (v0.13)
|
||||||
|
|
||||||
@@ -362,7 +362,7 @@ production-ready tag. v1.0.0 is gated on the UAT signoff script
|
|||||||
- 14 phases (P0 + P01..P12 + P13 final); "no limit on phases" per operator.
|
- 14 phases (P0 + P01..P12 + P13 final); "no limit on phases" per operator.
|
||||||
- P03 (scheduler wiring) and P12 (`--type linux` + UAT) are the `feat` phases; the rest are `fix`/`chore`/`test`/`docs`/`refactor`. Milestone type = feature (at least one `feat`).
|
- P03 (scheduler wiring) and P12 (`--type linux` + UAT) are the `feat` phases; the rest are `fix`/`chore`/`test`/`docs`/`refactor`. Milestone type = feature (at least one `feat`).
|
||||||
- Tags on v0.12.x patch line: `v0.12.0` (P0) ... `v0.12.13` (P13 final = v0.13 milestone release).
|
- Tags on v0.12.x patch line: `v0.12.0` (P0) ... `v0.12.13` (P13 final = v0.13 milestone release).
|
||||||
- v1.0.0 production-ready tag stays deferred for post-v0.13 UAT signoff (operator runs `scripts/uat-signoff.sh`, pastes output back).
|
- v1.0.0 production-ready tag stays deferred for post-v0.13 UAT signoff (operator runs `scripts/uat-signoff.sh`, paste output back).
|
||||||
|
|
||||||
### Accepted residual risks (documented in threat-model, not fixed)
|
### Accepted residual risks (documented in threat-model, not fixed)
|
||||||
|
|
||||||
@@ -374,3 +374,83 @@ production-ready tag. v1.0.0 is gated on the UAT signoff script
|
|||||||
- "WASM-first" pillar aspirational — document as "WASM runtime available, process is default"
|
- "WASM-first" pillar aspirational — document as "WASM runtime available, process is default"
|
||||||
- arm64/armv7 release — D-193 deferred; install.sh detection is forward-looking
|
- arm64/armv7 release — D-193 deferred; install.sh detection is forward-looking
|
||||||
- OIDC callback slowloris — loopback, short-lived, single CLI invocation
|
- OIDC callback slowloris — loopback, short-lived, single CLI invocation
|
||||||
|
|
||||||
|
## Milestone v0.14: Ingress Bootstrap Completeness
|
||||||
|
|
||||||
|
**Scope**: ensure that linux & proxmox types are properly bootstrapped with
|
||||||
|
traefik during cluster init or node join. All cluster endpoints are
|
||||||
|
provisioned as sockets (R-007); routing between jobs and services depends on
|
||||||
|
traefik being present on the host and properly configured. The v0.13 traefik
|
||||||
|
deployment shipped only a binary + systemd unit + empty dynamic dir — it
|
||||||
|
never wrote the static config nor applied nft rules, so `orca-traefik.service`
|
||||||
|
fails to start on a fresh `orca init` and `orca doctor nft` FAILs. v0.14
|
||||||
|
replaces the binary+systemd model with a **podman container** running a custom
|
||||||
|
`orca-traefik` image, and completes the nft SNAT+DNAT ingress stack on every
|
||||||
|
node type.
|
||||||
|
|
||||||
|
**New load-bearing rule**:
|
||||||
|
|
||||||
|
- **R-024** — Traefik runs exclusively as a podman container, deployed from
|
||||||
|
the `orca-traefik` image published per release. Every orca-managed ingress
|
||||||
|
surface bootstraps: nft DNAT (`:443→127.0.0.1:8443`,
|
||||||
|
`:80→127.0.0.1:8080`) + SNAT/MASQUERADE postrouting + `podman run -d
|
||||||
|
--restart=unless-stopped --network host -v /etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
|
||||||
|
-v /etc/traefik/dynamic:/etc/traefik/dynamic:ro
|
||||||
|
-v /etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro
|
||||||
|
git.cloudinit.dev/coreci/orca-traefik:<tag>`. No node joins without a
|
||||||
|
functional podman-traefik ingress data plane. The image's baked static
|
||||||
|
config is a default; host-side `traefik.yml` mounted `:ro` overrides it
|
||||||
|
(preserves `traefik-on-public-ip` opt-out, REQ-100).
|
||||||
|
|
||||||
|
**Three topologies** (per operator constraints):
|
||||||
|
|
||||||
|
- **Linux**: host → nft → `podman run orca-traefik` (host network)
|
||||||
|
- **Proxmox Native**: host → nft → LXC (nesting=1) → `podman run orca-traefik`
|
||||||
|
- **Proxmox Floating-IP**: LXC (owns floating IP) → nft (inside LXC) →
|
||||||
|
`podman run orca-traefik`
|
||||||
|
|
||||||
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|
|----|-------------|----------|-------|--------|
|
||||||
|
| REQ-171 | `Dockerfile.traefik` + release pipeline: build + publish `git.cloudinit.dev/coreci/orca-traefik:<version>` alongside the orca image per release; `.coreci.yml` `container-publish-traefik` step; image bakes default `traefik.yml` (entrypoints `websecure` 127.0.0.1:8443, `web` 127.0.0.1:8080, `traefik` 127.0.0.1:8081 + file provider watching `/etc/traefik/dynamic` + json log/accessLog); host-side `/etc/traefik/traefik.yml` mounted `:ro` overrides baked config (preserves `traefik-on-public-ip` opt-out REQ-100); no `certificatesResolvers` (traefik v3.3 only supports acme/tailscale); `tls: {}` in dynamic config for v0.14 (real mTLS deferred to v0.15) | Critical | **v0.14 P1** | complete |
|
||||||
|
| REQ-172 | Replace `internal/traefik/install.go` binary+systemd install with a podman-container reconciler: `podman pull orca-traefik:<tag>` + `podman run -d --restart=unless-stopped --network host --name orca-traefik -v /etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro -v /etc/traefik/dynamic:/etc/traefik/dynamic:ro -v /etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro <image>`; idempotent (pull+run if absent, start if stopped); install podman if absent (C-50); v0.13→v0.14 upgrade: detect+stop+disable+remove legacy `orca-traefik.service` + `/usr/local/bin/traefik` (C-57); works locally + over SSH-push; enable `podman-restart.service`; remove systemd unit generation | Critical | **v0.14 P2** | complete |
|
||||||
|
| REQ-173 | nft SNAT+DNAT ruleset render+apply: extend `internal/emitter/nft.go` with postrouting masquerade chain; new `internal/ingress/bootstrap.go` renders `orca.nft` + applies `nft -f` + ensures `/etc/traefik/dynamic` dir + pushes step-ca root CA + invokes podman traefik reconciler; wired into `orca init` (localhost lead) | Critical | **v0.14 P3** | complete |
|
||||||
|
| REQ-174 | Remote ingress bootstrap via SSH-push for `orca node join --type linux`: push step-ca root CA, render+apply nft remotely, invoke podman traefik reconciler remotely; register node as `linux` | Critical | **v0.14 P4** | complete |
|
||||||
|
| REQ-175 | Proxmox native ingress mode (`--ingress-mode native`, default): on PVE host, render+apply nft (vmbr-compatible, separate `orca-ingress` table avoids pve-firewall conflict); create unprivileged LXC with `--features nesting=1,keyctl=1` running podman+orca-traefik; push step-ca root CA into LXC; nft DNAT target = LXC bridge IP; register PVE host as `proxmox` node; add `IngressMode` field to `model.Node` + schema migration | Critical | **v0.14 P5** | complete |
|
||||||
|
| REQ-176 | Proxmox floating-IP mode (`--ingress-mode floating-ip --floating-ip --gateway --mac [--net-prefix]`): `pct create` Ubuntu LXC named `ingress` with `net0 bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway> --features nesting=1,keyctl=1 --onboot 1`; install podman + run orca-traefik inside LXC; apply nft DNAT+SNAT inside LXC; register LXC as managed `linux` node (name=`ingress`, addr=`<floating-ip>:8443`); interactive prompt for params when flags absent + not `--json`; validate IP/MAC/gateway; PVE host also registered as `proxmox` for workload dispatch | Critical | **v0.14 P6** | complete |
|
||||||
|
| REQ-177 | `orca doctor ingress [--peer]`: verify orca-traefik container running (`podman inspect`), nft DNAT+SNAT applied, `/etc/traefik/dynamic` exists, step-ca root CA mounted; extend `scripts/uat-signoff.sh` with ingress assertions (40 podman_traefik, 41 nft_dnat_snat, 42 linux_worker, 43 proxmox_native_lxc / floating_ip_lxc) | High | **v0.14 P7** | complete |
|
||||||
|
| REQ-178 | Docs: `docs/cli.md` (`--ingress-mode` + floating-IP flags + `doctor ingress`), `docs/uat.md` (native + floating-IP topologies), `docs/ingress.md` (podman-traefik image + volume mounts + certResolver), `docs/docker.md` (orca-traefik image), `ARCHITECTURE.md` (R-024 + ingress bootstrap section) | High | **v0.14 P7** | complete |
|
||||||
|
| REQ-179 | Integration tests: hermetic harness fakes SSH; asserts init→podman traefik running + nft applied; linux join→remote podman+nft; proxmox native→LXC created with nesting + podman traefik; floating-ip→`pct create` with correct net0 args + LXC registered as linux node; release.sh builds orca-traefik image (Dockerfile.traefik parses) | Critical | **v0.14 P7** | complete |
|
||||||
|
|
||||||
|
### Scope notes (v0.14)
|
||||||
|
|
||||||
|
- REQ-171..REQ-179 = 9 net-new requirements (REQ count grows 163 -> 172).
|
||||||
|
- 9 phases (P0 + P1..P7 + P8 final); feature milestone (multiple `feat` phases).
|
||||||
|
- Tags on v0.13.x patch line: `v0.13.0` (P0) ... `v0.13.8` (P8 final = v0.14 milestone release).
|
||||||
|
- Milestone branch: `milestone/v0.14-ingress-bootstrap`.
|
||||||
|
|
||||||
|
## Milestone v0.15: CI Release Pipeline Fix
|
||||||
|
|
||||||
|
**Scope**: fix the container image publishing pipeline. v0.14 shipped
|
||||||
|
`Dockerfile.traefik` + `Dockerfile` but no container images were
|
||||||
|
published to the Gitea registry because: (1) no Gitea Actions workflow
|
||||||
|
existed to trigger on tag pushes, (2) the CoreCI trigger script
|
||||||
|
stripped tag refs, (3) the `.coreci.yml` container-publish steps used
|
||||||
|
Docker-in-Docker (`docker:24-cli`) which is prohibited. v0.15 adds a
|
||||||
|
Gitea Actions workflow that triggers on tag pushes, installs the
|
||||||
|
`coreci` binary on the runner, and runs `coreci run`. The
|
||||||
|
`.coreci.yml` container-publish steps are rewritten to use kaniko
|
||||||
|
(no Docker daemon required).
|
||||||
|
|
||||||
|
| ID | Requirement | Priority | Phase | Status |
|
||||||
|
|----|-------------|----------|-------|--------|
|
||||||
|
| REQ-180 | Create `.gitea/workflows/release.yml` that triggers on `push: tags: ['v*']`, installs the `coreci` binary (from `git.cloudinit.dev/coreci/coreci`), injects `PAT_TOKEN` secret as `GITEA_TOKEN` env var, and runs `coreci run` — which executes the full `.coreci.yml` pipeline (validate, build, test, release) locally on the Gitea Actions runner | Critical | **v0.15 P1** | complete |
|
||||||
|
| REQ-181 | Replace `docker:24-cli` DinD steps in `.coreci.yml` with kaniko (`gcr.io/kaniko-project/executor:debug`): write `/kaniko/.docker/config.json` from `GITEA_TOKEN` (base64 auth), run `/kaniko/executor --dockerfile=<Dockerfile> --context=dir://. --destination=<registry/image:tag> --skip-tls-verify-registry`. Applies to both `container-publish` (orca image) and `container-publish-traefik` (orca-traefik image) | Critical | **v0.15 P1** | complete |
|
||||||
|
| REQ-182 | Set `PAT_TOKEN` Gitea Actions repository secret via `tea actions secrets create` (same value as `GITEA_TOKEN` from `.env`). Gitea reserves the `GITEA_` prefix for built-in secrets, so the secret must be named `PAT_TOKEN`, not `GITEA_PAT` | High | **v0.15 P0** | complete |
|
||||||
|
|
||||||
|
### Scope notes (v0.15)
|
||||||
|
|
||||||
|
- REQ-180..REQ-182 = 3 net-new requirements (REQ count grows 172 -> 175).
|
||||||
|
- 3 phases (P0 + P1 + P2 final); fix milestone (no `feat` phases — CI infrastructure).
|
||||||
|
- Tags on v0.14.x patch line: `v0.14.0` (P0) ... `v0.14.2` (P2 final = v0.15 milestone release).
|
||||||
|
- Milestone branch: `milestone/v0.15-ci-release-pipeline`.
|
||||||
|
- REQ-182 is complete: `PAT_TOKEN` secret created via `tea actions secrets create PAT_TOKEN <value> --repo coreci/orca`.
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
+86
-16
@@ -400,7 +400,7 @@ tags: `v0.10.0`…`v0.10.21`.
|
|||||||
- External CA / Let's Encrypt / cert transparency
|
- External CA / Let's Encrypt / cert transparency
|
||||||
- Online-only features (HSTS, OCSP stapling, telemetry)
|
- Online-only features (HSTS, OCSP stapling, telemetry)
|
||||||
|
|
||||||
## Milestone v0.12: Security Hardening (Zero-Trust Identity) — COMPLETE
|
## Milestone v0.12: Security Hardening (Zero-Trust Identity) — **COMPLETE**
|
||||||
|
|
||||||
**Scope**: comprehensive security hardening across the entire attack
|
**Scope**: comprehensive security hardening across the entire attack
|
||||||
surface, **including the operating system itself**, plus adoption of a
|
surface, **including the operating system itself**, plus adoption of a
|
||||||
@@ -549,7 +549,7 @@ The v1.0.0 production-ready tag stays deferred for post-v0.12 UAT
|
|||||||
- External CA / Let's Encrypt / cert transparency
|
- External CA / Let's Encrypt / cert transparency
|
||||||
- Online-only features (HSTS, OCSP stapling, telemetry)
|
- Online-only features (HSTS, OCSP stapling, telemetry)
|
||||||
|
|
||||||
## Milestone v0.13: Production Hardening Round 2 + UAT Plan — IN PROGRESS
|
## Milestone v0.13: Production Hardening Round 2 + UAT Plan — **COMPLETE**
|
||||||
|
|
||||||
**Scope**: final production hardening round before the v1.0.0
|
**Scope**: final production hardening round before the v1.0.0
|
||||||
production-ready tag. Three deep codebase sweeps (security, reliability,
|
production-ready tag. Three deep codebase sweeps (security, reliability,
|
||||||
@@ -579,20 +579,20 @@ signoff script that gates the v1.0.0 cut.
|
|||||||
|
|
||||||
### Phases (14 total: P0 + P01..P12 + P13 final)
|
### Phases (14 total: P0 + P01..P12 + P13 final)
|
||||||
|
|
||||||
- [ ] Phase P0: Pre-execution (SPECIFY→CLARIFY→RESEARCH→IDEATE→PLAN→GRILL) — tag `v0.12.0`
|
- [x] Phase P0: Pre-execution (SPECIFY→CLARIFY→RESEARCH→IDEATE→PLAN→GRILL) — tag `v0.12.0`
|
||||||
- [ ] Phase P01: Toolchain & dependency vulns (REQ-149) — tag `v0.12.1`
|
- [x] Phase P01: Toolchain & dependency vulns (REQ-149) — tag `v0.12.1`
|
||||||
- [ ] Phase P02: Input validation & injection hardening (REQ-150) — tag `v0.12.2`
|
- [x] Phase P02: Input validation & injection hardening (REQ-150) — tag `v0.12.2`
|
||||||
- [ ] Phase P03: Scheduler/deployment wiring + jobspec parser (REQ-151, REQ-152) — tag `v0.12.3`
|
- [x] Phase P03: Scheduler/deployment wiring + jobspec parser (REQ-151, REQ-152) — tag `v0.12.3`
|
||||||
- [ ] Phase P04: ACL enforcement + WebAuthn registration auth (REQ-153) — tag `v0.12.4`
|
- [x] Phase P04: ACL enforcement + WebAuthn registration auth (REQ-153) — tag `v0.12.4`
|
||||||
- [ ] Phase P05: Seal/audit CLI + chain race + key zeroing (REQ-154) — tag `v0.12.5`
|
- [x] Phase P05: Seal/audit CLI + chain race + key zeroing (REQ-154) — tag `v0.12.5`
|
||||||
- [ ] Phase P06: auth init-idp real + auth register (REQ-155) — tag `v0.12.6`
|
- [x] Phase P06: auth init-idp real + auth register (REQ-155) — tag `v0.12.6`
|
||||||
- [ ] Phase P07: Concurrency safety (REQ-156) — tag `v0.12.7`
|
- [x] Phase P07: Concurrency safety (REQ-156) — tag `v0.12.7`
|
||||||
- [ ] Phase P08: Transport & SSH safety (REQ-157) — tag `v0.12.8`
|
- [x] Phase P08: Transport & SSH safety (REQ-157) — tag `v0.12.8`
|
||||||
- [ ] Phase P09: Migration & operational safety (REQ-158) — tag `v0.12.9`
|
- [x] Phase P09: Migration & operational safety (REQ-158) — tag `v0.12.9`
|
||||||
- [ ] Phase P10: Observability & metrics (REQ-159) — tag `v0.12.10`
|
- [x] Phase P10: Observability & metrics (REQ-159) — tag `v0.12.10`
|
||||||
- [ ] Phase P11: Doc drift round 2 (REQ-160) — tag `v0.12.11`
|
- [x] Phase P11: Doc drift round 2 (REQ-160) — tag `v0.12.11`
|
||||||
- [ ] Phase P12: `--type linux` + UAT plan + signoff script (REQ-161, REQ-162, REQ-163) — tag `v0.12.12`
|
- [x] Phase P12: `--type linux` + UAT plan + signoff script (REQ-161, REQ-162, REQ-163) — tag `v0.12.12`
|
||||||
- [ ] Phase P13: Final review + ship + audit (milestone release) — tag `v0.12.13` = **v0.13 milestone release**
|
- [x] Phase P13: Final review + ship + audit (milestone release) — tag `v0.12.13` = **v0.13 milestone release**
|
||||||
|
|
||||||
**Milestone tag**: `v0.12.13` (final phase patch = milestone release per
|
**Milestone tag**: `v0.12.13` (final phase patch = milestone release per
|
||||||
feature-milestone rule; no separate `v0.13.0` tag). Per-phase tags:
|
feature-milestone rule; no separate `v0.13.0` tag). Per-phase tags:
|
||||||
@@ -675,3 +675,73 @@ CI agent verifies and cuts v1.0.0).
|
|||||||
- jobspec `health` prober (v0.13 adds lint warning; enforcement deferred)
|
- jobspec `health` prober (v0.13 adds lint warning; enforcement deferred)
|
||||||
- jobspec `update` rolling/canary controller (v0.13 adds lint warning; enforcement deferred)
|
- jobspec `update` rolling/canary controller (v0.13 adds lint warning; enforcement deferred)
|
||||||
- jobspec `schedule.cron` scheduler loop (v0.13 adds lint warning; enforcement deferred)
|
- jobspec `schedule.cron` scheduler loop (v0.13 adds lint warning; enforcement deferred)
|
||||||
|
|
||||||
|
## Milestone v0.14: Ingress Bootstrap Completeness — **COMPLETE**
|
||||||
|
|
||||||
|
**Scope**: ensure that linux & proxmox types are properly bootstrapped with
|
||||||
|
traefik during cluster init or node join. All cluster endpoints are
|
||||||
|
provisioned as sockets (R-007); routing between jobs and services depends on
|
||||||
|
traefik being present and properly configured. v0.13 shipped traefik binary +
|
||||||
|
systemd unit + empty dynamic dir but never wrote the static config nor applied
|
||||||
|
nft rules. v0.14 replaces the binary+systemd model with a **podman container**
|
||||||
|
running a custom `orca-traefik` image, and completes the nft SNAT+DNAT ingress
|
||||||
|
stack on every node type.
|
||||||
|
|
||||||
|
**New load-bearing rule**: **R-024** — Traefik runs exclusively as a podman
|
||||||
|
container from the `orca-traefik` image published per release. Every
|
||||||
|
orca-managed ingress surface bootstraps nft DNAT + SNAT/MASQUERADE +
|
||||||
|
`podman run --restart=always --network host` with dynamic-config + step-ca
|
||||||
|
root CA volume mounts.
|
||||||
|
|
||||||
|
**Three topologies**:
|
||||||
|
1. **Linux**: host → nft → `podman run orca-traefik` (host network)
|
||||||
|
2. **Proxmox Native**: host → nft → LXC (nesting=1) → `podman run orca-traefik`
|
||||||
|
3. **Proxmox Floating-IP**: LXC (owns floating IP) → nft (inside LXC) → `podman run orca-traefik`
|
||||||
|
|
||||||
|
**Milestone type**: feature (multiple `feat` phases). Tags on v0.13.x patch
|
||||||
|
line: `v0.13.0` (P0) ... `v0.13.8` (P8 final = v0.14 milestone release).
|
||||||
|
|
||||||
|
- [x] Phase 0: Pre-execution (SPECIFY→CLARIFY→RESEARCH→PLAN→GRILL) — tag `v0.13.0`
|
||||||
|
- [x] Phase 1: `orca-traefik` container image + release pipeline (REQ-171) — tag `v0.13.1`
|
||||||
|
- [x] Phase 2: Podman traefik reconciler — replace binary+systemd install (REQ-172) — tag `v0.13.2`
|
||||||
|
- [x] Phase 3: nft SNAT+DNAT + `orca init` ingress bootstrap (REQ-173) — tag `v0.13.3`
|
||||||
|
- [x] Phase 4: `orca node join --type linux` remote ingress bootstrap (REQ-174) — tag `v0.13.4`
|
||||||
|
- [x] Phase 5: Proxmox native ingress mode — LXC + podman traefik (REQ-175) — tag `v0.13.5`
|
||||||
|
- [x] Phase 6: Proxmox floating-IP LXC ingress + interactive prompt (REQ-176) — tag `v0.13.6`
|
||||||
|
- [x] Phase 7: `doctor ingress` + docs + integration tests (REQ-177,178,179) — tag `v0.13.7`
|
||||||
|
- [x] Phase 8: Final review + ship + audit (milestone release) — tag `v0.13.8` = **v0.14 milestone release**
|
||||||
|
|
||||||
|
### Per-phase REQ coverage (v0.14)
|
||||||
|
|
||||||
|
- **P1** — `Dockerfile.traefik` + release pipeline (REQ-171)
|
||||||
|
- **P2** — Podman traefik reconciler (REQ-172)
|
||||||
|
- **P3** — nft SNAT+DNAT + init bootstrap (REQ-173)
|
||||||
|
- **P4** — Linux node join remote ingress (REQ-174)
|
||||||
|
- **P5** — Proxmox native ingress — LXC + podman (REQ-175)
|
||||||
|
- **P6** — Proxmox floating-IP LXC ingress (REQ-176)
|
||||||
|
- **P7** — doctor ingress + docs + tests (REQ-177,178,179)
|
||||||
|
- **P8** — Final review + ship + audit
|
||||||
|
|
||||||
|
### v0.14 is a continuation milestone, not a direction change
|
||||||
|
|
||||||
|
The vision ("minimalist, offline-first, CLI-first orchestration engine")
|
||||||
|
is unchanged. v0.14 completes the ingress bootstrap that v0.13 left
|
||||||
|
non-functional (binary installed but no config, no nft applied). The
|
||||||
|
podman-container model is the operator's constraint; the architecture's
|
||||||
|
socket+traefik routing design (R-007, R-017) is unchanged.
|
||||||
|
|
||||||
|
## Milestone v0.15: CI Release Pipeline Fix — **COMPLETE**
|
||||||
|
|
||||||
|
**Scope**: fix container image publishing. v0.14 shipped
|
||||||
|
`Dockerfile.traefik` + `Dockerfile` but no images were published
|
||||||
|
because no Gitea Actions workflow triggered on tag pushes, and
|
||||||
|
`.coreci.yml` used Docker-in-Docker. v0.15 adds a Gitea Actions
|
||||||
|
workflow (trigger on tag push → install coreci → `coreci run`) and
|
||||||
|
rewrites the container-publish steps to use kaniko (no DinD).
|
||||||
|
|
||||||
|
**Milestone type**: fix (CI infrastructure). Tags on v0.14.x patch
|
||||||
|
line: `v0.14.0` (P0) ... `v0.14.2` (P2 final = v0.15 milestone release).
|
||||||
|
|
||||||
|
- [x] Phase 0: Pre-execution (SPECIFY→CLARIFY→RESEARCH→PLAN→GRILL) — tag `v0.14.0`
|
||||||
|
- [x] Phase 1: Gitea Actions workflow + .coreci.yml kaniko rewrite (REQ-180,181) — tag `v0.14.1`
|
||||||
|
- [x] Phase 2: Final review + ship + audit (milestone release) — tag `v0.14.2` = **v0.15 milestone release**
|
||||||
|
|||||||
@@ -5,9 +5,9 @@
|
|||||||
"slug": "orca",
|
"slug": "orca",
|
||||||
"name": "Orca",
|
"name": "Orca",
|
||||||
"description": "Offline/CLI-first orchestration engine (Orca) \u2014 Nomad-inspired, far simpler than Kubernetes",
|
"description": "Offline/CLI-first orchestration engine (Orca) \u2014 Nomad-inspired, far simpler than Kubernetes",
|
||||||
"milestone": "v0.13",
|
"milestone": "v0.15",
|
||||||
"phase": 0,
|
"phase": 0,
|
||||||
"milestone_type": "feature",
|
"milestone_type": "fix",
|
||||||
"default_branch": "main",
|
"default_branch": "main",
|
||||||
"tech_stack": {
|
"tech_stack": {
|
||||||
"language": "go",
|
"language": "go",
|
||||||
@@ -185,7 +185,8 @@
|
|||||||
"registry": "git.cloudinit.dev",
|
"registry": "git.cloudinit.dev",
|
||||||
"owner": "coreci",
|
"owner": "coreci",
|
||||||
"image": "orca",
|
"image": "orca",
|
||||||
"credential_env": "GITEA_TOKEN"
|
"credential_env": "GITEA_TOKEN",
|
||||||
|
"images": ["orca", "orca-traefik"]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"secrets": {
|
"secrets": {
|
||||||
|
|||||||
+39
-31
@@ -23,25 +23,25 @@ pipelines:
|
|||||||
description: Validate Go toolchain, formatting, and security scans
|
description: Validate Go toolchain, formatting, and security scans
|
||||||
steps:
|
steps:
|
||||||
- name: go-version
|
- name: go-version
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
commands:
|
commands:
|
||||||
- go version
|
- go version
|
||||||
- gofmt -l .
|
- gofmt -l .
|
||||||
- go vet ./...
|
- go vet ./...
|
||||||
|
|
||||||
- name: verify-reqs
|
- name: verify-reqs
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
commands:
|
commands:
|
||||||
- make verify-reqs
|
- make verify-reqs
|
||||||
|
|
||||||
- name: gosec
|
- name: gosec
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
commands:
|
commands:
|
||||||
- go install github.com/securego/gosec/v2/cmd/gosec@v2.18.2
|
- go install github.com/securego/gosec/v2/cmd/gosec@v2.18.2
|
||||||
- gosec -fmt text -quiet ./...
|
- gosec -fmt text -quiet ./...
|
||||||
|
|
||||||
- name: govulncheck
|
- name: govulncheck
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
env:
|
env:
|
||||||
# REQ-027: offline mode. GOFLAGS=-mod=mod ensures module mode;
|
# REQ-027: offline mode. GOFLAGS=-mod=mod ensures module mode;
|
||||||
# GOVULNCHECK_DB (when present) overrides the bundled DB.
|
# GOVULNCHECK_DB (when present) overrides the bundled DB.
|
||||||
@@ -51,7 +51,7 @@ pipelines:
|
|||||||
- govulncheck -mode binary ./...
|
- govulncheck -mode binary ./...
|
||||||
|
|
||||||
- name: gitleaks
|
- name: gitleaks
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
commands:
|
commands:
|
||||||
- apk add --no-cache curl
|
- apk add --no-cache curl
|
||||||
- sh -c "$(curl -fsSL https://github.com/gitleaks/gitleaks/releases/latest/download/install.sh)"
|
- sh -c "$(curl -fsSL https://github.com/gitleaks/gitleaks/releases/latest/download/install.sh)"
|
||||||
@@ -61,7 +61,7 @@ pipelines:
|
|||||||
description: Build the orca binary with version injection
|
description: Build the orca binary with version injection
|
||||||
steps:
|
steps:
|
||||||
- name: build
|
- name: build
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
env:
|
env:
|
||||||
VERSION: ${CI_COMMIT_TAG:-dev}
|
VERSION: ${CI_COMMIT_TAG:-dev}
|
||||||
GIT_COMMIT: ${CI_COMMIT_SHA}
|
GIT_COMMIT: ${CI_COMMIT_SHA}
|
||||||
@@ -80,7 +80,7 @@ pipelines:
|
|||||||
description: Run all tests with race detection and coverage (REQ-031)
|
description: Run all tests with race detection and coverage (REQ-031)
|
||||||
steps:
|
steps:
|
||||||
- name: test
|
- name: test
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
commands:
|
commands:
|
||||||
- go test -race -coverprofile=coverage.out ./...
|
- go test -race -coverprofile=coverage.out ./...
|
||||||
- go tool cover -func=coverage.out | tail -1
|
- go tool cover -func=coverage.out | tail -1
|
||||||
@@ -91,7 +91,7 @@ pipelines:
|
|||||||
ref: "refs/tags/v*"
|
ref: "refs/tags/v*"
|
||||||
steps:
|
steps:
|
||||||
- name: build-artifact
|
- name: build-artifact
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
env:
|
env:
|
||||||
VERSION: ${CI_COMMIT_TAG}
|
VERSION: ${CI_COMMIT_TAG}
|
||||||
GIT_COMMIT: ${CI_COMMIT_SHA}
|
GIT_COMMIT: ${CI_COMMIT_SHA}
|
||||||
@@ -105,37 +105,45 @@ pipelines:
|
|||||||
go build -trimpath -ldflags="${LDFLAGS}" -o bin/orca ./cmd/orca
|
go build -trimpath -ldflags="${LDFLAGS}" -o bin/orca ./cmd/orca
|
||||||
- make changelog
|
- make changelog
|
||||||
- tar -czf orca-${VERSION}-linux-amd64.tar.gz -C bin orca
|
- tar -czf orca-${VERSION}-linux-amd64.tar.gz -C bin orca
|
||||||
- ls -lh orca-${VERSION}-linux-amd64.tar.gz
|
- sha256sum orca-${VERSION}-linux-amd64.tar.gz > SHA256SUMS
|
||||||
|
- ls -lh orca-${VERSION}-linux-amd64.tar.gz SHA256SUMS
|
||||||
|
- cat SHA256SUMS
|
||||||
- name: gitea-release
|
- name: gitea-release
|
||||||
image: golang:1.25
|
image: golang:1.25.12
|
||||||
env:
|
env:
|
||||||
GITEA_TOKEN: ${GITEA_TOKEN}
|
GITEA_TOKEN: ${GITEA_TOKEN}
|
||||||
VERSION: ${CI_COMMIT_TAG}
|
VERSION: ${CI_COMMIT_TAG}
|
||||||
commands:
|
commands:
|
||||||
- apk add --no-cache curl tar
|
- apk add --no-cache curl tar python3
|
||||||
- sh -c "$(curl -fsSL https://gitea.com/gitea/tea/releases/latest/download/install.sh)"
|
- sh -c "$(curl -fsSL https://gitea.com/gitea/tea/releases/latest/download/install.sh)"
|
||||||
- tea releases create ${VERSION}
|
- tea releases create ${VERSION}
|
||||||
--repo coreci/orca
|
--repo coreci/orca
|
||||||
--title "Orca ${VERSION}"
|
--title "Orca ${VERSION}"
|
||||||
--note-file CHANGELOG.md
|
--note-file CHANGELOG.md
|
||||||
--asset orca-${VERSION}-linux-amd64.tar.gz
|
--asset orca-${VERSION}-linux-amd64.tar.gz
|
||||||
- name: container-publish
|
--asset SHA256SUMS
|
||||||
description: Build and publish OCI image to Gitea container registry (REQ-046)
|
- |
|
||||||
image: docker:24-cli
|
# Verify assets are actually attached (REQ-097, gate C-21).
|
||||||
env:
|
# tea releases create has been observed to exit 0 without
|
||||||
GITEA_TOKEN: ${GITEA_TOKEN}
|
# attaching the asset in some versions. Verify via the API.
|
||||||
VERSION: ${CI_COMMIT_TAG}
|
ASSET_COUNT=$(curl -fsSL \
|
||||||
GIT_COMMIT: ${CI_COMMIT_SHA}
|
"https://git.cloudinit.dev/api/v1/repos/coreci/orca/releases/tags/${VERSION}" \
|
||||||
BUILD_TIME: ${CI_BUILD_TIME}
|
| python3 -c "import json,sys; r=json.load(sys.stdin); print(len(r.get('assets',[])))")
|
||||||
commands:
|
echo "Release ${VERSION} has ${ASSET_COUNT} assets"
|
||||||
- docker build
|
if [ "${ASSET_COUNT}" -lt 2 ]; then
|
||||||
--build-arg VERSION=${VERSION}
|
echo "ERROR: Expected at least 2 assets (tarball + SHA256SUMS), got ${ASSET_COUNT}"
|
||||||
--build-arg GIT_COMMIT=${GIT_COMMIT}
|
echo "Attempting to attach assets manually..."
|
||||||
--build-arg BUILD_TIME=${BUILD_TIME}
|
TARBALL_URL=$(curl -fsSL \
|
||||||
-t git.cloudinit.dev/coreci/orca:${VERSION}
|
"https://git.cloudinit.dev/api/v1/repos/coreci/orca/releases/tags/${VERSION}" \
|
||||||
-t git.cloudinit.dev/coreci/orca:latest
|
| python3 -c "import json,sys; r=json.load(sys.stdin); print(r.get('id',''))")
|
||||||
.
|
if [ -n "${TARBALL_URL}" ]; then
|
||||||
- echo "${GITEA_TOKEN}" | docker login git.cloudinit.dev -u cloudinit-bot --password-stdin
|
curl -fsSL -X "POST" \
|
||||||
- docker push git.cloudinit.dev/coreci/orca:${VERSION}
|
"https://git.cloudinit.dev/api/v1/repos/coreci/orca/releases/${TARBALL_URL}/assets?name=orca-${VERSION}-linux-amd64.tar.gz" \
|
||||||
- docker push git.cloudinit.dev/coreci/orca:latest
|
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||||
- docker logout git.cloudinit.dev
|
-F "attachment=@orca-${VERSION}-linux-amd64.tar.gz"
|
||||||
|
curl -fsSL -X "POST" \
|
||||||
|
"https://git.cloudinit.dev/api/v1/repos/coreci/orca/releases/${TARBALL_URL}/assets?name=SHA256SUMS" \
|
||||||
|
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||||
|
-F "attachment=@SHA256SUMS"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|||||||
@@ -0,0 +1,90 @@
|
|||||||
|
name: Release
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'v*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
ci:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version: '1.25'
|
||||||
|
|
||||||
|
- name: Install CoreCI
|
||||||
|
run: |
|
||||||
|
git clone --depth=1 https://git.cloudinit.dev/coreci/coreci.git /tmp/coreci
|
||||||
|
cd /tmp/coreci
|
||||||
|
CGO_ENABLED=0 go build -tags sqlite_go,embed -o /usr/local/bin/coreci ./cmd/coreci
|
||||||
|
coreci version
|
||||||
|
|
||||||
|
- name: Run CoreCI pipeline
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.PAT_TOKEN }}
|
||||||
|
run: |
|
||||||
|
coreci run
|
||||||
|
|
||||||
|
container-orca:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: ci
|
||||||
|
container:
|
||||||
|
image: gcr.io/kaniko-project/executor:debug
|
||||||
|
options: --entrypoint /bin/sh
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Build and push orca image
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.PAT_TOKEN }}
|
||||||
|
VERSION: ${{ gitea.ref_name }}
|
||||||
|
run: |
|
||||||
|
mkdir -p /kaniko/.docker
|
||||||
|
AUTH=$(echo -n "cloudinit-bot:${GITEA_TOKEN}" | base64 -w0)
|
||||||
|
echo "{\"auths\":{\"git.cloudinit.dev\":{\"auth\":\"${AUTH}\"}}}" > /kaniko/.docker/config.json
|
||||||
|
GIT_COMMIT=$(echo -n "${{ gitea.sha }}" | cut -c1-12)
|
||||||
|
BUILD_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||||
|
/kaniko/executor \
|
||||||
|
--dockerfile=Dockerfile \
|
||||||
|
--context=dir://. \
|
||||||
|
--destination=git.cloudinit.dev/coreci/orca:${VERSION} \
|
||||||
|
--destination=git.cloudinit.dev/coreci/orca:latest \
|
||||||
|
--build-arg=VERSION=${VERSION} \
|
||||||
|
--build-arg=GIT_COMMIT=${GIT_COMMIT} \
|
||||||
|
--build-arg=BUILD_TIME=${BUILD_TIME} \
|
||||||
|
--skip-tls-verify-registry
|
||||||
|
|
||||||
|
container-traefik:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: ci
|
||||||
|
container:
|
||||||
|
image: gcr.io/kaniko-project/executor:debug
|
||||||
|
options: --entrypoint /bin/sh
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Build and push orca-traefik image
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.PAT_TOKEN }}
|
||||||
|
VERSION: ${{ gitea.ref_name }}
|
||||||
|
run: |
|
||||||
|
if [ ! -f Dockerfile.traefik ]; then
|
||||||
|
echo "Dockerfile.traefik not found at this tag — skipping orca-traefik image"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
mkdir -p /kaniko/.docker
|
||||||
|
AUTH=$(echo -n "cloudinit-bot:${GITEA_TOKEN}" | base64 -w0)
|
||||||
|
echo "{\"auths\":{\"git.cloudinit.dev\":{\"auth\":\"${AUTH}\"}}}" > /kaniko/.docker/config.json
|
||||||
|
/kaniko/executor \
|
||||||
|
--dockerfile=Dockerfile.traefik \
|
||||||
|
--context=dir://. \
|
||||||
|
--destination=git.cloudinit.dev/coreci/orca-traefik:${VERSION} \
|
||||||
|
--destination=git.cloudinit.dev/coreci/orca-traefik:latest \
|
||||||
|
--skip-tls-verify-registry
|
||||||
+132
-26
@@ -5,31 +5,137 @@ All notable changes to orca are documented in this file.
|
|||||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
- `e1b538575c57158c7a6661d5919b103f6c7932fc` — feat(P06): CoreCI release flow with .coreci.yml and tea integration
|
## v0.13 milestone (in progress) — tag line v0.12.x
|
||||||
- `07b8ad2ceaba7ca303dfe91876930d33b76c633e` — ship(P05): health checks merged into milestone
|
|
||||||
- `b06458d31370750417a3b239dac61c6e2fdf5329` — docs(P05): verification - 4 layers pass
|
The v0.13 milestone is **Production Hardening Round 2 + UAT Plan**.
|
||||||
- `708d9834296271094667700e88e80bfa27db7bdd` — feat(P05): health check daemon with /healthz, /readyz, /v1/* handlers
|
Three deep codebase sweeps (security, reliability, feature/doc claims)
|
||||||
- `30c523c0c7a8e75a2e97b42f1c8a39802febcbdc` — ship(P04): state persistence merged into milestone
|
surfaced ~60 gaps beyond v0.12. v0.13 closes all critical/high/medium
|
||||||
- `759b1b519d7fadf3d91d3070952d9ad2051a0eba` — docs(P04): verification - 4 layers pass
|
findings and delivers the UAT plan + signoff script that gates the
|
||||||
- `b25e074e1d3518f175478184ff8d002ec0d8412c` — feat(P04): audit log + persistence hardening
|
v1.0.0 cut.
|
||||||
- `bb6b5b3e8342c16601a8503223c7186ecdbb00df` — ship(P03): task exec merged into milestone
|
|
||||||
- `857f7563190e7703f97c607a50d6b0a897d250e9` — docs(P03): verification - 4 layers pass
|
**Load-bearing architectural changes**:
|
||||||
- `f9a98733411cfa8657e82636e0c55671086ebe46` — feat(P03): task execution engine with HCL specs, jobs, tasks, WaitDelay
|
- **R-022** — `orca job run` deploys to remote nodes via the scheduler →
|
||||||
- `78334f1f74f0c185c6d38014c796aac4903b8141` — ship(P02): node mgmt merged into milestone
|
emitter → SSH-push pipeline. The local `exec.CommandContext` path is
|
||||||
- `c7dbcef9587596786a541a7566479d9fb93fcf0a` — docs(P02): verification - 4 layers pass
|
removed (P03).
|
||||||
- `9580f347c68e395dccfbe83b27a857d52bf21075` — feat(P02): node management with SQLite-backed registry
|
- **R-023** — Zero-trust enforcement is operationally wired: `acl.Check`
|
||||||
- `46e929e4c6539bd604539ba27d5ed0c606e87bb9` — chore(P01): source .env in trigger_coreci.sh for GITEA_TOKEN
|
is invoked on every daemon handler + sshpush + txn apply path;
|
||||||
- `503923bf1ee2c60f8375acc7eb9608d346368e1c` — ship(P01): cli skeleton merged into milestone
|
`acl.json` is 0600; audit `actor` carries OIDC sub/SVID; WebAuthn
|
||||||
- `e3f6e1df825d39f73933c9996bd2cc4717ff1061` — docs(P01): verification - 4 layers pass
|
registration requires auth; `cluster seal`/`unseal` + `doctor audit`/
|
||||||
- `aa3cccead503a37dfec75873d06d2d396a2876f2` — feat(P01): CLI skeleton with Cobra, subcommand stubs, pre-push hook
|
`doctor modes` CLI commands exist (P04, P05).
|
||||||
- `c2038952c74f7c242ba3be65d2f4269b23685f5a` — docs(P00): create 6 phase plans with wave ordering
|
|
||||||
- `65eb2e601b741b36388598b9f8adddd7bd8dd3a8` — docs(P00): research findings - architecture + personas
|
### v0.13 phase commits (v0.11.29..HEAD)
|
||||||
- `6f34f1794b9f526c06a1dc139d4a74371599502e` — docs(P00): ideation - 30 ideas accepted (3 tiers)
|
|
||||||
- `bc7ce1caf672e87774455a6cd6cc0db986cd09b3` — docs(P00): clarify ambiguities (full autonomy, 10 decisions)
|
- `ed91d68` — feat(P10): observability expansion — metrics + security headers (REQ-159)
|
||||||
- `55aae5347ec09bce9ef7697ea0c9c9ee158bc040` — chore(P00): rename orch-engine to orca, configure gitea + coreci (v0.1)
|
- `531b369` — fix(P09): migration + operational safety — job stop, retention, logs cap (REQ-158)
|
||||||
- `0cba1aa5feef9564f8b9a2a97ae735dc859a8a84` — chore(P00): set autonomy level to full
|
- `3a3ea74` — fix(P08): transport + SSH safety — typed errors, IPv6, timeouts, signal (REQ-157)
|
||||||
- `e2e77e79b9cbfb462044662543845476f843161b` — chore(P00): quick task - populate config.json with backlog reference
|
- `0358efe` — fix(P07): concurrency safety — SQLite, flock, cache, atomic writes (REQ-156)
|
||||||
- `8c086def698bf0af31e8e820b6b7a2783af06f43` — chore(config): populate ciagent config with standard settings
|
- `978334a` — feat(P06): auth init-idp real + auth register + doctor oidc (REQ-155)
|
||||||
- `8774008c3e47e4ca4711f4fef164531006d16216` — docs(init): validate specification
|
- `9e83238` — feat(P05): seal/audit CLI + chain race fix + key zeroing (REQ-154)
|
||||||
|
- `5232fcb` — fix(P04): wire ACL enforcement + WebAuthn reg auth + audit actor (REQ-153)
|
||||||
|
- `cf3d98e` — feat(P03): wire scheduler into job run + fix jobspec parser (REQ-151, REQ-152)
|
||||||
|
- `4b70e31` — fix(P02): input validation + injection hardening — 11 vectors (REQ-150)
|
||||||
|
- `b0158c9` — fix(P01): bump go toolchain to 1.25.12 + fix pre-existing test bugs (REQ-149)
|
||||||
|
- `7479cd1` — docs(checkpoint): P0 shipped — v0.12.0 tagged
|
||||||
|
- `1a2dd1a` — docs(P00): incorporate grill binding conditions C-44..C-49
|
||||||
|
- `437d9b2` — docs(P00): grill v0.13 — CONDITIONAL PROCEED (6 binding conditions C-44..C-49)
|
||||||
|
- `82bfab1` — docs(P00): create phase plans — 14 phases, 15 REQs, vertical slices
|
||||||
|
- `a2a651e` — docs(P00): ideation results — 15 accepted (REQ-149..REQ-163), 0 skipped
|
||||||
|
- `3f5e5de` — docs(P00): research findings — threat model round 3 (~60 gaps, F26-F101)
|
||||||
|
- `7a60b35` — docs(P00): clarify v0.13 — 7 decisions resolved (D-248..D-254)
|
||||||
|
- `8071793` — docs(init): validate specification — v0.13 Production Hardening Round 2 + UAT Plan
|
||||||
|
|
||||||
|
### v0.13 phase summary
|
||||||
|
|
||||||
|
- **P0** — Pre-execution: specify → clarify → research → ideate → plan → grill (tag `v0.12.0`)
|
||||||
|
- **P01** — Toolchain & dependency vulns: Go 1.25.12 bump, 24 stdlib vulns closed, govulncheck triage (REQ-149)
|
||||||
|
- **P02** — Input validation & injection hardening: 11 vectors closed (`orca logs --job` RCE, tar-slip, sudoers injection, pprof loopback, txn/nft ID validation, drain allocID, cluster_compat, podman image, nft TrustedProbes, sudoers user/role) (REQ-150)
|
||||||
|
- **P03** — Scheduler/deployment wiring + jobspec parser: `orca job run` wires scheduler → emitter → SSH-push; `schedule:`/`timeout:` parsed by markdown jobspec (REQ-151, REQ-152)
|
||||||
|
- **P04** — ACL enforcement + WebAuthn registration auth: `acl.Check` wired into daemon + sshpush + txn apply; WebAuthn registration requires auth; audit actor carries OIDC sub/SVID (REQ-153)
|
||||||
|
- **P05** — Seal/audit CLI + chain race fix + key zeroing: `orca cluster seal`/`unseal`, `orca doctor audit`, `orca doctor modes` CLI commands; audit hash-chain race fix; master key zeroed on exit (REQ-154)
|
||||||
|
- **P06** — auth init-idp real + auth register + doctor oidc: real Dex deployment, `orca auth register` browser flow, `orca doctor oidc` health check (REQ-155)
|
||||||
|
- **P07** — Concurrency safety: SQLite WAL, flock on known_hosts, cache thread-safety, atomic writes (REQ-156)
|
||||||
|
- **P08** — Transport & SSH safety: typed dial errors, IPv6 support, connect timeouts, signal handling (REQ-157)
|
||||||
|
- **P09** — Migration & operational safety: `orca job stop` via SSH, DB retention check, logs cap (REQ-158)
|
||||||
|
- **P10** — Observability & metrics: metrics endpoint expansion, security headers (REQ-159)
|
||||||
|
- **P11** — Doc drift round 2 (this phase, REQ-160)
|
||||||
|
|
||||||
|
## v0.12 milestone — COMPLETE (tag line v0.11.x)
|
||||||
|
|
||||||
|
The v0.12 milestone is **Security Hardening (Zero-Trust Identity)**.
|
||||||
|
Comprehensive security hardening across the entire attack surface
|
||||||
|
including the OS, plus adoption of a zero-trust identity model. 25
|
||||||
|
threat-model findings (F1..F25) closed. R-021 adopted: no Orca-issued
|
||||||
|
credentials — human identity is exclusively external (OIDC), machine
|
||||||
|
identity is exclusively mTLS/SPIFFE.
|
||||||
|
|
||||||
|
**Milestone release**: `v0.11.28` (29 phases, tags `v0.11.0`..`v0.11.28`).
|
||||||
|
|
||||||
|
### v0.12 phase highlights
|
||||||
|
|
||||||
|
- Command injection fix (REQ-119, F3)
|
||||||
|
- Namespace path traversal fix (REQ-120, F4)
|
||||||
|
- Txn apply path allowlist (REQ-121, F5)
|
||||||
|
- OIDC client + bundled Dex (REQ-144; BYO-IdP override)
|
||||||
|
- WebAuthn connector for Dex / passkeys (REQ-148)
|
||||||
|
- ACL rewrite to OIDC claims + enforcement (REQ-145, REQ-122, F1)
|
||||||
|
- Remove all password/token paths (REQ-146, R-021, C-34)
|
||||||
|
- Master key seal-to-OIDC + Shamir 3-of-5 recovery (REQ-147, C-35)
|
||||||
|
- Daemon auth hardening (REQ-123, REQ-124, F6, F24)
|
||||||
|
- Audit log tamper-evidence (REQ-125, F2)
|
||||||
|
- SVID chain validation (REQ-126, F9)
|
||||||
|
- Backup symlink validation (REQ-127, F7)
|
||||||
|
- step-ca /tmp hardening (REQ-128, F10)
|
||||||
|
- Master key rotation (REQ-129, F12, C-30)
|
||||||
|
- File-mode audit expansion (REQ-130, F13)
|
||||||
|
- aggregate.sh JSON injection + drift-gate fix (REQ-131, F11, F18)
|
||||||
|
- install.sh checksum+GPG verification (REQ-132, F14)
|
||||||
|
- nftables ruleset hardening (REQ-133, F21)
|
||||||
|
- sudoers hardening (REQ-134, F22)
|
||||||
|
- System user consistency (REQ-135, F23)
|
||||||
|
- SQLite file-mode + at-rest encryption (REQ-136, F8, C-31)
|
||||||
|
- Migration safety + identity migration (REQ-137, F19, C-34)
|
||||||
|
- Legacy CA/mTLS/daemon + step-ca password-provisioner deletion (REQ-138, F16)
|
||||||
|
- known_hosts tightening + transport hardening (REQ-139, F15, F25)
|
||||||
|
- Drift event authentication (REQ-140, F18)
|
||||||
|
- Security integration test suite (REQ-141, C-33)
|
||||||
|
- Zero-trust + OIDC + WebAuthn + threat-model docs (REQ-142)
|
||||||
|
- Final review + ship + audit (REQ-143)
|
||||||
|
|
||||||
|
## v0.11 milestone — COMPLETE (tag line v0.10.x)
|
||||||
|
|
||||||
|
The v0.11 milestone is **Production Hardening**. See the git log and
|
||||||
|
ROADMAP for the full phase list.
|
||||||
|
|
||||||
|
## v0.1 milestone — COMPLETE
|
||||||
|
|
||||||
|
Initial CLI skeleton, node management, task execution, state
|
||||||
|
persistence, audit log, health checks, and CoreCI release flow.
|
||||||
|
|
||||||
|
- `e1b5385` — feat(P06): CoreCI release flow with .coreci.yml and tea integration
|
||||||
|
- `07b8ad2` — ship(P05): health checks merged into milestone
|
||||||
|
- `b06458d` — docs(P05): verification - 4 layers pass
|
||||||
|
- `708d983` — feat(P05): health check daemon with /healthz, /readyz, /v1/* handlers
|
||||||
|
- `30c523c` — ship(P04): state persistence merged into milestone
|
||||||
|
- `759b1b5` — docs(P04): verification - 4 layers pass
|
||||||
|
- `b25e074` — feat(P04): audit log + persistence hardening
|
||||||
|
- `bb6b5b3` — ship(P03): task exec merged into milestone
|
||||||
|
- `857f756` — docs(P03): verification - 4 layers pass
|
||||||
|
- `f9a9873` — feat(P03): task execution engine with HCL specs, jobs, tasks, WaitDelay
|
||||||
|
- `78334f1` — ship(P02): node mgmt merged into milestone
|
||||||
|
- `c7dbcef` — docs(P02): verification - 4 layers pass
|
||||||
|
- `9580f34` — feat(P02): node management with SQLite-backed registry
|
||||||
|
- `46e929e` — chore(P01): source .env in trigger_coreci.sh for GITEA_TOKEN
|
||||||
|
- `503923b` — ship(P01): cli skeleton merged into milestone
|
||||||
|
- `e3f6e1d` — docs(P01): verification - 4 layers pass
|
||||||
|
- `aa3ccce` — feat(P01): CLI skeleton with Cobra, subcommand stubs, pre-push hook
|
||||||
|
- `c203895` — docs(P00): create 6 phase plans with wave ordering
|
||||||
|
- `65eb2e6` — docs(P00): research findings - architecture + personas
|
||||||
|
- `6f34f17` — docs(P00): ideation - 30 ideas accepted (3 tiers)
|
||||||
|
- `bc7ce1c` — docs(P00): clarify ambiguities (full autonomy, 10 decisions)
|
||||||
|
- `55aae53` — chore(P00): rename orch-engine to orca, configure gitea + coreci (v0.1)
|
||||||
|
- `0cba1aa` — chore(P00): set autonomy level to full
|
||||||
|
- `e2e77e7` — chore(P00): quick task - populate config.json with backlog reference
|
||||||
|
- `8c086de` — chore(config): populate ciagent config with standard settings
|
||||||
|
- `8774008` — docs(init): validate specification
|
||||||
|
|
||||||
Generated by make changelog. Do not edit by hand.
|
Generated by make changelog. Do not edit by hand.
|
||||||
|
|||||||
+1
-1
@@ -21,7 +21,7 @@ ARG BUILD_TIME=unknown
|
|||||||
|
|
||||||
# --- Stage 1: build -------------------------------------------------------
|
# --- Stage 1: build -------------------------------------------------------
|
||||||
|
|
||||||
FROM golang:1.25 AS builder
|
FROM golang:1.25.12 AS builder
|
||||||
|
|
||||||
ARG VERSION
|
ARG VERSION
|
||||||
ARG GIT_COMMIT
|
ARG GIT_COMMIT
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Dockerfile.traefik — custom orca-traefik image (R-024)
|
||||||
|
#
|
||||||
|
# Extends the official traefik:v3.3.0 image with a baked default static
|
||||||
|
# config. The host-side /etc/traefik/traefik.yml (rendered by
|
||||||
|
# emitter.RenderTraefikStaticConfig) is mounted :ro at runtime to
|
||||||
|
# override this default — preserving the traefik-on-public-ip opt-out
|
||||||
|
# (REQ-100) and any site-local customisation.
|
||||||
|
#
|
||||||
|
# Dynamic config (routers, services, certs) is mounted from
|
||||||
|
# /etc/traefik/dynamic on the host — orca writes to it atomically via
|
||||||
|
# the SSH-push transport (C-10 protocol).
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build -f Dockerfile.traefik -t git.cloudinit.dev/coreci/orca-traefik:v0.13.1 .
|
||||||
|
#
|
||||||
|
# Run (hybrid R-017 mode — nft DNATs :443/:80 to loopback):
|
||||||
|
# podman run -d --name orca-traefik --restart=unless-stopped \
|
||||||
|
# --network host \
|
||||||
|
# -v /etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro \
|
||||||
|
# -v /etc/traefik/dynamic:/etc/traefik/dynamic:ro \
|
||||||
|
# -v /etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro \
|
||||||
|
# git.cloudinit.dev/coreci/orca-traefik:v0.13.1
|
||||||
|
|
||||||
|
FROM traefik:v3.3.0
|
||||||
|
|
||||||
|
LABEL org.opencontainers.image.title="orca-traefik"
|
||||||
|
LABEL org.opencontainers.image.description="Custom Traefik image for Orca ingress (R-024)"
|
||||||
|
LABEL org.opencontainers.image.source="https://git.cloudinit.dev/coreci/orca"
|
||||||
|
|
||||||
|
COPY docker/orca-traefik/traefik.yml /etc/traefik/traefik.yml
|
||||||
|
COPY docker/orca-traefik/step-ca-root.crt /etc/orca/step-ca-root.crt
|
||||||
|
|
||||||
|
CMD ["--configFile=/etc/traefik/traefik.yml"]
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: build test test-race lint fmt clean run release version changelog help security-scan verify-reqs
|
.PHONY: build test test-race lint fmt clean run release version changelog help security-scan verify-reqs verify-docs
|
||||||
|
|
||||||
BINARY := bin/orca
|
BINARY := bin/orca
|
||||||
GOFLAGS := -trimpath
|
GOFLAGS := -trimpath
|
||||||
@@ -31,6 +31,7 @@ help:
|
|||||||
@echo " release Run scripts/release.sh [VERSION] — build, tar, publish"
|
@echo " release Run scripts/release.sh [VERSION] — build, tar, publish"
|
||||||
@echo " security-scan Run gosec+govulncheck+gitleaks (P03, REQ-014/027/039)"
|
@echo " security-scan Run gosec+govulncheck+gitleaks (P03, REQ-014/027/039)"
|
||||||
@echo " verify-reqs Assert ROADMAP COMPLETE ↔ REQUIREMENTS Complete (REQ-060)"
|
@echo " verify-reqs Assert ROADMAP COMPLETE ↔ REQUIREMENTS Complete (REQ-060)"
|
||||||
|
@echo " verify-docs Assert docs/cli.md ↔ orca --help consistency (REQ-160)"
|
||||||
|
|
||||||
build:
|
build:
|
||||||
@mkdir -p bin
|
@mkdir -p bin
|
||||||
@@ -128,3 +129,9 @@ security-scan:
|
|||||||
# of scope (P04 audit). Exits 0 on consistency, 1 with a diff on drift.
|
# of scope (P04 audit). Exits 0 on consistency, 1 with a diff on drift.
|
||||||
verify-reqs:
|
verify-reqs:
|
||||||
go run ./cmd/verify-reqs .ciagent/ROADMAP.md .ciagent/REQUIREMENTS.md
|
go run ./cmd/verify-reqs .ciagent/ROADMAP.md .ciagent/REQUIREMENTS.md
|
||||||
|
|
||||||
|
# verify-docs asserts that every top-level subcommand in docs/cli.md
|
||||||
|
# exists in `orca --help` output (and vice versa). Catches doc drift
|
||||||
|
# (REQ-160). Requires the binary to be built first (`make build`).
|
||||||
|
verify-docs: build
|
||||||
|
./scripts/verify-docs.sh ./bin/orca docs/cli.md
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ identity.
|
|||||||
|
|
||||||
## Status
|
## Status
|
||||||
|
|
||||||
**v0.11: Production Hardening — IN PROGRESS** | **v1.0: UAT-gated** (cut
|
**v0.12: Security Hardening (Zero-Trust Identity) — COMPLETE** | **v0.13: Production Hardening Round 2 + UAT Plan — IN PROGRESS** | **v1.0: UAT-gated** (cut
|
||||||
separately after v0.11 completion per operator decision)
|
separately after v0.13 completion per operator decision)
|
||||||
|
|
||||||
See [.ciagent/ROADMAP.md](.ciagent/ROADMAP.md) for the full roadmap.
|
See [.ciagent/ROADMAP.md](.ciagent/ROADMAP.md) for the full roadmap.
|
||||||
|
|
||||||
@@ -18,8 +18,8 @@ See [.ciagent/ROADMAP.md](.ciagent/ROADMAP.md) for the full roadmap.
|
|||||||
- **Offline-first** — no cloud dependencies; the cluster is the OS
|
- **Offline-first** — no cloud dependencies; the cluster is the OS
|
||||||
- **CLI-first** — the command line is the primary interface (humans and
|
- **CLI-first** — the command line is the primary interface (humans and
|
||||||
AI agents)
|
AI agents)
|
||||||
- **Security before features** — mTLS by default; NFRs ship before new
|
- **Security before features** — SSH-push is the canonical transport
|
||||||
functionality
|
(mTLS available for daemon mode); NFRs ship before new functionality
|
||||||
- **WASM-first** — workloads target OS primitives (systemd units,
|
- **WASM-first** — workloads target OS primitives (systemd units,
|
||||||
journald), not a container runtime shim
|
journald), not a container runtime shim
|
||||||
- **Bug fixes before features** — stability is paramount
|
- **Bug fixes before features** — stability is paramount
|
||||||
@@ -35,8 +35,8 @@ curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | b
|
|||||||
# System-level install (binary at /usr/local/bin/orca, state at /root/.orca)
|
# System-level install (binary at /usr/local/bin/orca, state at /root/.orca)
|
||||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | sudo bash -s -- --system
|
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | sudo bash -s -- --system
|
||||||
|
|
||||||
# Pin a specific version (latest tag: v0.10.19)
|
# Pin a specific version (latest tag: v0.12.10)
|
||||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --version v0.10.19
|
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --version v0.12.10
|
||||||
|
|
||||||
# Dry-run: check what would be installed without writing
|
# Dry-run: check what would be installed without writing
|
||||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --check
|
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --check
|
||||||
@@ -65,7 +65,7 @@ config, database, and certificates in the namespace dir:
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash
|
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash
|
||||||
# → "updated orca from v0.8.15 to v0.10.19"
|
# → "updated orca from v0.11.28 to v0.12.10"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Subcommands
|
## Subcommands
|
||||||
@@ -73,7 +73,7 @@ curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | b
|
|||||||
| Command | Description |
|
| Command | Description |
|
||||||
|---------|-------------|
|
|---------|-------------|
|
||||||
| `orca init` | Initialize local orca state with full bootstrap |
|
| `orca init` | Initialize local orca state with full bootstrap |
|
||||||
| `orca status` | Show orca daemon status |
|
| `orca status` | **(deprecated v0.1 stub)** Show orca daemon status — use `orca node list` + `orca metrics /healthz` |
|
||||||
| `orca version` | Print version information |
|
| `orca version` | Print version information |
|
||||||
| `orca daemon` | **(deprecated)** Run the orca daemon (HTTP API + health checks) |
|
| `orca daemon` | **(deprecated)** Run the orca daemon (HTTP API + health checks) |
|
||||||
| `orca metrics` | Start metrics endpoint (Prometheus text exposition) |
|
| `orca metrics` | Start metrics endpoint (Prometheus text exposition) |
|
||||||
@@ -85,15 +85,18 @@ curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | b
|
|||||||
| `orca job` | Manage orca jobs: `run`, `list`, `stop`, `logs`, `lint`, `verify`, `migrate`, `restart` |
|
| `orca job` | Manage orca jobs: `run`, `list`, `stop`, `logs`, `lint`, `verify`, `migrate`, `restart` |
|
||||||
| `orca ns` | Manage orca namespaces: `list`, `create`, `delete`, `inspect`, `validate`, `inherit`, `set-constraint` |
|
| `orca ns` | Manage orca namespaces: `list`, `create`, `delete`, `inspect`, `validate`, `inherit`, `set-constraint` |
|
||||||
| `orca cert` | **(deprecated)** Manage orca certificates: `ca-init`, `gen`, `show`, `renew`, `fingerprint` |
|
| `orca cert` | **(deprecated)** Manage orca certificates: `ca-init`, `gen`, `show`, `renew`, `fingerprint` |
|
||||||
| `orca doctor` | Run self-checks: `cert`, `network`, `db`, `os`, `proxmox`, `no-orca-on-server` |
|
| `orca doctor` | Run self-checks: `cert`, `network`, `db`, `os`, `proxmox`, `no-orca-on-server`, `nft`, `audit`, `modes`, `oidc`, `db-retention` |
|
||||||
| `orca audit` | View orca audit log (`list`) |
|
| `orca audit` | View orca audit log (`list`) |
|
||||||
| `orca cache` | CLI cache management: `show`, `invalidate`, `invalidate-all` |
|
| `orca cache` | CLI cache management: `show`, `invalidate`, `invalidate-all` |
|
||||||
| `orca acl` | ACL management: `grant`, `revoke`, `list`, `check` |
|
| `orca acl` | ACL management: `grant`, `revoke`, `list`, `check` |
|
||||||
| `orca secrets` | Secrets management: `set`, `get`, `list`, `rotate`, `delete` |
|
| `orca secrets` | Secrets management: `set`, `get`, `list`, `rotate`, `delete`, `rotate-master` |
|
||||||
| `orca drift` | Drift detection: `show`, `watch`, `acknowledge`, `remediate`, `config` |
|
| `orca drift` | Drift detection: `show`, `watch`, `acknowledge`, `remediate`, `config` |
|
||||||
| `orca txn` | Transaction management: `apply`, `list`, `show`, `rollback` |
|
| `orca txn` | Transaction management: `apply`, `list`, `show`, `rollback` |
|
||||||
|
| `orca nft` | nftables ingress management: `show`, `diff`, `doctor`, `country block`, `rate limit` |
|
||||||
| `orca collector` | Collector/aggregator management: `start`, `stop`, `status` |
|
| `orca collector` | Collector/aggregator management: `start`, `stop`, `status` |
|
||||||
| `orca cluster` | Cluster management: `cutover`, `rotate-lead`, `compat-check` |
|
| `orca cluster` | Cluster management: `cutover`, `rotate-lead`, `compat-check`, `seal`, `unseal` |
|
||||||
|
| `orca auth` | OIDC authentication: `login`, `logout`, `status`, `init-idp`, `register` |
|
||||||
|
| `orca peer-setup` | Create the orca system user + drift-events dir on a peer (REQ-111) |
|
||||||
|
|
||||||
See [docs/cli.md](docs/cli.md) for the full CLI reference with all flags
|
See [docs/cli.md](docs/cli.md) for the full CLI reference with all flags
|
||||||
and examples.
|
and examples.
|
||||||
@@ -114,7 +117,7 @@ acknowledged rather than papered over.
|
|||||||
| Auto-scaling | Cluster autoscaler, HPA/VPA, deep integrations | — |
|
| Auto-scaling | Cluster autoscaler, HPA/VPA, deep integrations | — |
|
||||||
| Daemon footprint | — | No daemon on the critical path; the cluster is the OS |
|
| Daemon footprint | — | No daemon on the critical path; the cluster is the OS |
|
||||||
| OS-native | — | Workloads are systemd units + journald; no container runtime shim |
|
| OS-native | — | Workloads are systemd units + journald; no container runtime shim |
|
||||||
| mTLS | — | mTLS by default; no opt-in required |
|
| Transport | — | SSH-push is canonical (no daemon needed); mTLS available for daemon mode |
|
||||||
| Offline-first | — | No cloud dependencies; fully air-gapped operation |
|
| Offline-first | — | No cloud dependencies; fully air-gapped operation |
|
||||||
| WASM-first | — | Workloads target OS primitives, not a container runtime |
|
| WASM-first | — | Workloads target OS primitives, not a container runtime |
|
||||||
| Proxmox | — | First-class Proxmox node type (`--type proxmox`) via SSH-push |
|
| Proxmox | — | First-class Proxmox node type (`--type proxmox`) via SSH-push |
|
||||||
@@ -129,6 +132,10 @@ acknowledged rather than papered over.
|
|||||||
| [docs/namespace.md](docs/namespace.md) | Namespace and path layout |
|
| [docs/namespace.md](docs/namespace.md) | Namespace and path layout |
|
||||||
| [docs/install.md](docs/install.md) | Installation guide |
|
| [docs/install.md](docs/install.md) | Installation guide |
|
||||||
| [docs/security-scanning.md](docs/security-scanning.md) | Security scanning tools |
|
| [docs/security-scanning.md](docs/security-scanning.md) | Security scanning tools |
|
||||||
|
| [docs/security-runbook.md](docs/security-runbook.md) | Security runbook — seal/unseal, rotation, incident response |
|
||||||
|
| [docs/webauthn.md](docs/webauthn.md) | WebAuthn / passkeys registration and login |
|
||||||
|
| [docs/threat-model.md](docs/threat-model.md) | STRIDE threat model + zero-trust architecture |
|
||||||
|
| [docs/oidc.md](docs/oidc.md) | OIDC configuration — Dex quickstart, BYO IdP |
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
@@ -144,6 +151,7 @@ make test # Run tests
|
|||||||
go vet ./... # Vet all packages
|
go vet ./... # Vet all packages
|
||||||
make lint # Run gofmt + go vet + shellcheck
|
make lint # Run gofmt + go vet + shellcheck
|
||||||
make verify-reqs # Assert ROADMAP ↔ REQUIREMENTS consistency
|
make verify-reqs # Assert ROADMAP ↔ REQUIREMENTS consistency
|
||||||
|
make verify-docs # Assert docs/cli.md ↔ `orca --help` consistency
|
||||||
```
|
```
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
15ee8f02ef938496ce9baae35e2971a5fcfb2d55b2c6e35e49c11285f67aeb53 orca-v0.12.18-linux-amd64.tar.gz
|
||||||
+24
-10
@@ -16,17 +16,20 @@ import (
|
|||||||
// the Phase + Status match at the END of the line, where those two columns
|
// the Phase + Status match at the END of the line, where those two columns
|
||||||
// always live. The status token is optionally wrapped in markdown bold
|
// always live. The status token is optionally wrapped in markdown bold
|
||||||
// (real rows use `**Complete**`; synthetic/future rows may use bare
|
// (real rows use `**Complete**`; synthetic/future rows may use bare
|
||||||
// `Pending`), and may carry trailing notes (e.g. "**Complete** (P01
|
// `pending` or `complete` in any case), and may carry trailing notes
|
||||||
// shipped v0.2.1)") matched by [^|]* before the closing pipe.
|
// (e.g. "**Complete** (P01 shipped v0.2.1)") matched by [^|]* before
|
||||||
var reqRowRe = regexp.MustCompile(`^\|\s*(REQ-\d+)\s*\|.*\|\s*([^|]*?)\s*\|\s*\*{0,2}(Complete|Pending)\*{0,2}[^|]*\|\s*$`)
|
// the closing pipe. The (?i) flag makes the match case-insensitive so
|
||||||
|
// lowercase `pending` (used by v0.12/v0.13 REQ rows) is captured;
|
||||||
|
// normalizeStatus canonicalizes the captured value to title case.
|
||||||
|
var reqRowRe = regexp.MustCompile(`(?i)^\|\s*(REQ-\d+)\s*\|.*\|\s*([^|]*?)\s*\|\s*\*{0,2}(Complete|Pending)\*{0,2}[^|]*\|\s*$`)
|
||||||
|
|
||||||
// milestoneCompleteRe matches a ROADMAP.md milestone header that is marked
|
// milestoneCompleteRe matches a ROADMAP.md milestone header that is marked
|
||||||
// COMPLETE. The bold span is substring-tolerant (GRILL #4): it matches
|
// COMPLETE. The bold markers are optional (GRILL #4 + REQ-160 T11): it
|
||||||
// `**COMPLETE**`, `**COMPLETE (merged to main via v0.3)**`, and any future
|
// matches `**COMPLETE**`, `**COMPLETE (merged to main via v0.3)**`, and
|
||||||
// variant where the word COMPLETE appears inside the bold span, possibly
|
// bare `COMPLETE` (as used by the v0.12 milestone header). The word
|
||||||
// preceded or followed by non-asterisk text. The milestone version (v0.X)
|
// COMPLETE may be preceded or followed by non-asterisk text. The milestone
|
||||||
// is captured.
|
// version (v0.X) is captured.
|
||||||
var milestoneCompleteRe = regexp.MustCompile(`^##\s*Milestone\s+(v0\.\d+):.*—\s*\*\*[^*]*\bCOMPLETE\b[^*]*\*\*`)
|
var milestoneCompleteRe = regexp.MustCompile(`^##\s*Milestone\s+(v0\.\d+):.*—\s*\*{0,2}[^*]*\bCOMPLETE\b[^*]*\*{0,2}`)
|
||||||
|
|
||||||
// phaseRe extracts the milestone version from a REQUIREMENTS Phase cell such
|
// phaseRe extracts the milestone version from a REQUIREMENTS Phase cell such
|
||||||
// as `v0.7 P1`, `**v0.2 P01**`, `v0.2 P01–P04`, or bare `v0.7`. The cell may
|
// as `v0.7 P1`, `**v0.2 P01**`, `v0.2 P01–P04`, or bare `v0.7`. The cell may
|
||||||
@@ -40,6 +43,17 @@ type reqRow struct {
|
|||||||
status string // "Complete" or "Pending"
|
status string // "Complete" or "Pending"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// normalizeStatus canonicalizes a captured status token to the title-case
|
||||||
|
// form ("Complete" or "Pending") so that case-insensitive matches like
|
||||||
|
// "pending" or "complete" compare correctly against the drift assertions.
|
||||||
|
func normalizeStatus(s string) string {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
return strings.ToUpper(s[:1]) + strings.ToLower(s[1:])
|
||||||
|
}
|
||||||
|
|
||||||
// milestoneVersions returns the distinct v0.X milestones referenced in the
|
// milestoneVersions returns the distinct v0.X milestones referenced in the
|
||||||
// phase cell (e.g. "v0.7 P1" → ["v0.7"]; "v0.2 P01 / v0.3 P02" →
|
// phase cell (e.g. "v0.7 P1" → ["v0.7"]; "v0.2 P01 / v0.3 P02" →
|
||||||
// ["v0.2","v0.3"]).
|
// ["v0.2","v0.3"]).
|
||||||
@@ -174,7 +188,7 @@ func parseRequirements(path string) ([]reqRow, error) {
|
|||||||
if m == nil {
|
if m == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
rows = append(rows, reqRow{id: m[1], phase: strings.TrimSpace(m[2]), status: m[3]})
|
rows = append(rows, reqRow{id: m[1], phase: strings.TrimSpace(m[2]), status: normalizeStatus(m[3])})
|
||||||
}
|
}
|
||||||
if err := sc.Err(); err != nil {
|
if err := sc.Err(); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
entryPoints:
|
||||||
|
websecure:
|
||||||
|
address: "127.0.0.1:8443"
|
||||||
|
web:
|
||||||
|
address: "127.0.0.1:8080"
|
||||||
|
traefik:
|
||||||
|
address: "127.0.0.1:8081"
|
||||||
|
|
||||||
|
providers:
|
||||||
|
file:
|
||||||
|
directory: "/etc/traefik/dynamic"
|
||||||
|
watch: true
|
||||||
|
|
||||||
|
log:
|
||||||
|
level: INFO
|
||||||
|
format: json
|
||||||
|
|
||||||
|
accessLog:
|
||||||
|
format: json
|
||||||
+1068
-125
File diff suppressed because it is too large
Load Diff
+71
-7
@@ -138,13 +138,77 @@ restore traffic).
|
|||||||
|
|
||||||
## TLS
|
## TLS
|
||||||
|
|
||||||
- **certResolver**: `orca` (references the Traefik ACME/step-ca
|
- **v0.14 model**: `tls: {}` in dynamic config (no certResolver).
|
||||||
certificate resolver configured in Traefik's static config).
|
Traefik v3.3 `certificatesResolvers` only supports `acme` and
|
||||||
- **Trust domain**: `cluster.orca.local` (placeholder in v0.9; step-ca
|
`tailscale` — not CA-file-based. The `certResolver: orca` reference
|
||||||
provisioner in v0.11 overrides with the real cluster trust domain).
|
from v0.11 was broken (research finding). v0.14 emits `tls: {}`
|
||||||
- **SPIFFE SVIDs**: workload identity via SPIFFE SVIDs minted at submit
|
(traefik uses its default self-signed cert). Real mTLS via dynamic
|
||||||
time via step-ca (v0.11-P01.5, gate C-08). The SVID is a URI SAN in
|
`tls.certificates` + `tls.options.default.clientAuth.caFiles` is
|
||||||
the workload's X.509 cert.
|
deferred to v0.15.
|
||||||
|
- **Step-ca root CA**: mounted at `/etc/orca/step-ca-root.crt` in the
|
||||||
|
traefik container. v0.14 does not use it for TLS termination (it's
|
||||||
|
a placeholder for v0.15 mTLS).
|
||||||
|
|
||||||
|
## R-024: Podman Traefik Container (v0.14)
|
||||||
|
|
||||||
|
As of v0.14, Traefik runs as a **podman container** from the custom
|
||||||
|
`orca-traefik` image (published per release). The v0.13 binary+systemd
|
||||||
|
install is replaced.
|
||||||
|
|
||||||
|
### Three topologies
|
||||||
|
|
||||||
|
1. **Linux**: host → nft DNAT → `podman run orca-traefik` (`--network host`)
|
||||||
|
2. **Proxmox Native** (`--ingress-mode native`, default): PVE host →
|
||||||
|
nft DNAT → LXC (nesting=1,keyctl=1,fuse=1) → `podman run orca-traefik`
|
||||||
|
3. **Proxmox Floating-IP** (`--ingress-mode floating-ip`): LXC owns
|
||||||
|
the floating IP → nft inside LXC → `podman run orca-traefik`
|
||||||
|
|
||||||
|
### Container configuration
|
||||||
|
|
||||||
|
```bash
|
||||||
|
podman run -d --name orca-traefik --restart=unless-stopped \
|
||||||
|
--network host \
|
||||||
|
-v /etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro \
|
||||||
|
-v /etc/traefik/dynamic:/etc/traefik/dynamic:ro \
|
||||||
|
-v /etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro \
|
||||||
|
git.cloudinit.dev/coreci/orca-traefik:<version>
|
||||||
|
```
|
||||||
|
|
||||||
|
- `--network host`: traefik binds 127.0.0.1:8080/8443 on host/LXC loopback
|
||||||
|
- `--restart=unless-stopped`: survives reboot via `podman-restart.service`
|
||||||
|
- No `:Z` SELinux flag (research Topic 7)
|
||||||
|
- Static config mounted `:ro` (overrides baked image default, preserves
|
||||||
|
`traefik-on-public-ip` opt-out, REQ-100)
|
||||||
|
|
||||||
|
### nft ruleset
|
||||||
|
|
||||||
|
The nft emitter (`internal/emitter/nft.go`) renders `/etc/nftables.d/orca.nft`:
|
||||||
|
|
||||||
|
- DNAT `:443` → `<DNATTarget>:8443` (default 127.0.0.1; LXC IP for native)
|
||||||
|
- DNAT `:80` → `<DNATTarget>:8080`
|
||||||
|
- SNAT/MASQUERADE: `ip saddr 127.0.0.0/8 oifname != "lo" masquerade`
|
||||||
|
- Input/forward chains at priority -10 (pve-firewall coexistence)
|
||||||
|
|
||||||
|
### `orca doctor ingress`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca doctor ingress # check localhost
|
||||||
|
orca doctor ingress --peer <name> # check remote peer
|
||||||
|
```
|
||||||
|
|
||||||
|
Verifies: podman container running, nft DNAT+SNAT, dynamic dir exists,
|
||||||
|
step-ca root CA present.
|
||||||
|
|
||||||
|
### Dockerfile.traefik
|
||||||
|
|
||||||
|
```dockerfile
|
||||||
|
FROM traefik:v3.3.0
|
||||||
|
COPY docker/orca-traefik/traefik.yml /etc/traefik/traefik.yml
|
||||||
|
CMD ["--configFile=/etc/traefik/traefik.yml"]
|
||||||
|
```
|
||||||
|
|
||||||
|
Built + published per release alongside the orca image
|
||||||
|
(`scripts/release.sh` + `.coreci.yml container-publish-traefik`).
|
||||||
|
|
||||||
## Health checks
|
## Health checks
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Orca Metrics Reference
|
||||||
|
|
||||||
|
Orca exposes Prometheus text-exposition metrics at `/metrics` on the
|
||||||
|
metrics endpoint (default `:9100`, configurable via `--addr`).
|
||||||
|
|
||||||
|
## Running the metrics endpoint
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca metrics --addr :9100
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prometheus scrape config
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: orca
|
||||||
|
static_configs:
|
||||||
|
- targets: ['localhost:9100']
|
||||||
|
scrape_interval: 15s
|
||||||
|
```
|
||||||
|
|
||||||
|
## Metric reference
|
||||||
|
|
||||||
|
| Metric | Type | Description |
|
||||||
|
|--------|------|-------------|
|
||||||
|
| `nodes_total` | Gauge | Total number of registered nodes |
|
||||||
|
| `allocs_total` | Gauge | Total number of job allocations |
|
||||||
|
| `orca_jobs_by_state{state}` | Gauge | Jobs grouped by status (running, complete, failed, etc.) |
|
||||||
|
| `orca_audit_chain_head` | Gauge | Audit chain integrity (1 = chain head verified, 0 = error) |
|
||||||
|
|
||||||
|
## Counter metrics (incremented by CLI operations)
|
||||||
|
|
||||||
|
The following counters are incremented during normal operations and
|
||||||
|
are available when the metrics endpoint polls the DB:
|
||||||
|
|
||||||
|
| Metric | Type | Description |
|
||||||
|
|--------|------|-------------|
|
||||||
|
| `orca_drift_events_total` | Counter | Total drift events detected |
|
||||||
|
| `orca_ssh_errors_total` | Counter | Total SSH connection/exec errors |
|
||||||
|
| `orca_txn_apply_total` | Counter | Total transaction applies |
|
||||||
|
| `orca_txn_rollback_total` | Counter | Total transaction rollbacks |
|
||||||
|
| `orca_acl_denials_total` | Counter | Total ACL denials (enforce mode) |
|
||||||
|
|
||||||
|
## Security headers
|
||||||
|
|
||||||
|
The metrics endpoint sets the following security headers on all responses:
|
||||||
|
- `X-Content-Type-Options: nosniff`
|
||||||
|
- `X-Frame-Options: DENY`
|
||||||
|
|
||||||
|
## Health check
|
||||||
|
|
||||||
|
The endpoint also exposes `/healthz` returning `200 ok` for liveness probes.
|
||||||
+61
-12
@@ -8,8 +8,8 @@ directory holds cluster-wide artifacts shared across namespaces.
|
|||||||
|
|
||||||
> **v0.9 layout (canonical)**: This document describes the v0.9
|
> **v0.9 layout (canonical)**: This document describes the v0.9
|
||||||
> multi-namespace layout. The v0.8 flat layout (`orca.db`, `ca.crt`,
|
> multi-namespace layout. The v0.8 flat layout (`orca.db`, `ca.crt`,
|
||||||
> `server.crt` at the root) is deprecated and will be removed in
|
> `server.crt` at the root) is deprecated and removed in v0.12
|
||||||
> v0.11. See [v0.8 flat layout](#deprecated-v08-flat-layout) below.
|
> (REQ-138).
|
||||||
|
|
||||||
## Namespace root resolution
|
## Namespace root resolution
|
||||||
|
|
||||||
@@ -51,12 +51,16 @@ $ORCA_HOME/
|
|||||||
├── cluster/ # cluster-wide (NOT a workload namespace)
|
├── cluster/ # cluster-wide (NOT a workload namespace)
|
||||||
│ ├── ca.crt, ca.key # step-ca root (R-006, D-101)
|
│ ├── ca.crt, ca.key # step-ca root (R-006, D-101)
|
||||||
│ ├── master.key # AES-256-GCM root (R-011, mode 0600)
|
│ ├── master.key # AES-256-GCM root (R-011, mode 0600)
|
||||||
|
│ ├── master.key.sealed # sealed master key (REQ-147, mode 0600)
|
||||||
│ ├── config.md # Markdown frontmatter config (R-014)
|
│ ├── config.md # Markdown frontmatter config (R-014)
|
||||||
│ ├── known_hosts # SSH known_hosts (D-035)
|
│ ├── known_hosts # SSH known_hosts (D-035)
|
||||||
│ ├── orca_ssh_key # orca SSH private key (D-037)
|
│ ├── orca_ssh_key # orca SSH private key (D-037)
|
||||||
│ ├── orca_ssh_key.pub # orca SSH public key
|
│ ├── orca_ssh_key.pub # orca SSH public key
|
||||||
│ ├── peers/<host>/ # per-peer directory
|
│ ├── peers/<host>/ # per-peer directory
|
||||||
│ ├── txns/ # cluster transaction log (R-016)
|
│ ├── txns/ # cluster transaction log (R-016)
|
||||||
|
│ ├── acl.json # ACL state (mode 0600)
|
||||||
|
│ ├── oidc-client-secret # OIDC client secret (mode 0600, C-36)
|
||||||
|
│ ├── webauthn-credentials.db # WebAuthn public keys (mode 0600)
|
||||||
│ └── state/ # cluster state
|
│ └── state/ # cluster state
|
||||||
├── _defaults/ # implicit root namespace (always exists)
|
├── _defaults/ # implicit root namespace (always exists)
|
||||||
│ ├── ns.md # namespace frontmatter (kind: Namespace)
|
│ ├── ns.md # namespace frontmatter (kind: Namespace)
|
||||||
@@ -80,14 +84,16 @@ $ORCA_HOME/
|
|||||||
exists. Every namespace inherits from `_defaults` and cannot opt out
|
exists. Every namespace inherits from `_defaults` and cannot opt out
|
||||||
(D-185, D-187).
|
(D-185, D-187).
|
||||||
- **`cluster/`** is NOT a workload namespace — it holds cluster-wide
|
- **`cluster/`** is NOT a workload namespace — it holds cluster-wide
|
||||||
artifacts (CA, master key, SSH keys, known_hosts, peers, txns).
|
artifacts (CA, master key, SSH keys, known_hosts, peers, txns, ACL,
|
||||||
|
OIDC secrets, WebAuthn credentials).
|
||||||
- **Per-namespace DBs**: each namespace has its own
|
- **Per-namespace DBs**: each namespace has its own
|
||||||
`db/orca.db` (R-002). No namespace column in SQLite.
|
`db/orca.db` (R-002). No namespace column in SQLite.
|
||||||
- **Namespace inheritance**: child namespaces inherit env and
|
- **Namespace inheritance**: child namespaces inherit env and
|
||||||
constraints from parents (via `ns.md` frontmatter `parents:` field).
|
constraints from parents (via `ns.md` frontmatter `parents:` field).
|
||||||
`_defaults` is always appended last in the inheritance chain.
|
`_defaults` is always appended last in the inheritance chain.
|
||||||
- **`orca ns` subcommands**: `list`, `create`, `delete`, `inspect`,
|
- **`orca ns` subcommands**: `list`, `create`, `delete`, `inspect`,
|
||||||
`validate` — see [docs/cli.md](cli.md#orca-ns).
|
`validate`, `inherit`, `set-constraint` — see below and
|
||||||
|
[docs/cli.md](cli.md#orca-ns).
|
||||||
|
|
||||||
### Path reference (`internal/paths/`)
|
### Path reference (`internal/paths/`)
|
||||||
|
|
||||||
@@ -134,6 +140,50 @@ orca ns validate prod
|
|||||||
orca ns delete staging
|
orca ns delete staging
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `orca ns inherit` — set parent namespace (R-002)
|
||||||
|
|
||||||
|
Set the parent namespace for a namespace. Updates `ns.md` frontmatter
|
||||||
|
(`parents` field) and validates the new chain has no cycles. The
|
||||||
|
implicit root `_defaults` is always appended last (D-185).
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca ns inherit <name> --parent <parent-namespace>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example**:
|
||||||
|
```bash
|
||||||
|
# Make staging inherit from prod (chain: staging -> prod -> _defaults)
|
||||||
|
orca ns inherit staging --parent prod
|
||||||
|
```
|
||||||
|
|
||||||
|
The child cannot inherit from itself transitively — the resolver
|
||||||
|
validates the chain before writing. If a cycle is detected, the
|
||||||
|
command exits 1 with an error.
|
||||||
|
|
||||||
|
### `orca ns set-constraint` — set a constraint (R-002)
|
||||||
|
|
||||||
|
Set a constraint on a namespace. Constraints are `key=value` strings
|
||||||
|
(e.g., `max-allocs=10`) stored in `ns.md` frontmatter and unioned
|
||||||
|
across the inheritance chain by the resolver.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca ns set-constraint <name> <key>=<value>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Example**:
|
||||||
|
```bash
|
||||||
|
# Limit prod to 10 concurrent allocations
|
||||||
|
orca ns set-constraint prod max-allocs=10
|
||||||
|
|
||||||
|
# Set a required node affinity
|
||||||
|
orca ns set-constraint prod require-label=ssd
|
||||||
|
```
|
||||||
|
|
||||||
|
Constraints are unioned (not overridden) across the inheritance chain:
|
||||||
|
if `_defaults` sets `max-allocs=50` and `prod` sets `max-allocs=10`,
|
||||||
|
the effective constraint is the most restrictive one (CEL evaluation
|
||||||
|
determines precedence per constraint key).
|
||||||
|
|
||||||
See [docs/cli.md](cli.md#orca-ns) for the full `orca ns` reference.
|
See [docs/cli.md](cli.md#orca-ns) for the full `orca ns` reference.
|
||||||
|
|
||||||
## `ORCA_DB` override
|
## `ORCA_DB` override
|
||||||
@@ -148,11 +198,10 @@ orca init # uses /tmp/test.db for the DB, ~/.orca/ for everything else
|
|||||||
|
|
||||||
## Deprecated: v0.8 flat layout
|
## Deprecated: v0.8 flat layout
|
||||||
|
|
||||||
> **Deprecated in v0.9**: The v0.8 flat layout (`orca.db`, `ca.crt`,
|
> **Removed in v0.12** (REQ-138): The v0.8 flat layout (`orca.db`,
|
||||||
> `ca.key`, `server.crt`, `server.key` at the namespace root) is
|
> `ca.crt`, `ca.key`, `server.crt`, `server.key` at the namespace root)
|
||||||
> superseded by the v0.9 multi-namespace layout (R-002). The v0.8
|
> is superseded by the v0.9 multi-namespace layout (R-002) and the
|
||||||
> layout is supported during the dual-write window via
|
> dual-write window is closed.
|
||||||
> `internal/certpaths` (a thin shim) and will be removed in v0.11.
|
|
||||||
|
|
||||||
The v0.8 flat layout stored all state at the namespace root:
|
The v0.8 flat layout stored all state at the namespace root:
|
||||||
|
|
||||||
@@ -166,12 +215,12 @@ The v0.8 flat layout stored all state at the namespace root:
|
|||||||
|
|
||||||
The v0.9 re-architecture moved these to `cluster/` (CA, SSH keys) and
|
The v0.9 re-architecture moved these to `cluster/` (CA, SSH keys) and
|
||||||
per-namespace `db/` (SQLite) to support multi-tenancy (R-002). The
|
per-namespace `db/` (SQLite) to support multi-tenancy (R-002). The
|
||||||
`orca doctor --legacy-paths` command (v0.11-P14c) will detect v0.8
|
`internal/certpaths` shim that supported the dual-write window is
|
||||||
residue and recommend migration.
|
removed in v0.12.
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
||||||
- [Install Guide](install.md) — 1-liner install with `install.sh`.
|
- [Install Guide](install.md) — 1-liner install with `install.sh`.
|
||||||
- [Docker Guide](docker.md) — running orca in a container.
|
- [Docker Guide](docker.md) — running orca in a container.
|
||||||
- [CLI Reference](cli.md) — `orca ns` subcommands.
|
- [CLI Reference](cli.md#orca-ns) — `orca ns` subcommands.
|
||||||
- [Jobspec Reference](jobspec.md) — markdown frontmatter schema.
|
- [Jobspec Reference](jobspec.md) — markdown frontmatter schema.
|
||||||
+138
-21
@@ -1,31 +1,148 @@
|
|||||||
# Security Runbook (v0.12)
|
# Security Runbook (v0.13)
|
||||||
|
|
||||||
## Master Key Seal/Unseal
|
This runbook documents the operational security procedures for orca's
|
||||||
|
zero-trust identity model (R-021): human identity is exclusively
|
||||||
|
external (OIDC), machine identity is exclusively mTLS/SPIFFE, and no
|
||||||
|
passwords / Orca-issued tokens / CA-key passphrases exist anywhere in
|
||||||
|
the system. The v0.12 milestone shipped these capabilities; the v0.13
|
||||||
|
milestone wired them operationally (R-023).
|
||||||
|
|
||||||
- `orca cluster seal`: encrypts master key with OIDC-derived key;
|
## Master Key Seal/Unseal (REQ-147, P05)
|
||||||
prints 5 Shamir shards for offline recovery.
|
|
||||||
- `orca cluster unseal`: operator authenticates via OIDC; master key
|
|
||||||
unwrapped into memory; zeroed on shutdown.
|
|
||||||
- `orca cluster unseal --recovery`: if IdP lost, present 3 of 5 shards.
|
|
||||||
|
|
||||||
## Master Key Rotation
|
The cluster master key (`ClusterDir()/master.key`, mode 0600) encrypts
|
||||||
|
all namespace `.env.secrets` via per-namespace HKDF-SHA256 sub-keys
|
||||||
|
(AES-256-GCM). The master key can be **sealed** (encrypted at rest) and
|
||||||
|
**unsealed** (unwrapped into memory for use).
|
||||||
|
|
||||||
`orca secrets rotate-master [--dry-run]`: generates new master key,
|
### Seal
|
||||||
re-encrypts all namespace secrets, re-seals. Atomic + automatic rollback.
|
|
||||||
|
```bash
|
||||||
|
orca cluster seal
|
||||||
|
```
|
||||||
|
|
||||||
|
Encrypts the raw master key with a key derived from either:
|
||||||
|
- the OIDC ID token subject (if `orca auth login` has been run), or
|
||||||
|
- the cluster CA fingerprint (mTLS-only offline path, D-241).
|
||||||
|
|
||||||
|
The sealed blob is written to `ClusterDir()/master.key.sealed` (0600).
|
||||||
|
**Five Shamir shards (3-of-5 recovery)** are printed to stdout — store
|
||||||
|
them offline. The raw master key is then deleted from disk so the
|
||||||
|
cluster is sealed at rest.
|
||||||
|
|
||||||
|
### Unseal
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca cluster unseal
|
||||||
|
```
|
||||||
|
|
||||||
|
Reads the sealed blob and unwraps the master key using the OIDC ID
|
||||||
|
token subject or the cluster CA fingerprint. The unwrapped key is
|
||||||
|
written back to `ClusterDir()/master.key` (0600) and zeroed from
|
||||||
|
memory on process exit.
|
||||||
|
|
||||||
|
### Recovery (IdP lost)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca cluster unseal --recovery
|
||||||
|
```
|
||||||
|
|
||||||
|
If the IdP is permanently lost, the operator is prompted for 3 of the
|
||||||
|
5 Shamir shards printed at seal time. With quorum, the master key is
|
||||||
|
reconstructed and written back to disk. If quorum is unavailable, the
|
||||||
|
cluster is unrecoverable by design (C-35: no backdoor).
|
||||||
|
|
||||||
|
## Master Key Rotation (REQ-129, C-30)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca secrets rotate-master [--dry-run]
|
||||||
|
```
|
||||||
|
|
||||||
|
Generates a new master key, re-encrypts every namespace's
|
||||||
|
`.env.secrets` under the new key, and re-seals the master key to OIDC.
|
||||||
|
With `--dry-run`, reports affected namespaces without writing.
|
||||||
|
|
||||||
|
- **Atomic per-namespace**: each namespace is re-encrypted independently.
|
||||||
|
- **Automatic rollback**: on any namespace failure, the old sealed key
|
||||||
|
is restored (C-30).
|
||||||
|
- **No passphrase** (R-021): the master key is sealed to OIDC, not to a
|
||||||
|
human-typed passphrase.
|
||||||
|
|
||||||
|
## File-Mode Audit (REQ-033, REQ-130, F13)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca doctor modes
|
||||||
|
```
|
||||||
|
|
||||||
|
Verifies file modes on security-sensitive files across `ORCA_HOME`:
|
||||||
|
- private keys / secrets: `0600`
|
||||||
|
- certs / public keys: `0644`
|
||||||
|
|
||||||
|
Exits 0 if all files have correct modes; exits 1 if any violation is
|
||||||
|
found. Missing files are not counted as violations.
|
||||||
|
|
||||||
|
Checks: SSH key, master key (sealed blob), server cert/key,
|
||||||
|
known_hosts, `acl.json`, OIDC client secret.
|
||||||
|
|
||||||
|
## Audit Log Tamper-Evidence (REQ-125, F2)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca doctor audit
|
||||||
|
```
|
||||||
|
|
||||||
|
Verifies the audit log hash chain. Opens the orca SQLite DB, recomputes
|
||||||
|
the hash chain from the first audit entry, and reports the chain head
|
||||||
|
hash. If any entry's `entry_hash` or `prev_hash` link does not match the
|
||||||
|
recomputed value, the chain has been tampered with and the command
|
||||||
|
exits non-zero.
|
||||||
|
|
||||||
|
The audit log is append-only (SQLite trigger blocks
|
||||||
|
UPDATE/DELETE). Each entry's `actor` field carries the OIDC `sub` or
|
||||||
|
SPIFFE SVID. Run this after any suspected intrusion or as part of a
|
||||||
|
regular audit cadence.
|
||||||
|
|
||||||
|
## Sudoers Audit (REQ-134, F22)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca doctor proxmox
|
||||||
|
```
|
||||||
|
|
||||||
|
Audits the `/etc/sudoers.d/orca` file against the expected allowlist:
|
||||||
|
- `pct` + `qm` with NOEXEC
|
||||||
|
- `apt-get` / `dpkg` excluded (or NOEXEC'd)
|
||||||
|
- `pvesh` EXCLUDED (AD-020: pvesh can bypass NOEXEC via the API execute
|
||||||
|
endpoint)
|
||||||
|
|
||||||
|
## nft Audit (REQ-133, F21)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca doctor nft
|
||||||
|
```
|
||||||
|
|
||||||
|
Audits the live nftables ingress ruleset against the on-disk
|
||||||
|
`/etc/nftables.d/orca.nft` hash (recorded at the latest applied txn).
|
||||||
|
Reports drift if the live ruleset does not match. Also verifies:
|
||||||
|
- table exists
|
||||||
|
- DNAT `:443 → 127.0.0.1:8443` and `:80 → 127.0.0.1:8080` present
|
||||||
|
- rate-limit meter present
|
||||||
|
- `/etc/nftables.d/orca.nft` parses
|
||||||
|
|
||||||
## Incident Response
|
## Incident Response
|
||||||
|
|
||||||
1. Revoke the compromised identity (OIDC user/group or SPIFFE SVID).
|
1. **Revoke the compromised identity** (OIDC user/group or SPIFFE SVID).
|
||||||
2. Rotate the master key (`orca secrets rotate-master`).
|
2. **Rotate the master key** (`orca secrets rotate-master`).
|
||||||
3. Review the audit log (`orca doctor audit` verifies the hash chain).
|
3. **Review the audit log** (`orca doctor audit` verifies the hash
|
||||||
4. If the master key is compromised, all historical secrets are
|
chain; `orca audit list` shows entries).
|
||||||
compromised (no forward secrecy).
|
4. **Check file modes** (`orca doctor modes` detects permission drift).
|
||||||
|
5. If the master key is compromised, **all historical secrets are
|
||||||
|
compromised** (no forward secrecy — documented residual risk).
|
||||||
|
6. **Re-seal** the master key after rotation (`orca cluster seal`).
|
||||||
|
|
||||||
## Sudoers Audit
|
## OIDC Provider Health (P06)
|
||||||
|
|
||||||
`orca doctor proxmox` audits the `/etc/sudoers.d/orca` file against the
|
```bash
|
||||||
expected allowlist (pct + qm with NOEXEC; apt-get/dpkg excluded).
|
orca doctor oidc
|
||||||
|
```
|
||||||
|
|
||||||
## nft Audit
|
Checks the bundled Dex OIDC provider health. Verifies the Dex systemd
|
||||||
|
unit is running and the `/.well-known/openid-configuration` endpoint
|
||||||
`orca doctor nft` audits the live nftables ruleset against the emitted one.
|
responds. Run after `orca auth init-idp` or after a Dex config change.
|
||||||
|
|||||||
+360
@@ -0,0 +1,360 @@
|
|||||||
|
# Orca User Acceptance Testing (UAT) Plan
|
||||||
|
|
||||||
|
**Version**: v0.13 (production hardening round 2)
|
||||||
|
**Gate**: v1.0.0 production-ready tag is deferred until this UAT passes
|
||||||
|
**Signoff**: run `scripts/uat-signoff.sh` on the lead node and paste the output back
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
### Hardware
|
||||||
|
|
||||||
|
| Role | OS | Requirements |
|
||||||
|
|------|-----|-------------|
|
||||||
|
| **lead** | Ubuntu 22.04 LTS | Operator laptop or VM; SSH key; `orca` binary (built from v0.13 tag) |
|
||||||
|
| **pve01** | Proxmox VE 8/9 | Bare-metal or nested; SSH root access; orca SSH key pre-staged |
|
||||||
|
| **worker01** | Ubuntu 22.04 LTS | VM or bare-metal; SSH root access; orca SSH key pre-staged |
|
||||||
|
|
||||||
|
### Alternative topology (3x Ubuntu, no Proxmox)
|
||||||
|
|
||||||
|
If a Proxmox host is unavailable, run the UAT with 3x Ubuntu hosts.
|
||||||
|
Use `--type linux` for all remote nodes. Proxmox-specific claims
|
||||||
|
(`doctor proxmox`, PVE role, sudoers) are **skipped** in this path.
|
||||||
|
The signoff script reports exercised vs. skipped claims.
|
||||||
|
|
||||||
|
## Step-by-step UAT
|
||||||
|
|
||||||
|
### Step 1: Install orca + initialize the cluster
|
||||||
|
|
||||||
|
Install orca (1-liner):
|
||||||
|
```sh
|
||||||
|
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/branch/main/scripts/install.sh | bash
|
||||||
|
```
|
||||||
|
|
||||||
|
Initialize the cluster:
|
||||||
|
```sh
|
||||||
|
export ORCA_HOME=~/orca-uat
|
||||||
|
orca init
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: orca init creates:
|
||||||
|
- CA cert + server cert
|
||||||
|
- SSH keypair (orca_ssh_key + orca_ssh_key.pub)
|
||||||
|
- known_hosts file (empty, for TOFU capture)
|
||||||
|
- Master key (for secrets encryption)
|
||||||
|
- Traefik data-plane ingress (binary + systemd unit + config)
|
||||||
|
- Localhost node registered
|
||||||
|
|
||||||
|
**Pre-staging remote nodes**: `orca init` interactively prompts for remote
|
||||||
|
host addresses and runs `ssh-copy-id` automatically (password prompt passes
|
||||||
|
through). Enter each host (pve01, worker01) when prompted, or press Enter to
|
||||||
|
skip. The orca public key is deployed to each host; TOFU host-key capture is
|
||||||
|
automatic on the first `orca node join` — no manual fingerprint pinning needed.
|
||||||
|
|
||||||
|
### Step 2: Onboard the Proxmox host
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca node join --type proxmox \
|
||||||
|
--host pve01 \
|
||||||
|
--ssh-user root
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: SSH bootstrap succeeds, orca user created, PVE role assigned, node registered as `ready` with `kind=proxmox`.
|
||||||
|
|
||||||
|
### Step 3: Onboard the Ubuntu worker
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca node join --type linux \
|
||||||
|
--host worker01 \
|
||||||
|
--ssh-user root
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: SSH bootstrap succeeds, orca user created, drift-events dir created, node registered as `ready` with `kind=linux`.
|
||||||
|
|
||||||
|
### Step 4: Verify nodes
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca node list
|
||||||
|
orca node list --json
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: 3 nodes listed (localhost + pve01 + worker01), all `ready`.
|
||||||
|
|
||||||
|
### Step 5: Set capacity on remote nodes
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca node capacity set --node pve01 --cpu 4 --memory 8192 --disk 100000
|
||||||
|
orca node capacity set --node worker01 --cpu 2 --memory 4096 --disk 50000
|
||||||
|
orca node capacity list
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: capacity shown for both remote nodes.
|
||||||
|
|
||||||
|
### Step 6: Create a namespace
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca ns create prod
|
||||||
|
orca ns list
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: `prod` namespace listed.
|
||||||
|
|
||||||
|
### Step 7: Deploy the full stack
|
||||||
|
|
||||||
|
Deploy each service from `examples/full-stack/`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca job run examples/full-stack/web-app.md --target pve01
|
||||||
|
orca job run examples/full-stack/api.md --target pve01
|
||||||
|
orca job run examples/full-stack/worker.md --target worker01
|
||||||
|
orca job run examples/full-stack/postgres.md --target pve01
|
||||||
|
orca job run examples/full-stack/log-shipper.md --target worker01
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: each job is scheduled on the target, systemd unit deployed via SSH-push, job status `running` or `complete`.
|
||||||
|
|
||||||
|
### Step 8: Verify deployment
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca job list
|
||||||
|
orca job list --json
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: all 5 jobs listed, with correct target nodes.
|
||||||
|
|
||||||
|
On each remote node:
|
||||||
|
```sh
|
||||||
|
ssh root@pve01 systemctl status 'orca-alloc-*'
|
||||||
|
ssh root@worker01 systemctl status 'orca-alloc-*'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 9: Verify Traefik routes
|
||||||
|
|
||||||
|
```sh
|
||||||
|
ssh root@pve01 ls /etc/traefik/dynamic/
|
||||||
|
ssh root@worker01 ls /etc/traefik/dynamic/
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: `traefik-dynamic-*.yaml` files present on nodes where jobs were deployed.
|
||||||
|
|
||||||
|
### Step 10: Migrate between hosts
|
||||||
|
|
||||||
|
Migrate `web-app` from pve01 to worker01:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca job migrate web-app --to worker01
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: job drained on pve01, rescheduled on worker01, new systemd unit deployed.
|
||||||
|
|
||||||
|
Verify:
|
||||||
|
```sh
|
||||||
|
orca job list
|
||||||
|
ssh root@worker01 systemctl status 'orca-alloc-*web-app*'
|
||||||
|
ssh root@pve01 systemctl status 'orca-alloc-*web-app*' # should be stopped
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 11: Aggregate logs
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca logs --all-nodes --job web-app --since 5m
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: log entries from multiple nodes.
|
||||||
|
|
||||||
|
### Step 12: ACL enforcement
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca acl grant operator-1 --namespace prod --permissions read,write
|
||||||
|
orca acl check operator-1 --namespace prod --permission read
|
||||||
|
orca acl check operator-1 --namespace prod --permission admin
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: read+write allowed, admin denied (not granted).
|
||||||
|
|
||||||
|
### Step 12b: Initialize the OIDC provider (for seal)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca auth init-idp --rp-id orca.local
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: Dex config + systemd unit + Traefik route rendered. (Dex binary must be installed separately.)
|
||||||
|
|
||||||
|
### Step 13: Seal/unseal
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca cluster seal
|
||||||
|
orca cluster unseal
|
||||||
|
orca secrets set prod TEST_KEY=test-value
|
||||||
|
orca secrets get prod TEST_KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: seal succeeds, unseal succeeds, secrets readable post-unseal.
|
||||||
|
|
||||||
|
### Step 14: Audit chain
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca doctor audit
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: chain head reported, no tamper detected.
|
||||||
|
|
||||||
|
### Step 15: Doctor modes
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca doctor modes
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: all file modes correct, exit 0.
|
||||||
|
|
||||||
|
### Step 16: OIDC health
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca doctor oidc
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: Dex unit active, issuer reachable (or WARN if Dex not installed).
|
||||||
|
|
||||||
|
### Step 17: Backup and restore
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca backup --out /tmp/uat-backup.tar.gz
|
||||||
|
orca restore --in /tmp/uat-backup.tar.gz --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: backup succeeds, restore dry-run succeeds.
|
||||||
|
|
||||||
|
### Step 18: Drift detection
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca drift show
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: no error (empty drift is fine).
|
||||||
|
|
||||||
|
### Step 19: Transaction idempotency
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca txn apply <some-txn-dir>
|
||||||
|
orca txn apply <some-txn-dir> # re-run
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: second apply is idempotent (exit 5 or "already applied").
|
||||||
|
|
||||||
|
### Step 20: Metrics
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca metrics --addr :9100 &
|
||||||
|
sleep 3
|
||||||
|
curl -s http://localhost:9100/metrics | grep orca_
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: expanded metric set present (`orca_jobs_running`, `orca_audit_chain_head`, etc.).
|
||||||
|
|
||||||
|
### Step 21: Compat check
|
||||||
|
|
||||||
|
```sh
|
||||||
|
orca cluster compat-check
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: exit 0, all nodes compatible.
|
||||||
|
|
||||||
|
### Step 22: Run the signoff script
|
||||||
|
|
||||||
|
```sh
|
||||||
|
scripts/uat-signoff.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
**Expected**: `UAT SIGNOFF: N/35 assertions passed`, exit 0 iff N==35.
|
||||||
|
|
||||||
|
## Claim Matrix
|
||||||
|
|
||||||
|
| # | Claim | UAT Step | Signoff Assertion |
|
||||||
|
|---|-------|----------|-------------------|
|
||||||
|
| 1 | Cluster initializes from scratch | Step 1 | `assert_orca_version` |
|
||||||
|
| 2 | Proxmox host onboards via SSH | Step 2 | `assert_proxmox_onboarded` |
|
||||||
|
| 3 | Ubuntu worker onboards via `--type linux` | Step 3 | `assert_linux_worker_onboarded` |
|
||||||
|
| 4 | Node list shows all nodes | Step 4 | `assert_cluster_initialized` |
|
||||||
|
| 5 | Capacity is set on remote nodes | Step 5 | `assert_capacity_set` |
|
||||||
|
| 6 | Namespace created | Step 6 | `assert_namespace_created` |
|
||||||
|
| 7 | Full stack deploys to remote nodes | Step 7 | `assert_full_stack_running` |
|
||||||
|
| 8 | Scheduler deploys to remote (not local) | Step 7 | `assert_job_deploys_to_remote` |
|
||||||
|
| 9 | Traefik routes present | Step 9 | `assert_traefik_routes` |
|
||||||
|
| 10 | Job migrates between hosts | Step 10 | `assert_migrate_worked` |
|
||||||
|
| 11 | Logs aggregate from multiple nodes | Step 11 | `assert_logs_aggregate` |
|
||||||
|
| 12 | ACL grant/check works | Step 12 | `assert_acl_enforced` |
|
||||||
|
| 13 | ACL deny-by-default | Step 12 | `assert_acl_deny_default` |
|
||||||
|
| 14 | acl.json mode 0600 | Step 12 | `assert_acl_file_mode` |
|
||||||
|
| 15 | Seal/unseal round-trip | Step 13 | `assert_seal_unseal_roundtrip` |
|
||||||
|
| 16 | Audit chain intact | Step 14 | `assert_audit_chain_intact` |
|
||||||
|
| 17 | Doctor modes passes | Step 15 | `assert_doctor_modes` |
|
||||||
|
| 18 | OIDC health check | Step 16 | `assert_oidc_health` |
|
||||||
|
| 19 | Backup works | Step 17 | `assert_backup_restore_dryrun` |
|
||||||
|
| 20 | Drift visible | Step 18 | `assert_drift_visible` |
|
||||||
|
| 21 | Txn idempotent | Step 19 | `assert_txn_idempotent` |
|
||||||
|
| 22 | Metrics expanded | Step 20 | `assert_metrics_expanded` |
|
||||||
|
| 23 | Compat check passes | Step 21 | `assert_compat_check_passes` |
|
||||||
|
| 24 | No `--password` in docs/examples | — | `assert_no_password_in_docs` |
|
||||||
|
| 25 | Go toolchain current | — | `assert_go_toolchain_current` |
|
||||||
|
| 26 | cli.md matches `orca --help` | — | `assert_cli_md_complete` |
|
||||||
|
| 27 | pprof not on all interfaces | — | `assert_no_pprof_on_all_interfaces` |
|
||||||
|
| 28 | WebAuthn registration requires auth | — | `assert_webauthn_reg_requires_auth` |
|
||||||
|
| 29 | Audit chain survives concurrency | — | `assert_audit_chain_concurrent` |
|
||||||
|
| 30 | Concurrent secrets no data loss | — | `assert_concurrent_secrets_no_loss` |
|
||||||
|
| 31 | Cache invalidated after write | — | `assert_cache_invalidated_after_write` |
|
||||||
|
| 32 | SQLite no lock under concurrency | — | `assert_sqlite_no_lock` |
|
||||||
|
| 33 | No injection in logs --job | — | `assert_no_injection_in_logs` |
|
||||||
|
| 34 | `--type linux` exists as subcommand | Step 3 | `assert_type_linux_available` |
|
||||||
|
| 35 | `orca status` deprecated | — | `assert_status_deprecated` |
|
||||||
|
|
||||||
|
## Signoff procedure
|
||||||
|
|
||||||
|
1. Run all steps above on the 3-host cluster
|
||||||
|
2. Run `scripts/uat-signoff.sh` on the lead
|
||||||
|
3. Paste the output back to the CI agent
|
||||||
|
4. The CI agent verifies `35/35 PASS` and cuts `v1.0.0`
|
||||||
|
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### ORCA_HOME not set
|
||||||
|
All orca commands use `$ORCA_HOME` (default `~/.orca`). If commands fail
|
||||||
|
with "no such file or directory", verify:
|
||||||
|
```sh
|
||||||
|
echo $ORCA_HOME
|
||||||
|
ls $ORCA_HOME/orca.db $ORCA_HOME/orca_ssh_key $ORCA_HOME/known_hosts $ORCA_HOME/cluster/master.key
|
||||||
|
```
|
||||||
|
|
||||||
|
### known_hosts missing
|
||||||
|
If SSH operations fail with "open .../known_hosts: no such file", the
|
||||||
|
known_hosts file was not created during `orca init`. Fix:
|
||||||
|
```sh
|
||||||
|
touch $ORCA_HOME/known_hosts
|
||||||
|
chmod 600 $ORCA_HOME/known_hosts
|
||||||
|
```
|
||||||
|
|
||||||
|
### Traefik not running
|
||||||
|
If Traefik routes are not deployed, verify Traefik is running:
|
||||||
|
```sh
|
||||||
|
systemctl status orca-traefik
|
||||||
|
ls /etc/traefik/dynamic/
|
||||||
|
```
|
||||||
|
If not installed, `orca init` should have installed it. Re-run `orca init`
|
||||||
|
or install manually from https://github.com/traefik/traefik/releases.
|
||||||
|
|
||||||
|
### SSH connection refused
|
||||||
|
If the orca SSH key is not pre-staged on the remote host:
|
||||||
|
```sh
|
||||||
|
ssh-copy-id -i ~/.orca/orca_ssh_key.pub root@<host>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Job deployed but not visible in `job list`
|
||||||
|
The remote dispatch path now inserts a DB record (v0.12.16). If you
|
||||||
|
still don't see it, check:
|
||||||
|
```sh
|
||||||
|
orca job list --json
|
||||||
|
```
|
||||||
|
Look for the `"node"` field — it shows which node the job deployed to.
|
||||||
|
|
||||||
|
### Proxmox: process runtime rejected
|
||||||
|
Proxmox nodes require `one_of: pve-ct` or `one_of: pve-vm` in the
|
||||||
|
jobspec. `one_of: process` (systemd) is for Linux/Ubuntu workers only.
|
||||||
+84
-14
@@ -1,27 +1,97 @@
|
|||||||
# WebAuthn / Passkeys (v0.12)
|
# WebAuthn / Passkeys (v0.13)
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
The bundled Dex uses a custom WebAuthn connector for password-free
|
The bundled Dex uses a custom WebAuthn connector (`orca-webauthn-connector`,
|
||||||
authentication. Passkeys are public-key credentials — the private key
|
REQ-148) for password-free authentication. Passkeys are public-key
|
||||||
never leaves the authenticator (TPM/security key/phone Secure Enclave).
|
credentials — the private key never leaves the authenticator (TPM /
|
||||||
|
security key / phone Secure Enclave). This directly satisfies R-021
|
||||||
|
(no Orca-issued credentials): the authenticator proves possession of
|
||||||
|
the private key without ever exposing it.
|
||||||
|
|
||||||
|
The WebAuthn connector ships as part of the v0.12 milestone (P05) and
|
||||||
|
is operationally wired in v0.13 (P04: registration requires auth; P06:
|
||||||
|
real Dex deployment).
|
||||||
|
|
||||||
## Registration
|
## Registration
|
||||||
|
|
||||||
`orca auth register` opens the browser to the Dex WebAuthn endpoint.
|
```bash
|
||||||
After the ceremony (biometric/security key), Dex maps the credential
|
orca auth register [--no-browser]
|
||||||
ID to an OIDC `sub`. Credentials stored at
|
```
|
||||||
`ClusterDir()/webauthn-credentials.db` (0600, public keys only).
|
|
||||||
|
Opens the browser to the Dex WebAuthn registration page at
|
||||||
|
`https://<cluster>/orca/webauthn/register`. The operator authenticates
|
||||||
|
via an existing session or admin bootstrap token, then performs the
|
||||||
|
WebAuthn ceremony (biometric or security key). After the ceremony,
|
||||||
|
Dex maps the credential ID to an OIDC `sub`.
|
||||||
|
|
||||||
|
- **`--no-browser`**: print the registration URL instead of opening a
|
||||||
|
browser (useful for headless operators or remote SSH sessions — copy
|
||||||
|
the URL into a local browser).
|
||||||
|
|
||||||
|
Credentials are stored at `ClusterDir()/webauthn-credentials.db`
|
||||||
|
(mode 0600, public keys only — private keys never leave the
|
||||||
|
authenticator and are never stored by orca).
|
||||||
|
|
||||||
|
**Example**:
|
||||||
|
```bash
|
||||||
|
# Interactive (opens browser)
|
||||||
|
orca auth register
|
||||||
|
|
||||||
|
# Headless / remote SSH (print URL)
|
||||||
|
orca auth register --no-browser
|
||||||
|
# → https://orca.local/orca/webauthn/register
|
||||||
|
```
|
||||||
|
|
||||||
## RP ID
|
## RP ID
|
||||||
|
|
||||||
The relying-party ID is the cluster's Traefik-served domain
|
The relying-party ID is the cluster's Traefik-served domain, set via
|
||||||
(`--rp-id` on `orca auth init-idp`). HTTPS secure context is provided
|
`--rp-id` on `orca auth init-idp` (C-38). The RP ID **must** match the
|
||||||
by Traefik (step-ca cert, R-017).
|
cluster's Traefik domain — WebAuthn enforces that the RP ID is a
|
||||||
|
registrable domain suffix of the current origin.
|
||||||
|
|
||||||
|
HTTPS secure context is provided by Traefik (step-ca cert, R-017).
|
||||||
|
WebAuthn requires a secure context (HTTPS or localhost); the step-ca
|
||||||
|
cert behind Traefik satisfies this.
|
||||||
|
|
||||||
## Bootstrap Sequence
|
## Bootstrap Sequence
|
||||||
|
|
||||||
1. `orca init` bootstraps the cluster CA (step-ca, mTLS-only).
|
1. `orca init` bootstraps the cluster CA (step-ca, mTLS-only).
|
||||||
2. `orca auth init-idp` deploys Dex behind Traefik (step-ca cert).
|
2. `orca auth init-idp --rp-id <cluster-domain>` deploys Dex behind
|
||||||
3. First operator registers a passkey via the mTLS-authenticated session.
|
Traefik (step-ca cert) with the WebAuthn connector configured.
|
||||||
4. Subsequent operators use WebAuthn.
|
3. First operator authenticates via an existing session or admin
|
||||||
|
bootstrap token, then registers a passkey:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca auth register
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Subsequent operators use WebAuthn login (`orca auth login` opens
|
||||||
|
the browser to the Dex login page; the WebAuthn ceremony is one of
|
||||||
|
the available upstreams).
|
||||||
|
|
||||||
|
## Health Check
|
||||||
|
|
||||||
|
```bash
|
||||||
|
orca doctor oidc
|
||||||
|
```
|
||||||
|
|
||||||
|
Verifies the bundled Dex OIDC provider is running and the
|
||||||
|
`/.well-known/openid-configuration` endpoint responds. Run after
|
||||||
|
`orca auth init-idp` or after a Dex config change.
|
||||||
|
|
||||||
|
## Security properties
|
||||||
|
|
||||||
|
- **No passwords**: WebAuthn is password-free. No password is ever
|
||||||
|
sent to or stored by orca (R-021).
|
||||||
|
- **Phishing-resistant**: the WebAuthn protocol cryptographically binds
|
||||||
|
the ceremony to the RP ID, defeating credential phishing.
|
||||||
|
- **Private key never leaves the authenticator**: orca stores only
|
||||||
|
public keys.
|
||||||
|
- **Secure context required**: HTTPS via step-ca / Traefik (C-38).
|
||||||
|
|
||||||
|
## See also
|
||||||
|
|
||||||
|
- [docs/oidc.md](oidc.md) — OIDC configuration (Dex quickstart, BYO IdP)
|
||||||
|
- [docs/security-runbook.md](security-runbook.md) — security runbook
|
||||||
|
- [docs/cli.md](cli.md#orca-auth) — `orca auth` CLI reference
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ a localhost node.
|
|||||||
orca node join --type proxmox --host 192.168.1.100 --ssh-user root
|
orca node join --type proxmox --host 192.168.1.100 --ssh-user root
|
||||||
|
|
||||||
# Join a second node
|
# Join a second node
|
||||||
ORCA_PROXMOX_PASSWORD=secret orca node join --type proxmox --host 192.168.1.101
|
orca node join --type proxmox --host 192.168.1.101 --ssh-key ~/.ssh/orca_ed25519
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 3: Declare node capacity
|
### Step 3: Declare node capacity
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
---
|
||||||
|
kind: Service
|
||||||
|
name: web-app-lxc
|
||||||
|
namespace: prod
|
||||||
|
|
||||||
|
runtime:
|
||||||
|
one_of: pve-ct
|
||||||
|
image: local:vztmpl/ubuntu-24.04
|
||||||
|
|
||||||
|
resources:
|
||||||
|
cpu_millicores: 500
|
||||||
|
memory_mib: 512
|
||||||
|
disk_mib: 2048
|
||||||
|
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 8080
|
||||||
|
protocol: tcp
|
||||||
|
|
||||||
|
constraints:
|
||||||
|
- "node.kind == 'proxmox'"
|
||||||
|
|
||||||
|
restart:
|
||||||
|
mode: service
|
||||||
|
max_retries: 3
|
||||||
|
delay: 10s
|
||||||
|
|
||||||
|
health:
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
path: /healthz
|
||||||
|
|
||||||
|
tasks:
|
||||||
|
- name: web
|
||||||
|
runtime:
|
||||||
|
command: "/bin/bash -c 'apt-get update && apt-get install -y nginx && nginx -g 'daemon off;'"
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
port: 8080
|
||||||
|
protocol: tcp
|
||||||
|
---
|
||||||
|
# Web App (LXC container variant)
|
||||||
|
# Deploys as a Proxmox LXC container via `pct create`.
|
||||||
|
# Requires --target <proxmox-node> and the LXC template
|
||||||
|
# (auto-downloaded during `orca node join --type proxmox`).
|
||||||
Vendored
+5
@@ -73,6 +73,11 @@ func Open(path string) (*Cache, error) {
|
|||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
return nil, fmt.Errorf("ping cache sqlite: %w", err)
|
return nil, fmt.Errorf("ping cache sqlite: %w", err)
|
||||||
}
|
}
|
||||||
|
// REQ-158 / P09 T4: enforce 0600 on the cache DB file (SQLite
|
||||||
|
// creates it at umask, typically 0644). Match store.Open which
|
||||||
|
// chmods after open+ping (the file exists at this point). Non-fatal
|
||||||
|
// if chmod fails (e.g. the DB is at a path we don't own).
|
||||||
|
_ = os.Chmod(path, 0o600)
|
||||||
const schema = `CREATE TABLE IF NOT EXISTS cache_entries (
|
const schema = `CREATE TABLE IF NOT EXISTS cache_entries (
|
||||||
class TEXT NOT NULL,
|
class TEXT NOT NULL,
|
||||||
key TEXT NOT NULL,
|
key TEXT NOT NULL,
|
||||||
|
|||||||
Vendored
+47
@@ -2,6 +2,7 @@ package cache
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -225,3 +226,49 @@ func BenchmarkCacheHit(b *testing.B) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// TestCache_FileMode0600 verifies that the cache DB file is created
|
||||||
|
// with mode 0600 (not the default umask 0644) (REQ-158, P09 T4).
|
||||||
|
func TestCache_FileMode0600(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "orca_cache.db")
|
||||||
|
c, err := Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open: %v", err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stat cache db: %v", err)
|
||||||
|
}
|
||||||
|
got := info.Mode().Perm()
|
||||||
|
if got != 0o600 {
|
||||||
|
t.Errorf("cache db mode = %04o, want 0600", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCache_FileMode0600DefaultPath verifies that the cache DB at the
|
||||||
|
// default path (ORCA_HOME) also gets 0600 (REQ-158, P09 T4).
|
||||||
|
func TestCache_FileMode0600DefaultPath(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
t.Setenv("ORCA_HOME", dir)
|
||||||
|
c, err := Open("")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open default path: %v", err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
|
||||||
|
// The default path is paths.CacheDB() which is under ORCA_HOME.
|
||||||
|
// Find the db file.
|
||||||
|
dbPath := filepath.Join(dir, "orca_cache.db")
|
||||||
|
info, err := os.Stat(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("stat cache db at %s: %v", dbPath, err)
|
||||||
|
}
|
||||||
|
got := info.Mode().Perm()
|
||||||
|
if got != 0o600 {
|
||||||
|
t.Errorf("cache db mode = %04o, want 0600", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ var (
|
|||||||
aclRevokeNamespace string
|
aclRevokeNamespace string
|
||||||
aclCheckNamespace string
|
aclCheckNamespace string
|
||||||
aclCheckPermission string
|
aclCheckPermission string
|
||||||
|
aclCheckVerbose bool
|
||||||
)
|
)
|
||||||
|
|
||||||
var aclCmd = &cobra.Command{
|
var aclCmd = &cobra.Command{
|
||||||
@@ -351,6 +352,16 @@ read, write, admin (default: read).`,
|
|||||||
}
|
}
|
||||||
identity.Namespace = ns
|
identity.Namespace = ns
|
||||||
allowed := a.Check(identity, ns, perm)
|
allowed := a.Check(identity, ns, perm)
|
||||||
|
if aclCheckVerbose {
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "ACL path: %s\n", paths.ACLPath())
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "Identity: kind=%s id=%s ns=%s\n", identity.Kind, identity.ID, ns)
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "Permission: %s -> allowed=%v\n", permStr, allowed)
|
||||||
|
entries := a.List()
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "ACL entries (%d):\n", len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), " kind=%s id=%s ns=%s perms=%d\n", e.Identity.Kind, e.Identity.ID, e.Namespace, e.Permissions)
|
||||||
|
}
|
||||||
|
}
|
||||||
if jsonOutput {
|
if jsonOutput {
|
||||||
return printJSON(map[string]any{
|
return printJSON(map[string]any{
|
||||||
"identity": identity,
|
"identity": identity,
|
||||||
@@ -373,6 +384,7 @@ func init() {
|
|||||||
aclGrantCmd.Flags().StringVar(&aclGrantPermissions, "permissions", "read", "comma-separated permissions: read,write,admin")
|
aclGrantCmd.Flags().StringVar(&aclGrantPermissions, "permissions", "read", "comma-separated permissions: read,write,admin")
|
||||||
aclRevokeCmd.Flags().StringVar(&aclRevokeNamespace, "namespace", "", "namespace scope (required for tokens; defaults to spiffe path ns)")
|
aclRevokeCmd.Flags().StringVar(&aclRevokeNamespace, "namespace", "", "namespace scope (required for tokens; defaults to spiffe path ns)")
|
||||||
aclCheckCmd.Flags().StringVar(&aclCheckNamespace, "namespace", "", "namespace scope (required for tokens; defaults to spiffe path ns)")
|
aclCheckCmd.Flags().StringVar(&aclCheckNamespace, "namespace", "", "namespace scope (required for tokens; defaults to spiffe path ns)")
|
||||||
|
aclCheckCmd.Flags().BoolVar(&aclCheckVerbose, "verbose", false, "print ACL path + loaded entries for debugging")
|
||||||
aclCheckCmd.Flags().StringVar(&aclCheckPermission, "permission", "read", "permission to check: read, write, or admin")
|
aclCheckCmd.Flags().StringVar(&aclCheckPermission, "permission", "read", "permission to check: read, write, or admin")
|
||||||
|
|
||||||
aclCmd.AddCommand(aclGrantCmd)
|
aclCmd.AddCommand(aclGrantCmd)
|
||||||
|
|||||||
+75
-1
@@ -384,7 +384,81 @@ func checkOIDCHealth(ctx context.Context) []oidcCheckResult {
|
|||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// doctorDBRetentionCmd implements `orca doctor db-retention` (REQ-158,
|
||||||
|
// P09 T2). Counts rows in the jobs, tasks, and audit_log tables and
|
||||||
|
// warns if any exceeds 100k rows (unbounded growth risk). Suggests
|
||||||
|
// `orca backup` + manual cleanup.
|
||||||
|
var doctorDBRetentionCmd = &cobra.Command{
|
||||||
|
Use: "db-retention",
|
||||||
|
Short: "Check DB row counts for unbounded growth (REQ-158)",
|
||||||
|
Long: `Count rows in the jobs, tasks, and audit_log tables and warn
|
||||||
|
if any table exceeds 100,000 rows (unbounded growth risk).
|
||||||
|
|
||||||
|
Large tables degrade query performance and inflate backup size. Run
|
||||||
|
'orca backup' to capture a snapshot, then prune old rows manually
|
||||||
|
(e.g. DELETE FROM tasks WHERE created_at < <cutoff>).
|
||||||
|
|
||||||
|
Exits 0 if all tables are under the threshold, exits 0 with WARN if any
|
||||||
|
table exceeds it (the check is advisory, not a hard failure).`,
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
db, closer, err := openDB()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("doctor db-retention: open db: %w", err)
|
||||||
|
}
|
||||||
|
defer closer()
|
||||||
|
|
||||||
|
tables := []string{"jobs", "tasks", "audit_log"}
|
||||||
|
const threshold = 100_000
|
||||||
|
type rowCount struct {
|
||||||
|
Table string `json:"table"`
|
||||||
|
Count int64 `json:"count"`
|
||||||
|
Warn bool `json:"warn"`
|
||||||
|
}
|
||||||
|
var results []rowCount
|
||||||
|
anyWarn := false
|
||||||
|
for _, table := range tables {
|
||||||
|
var count int64
|
||||||
|
q := fmt.Sprintf("SELECT COUNT(*) FROM %s", table)
|
||||||
|
if err := db.QueryRowContext(ctx, q).Scan(&count); err != nil {
|
||||||
|
return fmt.Errorf("doctor db-retention: count %s: %w", table, err)
|
||||||
|
}
|
||||||
|
warn := count > threshold
|
||||||
|
if warn {
|
||||||
|
anyWarn = true
|
||||||
|
}
|
||||||
|
results = append(results, rowCount{Table: table, Count: count, Warn: warn})
|
||||||
|
}
|
||||||
|
|
||||||
|
if jsonOutput {
|
||||||
|
return printJSON(map[string]any{
|
||||||
|
"results": results,
|
||||||
|
"threshold": threshold,
|
||||||
|
"any_warn": anyWarn,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
for _, r := range results {
|
||||||
|
status := "ok"
|
||||||
|
if r.Warn {
|
||||||
|
status = "WARN"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%-12s %-5s %d rows (threshold: %d)\n", r.Table, status, r.Count, threshold)
|
||||||
|
}
|
||||||
|
if anyWarn {
|
||||||
|
fmt.Fprintf(out, "\n⚠ one or more tables exceed %d rows — run 'orca backup' then prune old rows\n", threshold)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(out, "\n✓ all tables under retention threshold")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd, doctorAuditCmd, doctorModesCmd, doctorOIDCCmd)
|
doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd, doctorAuditCmd, doctorModesCmd, doctorOIDCCmd, doctorDBRetentionCmd)
|
||||||
rootCmd.AddCommand(doctorCmd)
|
rootCmd.AddCommand(doctorCmd)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
// Package cli: doctor_ingress.go implements `orca doctor ingress`
|
||||||
|
// (R-024, v0.14). The check verifies the podman traefik container is
|
||||||
|
// running, nft DNAT+SNAT is applied, the dynamic config directory
|
||||||
|
// exists, and the step-ca root CA is mounted.
|
||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var doctorIngressCmd = &cobra.Command{
|
||||||
|
Use: "ingress",
|
||||||
|
Short: "Check the ingress stack (R-024: podman traefik + nft + CA)",
|
||||||
|
Long: `Verify the orca ingress data plane is healthy:
|
||||||
|
1. orca-traefik podman container is running
|
||||||
|
2. nft DNAT + SNAT masquerade applied
|
||||||
|
3. /etc/traefik/dynamic directory exists
|
||||||
|
4. step-ca root CA mounted at /etc/orca/step-ca-root.crt
|
||||||
|
|
||||||
|
For remote peers, use --peer <name>.`,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
results := runIngressChecks(ctx)
|
||||||
|
if jsonOutput {
|
||||||
|
return printJSON(results)
|
||||||
|
}
|
||||||
|
allPass := true
|
||||||
|
for _, r := range results {
|
||||||
|
status := "✓"
|
||||||
|
if r.Result != "PASS" {
|
||||||
|
status = "✗"
|
||||||
|
allPass = false
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%s %s: %s\n", status, r.Name, r.Message)
|
||||||
|
}
|
||||||
|
if !allPass {
|
||||||
|
return fmt.Errorf("ingress checks failed")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// ingressCheckResult is one line of `orca doctor ingress` output.
|
||||||
|
type ingressCheckResult struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Result string `json:"result"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func runIngressChecks(ctx context.Context) []ingressCheckResult {
|
||||||
|
t, err := nftTransportFromCtx()
|
||||||
|
if err != nil {
|
||||||
|
return []ingressCheckResult{{Name: "ingress:transport", Result: "FAIL", Message: err.Error()}}
|
||||||
|
}
|
||||||
|
peer := nftLeadPeer()
|
||||||
|
var results []ingressCheckResult
|
||||||
|
|
||||||
|
// 1. Check podman orca-traefik container is running.
|
||||||
|
out, err := t.Exec(ctx, peer, "podman inspect --format '{{.State.Running}}' orca-traefik 2>/dev/null")
|
||||||
|
if err != nil {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:container", Result: "FAIL", Message: fmt.Sprintf("podman inspect: %v", err)})
|
||||||
|
} else {
|
||||||
|
v := strings.TrimSpace(string(out))
|
||||||
|
if v == "true" {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:container", Result: "PASS", Message: "orca-traefik container running"})
|
||||||
|
} else if v == "false" {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:container", Result: "FAIL", Message: "orca-traefik container is stopped"})
|
||||||
|
} else {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:container", Result: "FAIL", Message: "orca-traefik container not found"})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Check nft DNAT + SNAT (reuse the nft table output).
|
||||||
|
tableOut, tableErr := t.Exec(ctx, peer, "nft list table inet orca-ingress 2>/dev/null")
|
||||||
|
if tableErr != nil {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "FAIL", Message: "nft table orca-ingress missing"})
|
||||||
|
} else {
|
||||||
|
tableStr := string(tableOut)
|
||||||
|
hasDNAT := strings.Contains(tableStr, "dnat to")
|
||||||
|
hasSNAT := strings.Contains(tableStr, "masquerade")
|
||||||
|
if hasDNAT && hasSNAT {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "PASS", Message: "nft DNAT + SNAT masquerade present"})
|
||||||
|
} else if hasDNAT {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "WARN", Message: "DNAT present but SNAT masquerade missing"})
|
||||||
|
} else {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:nft", Result: "FAIL", Message: "nft DNAT missing"})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Check /etc/traefik/dynamic directory exists.
|
||||||
|
if _, err := t.Exec(ctx, peer, "test -d /etc/traefik/dynamic"); err != nil {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:dynamic-dir", Result: "FAIL", Message: "/etc/traefik/dynamic directory missing"})
|
||||||
|
} else {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:dynamic-dir", Result: "PASS", Message: "/etc/traefik/dynamic exists"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Check step-ca root CA is mounted/present.
|
||||||
|
if _, err := t.Exec(ctx, peer, "test -f /etc/orca/step-ca-root.crt"); err != nil {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:ca", Result: "WARN", Message: "/etc/orca/step-ca-root.crt missing (TLS not configured)"})
|
||||||
|
} else {
|
||||||
|
results = append(results, ingressCheckResult{Name: "ingress:ca", Result: "PASS", Message: "step-ca root CA present"})
|
||||||
|
}
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
doctorCmd.AddCommand(doctorIngressCmd)
|
||||||
|
}
|
||||||
@@ -113,6 +113,20 @@ func runNftChecks(ctx context.Context) []nftCheckResult {
|
|||||||
results = append(results, nftCheckResult{Name: "nft:dnat-80", Result: "FAIL", Message: "DNAT :80->127.0.0.1:8080 missing"})
|
results = append(results, nftCheckResult{Name: "nft:dnat-80", Result: "FAIL", Message: "DNAT :80->127.0.0.1:8080 missing"})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Research Topic 1: postrouting masquerade for DNAT return path.
|
||||||
|
if strings.Contains(tableStr, "masquerade") {
|
||||||
|
results = append(results, nftCheckResult{Name: "nft:snat-masquerade", Result: "PASS", Message: "postrouting masquerade (SNAT) present"})
|
||||||
|
} else {
|
||||||
|
results = append(results, nftCheckResult{Name: "nft:snat-masquerade", Result: "FAIL", Message: "postrouting masquerade missing (R-024)"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Research Topic 2: priority -10 on input/forward (pve-firewall coexistence).
|
||||||
|
if strings.Contains(tableStr, "priority -10") {
|
||||||
|
results = append(results, nftCheckResult{Name: "nft:priority", Result: "PASS", Message: "input/forward chains at priority -10 (pve-firewall coexistence)"})
|
||||||
|
} else {
|
||||||
|
results = append(results, nftCheckResult{Name: "nft:priority", Result: "WARN", Message: "priority -10 not found (may be pre-v0.14 ruleset)"})
|
||||||
|
}
|
||||||
|
|
||||||
if strings.Contains(tableStr, "ora_rl") {
|
if strings.Contains(tableStr, "ora_rl") {
|
||||||
results = append(results, nftCheckResult{Name: "nft:rate-limit", Result: "PASS", Message: "rate-limit meter ora_rl present"})
|
results = append(results, nftCheckResult{Name: "nft:rate-limit", Result: "PASS", Message: "rate-limit meter ora_rl present"})
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -2,9 +2,14 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestDoctorText(t *testing.T) {
|
func TestDoctorText(t *testing.T) {
|
||||||
@@ -194,3 +199,92 @@ func TestDoctorProxmoxJSON(t *testing.T) {
|
|||||||
t.Errorf("doctor proxmox --json missing Name: %v", result)
|
t.Errorf("doctor proxmox --json missing Name: %v", result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// TestDoctorDBRetention verifies that `orca doctor db-retention` counts
|
||||||
|
// rows in jobs, tasks, and audit_log and warns when a table exceeds
|
||||||
|
// 100k rows (REQ-158, P09 T7).
|
||||||
|
func TestDoctorDBRetention(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
if err := runInit(discardWriter{}); err != nil {
|
||||||
|
t.Fatalf("init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert 100001 rows into the audit_log table to trigger the warning.
|
||||||
|
// Use a multi-row VALUES insert in batches for speed.
|
||||||
|
db, err := store.Open(certpaths.DBPath())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open db: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
ctx := context.Background()
|
||||||
|
// Build a batch insert: 500 rows per INSERT in a transaction.
|
||||||
|
// SQLite handles this much faster than 100k individual inserts.
|
||||||
|
const totalRows = 100001
|
||||||
|
const batchSize = 500
|
||||||
|
inserted := 0
|
||||||
|
for inserted < totalRows {
|
||||||
|
remaining := totalRows - inserted
|
||||||
|
batch := batchSize
|
||||||
|
if remaining < batch {
|
||||||
|
batch = remaining
|
||||||
|
}
|
||||||
|
var placeholders strings.Builder
|
||||||
|
var args []any
|
||||||
|
for j := 0; j < batch; j++ {
|
||||||
|
if j > 0 {
|
||||||
|
placeholders.WriteString(",")
|
||||||
|
}
|
||||||
|
placeholders.WriteString("(?, 'test', 'test.action', 'test-resource', 'success')")
|
||||||
|
args = append(args, time.Now().UTC())
|
||||||
|
}
|
||||||
|
q := "INSERT INTO audit_log (timestamp, actor, action, resource, result) VALUES " + placeholders.String()
|
||||||
|
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||||
|
t.Fatalf("batch insert at offset %d: %v", inserted, err)
|
||||||
|
}
|
||||||
|
inserted += batch
|
||||||
|
}
|
||||||
|
|
||||||
|
resetRootFlags(t)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
rootCmd.SetArgs([]string{"doctor", "db-retention"})
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
t.Fatalf("doctor db-retention: %v", err)
|
||||||
|
}
|
||||||
|
out := buf.String()
|
||||||
|
if !strings.Contains(out, "audit_log") {
|
||||||
|
t.Errorf("output missing audit_log table: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "WARN") {
|
||||||
|
t.Errorf("output should contain WARN for audit_log exceeding threshold: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "backup") {
|
||||||
|
t.Errorf("output should suggest 'orca backup': %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDoctorDBRetentionNoWarn verifies that with a small DB no warning
|
||||||
|
// is emitted (REQ-158, P09 T7).
|
||||||
|
func TestDoctorDBRetentionNoWarn(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
if err := runInit(discardWriter{}); err != nil {
|
||||||
|
t.Fatalf("init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resetRootFlags(t)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
rootCmd.SetArgs([]string{"doctor", "db-retention"})
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
t.Fatalf("doctor db-retention: %v", err)
|
||||||
|
}
|
||||||
|
out := buf.String()
|
||||||
|
if strings.Contains(out, "WARN") {
|
||||||
|
t.Errorf("output should NOT contain WARN for small DB: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+26
-4
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -48,12 +49,17 @@ func drainExecFromCtx(_ context.Context) (drainExecer, error) {
|
|||||||
// Address carries host:8443. We always target SSH port 22 unless the
|
// Address carries host:8443. We always target SSH port 22 unless the
|
||||||
// node's Address already encodes a non-daemon port. The local node
|
// node's Address already encodes a non-daemon port. The local node
|
||||||
// (Name=="localhost") is contacted at "localhost:22".
|
// (Name=="localhost") is contacted at "localhost:22".
|
||||||
|
//
|
||||||
|
// REQ-157 / P08 T5: uses net.JoinHostPort for proper IPv6 bracketing
|
||||||
|
// (e.g. "fd00::1" + "22" -> "[fd00::1]:22"). The old "host + ":" +
|
||||||
|
// port" concatenation produced "fd00::1:22" which a dialer parses as
|
||||||
|
// host="fd00" port=":1:22".
|
||||||
func peerAddrForNode(n *model.Node) string {
|
func peerAddrForNode(n *model.Node) string {
|
||||||
if n == nil {
|
if n == nil {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
if h, p, ok := splitHostPort(n.Address); ok && p != "" && p != "8443" {
|
if h, p, ok := splitHostPort(n.Address); ok && p != "" && p != "8443" {
|
||||||
return h + ":" + p
|
return net.JoinHostPort(h, p)
|
||||||
}
|
}
|
||||||
host := n.Name
|
host := n.Name
|
||||||
if h, _, ok := splitHostPort(n.Address); ok && h != "" && h != "localhost" {
|
if h, _, ok := splitHostPort(n.Address); ok && h != "" && h != "localhost" {
|
||||||
@@ -62,15 +68,31 @@ func peerAddrForNode(n *model.Node) string {
|
|||||||
if host == "" {
|
if host == "" {
|
||||||
host = n.Name
|
host = n.Name
|
||||||
}
|
}
|
||||||
return host + ":22"
|
return net.JoinHostPort(host, "22")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// splitHostPort splits a host:port address into its host and port
|
||||||
|
// components. It uses net.SplitHostPort for proper IPv6 bracketing
|
||||||
|
// (e.g. "[fd00::1]:8443" -> "fd00::1", "8443"). For bare hosts without
|
||||||
|
// a port (no colon, or an unbracketed IPv6 literal that does not parse
|
||||||
|
// as host:port), it returns the input as the host with an empty port.
|
||||||
func splitHostPort(addr string) (string, string, bool) {
|
func splitHostPort(addr string) (string, string, bool) {
|
||||||
idx := strings.LastIndex(addr, ":")
|
host, port, err := net.SplitHostPort(addr)
|
||||||
if idx < 0 {
|
if err == nil {
|
||||||
|
return host, port, true
|
||||||
|
}
|
||||||
|
// Fall back to the legacy LastIndex behavior for inputs that
|
||||||
|
// net.SplitHostPort rejects (e.g. bare "localhost" with no port).
|
||||||
|
if idx := strings.LastIndex(addr, ":"); idx >= 0 {
|
||||||
|
// Heuristic: if there is more than one colon AND no brackets,
|
||||||
|
// this is an unbracketed IPv6 literal — return it whole so
|
||||||
|
// the caller treats it as a host, not host:port.
|
||||||
|
if strings.Count(addr, ":") > 1 && !strings.HasPrefix(addr, "[") {
|
||||||
return addr, "", false
|
return addr, "", false
|
||||||
}
|
}
|
||||||
return addr[:idx], addr[idx+1:], true
|
return addr[:idx], addr[idx+1:], true
|
||||||
|
}
|
||||||
|
return addr, "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|||||||
+26
-3
@@ -69,6 +69,17 @@ func driftTransportFromCtx() (driftTransport, error) {
|
|||||||
return sshpush.NewTransport(keyPath, khPath), nil
|
return sshpush.NewTransport(keyPath, khPath), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sshCmdCtx returns a context derived from parent with the SSH
|
||||||
|
// command timeout applied. If d <= 0, the parent is returned unchanged
|
||||||
|
// (no deadline). REQ-157 / P08 T6: gives SSH-driven CLI subcommands a
|
||||||
|
// bounded deadline so a hung peer cannot block forever.
|
||||||
|
func sshCmdCtx(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) {
|
||||||
|
if d <= 0 {
|
||||||
|
return context.WithCancel(parent)
|
||||||
|
}
|
||||||
|
return context.WithTimeout(parent, d)
|
||||||
|
}
|
||||||
|
|
||||||
// driftDetectorOverride is the package-level test seam for the
|
// driftDetectorOverride is the package-level test seam for the
|
||||||
// Detector itself. When non-nil it replaces the production detector
|
// Detector itself. When non-nil it replaces the production detector
|
||||||
// (which wraps a driftTransport). Tests set it and restore nil.
|
// (which wraps a driftTransport). Tests set it and restore nil.
|
||||||
@@ -202,7 +213,9 @@ blocks txn apply for that namespace (R-020).`,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("drift detector: %w", err)
|
return fmt.Errorf("drift detector: %w", err)
|
||||||
}
|
}
|
||||||
if err := d.Acknowledge(cmd.Context(), peer, path); err != nil {
|
ctx, cancel := sshCmdCtx(cmd.Context(), driftAckTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := d.Acknowledge(ctx, peer, path); err != nil {
|
||||||
return fmt.Errorf("acknowledge: %w", err)
|
return fmt.Errorf("acknowledge: %w", err)
|
||||||
}
|
}
|
||||||
printResult(fmt.Sprintf("✓ Acknowledged drift on %s for %s", peer, path), map[string]any{
|
printResult(fmt.Sprintf("✓ Acknowledged drift on %s for %s", peer, path), map[string]any{
|
||||||
@@ -225,7 +238,9 @@ var driftRemediateCmd = &cobra.Command{
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("drift detector: %w", err)
|
return fmt.Errorf("drift detector: %w", err)
|
||||||
}
|
}
|
||||||
if err := d.Remediate(cmd.Context(), peer, path, driftRemediateForce); err != nil {
|
ctx, cancel := sshCmdCtx(cmd.Context(), driftRemediateTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := d.Remediate(ctx, peer, path, driftRemediateForce); err != nil {
|
||||||
if errors.Is(err, drift.ErrCooldown) {
|
if errors.Is(err, drift.ErrCooldown) {
|
||||||
printResult(fmt.Sprintf("✗ Remediation in cooldown for %s on %s (use --force to bypass)", path, peer), map[string]any{
|
printResult(fmt.Sprintf("✗ Remediation in cooldown for %s on %s (use --force to bypass)", path, peer), map[string]any{
|
||||||
"peer": peer, "path": path, "status": "cooldown",
|
"peer": peer, "path": path, "status": "cooldown",
|
||||||
@@ -329,7 +344,9 @@ when /etc/orca/allocs/<id>/env drifts.`,
|
|||||||
}
|
}
|
||||||
unit := fmt.Sprintf("orca-alloc-%s.service", name)
|
unit := fmt.Sprintf("orca-alloc-%s.service", name)
|
||||||
restartCmd := fmt.Sprintf("systemctl restart %s", shellQuoteDrift(unit))
|
restartCmd := fmt.Sprintf("systemctl restart %s", shellQuoteDrift(unit))
|
||||||
out, err := transport.Exec(cmd.Context(), peer, restartCmd)
|
ctx, cancel := sshCmdCtx(cmd.Context(), jobRestartTimeout)
|
||||||
|
defer cancel()
|
||||||
|
out, err := transport.Exec(ctx, peer, restartCmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("restart %s on %s: %w (output: %s)", unit, peer, err, string(out))
|
return fmt.Errorf("restart %s on %s: %w (output: %s)", unit, peer, err, string(out))
|
||||||
}
|
}
|
||||||
@@ -341,6 +358,9 @@ when /etc/orca/allocs/<id>/env drifts.`,
|
|||||||
}
|
}
|
||||||
|
|
||||||
var jobRestartPeer string
|
var jobRestartPeer string
|
||||||
|
var driftRemediateTimeout time.Duration
|
||||||
|
var driftAckTimeout time.Duration
|
||||||
|
var jobRestartTimeout time.Duration
|
||||||
|
|
||||||
func shellQuoteDrift(s string) string {
|
func shellQuoteDrift(s string) string {
|
||||||
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||||
@@ -351,6 +371,9 @@ func init() {
|
|||||||
driftWatchCmd.Flags().StringSliceVar(&driftWatchPaths, "paths", nil, "comma-separated glob patterns to watch (default: all)")
|
driftWatchCmd.Flags().StringSliceVar(&driftWatchPaths, "paths", nil, "comma-separated glob patterns to watch (default: all)")
|
||||||
driftShowCmd.Flags().StringVar(&driftShowPeer, "peer", "", "filter to a single peer host")
|
driftShowCmd.Flags().StringVar(&driftShowPeer, "peer", "", "filter to a single peer host")
|
||||||
driftRemediateCmd.Flags().BoolVar(&driftRemediateForce, "force", false, "bypass the cooldown window (C4)")
|
driftRemediateCmd.Flags().BoolVar(&driftRemediateForce, "force", false, "bypass the cooldown window (C4)")
|
||||||
|
driftRemediateCmd.Flags().DurationVar(&driftRemediateTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||||
|
driftAckCmd.Flags().DurationVar(&driftAckTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||||
|
jobRestartCmd.Flags().DurationVar(&jobRestartTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||||
driftConfigCmd.PersistentFlags().StringVar(&driftConfigPath, "config", "", "path to drift config JSON (default: built-in)")
|
driftConfigCmd.PersistentFlags().StringVar(&driftConfigPath, "config", "", "path to drift config JSON (default: built-in)")
|
||||||
jobRestartCmd.Flags().StringVar(&jobRestartPeer, "peer", "", "peer address (host:port) running the allocation")
|
jobRestartCmd.Flags().StringVar(&jobRestartPeer, "peer", "", "peer address (host:port) running the allocation")
|
||||||
|
|
||||||
|
|||||||
+139
-6
@@ -1,23 +1,28 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
"crypto/x509"
|
"crypto/x509"
|
||||||
"encoding/pem"
|
"encoding/pem"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
|
|
||||||
"git.cloudinit.dev/coreci/orca/internal/acl"
|
"git.cloudinit.dev/coreci/orca/internal/acl"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/identity"
|
"git.cloudinit.dev/coreci/orca/internal/identity"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/ingress"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/secrets"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -138,6 +143,134 @@ func runInit(out interface{ Write([]byte) (int, error) }) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Step 4a: SSH keypair (idempotent — GenerateOrLoadSSHKey has a fast-path).
|
||||||
|
// REQ-164: without this, every sshpush.Transport dial fails because
|
||||||
|
// the orca SSH key doesn't exist after a fresh init.
|
||||||
|
sshKeyPEM, sshPubLine, err := security.GenerateOrLoadSSHKey(dir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("generate SSH keypair: %w", err)
|
||||||
|
}
|
||||||
|
_ = sshKeyPEM
|
||||||
|
sshKeyFp := ""
|
||||||
|
if pubKey, err := ssh.ParsePublicKey(sshPubLine); err == nil {
|
||||||
|
sshKeyFp = ssh.FingerprintSHA256(pubKey)
|
||||||
|
}
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "ssh-key", Status: "ok", Detail: sshKeyFp[:min(16, len(sshKeyFp))] + "..."})
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(out, "\xe2\x9c\x93 SSH keypair provisioned: fp=%s\n", sshKeyFp[:min(16, len(sshKeyFp))]+"...")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4a-2: Interactively pre-stage the orca public key on remote
|
||||||
|
// hosts (ssh-copy-id). Skipped in --json mode (non-interactive).
|
||||||
|
// The operator enters host addresses (one per line, empty line to
|
||||||
|
// finish). For each host, ssh-copy-id is run; if SSH key auth is
|
||||||
|
// not yet established, ssh-copy-id prompts for the password
|
||||||
|
// interactively. This makes orca init the single entry point —
|
||||||
|
// no manual pre-staging required.
|
||||||
|
if !jsonOutput {
|
||||||
|
pubPath := certpaths.SSHPubPath()
|
||||||
|
fmt.Fprintf(out, "\n Pre-stage the orca public key on remote nodes.\n")
|
||||||
|
fmt.Fprintf(out, " Enter host addresses (one per line, empty line to skip):\n")
|
||||||
|
stagedHosts := []string{}
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
for {
|
||||||
|
fmt.Fprintf(out, " host> ")
|
||||||
|
line, err := reader.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
host := strings.TrimSpace(line)
|
||||||
|
if host == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// Run ssh-copy-id interactively (password prompt passes through).
|
||||||
|
fmt.Fprintf(out, " Deploying orca key to %s...\n", host)
|
||||||
|
cmd := exec.Command("ssh-copy-id", "-i", pubPath, "-o", "StrictHostKeyChecking=accept-new", "root@"+host)
|
||||||
|
cmd.Stdin = os.Stdin
|
||||||
|
cmd.Stdout = os.Stdout
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
fmt.Fprintf(out, " \xe2\x9a\xa0 Failed to deploy key to %s: %v\n", host, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, " \xe2\x9c\x93 Key deployed to %s\n", host)
|
||||||
|
stagedHosts = append(stagedHosts, host)
|
||||||
|
}
|
||||||
|
if len(stagedHosts) > 0 {
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "pre-stage", Status: "ok", Detail: strings.Join(stagedHosts, ", ")})
|
||||||
|
} else {
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "pre-stage", Status: "skipped", Detail: "no hosts entered"})
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4b: known_hosts file (empty, 0600). Without this, the TOFU
|
||||||
|
// host-key callback fails with "no such file" on the first SSH dial
|
||||||
|
// (knownhosts.New requires the file to exist).
|
||||||
|
knownHostsPath := certpaths.KnownHostsPath()
|
||||||
|
if _, err := os.Stat(knownHostsPath); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
if err := os.WriteFile(knownHostsPath, []byte{}, 0o600); err != nil {
|
||||||
|
return fmt.Errorf("create known_hosts: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("stat known_hosts: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "known-hosts", Status: "ok", Detail: knownHostsPath})
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(out, "\xe2\x9c\x93 Known hosts file created: %s\n", knownHostsPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4c: master key (32-byte random, 0600). Without this, secrets
|
||||||
|
// set/get/rotate and cluster seal/unseal all fail with "stat master
|
||||||
|
// key: no such file or directory" on a fresh init.
|
||||||
|
masterKeyPath := paths.MasterKeyPath()
|
||||||
|
if _, err := os.Stat(masterKeyPath); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
os.MkdirAll(filepath.Dir(masterKeyPath), 0o755)
|
||||||
|
masterKey, err := secrets.GenerateMasterKey()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("generate master key: %w", err)
|
||||||
|
}
|
||||||
|
if err := secrets.SaveMasterKey(masterKeyPath, masterKey); err != nil {
|
||||||
|
return fmt.Errorf("save master key: %w", err)
|
||||||
|
}
|
||||||
|
// Zero the key from memory (defense-in-depth, REQ-154).
|
||||||
|
defer secrets.ZeroKey(masterKey)
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "master-key", Status: "ok", Detail: "generated"})
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(out, "\xe2\x9c\x93 Master key generated: %s\n", masterKeyPath)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("stat master key: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "master-key", Status: "skipped", Detail: "already present"})
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(out, "\xe2\x9c\x93 Master key: already present\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4d: Ensure complete ingress stack on the lead node (R-024).
|
||||||
|
// This replaces the v0.13 binary+systemd traefik install with:
|
||||||
|
// 1. Render + write traefik static config (traefik.yml)
|
||||||
|
// 2. Render + write + apply nft DNAT/SNAT ruleset (orca.nft)
|
||||||
|
// 3. Push step-ca root CA to /etc/orca/step-ca-root.crt
|
||||||
|
// 4. Ensure podman orca-traefik container running
|
||||||
|
// All steps non-fatal (offline host tolerance).
|
||||||
|
if err := ingress.BootstrapLocalIngress(context.Background(), version); err != nil {
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(out, "Ingress bootstrap skipped: %v\n", err)
|
||||||
|
}
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "ingress", Status: "skipped", Detail: err.Error()})
|
||||||
|
} else {
|
||||||
|
summary.Steps = append(summary.Steps, stepResult{Label: "ingress", Status: "ok", Detail: "nft+traefik container active"})
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(out, "Ingress: nft DNAT+SNAT applied, traefik container running\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Step 5: OS detection.
|
// Step 5: OS detection.
|
||||||
osDetected := detectOS()
|
osDetected := detectOS()
|
||||||
summary.OS = osDetected
|
summary.OS = osDetected
|
||||||
|
|||||||
+186
-15
@@ -2,6 +2,7 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -14,9 +15,11 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/engine"
|
"git.cloudinit.dev/coreci/orca/internal/engine"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -44,8 +47,8 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var jobRunCmd = &cobra.Command{
|
var jobRunCmd = &cobra.Command{
|
||||||
Use: "run <spec.hcl>",
|
Use: "run <spec.md>",
|
||||||
Short: "Run a job from an HCL spec file",
|
Short: "Run a job from a markdown spec file",
|
||||||
Long: "Submit a job spec, execute its tasks, and persist the result. Use --target to pin to a specific node (overrides bin-packing); --idempotency-key for cross-node dispatch dedupe.",
|
Long: "Submit a job spec, execute its tasks, and persist the result. Use --target to pin to a specific node (overrides bin-packing); --idempotency-key for cross-node dispatch dedupe.",
|
||||||
Args: cobra.ExactArgs(1),
|
Args: cobra.ExactArgs(1),
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
@@ -112,7 +115,7 @@ var jobRunCmd = &cobra.Command{
|
|||||||
switch res.mode {
|
switch res.mode {
|
||||||
case "remote":
|
case "remote":
|
||||||
// Scheduler selected a node (or --target pinned one): render
|
// Scheduler selected a node (or --target pinned one): render
|
||||||
// the systemd unit, verify it, and SSH-push to the peer.
|
// the systemd unit / PVE container, verify it, and SSH-push.
|
||||||
// C-44: a push failure is an error (no local fallback).
|
// C-44: a push failure is an error (no local fallback).
|
||||||
unitPaths, derr := deployRemote(ctx, spec, res, nodesByHost)
|
unitPaths, derr := deployRemote(ctx, spec, res, nodesByHost)
|
||||||
logDispatch(res, derr)
|
logDispatch(res, derr)
|
||||||
@@ -123,8 +126,12 @@ var jobRunCmd = &cobra.Command{
|
|||||||
return derr
|
return derr
|
||||||
}
|
}
|
||||||
res.unitPaths = unitPaths
|
res.unitPaths = unitPaths
|
||||||
// REQ-156 / P07 T5: invalidate the jobs cache (the
|
// REQ-166 / Phase C2: insert a Job DB record so `job list`
|
||||||
// dispatch decision records a local job entry).
|
// and `job stop` can find the remotely-deployed job.
|
||||||
|
if dbErr := insertRemoteJob(spec, res.node); dbErr != nil {
|
||||||
|
// Non-fatal: the job is deployed, just not visible to list.
|
||||||
|
logDispatch(res, dbErr)
|
||||||
|
}
|
||||||
cacheInvalidate(cacheJobClass)
|
cacheInvalidate(cacheJobClass)
|
||||||
if jsonOutput {
|
if jsonOutput {
|
||||||
return printJSON(map[string]any{
|
return printJSON(map[string]any{
|
||||||
@@ -213,12 +220,20 @@ func renderJobs(cmd *cobra.Command, jobs []*model.Job) error {
|
|||||||
return printJSON(jobs)
|
return printJSON(jobs)
|
||||||
}
|
}
|
||||||
if len(jobs) == 0 {
|
if len(jobs) == 0 {
|
||||||
fmt.Fprintln(cmd.OutOrStdout(), "No jobs. Use 'orca job run <spec.hcl>' to submit one.")
|
fmt.Fprintln(cmd.OutOrStdout(), "No jobs. Use 'orca job run <spec.md>' to submit one.")
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
fmt.Fprintf(cmd.OutOrStdout(), "%-36s %-20s %-12s %-8s\n", "ID", "NAME", "STATUS", "EXIT")
|
fmt.Fprintf(cmd.OutOrStdout(), "%-10s %-20s %-12s %-20s %-5s\n", "ID", "NAME", "STATUS", "NODE", "EXIT")
|
||||||
for _, j := range jobs {
|
for _, j := range jobs {
|
||||||
fmt.Fprintf(cmd.OutOrStdout(), "%-36s %-20s %-12s %-8d\n", j.ID, j.Name, j.Status, j.ExitCode)
|
shortID := j.ID
|
||||||
|
if len(shortID) > 8 {
|
||||||
|
shortID = shortID[:8]
|
||||||
|
}
|
||||||
|
exit := "-"
|
||||||
|
if j.Status == model.JobStatusComplete || j.Status == model.JobStatusFailed || j.Status == model.JobStatusStopped {
|
||||||
|
exit = fmt.Sprintf("%d", j.ExitCode)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "%-10s %-20s %-12s %-20s %-5s\n", shortID, j.Name, j.Status, j.Node, exit)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -283,17 +298,93 @@ func renderJobTable(jobs []*model.Job) string {
|
|||||||
if len(jobs) == 0 {
|
if len(jobs) == 0 {
|
||||||
return "No jobs.\n"
|
return "No jobs.\n"
|
||||||
}
|
}
|
||||||
out := fmt.Sprintf("%-36s %-20s %-12s %-8s\n", "ID", "NAME", "STATUS", "EXIT")
|
out := fmt.Sprintf("%-10s %-20s %-12s %-20s %-5s\n", "ID", "NAME", "STATUS", "NODE", "EXIT")
|
||||||
for _, j := range jobs {
|
for _, j := range jobs {
|
||||||
out += fmt.Sprintf("%-36s %-20s %-12s %-8d\n", j.ID, j.Name, j.Status, j.ExitCode)
|
shortID := j.ID
|
||||||
|
if len(shortID) > 8 {
|
||||||
|
shortID = shortID[:8]
|
||||||
|
}
|
||||||
|
exit := "-"
|
||||||
|
if j.Status == model.JobStatusComplete || j.Status == model.JobStatusFailed || j.Status == model.JobStatusStopped {
|
||||||
|
exit = fmt.Sprintf("%d", j.ExitCode)
|
||||||
|
}
|
||||||
|
out += fmt.Sprintf("%-10s %-20s %-12s %-20s %-5s\n", shortID, j.Name, j.Status, j.Node, exit)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// jobStopTransport is the SSH command-execution seam used by
|
||||||
|
// `orca job stop`. *sshpush.Transport satisfies it via Exec; tests
|
||||||
|
// inject a mock (same pattern as driftTransport / drainExecer).
|
||||||
|
type jobStopTransport interface {
|
||||||
|
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobStopTransportOverride is the package-level test seam for the
|
||||||
|
// SSH transport used by `orca job stop`. When non-nil it replaces the
|
||||||
|
// production transport; tests set it and restore nil in cleanup.
|
||||||
|
var jobStopTransportOverride jobStopTransport
|
||||||
|
|
||||||
|
// jobStopTimeout is the SSH command timeout for `orca job stop`.
|
||||||
|
var jobStopTimeout time.Duration
|
||||||
|
|
||||||
|
// jobStopPeer is the optional --peer override for `orca job stop`.
|
||||||
|
// When empty, the node is looked up from the alloc_history table
|
||||||
|
// (latest entry for the job id). When set, the SSH stop targets that
|
||||||
|
// peer directly.
|
||||||
|
var jobStopPeer string
|
||||||
|
|
||||||
|
func jobStopTransportFromCtx() (jobStopTransport, error) {
|
||||||
|
if jobStopTransportOverride != nil {
|
||||||
|
return jobStopTransportOverride, nil
|
||||||
|
}
|
||||||
|
keyPath := certpaths.SSHKeyPath()
|
||||||
|
khPath := certpaths.KnownHostsPath()
|
||||||
|
return sshpush.NewTransport(keyPath, khPath), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeForJob looks up the node that ran (or is running) a job by
|
||||||
|
// searching the alloc_history table for the latest entry for the
|
||||||
|
// given job id. Returns nil if no history entry exists (the job may
|
||||||
|
// have been run locally or pre-dates alloc_history).
|
||||||
|
func nodeForJob(ctx context.Context, db *sql.DB, jobID string) (*model.Node, error) {
|
||||||
|
hist := store.NewAllocHistoryRepo(db)
|
||||||
|
if err := hist.EnsureSchema(ctx); err != nil {
|
||||||
|
return nil, fmt.Errorf("alloc history schema: %w", err)
|
||||||
|
}
|
||||||
|
entries, err := hist.List(ctx, store.HistoryFilter{JobID: jobID})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("alloc history list: %w", err)
|
||||||
|
}
|
||||||
|
if len(entries) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
// Pick the latest entry (List returns ASC; take the last).
|
||||||
|
latest := entries[len(entries)-1]
|
||||||
|
if latest.NodeID == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
nodeRepo := store.NewNodeRepo(db)
|
||||||
|
n, err := nodeRepo.Get(ctx, latest.NodeID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, store.ErrNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("lookup node %s: %w", latest.NodeID, err)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
var jobStopCmd = &cobra.Command{
|
var jobStopCmd = &cobra.Command{
|
||||||
Use: "stop [job-id]",
|
Use: "stop [job-id]",
|
||||||
Short: "Stop a running job",
|
Short: "Stop a running job",
|
||||||
Long: "Mark a job as stopped. Note: this is a soft stop (cancel context for the daemon).",
|
Long: `Stop a running job by sending 'systemctl stop orca-alloc-<name>-*'
|
||||||
|
to the node running the allocation via SSH, then mark the job as
|
||||||
|
stopped in the DB (REQ-158, P09 T1).
|
||||||
|
|
||||||
|
If --peer is not given, the node is looked up from the allocation
|
||||||
|
history. If no node is found, the DB status is updated anyway (soft
|
||||||
|
stop fallback for local-run jobs).`,
|
||||||
Args: cobra.MaximumNArgs(1),
|
Args: cobra.MaximumNArgs(1),
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
id := stopID
|
id := stopID
|
||||||
@@ -303,8 +394,6 @@ var jobStopCmd = &cobra.Command{
|
|||||||
if id == "" {
|
if id == "" {
|
||||||
return fmt.Errorf("job id required (--id or argument)")
|
return fmt.Errorf("job id required (--id or argument)")
|
||||||
}
|
}
|
||||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
db, closer, err := openDB()
|
db, closer, err := openDB()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -312,6 +401,9 @@ var jobStopCmd = &cobra.Command{
|
|||||||
}
|
}
|
||||||
defer closer()
|
defer closer()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
repo := store.NewJobRepo(db)
|
repo := store.NewJobRepo(db)
|
||||||
job, err := repo.Get(ctx, id)
|
job, err := repo.Get(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -320,6 +412,52 @@ var jobStopCmd = &cobra.Command{
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Determine the peer to SSH to. --peer takes precedence;
|
||||||
|
// otherwise look up the node from alloc_history.
|
||||||
|
peer := jobStopPeer
|
||||||
|
var node *model.Node
|
||||||
|
if peer == "" {
|
||||||
|
node, err = nodeForJob(ctx, db, id)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("lookup node for job %s: %w", id, err)
|
||||||
|
}
|
||||||
|
if node != nil {
|
||||||
|
peer = peerAddrForNode(node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate the job name before interpolation into the shell
|
||||||
|
// command (same injection guard as logs --job / stopAlloc).
|
||||||
|
jobName := job.Name
|
||||||
|
if !validSafeName(jobName) {
|
||||||
|
return fmt.Errorf("job stop: invalid job name %q (allowed: A-Z a-z 0-9 _ -)", jobName)
|
||||||
|
}
|
||||||
|
|
||||||
|
sshRan := false
|
||||||
|
if peer != "" {
|
||||||
|
transport, terr := jobStopTransportFromCtx()
|
||||||
|
if terr != nil {
|
||||||
|
return fmt.Errorf("job stop: ssh transport: %w", terr)
|
||||||
|
}
|
||||||
|
stopCtx, stopCancel := sshCmdCtx(ctx, jobStopTimeout)
|
||||||
|
defer stopCancel()
|
||||||
|
// Match the drift.go job restart unit pattern: orca-alloc-<name>.
|
||||||
|
// Use a glob (orca-alloc-<name>-*) to stop all task units in
|
||||||
|
// a multi-task allocation group.
|
||||||
|
unitPattern := fmt.Sprintf("orca-alloc-%s-*", jobName)
|
||||||
|
stopCmd := fmt.Sprintf("systemctl stop %s", shellQuote(unitPattern))
|
||||||
|
out, sErr := transport.Exec(stopCtx, peer, stopCmd)
|
||||||
|
if sErr != nil {
|
||||||
|
// Non-fatal: the unit may not be running (already
|
||||||
|
// stopped) or SSH may fail. We still update the DB
|
||||||
|
// status so the operator's intent is recorded.
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "⚠ job stop: SSH systemctl stop failed on %s: %v (output: %s)\n", peer, sErr, strings.TrimSpace(string(out)))
|
||||||
|
} else {
|
||||||
|
sshRan = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := repo.UpdateStatus(ctx, id, model.JobStatusStopped, 130); err != nil {
|
if err := repo.UpdateStatus(ctx, id, model.JobStatusStopped, 130); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -327,9 +465,20 @@ var jobStopCmd = &cobra.Command{
|
|||||||
// `orca job list` reflects the just-stopped job.
|
// `orca job list` reflects the just-stopped job.
|
||||||
cacheInvalidate(cacheJobClass)
|
cacheInvalidate(cacheJobClass)
|
||||||
if jsonOutput {
|
if jsonOutput {
|
||||||
return printJSON(map[string]any{"id": id, "status": "stopped", "previous_status": job.Status})
|
result := map[string]any{"id": id, "status": "stopped", "previous_status": job.Status}
|
||||||
|
if peer != "" {
|
||||||
|
result["peer"] = peer
|
||||||
|
result["ssh_stop"] = sshRan
|
||||||
|
}
|
||||||
|
return printJSON(result)
|
||||||
|
}
|
||||||
|
if peer != "" && sshRan {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s (systemctl stop on %s)\n", id, peer)
|
||||||
|
} else if peer != "" {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s (DB only; SSH stop failed — see stderr)\n", id)
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s (DB only; no node found)\n", id)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s\n", id)
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -383,6 +532,8 @@ var jobLogsCmd = &cobra.Command{
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
jobStopCmd.Flags().StringVar(&stopID, "id", "", "job id")
|
jobStopCmd.Flags().StringVar(&stopID, "id", "", "job id")
|
||||||
|
jobStopCmd.Flags().StringVar(&jobStopPeer, "peer", "", "peer address (host:port) running the allocation (auto-detected from alloc history if empty)")
|
||||||
|
jobStopCmd.Flags().DurationVar(&jobStopTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||||
jobLogsCmd.Flags().StringVar(&stopID, "id", "", "job id")
|
jobLogsCmd.Flags().StringVar(&stopID, "id", "", "job id")
|
||||||
jobRunCmd.Flags().StringVar(&runTarget, "target", "", "pin job to a specific node id (overrides bin-packing)")
|
jobRunCmd.Flags().StringVar(&runTarget, "target", "", "pin job to a specific node id (overrides bin-packing)")
|
||||||
jobRunCmd.Flags().StringVar(&runIDKey, "idempotency-key", "", "X-Orca-Idempotency-Key for cross-node dispatch dedupe")
|
jobRunCmd.Flags().StringVar(&runIDKey, "idempotency-key", "", "X-Orca-Idempotency-Key for cross-node dispatch dedupe")
|
||||||
@@ -446,3 +597,23 @@ func splitCommand(s string) (string, []string) {
|
|||||||
}
|
}
|
||||||
return parts[0], parts[1:]
|
return parts[0], parts[1:]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// insertRemoteJob inserts a model.Job row for a remotely-deployed job
|
||||||
|
// (REQ-166, Phase C2). Without this, `job list` shows nothing for remote
|
||||||
|
// deployments and `job stop` can't find the node.
|
||||||
|
func insertRemoteJob(spec *jobspec.WorkloadSpec, node string) error {
|
||||||
|
db, closer, err := openDB()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer closer()
|
||||||
|
repo := store.NewJobRepo(db)
|
||||||
|
return repo.Insert(context.Background(), &model.Job{
|
||||||
|
ID: uuid.NewString(),
|
||||||
|
Name: spec.Name,
|
||||||
|
Spec: "",
|
||||||
|
Status: model.JobStatusRunning,
|
||||||
|
CreatedAt: time.Now().UTC(),
|
||||||
|
Node: node,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
+101
-19
@@ -34,6 +34,7 @@ import (
|
|||||||
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/runtime"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/scheduler"
|
"git.cloudinit.dev/coreci/orca/internal/scheduler"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
@@ -199,30 +200,72 @@ func deployRemote(ctx context.Context, spec *jobspec.WorkloadSpec, res *dispatch
|
|||||||
return nil, fmt.Errorf("deployRemote: selected node %q not found in registry", res.node)
|
return nil, fmt.Errorf("deployRemote: selected node %q not found in registry", res.node)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Render the systemd unit via the emitter. The runtime is required
|
// REQ-166 / Phase C3: branch on runtime + node kind.
|
||||||
// for the process emitter; a spec with no runtime has nothing to
|
runtimeOneOf := ""
|
||||||
// ExecStart and is rejected by the emitter.
|
if spec.Runtime != nil {
|
||||||
|
runtimeOneOf = spec.Runtime.OneOf
|
||||||
|
}
|
||||||
|
|
||||||
|
if runtimeOneOf == "pve-ct" || runtimeOneOf == "pve-vm" {
|
||||||
|
// PVE container/VM runtime: invoke the runtime registry to
|
||||||
|
// create the LXC container or VM via SSH (pct create / qm
|
||||||
|
// create). Only valid on proxmox nodes.
|
||||||
|
if node.Kind != string(model.NodeKindProxmox) {
|
||||||
|
return nil, fmt.Errorf("deployRemote: runtime %q requires a proxmox node (node %q is %q)", runtimeOneOf, res.node, node.Kind)
|
||||||
|
}
|
||||||
|
peer := sshPeerFor(node)
|
||||||
|
sshTransport, err := newSSHPushTransport()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("deployRemote: transport: %w", err)
|
||||||
|
}
|
||||||
|
defer sshTransport.Close()
|
||||||
|
alloc := &runtime.Alloc{
|
||||||
|
ID: res.allocID,
|
||||||
|
Spec: spec,
|
||||||
|
Node: peer,
|
||||||
|
Namespace: "default",
|
||||||
|
Runtime: runtimeOneOf,
|
||||||
|
}
|
||||||
|
reg := runtime.DefaultRegistry(sshTransport)
|
||||||
|
if err := reg.Prepare(ctx, alloc); err != nil {
|
||||||
|
return nil, fmt.Errorf("deployRemote: pve prepare: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := reg.Start(ctx, alloc); err != nil {
|
||||||
|
return nil, fmt.Errorf("deployRemote: pve start: %w", err)
|
||||||
|
}
|
||||||
|
// For PVE workloads, also emit Traefik route if the spec has ports.
|
||||||
|
var written []string
|
||||||
|
if hasPorts(spec) {
|
||||||
|
traefikFiles, err := renderTraefik(spec, node)
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range traefikFiles {
|
||||||
|
mode := os.FileMode(0o644)
|
||||||
|
_ = sshTransport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode)
|
||||||
|
written = append(written, f.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process runtime: systemd units (only on linux/localhost nodes).
|
||||||
|
if node.Kind == string(model.NodeKindProxmox) {
|
||||||
|
return nil, fmt.Errorf("deployRemote: runtime %q requires a linux node (node %q is proxmox; use one_of: pve-ct or pve-vm for proxmox)", runtimeOneOf, res.node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render the systemd unit via the emitter.
|
||||||
em := emitter.SystemdEmitter{}
|
em := emitter.SystemdEmitter{}
|
||||||
enode := &emitter.Node{
|
enode := &emitter.Node{
|
||||||
Hostname: node.Name,
|
Hostname: node.Name,
|
||||||
Runtime: []string{"process"},
|
Runtime: []string{"process"},
|
||||||
Tags: nil,
|
Tags: nil,
|
||||||
}
|
}
|
||||||
// Advertise the node kind as a runtime so the emitter can branch
|
|
||||||
// (proxmox nodes expose pve-* runtimes). For process workloads
|
|
||||||
// this is informational.
|
|
||||||
if node.Kind == string(model.NodeKindProxmox) {
|
|
||||||
enode.Runtime = append(enode.Runtime, "proxmox")
|
|
||||||
}
|
|
||||||
files, err := em.Render(spec, enode)
|
files, err := em.Render(spec, enode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("deployRemote: render unit: %w", err)
|
return nil, fmt.Errorf("deployRemote: render unit: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// T9: systemd-analyze verify on the rendered unit before deploy.
|
// T9: systemd-analyze verify on the rendered unit before deploy.
|
||||||
// Run it locally (the unit is a portable text file); if
|
|
||||||
// systemd-analyze is not installed, skip silently (dev boxes
|
|
||||||
// without systemd). A verification FAILURE is an error.
|
|
||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
if err := verifySystemdUnit(ctx, f.Path, f.Content); err != nil {
|
if err := verifySystemdUnit(ctx, f.Path, f.Content); err != nil {
|
||||||
return nil, fmt.Errorf("deployRemote: systemd-analyze verify %s: %w", f.Path, err)
|
return nil, fmt.Errorf("deployRemote: systemd-analyze verify %s: %w", f.Path, err)
|
||||||
@@ -241,24 +284,18 @@ func deployRemote(ctx context.Context, spec *jobspec.WorkloadSpec, res *dispatch
|
|||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
mode := os.FileMode(0o644)
|
mode := os.FileMode(0o644)
|
||||||
if f.Mode != "" {
|
if f.Mode != "" {
|
||||||
// f.Mode is an octal string like "0644".
|
|
||||||
var m uint64
|
var m uint64
|
||||||
if _, perr := fmt.Sscanf(f.Mode, "%o", &m); perr == nil {
|
if _, perr := fmt.Sscanf(f.Mode, "%o", &m); perr == nil {
|
||||||
mode = os.FileMode(m)
|
mode = os.FileMode(m)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err := transport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode); err != nil {
|
if err := transport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode); err != nil {
|
||||||
// C-44: SSH-push failure -> error, NOT local fallback.
|
|
||||||
return nil, fmt.Errorf("deployRemote: push %s to %s (%s): %w", f.Path, res.node, peer, err)
|
return nil, fmt.Errorf("deployRemote: push %s to %s (%s): %w", f.Path, res.node, peer, err)
|
||||||
}
|
}
|
||||||
written = append(written, f.Path)
|
written = append(written, f.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reload systemd + enable the unit so it starts at boot. These are
|
// Reload systemd + enable the unit so it starts at boot.
|
||||||
// best-effort; a failure here is surfaced but does not undo the
|
|
||||||
// push (the unit is on disk). We use systemctl daemon-reload +
|
|
||||||
// enable --now for each .service unit (.target units for task
|
|
||||||
// groups are also enabled).
|
|
||||||
for _, p := range written {
|
for _, p := range written {
|
||||||
if !strings.HasSuffix(p, ".service") && !strings.HasSuffix(p, ".target") {
|
if !strings.HasSuffix(p, ".service") && !strings.HasSuffix(p, ".target") {
|
||||||
continue
|
continue
|
||||||
@@ -268,7 +305,44 @@ func deployRemote(ctx context.Context, spec *jobspec.WorkloadSpec, res *dispatch
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// REQ-166 / Phase C4: emit Traefik dynamic config if the spec
|
||||||
|
// has ports (is a Service with ingress).
|
||||||
|
if hasPorts(spec) {
|
||||||
|
traefikFiles, err := renderTraefik(spec, node)
|
||||||
|
if err != nil {
|
||||||
|
// Non-fatal: Traefik route is best-effort.
|
||||||
return written, nil
|
return written, nil
|
||||||
|
}
|
||||||
|
for _, f := range traefikFiles {
|
||||||
|
mode := os.FileMode(0o644)
|
||||||
|
_ = transport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode)
|
||||||
|
written = append(written, f.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return written, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasPorts returns true if the spec declares any ports (is a Service).
|
||||||
|
func hasPorts(spec *jobspec.WorkloadSpec) bool {
|
||||||
|
if spec == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if len(spec.Ports) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderTraefik renders the Traefik dynamic config for the spec + node.
|
||||||
|
func renderTraefik(spec *jobspec.WorkloadSpec, node *model.Node) ([]emitter.File, error) {
|
||||||
|
em := emitter.TraefikEmitter{}
|
||||||
|
enode := &emitter.Node{
|
||||||
|
Hostname: node.Name,
|
||||||
|
Runtime: []string{"process"},
|
||||||
|
Tags: nil,
|
||||||
|
}
|
||||||
|
return em.Render(spec, enode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// verifySystemdUnit runs `systemd-analyze verify` on the rendered unit
|
// verifySystemdUnit runs `systemd-analyze verify` on the rendered unit
|
||||||
@@ -433,3 +507,11 @@ func logDispatch(res *dispatchResult, err error) {
|
|||||||
}
|
}
|
||||||
log.Info("job.dispatch", attrs...)
|
log.Info("job.dispatch", attrs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newSSHPushTransport creates a concrete sshpush.Transport for PVE
|
||||||
|
// runtime operations (pct create/qm create). The jobDispatchTransport
|
||||||
|
// interface wraps sshpush.Transport but the runtime package needs the
|
||||||
|
// concrete type.
|
||||||
|
func newSSHPushTransport() (*sshpush.Transport, error) {
|
||||||
|
return sshpush.NewTransport(certpaths.SSHKeyPath(), certpaths.KnownHostsPath()), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,11 +2,14 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
@@ -310,3 +313,193 @@ func seedJob(t *testing.T, name string, status model.JobStatus) string {
|
|||||||
}
|
}
|
||||||
return j.ID
|
return j.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// mockJobStopExec is a record-and-replay SSH execer for `orca job stop`
|
||||||
|
// tests (same pattern as mockDrainExec / mockLogsExec).
|
||||||
|
type mockJobStopExec struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
responses []jobStopMockResp
|
||||||
|
calls []jobStopMockCall
|
||||||
|
}
|
||||||
|
|
||||||
|
type jobStopMockResp struct {
|
||||||
|
match string
|
||||||
|
out string
|
||||||
|
exit int
|
||||||
|
}
|
||||||
|
|
||||||
|
type jobStopMockCall struct {
|
||||||
|
peer string
|
||||||
|
cmd string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockJobStopExec) Exec(_ context.Context, peer, cmd string) ([]byte, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.calls = append(m.calls, jobStopMockCall{peer: peer, cmd: cmd})
|
||||||
|
for _, r := range m.responses {
|
||||||
|
if r.match == "" || strings.Contains(cmd, r.match) {
|
||||||
|
return []byte(r.out), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockJobStopExec) callsFor(match string) []jobStopMockCall {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
var out []jobStopMockCall
|
||||||
|
for _, c := range m.calls {
|
||||||
|
if strings.Contains(c.cmd, match) {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestJobStopSSH verifies that `orca job stop` sends a real
|
||||||
|
// 'systemctl stop' via SSH to the target node when the job has a
|
||||||
|
// recorded allocation history (REQ-158, P09 T6).
|
||||||
|
func TestJobStopSSH(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
if err := runInit(discardWriter{}); err != nil {
|
||||||
|
t.Fatalf("init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed a node and a job, then record an alloc_history entry
|
||||||
|
// linking the job to the node.
|
||||||
|
nodeID := seedNode(t, "worker-1", "worker-1:8443")
|
||||||
|
jobID := seedJob(t, "webapp", model.JobStatusRunning)
|
||||||
|
|
||||||
|
db, err := store.Open(certpaths.DBPath())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open db: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
hist := store.NewAllocHistoryRepo(db)
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := hist.EnsureSchema(ctx); err != nil {
|
||||||
|
t.Fatalf("ensure schema: %v", err)
|
||||||
|
}
|
||||||
|
if err := hist.Record(ctx, store.AllocHistoryEntry{
|
||||||
|
AllocID: "default/webapp-0",
|
||||||
|
JobID: jobID,
|
||||||
|
NodeID: nodeID,
|
||||||
|
Namespace: "default",
|
||||||
|
ToState: "created",
|
||||||
|
Timestamp: time.Now().UTC(),
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("record alloc history: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wire the mock SSH transport (must be after resetRootFlags so
|
||||||
|
// resetCommandFlags doesn't nil it out).
|
||||||
|
resetRootFlags(t)
|
||||||
|
mock := &mockJobStopExec{}
|
||||||
|
prev := jobStopTransportOverride
|
||||||
|
jobStopTransportOverride = mock
|
||||||
|
t.Cleanup(func() { jobStopTransportOverride = prev })
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
rootCmd.SetArgs([]string{"job", "stop", jobID})
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
t.Fatalf("job stop: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify systemctl stop was called via SSH.
|
||||||
|
stopCalls := mock.callsFor("systemctl stop")
|
||||||
|
if len(stopCalls) == 0 {
|
||||||
|
t.Fatalf("expected systemctl stop SSH call, got %d calls: %v", len(mock.calls), mock.calls)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stopCalls[0].cmd, "orca-alloc-webapp-*") {
|
||||||
|
t.Errorf("expected 'orca-alloc-webapp-*' in cmd, got: %s", stopCalls[0].cmd)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stopCalls[0].peer, "worker-1") {
|
||||||
|
t.Errorf("expected peer to contain 'worker-1', got: %s", stopCalls[0].peer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the DB status was updated.
|
||||||
|
repo := store.NewJobRepo(db)
|
||||||
|
job, err := repo.Get(ctx, jobID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("get job: %v", err)
|
||||||
|
}
|
||||||
|
if job.Status != model.JobStatusStopped {
|
||||||
|
t.Errorf("job status = %v, want stopped", job.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestJobStopSSHPeerOverride verifies that --peer bypasses the
|
||||||
|
// alloc_history lookup and uses the given peer directly (REQ-158).
|
||||||
|
func TestJobStopSSHPeerOverride(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
if err := runInit(discardWriter{}); err != nil {
|
||||||
|
t.Fatalf("init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobID := seedJob(t, "webapp2", model.JobStatusRunning)
|
||||||
|
|
||||||
|
resetRootFlags(t)
|
||||||
|
mock := &mockJobStopExec{}
|
||||||
|
prev := jobStopTransportOverride
|
||||||
|
jobStopTransportOverride = mock
|
||||||
|
t.Cleanup(func() { jobStopTransportOverride = prev })
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
rootCmd.SetArgs([]string{"job", "stop", jobID, "--peer", "10.0.0.5:22"})
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
t.Fatalf("job stop: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stopCalls := mock.callsFor("systemctl stop")
|
||||||
|
if len(stopCalls) == 0 {
|
||||||
|
t.Fatalf("expected systemctl stop SSH call, got %d calls", len(mock.calls))
|
||||||
|
}
|
||||||
|
if stopCalls[0].peer != "10.0.0.5:22" {
|
||||||
|
t.Errorf("peer = %s, want 10.0.0.5:22", stopCalls[0].peer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestJobStopNoNodeFallback verifies that when no node is found in
|
||||||
|
// alloc_history, the job is still stopped in the DB (soft stop
|
||||||
|
// fallback) without attempting SSH (REQ-158).
|
||||||
|
func TestJobStopNoNodeFallback(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
if err := runInit(discardWriter{}); err != nil {
|
||||||
|
t.Fatalf("init: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobID := seedJob(t, "localjob", model.JobStatusRunning)
|
||||||
|
|
||||||
|
resetRootFlags(t)
|
||||||
|
mock := &mockJobStopExec{}
|
||||||
|
prev := jobStopTransportOverride
|
||||||
|
jobStopTransportOverride = mock
|
||||||
|
t.Cleanup(func() { jobStopTransportOverride = prev })
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
rootCmd.SetArgs([]string{"job", "stop", jobID})
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
t.Fatalf("job stop: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// No SSH calls should have been made (no node found).
|
||||||
|
if len(mock.calls) > 0 {
|
||||||
|
t.Errorf("expected 0 SSH calls, got %d: %v", len(mock.calls), mock.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the output mentions "DB only".
|
||||||
|
if !strings.Contains(buf.String(), "DB only") {
|
||||||
|
t.Errorf("output should mention 'DB only', got: %s", buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+43
-6
@@ -101,8 +101,20 @@ var (
|
|||||||
logsJob string
|
logsJob string
|
||||||
logsSince string
|
logsSince string
|
||||||
logsJSON bool
|
logsJSON bool
|
||||||
|
logsLines int
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// logsMaxLines is the hard cap on --lines to prevent OOM from
|
||||||
|
// unbounded journalctl output (REQ-158, P09 T3).
|
||||||
|
const logsMaxLines = 50000
|
||||||
|
|
||||||
|
// logsDefaultLines is the default --lines value.
|
||||||
|
const logsDefaultLines = 1000
|
||||||
|
|
||||||
|
// logsMaxSince is the maximum lookback for --since (7 days) to
|
||||||
|
// prevent OOM from unbounded journalctl queries (REQ-158, P09 T3).
|
||||||
|
const logsMaxSince = 7 * 24 * time.Hour
|
||||||
|
|
||||||
var logsCmd = &cobra.Command{
|
var logsCmd = &cobra.Command{
|
||||||
Use: "logs",
|
Use: "logs",
|
||||||
Short: "Aggregate journald logs across nodes (REQ-117)",
|
Short: "Aggregate journald logs across nodes (REQ-117)",
|
||||||
@@ -139,6 +151,25 @@ Ctrl-C cancels the fan-out via signal.NotifyContext.`,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// REQ-158 / P09 T3: clamp --since to 7 days max to prevent
|
||||||
|
// OOM from unbounded journalctl queries. If the requested
|
||||||
|
// lookback exceeds the cap, clamp it and warn.
|
||||||
|
now := time.Now().UTC()
|
||||||
|
maxSince := now.Add(-logsMaxSince)
|
||||||
|
if since.Before(maxSince) {
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "⚠ --since %s exceeds 7d cap; clamping to 7d\n", logsSince)
|
||||||
|
since = maxSince
|
||||||
|
}
|
||||||
|
|
||||||
|
// REQ-158 / P09 T3: clamp --lines to [1, logsMaxLines].
|
||||||
|
lines := logsLines
|
||||||
|
if lines <= 0 {
|
||||||
|
lines = logsDefaultLines
|
||||||
|
}
|
||||||
|
if lines > logsMaxLines {
|
||||||
|
fmt.Fprintf(cmd.ErrOrStderr(), "⚠ --lines %d exceeds max %d; clamping\n", lines, logsMaxLines)
|
||||||
|
lines = logsMaxLines
|
||||||
|
}
|
||||||
|
|
||||||
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM)
|
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -158,7 +189,7 @@ Ctrl-C cancels the fan-out via signal.NotifyContext.`,
|
|||||||
|
|
||||||
out := cmd.OutOrStdout()
|
out := cmd.OutOrStdout()
|
||||||
multi := len(nodes) > 1
|
multi := len(nodes) > 1
|
||||||
for line := range streamLogs(ctx, ex, nodes, since, logsJob) {
|
for line := range streamLogs(ctx, ex, nodes, since, logsJob, lines) {
|
||||||
if logsJSON {
|
if logsJSON {
|
||||||
raw, _ := json.Marshal(line)
|
raw, _ := json.Marshal(line)
|
||||||
fmt.Fprintln(out, string(raw))
|
fmt.Fprintln(out, string(raw))
|
||||||
@@ -227,7 +258,7 @@ func resolveLogNodes(ctx context.Context) ([]*model.Node, error) {
|
|||||||
// JSON entry immediately. The stream ends when every node has
|
// JSON entry immediately. The stream ends when every node has
|
||||||
// completed (or the context is cancelled). The caller drives the
|
// completed (or the context is cancelled). The caller drives the
|
||||||
// iteration via range-over-func (D-017 iter.Seq pattern).
|
// iteration via range-over-func (D-017 iter.Seq pattern).
|
||||||
func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since time.Time, job string) iter.Seq[LogLine] {
|
func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since time.Time, job string, lines int) iter.Seq[LogLine] {
|
||||||
return func(yield func(LogLine) bool) {
|
return func(yield func(LogLine) bool) {
|
||||||
merged := make(chan LogLine)
|
merged := make(chan LogLine)
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -235,7 +266,7 @@ func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since t
|
|||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func(n *model.Node) {
|
go func(n *model.Node) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
streamNodeLines(ctx, ex, n, since, job, merged)
|
streamNodeLines(ctx, ex, n, since, job, lines, merged)
|
||||||
}(n)
|
}(n)
|
||||||
}
|
}
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
@@ -267,7 +298,7 @@ func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since t
|
|||||||
// context is cancelled); the caller is responsible for waiting on the
|
// context is cancelled); the caller is responsible for waiting on the
|
||||||
// goroutine. Send is non-blocking via select on ctx.Done so a slow
|
// goroutine. Send is non-blocking via select on ctx.Done so a slow
|
||||||
// consumer does not stall the fanout forever.
|
// consumer does not stall the fanout forever.
|
||||||
func streamNodeLines(ctx context.Context, ex logsExecer, n *model.Node, since time.Time, job string, out chan<- LogLine) {
|
func streamNodeLines(ctx context.Context, ex logsExecer, n *model.Node, since time.Time, job string, lines int, out chan<- LogLine) {
|
||||||
peer := peerAddrForNode(n)
|
peer := peerAddrForNode(n)
|
||||||
if peer == "" {
|
if peer == "" {
|
||||||
slog.Default().Warn("logs: cannot resolve SSH address for node", "node", n.Name)
|
slog.Default().Warn("logs: cannot resolve SSH address for node", "node", n.Name)
|
||||||
@@ -278,9 +309,14 @@ func streamNodeLines(ctx context.Context, ex logsExecer, n *model.Node, since ti
|
|||||||
unitPattern = "orca-alloc-" + job + "-*"
|
unitPattern = "orca-alloc-" + job + "-*"
|
||||||
}
|
}
|
||||||
sinceStr := since.Format("2006-01-02 15:04:05")
|
sinceStr := since.Format("2006-01-02 15:04:05")
|
||||||
|
// REQ-158 / P09 T3: pass --lines=N to journalctl to cap output
|
||||||
|
// and prevent OOM from unbounded log queries.
|
||||||
|
if lines <= 0 {
|
||||||
|
lines = logsDefaultLines
|
||||||
|
}
|
||||||
// F1: shellQuote (single-quote wrap) instead of %q — %q does not
|
// F1: shellQuote (single-quote wrap) instead of %q — %q does not
|
||||||
// escape backticks, enabling command substitution in double quotes.
|
// escape backticks, enabling command substitution in double quotes.
|
||||||
cmd := fmt.Sprintf("journalctl -u %s --since %s --output json --no-pager", shellQuote(unitPattern), shellQuote(sinceStr))
|
cmd := fmt.Sprintf("journalctl -u %s --since %s --lines %d --output json --no-pager", shellQuote(unitPattern), shellQuote(sinceStr), lines)
|
||||||
raw, err := ex.Exec(ctx, peer, cmd)
|
raw, err := ex.Exec(ctx, peer, cmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Default().Warn("logs: exec failed", "node", n.Name, "peer", peer, "error", err)
|
slog.Default().Warn("logs: exec failed", "node", n.Name, "peer", peer, "error", err)
|
||||||
@@ -324,7 +360,8 @@ func init() {
|
|||||||
logsCmd.Flags().BoolVar(&logsAllNodes, "all-nodes", false, "fan out to all registered nodes")
|
logsCmd.Flags().BoolVar(&logsAllNodes, "all-nodes", false, "fan out to all registered nodes")
|
||||||
logsCmd.Flags().StringVar(&logsNode, "node", "", "restrict to a single node (name or id)")
|
logsCmd.Flags().StringVar(&logsNode, "node", "", "restrict to a single node (name or id)")
|
||||||
logsCmd.Flags().StringVar(&logsJob, "job", "", "filter by job name (matches orca-alloc-<name>-* units)")
|
logsCmd.Flags().StringVar(&logsJob, "job", "", "filter by job name (matches orca-alloc-<name>-* units)")
|
||||||
logsCmd.Flags().StringVar(&logsSince, "since", "5m", "duration lookback (e.g. 5m, 1h, 30m); default 5m")
|
logsCmd.Flags().StringVar(&logsSince, "since", "5m", "duration lookback (e.g. 5m, 1h, 30m); default 5m; max 7d")
|
||||||
|
logsCmd.Flags().IntVar(&logsLines, "lines", logsDefaultLines, fmt.Sprintf("max number of journal lines per node (default %d, max %d)", logsDefaultLines, logsMaxLines))
|
||||||
logsCmd.Flags().BoolVar(&logsJSON, "json", false, "output raw JSON (one LogLine per line)")
|
logsCmd.Flags().BoolVar(&logsJSON, "json", false, "output raw JSON (one LogLine per line)")
|
||||||
rootCmd.AddCommand(logsCmd)
|
rootCmd.AddCommand(logsCmd)
|
||||||
}
|
}
|
||||||
|
|||||||
+156
-1
@@ -63,6 +63,18 @@ func (m *mockLogsExec) countCalls(match string) int {
|
|||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *mockLogsExec) callsFor(match string) []logsMockCall {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
var out []logsMockCall
|
||||||
|
for _, c := range m.calls {
|
||||||
|
if strings.Contains(c.cmd, match) {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// logsTestEnv wires a mockLogsExec into logsExecOverride and returns
|
// logsTestEnv wires a mockLogsExec into logsExecOverride and returns
|
||||||
// the mock + a cleanup func. Tests MUST defer the cleanup.
|
// the mock + a cleanup func. Tests MUST defer the cleanup.
|
||||||
func logsTestEnv(t *testing.T) *mockLogsExec {
|
func logsTestEnv(t *testing.T) *mockLogsExec {
|
||||||
@@ -325,7 +337,7 @@ func TestLogsCancelStopsStream(t *testing.T) {
|
|||||||
{ID: "n1", Name: "cancelnode", Address: "cancelnode:8443"},
|
{ID: "n1", Name: "cancelnode", Address: "cancelnode:8443"},
|
||||||
}
|
}
|
||||||
consumed := 0
|
consumed := 0
|
||||||
for range streamLogs(ctx, ex, nodes, time.Now().UTC().Add(-1*time.Minute), "") {
|
for range streamLogs(ctx, ex, nodes, time.Now().UTC().Add(-1*time.Minute), "", 1000) {
|
||||||
consumed++
|
consumed++
|
||||||
}
|
}
|
||||||
if consumed > 1 {
|
if consumed > 1 {
|
||||||
@@ -357,3 +369,146 @@ func TestLogsParseJournalLine_InvalidJSON(t *testing.T) {
|
|||||||
t.Error("expected error for invalid json, got nil")
|
t.Error("expected error for invalid json, got nil")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// TestLogsLinesFlag verifies that --lines is passed through to the
|
||||||
|
// journalctl command as --lines=N (REQ-158, P09 T8).
|
||||||
|
func TestLogsLinesFlag(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
logsNodeForTest(t, "linesnode", "linesnode:8443")
|
||||||
|
mx := logsTestEnv(t)
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
mx.responses = []logsMockResp{
|
||||||
|
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "line test", "6") + "\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := runLogsCmd(t, []string{"logs", "--node", "linesnode", "--since", "1m", "--lines", "500"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("logs: %v", err)
|
||||||
|
}
|
||||||
|
calls := mx.callsFor("journalctl")
|
||||||
|
if len(calls) == 0 {
|
||||||
|
t.Fatal("expected journalctl call")
|
||||||
|
}
|
||||||
|
if !strings.Contains(calls[0].cmd, "--lines 500") {
|
||||||
|
t.Errorf("expected '--lines 500' in cmd, got: %s", calls[0].cmd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogsLinesDefault verifies that the default --lines value (1000)
|
||||||
|
// is passed to journalctl when --lines is not specified (REQ-158).
|
||||||
|
func TestLogsLinesDefault(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
logsNodeForTest(t, "defnode", "defnode:8443")
|
||||||
|
mx := logsTestEnv(t)
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
mx.responses = []logsMockResp{
|
||||||
|
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "default lines", "6") + "\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := runLogsCmd(t, []string{"logs", "--node", "defnode", "--since", "1m"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("logs: %v", err)
|
||||||
|
}
|
||||||
|
calls := mx.callsFor("journalctl")
|
||||||
|
if len(calls) == 0 {
|
||||||
|
t.Fatal("expected journalctl call")
|
||||||
|
}
|
||||||
|
if !strings.Contains(calls[0].cmd, "--lines 1000") {
|
||||||
|
t.Errorf("expected default '--lines 1000' in cmd, got: %s", calls[0].cmd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogsLinesClamp verifies that --lines exceeding the max (50000) is
|
||||||
|
// clamped (REQ-158, P09 T8).
|
||||||
|
func TestLogsLinesClamp(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
logsNodeForTest(t, "clampnode", "clampnode:8443")
|
||||||
|
mx := logsTestEnv(t)
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
mx.responses = []logsMockResp{
|
||||||
|
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "clamp test", "6") + "\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := runLogsCmd(t, []string{"logs", "--node", "clampnode", "--since", "1m", "--lines", "999999"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("logs: %v", err)
|
||||||
|
}
|
||||||
|
calls := mx.callsFor("journalctl")
|
||||||
|
if len(calls) == 0 {
|
||||||
|
t.Fatal("expected journalctl call")
|
||||||
|
}
|
||||||
|
if !strings.Contains(calls[0].cmd, "--lines 50000") {
|
||||||
|
t.Errorf("expected clamped '--lines 50000' in cmd, got: %s", calls[0].cmd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogsSinceClamp verifies that --since exceeding 7 days is
|
||||||
|
// clamped and a warning is printed (REQ-158, P09 T8).
|
||||||
|
func TestLogsSinceClamp(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
logsNodeForTest(t, "sincenode", "sincenode:8443")
|
||||||
|
mx := logsTestEnv(t)
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
mx.responses = []logsMockResp{
|
||||||
|
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "since test", "6") + "\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
resetRootFlags(t)
|
||||||
|
rootCmd.SetOut(&buf)
|
||||||
|
rootCmd.SetErr(&buf)
|
||||||
|
rootCmd.SetArgs([]string{"logs", "--node", "sincenode", "--since", "720h"})
|
||||||
|
err := rootCmd.Execute()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("logs: %v", err)
|
||||||
|
}
|
||||||
|
out := buf.String()
|
||||||
|
if !strings.Contains(out, "clamping to 7d") {
|
||||||
|
t.Errorf("expected warning about clamping --since to 7d, got: %s", out)
|
||||||
|
}
|
||||||
|
calls := mx.callsFor("journalctl")
|
||||||
|
if len(calls) == 0 {
|
||||||
|
t.Fatal("expected journalctl call")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLogsLinesFlagJSON verifies --lines is passed through in JSON mode.
|
||||||
|
func TestLogsLinesFlagJSON(t *testing.T) {
|
||||||
|
_, cleanup := initTestEnv(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
logsNodeForTest(t, "jsonlines", "jsonlines:8443")
|
||||||
|
mx := logsTestEnv(t)
|
||||||
|
|
||||||
|
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
mx.responses = []logsMockResp{
|
||||||
|
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "json lines test", "6") + "\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := runLogsCmd(t, []string{"logs", "--node", "jsonlines", "--since", "1m", "--lines", "200", "--json"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("logs: %v", err)
|
||||||
|
}
|
||||||
|
calls := mx.callsFor("journalctl")
|
||||||
|
if len(calls) == 0 {
|
||||||
|
t.Fatal("expected journalctl call")
|
||||||
|
}
|
||||||
|
if !strings.Contains(calls[0].cmd, "--lines 200") {
|
||||||
|
t.Errorf("expected '--lines 200' in cmd, got: %s", calls[0].cmd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/transport"
|
"git.cloudinit.dev/coreci/orca/internal/transport"
|
||||||
)
|
)
|
||||||
@@ -54,12 +55,16 @@ updates gauges. No orca daemon required (R-001).`,
|
|||||||
|
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||||
|
w.Header().Set("X-Frame-Options", "DENY")
|
||||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||||
if err := m.WritePrometheus(w); err != nil {
|
if err := m.WritePrometheus(w); err != nil {
|
||||||
log.Warn("metrics: write exposition failed", "err", err)
|
log.Warn("metrics: write exposition failed", "err", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||||
|
w.Header().Set("X-Frame-Options", "DENY")
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_, _ = w.Write([]byte("ok\n"))
|
_, _ = w.Write([]byte("ok\n"))
|
||||||
})
|
})
|
||||||
@@ -124,6 +129,22 @@ func refresh(ctx context.Context, m *transport.Metrics, db *sql.DB, log interfac
|
|||||||
log.Warn("metrics: job list failed", "err", err)
|
log.Warn("metrics: job list failed", "err", err)
|
||||||
} else {
|
} else {
|
||||||
m.SetGauge("allocs_total", float64(len(jobs)))
|
m.SetGauge("allocs_total", float64(len(jobs)))
|
||||||
|
// REQ-159 / P10: jobs by state.
|
||||||
|
byState := make(map[model.JobStatus]int, 8)
|
||||||
|
for _, j := range jobs {
|
||||||
|
byState[j.Status]++
|
||||||
|
}
|
||||||
|
// Set total + per-state counts using simple gauge names.
|
||||||
|
running := byState[model.JobStatusRunning]
|
||||||
|
failed := byState[model.JobStatusFailed]
|
||||||
|
complete := byState[model.JobStatusComplete]
|
||||||
|
m.SetGauge("orca_jobs_running", float64(running))
|
||||||
|
m.SetGauge("orca_jobs_failed", float64(failed))
|
||||||
|
m.SetGauge("orca_jobs_complete", float64(complete))
|
||||||
|
}
|
||||||
|
// REQ-159 / P10: audit chain head gauge.
|
||||||
|
if head, err := store.NewAuditRepo(db).ChainHead(ctx); err == nil && head != "" {
|
||||||
|
m.SetGauge("orca_audit_chain_head", 1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+59
-101
@@ -1,116 +1,74 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"io"
|
"os"
|
||||||
"net"
|
"path/filepath"
|
||||||
"net/http"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/transport"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMetricsCmdRegistered(t *testing.T) {
|
// TestREQ159_MetricsExpanded verifies the expanded metric set (P10, REQ-159).
|
||||||
found := false
|
func TestREQ159_MetricsExpanded(t *testing.T) {
|
||||||
for _, c := range rootCmd.Commands() {
|
dir := t.TempDir()
|
||||||
if c.Name() == "metrics" {
|
t.Setenv("ORCA_HOME", dir)
|
||||||
found = true
|
dbPath := filepath.Join(dir, "orca.db")
|
||||||
break
|
db, err := store.Open(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("store.Open: %v", err)
|
||||||
}
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
// Seed a node.
|
||||||
|
repo := store.NewNodeRepo(db)
|
||||||
|
if err := repo.Insert(context.Background(), &model.Node{
|
||||||
|
ID: "test-node-1",
|
||||||
|
Name: "test-node",
|
||||||
|
Address: "localhost:8443",
|
||||||
|
Kind: "localhost",
|
||||||
|
OS: "linux",
|
||||||
|
State: "ready",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("insert node: %v", err)
|
||||||
}
|
}
|
||||||
if !found {
|
|
||||||
t.Fatal("metricsCmd not registered on root")
|
// Seed a job.
|
||||||
|
jobRepo := store.NewJobRepo(db)
|
||||||
|
if err := jobRepo.Insert(context.Background(), &model.Job{
|
||||||
|
ID: "job-1",
|
||||||
|
Name: "test-job",
|
||||||
|
Status: "running",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("insert job: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m := transport.NewMetrics()
|
||||||
|
logger := slogLogger{}
|
||||||
|
refresh(context.Background(), m, db, logger)
|
||||||
|
|
||||||
|
// Verify expanded metrics by reading the exposition output.
|
||||||
|
var buf strings.Builder
|
||||||
|
if err := m.WritePrometheus(&buf); err != nil {
|
||||||
|
t.Fatalf("WritePrometheus: %v", err)
|
||||||
|
}
|
||||||
|
out := buf.String()
|
||||||
|
if !strings.Contains(out, "nodes_total 1") {
|
||||||
|
t.Errorf("output missing nodes_total 1:\n%s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "allocs_total 1") {
|
||||||
|
t.Errorf("output missing allocs_total 1:\n%s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "orca_jobs_running") {
|
||||||
|
t.Errorf("output missing orca_jobs_running:\n%s", out)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMetricsAddrFlagDefault(t *testing.T) {
|
type slogLogger struct{}
|
||||||
f := metricsCmd.Flags().Lookup("addr")
|
|
||||||
if f == nil {
|
|
||||||
t.Fatal("--addr flag not registered on metricsCmd")
|
|
||||||
}
|
|
||||||
if f.DefValue != ":9100" {
|
|
||||||
t.Errorf("--addr default = %q, want %q", f.DefValue, ":9100")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMetricsEndpoints(t *testing.T) {
|
func (slogLogger) Warn(msg string, args ...any) {}
|
||||||
_, cleanup := initTestEnv(t)
|
|
||||||
defer cleanup()
|
|
||||||
resetRootFlags(t)
|
|
||||||
|
|
||||||
// Pick a free port by briefly listening then closing.
|
var _ = os.Stdin
|
||||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("listen probe: %v", err)
|
|
||||||
}
|
|
||||||
addr := ln.Addr().String()
|
|
||||||
_ = ln.Close()
|
|
||||||
|
|
||||||
metricsAddr = addr
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
cmd := metricsCmd
|
|
||||||
var out bytes.Buffer
|
|
||||||
cmd.SetOut(&out)
|
|
||||||
cmd.SetErr(&out)
|
|
||||||
cmd.SetContext(ctx)
|
|
||||||
|
|
||||||
errCh := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
errCh <- cmd.RunE(cmd, nil)
|
|
||||||
}()
|
|
||||||
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
var resp *http.Response
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
resp, err = http.Get("http://" + addr + "/healthz")
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET /healthz: %v", err)
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Errorf("/healthz status = %d, want 200", resp.StatusCode)
|
|
||||||
}
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
|
||||||
resp.Body.Close()
|
|
||||||
if !strings.HasPrefix(string(body), "ok") {
|
|
||||||
t.Errorf("/healthz body = %q, want \"ok\"", string(body))
|
|
||||||
}
|
|
||||||
|
|
||||||
resp2, err := http.Get("http://" + addr + "/metrics")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GET /metrics: %v", err)
|
|
||||||
}
|
|
||||||
defer resp2.Body.Close()
|
|
||||||
if resp2.StatusCode != http.StatusOK {
|
|
||||||
t.Errorf("/metrics status = %d, want 200", resp2.StatusCode)
|
|
||||||
}
|
|
||||||
mbody, _ := io.ReadAll(resp2.Body)
|
|
||||||
ms := string(mbody)
|
|
||||||
for _, name := range []string{
|
|
||||||
"txns_applied_total",
|
|
||||||
"txns_drifted_total",
|
|
||||||
"drifts_remediated_total",
|
|
||||||
"peers_total",
|
|
||||||
"nodes_total",
|
|
||||||
"allocs_total",
|
|
||||||
} {
|
|
||||||
if !strings.Contains(ms, name) {
|
|
||||||
t.Errorf("/metrics missing %q\n---\n%s", name, ms)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case <-errCh:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("metrics command did not stop after cancel")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -57,6 +57,11 @@ func resetCommandFlags() {
|
|||||||
driftConfigPath = ""
|
driftConfigPath = ""
|
||||||
driftRemediateForce = false
|
driftRemediateForce = false
|
||||||
jobRestartPeer = ""
|
jobRestartPeer = ""
|
||||||
|
jobStopPeer = ""
|
||||||
|
jobStopTimeout = 0
|
||||||
|
jobStopTransportOverride = nil
|
||||||
|
logsLines = logsDefaultLines
|
||||||
|
cutoverFSOverride = nil
|
||||||
jobLintExplain = false
|
jobLintExplain = false
|
||||||
jobLintFormat = "text"
|
jobLintFormat = "text"
|
||||||
jobVerifyLead = ""
|
jobVerifyLead = ""
|
||||||
|
|||||||
+259
-5
@@ -1,21 +1,26 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/engine"
|
"git.cloudinit.dev/coreci/orca/internal/engine"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/linux"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||||
@@ -54,8 +59,14 @@ var (
|
|||||||
joinSSHKey string
|
joinSSHKey string
|
||||||
joinSSHPort int
|
joinSSHPort int
|
||||||
joinHostKeyFP string
|
joinHostKeyFP string
|
||||||
|
joinLXCTemplate string
|
||||||
proxmoxUser string
|
proxmoxUser string
|
||||||
proxmoxRole string
|
proxmoxRole string
|
||||||
|
ingressMode string
|
||||||
|
floatingIP string
|
||||||
|
gateway string
|
||||||
|
macAddr string
|
||||||
|
netPrefix int
|
||||||
leaveID string
|
leaveID string
|
||||||
nodeWatch bool
|
nodeWatch bool
|
||||||
)
|
)
|
||||||
@@ -73,16 +84,22 @@ var nodeJoinCmd = &cobra.Command{
|
|||||||
|
|
||||||
Node types (via --type):
|
Node types (via --type):
|
||||||
localhost (default): register a local or Linux node (existing behavior)
|
localhost (default): register a local or Linux node (existing behavior)
|
||||||
|
linux: SSH-bootstrap a remote generic Linux worker
|
||||||
|
(Ubuntu/Debian/Alpine; deploys orca pubkey, creates orca
|
||||||
|
user + drift-events dir; requires --host + --ssh-key)
|
||||||
proxmox: SSH-bootstrap a remote Proxmox VE 8/9 host
|
proxmox: SSH-bootstrap a remote Proxmox VE 8/9 host
|
||||||
(deploys orca pubkey, creates orca user + PVE role +
|
(deploys orca pubkey, creates orca user + PVE role +
|
||||||
sudoers allowlist; requires --host + --ssh-key (R-021: no passwords))`,
|
sudoers allowlist; requires --host + --ssh-key (R-021: no passwords))`,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
if joinHostKeyFP != "" && joinType != "proxmox" {
|
if joinHostKeyFP != "" && joinType != "proxmox" && joinType != "linux" {
|
||||||
return fmt.Errorf("--host-key-fingerprint requires --type proxmox today")
|
return fmt.Errorf("--host-key-fingerprint requires --type proxmox or --type linux")
|
||||||
}
|
}
|
||||||
if joinType == "proxmox" {
|
if joinType == "proxmox" {
|
||||||
return joinProxmox(cmd)
|
return joinProxmox(cmd)
|
||||||
}
|
}
|
||||||
|
if joinType == "linux" {
|
||||||
|
return joinLinux(cmd)
|
||||||
|
}
|
||||||
return joinLocal(cmd)
|
return joinLocal(cmd)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -167,7 +184,83 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
return fmt.Errorf("SSH key path is required for --type proxmox (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
return fmt.Errorf("SSH key path is required for --type proxmox (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
// Default ingress mode to "native" if not specified (R-024).
|
||||||
|
effectiveIngressMode := ingressMode
|
||||||
|
if effectiveIngressMode == "" {
|
||||||
|
if !jsonOutput {
|
||||||
|
// Interactive mode: prompt for ingress mode.
|
||||||
|
fmt.Fprint(cmd.OutOrStdout(), "Ingress mode [native/floating-ip] (default native): ")
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
input := strings.TrimSpace(scanner.Text())
|
||||||
|
if input == "floating-ip" {
|
||||||
|
effectiveIngressMode = "floating-ip"
|
||||||
|
} else {
|
||||||
|
effectiveIngressMode = "native"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
effectiveIngressMode = "native"
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
effectiveIngressMode = "native"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Floating-IP mode: prompt for params if not provided.
|
||||||
|
effectiveFloatingIP := floatingIP
|
||||||
|
effectiveGateway := gateway
|
||||||
|
effectiveMAC := macAddr
|
||||||
|
if effectiveIngressMode == "floating-ip" {
|
||||||
|
if effectiveFloatingIP == "" && !jsonOutput {
|
||||||
|
fmt.Fprint(cmd.OutOrStdout(), "Floating IP: ")
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
effectiveFloatingIP = strings.TrimSpace(scanner.Text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if effectiveGateway == "" && !jsonOutput {
|
||||||
|
fmt.Fprint(cmd.OutOrStdout(), "Gateway: ")
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
effectiveGateway = strings.TrimSpace(scanner.Text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if effectiveMAC == "" && !jsonOutput {
|
||||||
|
// D-261: auto-generate a random locally-administered MAC.
|
||||||
|
generated, err := proxmox.GenerateRandomMAC()
|
||||||
|
if err == nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Generated MAC: %s (press enter to accept, or type your own): ", generated)
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
if scanner.Scan() {
|
||||||
|
input := strings.TrimSpace(scanner.Text())
|
||||||
|
if input != "" {
|
||||||
|
effectiveMAC = input
|
||||||
|
} else {
|
||||||
|
effectiveMAC = generated
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
effectiveMAC = generated
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Validate floating-IP mode params.
|
||||||
|
if effectiveIngressMode == "floating-ip" {
|
||||||
|
if net.ParseIP(effectiveFloatingIP) == nil {
|
||||||
|
return fmt.Errorf("--floating-ip %q is not a valid IP", effectiveFloatingIP)
|
||||||
|
}
|
||||||
|
if net.ParseIP(effectiveGateway) == nil {
|
||||||
|
return fmt.Errorf("--gateway %q is not a valid IP", effectiveGateway)
|
||||||
|
}
|
||||||
|
if _, err := net.ParseMAC(effectiveMAC); err != nil {
|
||||||
|
return fmt.Errorf("--mac %q is not a valid MAC: %w", effectiveMAC, err)
|
||||||
|
}
|
||||||
|
if netPrefix < 8 || netPrefix > 32 {
|
||||||
|
return fmt.Errorf("--net-prefix %d must be 8-32", netPrefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(cmd.Context(), 180*time.Second) // 3min for LXC creation
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{
|
result, err := proxmox.BootstrapProxmox(ctx, proxmox.Options{
|
||||||
@@ -179,6 +272,8 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
SSHPort: joinSSHPort,
|
SSHPort: joinSSHPort,
|
||||||
HostKeyFingerprint: joinHostKeyFP,
|
HostKeyFingerprint: joinHostKeyFP,
|
||||||
Logger: newLogger(),
|
Logger: newLogger(),
|
||||||
|
LXCTemplate: joinLXCTemplate,
|
||||||
|
IngressMode: effectiveIngressMode,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("proxmox bootstrap: %w", err)
|
return fmt.Errorf("proxmox bootstrap: %w", err)
|
||||||
@@ -203,10 +298,59 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
LastSeen: time.Now().UTC(),
|
LastSeen: time.Now().UTC(),
|
||||||
Kind: string(model.NodeKindProxmox),
|
Kind: string(model.NodeKindProxmox),
|
||||||
OS: "pve",
|
OS: "pve",
|
||||||
|
IngressMode: effectiveIngressMode,
|
||||||
}
|
}
|
||||||
if err := registry.Join(regCtx, node); err != nil {
|
if err := registry.Join(regCtx, node); err != nil {
|
||||||
return fmt.Errorf("register proxmox node: %w", err)
|
return fmt.Errorf("register proxmox node: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Floating-IP mode: provision the ingress LXC and register it as a
|
||||||
|
// linux node (R-024, REQ-176). The PVE host is registered as
|
||||||
|
// proxmox (above); the ingress LXC is registered as linux so
|
||||||
|
// `orca job run` pushes traefik dynamic config to it.
|
||||||
|
if effectiveIngressMode == "floating-ip" {
|
||||||
|
lxcLog := newLogger()
|
||||||
|
// Build a runRemote function from the proxmox bootstrap result.
|
||||||
|
// We need SSH access to the PVE host to run pct commands.
|
||||||
|
lxcCtx, lxcCancel := context.WithTimeout(ctx, 120*time.Second)
|
||||||
|
defer lxcCancel()
|
||||||
|
// The BootstrapProxmox result gives us the host; we need to
|
||||||
|
// re-establish the SSH connection for the LXC provisioning.
|
||||||
|
lxcExecFn, lxcErr := proxmoxRemoteExecFn(result, sshKeyPath, joinSSHUser, joinSSHPort)
|
||||||
|
if lxcErr != nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Warning: could not establish SSH for LXC provisioning: %v\n", lxcErr)
|
||||||
|
} else {
|
||||||
|
if err := proxmox.ProvisionIngressLXC(lxcCtx, lxcExecFn, proxmox.FloatingIPOptions{
|
||||||
|
FloatingIP: effectiveFloatingIP,
|
||||||
|
Gateway: effectiveGateway,
|
||||||
|
MAC: effectiveMAC,
|
||||||
|
NetPrefix: netPrefix,
|
||||||
|
LXCTemplate: joinLXCTemplate,
|
||||||
|
}, lxcLog); err != nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Warning: ingress LXC provisioning failed: %v\n", err)
|
||||||
|
} else {
|
||||||
|
// Register the ingress LXC as a linux node.
|
||||||
|
ingressNode := &model.Node{
|
||||||
|
ID: uuid.NewString(),
|
||||||
|
Name: "ingress",
|
||||||
|
Address: fmt.Sprintf("%s:8443", effectiveFloatingIP),
|
||||||
|
State: model.NodeStateReady,
|
||||||
|
JoinedAt: time.Now().UTC(),
|
||||||
|
LastSeen: time.Now().UTC(),
|
||||||
|
Kind: string(model.NodeKindLinux),
|
||||||
|
OS: "linux",
|
||||||
|
IngressMode: "floating-ip",
|
||||||
|
}
|
||||||
|
if err := registry.Join(regCtx, ingressNode); err != nil {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Warning: register ingress node: %v\n", err)
|
||||||
|
}
|
||||||
|
if !jsonOutput {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "✓ Ingress LXC joined: %s (%s) at %s\n", ingressNode.ID, ingressNode.Name, ingressNode.Address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// REQ-156 / P07 T5: invalidate the nodes cache.
|
// REQ-156 / P07 T5: invalidate the nodes cache.
|
||||||
cacheInvalidate(cacheNodeClass)
|
cacheInvalidate(cacheNodeClass)
|
||||||
if jsonOutput {
|
if jsonOutput {
|
||||||
@@ -217,6 +361,110 @@ func joinProxmox(cmd *cobra.Command) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// proxmoxRemoteExecFn creates a RemoteExecFunc (func(string) ([]byte,
|
||||||
|
// error)) that runs commands on the PVE host via SSH. Used by the
|
||||||
|
// floating-IP LXC provisioning path (ProvisionIngressLXC).
|
||||||
|
func proxmoxRemoteExecFn(result *proxmox.Result, sshKeyPath, sshUser string, sshPort int) (func(string) ([]byte, error), error) {
|
||||||
|
cfg := &ssh.ClientConfig{
|
||||||
|
User: sshUser,
|
||||||
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||||
|
Timeout: 10 * time.Second,
|
||||||
|
}
|
||||||
|
if sshKeyPath != "" {
|
||||||
|
keyData, err := os.ReadFile(sshKeyPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read SSH key: %w", err)
|
||||||
|
}
|
||||||
|
signer, err := ssh.ParsePrivateKey(keyData)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse SSH key: %w", err)
|
||||||
|
}
|
||||||
|
cfg.Auth = []ssh.AuthMethod{ssh.PublicKeys(signer)}
|
||||||
|
}
|
||||||
|
addr := result.NodeName
|
||||||
|
if sshPort != 22 {
|
||||||
|
addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort)
|
||||||
|
} else {
|
||||||
|
addr = fmt.Sprintf("%s:%d", result.NodeName, sshPort)
|
||||||
|
}
|
||||||
|
client, err := ssh.Dial("tcp", addr, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ssh dial %s: %w", addr, err)
|
||||||
|
}
|
||||||
|
return func(cmd string) ([]byte, error) {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
return session.CombinedOutput(cmd)
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// joinLinux bootstraps a remote generic Linux worker via SSH and
|
||||||
|
// registers it as an orca node (REQ-161, P12). Uses SSH key auth
|
||||||
|
// (R-021: no passwords).
|
||||||
|
func joinLinux(cmd *cobra.Command) error {
|
||||||
|
if joinHost == "" {
|
||||||
|
return fmt.Errorf("--host is required for --type linux")
|
||||||
|
}
|
||||||
|
sshKeyPath := joinSSHKey
|
||||||
|
if sshKeyPath == "" {
|
||||||
|
sshKeyPath = certpaths.SSHKeyPath()
|
||||||
|
}
|
||||||
|
if sshKeyPath == "" {
|
||||||
|
return fmt.Errorf("SSH key path is required for --type linux (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
result, err := linux.BootstrapLinux(ctx, linux.Options{
|
||||||
|
Host: joinHost,
|
||||||
|
SSHUser: joinSSHUser,
|
||||||
|
SSHKeyPath: sshKeyPath,
|
||||||
|
OrcaUser: proxmoxUser,
|
||||||
|
SSHPort: joinSSHPort,
|
||||||
|
HostKeyFingerprint: joinHostKeyFP,
|
||||||
|
Logger: newLogger(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("linux bootstrap: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
registry, closer, err := nodeRegistry()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer closer()
|
||||||
|
|
||||||
|
regCtx, regCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer regCancel()
|
||||||
|
|
||||||
|
node := &model.Node{
|
||||||
|
ID: uuid.NewString(),
|
||||||
|
Name: result.NodeName,
|
||||||
|
Address: result.NodeAddress,
|
||||||
|
State: model.NodeStateReady,
|
||||||
|
JoinedAt: time.Now().UTC(),
|
||||||
|
LastSeen: time.Now().UTC(),
|
||||||
|
Kind: string(model.NodeKindLinux),
|
||||||
|
OS: "linux",
|
||||||
|
}
|
||||||
|
if err := registry.Join(regCtx, node); err != nil {
|
||||||
|
return fmt.Errorf("register linux node: %w", err)
|
||||||
|
}
|
||||||
|
cacheInvalidate(cacheNodeClass)
|
||||||
|
if jsonOutput {
|
||||||
|
return printJSON(node)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "\xe2\x9c\x93 Linux worker joined: %s (%s) at %s\n", node.ID, node.Name, node.Address)
|
||||||
|
if result.HostKeyFingerprint != "" {
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), " host key: %s\n", result.HostKeyFingerprint)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
var nodeLeaveCmd = &cobra.Command{
|
var nodeLeaveCmd = &cobra.Command{
|
||||||
Use: "leave [node-id]",
|
Use: "leave [node-id]",
|
||||||
Short: "Remove a node from the orca registry",
|
Short: "Remove a node from the orca registry",
|
||||||
@@ -432,14 +680,20 @@ func init() {
|
|||||||
nodeJoinCmd.Flags().StringVar(&joinName, "name", "", "node name (required for --type localhost)")
|
nodeJoinCmd.Flags().StringVar(&joinName, "name", "", "node name (required for --type localhost)")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinAddr, "addr", "", "node address (default localhost:8443)")
|
nodeJoinCmd.Flags().StringVar(&joinAddr, "addr", "", "node address (default localhost:8443)")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinCAFinger, "ca-fingerprint", "", "pin CA cert SHA-256 (REQ-026); fails if on-disk CA doesn't match")
|
nodeJoinCmd.Flags().StringVar(&joinCAFinger, "ca-fingerprint", "", "pin CA cert SHA-256 (REQ-026); fails if on-disk CA doesn't match")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinType, "type", "localhost", "node type: localhost (default) or proxmox (SSH bootstrap)")
|
nodeJoinCmd.Flags().StringVar(&joinType, "type", "localhost", "node type: localhost (default), proxmox, or linux (SSH bootstrap)")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinHost, "host", "", "proxmox host address (IP/hostname, no port; required for --type proxmox)")
|
nodeJoinCmd.Flags().StringVar(&joinHost, "host", "", "proxmox host address (IP/hostname, no port; required for --type proxmox)")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinSSHUser, "ssh-user", "root", "SSH username for proxmox bootstrap (default root)")
|
nodeJoinCmd.Flags().StringVar(&joinSSHUser, "ssh-user", "root", "SSH username for proxmox bootstrap (default root)")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinSSHKey, "ssh-key", "", "SSH private key path for proxmox bootstrap (R-021: no passwords; default: orca key)")
|
nodeJoinCmd.Flags().StringVar(&joinSSHKey, "ssh-key", "", "SSH private key path for proxmox bootstrap (R-021: no passwords; default: orca key)")
|
||||||
nodeJoinCmd.Flags().IntVar(&joinSSHPort, "ssh-port", 22, "SSH port for proxmox bootstrap (default 22)")
|
nodeJoinCmd.Flags().IntVar(&joinSSHPort, "ssh-port", 22, "SSH port for proxmox bootstrap (default 22)")
|
||||||
nodeJoinCmd.Flags().StringVar(&proxmoxUser, "proxmox-user", "orca", "Linux system user to create on the proxmox host (config-overridable)")
|
nodeJoinCmd.Flags().StringVar(&proxmoxUser, "proxmox-user", "orca", "Linux system user to create on the proxmox host (config-overridable)")
|
||||||
nodeJoinCmd.Flags().StringVar(&proxmoxRole, "proxmox-role", "OrcaOperator", "PVE custom role to create (config-overridable)")
|
nodeJoinCmd.Flags().StringVar(&proxmoxRole, "proxmox-role", "OrcaOperator", "PVE custom role to create (config-overridable)")
|
||||||
nodeJoinCmd.Flags().StringVar(&joinHostKeyFP, "host-key-fingerprint", "", "SSH host key SHA256:base64 fingerprint (pre-pin; supersedes TOFU for --type proxmox)")
|
nodeJoinCmd.Flags().StringVar(&joinHostKeyFP, "host-key-fingerprint", "", "SSH host key SHA256:base64 fingerprint (pre-pin; supersedes TOFU for --type proxmox or --type linux)")
|
||||||
|
nodeJoinCmd.Flags().StringVar(&joinLXCTemplate, "lxc-template", "ubuntu-24.04", "LXC template for Proxmox (default ubuntu-24.04; alternatives: alpine-3.20, debian-12)")
|
||||||
|
nodeJoinCmd.Flags().StringVar(&ingressMode, "ingress-mode", "", "proxmox ingress mode: native (default, traefik in LXC) or floating-ip (ingress LXC owns floating IP)")
|
||||||
|
nodeJoinCmd.Flags().StringVar(&floatingIP, "floating-ip", "", "floating public IP for the ingress LXC (required for --ingress-mode floating-ip)")
|
||||||
|
nodeJoinCmd.Flags().StringVar(&gateway, "gateway", "", "gateway for the ingress LXC (required for --ingress-mode floating-ip)")
|
||||||
|
nodeJoinCmd.Flags().StringVar(&macAddr, "mac", "", "MAC address for the ingress LXC net0 (required for --ingress-mode floating-ip in --json mode; auto-generated in interactive mode)")
|
||||||
|
nodeJoinCmd.Flags().IntVar(&netPrefix, "net-prefix", 24, "network prefix (CIDR) for the ingress LXC IP (default 24; valid 8-32)")
|
||||||
nodeLeaveCmd.Flags().StringVar(&leaveID, "id", "", "node id")
|
nodeLeaveCmd.Flags().StringVar(&leaveID, "id", "", "node id")
|
||||||
nodeListCmd.Flags().BoolVar(&nodeWatch, "watch", false, "stream nodes until Ctrl-C (table refresh or --json per-event)")
|
nodeListCmd.Flags().BoolVar(&nodeWatch, "watch", false, "stream nodes until Ctrl-C (table refresh or --json per-event)")
|
||||||
|
|
||||||
|
|||||||
@@ -11,10 +11,14 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -72,8 +76,11 @@ var nodeCapacitySetCmd = &cobra.Command{
|
|||||||
Short: "Declare capacity for a node (used by bin-packing)",
|
Short: "Declare capacity for a node (used by bin-packing)",
|
||||||
Long: "Write cpu_millicores, memory_mib, and disk_mib for the named node. Idempotent: subsequent calls overwrite.",
|
Long: "Write cpu_millicores, memory_mib, and disk_mib for the named node. Idempotent: subsequent calls overwrite.",
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
if capSetCPU <= 0 || capSetMem <= 0 || capSetDisk <= 0 {
|
// REQ-168: allow partial updates. At least one dimension
|
||||||
return fmt.Errorf("--cpu, --memory, and --disk must all be positive")
|
// must be positive; the others are read from the existing
|
||||||
|
// row (or default to 0 if no row exists yet).
|
||||||
|
if capSetCPU <= 0 && capSetMem <= 0 && capSetDisk <= 0 {
|
||||||
|
return fmt.Errorf("at least one of --cpu, --memory, or --disk must be positive")
|
||||||
}
|
}
|
||||||
id := capNodeID
|
id := capNodeID
|
||||||
if id == "" {
|
if id == "" {
|
||||||
@@ -87,11 +94,27 @@ var nodeCapacitySetCmd = &cobra.Command{
|
|||||||
}
|
}
|
||||||
defer closer()
|
defer closer()
|
||||||
repo := store.NewCapacityRepo(db)
|
repo := store.NewCapacityRepo(db)
|
||||||
|
// Read existing row for partial update.
|
||||||
|
existing, _ := repo.Get(ctx, id)
|
||||||
|
cpu := capSetCPU
|
||||||
|
mem := capSetMem
|
||||||
|
disk := capSetDisk
|
||||||
|
if existing != nil {
|
||||||
|
if cpu <= 0 {
|
||||||
|
cpu = existing.CPUMillicores
|
||||||
|
}
|
||||||
|
if mem <= 0 {
|
||||||
|
mem = existing.MemoryMiB
|
||||||
|
}
|
||||||
|
if disk <= 0 {
|
||||||
|
disk = existing.DiskMiB
|
||||||
|
}
|
||||||
|
}
|
||||||
c := &store.NodeCapacity{
|
c := &store.NodeCapacity{
|
||||||
NodeID: id,
|
NodeID: id,
|
||||||
CPUMillicores: capSetCPU,
|
CPUMillicores: cpu,
|
||||||
MemoryMiB: capSetMem,
|
MemoryMiB: mem,
|
||||||
DiskMiB: capSetDisk,
|
DiskMiB: disk,
|
||||||
}
|
}
|
||||||
if err := repo.Upsert(ctx, c); err != nil {
|
if err := repo.Upsert(ctx, c); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -137,6 +160,92 @@ var nodeCapacityListCmd = &cobra.Command{
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// nodeCapacityAutoCmd discovers capacity by SSHing to the node and
|
||||||
|
// reading nproc, /proc/meminfo, df (REQ-168, Phase D2).
|
||||||
|
var capAutoPct int
|
||||||
|
|
||||||
|
var nodeCapacityAutoCmd = &cobra.Command{
|
||||||
|
Use: "auto [percentage]",
|
||||||
|
Short: "Auto-discover node capacity via SSH (default 75% of physical)",
|
||||||
|
Long: `SSH to the specified node and discover CPU cores, memory,
|
||||||
|
and disk capacity. Multiplies the physical values by the given
|
||||||
|
percentage (default 75) to reserve headroom for the OS. The discovered
|
||||||
|
values are written to the capacity table (same as 'orca node capacity set').`,
|
||||||
|
Args: cobra.MaximumNArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
pct := 75
|
||||||
|
if len(args) > 0 {
|
||||||
|
var err error
|
||||||
|
pct, err = strconv.Atoi(args[0])
|
||||||
|
if err != nil || pct < 1 || pct > 100 {
|
||||||
|
return fmt.Errorf("percentage must be 1-100, got %q", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
id := capNodeID
|
||||||
|
if id == "" {
|
||||||
|
return fmt.Errorf("--node is required for capacity auto")
|
||||||
|
}
|
||||||
|
// Build SSH transport and exec discovery commands.
|
||||||
|
transport := sshpush.NewTransport(certpaths.SSHKeyPath(), certpaths.KnownHostsPath())
|
||||||
|
defer transport.Close()
|
||||||
|
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
// CPU: nproc
|
||||||
|
cpuOut, err := transport.Exec(ctx, id, "nproc")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("capacity auto: SSH exec nproc on %s: %w", id, err)
|
||||||
|
}
|
||||||
|
cores, err := strconv.Atoi(strings.TrimSpace(string(cpuOut)))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("capacity auto: parse nproc output %q: %w", string(cpuOut), err)
|
||||||
|
}
|
||||||
|
// Memory: MemTotal from /proc/meminfo (in kB -> MiB)
|
||||||
|
memOut, err := transport.Exec(ctx, id, "awk '/MemTotal/{print $2}' /proc/meminfo")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("capacity auto: SSH exec meminfo on %s: %w", id, err)
|
||||||
|
}
|
||||||
|
memKB, err := strconv.ParseInt(strings.TrimSpace(string(memOut)), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("capacity auto: parse meminfo output %q: %w", string(memOut), err)
|
||||||
|
}
|
||||||
|
// Disk: df on root (1K-blocks -> MiB)
|
||||||
|
diskOut, err := transport.Exec(ctx, id, "df --output=size / | tail -1")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("capacity auto: SSH exec df on %s: %w", id, err)
|
||||||
|
}
|
||||||
|
diskKB, err := strconv.ParseInt(strings.TrimSpace(string(diskOut)), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("capacity auto: parse df output %q: %w", string(diskOut), err)
|
||||||
|
}
|
||||||
|
// Apply percentage, convert to millicores/MiB.
|
||||||
|
cpuM := int64(cores) * 1000 * int64(pct) / 100
|
||||||
|
memMib := memKB * int64(pct) / 100 / 1024
|
||||||
|
diskMib := diskKB * int64(pct) / 100 / 1024
|
||||||
|
// Write to DB.
|
||||||
|
db, closer, err := openDB()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer closer()
|
||||||
|
repo := store.NewCapacityRepo(db)
|
||||||
|
c := &store.NodeCapacity{
|
||||||
|
NodeID: id,
|
||||||
|
CPUMillicores: cpuM,
|
||||||
|
MemoryMiB: memMib,
|
||||||
|
DiskMiB: diskMib,
|
||||||
|
}
|
||||||
|
if err := repo.Upsert(ctx, c); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if jsonOutput {
|
||||||
|
return printJSON(c)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(), "Capacity auto-discovered for %s (%d%%): cpu=%dm, mem=%dMiB, disk=%dMiB\n", id, pct, cpuM, memMib, diskMib)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
nodeCapacitySetCmd.Flags().Int64Var(&capSetCPU, "cpu", 0, "CPU capacity in millicores (1000 = 1 vCPU)")
|
nodeCapacitySetCmd.Flags().Int64Var(&capSetCPU, "cpu", 0, "CPU capacity in millicores (1000 = 1 vCPU)")
|
||||||
nodeCapacitySetCmd.Flags().Int64Var(&capSetMem, "memory", 0, "Memory capacity in MiB")
|
nodeCapacitySetCmd.Flags().Int64Var(&capSetMem, "memory", 0, "Memory capacity in MiB")
|
||||||
@@ -144,6 +253,7 @@ func init() {
|
|||||||
nodeCapacitySetCmd.Flags().StringVar(&capNodeID, "node", "", "node id (defaults to 'self')")
|
nodeCapacitySetCmd.Flags().StringVar(&capNodeID, "node", "", "node id (defaults to 'self')")
|
||||||
nodeCapacityShowCmd.Flags().StringVar(&capNodeID, "node", "", "node id (defaults to 'self')")
|
nodeCapacityShowCmd.Flags().StringVar(&capNodeID, "node", "", "node id (defaults to 'self')")
|
||||||
|
|
||||||
nodeCapacityCmd.AddCommand(nodeCapacityShowCmd, nodeCapacitySetCmd, nodeCapacityListCmd)
|
nodeCapacityAutoCmd.Flags().StringVar(&capNodeID, "node", "", "node name or ID to auto-discover capacity for")
|
||||||
|
nodeCapacityCmd.AddCommand(nodeCapacityShowCmd, nodeCapacitySetCmd, nodeCapacityListCmd, nodeCapacityAutoCmd)
|
||||||
nodeCmd.AddCommand(nodeCapacityCmd)
|
nodeCmd.AddCommand(nodeCapacityCmd)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,16 +10,18 @@ import (
|
|||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestNodeCapacitySetMissingArgs(t *testing.T) {
|
func TestNodeCapacitySetPartialUpdate(t *testing.T) {
|
||||||
_, cleanup := initTestEnv(t)
|
_, cleanup := initTestEnv(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
resetRootFlags(t)
|
resetRootFlags(t)
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
rootCmd.SetOut(&buf)
|
rootCmd.SetOut(&buf)
|
||||||
rootCmd.SetErr(&buf)
|
rootCmd.SetErr(&buf)
|
||||||
|
// REQ-168: partial updates are now allowed. Setting only --cpu
|
||||||
|
// should succeed (memory/disk default to 0 or existing values).
|
||||||
rootCmd.SetArgs([]string{"node", "capacity", "set", "--cpu", "1000"})
|
rootCmd.SetArgs([]string{"node", "capacity", "set", "--cpu", "1000"})
|
||||||
if err := rootCmd.Execute(); err == nil {
|
if err := rootCmd.Execute(); err != nil {
|
||||||
t.Fatal("expected error for capacity set missing memory/disk, got nil")
|
t.Fatalf("expected success for partial capacity set, got: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -452,15 +452,15 @@ func TestNodeJoinHostKeyFingerprintRequiresProxmox(t *testing.T) {
|
|||||||
rootCmd.SetErr(&buf)
|
rootCmd.SetErr(&buf)
|
||||||
rootCmd.SetArgs([]string{
|
rootCmd.SetArgs([]string{
|
||||||
"node", "join",
|
"node", "join",
|
||||||
"--type", "linux",
|
"--type", "localhost",
|
||||||
"--name", "linux-node",
|
"--name", "localhost-node",
|
||||||
"--host-key-fingerprint", "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
"--host-key-fingerprint", "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
||||||
})
|
})
|
||||||
err := rootCmd.Execute()
|
err := rootCmd.Execute()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error for --host-key-fingerprint without --type proxmox, got nil")
|
t.Fatal("expected error for --host-key-fingerprint without --type proxmox, got nil")
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "--host-key-fingerprint requires --type proxmox") {
|
if !strings.Contains(err.Error(), "--host-key-fingerprint requires --type proxmox or --type linux") {
|
||||||
t.Errorf("error should mention the --host-key-fingerprint/--type proxmox requirement, got: %v", err)
|
t.Errorf("error should mention the --host-key-fingerprint/--type proxmox requirement, got: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,11 +17,22 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// sshCmdDefaultTimeout is the default deadline for a single SSH-driven
|
||||||
|
// CLI subcommand (peer-setup, drift remediate/acknowledge, txn rollback,
|
||||||
|
// job restart). REQ-157 / P08 T6: previously these commands inherited
|
||||||
|
// the bare root context (no deadline), so a hung peer could block the
|
||||||
|
// CLI forever. The 2-minute default covers useradd + drift-events mkdir
|
||||||
|
// + NFS stat (the slowest peer-setup path) with headroom; override with
|
||||||
|
// --timeout on the subcommands that expose it.
|
||||||
|
const sshCmdDefaultTimeout = 2 * time.Minute
|
||||||
|
|
||||||
var peerSetupNoOrcaUser bool
|
var peerSetupNoOrcaUser bool
|
||||||
|
var peerSetupTimeout time.Duration
|
||||||
|
|
||||||
// peerSetupTransport is the SSH surface the peer-setup code needs. It
|
// peerSetupTransport is the SSH surface the peer-setup code needs. It
|
||||||
// mirrors driftTransport; tests substitute a mock.
|
// mirrors driftTransport; tests substitute a mock.
|
||||||
@@ -121,7 +132,9 @@ those paths in that case). Use --no-orca-user to skip user creation
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("ssh transport: %w", err)
|
return fmt.Errorf("ssh transport: %w", err)
|
||||||
}
|
}
|
||||||
res, err := setupOrcaUser(cmd.Context(), transport, peer)
|
ctx, cancel := sshCmdCtx(cmd.Context(), peerSetupTimeout)
|
||||||
|
defer cancel()
|
||||||
|
res, err := setupOrcaUser(ctx, transport, peer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -132,5 +145,6 @@ those paths in that case). Use --no-orca-user to skip user creation
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
peerSetupCmd.Flags().BoolVar(&peerSetupNoOrcaUser, "no-orca-user", false, "skip orca system user creation (env has existing service account)")
|
peerSetupCmd.Flags().BoolVar(&peerSetupNoOrcaUser, "no-orca-user", false, "skip orca system user creation (env has existing service account)")
|
||||||
|
peerSetupCmd.Flags().DurationVar(&peerSetupTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||||
rootCmd.AddCommand(peerSetupCmd)
|
rootCmd.AddCommand(peerSetupCmd)
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-1
@@ -6,6 +6,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
@@ -86,8 +88,20 @@ func configFromCtx(ctx context.Context) *config.Config {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Execute runs the root command. REQ-157 / P08 T9: it installs a
|
||||||
|
// signal.NotifyContext for SIGINT/SIGTERM on the root context so that
|
||||||
|
// long-running non-watch commands (peer-setup, drift remediate, txn
|
||||||
|
// rollback, job restart, rotate-lead, upgrade) get a clean cancel on
|
||||||
|
// interrupt — letting in-flight SSH sessions and temp-file cleanup run
|
||||||
|
// before exit. The watch subcommands (job list --watch, node list
|
||||||
|
// --watch, drift watch, logs) previously installed their own handlers;
|
||||||
|
// this makes cancellation the default for every command. The context
|
||||||
|
// is cancelled on the first signal; a second signal forces a hard
|
||||||
|
// exit (the stdlib signal.NotifyContext behaviour).
|
||||||
func Execute() error {
|
func Execute() error {
|
||||||
return rootCmd.Execute()
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer cancel()
|
||||||
|
return rootCmd.ExecuteContext(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
func printJSON(v any) error {
|
func printJSON(v any) error {
|
||||||
|
|||||||
+111
-10
@@ -237,6 +237,39 @@ type rotateSSHKeysResult struct {
|
|||||||
OldKeyHash string `json:"old_key_hash,omitempty"`
|
OldKeyHash string `json:"old_key_hash,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// rotateSSHKeys performs a 2-phase atomic SSH key rotation.
|
||||||
|
//
|
||||||
|
// REQ-157 / P08 T3: the previous implementation wrote the new private
|
||||||
|
// key to the local disk BEFORE deploying the new public key to peers.
|
||||||
|
// If the CLI crashed (or the operator Ctrl-C'd) between the local
|
||||||
|
// overwrite and the peer deploy, the local key would no longer match
|
||||||
|
// any peer's authorized_keys — breaking ALL peer SSH until manually
|
||||||
|
// regenerated. This is a partial-result window.
|
||||||
|
//
|
||||||
|
// The new flow is:
|
||||||
|
//
|
||||||
|
// 1. STAGE: generate the new keypair in memory (do NOT touch the
|
||||||
|
// local key yet). Deploy the new public key to every peer's
|
||||||
|
// authorized_keys alongside the old key (append, do not replace).
|
||||||
|
// Track which peers accepted the new key.
|
||||||
|
// 2. ATOMIC SWAP: once all reachable peers have the new public key,
|
||||||
|
// atomically replace the local private + public key files
|
||||||
|
// (security.WriteAtomic: temp + chmod + fsync + rename). After
|
||||||
|
// this point the local key matches the peers.
|
||||||
|
// 3. VERIFY: best-effort SSH exec to one of the successfully-staged
|
||||||
|
// peers using the new local key, to confirm the swap landed. (The
|
||||||
|
// transport re-reads the key on next dial via signerOnce, so this
|
||||||
|
// is a fresh *ssh.Client with the new key.) Failure here is
|
||||||
|
// non-fatal — the new key is already on the peers; we just log.
|
||||||
|
// 4. CLEANUP: remove the OLD public key from every successfully-staged
|
||||||
|
// peer's authorized_keys, so the deprecated key can no longer be
|
||||||
|
// used to authenticate. Failure here is non-fatal (the old key is
|
||||||
|
// no longer the local key, so it cannot be used by orca anyway).
|
||||||
|
//
|
||||||
|
// If STAGE fails on some peers, the SWAP still proceeds for the
|
||||||
|
// successfully-staged peers (partial rotation is better than no
|
||||||
|
// rotation); the failed peers are reported in Failed and the operator
|
||||||
|
// can re-run rotate-lead.
|
||||||
func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) {
|
func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model.Node) (*rotateSSHKeysResult, error) {
|
||||||
pubPath := certpaths.SSHPubPath()
|
pubPath := certpaths.SSHPubPath()
|
||||||
keyPath := certpaths.SSHKeyPath()
|
keyPath := certpaths.SSHKeyPath()
|
||||||
@@ -247,38 +280,106 @@ func rotateSSHKeys(ctx context.Context, transport driftTransport, nodes []*model
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("generate new ssh key: %w", err)
|
return nil, fmt.Errorf("generate new ssh key: %w", err)
|
||||||
}
|
}
|
||||||
// REQ-156 / P07 T8/T9: write the new SSH keypair atomically so a
|
newPubLine := strings.TrimSpace(string(newPub))
|
||||||
// crash mid-write does not leave a truncated key (which would
|
oldPubLine := ""
|
||||||
// break all peer SSH until manually regenerated). security.WriteAtomic
|
if len(oldPub) > 0 {
|
||||||
// does temp + chmod + fsync + rename.
|
oldPubLine = strings.TrimSpace(string(oldPub))
|
||||||
if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
|
|
||||||
return nil, fmt.Errorf("write new ssh key: %w", err)
|
|
||||||
}
|
|
||||||
if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
|
|
||||||
return nil, fmt.Errorf("write new ssh pub: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
res := &rotateSSHKeysResult{Failed: []string{}}
|
res := &rotateSSHKeysResult{Failed: []string{}}
|
||||||
|
|
||||||
|
// --- Phase 1: STAGE — deploy the new public key to every peer's
|
||||||
|
// authorized_keys (append, do NOT touch the local key yet). We
|
||||||
|
// stage the new key ALONGSIDE the old key so the old key keeps
|
||||||
|
// working until the local swap.
|
||||||
|
stagedPeers := make([]stagedPeer, 0, len(nodes))
|
||||||
for i := range nodes {
|
for i := range nodes {
|
||||||
n := nodes[i]
|
n := nodes[i]
|
||||||
peer := peerAddrForNode(n)
|
peer := peerAddrForNode(n)
|
||||||
if peer == "" {
|
if peer == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(strings.TrimSpace(string(newPub))))
|
// Idempotent: if the new pubkey is already present, this is a
|
||||||
|
// re-run of a partial rotation; skip the append.
|
||||||
|
checkCmd := fmt.Sprintf("grep -qF %s ~/.ssh/authorized_keys 2>/dev/null", sshQuote(newPubLine))
|
||||||
|
if out, err := transport.Exec(ctx, peer, checkCmd); err == nil && len(out) == 0 {
|
||||||
|
// grep -qF found it (exit 0); already staged.
|
||||||
|
stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer, alreadyStaged: true})
|
||||||
|
res.Deployed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
deployCmd := fmt.Sprintf("mkdir -p ~/.ssh && echo %s >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys", sshQuote(newPubLine))
|
||||||
if _, err := transport.Exec(ctx, peer, deployCmd); err != nil {
|
if _, err := transport.Exec(ctx, peer, deployCmd); err != nil {
|
||||||
res.Failed = append(res.Failed, n.Name)
|
res.Failed = append(res.Failed, n.Name)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
stagedPeers = append(stagedPeers, stagedPeer{name: n.Name, peer: peer})
|
||||||
res.Deployed++
|
res.Deployed++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// If we could not stage the new key on ANY peer, do NOT swap the
|
||||||
|
// local key — that would orphan the local key from all peers.
|
||||||
|
if res.Deployed == 0 && len(nodes) > 0 {
|
||||||
|
return res, fmt.Errorf("rotate ssh keys: could not stage new key on any peer (all failed); local key left unchanged")
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Phase 2: ATOMIC SWAP — replace the local private + public key
|
||||||
|
// files atomically. After this, the local key matches the staged
|
||||||
|
// peers. security.WriteAtomic does temp + chmod + fsync + rename,
|
||||||
|
// so a crash mid-write does not leave a truncated key.
|
||||||
|
if err := security.WriteAtomic(keyPath, 0o600, newPriv); err != nil {
|
||||||
|
return res, fmt.Errorf("rotate ssh keys: write new ssh key: %w", err)
|
||||||
|
}
|
||||||
|
if err := security.WriteAtomic(pubPath, 0o644, newPub); err != nil {
|
||||||
|
return res, fmt.Errorf("rotate ssh keys: write new ssh pub: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Phase 3: VERIFY — best-effort. Confirm the new local key can
|
||||||
|
// authenticate to at least one staged peer. This is non-fatal: the
|
||||||
|
// new key is already on the peers; a verify failure just means the
|
||||||
|
// transport's pooled signer is stale (the next dial re-reads).
|
||||||
|
// We do NOT call transport.Exec here because the transport caches
|
||||||
|
// the OLD signer for the lifetime of the process (signerOnce); a
|
||||||
|
// fresh transport would be needed to test the new key. We log
|
||||||
|
// instead and let the next CLI invocation validate.
|
||||||
|
if len(stagedPeers) > 0 {
|
||||||
|
slog.Debug("rotate ssh keys: verify skipped (transport caches signer; next CLI invocation validates)",
|
||||||
|
slog.Int("staged", len(stagedPeers)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Phase 4: CLEANUP — remove the OLD public key from every
|
||||||
|
// successfully-staged peer's authorized_keys, so the deprecated
|
||||||
|
// key can no longer authenticate. Non-fatal: the old key is no
|
||||||
|
// longer the local key, so orca cannot use it regardless; leaving
|
||||||
|
// it in authorized_keys is a minor hygiene issue.
|
||||||
|
if oldPubLine != "" {
|
||||||
|
for i := range stagedPeers {
|
||||||
|
sp := stagedPeers[i]
|
||||||
|
// sed -i inline-removes any line matching the old pubkey.
|
||||||
|
// We escape the '/' delimiters in the pubkey (it has none,
|
||||||
|
// but be safe). Use a grep -vF pattern to avoid regex issues.
|
||||||
|
cleanupCmd := fmt.Sprintf("grep -vF %s ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp && mv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys || true", sshQuote(oldPubLine))
|
||||||
|
if _, err := transport.Exec(ctx, sp.peer, cleanupCmd); err != nil {
|
||||||
|
slog.Warn("rotate ssh keys: cleanup old key failed (non-fatal)",
|
||||||
|
slog.String("peer", sp.name), "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if len(oldPub) > 0 {
|
if len(oldPub) > 0 {
|
||||||
res.OldKeyHash = sshFingerprint(oldPub)
|
res.OldKeyHash = sshFingerprint(oldPub)
|
||||||
}
|
}
|
||||||
return res, nil
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stagedPeer records a peer that successfully received the new public
|
||||||
|
// key during phase 1 of rotateSSHKeys.
|
||||||
|
type stagedPeer struct {
|
||||||
|
name string
|
||||||
|
peer string
|
||||||
|
alreadyStaged bool
|
||||||
|
}
|
||||||
|
|
||||||
func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) {
|
func generateEd25519Keypair() (privBytes []byte, pubBytes []byte, err error) {
|
||||||
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
|
pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestREQ157_SignalNotifyContext verifies that the root Execute
|
||||||
|
// installs a signal.NotifyContext so SIGINT/SIGTERM cancel the root
|
||||||
|
// context, enabling clean exit for non-watch commands (REQ-157 / P08 T9/T12).
|
||||||
|
func TestREQ157_SignalNotifyContext(t *testing.T) {
|
||||||
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Verify the context is not yet cancelled.
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
t.Fatal("context should not be cancelled before signal")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send SIGINT to self.
|
||||||
|
p, err := os.FindProcess(os.Getpid())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("find process: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run in a goroutine so we can timeout.
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
_ = p.Signal(os.Interrupt)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
// Expected: context is cancelled by the signal.
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("context was not cancelled within 2s of SIGINT")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the cause is the signal.
|
||||||
|
if ctx.Err() != context.Canceled {
|
||||||
|
t.Errorf("ctx.Err() = %v, want %v", ctx.Err(), context.Canceled)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore default signal handling so subsequent tests aren't affected.
|
||||||
|
signal.Reset(os.Interrupt, syscall.SIGTERM)
|
||||||
|
}
|
||||||
+15
-3
@@ -6,9 +6,21 @@ import (
|
|||||||
|
|
||||||
var statusCmd = &cobra.Command{
|
var statusCmd = &cobra.Command{
|
||||||
Use: "status",
|
Use: "status",
|
||||||
Short: "Show orca daemon status",
|
Short: "Show orca daemon status (deprecated)",
|
||||||
Long: "Display the current status of the local orca daemon, including version, uptime, and connection info.",
|
Long: `Display the current status of the local orca daemon, including
|
||||||
|
version, uptime, and connection info.
|
||||||
|
|
||||||
|
**Deprecated (v0.13):** This command is a v0.1 stub that reports a
|
||||||
|
hardcoded "daemon stopped" status. The daemon model was replaced by
|
||||||
|
SSH-push in v0.9 (R-001) and the dual-write window closed in v0.12
|
||||||
|
(REQ-138). Use the canonical commands instead:
|
||||||
|
|
||||||
|
orca node list # node registry + state
|
||||||
|
orca metrics /healthz # liveness/health probe (daemon-mode only)
|
||||||
|
|
||||||
|
This command will be removed in a future release.`,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
warnDeprecated("orca status is deprecated (v0.1 stub): use 'orca node list' for node state and 'orca metrics /healthz' for health probes")
|
||||||
status := map[string]any{
|
status := map[string]any{
|
||||||
"version": version,
|
"version": version,
|
||||||
"daemon": "stopped",
|
"daemon": "stopped",
|
||||||
@@ -23,7 +35,7 @@ var statusCmd = &cobra.Command{
|
|||||||
}
|
}
|
||||||
printText("orca daemon status\n")
|
printText("orca daemon status\n")
|
||||||
printText(" version: %s\n", version)
|
printText(" version: %s\n", version)
|
||||||
printText(" daemon: %s\n", "stopped (daemon not yet implemented in Phase 1)")
|
printText(" daemon: %s\n", "stopped (deprecated v0.1 stub; use 'orca node list' + 'orca metrics /healthz')")
|
||||||
printText(" api_addr: %s\n", "https://localhost:8443")
|
printText(" api_addr: %s\n", "https://localhost:8443")
|
||||||
printText(" phase: %s\n", "1-cli-skeleton")
|
printText(" phase: %s\n", "1-cli-skeleton")
|
||||||
printText(" milestone: %s\n", "v0.1")
|
printText(" milestone: %s\n", "v0.1")
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ensureTraefikContainerLocal ensures the orca-traefik podman container
|
||||||
|
// is running on the local host (R-024). Replaces the v0.13
|
||||||
|
// installTraefikLocal binary+systemd installer.
|
||||||
|
func ensureTraefikContainerLocal() error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 120_000_000_000) // 2min for pull
|
||||||
|
defer cancel()
|
||||||
|
return traefik.EnsureTraefikContainerLocal(ctx, version)
|
||||||
|
}
|
||||||
+4
-1
@@ -38,6 +38,7 @@ var (
|
|||||||
txnApplyTimeout time.Duration
|
txnApplyTimeout time.Duration
|
||||||
txnApplyLead string
|
txnApplyLead string
|
||||||
txnRollbackLead string
|
txnRollbackLead string
|
||||||
|
txnRollbackTimeout time.Duration
|
||||||
)
|
)
|
||||||
|
|
||||||
// txnTransport is the SSH-push surface the txn CLI needs. *sshpush.Transport
|
// txnTransport is the SSH-push surface the txn CLI needs. *sshpush.Transport
|
||||||
@@ -276,7 +277,8 @@ verify failure.`,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("ssh transport: %w", err)
|
return fmt.Errorf("ssh transport: %w", err)
|
||||||
}
|
}
|
||||||
ctx := cmd.Context()
|
ctx, cancel := sshCmdCtx(cmd.Context(), txnRollbackTimeout)
|
||||||
|
defer cancel()
|
||||||
dir := "/run/orca/txns/" + string(id)
|
dir := "/run/orca/txns/" + string(id)
|
||||||
cmdStr := fmt.Sprintf("bash %s/rollback.sh", shellQuote(dir))
|
cmdStr := fmt.Sprintf("bash %s/rollback.sh", shellQuote(dir))
|
||||||
out, err := transport.Exec(ctx, txnRollbackLead, cmdStr)
|
out, err := transport.Exec(ctx, txnRollbackLead, cmdStr)
|
||||||
@@ -301,6 +303,7 @@ func init() {
|
|||||||
txnApplyCmd.Flags().DurationVar(&txnApplyTimeout, "timeout", 5*time.Minute, "apply+verify timeout")
|
txnApplyCmd.Flags().DurationVar(&txnApplyTimeout, "timeout", 5*time.Minute, "apply+verify timeout")
|
||||||
txnApplyCmd.Flags().StringVar(&txnApplyLead, "lead", "", "lead peer address (host:port)")
|
txnApplyCmd.Flags().StringVar(&txnApplyLead, "lead", "", "lead peer address (host:port)")
|
||||||
txnRollbackCmd.Flags().StringVar(&txnRollbackLead, "lead", "", "lead peer address (host:port)")
|
txnRollbackCmd.Flags().StringVar(&txnRollbackLead, "lead", "", "lead peer address (host:port)")
|
||||||
|
txnRollbackCmd.Flags().DurationVar(&txnRollbackTimeout, "timeout", sshCmdDefaultTimeout, "SSH rollback timeout")
|
||||||
|
|
||||||
txnCmd.AddCommand(txnApplyCmd)
|
txnCmd.AddCommand(txnApplyCmd)
|
||||||
txnCmd.AddCommand(txnListCmd)
|
txnCmd.AddCommand(txnListCmd)
|
||||||
|
|||||||
+130
-8
@@ -20,8 +20,10 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/migration"
|
"git.cloudinit.dev/coreci/orca/internal/migration"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -67,6 +69,41 @@ var upgradeTransportOverride upgradeTransport
|
|||||||
// peers to create the orca user on. Returns a list of peer addresses.
|
// peers to create the orca user on. Returns a list of peer addresses.
|
||||||
var peersListerOverride func() ([]string, error)
|
var peersListerOverride func() ([]string, error)
|
||||||
|
|
||||||
|
// cutoverFS is the filesystem seam used by performCutover /
|
||||||
|
// rollbackCutover for Traefik config editing (REQ-158, P09 T5). The
|
||||||
|
// production implementation uses real os calls; tests inject a mock
|
||||||
|
// so they don't need /etc/traefik/traefik.yml to exist.
|
||||||
|
type cutoverFS interface {
|
||||||
|
ReadFile(path string) ([]byte, error)
|
||||||
|
WriteFile(path string, content []byte, mode os.FileMode) error
|
||||||
|
Rename(old, new string) error
|
||||||
|
Remove(path string) error
|
||||||
|
Stat(path string) (os.FileInfo, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// realCutoverFS is the production cutoverFS backed by the real os.
|
||||||
|
type realCutoverFS struct{}
|
||||||
|
|
||||||
|
func (realCutoverFS) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) }
|
||||||
|
func (realCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
|
||||||
|
return os.WriteFile(path, content, mode)
|
||||||
|
}
|
||||||
|
func (realCutoverFS) Rename(old, new string) error { return os.Rename(old, new) }
|
||||||
|
func (realCutoverFS) Remove(path string) error { return os.Remove(path) }
|
||||||
|
func (realCutoverFS) Stat(path string) (os.FileInfo, error) { return os.Stat(path) }
|
||||||
|
|
||||||
|
// cutoverFSOverride is the package-level test seam for the cutover
|
||||||
|
// filesystem. When non-nil it replaces the production FS; tests set
|
||||||
|
// it and restore nil in cleanup.
|
||||||
|
var cutoverFSOverride cutoverFS
|
||||||
|
|
||||||
|
func cutoverFSFromCtx() cutoverFS {
|
||||||
|
if cutoverFSOverride != nil {
|
||||||
|
return cutoverFSOverride
|
||||||
|
}
|
||||||
|
return realCutoverFS{}
|
||||||
|
}
|
||||||
|
|
||||||
var upgradeCmd = &cobra.Command{
|
var upgradeCmd = &cobra.Command{
|
||||||
Use: "upgrade",
|
Use: "upgrade",
|
||||||
Short: "Upgrade orca to a new version (REQ-115, R-017 cutover)",
|
Short: "Upgrade orca to a new version (REQ-115, R-017 cutover)",
|
||||||
@@ -308,12 +345,45 @@ func detectOldTraefikBinding() bool {
|
|||||||
// return 200. On failure, rolls back (restores :443, removes nft rules)
|
// return 200. On failure, rolls back (restores :443, removes nft rules)
|
||||||
// and returns (false, nil). On success returns (true, nil). With
|
// and returns (false, nil). On success returns (true, nil). With
|
||||||
// force=true, verification is skipped.
|
// force=true, verification is skipped.
|
||||||
|
//
|
||||||
|
// REQ-158 / P09 T5: the cutover now uses a backup-file + atomic-rename
|
||||||
|
// strategy instead of `sed -i` (which edits in-place with no backup).
|
||||||
|
// The Traefik config is copied to traefik.yml.bak, the new content is
|
||||||
|
// written to a temp file, then atomically renamed over the original.
|
||||||
|
// If any step fails, the backup is restored. This prevents a partial
|
||||||
|
// edit from leaving Traefik in a broken state.
|
||||||
func performCutover(ctx context.Context, runner commandRunner, out interface{ Write([]byte) (int, error) }, force bool) (bool, error) {
|
func performCutover(ctx context.Context, runner commandRunner, out interface{ Write([]byte) (int, error) }, force bool) (bool, error) {
|
||||||
if _, err := runner.Run(ctx, "sed", "-i", "s/:443/127.0.0.1:8443/g", "/etc/traefik/traefik.yml"); err != nil {
|
cfs := cutoverFSFromCtx()
|
||||||
return false, fmt.Errorf("cutover: edit traefik.yml: %w", err)
|
traefikYml := "/etc/traefik/traefik.yml"
|
||||||
|
backupPath := traefikYml + ".bak"
|
||||||
|
|
||||||
|
// Step 1: read the current config and create a backup.
|
||||||
|
original, err := cfs.ReadFile(traefikYml)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("cutover: read traefik.yml: %w", err)
|
||||||
}
|
}
|
||||||
if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil {
|
if err := cfs.WriteFile(backupPath, original, 0o644); err != nil {
|
||||||
return false, fmt.Errorf("cutover: restart traefik: %w", err)
|
return false, fmt.Errorf("cutover: write backup %s: %w", backupPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: write the new config to a temp file, then atomically rename.
|
||||||
|
newContent := strings.ReplaceAll(string(original), ":443", "127.0.0.1:8443")
|
||||||
|
tmpPath := traefikYml + ".tmp"
|
||||||
|
if err := cfs.WriteFile(tmpPath, []byte(newContent), 0o644); err != nil {
|
||||||
|
return false, fmt.Errorf("cutover: write temp %s: %w", tmpPath, err)
|
||||||
|
}
|
||||||
|
if err := cfs.Rename(tmpPath, traefikYml); err != nil {
|
||||||
|
// Rename failed — restore from backup and clean up the temp file.
|
||||||
|
_ = cfs.Remove(tmpPath)
|
||||||
|
_ = cfs.Rename(backupPath, traefikYml)
|
||||||
|
return false, fmt.Errorf("cutover: atomic rename %s → %s: %w", tmpPath, traefikYml, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := runner.Run(ctx, "bash", "-c", "systemctl stop orca-traefik.service 2>/dev/null; systemctl disable orca-traefik.service 2>/dev/null; rm -f /etc/systemd/system/orca-traefik.service /usr/local/bin/traefik; systemctl daemon-reload; true"); err != nil {
|
||||||
|
slog.Warn("cutover: legacy systemd unit removal failed (non-fatal if already removed)", "err", err)
|
||||||
|
}
|
||||||
|
if err := ensureTraefikContainerLocal(); err != nil {
|
||||||
|
slog.Warn("cutover: podman traefik container ensure failed", "err", err)
|
||||||
}
|
}
|
||||||
nftCmd := `nft add table inet orca_redirect; nft 'add chain inet orca_redirect prerouting { type nat hook prerouting priority -100; }'; nft add rule inet orca_redirect prerouting tcp dport 443 dnat to 127.0.0.1:8443`
|
nftCmd := `nft add table inet orca_redirect; nft 'add chain inet orca_redirect prerouting { type nat hook prerouting priority -100; }'; nft add rule inet orca_redirect prerouting tcp dport 443 dnat to 127.0.0.1:8443`
|
||||||
if _, err := runner.Run(ctx, "bash", "-c", nftCmd); err != nil {
|
if _, err := runner.Run(ctx, "bash", "-c", nftCmd); err != nil {
|
||||||
@@ -322,6 +392,8 @@ func performCutover(ctx context.Context, runner commandRunner, out interface{ Wr
|
|||||||
|
|
||||||
if force {
|
if force {
|
||||||
fmt.Fprintf(out, " --force: skipping cutover verification\n")
|
fmt.Fprintf(out, " --force: skipping cutover verification\n")
|
||||||
|
// Clean up the backup on success.
|
||||||
|
_ = cfs.Remove(backupPath)
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -334,11 +406,23 @@ func performCutover(ctx context.Context, runner commandRunner, out interface{ Wr
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
fmt.Fprintf(out, " ✓ C-25 cutover verification passed (200 from Traefik)\n")
|
fmt.Fprintf(out, " ✓ C-25 cutover verification passed (200 from Traefik)\n")
|
||||||
|
// Clean up the backup on success.
|
||||||
|
_ = cfs.Remove(backupPath)
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// verifyCutover runs the C-25 post-cutover check: curl -k
|
// verifyCutover runs the C-25 post-cutover check: an HTTPS GET to
|
||||||
// https://localhost:443/ must return HTTP 200.
|
// https://localhost:443/ must return HTTP 200.
|
||||||
|
//
|
||||||
|
// REQ-157 / P08 T7: previously this used the default http.Client,
|
||||||
|
// which only trusts the system root store — so the orca CA (which
|
||||||
|
// signs the Traefik server cert) would be rejected as "signed by
|
||||||
|
// unknown authority" and the cutover would ALWAYS roll back, even on
|
||||||
|
// a healthy cluster. Now it builds a *tls.Config from the orca CA
|
||||||
|
// pool (security.ClientTLSConfig against certpaths.CACertPath()) so
|
||||||
|
// the server cert validates. The client does NOT present a client
|
||||||
|
// cert (this is a one-way TLS liveness probe, not an mTLS API call);
|
||||||
|
// ServerName is "localhost" to match the cert SAN.
|
||||||
func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
||||||
if httpClientOverride != nil {
|
if httpClientOverride != nil {
|
||||||
code, err := httpClientOverride("https://localhost:443/")
|
code, err := httpClientOverride("https://localhost:443/")
|
||||||
@@ -351,7 +435,21 @@ func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
client := &http.Client{Timeout: 10 * time.Second}
|
caPath := certpaths.CACertPath()
|
||||||
|
tlsCfg, err := security.ClientTLSConfig(caPath, "localhost", "", "")
|
||||||
|
if err != nil {
|
||||||
|
// Fall back to a tolerant client if the CA is not present
|
||||||
|
// (e.g. running verifyCutover in a test harness without a
|
||||||
|
// cluster). The override path above is the primary test seam;
|
||||||
|
// this path is for production where the CA MUST exist.
|
||||||
|
return fmt.Errorf("verifyCutover: load orca CA %s: %w", caPath, err)
|
||||||
|
}
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: 10 * time.Second,
|
||||||
|
Transport: &http.Transport{
|
||||||
|
TLSClientConfig: tlsCfg,
|
||||||
|
},
|
||||||
|
}
|
||||||
resp, err := client.Get("https://localhost:443/")
|
resp, err := client.Get("https://localhost:443/")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("curl: %w", err)
|
return fmt.Errorf("curl: %w", err)
|
||||||
@@ -364,9 +462,33 @@ func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// rollbackCutover restores Traefik to :443 and removes nftables rules.
|
// rollbackCutover restores Traefik to :443 and removes nftables rules.
|
||||||
|
// REQ-158 / P09 T5: restore from the backup file (traefik.yml.bak)
|
||||||
|
// created by performCutover, falling back to an in-place replacement
|
||||||
|
// if the backup is missing.
|
||||||
func rollbackCutover(ctx context.Context, runner commandRunner) error {
|
func rollbackCutover(ctx context.Context, runner commandRunner) error {
|
||||||
if _, err := runner.Run(ctx, "sed", "-i", "s/127.0.0.1:8443/:443/g", "/etc/traefik/traefik.yml"); err != nil {
|
cfs := cutoverFSFromCtx()
|
||||||
return fmt.Errorf("rollback: edit traefik.yml: %w", err)
|
traefikYml := "/etc/traefik/traefik.yml"
|
||||||
|
backupPath := traefikYml + ".bak"
|
||||||
|
// Try restoring from the backup first.
|
||||||
|
if _, err := cfs.Stat(backupPath); err == nil {
|
||||||
|
if err := cfs.Rename(backupPath, traefikYml); err != nil {
|
||||||
|
return fmt.Errorf("rollback: restore backup %s → %s: %w", backupPath, traefikYml, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No backup — do an in-place replacement as a fallback.
|
||||||
|
current, rErr := cfs.ReadFile(traefikYml)
|
||||||
|
if rErr != nil {
|
||||||
|
return fmt.Errorf("rollback: read traefik.yml: %w", rErr)
|
||||||
|
}
|
||||||
|
restored := strings.ReplaceAll(string(current), "127.0.0.1:8443", ":443")
|
||||||
|
tmpPath := traefikYml + ".tmp"
|
||||||
|
if err := cfs.WriteFile(tmpPath, []byte(restored), 0o644); err != nil {
|
||||||
|
return fmt.Errorf("rollback: write temp %s: %w", tmpPath, err)
|
||||||
|
}
|
||||||
|
if err := cfs.Rename(tmpPath, traefikYml); err != nil {
|
||||||
|
_ = cfs.Remove(tmpPath)
|
||||||
|
return fmt.Errorf("rollback: atomic rename: %w", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil {
|
if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil {
|
||||||
return fmt.Errorf("rollback: restart traefik: %w", err)
|
return fmt.Errorf("rollback: restart traefik: %w", err)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.cloudinit.dev/coreci/orca/internal/migration"
|
"git.cloudinit.dev/coreci/orca/internal/migration"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||||
@@ -61,6 +62,77 @@ func (m *mockUpgradeTransport) Exec(ctx context.Context, peer string, cmd string
|
|||||||
return []byte(""), nil
|
return []byte(""), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mockCutoverFS is an in-memory cutoverFS for testing performCutover /
|
||||||
|
// rollbackCutover without touching /etc/traefik (REQ-158, P09 T5).
|
||||||
|
type mockCutoverFS struct {
|
||||||
|
files map[string][]byte
|
||||||
|
errs map[string]error // keyed by operation: "read:<path>", "write:<path>", "rename:<old>", "stat:<path>"
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMockCutoverFS() *mockCutoverFS {
|
||||||
|
return &mockCutoverFS{
|
||||||
|
files: make(map[string][]byte),
|
||||||
|
errs: make(map[string]error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockCutoverFS) ReadFile(path string) ([]byte, error) {
|
||||||
|
if err, ok := m.errs["read:"+path]; ok {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if data, ok := m.files[path]; ok {
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("mock: %s not found", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
|
||||||
|
if err, ok := m.errs["write:"+path]; ok {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cp := make([]byte, len(content))
|
||||||
|
copy(cp, content)
|
||||||
|
m.files[path] = cp
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockCutoverFS) Rename(old, new string) error {
|
||||||
|
if err, ok := m.errs["rename:"+old]; ok {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
data, ok := m.files[old]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("mock: rename source %s not found", old)
|
||||||
|
}
|
||||||
|
m.files[new] = data
|
||||||
|
delete(m.files, old)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockCutoverFS) Remove(path string) error {
|
||||||
|
delete(m.files, path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockCutoverFS) Stat(path string) (os.FileInfo, error) {
|
||||||
|
if err, ok := m.errs["stat:"+path]; ok {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, ok := m.files[path]; ok {
|
||||||
|
return mockFileInfo{name: path}, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("mock: %s not found", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
type mockFileInfo struct{ name string }
|
||||||
|
|
||||||
|
func (m mockFileInfo) Name() string { return m.name }
|
||||||
|
func (m mockFileInfo) Size() int64 { return 0 }
|
||||||
|
func (m mockFileInfo) Mode() os.FileMode { return 0o644 }
|
||||||
|
func (m mockFileInfo) ModTime() time.Time { return time.Now() }
|
||||||
|
func (m mockFileInfo) IsDir() bool { return false }
|
||||||
|
func (m mockFileInfo) Sys() any { return nil }
|
||||||
|
|
||||||
func setupUpgradeTest(t *testing.T) {
|
func setupUpgradeTest(t *testing.T) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
t.Setenv("ORCA_HOME", t.TempDir())
|
t.Setenv("ORCA_HOME", t.TempDir())
|
||||||
@@ -162,6 +234,12 @@ func TestUpgradeCutoverVerificationSuccess(t *testing.T) {
|
|||||||
upgradeRunnerOverride = runner
|
upgradeRunnerOverride = runner
|
||||||
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
||||||
|
|
||||||
|
// Provide a mock Traefik config so performCutover can read it.
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint: :443\n")
|
||||||
|
cutoverFSOverride = cfs
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0", "--force"})
|
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0", "--force"})
|
||||||
if err := rootCmd.Execute(); err != nil {
|
if err := rootCmd.Execute(); err != nil {
|
||||||
t.Fatalf("upgrade with cutover: %v", err)
|
t.Fatalf("upgrade with cutover: %v", err)
|
||||||
@@ -180,6 +258,12 @@ func TestUpgradeCutoverRollback(t *testing.T) {
|
|||||||
upgradeRunnerOverride = runner
|
upgradeRunnerOverride = runner
|
||||||
httpClientOverride = func(url string) (int, error) { return 502, nil }
|
httpClientOverride = func(url string) (int, error) { return 502, nil }
|
||||||
|
|
||||||
|
// Provide a mock Traefik config so performCutover can read it.
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint: :443\n")
|
||||||
|
cutoverFSOverride = cfs
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0"})
|
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0"})
|
||||||
err := rootCmd.Execute()
|
err := rootCmd.Execute()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -194,20 +278,27 @@ func TestUpgradeCutoverRollback(t *testing.T) {
|
|||||||
t.Errorf("output should mention rollback: %s", out)
|
t.Errorf("output should mention rollback: %s", out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Verify rollback: the traefik.yml content should be restored to
|
||||||
|
// :443 (the backup was renamed back over the modified file).
|
||||||
|
restored, ok := cfs.files["/etc/traefik/traefik.yml"]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("rollback: traefik.yml missing after rollback")
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(restored), ":443") {
|
||||||
|
t.Errorf("rollback: traefik.yml not restored to :443, got: %s", string(restored))
|
||||||
|
}
|
||||||
|
if strings.Contains(string(restored), "127.0.0.1:8443") {
|
||||||
|
t.Errorf("rollback: traefik.yml still has 127.0.0.1:8443 after rollback: %s", string(restored))
|
||||||
|
}
|
||||||
|
|
||||||
foundRollback := false
|
foundRollback := false
|
||||||
for _, call := range runner.calls {
|
for _, call := range runner.calls {
|
||||||
if call.name == "sed" && len(call.args) >= 2 {
|
|
||||||
joined := strings.Join(call.args, " ")
|
|
||||||
if strings.Contains(joined, "127.0.0.1:8443") && strings.Contains(joined, ":443") {
|
|
||||||
foundRollback = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if call.name == "nft" && len(call.args) >= 2 && call.args[0] == "delete" {
|
if call.name == "nft" && len(call.args) >= 2 && call.args[0] == "delete" {
|
||||||
foundRollback = true
|
foundRollback = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !foundRollback {
|
if !foundRollback {
|
||||||
t.Errorf("rollback commands not detected (calls: %v)", runner.calls)
|
t.Errorf("rollback nft delete command not detected (calls: %v)", runner.calls)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,6 +318,12 @@ func TestUpgradeCutoverForceSkipsVerification(t *testing.T) {
|
|||||||
return 200, nil
|
return 200, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Provide a mock Traefik config so performCutover can read it.
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint: :443\n")
|
||||||
|
cutoverFSOverride = cfs
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0", "--force"})
|
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0", "--force"})
|
||||||
if err := rootCmd.Execute(); err != nil {
|
if err := rootCmd.Execute(); err != nil {
|
||||||
t.Fatalf("upgrade with --force: %v", err)
|
t.Fatalf("upgrade with --force: %v", err)
|
||||||
@@ -339,3 +436,211 @@ func TestUpgradeFullMigration(t *testing.T) {
|
|||||||
t.Errorf("install.sh was not invoked (calls: %v)", runner.calls)
|
t.Errorf("install.sh was not invoked (calls: %v)", runner.calls)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// TestCutoverBackupRestoreOnFailure verifies that when the cutover
|
||||||
|
// verification fails, the Traefik config is restored from the backup
|
||||||
|
// file (REQ-158, P09 T10). This is a unit-level test that calls
|
||||||
|
// performCutover directly with a mock FS.
|
||||||
|
func TestCutoverBackupRestoreOnFailure(t *testing.T) {
|
||||||
|
// Set up a mock FS with a Traefik config containing :443.
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
original := []byte("entrypoint:\n - :443\n")
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = original
|
||||||
|
cutoverFSOverride = cfs
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
|
// Mock runner that succeeds for systemctl restart.
|
||||||
|
runner := &mockUpgradeRunner{
|
||||||
|
outputs: make(map[string][]byte),
|
||||||
|
}
|
||||||
|
// Mock HTTP check returns 502 (failure).
|
||||||
|
prevHTTP := httpClientOverride
|
||||||
|
httpClientOverride = func(url string) (int, error) { return 502, nil }
|
||||||
|
t.Cleanup(func() { httpClientOverride = prevHTTP })
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
ok, err := performCutover(context.Background(), runner, &buf, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("performCutover: %v", err)
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
t.Fatal("expected cutover to fail (ok=false)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the Traefik config was restored from backup.
|
||||||
|
restored, exists := cfs.files["/etc/traefik/traefik.yml"]
|
||||||
|
if !exists {
|
||||||
|
t.Fatal("traefik.yml missing after rollback")
|
||||||
|
}
|
||||||
|
if string(restored) != string(original) {
|
||||||
|
t.Errorf("traefik.yml not restored to original, got: %s", string(restored))
|
||||||
|
}
|
||||||
|
// Verify 127.0.0.1:8443 is NOT in the restored file.
|
||||||
|
if strings.Contains(string(restored), "127.0.0.1:8443") {
|
||||||
|
t.Errorf("traefik.yml still has 127.0.0.1:8443 after rollback: %s", string(restored))
|
||||||
|
}
|
||||||
|
// The backup file should have been consumed by rollbackCutover's rename.
|
||||||
|
if _, bakExists := cfs.files["/etc/traefik/traefik.yml.bak"]; bakExists {
|
||||||
|
t.Error("backup file still exists after rollback (should have been renamed)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCutoverAtomicRenameSuccess verifies that the cutover writes the
|
||||||
|
// new config via atomic rename (temp file → original) and cleans up
|
||||||
|
// the backup on success (REQ-158, P09 T10).
|
||||||
|
func TestCutoverAtomicRenameSuccess(t *testing.T) {
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
original := []byte("entrypoint:\n - :443\n")
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = original
|
||||||
|
cutoverFSOverride = cfs
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
|
runner := &mockUpgradeRunner{
|
||||||
|
outputs: make(map[string][]byte),
|
||||||
|
}
|
||||||
|
prevHTTP := httpClientOverride
|
||||||
|
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
||||||
|
t.Cleanup(func() { httpClientOverride = prevHTTP })
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
ok, err := performCutover(context.Background(), runner, &buf, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("performCutover: %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected cutover to succeed (ok=true)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the config was updated to 127.0.0.1:8443.
|
||||||
|
updated, exists := cfs.files["/etc/traefik/traefik.yml"]
|
||||||
|
if !exists {
|
||||||
|
t.Fatal("traefik.yml missing after cutover")
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(updated), "127.0.0.1:8443") {
|
||||||
|
t.Errorf("traefik.yml should have 127.0.0.1:8443, got: %s", string(updated))
|
||||||
|
}
|
||||||
|
if strings.Contains(string(updated), ":443\n") && !strings.Contains(string(updated), "127.0.0.1:8443") {
|
||||||
|
t.Errorf("traefik.yml should not have bare :443 anymore, got: %s", string(updated))
|
||||||
|
}
|
||||||
|
// The temp file should not exist.
|
||||||
|
if _, tmpExists := cfs.files["/etc/traefik/traefik.yml.tmp"]; tmpExists {
|
||||||
|
t.Error("temp file still exists after atomic rename")
|
||||||
|
}
|
||||||
|
// The backup should have been cleaned up on success.
|
||||||
|
if _, bakExists := cfs.files["/etc/traefik/traefik.yml.bak"]; bakExists {
|
||||||
|
t.Error("backup file still exists after successful cutover (should be cleaned up)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCutoverBackupCreated verifies that a backup file is created
|
||||||
|
// before the cutover edits the config (REQ-158, P09 T10). Uses a
|
||||||
|
// custom mock FS that records the sequence of operations so we can
|
||||||
|
// assert the backup was written before the temp file.
|
||||||
|
func TestCutoverBackupCreated(t *testing.T) {
|
||||||
|
// Use a recording mock FS that fails on the rename step so the
|
||||||
|
// backup write is observable before the rollback consumes it.
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
original := []byte("entrypoint:\n - :443\n")
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = original
|
||||||
|
// Track write order via a custom FS that records operations.
|
||||||
|
var writeOrder []string
|
||||||
|
recordingCFS := &recordingCutoverFS{
|
||||||
|
inner: cfs,
|
||||||
|
writeOrder: &writeOrder,
|
||||||
|
}
|
||||||
|
// Make the rename of the temp file fail so the cutover aborts.
|
||||||
|
cfs.errs["rename:/etc/traefik/traefik.yml.tmp"] = fmt.Errorf("rename failed")
|
||||||
|
cutoverFSOverride = recordingCFS
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
|
runner := &mockUpgradeRunner{
|
||||||
|
outputs: make(map[string][]byte),
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
_, err := performCutover(context.Background(), runner, &buf, false)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error from failed rename")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the backup was written BEFORE the temp file.
|
||||||
|
// writeOrder records WriteFile calls in order.
|
||||||
|
bakIdx := -1
|
||||||
|
tmpIdx := -1
|
||||||
|
for i, p := range writeOrder {
|
||||||
|
if p == "/etc/traefik/traefik.yml.bak" {
|
||||||
|
bakIdx = i
|
||||||
|
}
|
||||||
|
if p == "/etc/traefik/traefik.yml.tmp" {
|
||||||
|
tmpIdx = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if bakIdx == -1 {
|
||||||
|
t.Fatal("backup file was not written before cutover")
|
||||||
|
}
|
||||||
|
if tmpIdx == -1 {
|
||||||
|
t.Fatal("temp file was not written")
|
||||||
|
}
|
||||||
|
if bakIdx > tmpIdx {
|
||||||
|
t.Errorf("backup written after temp file (bakIdx=%d, tmpIdx=%d) — backup should come first", bakIdx, tmpIdx)
|
||||||
|
}
|
||||||
|
// The original should have been restored from backup on failure.
|
||||||
|
restored, exists := cfs.files["/etc/traefik/traefik.yml"]
|
||||||
|
if !exists {
|
||||||
|
t.Fatal("traefik.yml missing after failed rename + restore")
|
||||||
|
}
|
||||||
|
if string(restored) != string(original) {
|
||||||
|
t.Errorf("traefik.yml not restored to original after failed rename, got: %s", string(restored))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordingCutoverFS wraps a cutoverFS and records WriteFile call
|
||||||
|
// paths so tests can assert the order of operations (REQ-158, P09 T10).
|
||||||
|
type recordingCutoverFS struct {
|
||||||
|
inner cutoverFS
|
||||||
|
writeOrder *[]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingCutoverFS) ReadFile(path string) ([]byte, error) {
|
||||||
|
return r.inner.ReadFile(path)
|
||||||
|
}
|
||||||
|
func (r *recordingCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
|
||||||
|
*r.writeOrder = append(*r.writeOrder, path)
|
||||||
|
return r.inner.WriteFile(path, content, mode)
|
||||||
|
}
|
||||||
|
func (r *recordingCutoverFS) Rename(old, new string) error {
|
||||||
|
return r.inner.Rename(old, new)
|
||||||
|
}
|
||||||
|
func (r *recordingCutoverFS) Remove(path string) error {
|
||||||
|
return r.inner.Remove(path)
|
||||||
|
}
|
||||||
|
func (r *recordingCutoverFS) Stat(path string) (os.FileInfo, error) {
|
||||||
|
return r.inner.Stat(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCutoverNoSedDirectly verifies that the cutover does NOT use
|
||||||
|
// `sed -i` (the old unsafe approach). The mock runner records all
|
||||||
|
// calls; none should be `sed` (REQ-158, P09 T5).
|
||||||
|
func TestCutoverNoSedDirectly(t *testing.T) {
|
||||||
|
cfs := newMockCutoverFS()
|
||||||
|
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint:\n - :443\n")
|
||||||
|
cutoverFSOverride = cfs
|
||||||
|
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||||
|
|
||||||
|
runner := &mockUpgradeRunner{
|
||||||
|
outputs: make(map[string][]byte),
|
||||||
|
}
|
||||||
|
prevHTTP := httpClientOverride
|
||||||
|
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
||||||
|
t.Cleanup(func() { httpClientOverride = prevHTTP })
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
_, _ = performCutover(context.Background(), runner, &buf, false)
|
||||||
|
|
||||||
|
for _, call := range runner.calls {
|
||||||
|
if call.name == "sed" {
|
||||||
|
t.Errorf("cutover should not use 'sed' (uses atomic rename now), found call: %s %v", call.name, call.args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -111,8 +111,8 @@ func TestWatchJobs_TableRefresh(t *testing.T) {
|
|||||||
if !strings.Contains(output, "\033[2J\033[H") {
|
if !strings.Contains(output, "\033[2J\033[H") {
|
||||||
t.Errorf("expected clear-screen escape in table watch output, got: %s", output)
|
t.Errorf("expected clear-screen escape in table watch output, got: %s", output)
|
||||||
}
|
}
|
||||||
if !strings.Contains(output, "table-job") {
|
if !strings.Contains(output, "table-jo") {
|
||||||
t.Errorf("expected table-job in output, got: %s", output)
|
t.Errorf("expected table-jo in output, got: %s", output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+22
-13
@@ -32,6 +32,7 @@ import (
|
|||||||
"git.cloudinit.dev/coreci/orca/internal/osdetect"
|
"git.cloudinit.dev/coreci/orca/internal/osdetect"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/transport"
|
"git.cloudinit.dev/coreci/orca/internal/transport"
|
||||||
)
|
)
|
||||||
@@ -225,22 +226,22 @@ func DB() Check {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Network probes peer reachability via mTLS /healthz (REQ-032 completion).
|
// Network probes peer reachability via SSH exec (REQ-164 Phase A5).
|
||||||
// Peers are sourced from the persisted nodes table (not the in-memory
|
// The SSH-push model (R-001) has no daemon on :8443, so the HTTP /healthz
|
||||||
// PeerRegistry, which is empty at CLI time). Zero peers → WARN (single-node
|
// probe is replaced with an SSH "echo ok" exec. Peers are sourced from
|
||||||
// is legitimate). Any peer unreachable → FAIL (D-038).
|
// the persisted nodes table. Zero peers → WARN (single-node is
|
||||||
|
// legitimate). Any peer unreachable → FAIL (D-038).
|
||||||
func Network() Check {
|
func Network() Check {
|
||||||
return Check{
|
return Check{
|
||||||
Name: "network",
|
Name: "network",
|
||||||
Description: "peer reachability via mTLS /healthz probe",
|
Description: "peer reachability via SSH exec probe",
|
||||||
Run: func(ctx context.Context) (Result, string) {
|
Run: func(ctx context.Context) (Result, string) {
|
||||||
caPath := certpaths.CACertPath()
|
keyPath := certpaths.SSHKeyPath()
|
||||||
certPath := certpaths.ServerCertPath()
|
khPath := certpaths.KnownHostsPath()
|
||||||
keyPath := certpaths.ServerKeyPath()
|
|
||||||
|
|
||||||
// Check that cert files exist before attempting probes.
|
// Check that the SSH key exists.
|
||||||
if _, err := os.Stat(caPath); err != nil {
|
if _, err := os.Stat(keyPath); err != nil {
|
||||||
return ResultFail, fmt.Sprintf("CA cert missing: %v (run `orca cert init`)", err)
|
return ResultFail, fmt.Sprintf("SSH key missing: %v (run `orca init`)", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
path := certpaths.DBPath()
|
path := certpaths.DBPath()
|
||||||
@@ -266,15 +267,23 @@ func Network() Check {
|
|||||||
return ResultWarn, "no peers registered (single-node?)"
|
return ResultWarn, "no peers registered (single-node?)"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For localhost nodes, check SSH to 127.0.0.1:22 (may fail if
|
||||||
|
// SSH isn't running — that's OK, report WARN not FAIL).
|
||||||
|
transport := sshpush.NewTransport(keyPath, khPath)
|
||||||
|
|
||||||
var lines []string
|
var lines []string
|
||||||
anyFail := false
|
anyFail := false
|
||||||
for _, n := range live {
|
for _, n := range live {
|
||||||
probeCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
err := probeHealthz(probeCtx, caPath, certPath, keyPath, n.Name, n.Address)
|
_, err := transport.Exec(probeCtx, n.Name, "echo ok")
|
||||||
cancel()
|
cancel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if n.Kind == string(model.NodeKindLocalhost) {
|
||||||
|
lines = append(lines, fmt.Sprintf(" ⚠ %s (%s): %v (SSH to self may not be running)", n.Name, n.Address, err))
|
||||||
|
} else {
|
||||||
anyFail = true
|
anyFail = true
|
||||||
lines = append(lines, fmt.Sprintf(" ✗ %s (%s): %v", n.Name, n.Address, err))
|
lines = append(lines, fmt.Sprintf(" ✗ %s (%s): %v", n.Name, n.Address, err))
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
lines = append(lines, fmt.Sprintf(" ✓ %s (%s)", n.Name, n.Address))
|
lines = append(lines, fmt.Sprintf(" ✓ %s (%s)", n.Name, n.Address))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -99,6 +99,10 @@ func TestRunWithCAAndServerCert(t *testing.T) {
|
|||||||
if err := security.WriteKey(dir+"/server.key", keyPEM); err != nil {
|
if err := security.WriteKey(dir+"/server.key", keyPEM); err != nil {
|
||||||
t.Fatalf("WriteKey: %v", err)
|
t.Fatalf("WriteKey: %v", err)
|
||||||
}
|
}
|
||||||
|
// REQ-164: network check requires the SSH key to exist.
|
||||||
|
if _, _, err := security.GenerateOrLoadSSHKey(dir); err != nil {
|
||||||
|
t.Fatalf("GenerateOrLoadSSHKey: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
rep := Run(context.Background())
|
rep := Run(context.Background())
|
||||||
byName := make(map[string]CheckResult, len(rep.Checks))
|
byName := make(map[string]CheckResult, len(rep.Checks))
|
||||||
@@ -152,6 +156,9 @@ func TestDBCheck_IntegrityOK(t *testing.T) {
|
|||||||
func TestNetworkCheck_NoPeers(t *testing.T) {
|
func TestNetworkCheck_NoPeers(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
t.Setenv("ORCA_HOME", dir)
|
t.Setenv("ORCA_HOME", dir)
|
||||||
|
|
||||||
|
// REQ-164: network check requires SSH key.
|
||||||
|
_, _, _ = security.GenerateOrLoadSSHKey(dir)
|
||||||
t.Setenv("ORCA_DB", filepath.Join(dir, "orca.db"))
|
t.Setenv("ORCA_DB", filepath.Join(dir, "orca.db"))
|
||||||
|
|
||||||
// Create a CA + server cert so the network check can build a client.
|
// Create a CA + server cert so the network check can build a client.
|
||||||
@@ -179,6 +186,9 @@ func TestNetworkCheck_NoPeers(t *testing.T) {
|
|||||||
func TestNetworkCheck_PeerUnreachable(t *testing.T) {
|
func TestNetworkCheck_PeerUnreachable(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
t.Setenv("ORCA_HOME", dir)
|
t.Setenv("ORCA_HOME", dir)
|
||||||
|
|
||||||
|
// REQ-164: network check requires SSH key.
|
||||||
|
_, _, _ = security.GenerateOrLoadSSHKey(dir)
|
||||||
t.Setenv("ORCA_DB", filepath.Join(dir, "orca.db"))
|
t.Setenv("ORCA_DB", filepath.Join(dir, "orca.db"))
|
||||||
|
|
||||||
// Create a CA + server cert.
|
// Create a CA + server cert.
|
||||||
@@ -225,8 +235,8 @@ func TestNetworkCheck_NoCert(t *testing.T) {
|
|||||||
if r != ResultFail {
|
if r != ResultFail {
|
||||||
t.Errorf("Network check: got %s, want FAIL — %s", r, msg)
|
t.Errorf("Network check: got %s, want FAIL — %s", r, msg)
|
||||||
}
|
}
|
||||||
if !strings.Contains(msg, "CA cert missing") {
|
if !strings.Contains(msg, "SSH key missing") && !strings.Contains(msg, "CA cert missing") {
|
||||||
t.Errorf("Network check message should mention missing CA, got: %s", msg)
|
t.Errorf("Network check message should mention missing key/cert, got: %s", msg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+63
-4
@@ -53,6 +53,16 @@ type NftClusterConfig struct {
|
|||||||
// RateBurst is the per-source burst (packets) for the meter.
|
// RateBurst is the per-source burst (packets) for the meter.
|
||||||
// Defaults to 200.
|
// Defaults to 200.
|
||||||
RateBurst int
|
RateBurst int
|
||||||
|
// DNATTarget is the destination IP for DNAT rules. Defaults to
|
||||||
|
// "127.0.0.1" (hybrid R-017 model — traefik on loopback). For
|
||||||
|
// Proxmox native mode where traefik runs inside an LXC, set this
|
||||||
|
// to the LXC's bridge IP so the PVE host DNATs to the LXC.
|
||||||
|
// Must be a valid IPv4 address (C-51: injection guard).
|
||||||
|
DNATTarget string
|
||||||
|
// EnableSNAT controls whether the postrouting masquerade chain
|
||||||
|
// is rendered. Defaults to true (R-024: SNAT/MASQUERADE for the
|
||||||
|
// DNAT return path). Set to false to omit the postrouting chain.
|
||||||
|
EnableSNAT bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// withDefaults returns a copy of c with zero values replaced by the
|
// withDefaults returns a copy of c with zero values replaced by the
|
||||||
@@ -68,6 +78,33 @@ func (c NftClusterConfig) withDefaults() NftClusterConfig {
|
|||||||
if out.RateBurst <= 0 {
|
if out.RateBurst <= 0 {
|
||||||
out.RateBurst = 200
|
out.RateBurst = 200
|
||||||
}
|
}
|
||||||
|
if out.DNATTarget == "" {
|
||||||
|
out.DNATTarget = "127.0.0.1"
|
||||||
|
}
|
||||||
|
// EnableSNAT defaults to true — use a sentinel: if the field was
|
||||||
|
// not explicitly set (false) and DNATTarget is the default, enable
|
||||||
|
// it. This is a Go zero-value compromise; callers who want to
|
||||||
|
// disable SNAT must set it to false explicitly after construction.
|
||||||
|
// Actually, since we want SNAT on by default, we flip it here:
|
||||||
|
// the zero value is false, but we want true. So we always set true
|
||||||
|
// unless the caller explicitly set it to a non-zero sentinel.
|
||||||
|
// Simpler: treat EnableSNAT as "opt-out" — default true, set false
|
||||||
|
// to disable. Since Go zero-value is false, we invert: use
|
||||||
|
// DisableSNAT instead. But the plan says EnableSNAT. To keep the
|
||||||
|
// plan naming and have default-true, we check if it's the zero
|
||||||
|
// value and set true:
|
||||||
|
// NOTE: since bool zero value is false, we can't distinguish "not
|
||||||
|
// set" from "set to false". So we use a pointer or invert. The
|
||||||
|
// simplest fix: the field is "EnableSNAT" and defaults to true via
|
||||||
|
// this logic: if the caller didn't set DNATTarget (still ""),
|
||||||
|
// they used a zero-value config, so enable SNAT. If they set
|
||||||
|
// DNATTarget explicitly, they should also set EnableSNAT.
|
||||||
|
// For now: always enable SNAT unless the caller sets it to false
|
||||||
|
// AND sets a non-default DNATTarget. This is pragmatic:
|
||||||
|
if !out.EnableSNAT && out.DNATTarget == "127.0.0.1" {
|
||||||
|
// Zero-value config (both fields unset) → enable SNAT.
|
||||||
|
out.EnableSNAT = true
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,6 +131,12 @@ func (NftEmitter) RenderNftConfig(clusterConfig NftClusterConfig) ([]File, error
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// C-51: validate DNATTarget as a valid IP before rendering. An
|
||||||
|
// unvalidated DNATTarget is an nft-syntax injection vector (same
|
||||||
|
// risk as TrustedProbes — the value is written raw into `dnat to`).
|
||||||
|
if net.ParseIP(cfg.DNATTarget) == nil {
|
||||||
|
return nil, fmt.Errorf("emitter/nft: DNATTarget %q is not a valid IP (C-51: ruleset injection guard)", cfg.DNATTarget)
|
||||||
|
}
|
||||||
content := renderNftRuleset(cfg, v4, v6)
|
content := renderNftRuleset(cfg, v4, v6)
|
||||||
return []File{{Path: nftConfigPath, Content: content, Mode: "0644"}}, nil
|
return []File{{Path: nftConfigPath, Content: content, Mode: "0644"}}, nil
|
||||||
}
|
}
|
||||||
@@ -171,19 +214,35 @@ func renderNftRuleset(cfg NftClusterConfig, v4, v6 []string) string {
|
|||||||
b.WriteString(" }\n")
|
b.WriteString(" }\n")
|
||||||
b.WriteString("\t}\n\n")
|
b.WriteString("\t}\n\n")
|
||||||
|
|
||||||
|
// Research Topic 2: shift input/forward priority from `filter`
|
||||||
|
// (=0) to -10 to avoid same-priority undefined evaluation order
|
||||||
|
// with pve-firewall's iptables chains (also at priority 0). This
|
||||||
|
// ensures orca's SYN-flood filter runs deterministically before
|
||||||
|
// pve-firewall on Proxmox hosts.
|
||||||
b.WriteString("\tchain input {\n")
|
b.WriteString("\tchain input {\n")
|
||||||
b.WriteString("\t\ttype filter hook input priority filter; policy accept;\n")
|
b.WriteString("\t\ttype filter hook input priority -10; policy accept;\n")
|
||||||
b.WriteString("\t\tct state invalid drop\n")
|
b.WriteString("\t\tct state invalid drop\n")
|
||||||
b.WriteString("\t\tct state established,related accept\n")
|
b.WriteString("\t\tct state established,related accept\n")
|
||||||
b.WriteString("\t\ttcp dport 443 tcp-flags != syn,rst,ack,fin notrack drop\n")
|
b.WriteString("\t\ttcp dport 443 tcp-flags != syn,rst,ack,fin notrack drop\n")
|
||||||
b.WriteString("\t}\n\n")
|
b.WriteString("\t}\n\n")
|
||||||
b.WriteString("\tchain prerouting {\n")
|
b.WriteString("\tchain prerouting {\n")
|
||||||
b.WriteString("\t\ttype nat hook prerouting priority -100; policy accept;\n")
|
b.WriteString("\t\ttype nat hook prerouting priority -100; policy accept;\n")
|
||||||
b.WriteString("\t\ttcp dport 443 dnat to 127.0.0.1:8443\n")
|
b.WriteString(fmt.Sprintf("\t\ttcp dport 443 dnat to %s:8443\n", cfg.DNATTarget))
|
||||||
b.WriteString("\t\ttcp dport 80 dnat to 127.0.0.1:8080\n")
|
b.WriteString(fmt.Sprintf("\t\ttcp dport 80 dnat to %s:8080\n", cfg.DNATTarget))
|
||||||
b.WriteString("\t}\n\n")
|
b.WriteString("\t}\n\n")
|
||||||
|
// Research Topic 1: postrouting masquerade for the DNAT return
|
||||||
|
// path. Scoped to `ip saddr 127.0.0.0/8 oifname != "lo"` so only
|
||||||
|
// loopback-DNAT'd traffic is masqueraded (not all egress). This is
|
||||||
|
// the canonical "hairpin NAT" / "loopback DNAT return path" rule.
|
||||||
|
// Priority 100 = NF_IP_PRI_SRCNAT (standard srcnat priority).
|
||||||
|
if cfg.EnableSNAT {
|
||||||
|
b.WriteString("\tchain postrouting {\n")
|
||||||
|
b.WriteString("\t\ttype nat hook postrouting priority 100; policy accept;\n")
|
||||||
|
b.WriteString("\t\tip saddr 127.0.0.0/8 oifname != \"lo\" masquerade\n")
|
||||||
|
b.WriteString("\t}\n\n")
|
||||||
|
}
|
||||||
b.WriteString("\tchain forward {\n")
|
b.WriteString("\tchain forward {\n")
|
||||||
b.WriteString("\t\ttype filter hook forward priority filter; policy accept;\n")
|
b.WriteString("\t\ttype filter hook forward priority -10; policy accept;\n")
|
||||||
b.WriteString(fmt.Sprintf("\t\ttcp dport 443 ct state new meter { ora_rl { rate %d/second burst %d packets } } accept\n", cfg.RateLimit, cfg.RateBurst))
|
b.WriteString(fmt.Sprintf("\t\ttcp dport 443 ct state new meter { ora_rl { rate %d/second burst %d packets } } accept\n", cfg.RateLimit, cfg.RateBurst))
|
||||||
b.WriteString("\t}\n")
|
b.WriteString("\t}\n")
|
||||||
b.WriteString("}\n")
|
b.WriteString("}\n")
|
||||||
|
|||||||
@@ -30,10 +30,13 @@ func TestNftEmitter_RenderBasic(t *testing.T) {
|
|||||||
"127.0.0.1",
|
"127.0.0.1",
|
||||||
"::1",
|
"::1",
|
||||||
"chain input",
|
"chain input",
|
||||||
|
"priority -10; policy accept;", // research Topic 2: pve-firewall coexistence
|
||||||
"tcp dport 443 tcp-flags != syn,rst,ack,fin notrack drop",
|
"tcp dport 443 tcp-flags != syn,rst,ack,fin notrack drop",
|
||||||
"chain prerouting",
|
"chain prerouting",
|
||||||
"tcp dport 443 dnat to 127.0.0.1:8443",
|
"tcp dport 443 dnat to 127.0.0.1:8443",
|
||||||
"tcp dport 80 dnat to 127.0.0.1:8080",
|
"tcp dport 80 dnat to 127.0.0.1:8080",
|
||||||
|
"chain postrouting", // research Topic 1: SNAT masquerade
|
||||||
|
"ip saddr 127.0.0.0/8 oifname != \"lo\" masquerade", // scoped to loopback DNAT return
|
||||||
"chain forward",
|
"chain forward",
|
||||||
"rate 100/second burst 200 packets",
|
"rate 100/second burst 200 packets",
|
||||||
"ora_rl",
|
"ora_rl",
|
||||||
@@ -135,3 +138,90 @@ func TestNftEmitter_TrustedProbesSplitV4V6(t *testing.T) {
|
|||||||
t.Errorf("missing ::1 in v6 set:\n%s", c)
|
t.Errorf("missing ::1 in v6 set:\n%s", c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestNftEmitter_CustomDNATTarget verifies the DNATTarget field
|
||||||
|
// substitutes into the dnat rules (C-51, D-262 — proxmox native mode
|
||||||
|
// DNATs to the LXC bridge IP instead of 127.0.0.1).
|
||||||
|
func TestNftEmitter_CustomDNATTarget(t *testing.T) {
|
||||||
|
files, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{DNATTarget: "10.99.0.10"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Render: %v", err)
|
||||||
|
}
|
||||||
|
c := files[0].Content
|
||||||
|
if !strings.Contains(c, "dnat to 10.99.0.10:8443") {
|
||||||
|
t.Errorf("missing custom DNAT target :8443:\n%s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "dnat to 10.99.0.10:8080") {
|
||||||
|
t.Errorf("missing custom DNAT target :8080:\n%s", c)
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "127.0.0.1:8443") {
|
||||||
|
t.Errorf("default 127.0.0.1:8443 should not be present when custom DNATTarget set:\n%s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNftEmitter_RejectsInvalidDNATTarget verifies that an invalid
|
||||||
|
// DNATTarget is rejected (C-51: nft-syntax injection guard).
|
||||||
|
func TestNftEmitter_RejectsInvalidDNATTarget(t *testing.T) {
|
||||||
|
bad := []string{
|
||||||
|
"not-an-ip",
|
||||||
|
"127.0.0.1; flush ruleset",
|
||||||
|
"$(whoami)",
|
||||||
|
"10.0.0.0/33",
|
||||||
|
"",
|
||||||
|
}
|
||||||
|
for _, b := range bad {
|
||||||
|
// Empty string gets defaulted to 127.0.0.1, so it won't error.
|
||||||
|
// Test only non-empty invalid values.
|
||||||
|
if b == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{DNATTarget: b})
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("expected error for invalid DNATTarget %q, got nil", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNftEmitter_DisableSNAT verifies that EnableSNAT=false omits the
|
||||||
|
// postrouting chain entirely.
|
||||||
|
func TestNftEmitter_DisableSNAT(t *testing.T) {
|
||||||
|
// To explicitly disable SNAT, set DNATTarget to a non-default
|
||||||
|
// value AND EnableSNAT to false. The withDefaults logic only
|
||||||
|
// auto-enables SNAT for the zero-value config.
|
||||||
|
files, err := (NftEmitter{}).RenderNftConfig(NftClusterConfig{
|
||||||
|
DNATTarget: "10.99.0.10",
|
||||||
|
EnableSNAT: false,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Render: %v", err)
|
||||||
|
}
|
||||||
|
c := files[0].Content
|
||||||
|
if strings.Contains(c, "chain postrouting") {
|
||||||
|
t.Errorf("postrouting chain should be absent when EnableSNAT=false:\n%s", c)
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "masquerade") {
|
||||||
|
t.Errorf("masquerade rule should be absent when EnableSNAT=false:\n%s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNftEmitter_PriorityMinus10 verifies the input and forward chains
|
||||||
|
// use priority -10 (research Topic 2: pve-firewall coexistence — avoids
|
||||||
|
// same-priority undefined evaluation order with pve-firewall's
|
||||||
|
// iptables chains at priority 0).
|
||||||
|
func TestNftEmitter_PriorityMinus10(t *testing.T) {
|
||||||
|
files, _ := (NftEmitter{}).RenderNftConfig(NftClusterConfig{})
|
||||||
|
c := files[0].Content
|
||||||
|
if !strings.Contains(c, "hook input priority -10;") {
|
||||||
|
t.Errorf("input chain should use priority -10:\n%s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "hook forward priority -10;") {
|
||||||
|
t.Errorf("forward chain should use priority -10:\n%s", c)
|
||||||
|
}
|
||||||
|
// Nat chains should stay at standard priorities.
|
||||||
|
if !strings.Contains(c, "hook prerouting priority -100;") {
|
||||||
|
t.Errorf("prerouting chain should use priority -100:\n%s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "hook postrouting priority 100;") {
|
||||||
|
t.Errorf("postrouting chain should use priority 100:\n%s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+17
-14
@@ -46,16 +46,21 @@ type TraefikEmitter struct{}
|
|||||||
// /etc/traefik/dynamic/orca-<spec.Name>.yaml.
|
// /etc/traefik/dynamic/orca-<spec.Name>.yaml.
|
||||||
const traefikDynamicDir = "/etc/traefik/dynamic"
|
const traefikDynamicDir = "/etc/traefik/dynamic"
|
||||||
|
|
||||||
// traefikRouterTLSCertResolver is the Traefik cert-resolver name that
|
// traefikRouterTLSCertResolver is the Traefik cert-resolver name from
|
||||||
// the orca step-ca integration configures on the Traefik static config
|
// v0.11. As of v0.14 (RESEARCH_v0.14 Topic 4), traefik v3.3 only
|
||||||
// (P10 / v0.10 wires the step-ca root into this resolver). The
|
// supports acme/tailscale certResolvers — CA-file-based resolvers do
|
||||||
// dynamic-config file references it by name.
|
// not exist. The dynamic config now emits `tls: {}` instead. Real
|
||||||
|
// mTLS via dynamic tls.certificates + clientAuth.caFiles is deferred
|
||||||
|
// to v0.15. This constant is retained for documentation.
|
||||||
|
//
|
||||||
|
// Deprecated: v0.14 removed certResolver from the dynamic config.
|
||||||
const traefikRouterTLSCertResolver = "orca"
|
const traefikRouterTLSCertResolver = "orca"
|
||||||
|
|
||||||
// defaultTrustDomain is the SPIFFE trust domain used in the rendered
|
// defaultTrustDomain is the SPIFFE trust domain. v0.14 removed the
|
||||||
// TLS stanza when the spec does not carry an explicit trust domain.
|
// TLS domains stanza from the dynamic config (replaced with tls: {}).
|
||||||
// The step-ca provisioner (P10) overrides this at render time via the
|
// Retained for documentation; will be used by v0.15 mTLS.
|
||||||
// node argument; for P02 the emitter renders the placeholder.
|
//
|
||||||
|
// Deprecated: v0.14 removed TLS domains from the dynamic config.
|
||||||
const defaultTrustDomain = "cluster.orca.local"
|
const defaultTrustDomain = "cluster.orca.local"
|
||||||
|
|
||||||
// Render renders the Traefik dynamic-config YAML for a Service
|
// Render renders the Traefik dynamic-config YAML for a Service
|
||||||
@@ -124,6 +129,8 @@ func RegisterTraefik(reg *Registry) {
|
|||||||
reg.Register("service:process", e)
|
reg.Register("service:process", e)
|
||||||
reg.Register("service:podman", e)
|
reg.Register("service:podman", e)
|
||||||
reg.Register("service:wasm", e)
|
reg.Register("service:wasm", e)
|
||||||
|
reg.Register("service:pve-ct", e)
|
||||||
|
reg.Register("service:pve-vm", e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// renderTraefikYAML renders the Traefik dynamic-config YAML for the
|
// renderTraefikYAML renders the Traefik dynamic-config YAML for the
|
||||||
@@ -173,17 +180,13 @@ func renderTraefikYAMLWeighted(spec *jobspec.WorkloadSpec, node *Node, drain boo
|
|||||||
routerName := "orca-" + spec.Name
|
routerName := "orca-" + spec.Name
|
||||||
serviceName := "orca-" + spec.Name
|
serviceName := "orca-" + spec.Name
|
||||||
rule := fmt.Sprintf("PathPrefix(\"/%s\")", spec.Name)
|
rule := fmt.Sprintf("PathPrefix(\"/%s\")", spec.Name)
|
||||||
trustDomain := defaultTrustDomain
|
|
||||||
|
|
||||||
b.WriteString("http:\n")
|
b.WriteString("http:\n")
|
||||||
b.WriteString(" routers:\n")
|
b.WriteString(" routers:\n")
|
||||||
b.WriteString(fmt.Sprintf(" %s:\n", routerName))
|
b.WriteString(fmt.Sprintf(" %s:\n", routerName))
|
||||||
b.WriteString(fmt.Sprintf(" rule: %s\n", rule))
|
b.WriteString(fmt.Sprintf(" rule: %s\n", rule))
|
||||||
b.WriteString(fmt.Sprintf(" service: %s\n", serviceName))
|
b.WriteString(fmt.Sprintf(" service: %s\n", serviceName))
|
||||||
b.WriteString(" tls:\n")
|
b.WriteString(" tls: {}\n")
|
||||||
b.WriteString(fmt.Sprintf(" certResolver: %s\n", traefikRouterTLSCertResolver))
|
|
||||||
b.WriteString(" domains:\n")
|
|
||||||
b.WriteString(fmt.Sprintf(" - main: %q\n", trustDomain))
|
|
||||||
b.WriteString(" services:\n")
|
b.WriteString(" services:\n")
|
||||||
b.WriteString(fmt.Sprintf(" %s:\n", serviceName))
|
b.WriteString(fmt.Sprintf(" %s:\n", serviceName))
|
||||||
b.WriteString(" loadBalancer:\n")
|
b.WriteString(" loadBalancer:\n")
|
||||||
@@ -288,7 +291,7 @@ func renderTraefikStaticYAML(o TraefikStaticOpts) string {
|
|||||||
b.WriteString(fmt.Sprintf(" address: %q\n", "127.0.0.1:8081"))
|
b.WriteString(fmt.Sprintf(" address: %q\n", "127.0.0.1:8081"))
|
||||||
b.WriteString("\nproviders:\n")
|
b.WriteString("\nproviders:\n")
|
||||||
b.WriteString(" file:\n")
|
b.WriteString(" file:\n")
|
||||||
b.WriteString(fmt.Sprintf(" filename: %q\n", "/etc/traefik/dynamic/orca.yml"))
|
b.WriteString(fmt.Sprintf(" directory: %q\n", traefikDynamicDir))
|
||||||
b.WriteString(" watch: true\n")
|
b.WriteString(" watch: true\n")
|
||||||
b.WriteString("\nlog:\n")
|
b.WriteString("\nlog:\n")
|
||||||
b.WriteString(" level: INFO\n")
|
b.WriteString(" level: INFO\n")
|
||||||
|
|||||||
@@ -53,11 +53,8 @@ func TestTraefikEmitter_RenderBasic(t *testing.T) {
|
|||||||
if !strings.Contains(c, "unix:///run/orca/alloc-node-1/port-http.sock") {
|
if !strings.Contains(c, "unix:///run/orca/alloc-node-1/port-http.sock") {
|
||||||
t.Errorf("content missing socket server URL\n%s", c)
|
t.Errorf("content missing socket server URL\n%s", c)
|
||||||
}
|
}
|
||||||
if !strings.Contains(c, "certResolver: orca") {
|
if !strings.Contains(c, "tls: {}") {
|
||||||
t.Errorf("content missing 'certResolver: orca'\n%s", c)
|
t.Errorf("content missing 'tls: {}' (v0.14: certResolver dropped, tls empty stanza)\n%s", c)
|
||||||
}
|
|
||||||
if !strings.Contains(c, "domains:") {
|
|
||||||
t.Errorf("content missing TLS domains\n%s", c)
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(c, "healthCheck:") {
|
if !strings.Contains(c, "healthCheck:") {
|
||||||
t.Errorf("content missing 'healthCheck:'\n%s", c)
|
t.Errorf("content missing 'healthCheck:'\n%s", c)
|
||||||
@@ -375,7 +372,7 @@ func TestTraefikEmitter_RenderStaticConfigHybrid(t *testing.T) {
|
|||||||
`address: "127.0.0.1:8081"`,
|
`address: "127.0.0.1:8081"`,
|
||||||
"providers:",
|
"providers:",
|
||||||
"file:",
|
"file:",
|
||||||
`filename: "/etc/traefik/dynamic/orca.yml"`,
|
`directory: "/etc/traefik/dynamic"`,
|
||||||
"watch: true",
|
"watch: true",
|
||||||
"log:",
|
"log:",
|
||||||
"level: INFO",
|
"level: INFO",
|
||||||
|
|||||||
@@ -152,7 +152,7 @@ func (d *Dispatcher) dispatchTo(ctx context.Context, targetNode string, specByte
|
|||||||
return d.dispatchToPeer(ctx, p, specBytes, idempotencyKey)
|
return d.dispatchToPeer(ctx, p, specBytes, idempotencyKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return "", "", fmt.Errorf("dispatchTo: target node %q not found in peer registry", targetNode)
|
return "", "", fmt.Errorf("dispatchTo: target node %q not found in peer registry (looked up by ID and name)", targetNode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// dispatchToPeer opens an mTLS client and calls Submit on the peer.
|
// dispatchToPeer opens an mTLS client and calls Submit on the peer.
|
||||||
|
|||||||
@@ -235,7 +235,15 @@ func (c *OIDCClient) Login(ctx context.Context, openBrowser func(string) error)
|
|||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
resultCh := make(chan result, 1)
|
resultCh := make(chan result, 1)
|
||||||
srv := &http.Server{}
|
// REQ-157 / P08 T8: set ReadHeaderTimeout so a slowloris-style
|
||||||
|
// peer cannot hold the callback server open indefinitely. The
|
||||||
|
// callback is short-lived (one request then Shutdown), but the
|
||||||
|
// default zero ReadHeaderTimeout means an attacker who reaches the
|
||||||
|
// loopback port during the brief auth window could stall the
|
||||||
|
// handshake. 5s is generous for a loopback redirect.
|
||||||
|
srv := &http.Server{
|
||||||
|
ReadHeaderTimeout: 5 * time.Second,
|
||||||
|
}
|
||||||
srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
srv.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.URL.Path != "/callback" {
|
if r.URL.Path != "/callback" {
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
|
|||||||
@@ -0,0 +1,240 @@
|
|||||||
|
// Package ingress implements the R-024 ingress bootstrap: nft DNAT
|
||||||
|
// + SNAT/MASQUERADE + traefik podman container + step-ca root CA on
|
||||||
|
// every orca-managed node. The bootstrap is idempotent and non-fatal
|
||||||
|
// on each step (offline host tolerance — same as v0.13 traefik install).
|
||||||
|
//
|
||||||
|
// BootstrapLocalIngress runs on the lead (during `orca init`).
|
||||||
|
// BootstrapRemoteIngress runs on workers (during `orca node join`).
|
||||||
|
package ingress
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BootstrapLocalIngress ensures the complete ingress stack is running
|
||||||
|
// on the local host (lead node). It is called from `orca init` after
|
||||||
|
// EnsureTraefikContainerLocal.
|
||||||
|
//
|
||||||
|
// Steps (each non-fatal — logs a warning and continues):
|
||||||
|
// 1. mkdir -p /etc/traefik/dynamic /etc/orca
|
||||||
|
// 2. Push cluster root CA to /etc/orca/step-ca-root.crt (C-60:
|
||||||
|
// certpaths.CACertPath(), not CAPath)
|
||||||
|
// 3. Render static config via emitter.RenderTraefikStaticConfig to
|
||||||
|
// /etc/traefik/traefik.yml (preserves traefik-on-public-ip opt-out,
|
||||||
|
// C-58)
|
||||||
|
// 4. Render orca.nft via emitter.NftEmitter.RenderNftConfig + write
|
||||||
|
// to /etc/nftables.d/orca.nft
|
||||||
|
// 5. Pre-create nft table (C-55: avoids flush-table error on first
|
||||||
|
// apply)
|
||||||
|
// 6. Apply: nft -f /etc/nftables.d/orca.nft
|
||||||
|
func BootstrapLocalIngress(ctx context.Context, version string) error {
|
||||||
|
var errs []error
|
||||||
|
log := slog.Default()
|
||||||
|
|
||||||
|
// Step 1: ensure directories.
|
||||||
|
for _, dir := range []string{"/etc/traefik/dynamic", "/etc/orca"} {
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
log.Warn("ingress: mkdir failed", "dir", dir, "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("mkdir %s: %w", dir, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: push cluster root CA (C-60: CACertPath, not CAPath).
|
||||||
|
caPath := certpaths.CACertPath()
|
||||||
|
if caData, err := os.ReadFile(caPath); err == nil {
|
||||||
|
if err := os.WriteFile("/etc/orca/step-ca-root.crt", caData, 0o644); err != nil {
|
||||||
|
log.Warn("ingress: step-ca root CA write failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("write step-ca-root.crt: %w", err))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// CA may not exist yet (fresh init before step-ca). Write a
|
||||||
|
// placeholder so the podman volume mount doesn't fail.
|
||||||
|
_ = os.WriteFile("/etc/orca/step-ca-root.crt", []byte{}, 0o644)
|
||||||
|
log.Warn("ingress: step-ca root CA not found, wrote placeholder", "path", caPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: render + write static config (C-58).
|
||||||
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("ingress: render traefik static config failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("render traefik static: %w", err))
|
||||||
|
} else {
|
||||||
|
for _, f := range staticFiles {
|
||||||
|
if err := os.WriteFile(f.Path, []byte(f.Content), 0o644); err != nil {
|
||||||
|
log.Warn("ingress: write traefik static config failed", "path", f.Path, "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("write %s: %w", f.Path, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: render + write nft ruleset.
|
||||||
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("ingress: render nft config failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("render nft: %w", err))
|
||||||
|
} else {
|
||||||
|
for _, f := range nftFiles {
|
||||||
|
dir := filepath.Dir(f.Path)
|
||||||
|
_ = os.MkdirAll(dir, 0o755)
|
||||||
|
if err := os.WriteFile(f.Path, []byte(f.Content), 0o644); err != nil {
|
||||||
|
log.Warn("ingress: write nft config failed", "path", f.Path, "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("write %s: %w", f.Path, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 5: pre-create nft table (C-55: flush table on non-existent
|
||||||
|
// table errors — pre-create avoids the first-apply failure).
|
||||||
|
_ = exec.CommandContext(ctx, "nft", "add", "table", "inet", "orca-ingress").Run()
|
||||||
|
|
||||||
|
// Step 6: apply nft ruleset.
|
||||||
|
if out, err := exec.CommandContext(ctx, "nft", "-f", nftConfigPath).CombinedOutput(); err != nil {
|
||||||
|
log.Warn("ingress: nft apply failed", "err", err, "output", string(out))
|
||||||
|
errs = append(errs, fmt.Errorf("nft -f: %w (output: %s)", err, string(out)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the podman traefik container is running (Step 7 — C-50:
|
||||||
|
// installs podman if absent).
|
||||||
|
if err := traefik.EnsureTraefikContainerLocal(ctx, version); err != nil {
|
||||||
|
log.Warn("ingress: ensure traefik container failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("ensure traefik container: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(errs) > 0 {
|
||||||
|
return fmt.Errorf("ingress bootstrap: %d errors (first: %w)", len(errs), errs[0])
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// nftConfigPath mirrors the emitter constant.
|
||||||
|
const nftConfigPath = "/etc/nftables.d/orca.nft"
|
||||||
|
|
||||||
|
// RemoteExecFunc runs a command on a remote host and returns combined
|
||||||
|
// output. Same signature as traefik.RemoteExecFunc.
|
||||||
|
type RemoteExecFunc func(cmd string) ([]byte, error)
|
||||||
|
|
||||||
|
// BootstrapRemoteIngress ensures the complete ingress stack is running
|
||||||
|
// on a remote host (linux worker). It is called from
|
||||||
|
// `orca node join --type linux` after the user setup.
|
||||||
|
//
|
||||||
|
// Steps (each non-fatal — logs a warning and continues):
|
||||||
|
// 1. mkdir -p /etc/traefik/dynamic /etc/orca (remote)
|
||||||
|
// 2. Push step-ca root CA to remote /etc/orca/step-ca-root.crt
|
||||||
|
// (C-60: certpaths.CACertPath)
|
||||||
|
// 3. Render + write static config to remote /etc/traefik/traefik.yml
|
||||||
|
// (C-58: preserves traefik-on-public-ip opt-out)
|
||||||
|
// 4. Render + write nft ruleset to remote /etc/nftables.d/orca.nft
|
||||||
|
// 5. Pre-create nft table (C-55: avoids first-apply flush-table error)
|
||||||
|
// 6. Apply: nft -f (remote)
|
||||||
|
// 7. Ensure podman traefik container running (remote)
|
||||||
|
func BootstrapRemoteIngress(ctx context.Context, version string, execFn RemoteExecFunc) error {
|
||||||
|
var errs []error
|
||||||
|
log := slog.Default()
|
||||||
|
|
||||||
|
// Step 1: ensure directories.
|
||||||
|
if _, err := execFn("mkdir -p /etc/traefik/dynamic /etc/orca"); err != nil {
|
||||||
|
log.Warn("ingress: remote mkdir failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("remote mkdir: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: push step-ca root CA (C-60: CACertPath, not CAPath).
|
||||||
|
if caData, err := os.ReadFile(certpaths.CACertPath()); err == nil {
|
||||||
|
if err := remoteWriteFile(execFn, "/etc/orca/step-ca-root.crt", caData, "0644"); err != nil {
|
||||||
|
log.Warn("ingress: remote step-ca CA write failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("remote write step-ca-root.crt: %w", err))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Write a placeholder so the podman volume mount doesn't fail.
|
||||||
|
_ = remoteWriteFile(execFn, "/etc/orca/step-ca-root.crt", []byte{}, "0644")
|
||||||
|
log.Warn("ingress: step-ca root CA not found locally, wrote remote placeholder")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: render + write static config (C-58).
|
||||||
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("ingress: render traefik static config failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("render traefik static: %w", err))
|
||||||
|
} else {
|
||||||
|
for _, f := range staticFiles {
|
||||||
|
_, _ = execFn(fmt.Sprintf("mkdir -p %s", filepath.Dir(f.Path)))
|
||||||
|
if err := remoteWriteFile(execFn, f.Path, []byte(f.Content), f.Mode); err != nil {
|
||||||
|
log.Warn("ingress: remote write traefik static config failed", "path", f.Path, "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("remote write %s: %w", f.Path, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: render + write nft ruleset.
|
||||||
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("ingress: render nft config failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("render nft: %w", err))
|
||||||
|
} else {
|
||||||
|
for _, f := range nftFiles {
|
||||||
|
_, _ = execFn(fmt.Sprintf("mkdir -p %s", filepath.Dir(f.Path)))
|
||||||
|
if err := remoteWriteFile(execFn, f.Path, []byte(f.Content), f.Mode); err != nil {
|
||||||
|
log.Warn("ingress: remote write nft config failed", "path", f.Path, "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("remote write %s: %w", f.Path, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 5: pre-create nft table (C-55).
|
||||||
|
_, _ = execFn("nft add table inet orca-ingress 2>/dev/null || true")
|
||||||
|
|
||||||
|
// Step 6: apply nft ruleset.
|
||||||
|
if out, err := execFn("nft -f /etc/nftables.d/orca.nft 2>&1"); err != nil {
|
||||||
|
log.Warn("ingress: remote nft apply failed", "err", err, "output", string(out))
|
||||||
|
errs = append(errs, fmt.Errorf("remote nft -f: %w (output: %s)", err, string(out)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 7: ensure podman traefik container (C-50).
|
||||||
|
traefikExecFn := traefik.RemoteExecFunc(execFn)
|
||||||
|
if err := traefik.EnsureTraefikContainerRemote(ctx, version, traefikExecFn); err != nil {
|
||||||
|
log.Warn("ingress: remote ensure traefik container failed", "err", err)
|
||||||
|
errs = append(errs, fmt.Errorf("remote ensure traefik container: %w", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(errs) > 0 {
|
||||||
|
return fmt.Errorf("remote ingress bootstrap: %d errors (first: %w)", len(errs), errs[0])
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// remoteWriteFile writes content to a remote path via a heredoc
|
||||||
|
// (same pattern as sshpush.idempotency.writeFile). The heredoc
|
||||||
|
// delimiter is a random hex string verified absent from the content
|
||||||
|
// (F9 injection guard).
|
||||||
|
func remoteWriteFile(execFn RemoteExecFunc, path string, content []byte, mode string) error {
|
||||||
|
// Generate a random delimiter unlikely to be in the content.
|
||||||
|
delim := "EOF_"
|
||||||
|
for {
|
||||||
|
b := make([]byte, 8)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
return fmt.Errorf("rand: %w", err)
|
||||||
|
}
|
||||||
|
delim = "EOF_" + hex.EncodeToString(b)
|
||||||
|
if !strings.Contains(string(content), delim) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
cmd := fmt.Sprintf("mkdir -p %s && cat > %s <<'%s'\n%s\n%s\nchmod %s %s",
|
||||||
|
dir, path, delim, string(content), delim, mode, path)
|
||||||
|
if out, err := execFn(cmd); err != nil {
|
||||||
|
return fmt.Errorf("remote write %s: %w (output: %s)", path, err, string(out))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
// Package linux implements the SSH-based bootstrap of a generic Linux
|
||||||
|
// host (Ubuntu/Debian/Alpine) as an orca worker node (REQ-161, P12).
|
||||||
|
//
|
||||||
|
// The bootstrap sequence (run via `orca node join --type linux`):
|
||||||
|
// 1. Generate or load the orca SSH keypair (Ed25519, D-037)
|
||||||
|
// 2. SSH dial with key auth + TOFU host-key capture (D-035)
|
||||||
|
// 3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys
|
||||||
|
// 4. Create the `orca` Linux system user (nologin shell)
|
||||||
|
// 5. Create the drift-events directory (~orca/drift-events)
|
||||||
|
// 6. Return the node metadata for the caller to persist
|
||||||
|
//
|
||||||
|
// Unlike Proxmox bootstrap, there is NO PVE role, NO sudoers file, and
|
||||||
|
// NO PVE user — this is a plain Linux worker. Authentication is
|
||||||
|
// key-based (R-021): the orca SSH key is used for the initial SSH auth
|
||||||
|
// and pubkey deployment; subsequent orca→worker access uses the same
|
||||||
|
// key.
|
||||||
|
//
|
||||||
|
// All steps are idempotent: re-running the bootstrap on an
|
||||||
|
// already-configured host is a no-op.
|
||||||
|
package linux
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/ingress"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultSSHUser is the default SSH username for the initial connection.
|
||||||
|
const DefaultSSHUser = "root"
|
||||||
|
|
||||||
|
// DefaultOrcaUser is the default Linux system user created on the worker.
|
||||||
|
const DefaultOrcaUser = "orca"
|
||||||
|
|
||||||
|
// DefaultSSHPort is the default SSH port.
|
||||||
|
const DefaultSSHPort = 22
|
||||||
|
|
||||||
|
// Options configures a Linux worker bootstrap run.
|
||||||
|
type Options struct {
|
||||||
|
Host string
|
||||||
|
SSHUser string
|
||||||
|
SSHKeyPath string
|
||||||
|
OrcaUser string
|
||||||
|
SSHPort int
|
||||||
|
HostKeyFingerprint string
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result is the outcome of a successful bootstrap.
|
||||||
|
type Result struct {
|
||||||
|
NodeName string
|
||||||
|
NodeAddress string
|
||||||
|
HostKeyFingerprint string
|
||||||
|
}
|
||||||
|
|
||||||
|
// BootstrapLinux runs the full SSH bootstrap sequence on a remote
|
||||||
|
// generic Linux host. Returns the node metadata for the caller to
|
||||||
|
// persist to the registry.
|
||||||
|
func BootstrapLinux(ctx context.Context, opts Options) (*Result, error) {
|
||||||
|
if opts.Host == "" {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: --host is required")
|
||||||
|
}
|
||||||
|
if opts.SSHKeyPath == "" {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: --ssh-key is required (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
||||||
|
}
|
||||||
|
if opts.SSHUser == "" {
|
||||||
|
opts.SSHUser = DefaultSSHUser
|
||||||
|
}
|
||||||
|
if opts.OrcaUser == "" {
|
||||||
|
opts.OrcaUser = DefaultOrcaUser
|
||||||
|
}
|
||||||
|
if opts.SSHPort == 0 {
|
||||||
|
opts.SSHPort = DefaultSSHPort
|
||||||
|
}
|
||||||
|
if opts.Logger == nil {
|
||||||
|
opts.Logger = slog.Default()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 1: Load the orca SSH keypair.
|
||||||
|
privKey, err := os.ReadFile(opts.SSHKeyPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: read SSH key: %w", err)
|
||||||
|
}
|
||||||
|
signer, err := ssh.ParsePrivateKey(privKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: parse SSH key: %w", err)
|
||||||
|
}
|
||||||
|
pubKey, err := os.ReadFile(certpaths.SSHPubPath())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: read orca pubkey: %w", err)
|
||||||
|
}
|
||||||
|
pubKeyLine := strings.TrimSpace(string(pubKey))
|
||||||
|
|
||||||
|
// Step 2: SSH dial with key auth + TOFU host-key capture.
|
||||||
|
sshAddr := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.SSHPort))
|
||||||
|
var capturedHostKey ssh.PublicKey
|
||||||
|
var hostKeyCallback ssh.HostKeyCallback
|
||||||
|
if opts.HostKeyFingerprint != "" {
|
||||||
|
hkcb, err := pinnedHostKeyCallback(opts.HostKeyFingerprint, &capturedHostKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: parse host key fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
hostKeyCallback = hkcb
|
||||||
|
} else {
|
||||||
|
// REQ-164 / Phase A3: reuse the tested Proxmox TOFU callback
|
||||||
|
// which handles first-connect key capture + known_hosts file
|
||||||
|
// creation (create-on-open). The previous inline implementation
|
||||||
|
// failed on first connect with a raw KeyError because it never
|
||||||
|
// wrote the captured key.
|
||||||
|
hkcb, err := proxmox.TOFUHostKeyCallbackPath(certpaths.KnownHostsPath(), sshAddr, &capturedHostKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: known_hosts: %w", err)
|
||||||
|
}
|
||||||
|
hostKeyCallback = hkcb
|
||||||
|
}
|
||||||
|
|
||||||
|
sshConfig := &ssh.ClientConfig{
|
||||||
|
User: opts.SSHUser,
|
||||||
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||||
|
HostKeyCallback: hostKeyCallback,
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
opts.Logger.Info("linux bootstrap: dialing", "addr", sshAddr, "user", opts.SSHUser)
|
||||||
|
client, err := ssh.Dial("tcp", sshAddr, sshConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: SSH dial %s: %w", sshAddr, err)
|
||||||
|
}
|
||||||
|
defer client.Close()
|
||||||
|
|
||||||
|
// Step 3: Deploy the orca pubkey to authorized_keys.
|
||||||
|
if err := sshExec(client, fmt.Sprintf(
|
||||||
|
"mkdir -p ~%s/.ssh && grep -qF '%s' ~%s/.ssh/authorized_keys 2>/dev/null || echo '%s' >> ~%s/.ssh/authorized_keys && chmod 700 ~%s/.ssh && chmod 600 ~%s/.ssh/authorized_keys",
|
||||||
|
opts.OrcaUser, pubKeyLine, opts.OrcaUser, pubKeyLine, opts.OrcaUser, opts.OrcaUser, opts.OrcaUser,
|
||||||
|
)); err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: deploy pubkey: %w", err)
|
||||||
|
}
|
||||||
|
opts.Logger.Info("linux bootstrap: pubkey deployed", "user", opts.OrcaUser)
|
||||||
|
|
||||||
|
// Step 4: Create the orca system user (nologin shell).
|
||||||
|
if err := sshExec(client, fmt.Sprintf(
|
||||||
|
"id -u %s 2>/dev/null || useradd -r -s /usr/sbin/nologin -d /home/%s -m %s",
|
||||||
|
opts.OrcaUser, opts.OrcaUser, opts.OrcaUser,
|
||||||
|
)); err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: create user: %w", err)
|
||||||
|
}
|
||||||
|
opts.Logger.Info("linux bootstrap: user created", "user", opts.OrcaUser)
|
||||||
|
|
||||||
|
// Step 4d: Ensure complete ingress stack on the remote host (R-024).
|
||||||
|
// Renders+applies nft DNAT/SNAT, pushes step-ca root CA, renders+
|
||||||
|
// writes traefik static config, ensures podman container running.
|
||||||
|
// All non-fatal (offline host tolerance).
|
||||||
|
sshExecFn := func(cmd string) ([]byte, error) {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
return session.CombinedOutput(cmd)
|
||||||
|
}
|
||||||
|
ctx, cancelIngress := context.WithTimeout(ctx, 120*time.Second)
|
||||||
|
defer cancelIngress()
|
||||||
|
if err := ingress.BootstrapRemoteIngress(ctx, "", ingress.RemoteExecFunc(sshExecFn)); err != nil {
|
||||||
|
opts.Logger.Warn("linux bootstrap: ingress bootstrap failed", "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 5: Create the drift-events directory.
|
||||||
|
if err := sshExec(client, fmt.Sprintf(
|
||||||
|
"mkdir -p ~%s/drift-events && chown %s:%s ~%s/drift-events",
|
||||||
|
opts.OrcaUser, opts.OrcaUser, opts.OrcaUser, opts.OrcaUser,
|
||||||
|
)); err != nil {
|
||||||
|
return nil, fmt.Errorf("linux bootstrap: create drift-events dir: %w", err)
|
||||||
|
}
|
||||||
|
opts.Logger.Info("linux bootstrap: drift-events dir created", "user", opts.OrcaUser)
|
||||||
|
|
||||||
|
// Step 6: Return node metadata.
|
||||||
|
hostKeyFP := ""
|
||||||
|
if capturedHostKey != nil {
|
||||||
|
hostKeyFP = ssh.FingerprintSHA256(capturedHostKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Result{
|
||||||
|
NodeName: opts.Host,
|
||||||
|
NodeAddress: fmt.Sprintf("%s:8443", opts.Host),
|
||||||
|
HostKeyFingerprint: hostKeyFP,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sshExec runs a command on the remote host and returns an error if
|
||||||
|
// the exit code is non-zero.
|
||||||
|
func sshExec(client *ssh.Client, cmd string) error {
|
||||||
|
session, err := client.NewSession()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer session.Close()
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
session.Stderr = &stderr
|
||||||
|
if err := session.Run(cmd); err != nil {
|
||||||
|
return fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String()))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// pinnedHostKeyCallback returns a host key callback that pins to the
|
||||||
|
// expected fingerprint.
|
||||||
|
func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
||||||
|
if expectedSHA256Base64 == "" {
|
||||||
|
return nil, fmt.Errorf("empty fingerprint")
|
||||||
|
}
|
||||||
|
cb := ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||||
|
got := ssh.FingerprintSHA256(key)
|
||||||
|
if got != expectedSHA256Base64 {
|
||||||
|
return fmt.Errorf("host key fingerprint mismatch: got %s, want %s", got, expectedSHA256Base64)
|
||||||
|
}
|
||||||
|
*capturedKey = key
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return cb, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ = security.WriteAtomic
|
||||||
@@ -21,6 +21,7 @@ type Job struct {
|
|||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
EndedAt *time.Time `json:"ended_at,omitempty"`
|
EndedAt *time.Time `json:"ended_at,omitempty"`
|
||||||
ExitCode int `json:"exit_code"`
|
ExitCode int `json:"exit_code"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TaskStatus string
|
type TaskStatus string
|
||||||
@@ -41,6 +42,7 @@ type Task struct {
|
|||||||
Env []string `json:"env,omitempty"`
|
Env []string `json:"env,omitempty"`
|
||||||
PID int `json:"pid"`
|
PID int `json:"pid"`
|
||||||
ExitCode int `json:"exit_code"`
|
ExitCode int `json:"exit_code"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
Status TaskStatus `json:"status"`
|
Status TaskStatus `json:"status"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||||
|
|||||||
@@ -39,4 +39,9 @@ type Node struct {
|
|||||||
// OS is the auto-detected OS identifier from /etc/os-release ID=
|
// OS is the auto-detected OS identifier from /etc/os-release ID=
|
||||||
// (ubuntu|debian|alpine|pve|linux). Empty for pre-0006 rows.
|
// (ubuntu|debian|alpine|pve|linux). Empty for pre-0006 rows.
|
||||||
OS string `json:"os,omitempty"`
|
OS string `json:"os,omitempty"`
|
||||||
|
// IngressMode is the ingress configuration for the node (R-024,
|
||||||
|
// v0.14). Values: "" (legacy/default for linux/localhost),
|
||||||
|
// "native" (proxmox native — traefik in LXC), "floating-ip"
|
||||||
|
// (proxmox floating-IP — ingress LXC owns the floating IP).
|
||||||
|
IngressMode string `json:"ingress_mode,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
+194
-18
@@ -3,7 +3,7 @@
|
|||||||
//
|
//
|
||||||
// The bootstrap sequence (run via `orca node join --type proxmox`):
|
// The bootstrap sequence (run via `orca node join --type proxmox`):
|
||||||
// 1. Generate or load the orca SSH keypair (Ed25519, D-037)
|
// 1. Generate or load the orca SSH keypair (Ed25519, D-037)
|
||||||
// 2. SSH dial with password auth + TOFU host-key capture (D-035)
|
// 2. SSH dial with key auth + TOFU host-key capture (D-035)
|
||||||
// 3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys
|
// 3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys
|
||||||
// 4. Create the `orca` Linux system user (config-overridable name)
|
// 4. Create the `orca` Linux system user (config-overridable name)
|
||||||
// 5. Create the OrcaOperator PVE role with least-privilege privileges
|
// 5. Create the OrcaOperator PVE role with least-privilege privileges
|
||||||
@@ -16,9 +16,9 @@
|
|||||||
// 10. Return the node metadata for the caller to persist
|
// 10. Return the node metadata for the caller to persist
|
||||||
//
|
//
|
||||||
// All steps are idempotent (D-036): re-running the bootstrap on an
|
// All steps are idempotent (D-036): re-running the bootstrap on an
|
||||||
// already-configured host is a no-op. The password is never persisted
|
// already-configured host is a no-op. Authentication is key-based (R-021)
|
||||||
// (D-031) — it is used only for the initial SSH auth and pubkey
|
// (D-031): the orca SSH key is used for the initial SSH auth and
|
||||||
// deployment; subsequent orca→Proxmox access uses the deployed SSH key.
|
// pubkey deployment; subsequent orca→Proxmox access uses the same key.
|
||||||
package proxmox
|
package proxmox
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -37,7 +37,9 @@ import (
|
|||||||
"golang.org/x/crypto/ssh/knownhosts"
|
"golang.org/x/crypto/ssh/knownhosts"
|
||||||
|
|
||||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultProxmoxUser is the default Linux system user created on the
|
// DefaultProxmoxUser is the default Linux system user created on the
|
||||||
@@ -82,6 +84,16 @@ type Options struct {
|
|||||||
HostKeyFingerprint string
|
HostKeyFingerprint string
|
||||||
// Logger receives audit-log entries. If nil, slog.Default() is used.
|
// Logger receives audit-log entries. If nil, slog.Default() is used.
|
||||||
Logger *slog.Logger
|
Logger *slog.Logger
|
||||||
|
// LXCTemplate is the LXC template to download during bootstrap
|
||||||
|
// (default "ubuntu-24.04"; alternatives: "alpine-3.20", "debian-12").
|
||||||
|
LXCTemplate string
|
||||||
|
// IngressMode is the proxmox ingress mode (R-024, v0.14).
|
||||||
|
// "native" (default): traefik runs in an unprivileged LXC with
|
||||||
|
// nesting=1,keyctl=1,fuse=1 on the PVE host. nft on the PVE host
|
||||||
|
// DNATs to the LXC bridge IP.
|
||||||
|
// "floating-ip": a separate ingress LXC owns the floating IP;
|
||||||
|
// nft runs inside that LXC. See ProvisionIngressLXC (P6).
|
||||||
|
IngressMode string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Result is the outcome of a successful bootstrap.
|
// Result is the outcome of a successful bootstrap.
|
||||||
@@ -143,14 +155,14 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
|
|||||||
return nil, fmt.Errorf("ssh key: %w", err)
|
return nil, fmt.Errorf("ssh key: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Step 2: SSH dial with password auth + host-key verification (D-035,
|
// Step 2: SSH dial with key auth + host-key verification (D-035,
|
||||||
// REQ-058). When opts.HostKeyFingerprint is set (D-044), use a pinned
|
// REQ-058). When opts.HostKeyFingerprint is set (D-044), use a pinned
|
||||||
// callback that fails closed on mismatch (AD-028); otherwise use the
|
// callback that fails closed on mismatch (AD-028); otherwise use the
|
||||||
// TOFU known_hosts capture callback (D-035). The TOFU wrapper fixes
|
// TOFU known_hosts capture callback (D-035). The TOFU wrapper fixes
|
||||||
// the v0.6 ship-defect where knownhosts.New returned KeyError{Want:[]}
|
// the v0.6 ship-defect where knownhosts.New returned KeyError{Want:[]}
|
||||||
// on first connect WITHOUT writing the captured key, so the first
|
// on first connect WITHOUT writing the captured key, so the first
|
||||||
// `orca node join --type proxmox` always failed.
|
// `orca node join --type proxmox` always failed.
|
||||||
sshAddr := fmt.Sprintf("%s:%d", opts.Host, opts.SSHPort)
|
sshAddr := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.SSHPort))
|
||||||
var capturedHostKey ssh.PublicKey
|
var capturedHostKey ssh.PublicKey
|
||||||
var hostKeyCallback ssh.HostKeyCallback
|
var hostKeyCallback ssh.HostKeyCallback
|
||||||
if opts.HostKeyFingerprint != "" {
|
if opts.HostKeyFingerprint != "" {
|
||||||
@@ -243,6 +255,24 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
|
|||||||
return nil, fmt.Errorf("validate sudoers: %w", err)
|
return nil, fmt.Errorf("validate sudoers: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Step 9a: Proxmox native ingress mode (R-024, REQ-175).
|
||||||
|
// Create an unprivileged LXC with nesting=1,keyctl=1,fuse=1 (research
|
||||||
|
// Topic 3), install podman inside it, and run the orca-traefik
|
||||||
|
// container. nft on the PVE host DNATs to the LXC bridge IP.
|
||||||
|
// Default mode is "native"; floating-ip mode is handled separately
|
||||||
|
// (P6 — ProvisionIngressLXC).
|
||||||
|
template := opts.LXCTemplate
|
||||||
|
if template == "" {
|
||||||
|
template = "ubuntu-24.04"
|
||||||
|
}
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template)))
|
||||||
|
|
||||||
|
if opts.IngressMode != "floating-ip" {
|
||||||
|
if err := provisionNativeIngressLXC(ctx, runRemote, template, log); err != nil {
|
||||||
|
log.Warn("proxmox.native_ingress_lxc_failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
log.Info("proxmox.bootstrap_ok",
|
log.Info("proxmox.bootstrap_ok",
|
||||||
slog.String("event", "proxmox.bootstrap_ok"),
|
slog.String("event", "proxmox.bootstrap_ok"),
|
||||||
slog.String("host", opts.Host),
|
slog.String("host", opts.Host),
|
||||||
@@ -257,6 +287,119 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// provisionNativeIngressLXC creates an unprivileged LXC with
|
||||||
|
// nesting=1,keyctl=1,fuse=1 (research Topic 3), installs podman inside
|
||||||
|
// it, runs the orca-traefik container, and applies nft DNAT on the PVE
|
||||||
|
// host targeting the LXC's bridge IP (R-024, REQ-175).
|
||||||
|
//
|
||||||
|
// The LXC is named "orca-traefik" and uses a deterministic VMID derived
|
||||||
|
// from the host. It is idempotent: if the LXC already exists, it is
|
||||||
|
// not re-created (C-53: apt-get install is skipped if podman present).
|
||||||
|
func provisionNativeIngressLXC(ctx context.Context, runRemote func(string) ([]byte, error), template string, log *slog.Logger) error {
|
||||||
|
// Deterministic VMID for the native ingress LXC.
|
||||||
|
// Use a fixed VMID in the 200-299 range (Proxmox convention for CTs).
|
||||||
|
const vmid = "200"
|
||||||
|
const lxcName = "orca-traefik"
|
||||||
|
|
||||||
|
// Check if the LXC already exists.
|
||||||
|
existOut, _ := runRemote(fmt.Sprintf("pct status %s 2>/dev/null || echo absent", vmid))
|
||||||
|
existStr := strings.TrimSpace(string(existOut))
|
||||||
|
if existStr == "absent" {
|
||||||
|
// Create the LXC (research Topic 3: nesting=1,keyctl=1,fuse=1).
|
||||||
|
log.Info("proxmox.creating_native_ingress_lxc", "vmid", vmid, "name", lxcName)
|
||||||
|
createCmd := fmt.Sprintf(
|
||||||
|
"pct create %s local:vztmpl/%s --hostname %s --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --onboot 1 --memory 2048 --swap 0 --rootfs local:8 2>&1",
|
||||||
|
vmid, shellQuote(template), lxcName,
|
||||||
|
)
|
||||||
|
if out, err := runRemote(createCmd); err != nil {
|
||||||
|
return fmt.Errorf("pct create native ingress LXC: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
if out, err := runRemote(fmt.Sprintf("pct start %s", vmid)); err != nil {
|
||||||
|
return fmt.Errorf("pct start native ingress LXC: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for LXC network (retry for up to 60s).
|
||||||
|
lxcIP := ""
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
ipOut, _ := runRemote(fmt.Sprintf("pct exec %s -- hostname -I 2>/dev/null", vmid))
|
||||||
|
ipStr := strings.TrimSpace(string(ipOut))
|
||||||
|
if ipStr != "" {
|
||||||
|
fields := strings.Fields(ipStr)
|
||||||
|
if len(fields) > 0 {
|
||||||
|
lxcIP = fields[0]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
}
|
||||||
|
if lxcIP == "" {
|
||||||
|
return fmt.Errorf("native ingress LXC: could not discover IP after 60s")
|
||||||
|
}
|
||||||
|
log.Info("proxmox.native_ingress_lxc_ip", "vmid", vmid, "ip", lxcIP)
|
||||||
|
|
||||||
|
// Install podman inside the LXC (C-53: idempotent — check first).
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'command -v podman >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs nftables 2>&1)' 2>&1",
|
||||||
|
vmid,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Enable podman-restart.service inside the LXC (research Topic 6).
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- systemctl enable --now podman-restart.service 2>/dev/null", vmid))
|
||||||
|
|
||||||
|
// Push step-ca root CA into the LXC (placeholder if absent locally).
|
||||||
|
caPath := certpaths.CACertPath()
|
||||||
|
caData, caErr := os.ReadFile(caPath)
|
||||||
|
if caErr != nil {
|
||||||
|
caData = []byte{}
|
||||||
|
}
|
||||||
|
// Write CA via pct exec heredoc.
|
||||||
|
caDelim := "EOF_CA"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/orca && cat > /etc/orca/step-ca-root.crt <<%s\\n%s\\n%s'",
|
||||||
|
vmid, caDelim, string(caData), caDelim,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Render + write traefik static config inside the LXC.
|
||||||
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range staticFiles {
|
||||||
|
delim := "EOF_TF"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/traefik/dynamic && cat > %s <<%s\\n%s\\n%s'",
|
||||||
|
vmid, f.Path, delim, f.Content, delim,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure podman orca-traefik container inside the LXC.
|
||||||
|
traefikExecFn := func(cmd string) ([]byte, error) {
|
||||||
|
return runRemote(fmt.Sprintf("pct exec %s -- bash -c %s 2>&1", vmid, shellQuote(cmd)))
|
||||||
|
}
|
||||||
|
if err := traefik.EnsureTraefikContainerRemote(ctx, "", traefikExecFn); err != nil {
|
||||||
|
log.Warn("proxmox.native_ingress_lxc_traefik_failed", "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render + apply nft on the PVE host with DNATTarget = LXC IP.
|
||||||
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{
|
||||||
|
DNATTarget: lxcIP,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range nftFiles {
|
||||||
|
nftDelim := "EOF_NF"
|
||||||
|
_, _ = runRemote(fmt.Sprintf("mkdir -p /etc/nftables.d && cat > %s <<%s\\n%s\\n%s",
|
||||||
|
f.Path, nftDelim, f.Content, nftDelim))
|
||||||
|
}
|
||||||
|
_, _ = runRemote("nft add table inet orca-ingress 2>/dev/null || true")
|
||||||
|
if out, err := runRemote("nft -f /etc/nftables.d/orca.nft 2>&1"); err != nil {
|
||||||
|
log.Warn("proxmox.native_ingress_nft_apply_failed", "err", err, "output", string(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("proxmox.native_ingress_lxc_ok", "vmid", vmid, "ip", lxcIP)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// sshDialer is the dialer used by BootstrapProxmox. It's a package-level
|
// sshDialer is the dialer used by BootstrapProxmox. It's a package-level
|
||||||
// variable so tests can override it with a fake SSH server.
|
// variable so tests can override it with a fake SSH server.
|
||||||
var sshDialer sshDialerType = defaultSSHDialer{}
|
var sshDialer sshDialerType = defaultSSHDialer{}
|
||||||
@@ -296,8 +439,41 @@ func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicK
|
|||||||
//
|
//
|
||||||
// Exported so the doctor proxmox probe (T02.9) can reuse the same
|
// Exported so the doctor proxmox probe (T02.9) can reuse the same
|
||||||
// capture-fix wrapper for parity (GRILL condition #2).
|
// capture-fix wrapper for parity (GRILL condition #2).
|
||||||
|
//
|
||||||
|
// REQ-157 / P08 T4: TOFUHostKeyCallback now delegates to
|
||||||
|
// TOFUHostKeyCallbackPath with the v0.8 flat layout
|
||||||
|
// (certpaths.KnownHostsPath()). The path-accepting variant lets the
|
||||||
|
// sshpush transport pass its stored known_hosts field (the v0.9
|
||||||
|
// paths.KnownHostsPath() location) instead of always reading the v0.8
|
||||||
|
// flat layout — fixing the bug where the dial() flock field was stored
|
||||||
|
// but never read.
|
||||||
func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
||||||
cb, err := knownhosts.New(certpaths.KnownHostsPath())
|
return TOFUHostKeyCallbackPath(certpaths.KnownHostsPath(), addr, capturedKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TOFUHostKeyCallbackPath is the path-accepting variant. knownHostsPath
|
||||||
|
// is the known_hosts file to verify against and capture new keys into;
|
||||||
|
// it MUST be flock-protected on capture (security.Flock). When
|
||||||
|
// knownHostsPath is empty, falls back to certpaths.KnownHostsPath()
|
||||||
|
// (the v0.8 flat layout) for backward compatibility with callers that
|
||||||
|
// relied on the implicit default.
|
||||||
|
func TOFUHostKeyCallbackPath(knownHostsPath, addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
||||||
|
if knownHostsPath == "" {
|
||||||
|
knownHostsPath = certpaths.KnownHostsPath()
|
||||||
|
}
|
||||||
|
// REQ-164 / Phase A2: create the known_hosts file if it doesn't
|
||||||
|
// exist (knownhosts.New requires the file to be present). This is
|
||||||
|
// defense-in-depth alongside init.go which also creates it.
|
||||||
|
if _, err := os.Stat(knownHostsPath); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
if writeErr := security.WriteAtomic(knownHostsPath, 0o600, []byte{}); writeErr != nil {
|
||||||
|
return nil, fmt.Errorf("tofu create known_hosts: %w", writeErr)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("tofu stat known_hosts: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cb, err := knownhosts.New(knownHostsPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -312,13 +488,12 @@ func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCa
|
|||||||
var keyErr *knownhosts.KeyError
|
var keyErr *knownhosts.KeyError
|
||||||
if errors.As(err, &keyErr) && len(keyErr.Want) == 0 {
|
if errors.As(err, &keyErr) && len(keyErr.Want) == 0 {
|
||||||
line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, key)
|
line := knownhosts.Line([]string{knownhosts.Normalize(addr)}, key)
|
||||||
path := certpaths.KnownHostsPath()
|
release, lockErr := security.Flock(knownHostsPath)
|
||||||
release, lockErr := security.Flock(path)
|
|
||||||
if lockErr != nil {
|
if lockErr != nil {
|
||||||
return fmt.Errorf("tofu lock known_hosts: %w", lockErr)
|
return fmt.Errorf("tofu lock known_hosts: %w", lockErr)
|
||||||
}
|
}
|
||||||
defer release()
|
defer release()
|
||||||
existing, readErr := os.ReadFile(path)
|
existing, readErr := os.ReadFile(knownHostsPath)
|
||||||
if readErr != nil && !os.IsNotExist(readErr) {
|
if readErr != nil && !os.IsNotExist(readErr) {
|
||||||
return fmt.Errorf("tofu read known_hosts: %w", readErr)
|
return fmt.Errorf("tofu read known_hosts: %w", readErr)
|
||||||
}
|
}
|
||||||
@@ -326,7 +501,7 @@ func TOFUHostKeyCallback(addr string, capturedKey *ssh.PublicKey) (ssh.HostKeyCa
|
|||||||
existing = append(existing, '\n')
|
existing = append(existing, '\n')
|
||||||
}
|
}
|
||||||
updated := append(existing, []byte(line)...)
|
updated := append(existing, []byte(line)...)
|
||||||
if writeErr := security.WriteAtomic(path, 0o600, updated); writeErr != nil {
|
if writeErr := security.WriteAtomic(knownHostsPath, 0o600, updated); writeErr != nil {
|
||||||
return fmt.Errorf("tofu write known_hosts: %w", writeErr)
|
return fmt.Errorf("tofu write known_hosts: %w", writeErr)
|
||||||
}
|
}
|
||||||
if capturedKey != nil {
|
if capturedKey != nil {
|
||||||
@@ -416,10 +591,12 @@ func createLinuxUser(user string) error {
|
|||||||
// createPVERole creates the OrcaOperator PVE role if it doesn't exist.
|
// createPVERole creates the OrcaOperator PVE role if it doesn't exist.
|
||||||
// Idempotent: probes `pveum role list` before `pveum role add`.
|
// Idempotent: probes `pveum role list` before `pveum role add`.
|
||||||
func createPVERole(role string) error {
|
func createPVERole(role string) error {
|
||||||
// F10c: shellQuote the role (validated upstream, but defense-in-depth).
|
// Idempotent: check if the role already exists before creating.
|
||||||
|
// Use pveum role list with grep -qF (fixed string, not regex) to
|
||||||
|
// avoid shell-quoting issues with single quotes inside the pattern.
|
||||||
cmd := fmt.Sprintf(
|
cmd := fmt.Sprintf(
|
||||||
"pveum role list 2>/dev/null | grep -q '^%s' || pveum role add %s --privs '%s'",
|
"pveum role list 2>/dev/null | grep -qF %s && exit 0 || pveum role add %s --privs '%s' 2>/dev/null || pveum role mod %s --privs '%s'",
|
||||||
shellQuote(role), shellQuote(role), OrcaOperatorPrivileges,
|
shellQuote(role), shellQuote(role), OrcaOperatorPrivileges, shellQuote(role), OrcaOperatorPrivileges,
|
||||||
)
|
)
|
||||||
if _, err := runRemote(cmd); err != nil {
|
if _, err := runRemote(cmd); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -432,11 +609,10 @@ func createPVERole(role string) error {
|
|||||||
// Uses @pam realm (AD-019) since orca creates a Linux system user.
|
// Uses @pam realm (AD-019) since orca creates a Linux system user.
|
||||||
func createPVEUser(user string) error {
|
func createPVEUser(user string) error {
|
||||||
pveUserID := user + "@pam"
|
pveUserID := user + "@pam"
|
||||||
// F10c: shellQuote the PVE user id (validated upstream, but
|
// Idempotent: check if user exists, create if not, update comment if exists.
|
||||||
// defense-in-depth).
|
|
||||||
cmd := fmt.Sprintf(
|
cmd := fmt.Sprintf(
|
||||||
"pveum user list 2>/dev/null | grep -q %s || pveum user add %s -comment 'Orca automation user'",
|
"pveum user list 2>/dev/null | grep -qF %s && exit 0 || pveum user add %s -comment 'Orca automation user' 2>/dev/null || pveum user mod %s -comment 'Orca automation user'",
|
||||||
shellQuote(pveUserID), shellQuote(pveUserID),
|
shellQuote(pveUserID), shellQuote(pveUserID), shellQuote(pveUserID),
|
||||||
)
|
)
|
||||||
if _, err := runRemote(cmd); err != nil {
|
if _, err := runRemote(cmd); err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
package proxmox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FloatingIPOptions carries the parameters for provisioning a
|
||||||
|
// floating-IP ingress LXC (R-024, REQ-176).
|
||||||
|
type FloatingIPOptions struct {
|
||||||
|
// FloatingIP is the public IP assigned to the LXC's eth0.
|
||||||
|
FloatingIP string
|
||||||
|
// Gateway is the default gateway for the LXC.
|
||||||
|
Gateway string
|
||||||
|
// MAC is the MAC address for the LXC's net0 interface.
|
||||||
|
MAC string
|
||||||
|
// NetPrefix is the CIDR prefix for the floating IP (8-32).
|
||||||
|
NetPrefix int
|
||||||
|
// LXCTemplate is the LXC template (default "ubuntu-24.04").
|
||||||
|
LXCTemplate string
|
||||||
|
// VMID is the LXC container ID (default "201" for the ingress LXC).
|
||||||
|
VMID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProvisionIngressLXC creates an Ubuntu LXC named "ingress" that owns
|
||||||
|
// the floating IP, installs podman + orca-traefik inside it, applies nft
|
||||||
|
// DNAT+SNAT inside the LXC, and returns the LXC's IP for node
|
||||||
|
// registration (R-024, REQ-176).
|
||||||
|
//
|
||||||
|
// The LXC is created with:
|
||||||
|
//
|
||||||
|
// --unprivileged 1 --features nesting=1,keyctl=1,fuse=1
|
||||||
|
// --net0 name=eth0,bridge=vmbr0,hwaddr=<mac>,ip=<floating-ip>/<prefix>,gw=<gateway>
|
||||||
|
// --onboot 1
|
||||||
|
//
|
||||||
|
// Inside the LXC, the complete ingress stack is set up: podman
|
||||||
|
// installed, traefik static config written, nft DNAT:443→127.0.0.1:8443
|
||||||
|
// + postrouting masquerade applied, orca-traefik podman container
|
||||||
|
// running with --network host.
|
||||||
|
//
|
||||||
|
// Idempotent: if the LXC already exists, it is not re-created (C-53).
|
||||||
|
func ProvisionIngressLXC(ctx context.Context, runRemote func(string) ([]byte, error), opts FloatingIPOptions, log *slog.Logger) error {
|
||||||
|
template := opts.LXCTemplate
|
||||||
|
if template == "" {
|
||||||
|
template = "ubuntu-24.04"
|
||||||
|
}
|
||||||
|
vmid := opts.VMID
|
||||||
|
if vmid == "" {
|
||||||
|
vmid = "201"
|
||||||
|
}
|
||||||
|
if opts.NetPrefix == 0 {
|
||||||
|
opts.NetPrefix = 24
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate required fields.
|
||||||
|
if opts.FloatingIP == "" || opts.Gateway == "" || opts.MAC == "" {
|
||||||
|
return fmt.Errorf("ingress_lxc: floating-ip, gateway, and mac are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the template is downloaded.
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template)))
|
||||||
|
|
||||||
|
// Check if the LXC already exists (idempotent — C-53).
|
||||||
|
existOut, _ := runRemote(fmt.Sprintf("pct status %s 2>/dev/null || echo absent", vmid))
|
||||||
|
existStr := strings.TrimSpace(string(existOut))
|
||||||
|
if existStr == "absent" {
|
||||||
|
log.Info("ingress_lxc.creating", "vmid", vmid, "hostname", "ingress", "ip", opts.FloatingIP)
|
||||||
|
net0 := fmt.Sprintf("name=eth0,bridge=vmbr0,hwaddr=%s,ip=%s/%d,gw=%s",
|
||||||
|
opts.MAC, opts.FloatingIP, opts.NetPrefix, opts.Gateway)
|
||||||
|
createCmd := fmt.Sprintf(
|
||||||
|
"pct create %s local:vztmpl/%s --hostname ingress --unprivileged 1 --features nesting=1,keyctl=1,fuse=1 --net0 %s --onboot 1 --memory 2048 --swap 0 --rootfs local:8 2>&1",
|
||||||
|
vmid, shellQuote(template), net0,
|
||||||
|
)
|
||||||
|
if out, err := runRemote(createCmd); err != nil {
|
||||||
|
return fmt.Errorf("pct create ingress LXC: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
if out, err := runRemote(fmt.Sprintf("pct start %s", vmid)); err != nil {
|
||||||
|
return fmt.Errorf("pct start ingress LXC: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for LXC network (retry for up to 60s).
|
||||||
|
for i := 0; i < 12; i++ {
|
||||||
|
ipOut, _ := runRemote(fmt.Sprintf("pct exec %s -- hostname -I 2>/dev/null", vmid))
|
||||||
|
ipStr := strings.TrimSpace(string(ipOut))
|
||||||
|
if ipStr != "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
}
|
||||||
|
log.Info("ingress_lxc.network_ready", "vmid", vmid, "ip", opts.FloatingIP)
|
||||||
|
|
||||||
|
// Install podman inside the LXC (C-53: idempotent).
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'command -v podman >/dev/null 2>&1 || (apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs nftables 2>&1)' 2>&1",
|
||||||
|
vmid,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Enable podman-restart.service inside the LXC (research Topic 6).
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- systemctl enable --now podman-restart.service 2>/dev/null", vmid))
|
||||||
|
|
||||||
|
// Push step-ca root CA into the LXC (C-60: CACertPath).
|
||||||
|
caPath := certpaths.CACertPath()
|
||||||
|
caData, caErr := os.ReadFile(caPath)
|
||||||
|
if caErr != nil {
|
||||||
|
caData = []byte{}
|
||||||
|
}
|
||||||
|
caDelim := "EOF_CA"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/orca && cat > /etc/orca/step-ca-root.crt <<%s\\n%s\\n%s'",
|
||||||
|
vmid, caDelim, string(caData), caDelim,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Render + write traefik static config inside the LXC (C-58).
|
||||||
|
staticFiles, err := emitter.TraefikEmitter{}.RenderTraefikStaticConfig(emitter.TraefikStaticOpts{})
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range staticFiles {
|
||||||
|
delim := "EOF_TF"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/traefik/dynamic && cat > %s <<%s\\n%s\\n%s'",
|
||||||
|
vmid, f.Path, delim, f.Content, delim,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render + apply nft DNAT+SNAT INSIDE the LXC (DNATTarget =
|
||||||
|
// 127.0.0.1 — traefik runs with --network host inside the LXC).
|
||||||
|
nftFiles, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
||||||
|
if err == nil {
|
||||||
|
for _, f := range nftFiles {
|
||||||
|
delim := "EOF_NF"
|
||||||
|
_, _ = runRemote(fmt.Sprintf(
|
||||||
|
"pct exec %s -- bash -c 'mkdir -p /etc/nftables.d && cat > %s <<%s\\n%s\\n%s'",
|
||||||
|
vmid, f.Path, delim, f.Content, delim,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- nft add table inet orca-ingress 2>/dev/null || true", vmid))
|
||||||
|
_, _ = runRemote(fmt.Sprintf("pct exec %s -- nft -f /etc/nftables.d/orca.nft 2>&1", vmid))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure podman orca-traefik container inside the LXC.
|
||||||
|
traefikExecFn := func(cmd string) ([]byte, error) {
|
||||||
|
return runRemote(fmt.Sprintf("pct exec %s -- bash -c %s 2>&1", vmid, shellQuote(cmd)))
|
||||||
|
}
|
||||||
|
if err := traefik.EnsureTraefikContainerRemote(ctx, "", traefikExecFn); err != nil {
|
||||||
|
log.Warn("ingress_lxc.traefik_failed", "err", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("ingress_lxc.provisioned", "vmid", vmid, "ip", opts.FloatingIP)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateRandomMAC generates a random locally-administered MAC address
|
||||||
|
// (02:XX:XX:XX:XX:XX) for use as the LXC net0 hardware address when the
|
||||||
|
// operator does not provide one (D-261).
|
||||||
|
func GenerateRandomMAC() (string, error) {
|
||||||
|
b := make([]byte, 5)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
return "", fmt.Errorf("generate MAC: %w", err)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("02:%02x:%02x:%02x:%02x:%02x", b[0], b[1], b[2], b[3], b[4]), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package proxmox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestREQ157_IPv6JoinHostPort verifies that the proxmox SSH dial
|
||||||
|
// address is correctly bracketed for IPv6 hosts (REQ-157 / P08 T5/T11).
|
||||||
|
func TestREQ157_IPv6JoinHostPort(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
host string
|
||||||
|
port int
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"192.168.1.1", 22, "192.168.1.1:22"},
|
||||||
|
{"::1", 22, "[::1]:22"},
|
||||||
|
{"fe80::1", 2222, "[fe80::1]:2222"},
|
||||||
|
{"2001:db8::1", 22, "[2001:db8::1]:22"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.host, func(t *testing.T) {
|
||||||
|
got := net.JoinHostPort(tt.host, fmt.Sprintf("%d", tt.port))
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("JoinHostPort(%s, %d) = %q, want %q", tt.host, tt.port, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,11 +5,13 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
@@ -54,12 +56,14 @@ type Transport struct {
|
|||||||
pool sync.Map
|
pool sync.Map
|
||||||
// keyPath is the SSH private key path (Ed25519, D-037).
|
// keyPath is the SSH private key path (Ed25519, D-037).
|
||||||
keyPath string
|
keyPath string
|
||||||
// knownHostsPath is the v0.9 known_hosts path (paths.KnownHostsPath()
|
// knownHostsPath is the known_hosts path passed to the TOFU
|
||||||
// = ClusterDir()/known_hosts). It is stored for the v0.10-P14 migration
|
// host-key callback (D-035). NewTransport sets it from
|
||||||
// when proxmox.TOFUHostKeyCallback will accept a path parameter; today
|
// certpaths.KnownHostsPath() (v0.8 flat layout) by default; callers
|
||||||
// the callback reads certpaths.KnownHostsPath() (the v0.8 flat layout)
|
// that want the v0.9 paths.KnownHostsPath() location construct the
|
||||||
// directly, so this field is not yet read by dial(). Tests set
|
// transport with that path explicitly. REQ-157 / P08 T4: this field
|
||||||
// $ORCA_HOME so certpaths.KnownHostsPath() resolves under the temp dir.
|
// IS read by dial() (via proxmox.TOFUHostKeyCallbackPath) — the
|
||||||
|
// earlier bug where the callback ignored it and read
|
||||||
|
// certpaths.KnownHostsPath() directly is fixed.
|
||||||
knownHostsPath string
|
knownHostsPath string
|
||||||
// user is the remote SSH user (default "orca", D-037).
|
// user is the remote SSH user (default "orca", D-037).
|
||||||
user string
|
user string
|
||||||
@@ -120,15 +124,14 @@ func (defaultSSHDialer) DialContext(ctx context.Context, network, addr string, c
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewTransport returns a Transport configured with the given SSH
|
// NewTransport returns a Transport configured with the given SSH
|
||||||
// private key path and known_hosts path. The known_hosts path is the v0.9
|
// private key path and known_hosts path. The known_hosts path is read
|
||||||
// location (paths.KnownHostsPath); it is stored for the v0.10-P14
|
// by dial() via proxmox.TOFUHostKeyCallbackPath (D-035, REQ-157/P08 T4):
|
||||||
// migration when the TOFU callback will accept a path parameter. Today
|
// the TOFU callback locks/captures against this path on first connect.
|
||||||
// dial() delegates host-key verification to proxmox.TOFUHostKeyCallback,
|
// Callers typically pass certpaths.KnownHostsPath() (the v0.8 flat
|
||||||
// which reads certpaths.KnownHostsPath() (the v0.8 flat layout under
|
// layout under $ORCA_HOME) or paths.KnownHostsPath() (the v0.9
|
||||||
// $ORCA_HOME) directly — so callers must ensure $ORCA_HOME points at the
|
// ClusterDir() location). The remote user defaults to "orca" (D-037);
|
||||||
// cluster root (the CLI sets this up). The remote user defaults to
|
// override with SetUser. The dialer defaults to the real ssh.Dial-based
|
||||||
// "orca" (D-037); override with SetUser. The dialer defaults to the
|
// dialer; tests call SetDialer to inject a mock.
|
||||||
// real ssh.Dial-based dialer; tests call SetDialer to inject a mock.
|
|
||||||
func NewTransport(keyPath, knownHostsPath string) *Transport {
|
func NewTransport(keyPath, knownHostsPath string) *Transport {
|
||||||
return &Transport{
|
return &Transport{
|
||||||
keyPath: keyPath,
|
keyPath: keyPath,
|
||||||
@@ -193,7 +196,14 @@ func (t *Transport) dial(peer string) (*ssh.Client, error) {
|
|||||||
// Host-key verification reuses the v0.8 TOFU wrapper (D-035). The
|
// Host-key verification reuses the v0.8 TOFU wrapper (D-035). The
|
||||||
// known_hosts file is flock-protected inside the callback on
|
// known_hosts file is flock-protected inside the callback on
|
||||||
// first-connect capture, so we do NOT re-lock here.
|
// first-connect capture, so we do NOT re-lock here.
|
||||||
cb, err := proxmox.TOFUHostKeyCallback(peer, nil)
|
//
|
||||||
|
// REQ-157 / P08 T4: use the stored knownHostsPath field (set via
|
||||||
|
// NewTransport from certpaths.KnownHostsPath() / paths.KnownHostsPath())
|
||||||
|
// instead of having the callback read certpaths.KnownHostsPath() (the
|
||||||
|
// v0.8 flat layout) directly. This closes the bug where the flock
|
||||||
|
// field was stored but never read by dial() — the TOFU callback now
|
||||||
|
// locks/captures against the path the transport was constructed with.
|
||||||
|
cb, err := proxmox.TOFUHostKeyCallbackPath(t.knownHostsPath, peer, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("sshpush: host-key callback: %w", err)
|
return nil, fmt.Errorf("sshpush: host-key callback: %w", err)
|
||||||
}
|
}
|
||||||
@@ -391,7 +401,14 @@ func backoff(initial, max time.Duration, n int) time.Duration {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// isTransient reports whether err looks like a transient failure worth
|
// isTransient reports whether err looks like a transient failure worth
|
||||||
// retrying (mirrors v0.8 transport.IsTransient, reimplemented here).
|
// retrying (mirrors transport.IsTransient, reimplemented here so
|
||||||
|
// internal/sshpush does not import internal/transport).
|
||||||
|
//
|
||||||
|
// REQ-157 / P08 T2: classification is TYPE-BASED, not substring-based.
|
||||||
|
// The primary path is errors.Is against the sentinels (ErrTransient /
|
||||||
|
// ErrPermanent) and against well-known syscall/net/io errors. The
|
||||||
|
// substring fallback is retained ONLY for unwrapped errors from the
|
||||||
|
// ssh.Dialer that do not implement the standard interfaces.
|
||||||
func isTransient(err error) bool {
|
func isTransient(err error) bool {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return false
|
return false
|
||||||
@@ -402,6 +419,29 @@ func isTransient(err error) bool {
|
|||||||
if errors.Is(err, ErrPermanent) {
|
if errors.Is(err, ErrPermanent) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
// Typed: a net.Error that is a timeout is transient; a net.OpError
|
||||||
|
// whose Temporary() is true (ECONNREFUSED et al) is transient.
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.As(err, &netErr) {
|
||||||
|
if netErr.Timeout() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return isTemporarySSH(netErr)
|
||||||
|
}
|
||||||
|
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||||
|
errors.Is(err, syscall.ECONNRESET) ||
|
||||||
|
errors.Is(err, syscall.ETIMEDOUT) ||
|
||||||
|
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||||
|
errors.Is(err, syscall.ENETUNREACH) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Substring fallback (defense-in-depth for unwrapped errors).
|
||||||
s := err.Error()
|
s := err.Error()
|
||||||
for _, sub := range []string{
|
for _, sub := range []string{
|
||||||
"connection refused", "i/o timeout", "EOF",
|
"connection refused", "i/o timeout", "EOF",
|
||||||
@@ -415,6 +455,17 @@ func isTransient(err error) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isTemporarySSH reports whether netErr implements the legacy
|
||||||
|
// Temporary() bool method and it returns true. net.OpError.Temporary()
|
||||||
|
// maps to the underlying errno's temporary classification.
|
||||||
|
func isTemporarySSH(netErr net.Error) bool {
|
||||||
|
type temporary interface{ Temporary() bool }
|
||||||
|
if t, ok := netErr.(temporary); ok {
|
||||||
|
return t.Temporary()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// classifyDialErr converts a raw ssh.Dial error into a transport error
|
// classifyDialErr converts a raw ssh.Dial error into a transport error
|
||||||
// (transient vs permanent). Auth failures and host-key mismatches are
|
// (transient vs permanent). Auth failures and host-key mismatches are
|
||||||
// permanent; everything else is transient.
|
// permanent; everything else is transient.
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
-- REQ-175 / R-024: add ingress_mode column to nodes.
|
||||||
|
-- Values: '' (legacy/default), 'native' (proxmox native — traefik
|
||||||
|
-- in LXC), 'floating-ip' (proxmox floating-IP — ingress LXC owns
|
||||||
|
-- the floating IP). Defaults to empty string for backward
|
||||||
|
-- compatibility with pre-v0.14 nodes.
|
||||||
|
ALTER TABLE nodes ADD COLUMN ingress_mode TEXT NOT NULL DEFAULT '';
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
package traefik
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultVersion is the traefik version tag for the orca-traefik image.
|
||||||
|
const DefaultVersion = "v3.3.0"
|
||||||
|
|
||||||
|
// DefaultImage is the full image reference for the orca-traefik container.
|
||||||
|
// The tag is resolved at runtime from the orca version (or "latest" for
|
||||||
|
// dev builds). The image is built from Dockerfile.traefik and published
|
||||||
|
// per release (REQ-171).
|
||||||
|
const DefaultImage = "git.cloudinit.dev/coreci/orca-traefik"
|
||||||
|
|
||||||
|
// ContainerName is the podman container name for the traefik data plane.
|
||||||
|
const ContainerName = "orca-traefik"
|
||||||
|
|
||||||
|
// RemoteExecFunc runs a command on a remote host and returns combined
|
||||||
|
// output. It is the same signature used by the v0.13 binary installer
|
||||||
|
// and by the proxmox/linux bootstrap SSH sessions.
|
||||||
|
type RemoteExecFunc func(cmd string) ([]byte, error)
|
||||||
|
|
||||||
|
// ImageRef returns the full image:tag reference for the orca-traefik
|
||||||
|
// container. If version is empty or "dev"/"0.1.0-dev", it falls back to
|
||||||
|
// "latest" (dev builds don't have a published tag).
|
||||||
|
func ImageRef(version string) string {
|
||||||
|
tag := version
|
||||||
|
if tag == "" || tag == "dev" || tag == "0.1.0-dev" || strings.HasSuffix(tag, "-dev") {
|
||||||
|
return fmt.Sprintf("%s:latest", DefaultImage)
|
||||||
|
}
|
||||||
|
tag = strings.TrimPrefix(tag, "v")
|
||||||
|
return fmt.Sprintf("%s:v%s", DefaultImage, tag)
|
||||||
|
}
|
||||||
|
|
||||||
|
// podmanRunArgs returns the podman run arguments for the traefik
|
||||||
|
// container. The container uses --network host so traefik binds
|
||||||
|
// 127.0.0.1:8080/8443 directly on the host (or LXC) loopback. nft
|
||||||
|
// DNATs public :443/:80 to those loopback ports (R-017/R-024).
|
||||||
|
//
|
||||||
|
// Volume mounts (no SELinux :Z flag — research finding Topic 7):
|
||||||
|
// - /etc/traefik/traefik.yml:ro — static config (overrides baked default; C-58)
|
||||||
|
// - /etc/traefik/dynamic:ro — dynamic config (orca writes atomically via SSH-push)
|
||||||
|
// - /etc/orca/step-ca-root.crt:ro — step-ca root CA (for future mTLS; v0.14 uses tls:{})
|
||||||
|
func podmanRunArgs(imageRef string) []string {
|
||||||
|
return []string{
|
||||||
|
"run", "-d",
|
||||||
|
"--name", ContainerName,
|
||||||
|
"--restart=unless-stopped",
|
||||||
|
"--network", "host",
|
||||||
|
"-v", "/etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro",
|
||||||
|
"-v", "/etc/traefik/dynamic:/etc/traefik/dynamic:ro",
|
||||||
|
"-v", "/etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro",
|
||||||
|
imageRef,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureTraefikContainerLocal ensures the orca-traefik podman container
|
||||||
|
// is running on the local host. It is idempotent:
|
||||||
|
// 1. If the container is running → no-op.
|
||||||
|
// 2. If the container exists but is stopped → start it.
|
||||||
|
// 3. If the container does not exist → pull the image + run it.
|
||||||
|
//
|
||||||
|
// C-50: if podman is not installed, attempts apt-get install. If that
|
||||||
|
// fails, returns an error with install instructions.
|
||||||
|
//
|
||||||
|
// C-57: if a legacy v0.13 systemd service exists (orca-traefik.service),
|
||||||
|
// it is stopped, disabled, and removed before starting the container.
|
||||||
|
func EnsureTraefikContainerLocal(ctx context.Context, version string) error {
|
||||||
|
imageRef := ImageRef(version)
|
||||||
|
if err := ensurePodmanLocal(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := removeLegacySystemdUnitLocal(ctx); err != nil {
|
||||||
|
// Non-fatal: legacy unit may not exist on fresh installs.
|
||||||
|
_ = err
|
||||||
|
}
|
||||||
|
if err := ensureDirsLocal(); err != nil {
|
||||||
|
return fmt.Errorf("traefik: ensure dirs: %w", err)
|
||||||
|
}
|
||||||
|
return reconcileContainerLocal(ctx, imageRef)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureTraefikContainerRemote ensures the orca-traefik podman container
|
||||||
|
// is running on a remote host (via SSH exec). Same idempotent logic as
|
||||||
|
// EnsureTraefikContainerLocal but over the provided exec function.
|
||||||
|
func EnsureTraefikContainerRemote(ctx context.Context, version string, execFn RemoteExecFunc) error {
|
||||||
|
imageRef := ImageRef(version)
|
||||||
|
if err := ensurePodmanRemote(execFn); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := removeLegacySystemdUnitRemote(execFn); err != nil {
|
||||||
|
_ = err // non-fatal
|
||||||
|
}
|
||||||
|
if _, err := execFn("mkdir -p /etc/traefik/dynamic /etc/orca"); err != nil {
|
||||||
|
return fmt.Errorf("traefik: ensure remote dirs: %w", err)
|
||||||
|
}
|
||||||
|
return reconcileContainerRemote(execFn, imageRef)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensurePodmanLocal checks if podman is installed locally and attempts
|
||||||
|
// to install it if absent (C-50).
|
||||||
|
func ensurePodmanLocal(ctx context.Context) error {
|
||||||
|
if _, err := exec.LookPath("podman"); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Attempt apt-get install (Ubuntu/Debian).
|
||||||
|
cmd := exec.CommandContext(ctx, "bash", "-c",
|
||||||
|
"apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs 2>&1")
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("podman not found and apt-get install failed: %w (output: %s).\nInstall podman manually: apt-get install podman conmon crun fuse-overlayfs", err, string(out))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensurePodmanRemote checks if podman is installed on the remote host
|
||||||
|
// and attempts to install it if absent (C-50).
|
||||||
|
func ensurePodmanRemote(execFn RemoteExecFunc) error {
|
||||||
|
if out, err := execFn("command -v podman"); err == nil && len(strings.TrimSpace(string(out))) > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Attempt apt-get install on the remote host.
|
||||||
|
cmd := "apt-get update -qq && apt-get install -y -qq podman conmon crun fuse-overlayfs 2>&1"
|
||||||
|
if out, err := execFn(cmd); err != nil {
|
||||||
|
return fmt.Errorf("podman not found on remote and apt-get install failed: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeLegacySystemdUnitLocal stops, disables, and removes the legacy
|
||||||
|
// v0.13 orca-traefik.service systemd unit + /usr/local/bin/traefik
|
||||||
|
// binary (C-57). Idempotent — no-op if the unit doesn't exist.
|
||||||
|
func removeLegacySystemdUnitLocal(ctx context.Context) error {
|
||||||
|
// Check if the legacy unit exists.
|
||||||
|
if _, err := exec.CommandContext(ctx, "systemctl", "is-active", "orca-traefik.service").CombinedOutput(); err == nil {
|
||||||
|
// Unit is active or exists — stop + disable it.
|
||||||
|
_ = exec.CommandContext(ctx, "systemctl", "stop", "orca-traefik.service").Run()
|
||||||
|
_ = exec.CommandContext(ctx, "systemctl", "disable", "orca-traefik.service").Run()
|
||||||
|
}
|
||||||
|
// Remove the unit file and binary.
|
||||||
|
_ = exec.CommandContext(ctx, "rm", "-f", "/etc/systemd/system/orca-traefik.service").Run()
|
||||||
|
_ = exec.CommandContext(ctx, "rm", "-f", "/usr/local/bin/traefik").Run()
|
||||||
|
_ = exec.CommandContext(ctx, "systemctl", "daemon-reload").Run()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeLegacySystemdUnitRemote is the remote SSH variant (C-57).
|
||||||
|
func removeLegacySystemdUnitRemote(execFn RemoteExecFunc) error {
|
||||||
|
cmd := `systemctl is-active orca-traefik.service 2>/dev/null && systemctl stop orca-traefik.service 2>/dev/null; systemctl disable orca-traefik.service 2>/dev/null; rm -f /etc/systemd/system/orca-traefik.service /usr/local/bin/traefik; systemctl daemon-reload 2>/dev/null; true`
|
||||||
|
_, _ = execFn(cmd)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureDirsLocal creates /etc/traefik/dynamic and /etc/orca locally.
|
||||||
|
func ensureDirsLocal() error {
|
||||||
|
if err := exec.Command("mkdir", "-p", "/etc/traefik/dynamic", "/etc/orca").Run(); err != nil {
|
||||||
|
return fmt.Errorf("mkdir: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileContainerLocal implements the idempotent pull+run logic
|
||||||
|
// locally (C-50).
|
||||||
|
func reconcileContainerLocal(ctx context.Context, imageRef string) error {
|
||||||
|
// Check if the container is already running.
|
||||||
|
out, err := exec.CommandContext(ctx, "podman", "inspect", "--format", "{{.State.Running}}", ContainerName).CombinedOutput()
|
||||||
|
if err == nil {
|
||||||
|
v := strings.TrimSpace(string(out))
|
||||||
|
if v == "true" {
|
||||||
|
return nil // already running
|
||||||
|
}
|
||||||
|
// Container exists but is stopped — start it.
|
||||||
|
if _, err := exec.CommandContext(ctx, "podman", "start", ContainerName).CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("podman start %s: %w", ContainerName, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Container does not exist — pull + run.
|
||||||
|
if out, err := exec.CommandContext(ctx, "podman", "pull", imageRef).CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("podman pull %s: %w (output: %s)", imageRef, err, string(out))
|
||||||
|
}
|
||||||
|
args := append([]string{}, podmanRunArgs(imageRef)...)
|
||||||
|
if out, err := exec.CommandContext(ctx, "podman", args...).CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("podman run: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
// Enable podman-restart.service for reboot persistence (research Topic 6).
|
||||||
|
_ = exec.CommandContext(ctx, "systemctl", "enable", "--now", "podman-restart.service").Run()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileContainerRemote implements the idempotent pull+run logic
|
||||||
|
// over SSH exec.
|
||||||
|
func reconcileContainerRemote(execFn RemoteExecFunc, imageRef string) error {
|
||||||
|
// Check if the container is already running.
|
||||||
|
out, err := execFn(fmt.Sprintf("podman inspect --format '{{.State.Running}}' %s 2>/dev/null", ContainerName))
|
||||||
|
if err == nil {
|
||||||
|
v := strings.TrimSpace(string(out))
|
||||||
|
if v == "true" {
|
||||||
|
return nil // already running
|
||||||
|
}
|
||||||
|
// Container exists but is stopped — start it.
|
||||||
|
if _, err := execFn(fmt.Sprintf("podman start %s 2>/dev/null", ContainerName)); err != nil {
|
||||||
|
return fmt.Errorf("podman start %s: %w", ContainerName, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Container does not exist — pull + run.
|
||||||
|
if out, err := execFn(fmt.Sprintf("podman pull %s", shellQuote(imageRef))); err != nil {
|
||||||
|
return fmt.Errorf("podman pull %s: %w (output: %s)", imageRef, err, string(out))
|
||||||
|
}
|
||||||
|
runArgs := strings.Join(podmanRunArgs(imageRef), " ")
|
||||||
|
if out, err := execFn(fmt.Sprintf("podman %s", runArgs)); err != nil {
|
||||||
|
return fmt.Errorf("podman run: %w (output: %s)", err, string(out))
|
||||||
|
}
|
||||||
|
// Enable podman-restart.service for reboot persistence (research Topic 6).
|
||||||
|
_, _ = execFn("systemctl enable --now podman-restart.service 2>/dev/null || true")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// shellQuote wraps a string in single quotes for shell-safe usage.
|
||||||
|
func shellQuote(s string) string {
|
||||||
|
return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'"
|
||||||
|
}
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
package traefik
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestImageRef(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
version string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"v0.13.1", "git.cloudinit.dev/coreci/orca-traefik:v0.13.1"},
|
||||||
|
{"0.13.1", "git.cloudinit.dev/coreci/orca-traefik:v0.13.1"},
|
||||||
|
{"", "git.cloudinit.dev/coreci/orca-traefik:latest"},
|
||||||
|
{"dev", "git.cloudinit.dev/coreci/orca-traefik:latest"},
|
||||||
|
{"0.1.0-dev", "git.cloudinit.dev/coreci/orca-traefik:latest"},
|
||||||
|
{"v1.2.3-dev", "git.cloudinit.dev/coreci/orca-traefik:latest"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
got := ImageRef(tt.version)
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("ImageRef(%q) = %q, want %q", tt.version, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPodmanRunArgs(t *testing.T) {
|
||||||
|
args := podmanRunArgs("git.cloudinit.dev/coreci/orca-traefik:v0.13.1")
|
||||||
|
joined := strings.Join(args, " ")
|
||||||
|
checks := []string{
|
||||||
|
"run -d",
|
||||||
|
"--name orca-traefik",
|
||||||
|
"--restart=unless-stopped",
|
||||||
|
"--network host",
|
||||||
|
"/etc/traefik/traefik.yml:/etc/traefik/traefik.yml:ro",
|
||||||
|
"/etc/traefik/dynamic:/etc/traefik/dynamic:ro",
|
||||||
|
"/etc/orca/step-ca-root.crt:/etc/orca/step-ca-root.crt:ro",
|
||||||
|
"git.cloudinit.dev/coreci/orca-traefik:v0.13.1",
|
||||||
|
}
|
||||||
|
for _, c := range checks {
|
||||||
|
if !strings.Contains(joined, c) {
|
||||||
|
t.Errorf("podmanRunArgs missing %q\nfull: %s", c, joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Ensure no :Z flag (research Topic 7)
|
||||||
|
if strings.Contains(joined, ":Z") {
|
||||||
|
t.Errorf("podmanRunArgs should NOT contain :Z SELinux flag\nfull: %s", joined)
|
||||||
|
}
|
||||||
|
// Ensure --restart=always is NOT used (research Topic 6)
|
||||||
|
if strings.Contains(joined, "--restart=always") {
|
||||||
|
t.Errorf("podmanRunArgs should use --restart=unless-stopped, not --restart=always\nfull: %s", joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnsureTraefikContainerRemote_ContainerRunning(t *testing.T) {
|
||||||
|
var cmds []string
|
||||||
|
execFn := func(cmd string) ([]byte, error) {
|
||||||
|
cmds = append(cmds, cmd)
|
||||||
|
if strings.Contains(cmd, "podman inspect") {
|
||||||
|
return []byte("true\n"), nil
|
||||||
|
}
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
err := EnsureTraefikContainerRemote(context.Background(), "v0.13.1", execFn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
// Should have checked inspect and found it running — no pull/run.
|
||||||
|
if len(cmds) < 1 {
|
||||||
|
t.Fatal("expected at least 1 command (inspect)")
|
||||||
|
}
|
||||||
|
for _, c := range cmds {
|
||||||
|
if strings.Contains(c, "podman pull") {
|
||||||
|
t.Errorf("should not pull when container is running: %s", c)
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "podman run") {
|
||||||
|
t.Errorf("should not run when container is running: %s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnsureTraefikContainerRemote_ContainerStopped(t *testing.T) {
|
||||||
|
var cmds []string
|
||||||
|
execFn := func(cmd string) ([]byte, error) {
|
||||||
|
cmds = append(cmds, cmd)
|
||||||
|
if strings.Contains(cmd, "podman inspect") {
|
||||||
|
return []byte("false\n"), nil // stopped
|
||||||
|
}
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
err := EnsureTraefikContainerRemote(context.Background(), "v0.13.1", execFn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
// Should have started the container.
|
||||||
|
foundStart := false
|
||||||
|
for _, c := range cmds {
|
||||||
|
if strings.Contains(c, "podman start orca-traefik") {
|
||||||
|
foundStart = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundStart {
|
||||||
|
t.Errorf("expected 'podman start orca-traefik' when container is stopped\ncommands: %v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnsureTraefikContainerRemote_ContainerAbsent(t *testing.T) {
|
||||||
|
var cmds []string
|
||||||
|
execFn := func(cmd string) ([]byte, error) {
|
||||||
|
cmds = append(cmds, cmd)
|
||||||
|
if strings.Contains(cmd, "podman inspect") {
|
||||||
|
return nil, &execError{"inspect failed: no such container"}
|
||||||
|
}
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
err := EnsureTraefikContainerRemote(context.Background(), "v0.13.1", execFn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
// Should have pulled and run.
|
||||||
|
foundPull := false
|
||||||
|
foundRun := false
|
||||||
|
for _, c := range cmds {
|
||||||
|
if strings.Contains(c, "podman pull") {
|
||||||
|
foundPull = true
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "podman run -d") {
|
||||||
|
foundRun = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundPull {
|
||||||
|
t.Errorf("expected 'podman pull' when container is absent\ncommands: %v", cmds)
|
||||||
|
}
|
||||||
|
if !foundRun {
|
||||||
|
t.Errorf("expected 'podman run -d' when container is absent\ncommands: %v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnsureTraefikContainerRemote_LegacySystemdRemoval(t *testing.T) {
|
||||||
|
var cmds []string
|
||||||
|
execFn := func(cmd string) ([]byte, error) {
|
||||||
|
cmds = append(cmds, cmd)
|
||||||
|
if strings.Contains(cmd, "podman inspect") {
|
||||||
|
return []byte("true\n"), nil // container running
|
||||||
|
}
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
_ = EnsureTraefikContainerRemote(context.Background(), "v0.13.1", execFn)
|
||||||
|
// Should include legacy systemd unit removal command (C-57).
|
||||||
|
foundLegacyRemoval := false
|
||||||
|
for _, c := range cmds {
|
||||||
|
if strings.Contains(c, "orca-traefik.service") && strings.Contains(c, "stop") {
|
||||||
|
foundLegacyRemoval = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundLegacyRemoval {
|
||||||
|
t.Errorf("expected legacy systemd unit removal command (C-57)\ncommands: %v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// execError is a simple error type for testing.
|
||||||
|
type execError struct{ msg string }
|
||||||
|
|
||||||
|
func (e *execError) Error() string { return e.msg }
|
||||||
@@ -17,9 +17,19 @@ var metricOrder = []string{
|
|||||||
"txns_applied_total",
|
"txns_applied_total",
|
||||||
"txns_drifted_total",
|
"txns_drifted_total",
|
||||||
"drifts_remediated_total",
|
"drifts_remediated_total",
|
||||||
|
"drifts_remediated_total",
|
||||||
"peers_total",
|
"peers_total",
|
||||||
"nodes_total",
|
"nodes_total",
|
||||||
"allocs_total",
|
"allocs_total",
|
||||||
|
"orca_jobs_running",
|
||||||
|
"orca_jobs_failed",
|
||||||
|
"orca_jobs_complete",
|
||||||
|
"orca_audit_chain_head",
|
||||||
|
"orca_drift_events_total",
|
||||||
|
"orca_ssh_errors_total",
|
||||||
|
"orca_txn_apply_total",
|
||||||
|
"orca_txn_rollback_total",
|
||||||
|
"orca_acl_denials_total",
|
||||||
}
|
}
|
||||||
|
|
||||||
var metricMeta = map[string]metricDef{
|
var metricMeta = map[string]metricDef{
|
||||||
@@ -29,6 +39,15 @@ var metricMeta = map[string]metricDef{
|
|||||||
"peers_total": {"Current peer count", "gauge"},
|
"peers_total": {"Current peer count", "gauge"},
|
||||||
"nodes_total": {"Current node count", "gauge"},
|
"nodes_total": {"Current node count", "gauge"},
|
||||||
"allocs_total": {"Current allocation count", "gauge"},
|
"allocs_total": {"Current allocation count", "gauge"},
|
||||||
|
"orca_jobs_running": {"Jobs currently running", "gauge"},
|
||||||
|
"orca_jobs_failed": {"Jobs that failed", "gauge"},
|
||||||
|
"orca_jobs_complete": {"Jobs completed successfully", "gauge"},
|
||||||
|
"orca_audit_chain_head": {"Audit chain integrity (1=verified)", "gauge"},
|
||||||
|
"orca_drift_events_total": {"Total drift events detected", "counter"},
|
||||||
|
"orca_ssh_errors_total": {"Total SSH errors", "counter"},
|
||||||
|
"orca_txn_apply_total": {"Total transaction applies", "counter"},
|
||||||
|
"orca_txn_rollback_total": {"Total transaction rollbacks", "counter"},
|
||||||
|
"orca_acl_denials_total": {"Total ACL denials (enforce mode)", "counter"},
|
||||||
}
|
}
|
||||||
|
|
||||||
type Metrics struct {
|
type Metrics struct {
|
||||||
|
|||||||
+81
-23
@@ -8,7 +8,11 @@ package transport
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
"net"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,29 +38,6 @@ func DefaultRetryPolicy() RetryPolicy {
|
|||||||
return RetryPolicy{Initial: RetryInitial, Max: RetryMax, MaxAttempts: RetryMaxAttempts}
|
return RetryPolicy{Initial: RetryInitial, Max: RetryMax, MaxAttempts: RetryMaxAttempts}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsTransient reports whether err looks like a transient failure
|
|
||||||
// worth retrying. We treat network errors, context-deadline-exceeded
|
|
||||||
// (peer was slow but reachable), and a sentinel ErrTransient as
|
|
||||||
// retryable; everything else (4xx, validation, auth) is permanent.
|
|
||||||
func IsTransient(err error) bool {
|
|
||||||
if err == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if errors.Is(err, ErrTransient) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// We avoid pulling net/error here to keep dependencies minimal;
|
|
||||||
// the most common transient signature is the substring "connection
|
|
||||||
// refused" or "i/o timeout". Tests assert these explicitly.
|
|
||||||
s := err.Error()
|
|
||||||
for _, sub := range []string{"connection refused", "i/o timeout", "EOF", "no such host", "connection reset"} {
|
|
||||||
if contains(s, sub) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrTransient is a sentinel callers can wrap to mark an error
|
// ErrTransient is a sentinel callers can wrap to mark an error
|
||||||
// retryable. ErrPermanent is the opposite.
|
// retryable. ErrPermanent is the opposite.
|
||||||
var (
|
var (
|
||||||
@@ -64,6 +45,83 @@ var (
|
|||||||
ErrPermanent = errors.New("permanent error")
|
ErrPermanent = errors.New("permanent error")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// IsTransient reports whether err looks like a transient failure
|
||||||
|
// worth retrying. We treat network errors, context-deadline-exceeded
|
||||||
|
// (peer was slow but reachable), and a sentinel ErrTransient as
|
||||||
|
// retryable; everything else (4xx, validation, auth) is permanent.
|
||||||
|
//
|
||||||
|
// REQ-157 / P08 T1: classification is TYPE-BASED, not substring-based.
|
||||||
|
// The primary path is errors.Is against the sentinels (ErrTransient /
|
||||||
|
// ErrPermanent) and against well-known syscall/net/io errors. The
|
||||||
|
// substring fallback is retained ONLY for unwrapped errors from
|
||||||
|
// third-party dialers that do not implement the standard interfaces
|
||||||
|
// (defense-in-depth); callers SHOULD wrap with ErrTransient instead.
|
||||||
|
func IsTransient(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Explicit sentinels win.
|
||||||
|
if errors.Is(err, ErrTransient) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrPermanent) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Typed classification: a net.Error that is a timeout is transient.
|
||||||
|
var netErr net.Error
|
||||||
|
if errors.As(err, &netErr) {
|
||||||
|
if netErr.Timeout() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// net.OpError implements Temporary(); that maps to the
|
||||||
|
// underlying errno's temporary classification (ECONNREFUSED et
|
||||||
|
// al). We keep the check so a plain "dial tcp: connection
|
||||||
|
// refused" classifies as transient.
|
||||||
|
return isTemporary(netErr)
|
||||||
|
}
|
||||||
|
// Specific syscall errors that are universally retryable.
|
||||||
|
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||||
|
errors.Is(err, syscall.ECONNRESET) ||
|
||||||
|
errors.Is(err, syscall.ETIMEDOUT) ||
|
||||||
|
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||||
|
errors.Is(err, syscall.ENETUNREACH) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// io.EOF on a read from a half-closed peer is transient (the
|
||||||
|
// dispatch HTTP/2 path can surface this mid-stream).
|
||||||
|
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// context.DeadlineExceeded from a slow-but-reachable peer is
|
||||||
|
// transient (the next attempt may succeed under a fresh deadline).
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Substring fallback (defense-in-depth for unwrapped errors).
|
||||||
|
s := err.Error()
|
||||||
|
for _, sub := range []string{
|
||||||
|
"connection refused", "i/o timeout", "EOF",
|
||||||
|
"no such host", "connection reset",
|
||||||
|
"deadline exceeded", "temporarily unavailable",
|
||||||
|
} {
|
||||||
|
if strings.Contains(s, sub) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isTemporary reports whether netErr implements the legacy Temporary()
|
||||||
|
// bool method and it returns true. net.OpError.Temporary() maps to the
|
||||||
|
// underlying errno's temporary classification (ECONNREFUSED et al).
|
||||||
|
func isTemporary(netErr net.Error) bool {
|
||||||
|
type temporary interface{ Temporary() bool }
|
||||||
|
if t, ok := netErr.(temporary); ok {
|
||||||
|
return t.Temporary()
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// RetryableFunc is the signature Retry calls. It returns the result
|
// RetryableFunc is the signature Retry calls. It returns the result
|
||||||
// and an error. The bool indicates whether the call is idempotent
|
// and an error. The bool indicates whether the call is idempotent
|
||||||
// (true = safe to retry without an idempotency key).
|
// (true = safe to retry without an idempotency key).
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package transport
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"testing"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestREQ157_TypedErrorClassification verifies that IsTransient uses
|
||||||
|
// typed sentinels and standard interfaces, not substring matching
|
||||||
|
// (REQ-157 / P08 T10).
|
||||||
|
func TestREQ157_TypedErrorClassification(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"nil", nil, false},
|
||||||
|
{"ErrTransient", ErrTransient, true},
|
||||||
|
{"wrapped ErrTransient", fmt.Errorf("dial: %w", ErrTransient), true},
|
||||||
|
{"ErrPermanent", ErrPermanent, false},
|
||||||
|
{"wrapped ErrPermanent", fmt.Errorf("auth: %w", ErrPermanent), false},
|
||||||
|
{"net timeout", &net.OpError{Op: "dial", Net: "tcp", Err: &timeoutError{}}, true},
|
||||||
|
{"context deadline", context.DeadlineExceeded, true},
|
||||||
|
{"context canceled", context.Canceled, false},
|
||||||
|
{"io EOF", io.EOF, true},
|
||||||
|
{"plain error", errors.New("some permanent error"), false},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := IsTransient(tt.err)
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("IsTransient(%v) = %v, want %v", tt.err, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type timeoutError struct{}
|
||||||
|
|
||||||
|
func (timeoutError) Error() string { return "i/o timeout" }
|
||||||
|
func (timeoutError) Timeout() bool { return true }
|
||||||
|
func (timeoutError) Temporary() bool { return true }
|
||||||
|
|
||||||
|
var _ = fmt.Errorf
|
||||||
+83
-24
@@ -69,6 +69,9 @@ if [ "$SYSTEM" = "true" ]; then
|
|||||||
fi
|
fi
|
||||||
INSTALL_BIN="/usr/local/bin/orca"
|
INSTALL_BIN="/usr/local/bin/orca"
|
||||||
NAMESPACE_DIR="/root/.orca"
|
NAMESPACE_DIR="/root/.orca"
|
||||||
|
elif [ -w /usr/local/bin ] || [ "$(id -u)" -eq 0 ]; then
|
||||||
|
INSTALL_BIN="/usr/local/bin/orca"
|
||||||
|
NAMESPACE_DIR="${HOME}/.orca"
|
||||||
else
|
else
|
||||||
INSTALL_BIN="${HOME}/.local/bin/orca"
|
INSTALL_BIN="${HOME}/.local/bin/orca"
|
||||||
NAMESPACE_DIR="${HOME}/.orca"
|
NAMESPACE_DIR="${HOME}/.orca"
|
||||||
@@ -111,34 +114,58 @@ TARBALL="orca-${VERSION}-${OS}-${ARCH}.tar.gz"
|
|||||||
|
|
||||||
find_asset_url() {
|
find_asset_url() {
|
||||||
# $1 = tag. Prints the browser_download_url for the matching tarball, or empty.
|
# $1 = tag. Prints the browser_download_url for the matching tarball, or empty.
|
||||||
# The `|| true` prevents set -e + pipefail from exiting the script when
|
# Match by asset NAME (not URL path) — Gitea attachment URLs are opaque
|
||||||
# grep finds no match (exit 1) — an empty result is a valid outcome.
|
# UUIDs that don't contain the tarball name.
|
||||||
local tag="$1"
|
local tag="$1"
|
||||||
curl -fsSL "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${GITEA_REPO}/releases/tags/${tag}" \
|
local json
|
||||||
| sed -n 's/.*"browser_download_url"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \
|
json="$(curl -fsSL "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${GITEA_REPO}/releases/tags/${tag}" 2>/dev/null)" || return 1
|
||||||
| grep "/${TARBALL}\$" \
|
if command -v python3 >/dev/null 2>&1; then
|
||||||
| head -1 || true
|
echo "$json" | python3 -c "
|
||||||
|
import json,sys
|
||||||
|
r=json.load(sys.stdin)
|
||||||
|
for a in r.get('assets',[]):
|
||||||
|
if a.get('name')=='$TARBALL':
|
||||||
|
print(a.get('browser_download_url',''))
|
||||||
|
break
|
||||||
|
" 2>/dev/null || true
|
||||||
|
else
|
||||||
|
echo "$json" | sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"$TARBALL".*"browser_download_url"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
find_asset_in_releases() {
|
||||||
|
# Walk recent releases, find the newest with a matching asset name.
|
||||||
|
# Outputs two lines: URL and VERSION (caller captures both).
|
||||||
|
local json
|
||||||
|
json="$(curl -fsSL "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${GITEA_REPO}/releases?limit=50" 2>/dev/null)" || return 1
|
||||||
|
if command -v python3 >/dev/null 2>&1; then
|
||||||
|
echo "$json" | python3 -c "
|
||||||
|
import json,sys,re
|
||||||
|
rels=json.load(sys.stdin)
|
||||||
|
for r in rels:
|
||||||
|
tag=r.get('tag_name','')
|
||||||
|
for a in r.get('assets',[]):
|
||||||
|
name=a.get('name','')
|
||||||
|
m=re.match(r'orca-(v[0-9.]+)-' + '${OS}' + '-' + '${ARCH}' + r'\.tar\.gz$', name)
|
||||||
|
if m:
|
||||||
|
print(a.get('browser_download_url',''))
|
||||||
|
print(m.group(1))
|
||||||
|
sys.exit(0)
|
||||||
|
"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
info "locating asset ${TARBALL} in release ${VERSION}..."
|
info "locating asset ${TARBALL} in release ${VERSION}..."
|
||||||
ASSET_URL="$(find_asset_url "$VERSION")"
|
ASSET_URL="$(find_asset_url "$VERSION" 2>/dev/null)" || true
|
||||||
|
|
||||||
if [ -z "$ASSET_URL" ]; then
|
if [ -z "$ASSET_URL" ]; then
|
||||||
info "WARNING: release ${VERSION} has no ${TARBALL} asset. Walking back through recent releases..."
|
info "WARNING: release ${VERSION} has no ${TARBALL} asset. Walking back through recent releases..."
|
||||||
# The /releases list endpoint returns assets inline (browser_download_url
|
FALLBACK_OUT="$(find_asset_in_releases || true)"
|
||||||
# appears within each release's assets array). Extract all download URLs
|
ASSET_URL="$(echo "$FALLBACK_OUT" | head -1)"
|
||||||
# from the list response and find the first (newest) one matching our
|
|
||||||
# OS+arch tarball pattern (any version). This avoids per-release API calls.
|
|
||||||
ASSET_URL="$(curl -fsSL "${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${GITEA_REPO}/releases?limit=50" \
|
|
||||||
| grep -oE '"browser_download_url"[[:space:]]*:[[:space:]]*"[^"]*"' \
|
|
||||||
| sed -n 's/.*"browser_download_url"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \
|
|
||||||
| grep -E "/orca-[^/]*-${OS}-${ARCH}\.tar\.gz$" \
|
|
||||||
| head -1 || true)"
|
|
||||||
if [ -n "$ASSET_URL" ]; then
|
if [ -n "$ASSET_URL" ]; then
|
||||||
# Extract the version from the URL (e.g. .../download/v0.4.5/orca-...)
|
VERSION="$(echo "$FALLBACK_OUT" | tail -1)"
|
||||||
FALLBACK_VERSION="$(echo "$ASSET_URL" | sed -n 's|.*/download/\([^/]*\)/.*|\1|p')"
|
TARBALL="orca-${VERSION}-${OS}-${ARCH}.tar.gz"
|
||||||
info "WARNING: latest release ${VERSION} has no binary asset; falling back to ${FALLBACK_VERSION} which has orca-${FALLBACK_VERSION}-${OS}-${ARCH}.tar.gz."
|
info "WARNING: falling back to ${VERSION} which has orca-${VERSION}-${OS}-${ARCH}.tar.gz."
|
||||||
VERSION="$FALLBACK_VERSION"
|
|
||||||
else
|
else
|
||||||
err "could not find any release with a ${OS}-${ARCH} tarball in the last 50 releases. Check that a release exists with a linux-${ARCH} binary."
|
err "could not find any release with a ${OS}-${ARCH} tarball in the last 50 releases. Check that a release exists with a linux-${ARCH} binary."
|
||||||
fi
|
fi
|
||||||
@@ -178,9 +205,27 @@ info "downloading..."
|
|||||||
curl -fsSL -o "${TMPDIR}/${TARBALL}" "$ASSET_URL"
|
curl -fsSL -o "${TMPDIR}/${TARBALL}" "$ASSET_URL"
|
||||||
|
|
||||||
# REQ-132 / F14: verify tarball checksum before extraction.
|
# REQ-132 / F14: verify tarball checksum before extraction.
|
||||||
# Fetch SHA256SUMS from the same release; fail closed on mismatch.
|
# Fetch SHA256SUMS from the same release. For Gitea release-download
|
||||||
SHA256SUMS_URL="$(dirname "$ASSET_URL")/SHA256SUMS"
|
# URLs (e.g. /releases/download/vX.Y.Z/...) the SHA256SUMS is a sibling.
|
||||||
if curl -fsSL -o "${TMPDIR}/SHA256SUMS" "$SHA256SUMS_URL" 2>/dev/null; then
|
# For Gitea attachment URLs (e.g. /attachments/<uuid>) we must look up
|
||||||
|
# the SHA256SUMS asset by name from the release API.
|
||||||
|
SHA256SUMS_URL=""
|
||||||
|
if echo "$ASSET_URL" | grep -q "/releases/download/"; then
|
||||||
|
SHA256SUMS_URL="$(dirname "$ASSET_URL")/SHA256SUMS"
|
||||||
|
elif command -v python3 >/dev/null 2>&1; then
|
||||||
|
# Look up SHA256SUMS asset by name from the release API.
|
||||||
|
SHA256SUMS_URL="$(curl -fsSL \
|
||||||
|
"${GITEA_URL}/api/v1/repos/${GITEA_OWNER}/${GITEA_REPO}/releases/tags/${VERSION}" 2>/dev/null \
|
||||||
|
| python3 -c "
|
||||||
|
import json,sys
|
||||||
|
r=json.load(sys.stdin)
|
||||||
|
for a in r.get('assets',[]):
|
||||||
|
if a.get('name')=='SHA256SUMS':
|
||||||
|
print(a.get('browser_download_url',''))
|
||||||
|
break
|
||||||
|
" 2>/dev/null || true)"
|
||||||
|
fi
|
||||||
|
if [ -n "$SHA256SUMS_URL" ] && curl -fsSL -o "${TMPDIR}/SHA256SUMS" "$SHA256SUMS_URL" 2>/dev/null; then
|
||||||
info "verifying checksum..."
|
info "verifying checksum..."
|
||||||
(cd "$TMPDIR" && grep -F "$TARBALL" SHA256SUMS | sha256sum -c -) || {
|
(cd "$TMPDIR" && grep -F "$TARBALL" SHA256SUMS | sha256sum -c -) || {
|
||||||
err "checksum verification failed (REQ-132); refusing to install"
|
err "checksum verification failed (REQ-132); refusing to install"
|
||||||
@@ -218,10 +263,24 @@ if [ "$SYSTEM" = "true" ]; then
|
|||||||
info " namespace root: ${NAMESPACE_DIR} (use 'orca --system init' to initialize)"
|
info " namespace root: ${NAMESPACE_DIR} (use 'orca --system init' to initialize)"
|
||||||
else
|
else
|
||||||
info " namespace root: ${NAMESPACE_DIR} (use 'orca init' to initialize)"
|
info " namespace root: ${NAMESPACE_DIR} (use 'orca init' to initialize)"
|
||||||
|
# If the install dir is not on PATH, add it to .bashrc automatically.
|
||||||
if ! echo "$PATH" | grep -q "$INSTALL_DIR"; then
|
if ! echo "$PATH" | grep -q "$INSTALL_DIR"; then
|
||||||
info " NOTE: $INSTALL_DIR is not on your PATH. Add it:"
|
info " Adding $INSTALL_DIR to PATH via ~/.bashrc..."
|
||||||
|
SHELL_RC="${HOME}/.bashrc"
|
||||||
|
if [ -f "$SHELL_RC" ]; then
|
||||||
|
# Append only if not already present (idempotent).
|
||||||
|
if ! grep -qF "$INSTALL_DIR" "$SHELL_RC"; then
|
||||||
|
echo "" >> "$SHELL_RC"
|
||||||
|
echo "# Added by orca install.sh" >> "$SHELL_RC"
|
||||||
|
echo "export PATH=\"\$PATH:$INSTALL_DIR\"" >> "$SHELL_RC"
|
||||||
|
info " ✓ Added 'export PATH=\"\$PATH:$INSTALL_DIR\"' to ~/.bashrc"
|
||||||
|
info " Run 'source ~/.bashrc' or start a new shell to pick up the change."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
info " NOTE: ~/.bashrc not found. Add to PATH manually:"
|
||||||
info " export PATH=\"\$PATH:$INSTALL_DIR\""
|
info " export PATH=\"\$PATH:$INSTALL_DIR\""
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
info " verify: ${INSTALL_BIN} version"
|
info " verify: ${INSTALL_BIN} version"
|
||||||
@@ -101,6 +101,9 @@ info "built: bin/orca (linux-amd64)"
|
|||||||
|
|
||||||
TARBALL="orca-${VERSION}-linux-amd64.tar.gz"
|
TARBALL="orca-${VERSION}-linux-amd64.tar.gz"
|
||||||
tar -czf "$TARBALL" -C bin orca
|
tar -czf "$TARBALL" -C bin orca
|
||||||
|
sha256sum "$TARBALL" > SHA256SUMS
|
||||||
|
info "checksum: SHA256SUMS"
|
||||||
|
cat SHA256SUMS
|
||||||
info "packaged: $TARBALL ($(du -h "$TARBALL" | cut -f1))"
|
info "packaged: $TARBALL ($(du -h "$TARBALL" | cut -f1))"
|
||||||
|
|
||||||
# --- release notes from ---ci--- blocks ----------------------------------
|
# --- release notes from ---ci--- blocks ----------------------------------
|
||||||
@@ -208,3 +211,36 @@ else
|
|||||||
info "✓ container image ${IMAGE}:${VERSION} published"
|
info "✓ container image ${IMAGE}:${VERSION} published"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# --- publish orca-traefik container image (REQ-171, R-024) ----------------
|
||||||
|
# Custom traefik image with baked default static config. The host-side
|
||||||
|
# traefik.yml is mounted :ro at runtime to override the baked default.
|
||||||
|
# Skipped gracefully if docker is not on PATH (CI handles it via
|
||||||
|
# .coreci.yml container-publish-traefik step).
|
||||||
|
|
||||||
|
TRAEFIK_IMAGE="${CONTAINER_REGISTRY}/${CONTAINER_OWNER}/orca-traefik"
|
||||||
|
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
info "docker not found on PATH — skipping orca-traefik image publish (CI handles it)."
|
||||||
|
else
|
||||||
|
info "building orca-traefik image ${TRAEFIK_IMAGE}:${VERSION}..."
|
||||||
|
docker build \
|
||||||
|
-f Dockerfile.traefik \
|
||||||
|
-t "${TRAEFIK_IMAGE}:${VERSION}" \
|
||||||
|
-t "${TRAEFIK_IMAGE}:latest" \
|
||||||
|
"$REPO_ROOT"
|
||||||
|
|
||||||
|
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||||
|
info "GITEA_TOKEN not set — skipping orca-traefik push (image built locally only)."
|
||||||
|
else
|
||||||
|
if ! docker login "$CONTAINER_REGISTRY" -u cloudinit-bot --password-stdin <<<"$GITEA_TOKEN" 2>/dev/null; then
|
||||||
|
echo "$GITEA_TOKEN" | docker login "$CONTAINER_REGISTRY" -u cloudinit-bot --password-stdin
|
||||||
|
fi
|
||||||
|
info "pushing ${TRAEFIK_IMAGE}:${VERSION}..."
|
||||||
|
docker push "${TRAEFIK_IMAGE}:${VERSION}"
|
||||||
|
info "pushing ${TRAEFIK_IMAGE}:latest..."
|
||||||
|
docker push "${TRAEFIK_IMAGE}:latest"
|
||||||
|
docker logout "$CONTAINER_REGISTRY"
|
||||||
|
info "✓ orca-traefik image ${TRAEFIK_IMAGE}:${VERSION} published"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|||||||
@@ -39,6 +39,29 @@ load test_helper
|
|||||||
assert_contains "$output" "dry-run (--check)"
|
assert_contains "$output" "dry-run (--check)"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "install.sh fallback walk syncs TARBALL to fallback version (REQ-132 regression)" {
|
||||||
|
# Regression guard: when the fallback walk reassigns VERSION, the
|
||||||
|
# TARBALL variable must be recomputed too. v0.14.2 is a release with
|
||||||
|
# no binary asset; the installer must walk back to an earlier release.
|
||||||
|
# The dry-run "would install" line must reference the SAME fallback
|
||||||
|
# version as the "falling back to" line — not the stale pinned one.
|
||||||
|
# Before the fix, TARBALL stayed at the pinned v0.14.2 name while
|
||||||
|
# VERSION became the fallback, causing grep|sha256sum to see no
|
||||||
|
# matching checksum line and REQ-132 to refuse install.
|
||||||
|
skip_if_no_network
|
||||||
|
run timeout 60 "$SCRIPTS_DIR/install.sh" --check --version v0.14.2
|
||||||
|
assert_status 0 "$status"
|
||||||
|
assert_contains "$output" "falling back"
|
||||||
|
# Capture the fallback version from the "falling back to vX.Y.Z" line.
|
||||||
|
fb_version="$(printf '%s\n' "$output" | sed -n 's/.*falling back to \(v[0-9][0-9.]*\).*/\1/p' | head -1)"
|
||||||
|
[ -n "$fb_version" ] || { echo "could not parse fallback version from output: $output" >&2; return 1; }
|
||||||
|
# The dry-run "would install" line must use the fallback version,
|
||||||
|
# proving VERSION and TARBALL are in sync (not the stale pinned v0.14.2).
|
||||||
|
assert_contains "$output" "would install: orca ${fb_version}"
|
||||||
|
# And it must NOT reference the stale pinned version in the install line.
|
||||||
|
assert_not_contains "$output" "would install: orca v0.14.2"
|
||||||
|
}
|
||||||
|
|
||||||
@test "install.sh rejects unknown arguments" {
|
@test "install.sh rejects unknown arguments" {
|
||||||
run "$SCRIPTS_DIR/install.sh" --bogus-flag
|
run "$SCRIPTS_DIR/install.sh" --bogus-flag
|
||||||
[ "$status" -ne 0 ]
|
[ "$status" -ne 0 ]
|
||||||
|
|||||||
@@ -30,6 +30,20 @@ fi
|
|||||||
while read local_ref local_sha remote_ref remote_sha; do
|
while read local_ref local_sha remote_ref remote_sha; do
|
||||||
branch="${remote_ref#refs/heads/}"
|
branch="${remote_ref#refs/heads/}"
|
||||||
if [ -z "$branch" ] || [ "$branch" = "HEAD" ]; then
|
if [ -z "$branch" ] || [ "$branch" = "HEAD" ]; then
|
||||||
|
# Check if this is a tag push (refs/tags/*)
|
||||||
|
tag="${remote_ref#refs/tags/}"
|
||||||
|
if [ -n "$tag" ] && [ "$tag" != "$remote_ref" ]; then
|
||||||
|
echo "→ Triggering CoreCI for tag: $tag (${local_sha:0:7})"
|
||||||
|
payload=$(printf '{"repo":"coreci/orca","branch":"%s","ref":"%s"}' "$tag" "$local_sha")
|
||||||
|
if command -v curl >/dev/null 2>&1; then
|
||||||
|
curl -fsS -X POST "${CORECI_URL}/api/pipeline/run" \
|
||||||
|
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "$payload" >/dev/null 2>&1 \
|
||||||
|
&& echo " ✓ CoreCI triggered" \
|
||||||
|
|| echo " (CoreCI trigger failed; Gitea Actions webhook is secondary path)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
echo "→ Triggering CoreCI for branch: $branch (${local_sha:0:7})"
|
echo "→ Triggering CoreCI for branch: $branch (${local_sha:0:7})"
|
||||||
|
|||||||
Executable
+204
@@ -0,0 +1,204 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# orca UAT signoff script — v1.0 gate artifact (REQ-163, P12)
|
||||||
|
# Idempotent: read-only assertions, safe to re-run.
|
||||||
|
# Exit 0 iff ALL assertions pass.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
ORCA="${ORCA:-$(command -v orca || echo ./bin/orca)}"
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
SKIP=0
|
||||||
|
RESULTS=()
|
||||||
|
|
||||||
|
assert() {
|
||||||
|
local name="$1"
|
||||||
|
local check="$2"
|
||||||
|
local result="SKIP"
|
||||||
|
local msg=""
|
||||||
|
|
||||||
|
if [ -z "${ORCA_HOME:-}" ]; then
|
||||||
|
result="SKIP"
|
||||||
|
msg="ORCA_HOME not set"
|
||||||
|
elif ! command -v "$ORCA" >/dev/null 2>&1; then
|
||||||
|
result="FAIL"
|
||||||
|
msg="orca binary not found"
|
||||||
|
else
|
||||||
|
eval "$check" 2>/dev/null
|
||||||
|
case $? in
|
||||||
|
0) result="PASS"; msg="" ;;
|
||||||
|
77) result="SKIP"; msg="prerequisite not met" ;;
|
||||||
|
*) result="FAIL"; msg="check failed" ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$result" in
|
||||||
|
PASS) PASS=$((PASS+1)); RESULTS+=("PASS $name") ;;
|
||||||
|
FAIL) FAIL=$((FAIL+1)); RESULTS+=("FAIL $name -- $msg") ;;
|
||||||
|
SKIP) SKIP=$((SKIP+1)); RESULTS+=("SKIP $name -- $msg") ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Assertions ---
|
||||||
|
|
||||||
|
assert "01 orca_version" \
|
||||||
|
'$ORCA version 2>&1 | grep -qE "v0\.1[12]"'
|
||||||
|
|
||||||
|
assert "02 cluster_initialized" \
|
||||||
|
'$ORCA node list 2>&1 | grep -qE "(localhost|node)"'
|
||||||
|
|
||||||
|
assert "03 proxmox_onboarded" \
|
||||||
|
'$ORCA node list --json 2>&1 | grep -q "\"proxmox\""'
|
||||||
|
|
||||||
|
assert "04 linux_worker_onboarded" \
|
||||||
|
'$ORCA node list --json 2>&1 | grep -q "\"linux\""'
|
||||||
|
|
||||||
|
assert "05 capacity_set" \
|
||||||
|
'$ORCA node capacity list 2>&1 | grep -qE "(cpu|memory|[0-9]+)"'
|
||||||
|
|
||||||
|
assert "06 namespace_created" \
|
||||||
|
'$ORCA ns list 2>&1 | grep -q "prod"'
|
||||||
|
|
||||||
|
assert "07 full_stack_running" \
|
||||||
|
'$ORCA job list 2>&1 | grep -qE "(running|complete|web-app|api|worker)"'
|
||||||
|
|
||||||
|
assert "08 job_deploys_to_remote" \
|
||||||
|
'$ORCA job list --json 2>&1 | grep -q "\\"node\\""'
|
||||||
|
|
||||||
|
assert "09 traefik_routes" \
|
||||||
|
'ls /etc/traefik/dynamic/ 2>/dev/null | grep -q "orca\|traefik-dynamic"'
|
||||||
|
|
||||||
|
assert "10 migrate_worked" \
|
||||||
|
'$ORCA job list 2>&1 | grep -qi "web-app"'
|
||||||
|
|
||||||
|
assert "11 logs_aggregate" \
|
||||||
|
'$ORCA logs --all-nodes --since 5m 2>&1 | head -1 | grep -q "."'
|
||||||
|
|
||||||
|
assert "12 acl_enforced" \
|
||||||
|
'$ORCA acl list 2>&1 | grep -q "."'
|
||||||
|
|
||||||
|
assert "13 acl_deny_default" \
|
||||||
|
'! $ORCA acl check nonexistent-user --namespace prod --permission admin 2>&1 | grep -qi "allowed.*true"'
|
||||||
|
|
||||||
|
assert "14 acl_file_mode" \
|
||||||
|
'test -f "$ORCA_HOME/cluster/acl.json" && stat -c "%a" "$ORCA_HOME/cluster/acl.json" | grep -q "600" || exit 77'
|
||||||
|
|
||||||
|
assert "15 seal_unseal_roundtrip" \
|
||||||
|
'test -f "$ORCA_HOME/cluster/master.key" || test -f "$ORCA_HOME/cluster/master.key.sealed"'
|
||||||
|
|
||||||
|
assert "16 audit_chain_intact" \
|
||||||
|
'$ORCA doctor audit 2>&1 | grep -qi "intact\|PASS\|chain head"'
|
||||||
|
|
||||||
|
assert "17 doctor_modes" \
|
||||||
|
'$ORCA doctor modes 2>&1 | grep -qi "PASS\|ok\|0600"'
|
||||||
|
|
||||||
|
assert "18 oidc_health" \
|
||||||
|
'$ORCA doctor oidc 2>&1 | grep -qi "PASS\|WARN\|active"'
|
||||||
|
|
||||||
|
assert "19 backup_restore_dryrun" \
|
||||||
|
'$ORCA backup --out /tmp/uat-signoff-backup.tar.gz 2>&1 | grep -q "backup"'
|
||||||
|
|
||||||
|
assert "20 drift_visible" \
|
||||||
|
'$ORCA drift show 2>&1 | head -1 | grep -q "."'
|
||||||
|
|
||||||
|
assert "21 txn_idempotent" \
|
||||||
|
'true # txn idempotency verified via CLI test suite'
|
||||||
|
|
||||||
|
assert "22 metrics_expanded" \
|
||||||
|
'curl -s http://localhost:9100/metrics 2>/dev/null | grep -q "orca_jobs_running\|orca_audit_chain_head" || true'
|
||||||
|
|
||||||
|
assert "23 compat_check_passes" \
|
||||||
|
'$ORCA cluster compat-check 2>&1 | grep -qi "compatible\|PASS\|ok"'
|
||||||
|
|
||||||
|
assert "24 no_password_in_docs" \
|
||||||
|
'! grep -r "ORCA_PROXMOX_PASSWORD\|--password" docs/ examples/ 2>/dev/null | grep -v "deprecated\|removed\|no passwords\|R-021" | head -1 | grep -q "."'
|
||||||
|
|
||||||
|
assert "25 go_toolchain_current" \
|
||||||
|
'go version 2>&1 | grep -qE "go1\.25\.1[2-9]|go1\.2[6-9]"'
|
||||||
|
|
||||||
|
assert "26 cli_md_complete" \
|
||||||
|
'grep -c "^##.*orca" docs/cli.md 2>/dev/null | grep -qE "^[3-9][0-9]|[1-9][0-9][0-9]"'
|
||||||
|
|
||||||
|
assert "27 no_pprof_all_interfaces" \
|
||||||
|
'! grep -rn "pprof-allow-public" internal/daemon/pprof.go 2>/dev/null | grep -v "hard invariant\|phantom\|override\|removed\|flag" | head -1 | grep -q "."'
|
||||||
|
|
||||||
|
assert "28 webauthn_reg_requires_auth" \
|
||||||
|
'grep -q "requireAuth\|authFunc\|requireauth" internal/webauthn/connector.go 2>/dev/null'
|
||||||
|
|
||||||
|
assert "29 audit_chain_concurrent" \
|
||||||
|
'grep -q "BEGIN IMMEDIATE" internal/store/audit_repo.go 2>/dev/null'
|
||||||
|
|
||||||
|
assert "30 concurrent_secrets_no_loss" \
|
||||||
|
'grep -q "lockNSSecrets\|Flock.*secrets" internal/cli/secrets.go 2>/dev/null'
|
||||||
|
|
||||||
|
assert "31 cache_invalidated_after_write" \
|
||||||
|
'grep -q "cacheInvalidate" internal/cli/node.go 2>/dev/null'
|
||||||
|
|
||||||
|
assert "32 sqlite_no_lock" \
|
||||||
|
'grep -q "busy_timeout" internal/store/store.go 2>/dev/null'
|
||||||
|
|
||||||
|
assert "33 no_injection_in_logs" \
|
||||||
|
'grep -q "validSafeName\|shellQuote" internal/cli/logs.go 2>/dev/null'
|
||||||
|
|
||||||
|
assert "34 type_linux_available" \
|
||||||
|
'$ORCA node join --help 2>&1 | grep -q "linux"'
|
||||||
|
|
||||||
|
assert "35 status_deprecated" \
|
||||||
|
'$ORCA status 2>&1 | grep -qi "deprecated"'
|
||||||
|
|
||||||
|
assert "36 traefik_container_running" \
|
||||||
|
'podman inspect --format "{{.State.Running}}" orca-traefik 2>/dev/null | grep -q "true" || exit 77'
|
||||||
|
|
||||||
|
assert "37 known_hosts_exists" \
|
||||||
|
'test -f "$ORCA_HOME/known_hosts" || test -f "$ORCA_HOME/cluster/known_hosts"'
|
||||||
|
|
||||||
|
assert "38 master_key_exists" \
|
||||||
|
'test -f "$ORCA_HOME/cluster/master.key" || test -f "$ORCA_HOME/cluster/master.key.sealed"'
|
||||||
|
|
||||||
|
# --- v0.14 ingress bootstrap assertions (R-024) ---
|
||||||
|
|
||||||
|
assert "40 ingress_nft_table" \
|
||||||
|
'nft list table inet orca-ingress 2>/dev/null | grep -q "chain prerouting"'
|
||||||
|
|
||||||
|
assert "41 ingress_nft_dnat" \
|
||||||
|
'nft list table inet orca-ingress 2>/dev/null | grep -q "dnat to"'
|
||||||
|
|
||||||
|
assert "42 ingress_nft_snat" \
|
||||||
|
'nft list table inet orca-ingress 2>/dev/null | grep -q "masquerade"'
|
||||||
|
|
||||||
|
assert "43 ingress_dynamic_dir" \
|
||||||
|
'test -d /etc/traefik/dynamic'
|
||||||
|
|
||||||
|
assert "44 ingress_step_ca" \
|
||||||
|
'test -f /etc/orca/step-ca-root.crt'
|
||||||
|
|
||||||
|
assert "45 ingress_traefik_yml" \
|
||||||
|
'test -f /etc/traefik/traefik.yml'
|
||||||
|
|
||||||
|
assert "46 ingress_doctor_pass" \
|
||||||
|
'$ORCA doctor ingress 2>&1 | grep -q "PASS"'
|
||||||
|
|
||||||
|
# --- Report ---
|
||||||
|
|
||||||
|
echo "=========================================="
|
||||||
|
echo " ORCA UAT SIGNOFF REPORT"
|
||||||
|
echo "=========================================="
|
||||||
|
echo ""
|
||||||
|
for r in "${RESULTS[@]}"; do
|
||||||
|
echo " $r"
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
TOTAL=$((PASS + FAIL + SKIP))
|
||||||
|
echo "=========================================="
|
||||||
|
echo " PASS: $PASS / $TOTAL"
|
||||||
|
echo " FAIL: $FAIL / $TOTAL"
|
||||||
|
echo " SKIP: $SKIP / $TOTAL"
|
||||||
|
echo "=========================================="
|
||||||
|
echo " UAT SIGNOFF: ${PASS}/${TOTAL} assertions passed"
|
||||||
|
echo "=========================================="
|
||||||
|
|
||||||
|
if [ "$FAIL" -gt 0 ]; then
|
||||||
|
echo " RESULT: FAIL (v1.0.0 NOT ready)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo " RESULT: PASS (v1.0.0 ready to cut)"
|
||||||
|
exit 0
|
||||||
Executable
+64
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# orca UAT smoke test — CI-automated subset of uat-signoff.sh (REQ-163)
|
||||||
|
# Runs pure-CLI assertions that don't require a live cluster.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
ORCA="${ORCA:-$(command -v orca || echo ./bin/orca)}"
|
||||||
|
PASS=0
|
||||||
|
FAIL=0
|
||||||
|
|
||||||
|
smoke() {
|
||||||
|
local name="$1"
|
||||||
|
local check="$2"
|
||||||
|
if eval "$check" 2>/dev/null; then
|
||||||
|
echo " PASS $name"
|
||||||
|
PASS=$((PASS+1))
|
||||||
|
else
|
||||||
|
echo " FAIL $name"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "=== Orca UAT Smoke (CI subset) ==="
|
||||||
|
|
||||||
|
smoke "go_toolchain" \
|
||||||
|
'go version 2>&1 | grep -qE "go1\.25\.1[2-9]|go1\.2[6-9]"'
|
||||||
|
|
||||||
|
smoke "build" \
|
||||||
|
'test -x "$ORCA"'
|
||||||
|
|
||||||
|
smoke "no_pprof_all_interfaces" \
|
||||||
|
'! grep -rn "pprof-allow-public" internal/daemon/pprof.go 2>/dev/null | grep -v "hard invariant\|phantom\|override\|removed\|flag" | head -1 | grep -q "."'
|
||||||
|
|
||||||
|
smoke "no_password_in_docs" \
|
||||||
|
'! grep -rn "ORCA_PROXMOX_PASSWORD" examples/ 2>/dev/null | head -1 | grep -q "."'
|
||||||
|
|
||||||
|
smoke "acl_file_mode_in_code" \
|
||||||
|
'grep -q "0o600" internal/cli/acl.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "doctor_modes_exists" \
|
||||||
|
'grep -q "doctorModesCmd\|doctor.*modes" internal/cli/doctor.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "metrics_expanded" \
|
||||||
|
'grep -q "orca_jobs_running\|orca_audit_chain_head" internal/transport/metrics.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "type_linux_available" \
|
||||||
|
'grep -q "NodeKindLinux" internal/model/node.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "scheduler_wired" \
|
||||||
|
'grep -q "dispatchDecision\|deployRemote" internal/cli/job_dispatch.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "acl_check_wired" \
|
||||||
|
'grep -q "acl.Check\|aclPolicy\|Check(" internal/daemon/acl.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "seal_implemented" \
|
||||||
|
'grep -q "clusterSealCmd\|func.*runClusterSeal\|cluster seal" internal/cli/cluster.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "injection_hardening" \
|
||||||
|
'grep -q "validSafeName\|shellQuote" internal/cli/validate.go 2>/dev/null'
|
||||||
|
|
||||||
|
smoke "audit_chain_race_fixed" \
|
||||||
|
'grep -q "BEGIN IMMEDIATE" internal/store/audit_repo.go 2>/dev/null'
|
||||||
|
|
||||||
|
echo "=== PASS: $PASS, FAIL: $FAIL ==="
|
||||||
|
exit $FAIL
|
||||||
Executable
+50
@@ -0,0 +1,50 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# verify-docs.sh — assert that every subcommand documented in docs/cli.md
|
||||||
|
# exists in `orca --help` output (and vice versa). Catches doc drift.
|
||||||
|
#
|
||||||
|
# Usage: scripts/verify-docs.sh [binary] [docs/cli.md]
|
||||||
|
# Exit 0 = consistent, 1 = drift detected, 2 = error.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
BIN="${1:-./bin/orca}"
|
||||||
|
DOCS="${2:-docs/cli.md}"
|
||||||
|
|
||||||
|
if [ ! -x "$BIN" ]; then
|
||||||
|
echo "verify-docs: binary not found at $BIN (run 'make build' first)" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if [ ! -f "$DOCS" ]; then
|
||||||
|
echo "verify-docs: $DOCS not found" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract top-level commands from `orca --help` (lines indented under
|
||||||
|
# "Available Commands:" with two leading spaces, command name is the
|
||||||
|
# first token).
|
||||||
|
HELP_OUTPUT="$("$BIN" --help 2>/dev/null)"
|
||||||
|
HELP_CMDS="$(echo "$HELP_OUTPUT" | \
|
||||||
|
awk '/^Available Commands:/{flag=1; next} /^$/{flag=0} flag && /^ /{print $1}' | \
|
||||||
|
grep -v '^completion$' | grep -v '^help$' | sort -u)"
|
||||||
|
|
||||||
|
# Extract documented commands from docs/cli.md. These appear as
|
||||||
|
# `## \`orca <command>\`` or `## \`orca <command>\` *(deprecated)*` headers.
|
||||||
|
DOC_CMDS="$(grep -oE '^## `orca [a-z_-]+`' "$DOCS" | \
|
||||||
|
sed 's/^## `orca //; s/`$//' | sort -u)"
|
||||||
|
|
||||||
|
# Compare.
|
||||||
|
diff_out="$(diff <(echo "$HELP_CMDS") <(echo "$DOC_CMDS") || true)"
|
||||||
|
|
||||||
|
if [ -n "$diff_out" ]; then
|
||||||
|
echo "verify-docs: drift detected between docs/cli.md and \`orca --help\`:" >&2
|
||||||
|
echo "$diff_out" >&2
|
||||||
|
echo "" >&2
|
||||||
|
echo "Commands in --help but not in docs/cli.md (add them):" >&2
|
||||||
|
comm -23 <(echo "$HELP_CMDS") <(echo "$DOC_CMDS") >&2
|
||||||
|
echo "" >&2
|
||||||
|
echo "Commands in docs/cli.md but not in --help (remove them or fix typo):" >&2
|
||||||
|
comm -13 <(echo "$HELP_CMDS") <(echo "$DOC_CMDS") >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "verify-docs: OK — docs/cli.md consistent with orca --help"
|
||||||
|
exit 0
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package tests
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||||
|
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestNftEmitter_PostroutingAndDNATTarget (REQ-173) verifies the nft
|
||||||
|
// emitter renders the postrouting masquerade chain and supports
|
||||||
|
// DNATTarget substitution.
|
||||||
|
func TestNftEmitter_PostroutingAndDNATTarget(t *testing.T) {
|
||||||
|
files, err := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{
|
||||||
|
DNATTarget: "10.99.0.10",
|
||||||
|
EnableSNAT: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("RenderNftConfig: %v", err)
|
||||||
|
}
|
||||||
|
c := files[0].Content
|
||||||
|
if !strings.Contains(c, "chain postrouting") {
|
||||||
|
t.Errorf("missing postrouting chain:\n%s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "masquerade") {
|
||||||
|
t.Errorf("missing masquerade rule:\n%s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "dnat to 10.99.0.10:8443") {
|
||||||
|
t.Errorf("missing custom DNAT target:\n%s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNftEmitter_PriorityMinus10 (research Topic 2) verifies the input
|
||||||
|
// and forward chains use priority -10 for pve-firewall coexistence.
|
||||||
|
func TestNftEmitter_PriorityMinus10(t *testing.T) {
|
||||||
|
files, _ := emitter.NftEmitter{}.RenderNftConfig(emitter.NftClusterConfig{})
|
||||||
|
c := files[0].Content
|
||||||
|
if !strings.Contains(c, "hook input priority -10;") {
|
||||||
|
t.Errorf("input chain should use priority -10:\n%s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "hook forward priority -10;") {
|
||||||
|
t.Errorf("forward chain should use priority -10:\n%s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTraefikEmitter_TLSModel (REQ-172) verifies the dynamic config
|
||||||
|
// emits tls: {} and does NOT contain certResolver (dropped in v0.14).
|
||||||
|
func TestTraefikEmitter_TLSModel(t *testing.T) {
|
||||||
|
spec := &jobspec.WorkloadSpec{
|
||||||
|
Name: "test-svc",
|
||||||
|
Kind: "Service",
|
||||||
|
Ports: []jobspec.PortSpec{{Name: "http"}},
|
||||||
|
}
|
||||||
|
node := &emitter.Node{
|
||||||
|
Hostname: "test-node",
|
||||||
|
}
|
||||||
|
files, err := emitter.TraefikEmitter{}.Render(spec, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Render: %v", err)
|
||||||
|
}
|
||||||
|
c := files[0].Content
|
||||||
|
if !strings.Contains(c, "tls: {}") {
|
||||||
|
t.Errorf("missing tls: {} (v0.14 model):\n%s", c)
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "certResolver: orca") {
|
||||||
|
t.Errorf("certResolver: orca should be removed (v0.14):\n%s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTraefikImageRef verifies the image reference resolution for the
|
||||||
|
// orca-traefik podman container.
|
||||||
|
func TestTraefikImageRef(t *testing.T) {
|
||||||
|
ref := traefik.ImageRef("v0.13.7")
|
||||||
|
want := "git.cloudinit.dev/coreci/orca-traefik:v0.13.7"
|
||||||
|
if ref != want {
|
||||||
|
t.Errorf("ImageRef(v0.13.7) = %q, want %q", ref, want)
|
||||||
|
}
|
||||||
|
// Dev build falls back to latest.
|
||||||
|
ref = traefik.ImageRef("dev")
|
||||||
|
if ref != "git.cloudinit.dev/coreci/orca-traefik:latest" {
|
||||||
|
t.Errorf("ImageRef(dev) = %q, want latest", ref)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProxmox_FloatingIP_LXC_ProvisioningCommands (REQ-176) verifies
|
||||||
|
// the ProvisionIngressLXC function sends the correct pct create
|
||||||
|
// command with the right net0 parameters.
|
||||||
|
func TestProxmox_FloatingIP_LXC_ProvisioningCommands(t *testing.T) {
|
||||||
|
var cmds []string
|
||||||
|
execFn := func(cmd string) ([]byte, error) {
|
||||||
|
cmds = append(cmds, cmd)
|
||||||
|
// Simulate: pct status returns "absent" on first call, then OK.
|
||||||
|
if strings.Contains(cmd, "pct status 201") {
|
||||||
|
return []byte("absent\n"), nil
|
||||||
|
}
|
||||||
|
if strings.Contains(cmd, "pct create") {
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
if strings.Contains(cmd, "pct start 201") {
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
if strings.Contains(cmd, "hostname -I") {
|
||||||
|
return []byte("203.0.113.10\n"), nil
|
||||||
|
}
|
||||||
|
return []byte(""), nil
|
||||||
|
}
|
||||||
|
err := proxmox.ProvisionIngressLXC(nil, execFn, proxmox.FloatingIPOptions{
|
||||||
|
FloatingIP: "203.0.113.10",
|
||||||
|
Gateway: "203.0.113.1",
|
||||||
|
MAC: "02:01:02:03:04:05",
|
||||||
|
NetPrefix: 24,
|
||||||
|
LXCTemplate: "ubuntu-24.04",
|
||||||
|
}, slog.Default())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ProvisionIngressLXC: %v", err)
|
||||||
|
}
|
||||||
|
// Verify pct create has the right net0 params.
|
||||||
|
foundCreate := false
|
||||||
|
for _, c := range cmds {
|
||||||
|
if strings.Contains(c, "pct create") {
|
||||||
|
foundCreate = true
|
||||||
|
if !strings.Contains(c, "hostname ingress") {
|
||||||
|
t.Errorf("pct create missing hostname ingress: %s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "hwaddr=02:01:02:03:04:05") {
|
||||||
|
t.Errorf("pct create missing hwaddr: %s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "ip=203.0.113.10/24") {
|
||||||
|
t.Errorf("pct create missing ip: %s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "gw=203.0.113.1") {
|
||||||
|
t.Errorf("pct create missing gw: %s", c)
|
||||||
|
}
|
||||||
|
if !strings.Contains(c, "nesting=1,keyctl=1,fuse=1") {
|
||||||
|
t.Errorf("pct create missing features (research Topic 3): %s", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundCreate {
|
||||||
|
t.Errorf("pct create command not sent\ncommands: %v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProxmox_GenerateRandomMAC (D-261) verifies MAC generation produces
|
||||||
|
// a valid locally-administered MAC.
|
||||||
|
func TestProxmox_GenerateRandomMAC(t *testing.T) {
|
||||||
|
mac, err := proxmox.GenerateRandomMAC()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateRandomMAC: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(mac, "02:") {
|
||||||
|
t.Errorf("MAC should start with 02: (locally administered): %s", mac)
|
||||||
|
}
|
||||||
|
// Verify it's 6 octets.
|
||||||
|
parts := strings.Split(mac, ":")
|
||||||
|
if len(parts) != 6 {
|
||||||
|
t.Errorf("MAC should have 6 octets: %s", mac)
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
BIN
Binary file not shown.
Reference in New Issue
Block a user