Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Jon Chery
|
790109ea24 | ||
|
Jon Chery
|
64cbbd543e | ||
|
Jon Chery
|
16440a89f2 | ||
|
Jon Chery
|
a6ceb13491 | ||
|
Jon Chery
|
c0ba30824e | ||
|
Jon Chery
|
52e17aefbf | ||
|
Jon Chery
|
b6dd86fdf3 | ||
|
Jon Chery
|
ed91d68fbf | ||
|
Jon Chery
|
531b36924c |
@@ -1,21 +1,26 @@
|
||||
{
|
||||
"phase": 1,
|
||||
"phase": 13,
|
||||
"stage": "complete",
|
||||
"milestone": "v0.13",
|
||||
"milestone_slug": "production-hardening-2",
|
||||
"phase_role": "execution",
|
||||
"phase_role": "final",
|
||||
"attempts": 0,
|
||||
"updated_at": "2026-08-07T19:05:00Z",
|
||||
"milestone_complete": false,
|
||||
"updated_at": "2026-08-10T14:30:00Z",
|
||||
"milestone_complete": true,
|
||||
"previous_milestone": "v0.12",
|
||||
"phase_count": 14,
|
||||
"phases_shipped": ["P0", "P1"],
|
||||
"tags_shipped": ["v0.12.0", "v0.12.1"],
|
||||
"phases_shipped": ["P0","P1","P2","P3","P4","P5","P6","P7","P8","P9","P10","P11","P12","P13"],
|
||||
"tags_shipped": ["v0.12.0","v0.12.1","v0.12.2","v0.12.3","v0.12.4","v0.12.5","v0.12.6","v0.12.7","v0.12.8","v0.12.9","v0.12.10","v0.12.11","v0.12.12"],
|
||||
"requirements": {
|
||||
"covered": [149],
|
||||
"covered": [149,150,151,152,153,154,155,156,157,158,159,160,161,162,163],
|
||||
"partial": []
|
||||
},
|
||||
"binding_conditions": ["C-39","C-40","C-41","C-42","C-43","C-44","C-45","C-46","C-47","C-48","C-49"],
|
||||
"load_bearing_rule": "R-022",
|
||||
"next_milestone": "v1.0"
|
||||
"next_milestone": "v1.0",
|
||||
"ship": {
|
||||
"tag": "v0.12.13",
|
||||
"merged_to_milestone": true,
|
||||
"milestone_release": "v0.13"
|
||||
}
|
||||
}
|
||||
|
||||
+47
-47
@@ -254,7 +254,7 @@ operator decision Q2=C.
|
||||
|
||||
## v0.12 Milestone Summary — Security Hardening (Zero-Trust Identity)
|
||||
|
||||
**Status**: in progress (Phase 0). 30 net-new requirements (REQ-119..REQ-148)
|
||||
**Status**: complete (shipped as v0.11.x tags; milestone release v0.11.28). 30 net-new requirements (REQ-119..REQ-148)
|
||||
derived from the v0.12 threat-model review (25 findings F1..F25) and the
|
||||
zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
||||
(P0 + P01..P27 + P28 final) and RESEARCH_v0.12.md for the full threat model.
|
||||
@@ -263,41 +263,41 @@ zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-119 | Command injection fix in `internal/runtime/podman.go` & `wasm.go`: shell-quote `cmdStr` via `shellQuote` in SSH exec interpolation (`podman.go:57`, `wasm.go:39`); add injection regression tests (bats + Go) covering `;`, `\|`, `$()`, backticks, newline injection (F3) | High | **v0.12 P01** | pending |
|
||||
| REQ-120 | Namespace path traversal fix: `validateNamespaceName` in `internal/ns/` rejects `..`, `/`, leading `-`, null bytes, control chars in `ns create`/`ns inherit`/`ns set-constraint`; add fuzz test (F4) | High | **v0.12 P02** | pending |
|
||||
| REQ-121 | Txn apply path allowlist: `apply.sh` python heredoc validates every `path` in `desired-state.json` against a prefix allowlist (`/etc/orca/`, `/etc/traefik/orca*`, `/etc/systemd/system/orca-*`, `/etc/nftables.d/orca*`, `/etc/syncthing/orca*`); rejects otherwise; HMAC-signed manifest unchanged (F5) | High | **v0.12 P03** | pending |
|
||||
| REQ-119 | Command injection fix in `internal/runtime/podman.go` & `wasm.go`: shell-quote `cmdStr` via `shellQuote` in SSH exec interpolation (`podman.go:57`, `wasm.go:39`); add injection regression tests (bats + Go) covering `;`, `\|`, `$()`, backticks, newline injection (F3) | High | **v0.12 P01** | complete |
|
||||
| REQ-120 | Namespace path traversal fix: `validateNamespaceName` in `internal/ns/` rejects `..`, `/`, leading `-`, null bytes, control chars in `ns create`/`ns inherit`/`ns set-constraint`; add fuzz test (F4) | High | **v0.12 P02** | complete |
|
||||
| REQ-121 | Txn apply path allowlist: `apply.sh` python heredoc validates every `path` in `desired-state.json` against a prefix allowlist (`/etc/orca/`, `/etc/traefik/orca*`, `/etc/systemd/system/orca-*`, `/etc/nftables.d/orca*`, `/etc/syncthing/orca*`); rejects otherwise; HMAC-signed manifest unchanged (F5) | High | **v0.12 P03** | complete |
|
||||
|
||||
### Wave B — Zero-trust identity
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-122 | ACL enforcement wiring: `acl.Check` invoked in daemon handlers (read/write/admin by route) and SSH-push applier (validates `ORCA_OIDC_TOKEN` env var against JWKS before applying any txn); deny-by-default enforced; actor recorded in audit (F1, foundational for REQ-145) | High | **v0.12 P06** | pending |
|
||||
| REQ-123 | Daemon auth hardening: mandatory mTLS (remove plaintext mode entirely); OIDC bearer accepted as second factor on human-facing endpoints; `MaxBytesReader` body limits; pprof loopback-only by default, refuse non-loopback without `--pprof-allow-public` confirmation (F6, F24) | High | **v0.12 P09** | pending |
|
||||
| REQ-124 | HTTP request body size limits: `http.MaxBytesReader` on all JSON-decoding handlers; `MaxHeaderBytes` set; rejects oversized bodies (F24) | Medium | **v0.12 P09** | pending |
|
||||
| REQ-125 | Audit log tamper-evidence: hash-chained entries (`prev_hash = sha256(prev_row \|\| payload)`), HMAC-SHA256 under master key on the chain head; `orca doctor audit` verifies the chain; append-only enforcement via SQLite trigger blocking UPDATE/DELETE; actor field carries OIDC `sub` or SPIFFE SVID (F2) | High | **v0.12 P10** | pending |
|
||||
| REQ-126 | SVID chain validation: `VerifySVID` validates the full cert chain against the CA pool, not just the URI SAN; reject certs signed by unknown CAs even with correct URI (F9) | High | **v0.12 P11** | pending |
|
||||
| REQ-127 | Backup symlink validation: `Restore` rejects `Linkname` that's absolute, contains `..`, or points outside `ORCA_HOME`; add regression test with crafted tarball (F7) | High | **v0.12 P12** | pending |
|
||||
| REQ-128 | step-ca /tmp hardening: `step ca certificate` writes to 0600 temp under `ClusterDir()/step-tmp/` (or `TMPDIR` override), not world-readable `/tmp`; cleanup in `defer` (F10) | High | **v0.12 P13** | pending |
|
||||
| REQ-129 | Master key rotation: `orca secrets rotate-master` re-encrypts all namespace secrets under a new master key; new master key re-sealed to OIDC as part of the same operation; `--dry-run` + atomic + automatic rollback to old sealed key on any ns failure; no passphrase (R-021) (F12) | High | **v0.12 P14** | pending |
|
||||
| REQ-130 | File-mode audit expansion: `EnforceFileModes` extended to SSH key, master key (sealed blob), server cert/key, known_hosts; `orca doctor modes` checks all; startup refuses to run on violation (F13) | Medium | **v0.12 P15** | pending |
|
||||
| REQ-131 | aggregate.sh JSON injection fix + drift-gate parse fix: replace `printf` interpolation with `jq`-based JSON construction (or Go-side aggregator emitting JSON); fix `orca-pull.sh` R-020 parsing to use `jq` instead of grep (F11, F18) | High | **v0.12 P16** | pending |
|
||||
| REQ-132 | install.sh checksum+GPG verification: release.sh publishes `SHA256SUMS` + `SHA256SUMS.asc` (GPG-signed) alongside tarball; install.sh verifies before `tar -xzf`; fail closed on mismatch (F14) | High | **v0.12 P17** | pending |
|
||||
| REQ-133 | nftables ruleset hardening: add conntrack bounds (`ct state established,related accept`), input default-deny on orca chain, drop invalid packets; `orca doctor nft` audits live ruleset against emitted one (F21) | Medium | **v0.12 P18** | pending |
|
||||
| REQ-134 | sudoers hardening: add NOEXEC to `apt-get`/`dpkg` (or remove if unused); `orca doctor proxmox` audits sudoers file against expected allowlist (F22) | Medium | **v0.12 P19** | pending |
|
||||
| REQ-135 | System user consistency: Proxmox bootstrap creates `nologin` system user (`-r -s /usr/sbin/nologin`), matching peer-setup; `orca doctor` flags inconsistency on existing peers; `orca upgrade` migrates (F23) | Medium | **v0.12 P20** | pending |
|
||||
| REQ-136 | SQLite file-mode + at-rest encryption: `store.Open` sets DB file mode 0600; optional `--encrypt-db` (CGO-free fallback per C-31: file-mode 0600 + documented threat if SQLCipher needs CGO); no CGO (F8) | High | **v0.12 P21** | pending |
|
||||
| REQ-137 | Migration safety: `copyFile` -> atomic temp+rename; `migrateDBSchema` runs in transaction with `foreign_keys(ON)`; pre-migration backup step (uses `internal/backup`); document manual rollback; v0.11->v0.12 identity migration: `orca upgrade` refuses clusters using `--password`/bare-tokens without `--accept-identity-migration` (F19, C-34) | High | **v0.12 P22** | pending |
|
||||
| REQ-138 | Legacy CA/mTLS/daemon + step-ca password-provisioner deletion: remove `internal/security/ca.go` legacy CA, `internal/transport/mtls.go` deprecated path, daemon plaintext mode; migrate `orca init`/`orca cert *` to step-ca exclusively; `certpaths` (v0.8 layout) removed; delete step-ca `--password-file` provisioner (replaced by OIDC provisioner); **gate: P06/P08/P09/P11 all shipped** (F16) | High | **v0.12 P23** | pending |
|
||||
| REQ-139 | known_hosts tightening + transport hardening: `Flock` tightens pre-existing looser perms to 0600; `classifyDialErr` switched from substring to typed errors; add SSH-exec rate limiting (token bucket per peer) (F15, F25) | Medium | **v0.12 P24** | pending |
|
||||
| REQ-140 | Drift event authentication: drift events signed with per-peer HMAC key (derived from master key); aggregator rejects unsigned/forged events; `orca-drift-notify.sh` reads key from 0600 file owned by `orca` (F18) | Medium | **v0.12 P25** | pending |
|
||||
| REQ-141 | Security integration test suite: hermetic harness exercising injection, traversal, symlink, drift-forgery, audit-tamper, daemon-auth-negative, OIDC mock-IdP flow, ACL-with-OIDC-claims negative tests, unseal/seal, WebAuthn virtual-authenticator ceremony, password-removal regression (assert `--password` is rejected); gates in `.coreci.yml` `validate` (C-33) | High | **v0.12 P26** | pending |
|
||||
| REQ-142 | Zero-trust + OIDC + WebAuthn + threat-model docs: `docs/threat-model.md` (STRIDE + zero-trust model + OIDC data-flow), `docs/oidc.md` (configure your IdP, Dex offline quickstart, claim-to-namespace mapping), `docs/webauthn.md` (passkey registration, RP ID, secure context), `docs/security-runbook.md` (unseal/seal, master key rotation, incident response, sudoers audit, nft audit); README security section names "no orca credentials" as an invariant | Medium | **v0.12 P27** | pending |
|
||||
| REQ-143 | Final review + ship + audit: multi-persona review across all phases, `ciagent-audit` reconstruction test, milestone merge to main, tag `v0.11.29` (= v0.12 milestone release per feature-milestone rule) | High | **v0.12 P28** | pending |
|
||||
| REQ-144 | OIDC client + bundled Dex: `orca auth login`/`logout`/`status`/`init-idp`; OIDC config block (`oidc.issuer`, `client_id`, `client_secret`, `scopes`); bundled Dex systemd unit + Traefik route on the lead; BYO external IdP override via `oidc.issuer` repoint; JWKS caching + refresh; token storage at `~/.orca/credentials.json` (0600); `--oidc` flag on commands requiring identity; browser auth-code + PKCE + local loopback redirect; headless device-code fallback (D-238..D-247) | High | **v0.12 P04** | pending |
|
||||
| REQ-145 | ACL rewrite to OIDC claims: remove `KindToken` entirely; `KindSpiffe` stays for machine identity; new `KindOidc` maps `sub`+`groups` -> namespace permissions; `acl.Check` takes OIDC claims struct; deny-by-default enforced in daemon + SSH-push applier; `acl.json` mode tightened to 0600 (F1) | High | **v0.12 P06** | pending |
|
||||
| REQ-146 | Remove all password/token paths (breaking): delete `--password`/`$ORCA_PROXMOX_PASSWORD` from Proxmox join (replace with pre-staged-key-only or `step ssh` OIDC cert exchange); delete step-ca `--password-file` provisioner (migrate to OIDC provisioner); delete any bare-token CLI paths; documented in migration guide (R-021, C-34) | High | **v0.12 P07** | pending |
|
||||
| REQ-147 | Master key seal-to-OIDC + Shamir recovery: master key encrypted with key derived from OIDC token exchange at unseal; `orca cluster unseal`/`seal`; sealed blob at `ClusterDir()/master.key.sealed` (0600); raw key never on disk; Shamir 3-of-5 shards printed at seal time; recovery via `--recovery` + 3 shards; mTLS-only offline path derives seal key from cluster CA (D-241, C-35) | High | **v0.12 P08** | pending |
|
||||
| REQ-148 | WebAuthn connector for Dex (passkeys): `orca-webauthn-connector` (~300 LoC Go, `go-webauthn`); register/login ceremonies at `/orca/webauthn/{register,login}` behind Traefik; `orca auth register` browser flow; passkey storage SQLite `ClusterDir()/webauthn-credentials.db` (0600, public keys only); RP ID = cluster Traefik domain; secure context via step-ca cert; headless device-code fallback; virtual-authenticator integration tests (D-240, D-243, D-244, C-38) | High | **v0.12 P05** | pending |
|
||||
| REQ-122 | ACL enforcement wiring: `acl.Check` invoked in daemon handlers (read/write/admin by route) and SSH-push applier (validates `ORCA_OIDC_TOKEN` env var against JWKS before applying any txn); deny-by-default enforced; actor recorded in audit (F1, foundational for REQ-145) | High | **v0.12 P06** | complete |
|
||||
| REQ-123 | Daemon auth hardening: mandatory mTLS (remove plaintext mode entirely); OIDC bearer accepted as second factor on human-facing endpoints; `MaxBytesReader` body limits; pprof loopback-only by default, refuse non-loopback without `--pprof-allow-public` confirmation (F6, F24) | High | **v0.12 P09** | complete |
|
||||
| REQ-124 | HTTP request body size limits: `http.MaxBytesReader` on all JSON-decoding handlers; `MaxHeaderBytes` set; rejects oversized bodies (F24) | Medium | **v0.12 P09** | complete |
|
||||
| REQ-125 | Audit log tamper-evidence: hash-chained entries (`prev_hash = sha256(prev_row \|\| payload)`), HMAC-SHA256 under master key on the chain head; `orca doctor audit` verifies the chain; append-only enforcement via SQLite trigger blocking UPDATE/DELETE; actor field carries OIDC `sub` or SPIFFE SVID (F2) | High | **v0.12 P10** | complete |
|
||||
| REQ-126 | SVID chain validation: `VerifySVID` validates the full cert chain against the CA pool, not just the URI SAN; reject certs signed by unknown CAs even with correct URI (F9) | High | **v0.12 P11** | complete |
|
||||
| REQ-127 | Backup symlink validation: `Restore` rejects `Linkname` that's absolute, contains `..`, or points outside `ORCA_HOME`; add regression test with crafted tarball (F7) | High | **v0.12 P12** | complete |
|
||||
| REQ-128 | step-ca /tmp hardening: `step ca certificate` writes to 0600 temp under `ClusterDir()/step-tmp/` (or `TMPDIR` override), not world-readable `/tmp`; cleanup in `defer` (F10) | High | **v0.12 P13** | complete |
|
||||
| REQ-129 | Master key rotation: `orca secrets rotate-master` re-encrypts all namespace secrets under a new master key; new master key re-sealed to OIDC as part of the same operation; `--dry-run` + atomic + automatic rollback to old sealed key on any ns failure; no passphrase (R-021) (F12) | High | **v0.12 P14** | complete |
|
||||
| REQ-130 | File-mode audit expansion: `EnforceFileModes` extended to SSH key, master key (sealed blob), server cert/key, known_hosts; `orca doctor modes` checks all; startup refuses to run on violation (F13) | Medium | **v0.12 P15** | complete |
|
||||
| REQ-131 | aggregate.sh JSON injection fix + drift-gate parse fix: replace `printf` interpolation with `jq`-based JSON construction (or Go-side aggregator emitting JSON); fix `orca-pull.sh` R-020 parsing to use `jq` instead of grep (F11, F18) | High | **v0.12 P16** | complete |
|
||||
| REQ-132 | install.sh checksum+GPG verification: release.sh publishes `SHA256SUMS` + `SHA256SUMS.asc` (GPG-signed) alongside tarball; install.sh verifies before `tar -xzf`; fail closed on mismatch (F14) | High | **v0.12 P17** | complete |
|
||||
| REQ-133 | nftables ruleset hardening: add conntrack bounds (`ct state established,related accept`), input default-deny on orca chain, drop invalid packets; `orca doctor nft` audits live ruleset against emitted one (F21) | Medium | **v0.12 P18** | complete |
|
||||
| REQ-134 | sudoers hardening: add NOEXEC to `apt-get`/`dpkg` (or remove if unused); `orca doctor proxmox` audits sudoers file against expected allowlist (F22) | Medium | **v0.12 P19** | complete |
|
||||
| REQ-135 | System user consistency: Proxmox bootstrap creates `nologin` system user (`-r -s /usr/sbin/nologin`), matching peer-setup; `orca doctor` flags inconsistency on existing peers; `orca upgrade` migrates (F23) | Medium | **v0.12 P20** | complete |
|
||||
| REQ-136 | SQLite file-mode + at-rest encryption: `store.Open` sets DB file mode 0600; optional `--encrypt-db` (CGO-free fallback per C-31: file-mode 0600 + documented threat if SQLCipher needs CGO); no CGO (F8) | High | **v0.12 P21** | complete |
|
||||
| REQ-137 | Migration safety: `copyFile` -> atomic temp+rename; `migrateDBSchema` runs in transaction with `foreign_keys(ON)`; pre-migration backup step (uses `internal/backup`); document manual rollback; v0.11->v0.12 identity migration: `orca upgrade` refuses clusters using `--password`/bare-tokens without `--accept-identity-migration` (F19, C-34) | High | **v0.12 P22** | complete |
|
||||
| REQ-138 | Legacy CA/mTLS/daemon + step-ca password-provisioner deletion: remove `internal/security/ca.go` legacy CA, `internal/transport/mtls.go` deprecated path, daemon plaintext mode; migrate `orca init`/`orca cert *` to step-ca exclusively; `certpaths` (v0.8 layout) removed; delete step-ca `--password-file` provisioner (replaced by OIDC provisioner); **gate: P06/P08/P09/P11 all shipped** (F16) | High | **v0.12 P23** | complete |
|
||||
| REQ-139 | known_hosts tightening + transport hardening: `Flock` tightens pre-existing looser perms to 0600; `classifyDialErr` switched from substring to typed errors; add SSH-exec rate limiting (token bucket per peer) (F15, F25) | Medium | **v0.12 P24** | complete |
|
||||
| REQ-140 | Drift event authentication: drift events signed with per-peer HMAC key (derived from master key); aggregator rejects unsigned/forged events; `orca-drift-notify.sh` reads key from 0600 file owned by `orca` (F18) | Medium | **v0.12 P25** | complete |
|
||||
| REQ-141 | Security integration test suite: hermetic harness exercising injection, traversal, symlink, drift-forgery, audit-tamper, daemon-auth-negative, OIDC mock-IdP flow, ACL-with-OIDC-claims negative tests, unseal/seal, WebAuthn virtual-authenticator ceremony, password-removal regression (assert `--password` is rejected); gates in `.coreci.yml` `validate` (C-33) | High | **v0.12 P26** | complete |
|
||||
| REQ-142 | Zero-trust + OIDC + WebAuthn + threat-model docs: `docs/threat-model.md` (STRIDE + zero-trust model + OIDC data-flow), `docs/oidc.md` (configure your IdP, Dex offline quickstart, claim-to-namespace mapping), `docs/webauthn.md` (passkey registration, RP ID, secure context), `docs/security-runbook.md` (unseal/seal, master key rotation, incident response, sudoers audit, nft audit); README security section names "no orca credentials" as an invariant | Medium | **v0.12 P27** | complete |
|
||||
| REQ-143 | Final review + ship + audit: multi-persona review across all phases, `ciagent-audit` reconstruction test, milestone merge to main, tag `v0.11.29` (= v0.12 milestone release per feature-milestone rule) | High | **v0.12 P28** | complete |
|
||||
| REQ-144 | OIDC client + bundled Dex: `orca auth login`/`logout`/`status`/`init-idp`; OIDC config block (`oidc.issuer`, `client_id`, `client_secret`, `scopes`); bundled Dex systemd unit + Traefik route on the lead; BYO external IdP override via `oidc.issuer` repoint; JWKS caching + refresh; token storage at `~/.orca/credentials.json` (0600); `--oidc` flag on commands requiring identity; browser auth-code + PKCE + local loopback redirect; headless device-code fallback (D-238..D-247) | High | **v0.12 P04** | complete |
|
||||
| REQ-145 | ACL rewrite to OIDC claims: remove `KindToken` entirely; `KindSpiffe` stays for machine identity; new `KindOidc` maps `sub`+`groups` -> namespace permissions; `acl.Check` takes OIDC claims struct; deny-by-default enforced in daemon + SSH-push applier; `acl.json` mode tightened to 0600 (F1) | High | **v0.12 P06** | complete |
|
||||
| REQ-146 | Remove all password/token paths (breaking): delete `--password`/`$ORCA_PROXMOX_PASSWORD` from Proxmox join (replace with pre-staged-key-only or `step ssh` OIDC cert exchange); delete step-ca `--password-file` provisioner (migrate to OIDC provisioner); delete any bare-token CLI paths; documented in migration guide (R-021, C-34) | High | **v0.12 P07** | complete |
|
||||
| REQ-147 | Master key seal-to-OIDC + Shamir recovery: master key encrypted with key derived from OIDC token exchange at unseal; `orca cluster unseal`/`seal`; sealed blob at `ClusterDir()/master.key.sealed` (0600); raw key never on disk; Shamir 3-of-5 shards printed at seal time; recovery via `--recovery` + 3 shards; mTLS-only offline path derives seal key from cluster CA (D-241, C-35) | High | **v0.12 P08** | complete |
|
||||
| REQ-148 | WebAuthn connector for Dex (passkeys): `orca-webauthn-connector` (~300 LoC Go, `go-webauthn`); register/login ceremonies at `/orca/webauthn/{register,login}` behind Traefik; `orca auth register` browser flow; passkey storage SQLite `ClusterDir()/webauthn-credentials.db` (0600, public keys only); RP ID = cluster Traefik domain; secure context via step-ca cert; headless device-code fallback; virtual-authenticator integration tests (D-240, D-243, D-244, C-38) | High | **v0.12 P05** | complete |
|
||||
|
||||
### Scope notes (v0.12)
|
||||
|
||||
@@ -311,7 +311,7 @@ zero-trust identity model (R-021). See ROADMAP.md for the 29-phase plan
|
||||
|
||||
## Milestone v0.13: Production Hardening Round 2 + UAT Plan
|
||||
|
||||
**Status**: in progress (2026-08-07). v0.12 (Security Hardening) is
|
||||
**Status**: complete (2026-08-10). v0.12 (Security Hardening) is
|
||||
COMPLETE; v0.13 is the final hardening round before the v1.0.0
|
||||
production-ready tag. v1.0.0 is gated on the UAT signoff script
|
||||
(`scripts/uat-signoff.sh`) delivered by this milestone.
|
||||
@@ -320,41 +320,41 @@ production-ready tag. v1.0.0 is gated on the UAT signoff script
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-149 | Go toolchain bump to 1.25.12+ (closes 24 stdlib vulns: archive/tar GO-2025-4014/GO-2026-4869, crypto/tls GO-2026-5856/GO-2025-4008, crypto/x509 GO-2026-5037/4947/4946/GO-2025-4175/4155/4013, net/http GO-2026-4918/GO-2025-4012, net/url GO-2026-4601/4341/GO-2025-4010, encoding/pem GO-2025-4009, os GO-2026-4602); `govulncheck -show verbose` triage of 6 imported third-party vulns; bump deps with reachable traces | High | **v0.13 P01** | pending |
|
||||
| REQ-150 | Input validation & injection hardening: (a) `orca logs --job` validate against `^[A-Za-z0-9_-]+$`, use `shellQuote` not `%q` (critical: backtick RCE via SSH fanout); (b) pprof `isLoopback(":6060")` treat empty host as non-loopback/bind-all, reject unless explicit public-allow flag wired; remove phantom `--pprof-allow-public` references, make loopback-only a hard invariant; (c) backup restore tar-slip fix: use `filepath.Rel(target, dest)` containment check instead of `HasPrefix(name, "..")`; (d) `orca txn rollback` validate txn ID against `^T-[0-9a-f]{16}$`; (e) `orca nft diff --against` validate txn ID before `filepath.Join`; (f) `drain stopAlloc` validate `allocID` against `^[A-Za-z0-9_-]+$` before `systemctl stop`; (g) `cluster_compat` `shellQuote(first)` for peer dir name; (h) `runtime/podman.go` use `shellQuote(image)` not `%q`; (i) nft `TrustedProbes` validate each entry with `net.ParseIP`/`net.ParseCIDR`; (j) sudoers: validate `--proxmox-user`/`--proxmox-role` against `^[a-z_][a-z0-9_-]{0,31}$`; write to fixed `/etc/sudoers.d/orca`; `shellQuote` all pveum/useradd; `validateSudoers` check the actual file written; (k) `nft country block add` validate `^[A-Z]{2}$` | Critical | **v0.13 P02** | pending |
|
||||
| REQ-149 | Go toolchain bump to 1.25.12+ (closes 24 stdlib vulns: archive/tar GO-2025-4014/GO-2026-4869, crypto/tls GO-2026-5856/GO-2025-4008, crypto/x509 GO-2026-5037/4947/4946/GO-2025-4175/4155/4013, net/http GO-2026-4918/GO-2025-4012, net/url GO-2026-4601/4341/GO-2025-4010, encoding/pem GO-2025-4009, os GO-2026-4602); `govulncheck -show verbose` triage of 6 imported third-party vulns; bump deps with reachable traces | High | **v0.13 P01** | complete |
|
||||
| REQ-150 | Input validation & injection hardening: (a) `orca logs --job` validate against `^[A-Za-z0-9_-]+$`, use `shellQuote` not `%q` (critical: backtick RCE via SSH fanout); (b) pprof `isLoopback(":6060")` treat empty host as non-loopback/bind-all, reject unless explicit public-allow flag wired; remove phantom `--pprof-allow-public` references, make loopback-only a hard invariant; (c) backup restore tar-slip fix: use `filepath.Rel(target, dest)` containment check instead of `HasPrefix(name, "..")`; (d) `orca txn rollback` validate txn ID against `^T-[0-9a-f]{16}$`; (e) `orca nft diff --against` validate txn ID before `filepath.Join`; (f) `drain stopAlloc` validate `allocID` against `^[A-Za-z0-9_-]+$` before `systemctl stop`; (g) `cluster_compat` `shellQuote(first)` for peer dir name; (h) `runtime/podman.go` use `shellQuote(image)` not `%q`; (i) nft `TrustedProbes` validate each entry with `net.ParseIP`/`net.ParseCIDR`; (j) sudoers: validate `--proxmox-user`/`--proxmox-role` against `^[a-z_][a-z0-9_-]{0,31}$`; write to fixed `/etc/sudoers.d/orca`; `shellQuote` all pveum/useradd; `validateSudoers` check the actual file written; (k) `nft country block add` validate `^[A-Z]{2}$` | Critical | **v0.13 P02** | complete |
|
||||
|
||||
### Wave B — Scheduler wiring & jobspec parser (architectural)
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-151 | Scheduler/deployment wiring: wire `internal/scheduler.Schedule()` into `orca job run` — replace local `exec.CommandContext` path with: evaluate constraints/capacity/affinity via scheduler → render systemd units via `internal/emitter` → SSH-push to target via `internal/sshpush`; `--target` overrides scheduler selection; capacity enforced (reject job if no node fits); CEL constraints evaluated; affinity weighted scoring; `systemd-analyze verify` on rendered unit before deploy; `job run` without `--target` uses scheduler bin-packing across registered nodes | Critical | **v0.13 P03** | pending |
|
||||
| REQ-152 | jobspec parser fixes: add `case "schedule":` and `case "timeout":` to top-level switch in `internal/jobspec/markdown.go` (currently silently dropped); fix DaemonSet — parser must not default Count to 1 for DaemonSet (validator rejects Count!=0); DaemonSet schedule block actually parsed and stored; `timeout:` on Jobs parsed and enforced (kill after duration); `restart:` policy translated to systemd `Restart=`/`StartLimitBurst` in emitter; add `job lint` warnings for advisory-only fields (cron, health, update, affinity) with honest "not enforced in this version" message | Critical | **v0.13 P03** | pending |
|
||||
| REQ-151 | Scheduler/deployment wiring: wire `internal/scheduler.Schedule()` into `orca job run` — replace local `exec.CommandContext` path with: evaluate constraints/capacity/affinity via scheduler → render systemd units via `internal/emitter` → SSH-push to target via `internal/sshpush`; `--target` overrides scheduler selection; capacity enforced (reject job if no node fits); CEL constraints evaluated; affinity weighted scoring; `systemd-analyze verify` on rendered unit before deploy; `job run` without `--target` uses scheduler bin-packing across registered nodes | Critical | **v0.13 P03** | complete |
|
||||
| REQ-152 | jobspec parser fixes: add `case "schedule":` and `case "timeout":` to top-level switch in `internal/jobspec/markdown.go` (currently silently dropped); fix DaemonSet — parser must not default Count to 1 for DaemonSet (validator rejects Count!=0); DaemonSet schedule block actually parsed and stored; `timeout:` on Jobs parsed and enforced (kill after duration); `restart:` policy translated to systemd `Restart=`/`StartLimitBurst` in emitter; add `job lint` warnings for advisory-only fields (cron, health, update, affinity) with honest "not enforced in this version" message | Critical | **v0.13 P03** | complete |
|
||||
|
||||
### Wave C — Zero-trust enforcement wiring
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-153 | ACL enforcement + WebAuthn registration auth: (a) wire `acl.Check` into all 5 daemon handlers (`dispatch`/`jobs`/`nodes`/`tasks`/`health`) — extract OIDC sub/SPIFFE SVID from mTLS peer cert, check against ACL for namespace+verb, deny-by-default; (b) wire `acl.Check` into sshpush applier + txn apply path (validate `ORCA_OIDC_TOKEN` bearer against JWKS); (c) thread OIDC sub/SVID into audit `actor` field (replaces "cli"/"daemon"); (d) fix `acl.json` mode 0644→0600; (e) fix WebAuthn unauthenticated registration — `/orca/webauthn/register` requires existing authenticated session or admin bootstrap token; do not allow overwriting existing credentials without re-auth; (f) add flock on `acl.json` for concurrent grant/revoke | Critical | **v0.13 P04** | pending |
|
||||
| REQ-154 | Seal/audit CLI + chain race + key zeroing: (a) implement `orca cluster seal`/`unseal` (OIDC token exchange→unwrap master key→zeroed on shutdown; Shamir 3-of-5 shards printed at seal time; sealed blob at `ClusterDir()/master.key.sealed` 0600); (b) implement `orca doctor audit` (invokes `AuditRepo.VerifyChain`); (c) implement `orca doctor modes` (invokes `EnforceFileModes` across ORCA_HOME); (d) fix audit hash-chain race — `Append` uses `BEGIN IMMEDIATE` transaction; (e) fix `secrets rotate-master` to actually re-seal to OIDC; (f) zero master key / namespace keys / SVID private keys after use (defense-in-depth against pprof heap extraction) | High | **v0.13 P05** | pending |
|
||||
| REQ-155 | auth init-idp real + auth register: (a) implement `orca auth init-idp` — render Dex systemd unit + config template + Traefik dynamic route from `internal/webauthn/` connector at `https://<cluster>/orca/webauthn/{register,login}`; RP ID = cluster Traefik domain (C-38); HTTPS secure context via step-ca cert; atomic deploy with rollback; (b) implement `orca auth register` (browser flow to WebAuthn registration endpoint); (c) `loadOIDCConfig` config-file loading (`oidc.issuer` in config, not flags-only); (d) `orca doctor oidc` health check | High | **v0.13 P06** | pending |
|
||||
| REQ-153 | ACL enforcement + WebAuthn registration auth: (a) wire `acl.Check` into all 5 daemon handlers (`dispatch`/`jobs`/`nodes`/`tasks`/`health`) — extract OIDC sub/SPIFFE SVID from mTLS peer cert, check against ACL for namespace+verb, deny-by-default; (b) wire `acl.Check` into sshpush applier + txn apply path (validate `ORCA_OIDC_TOKEN` bearer against JWKS); (c) thread OIDC sub/SVID into audit `actor` field (replaces "cli"/"daemon"); (d) fix `acl.json` mode 0644→0600; (e) fix WebAuthn unauthenticated registration — `/orca/webauthn/register` requires existing authenticated session or admin bootstrap token; do not allow overwriting existing credentials without re-auth; (f) add flock on `acl.json` for concurrent grant/revoke | Critical | **v0.13 P04** | complete |
|
||||
| REQ-154 | Seal/audit CLI + chain race + key zeroing: (a) implement `orca cluster seal`/`unseal` (OIDC token exchange→unwrap master key→zeroed on shutdown; Shamir 3-of-5 shards printed at seal time; sealed blob at `ClusterDir()/master.key.sealed` 0600); (b) implement `orca doctor audit` (invokes `AuditRepo.VerifyChain`); (c) implement `orca doctor modes` (invokes `EnforceFileModes` across ORCA_HOME); (d) fix audit hash-chain race — `Append` uses `BEGIN IMMEDIATE` transaction; (e) fix `secrets rotate-master` to actually re-seal to OIDC; (f) zero master key / namespace keys / SVID private keys after use (defense-in-depth against pprof heap extraction) | High | **v0.13 P05** | complete |
|
||||
| REQ-155 | auth init-idp real + auth register: (a) implement `orca auth init-idp` — render Dex systemd unit + config template + Traefik dynamic route from `internal/webauthn/` connector at `https://<cluster>/orca/webauthn/{register,login}`; RP ID = cluster Traefik domain (C-38); HTTPS secure context via step-ca cert; atomic deploy with rollback; (b) implement `orca auth register` (browser flow to WebAuthn registration endpoint); (c) `loadOIDCConfig` config-file loading (`oidc.issuer` in config, not flags-only); (d) `orca doctor oidc` health check | High | **v0.13 P06** | complete |
|
||||
|
||||
### Wave D — Concurrency, transport, migration safety
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-156 | Concurrency safety: (a) SQLite `busy_timeout(5000)` + `SetMaxOpenConns(1)` on all DSNs (store, cache, recovery, webauthn); (b) secrets file flock (concurrent `secrets set` on same ns no longer loses data); (c) upgrade lock file (refuse concurrent `orca upgrade`); (d) backup lock file; (e) cache invalidation by write commands (`node join`/`leave`, `ns create`/`delete`, `job run`/`stop` invalidate relevant cache class — read-after-write consistency); (f) `Executor.Run` mutex scope fix (hold only for DB inserts, not whole job duration); (g) `ns create` atomic dir+ns.md write; (h) `writeCurrentLead` atomic write; (i) consolidate 3 divergent `writeAtomic` impls onto `security.WriteAtomic`; (j) WebAuthn session stores guarded with `sync.Mutex` | High | **v0.13 P07** | pending |
|
||||
| REQ-157 | Transport & SSH safety: (a) replace substring matching in `transport.IsTransient` AND `sshpush.isTransient` with typed sentinels (`errors.Is`); (b) `rotateSSHKeys` 2-phase atomic swap (stage new key on all peers → atomic swap → verify → cleanup old); (c) `known_hosts` flock field actually read by `dial()` (TOFU callback uses new field, not v0.8 `certpaths.KnownHostsPath()`); (d) IPv6 `net.JoinHostPort` in proxmox SSH dial + drain `splitHostPort`; (e) explicit timeouts for all SSH commands (peer-setup, drift remediate/ack, txn rollback, job restart — use `context.WithTimeout`); (f) `verifyCutover` use `security.ClientTLSConfig` with orca CA pool; (g) OIDC callback server `ReadHeaderTimeout: 5s`; (h) root SIGINT/SIGTERM handler for non-watch commands (clean SSH session + temp file cleanup) | High | **v0.13 P08** | pending |
|
||||
| REQ-158 | Migration & operational safety: (a) migration transaction + torn-write fix — `migrateDBSchema` wraps ALTER TABLE in transaction; crash after `os.Rename` but before schema fixup is recoverable; (b) `job stop` real `systemctl stop` via SSH (matches `job restart` pattern; honest semantics); (c) DB retention/compaction for `jobs`/`tasks`/`audit_log` tables (retention policy + `orca doctor db` compaction check); (d) `orca logs --lines` cap + `--since` upper bound (prevent OOM from unbounded journalctl output); (e) cache DB mode 0600 (matches `store.Open`); (f) `upgrade.go` cutover backup-file + atomic-rename (replace direct `sed -i`) | High | **v0.13 P09** | pending |
|
||||
| REQ-156 | Concurrency safety: (a) SQLite `busy_timeout(5000)` + `SetMaxOpenConns(1)` on all DSNs (store, cache, recovery, webauthn); (b) secrets file flock (concurrent `secrets set` on same ns no longer loses data); (c) upgrade lock file (refuse concurrent `orca upgrade`); (d) backup lock file; (e) cache invalidation by write commands (`node join`/`leave`, `ns create`/`delete`, `job run`/`stop` invalidate relevant cache class — read-after-write consistency); (f) `Executor.Run` mutex scope fix (hold only for DB inserts, not whole job duration); (g) `ns create` atomic dir+ns.md write; (h) `writeCurrentLead` atomic write; (i) consolidate 3 divergent `writeAtomic` impls onto `security.WriteAtomic`; (j) WebAuthn session stores guarded with `sync.Mutex` | High | **v0.13 P07** | complete |
|
||||
| REQ-157 | Transport & SSH safety: (a) replace substring matching in `transport.IsTransient` AND `sshpush.isTransient` with typed sentinels (`errors.Is`); (b) `rotateSSHKeys` 2-phase atomic swap (stage new key on all peers → atomic swap → verify → cleanup old); (c) `known_hosts` flock field actually read by `dial()` (TOFU callback uses new field, not v0.8 `certpaths.KnownHostsPath()`); (d) IPv6 `net.JoinHostPort` in proxmox SSH dial + drain `splitHostPort`; (e) explicit timeouts for all SSH commands (peer-setup, drift remediate/ack, txn rollback, job restart — use `context.WithTimeout`); (f) `verifyCutover` use `security.ClientTLSConfig` with orca CA pool; (g) OIDC callback server `ReadHeaderTimeout: 5s`; (h) root SIGINT/SIGTERM handler for non-watch commands (clean SSH session + temp file cleanup) | High | **v0.13 P08** | complete |
|
||||
| REQ-158 | Migration & operational safety: (a) migration transaction + torn-write fix — `migrateDBSchema` wraps ALTER TABLE in transaction; crash after `os.Rename` but before schema fixup is recoverable; (b) `job stop` real `systemctl stop` via SSH (matches `job restart` pattern; honest semantics); (c) DB retention/compaction for `jobs`/`tasks`/`audit_log` tables (retention policy + `orca doctor db` compaction check); (d) `orca logs --lines` cap + `--since` upper bound (prevent OOM from unbounded journalctl output); (e) cache DB mode 0600 (matches `store.Open`); (f) `upgrade.go` cutover backup-file + atomic-rename (replace direct `sed -i`) | High | **v0.13 P09** | complete |
|
||||
|
||||
### Wave E — Observability, docs, UAT
|
||||
|
||||
| ID | Requirement | Priority | Phase | Status |
|
||||
|----|-------------|----------|-------|--------|
|
||||
| REQ-159 | Observability expansion: metrics add `orca_jobs_by_state` histogram, `orca_drift_events_total` counter, `orca_ssh_errors_total` counter, `orca_txn_apply_total`/`orca_txn_rollback_total` counters, `orca_acl_denials_total` counter, `orca_audit_chain_head` gauge; new `docs/metrics.md` with Prometheus scrape config; security headers middleware on daemon (`X-Content-Type-Options`, `X-Frame-Options`) | Medium | **v0.13 P10** | pending |
|
||||
| REQ-160 | Doc drift round 2: (a) README — update status banner (v0.12+v0.13 complete), latest tag, subcommand table (add `auth`/`nft`/`peer-setup`/`secrets rotate-master`), correct "mTLS by default" claim (SSH-push is canonical, mTLS deprecated), add missing docs to table; (b) `docs/cli.md` — complete rewrite covering all ~40 subcommands; (c) CHANGELOG regen; (d) help text fixes (`job run` HCL→markdown, `job stop` daemon→SSH-push); (e) `docs/webauthn.md` add `auth register`; (f) `docs/namespace.md` add `inherit`/`set-constraint`; (g) `docs/install.md`+`docker.md` update version refs; (h) `docs/security-runbook.md` match P05 reality; (i) fix `verify-reqs` bold-format regex (currently bypasses v0.12); (j) fix ROADMAP/REQUIREMENTS v0.12 status hygiene; (k) `docs/security-scanning.md` gosec.json; (l) `internal/proxmox/bootstrap.go` comments (password→key auth); (m) deprecate `orca status` stub; (n) `make verify-docs` target (cli.md ↔ `orca --help` consistency) | High | **v0.13 P11** | pending |
|
||||
| REQ-161 | `--type linux` SSH-join: implement `NodeKindLinux` path (reserved at `model/node.go:29`); new `internal/linux/bootstrap.go` mirroring Proxmox pattern — orca pubkey deploy → `orca` system user → drift-events dir → no PVE role; key-auth only (R-021); `orca node join --type linux --host <ip> --ssh-user root --ssh-key <path>`; `peer-setup.go` kept as documented fallback | High | **v0.13 P12** | pending |
|
||||
| REQ-162 | UAT plan: `docs/uat.md` — 3-host topology (lead Ubuntu 22.04 + pve01 Proxmox VE 8/9 + worker01 Ubuntu 22.04); step-by-step with exact commands (bootstrap→onboard Proxmox→onboard Ubuntu worker→capacity→namespace→deploy full stack→migrate between hosts→exercise every claim); claim matrix mapping ~35 feature claims to UAT steps; signoff procedure (run `scripts/uat-signoff.sh`, paste output) | Critical | **v0.13 P12** | pending |
|
||||
| REQ-163 | UAT signoff script: `scripts/uat-signoff.sh` — idempotent, `set -euo pipefail`, ~35 named assertions covering all feature claims; read + non-mutating only (doctor, list, --dry-run); exit 0 iff all pass; `scripts/uat-smoke.sh` — pure-CLI subset for CI `validate` (version, acl file mode, doctor modes, no-password grep, metrics shape); tests for both scripts | Critical | **v0.13 P12** | pending |
|
||||
| REQ-159 | Observability expansion: metrics add `orca_jobs_by_state` histogram, `orca_drift_events_total` counter, `orca_ssh_errors_total` counter, `orca_txn_apply_total`/`orca_txn_rollback_total` counters, `orca_acl_denials_total` counter, `orca_audit_chain_head` gauge; new `docs/metrics.md` with Prometheus scrape config; security headers middleware on daemon (`X-Content-Type-Options`, `X-Frame-Options`) | Medium | **v0.13 P10** | complete |
|
||||
| REQ-160 | Doc drift round 2: (a) README — update status banner (v0.12+v0.13 complete), latest tag, subcommand table (add `auth`/`nft`/`peer-setup`/`secrets rotate-master`), correct "mTLS by default" claim (SSH-push is canonical, mTLS deprecated), add missing docs to table; (b) `docs/cli.md` — complete rewrite covering all ~40 subcommands; (c) CHANGELOG regen; (d) help text fixes (`job run` HCL→markdown, `job stop` daemon→SSH-push); (e) `docs/webauthn.md` add `auth register`; (f) `docs/namespace.md` add `inherit`/`set-constraint`; (g) `docs/install.md`+`docker.md` update version refs; (h) `docs/security-runbook.md` match P05 reality; (i) fix `verify-reqs` bold-format regex (currently bypasses v0.12); (j) fix ROADMAP/REQUIREMENTS v0.12 status hygiene; (k) `docs/security-scanning.md` gosec.json; (l) `internal/proxmox/bootstrap.go` comments (password→key auth); (m) deprecate `orca status` stub; (n) `make verify-docs` target (cli.md ↔ `orca --help` consistency) | High | **v0.13 P11** | complete |
|
||||
| REQ-161 | `--type linux` SSH-join: implement `NodeKindLinux` path (reserved at `model/node.go:29`); new `internal/linux/bootstrap.go` mirroring Proxmox pattern — orca pubkey deploy → `orca` system user → drift-events dir → no PVE role; key-auth only (R-021); `orca node join --type linux --host <ip> --ssh-user root --ssh-key <path>`; `peer-setup.go` kept as documented fallback | High | **v0.13 P12** | complete |
|
||||
| REQ-162 | UAT plan: `docs/uat.md` — 3-host topology (lead Ubuntu 22.04 + pve01 Proxmox VE 8/9 + worker01 Ubuntu 22.04); step-by-step with exact commands (bootstrap→onboard Proxmox→onboard Ubuntu worker→capacity→namespace→deploy full stack→migrate between hosts→exercise every claim); claim matrix mapping ~35 feature claims to UAT steps; signoff procedure (run `scripts/uat-signoff.sh`, paste output) | Critical | **v0.13 P12** | complete |
|
||||
| REQ-163 | UAT signoff script: `scripts/uat-signoff.sh` — idempotent, `set -euo pipefail`, ~35 named assertions covering all feature claims; read + non-mutating only (doctor, list, --dry-run); exit 0 iff all pass; `scripts/uat-smoke.sh` — pure-CLI subset for CI `validate` (version, acl file mode, doctor modes, no-password grep, metrics shape); tests for both scripts | Critical | **v0.13 P12** | complete |
|
||||
|
||||
### Scope notes (v0.13)
|
||||
|
||||
|
||||
+16
-16
@@ -400,7 +400,7 @@ tags: `v0.10.0`…`v0.10.21`.
|
||||
- External CA / Let's Encrypt / cert transparency
|
||||
- Online-only features (HSTS, OCSP stapling, telemetry)
|
||||
|
||||
## Milestone v0.12: Security Hardening (Zero-Trust Identity) — COMPLETE
|
||||
## Milestone v0.12: Security Hardening (Zero-Trust Identity) — **COMPLETE**
|
||||
|
||||
**Scope**: comprehensive security hardening across the entire attack
|
||||
surface, **including the operating system itself**, plus adoption of a
|
||||
@@ -549,7 +549,7 @@ The v1.0.0 production-ready tag stays deferred for post-v0.12 UAT
|
||||
- External CA / Let's Encrypt / cert transparency
|
||||
- Online-only features (HSTS, OCSP stapling, telemetry)
|
||||
|
||||
## Milestone v0.13: Production Hardening Round 2 + UAT Plan — IN PROGRESS
|
||||
## Milestone v0.13: Production Hardening Round 2 + UAT Plan — **COMPLETE**
|
||||
|
||||
**Scope**: final production hardening round before the v1.0.0
|
||||
production-ready tag. Three deep codebase sweeps (security, reliability,
|
||||
@@ -579,20 +579,20 @@ signoff script that gates the v1.0.0 cut.
|
||||
|
||||
### Phases (14 total: P0 + P01..P12 + P13 final)
|
||||
|
||||
- [ ] Phase P0: Pre-execution (SPECIFY→CLARIFY→RESEARCH→IDEATE→PLAN→GRILL) — tag `v0.12.0`
|
||||
- [ ] Phase P01: Toolchain & dependency vulns (REQ-149) — tag `v0.12.1`
|
||||
- [ ] Phase P02: Input validation & injection hardening (REQ-150) — tag `v0.12.2`
|
||||
- [ ] Phase P03: Scheduler/deployment wiring + jobspec parser (REQ-151, REQ-152) — tag `v0.12.3`
|
||||
- [ ] Phase P04: ACL enforcement + WebAuthn registration auth (REQ-153) — tag `v0.12.4`
|
||||
- [ ] Phase P05: Seal/audit CLI + chain race + key zeroing (REQ-154) — tag `v0.12.5`
|
||||
- [ ] Phase P06: auth init-idp real + auth register (REQ-155) — tag `v0.12.6`
|
||||
- [ ] Phase P07: Concurrency safety (REQ-156) — tag `v0.12.7`
|
||||
- [ ] Phase P08: Transport & SSH safety (REQ-157) — tag `v0.12.8`
|
||||
- [ ] Phase P09: Migration & operational safety (REQ-158) — tag `v0.12.9`
|
||||
- [ ] Phase P10: Observability & metrics (REQ-159) — tag `v0.12.10`
|
||||
- [ ] Phase P11: Doc drift round 2 (REQ-160) — tag `v0.12.11`
|
||||
- [ ] Phase P12: `--type linux` + UAT plan + signoff script (REQ-161, REQ-162, REQ-163) — tag `v0.12.12`
|
||||
- [ ] Phase P13: Final review + ship + audit (milestone release) — tag `v0.12.13` = **v0.13 milestone release**
|
||||
- [x] Phase P0: Pre-execution (SPECIFY→CLARIFY→RESEARCH→IDEATE→PLAN→GRILL) — tag `v0.12.0`
|
||||
- [x] Phase P01: Toolchain & dependency vulns (REQ-149) — tag `v0.12.1`
|
||||
- [x] Phase P02: Input validation & injection hardening (REQ-150) — tag `v0.12.2`
|
||||
- [x] Phase P03: Scheduler/deployment wiring + jobspec parser (REQ-151, REQ-152) — tag `v0.12.3`
|
||||
- [x] Phase P04: ACL enforcement + WebAuthn registration auth (REQ-153) — tag `v0.12.4`
|
||||
- [x] Phase P05: Seal/audit CLI + chain race + key zeroing (REQ-154) — tag `v0.12.5`
|
||||
- [x] Phase P06: auth init-idp real + auth register (REQ-155) — tag `v0.12.6`
|
||||
- [x] Phase P07: Concurrency safety (REQ-156) — tag `v0.12.7`
|
||||
- [x] Phase P08: Transport & SSH safety (REQ-157) — tag `v0.12.8`
|
||||
- [x] Phase P09: Migration & operational safety (REQ-158) — tag `v0.12.9`
|
||||
- [x] Phase P10: Observability & metrics (REQ-159) — tag `v0.12.10`
|
||||
- [x] Phase P11: Doc drift round 2 (REQ-160) — tag `v0.12.11`
|
||||
- [x] Phase P12: `--type linux` + UAT plan + signoff script (REQ-161, REQ-162, REQ-163) — tag `v0.12.12`
|
||||
- [x] Phase P13: Final review + ship + audit (milestone release) — tag `v0.12.13` = **v0.13 milestone release**
|
||||
|
||||
**Milestone tag**: `v0.12.13` (final phase patch = milestone release per
|
||||
feature-milestone rule; no separate `v0.13.0` tag). Per-phase tags:
|
||||
|
||||
+132
-26
@@ -5,31 +5,137 @@ All notable changes to orca are documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
- `e1b538575c57158c7a6661d5919b103f6c7932fc` — feat(P06): CoreCI release flow with .coreci.yml and tea integration
|
||||
- `07b8ad2ceaba7ca303dfe91876930d33b76c633e` — ship(P05): health checks merged into milestone
|
||||
- `b06458d31370750417a3b239dac61c6e2fdf5329` — docs(P05): verification - 4 layers pass
|
||||
- `708d9834296271094667700e88e80bfa27db7bdd` — feat(P05): health check daemon with /healthz, /readyz, /v1/* handlers
|
||||
- `30c523c0c7a8e75a2e97b42f1c8a39802febcbdc` — ship(P04): state persistence merged into milestone
|
||||
- `759b1b519d7fadf3d91d3070952d9ad2051a0eba` — docs(P04): verification - 4 layers pass
|
||||
- `b25e074e1d3518f175478184ff8d002ec0d8412c` — feat(P04): audit log + persistence hardening
|
||||
- `bb6b5b3e8342c16601a8503223c7186ecdbb00df` — ship(P03): task exec merged into milestone
|
||||
- `857f7563190e7703f97c607a50d6b0a897d250e9` — docs(P03): verification - 4 layers pass
|
||||
- `f9a98733411cfa8657e82636e0c55671086ebe46` — feat(P03): task execution engine with HCL specs, jobs, tasks, WaitDelay
|
||||
- `78334f1f74f0c185c6d38014c796aac4903b8141` — ship(P02): node mgmt merged into milestone
|
||||
- `c7dbcef9587596786a541a7566479d9fb93fcf0a` — docs(P02): verification - 4 layers pass
|
||||
- `9580f347c68e395dccfbe83b27a857d52bf21075` — feat(P02): node management with SQLite-backed registry
|
||||
- `46e929e4c6539bd604539ba27d5ed0c606e87bb9` — chore(P01): source .env in trigger_coreci.sh for GITEA_TOKEN
|
||||
- `503923bf1ee2c60f8375acc7eb9608d346368e1c` — ship(P01): cli skeleton merged into milestone
|
||||
- `e3f6e1df825d39f73933c9996bd2cc4717ff1061` — docs(P01): verification - 4 layers pass
|
||||
- `aa3cccead503a37dfec75873d06d2d396a2876f2` — feat(P01): CLI skeleton with Cobra, subcommand stubs, pre-push hook
|
||||
- `c2038952c74f7c242ba3be65d2f4269b23685f5a` — docs(P00): create 6 phase plans with wave ordering
|
||||
- `65eb2e601b741b36388598b9f8adddd7bd8dd3a8` — docs(P00): research findings - architecture + personas
|
||||
- `6f34f1794b9f526c06a1dc139d4a74371599502e` — docs(P00): ideation - 30 ideas accepted (3 tiers)
|
||||
- `bc7ce1caf672e87774455a6cd6cc0db986cd09b3` — docs(P00): clarify ambiguities (full autonomy, 10 decisions)
|
||||
- `55aae5347ec09bce9ef7697ea0c9c9ee158bc040` — chore(P00): rename orch-engine to orca, configure gitea + coreci (v0.1)
|
||||
- `0cba1aa5feef9564f8b9a2a97ae735dc859a8a84` — chore(P00): set autonomy level to full
|
||||
- `e2e77e79b9cbfb462044662543845476f843161b` — chore(P00): quick task - populate config.json with backlog reference
|
||||
- `8c086def698bf0af31e8e820b6b7a2783af06f43` — chore(config): populate ciagent config with standard settings
|
||||
- `8774008c3e47e4ca4711f4fef164531006d16216` — docs(init): validate specification
|
||||
## v0.13 milestone (in progress) — tag line v0.12.x
|
||||
|
||||
The v0.13 milestone is **Production Hardening Round 2 + UAT Plan**.
|
||||
Three deep codebase sweeps (security, reliability, feature/doc claims)
|
||||
surfaced ~60 gaps beyond v0.12. v0.13 closes all critical/high/medium
|
||||
findings and delivers the UAT plan + signoff script that gates the
|
||||
v1.0.0 cut.
|
||||
|
||||
**Load-bearing architectural changes**:
|
||||
- **R-022** — `orca job run` deploys to remote nodes via the scheduler →
|
||||
emitter → SSH-push pipeline. The local `exec.CommandContext` path is
|
||||
removed (P03).
|
||||
- **R-023** — Zero-trust enforcement is operationally wired: `acl.Check`
|
||||
is invoked on every daemon handler + sshpush + txn apply path;
|
||||
`acl.json` is 0600; audit `actor` carries OIDC sub/SVID; WebAuthn
|
||||
registration requires auth; `cluster seal`/`unseal` + `doctor audit`/
|
||||
`doctor modes` CLI commands exist (P04, P05).
|
||||
|
||||
### v0.13 phase commits (v0.11.29..HEAD)
|
||||
|
||||
- `ed91d68` — feat(P10): observability expansion — metrics + security headers (REQ-159)
|
||||
- `531b369` — fix(P09): migration + operational safety — job stop, retention, logs cap (REQ-158)
|
||||
- `3a3ea74` — fix(P08): transport + SSH safety — typed errors, IPv6, timeouts, signal (REQ-157)
|
||||
- `0358efe` — fix(P07): concurrency safety — SQLite, flock, cache, atomic writes (REQ-156)
|
||||
- `978334a` — feat(P06): auth init-idp real + auth register + doctor oidc (REQ-155)
|
||||
- `9e83238` — feat(P05): seal/audit CLI + chain race fix + key zeroing (REQ-154)
|
||||
- `5232fcb` — fix(P04): wire ACL enforcement + WebAuthn reg auth + audit actor (REQ-153)
|
||||
- `cf3d98e` — feat(P03): wire scheduler into job run + fix jobspec parser (REQ-151, REQ-152)
|
||||
- `4b70e31` — fix(P02): input validation + injection hardening — 11 vectors (REQ-150)
|
||||
- `b0158c9` — fix(P01): bump go toolchain to 1.25.12 + fix pre-existing test bugs (REQ-149)
|
||||
- `7479cd1` — docs(checkpoint): P0 shipped — v0.12.0 tagged
|
||||
- `1a2dd1a` — docs(P00): incorporate grill binding conditions C-44..C-49
|
||||
- `437d9b2` — docs(P00): grill v0.13 — CONDITIONAL PROCEED (6 binding conditions C-44..C-49)
|
||||
- `82bfab1` — docs(P00): create phase plans — 14 phases, 15 REQs, vertical slices
|
||||
- `a2a651e` — docs(P00): ideation results — 15 accepted (REQ-149..REQ-163), 0 skipped
|
||||
- `3f5e5de` — docs(P00): research findings — threat model round 3 (~60 gaps, F26-F101)
|
||||
- `7a60b35` — docs(P00): clarify v0.13 — 7 decisions resolved (D-248..D-254)
|
||||
- `8071793` — docs(init): validate specification — v0.13 Production Hardening Round 2 + UAT Plan
|
||||
|
||||
### v0.13 phase summary
|
||||
|
||||
- **P0** — Pre-execution: specify → clarify → research → ideate → plan → grill (tag `v0.12.0`)
|
||||
- **P01** — Toolchain & dependency vulns: Go 1.25.12 bump, 24 stdlib vulns closed, govulncheck triage (REQ-149)
|
||||
- **P02** — Input validation & injection hardening: 11 vectors closed (`orca logs --job` RCE, tar-slip, sudoers injection, pprof loopback, txn/nft ID validation, drain allocID, cluster_compat, podman image, nft TrustedProbes, sudoers user/role) (REQ-150)
|
||||
- **P03** — Scheduler/deployment wiring + jobspec parser: `orca job run` wires scheduler → emitter → SSH-push; `schedule:`/`timeout:` parsed by markdown jobspec (REQ-151, REQ-152)
|
||||
- **P04** — ACL enforcement + WebAuthn registration auth: `acl.Check` wired into daemon + sshpush + txn apply; WebAuthn registration requires auth; audit actor carries OIDC sub/SVID (REQ-153)
|
||||
- **P05** — Seal/audit CLI + chain race fix + key zeroing: `orca cluster seal`/`unseal`, `orca doctor audit`, `orca doctor modes` CLI commands; audit hash-chain race fix; master key zeroed on exit (REQ-154)
|
||||
- **P06** — auth init-idp real + auth register + doctor oidc: real Dex deployment, `orca auth register` browser flow, `orca doctor oidc` health check (REQ-155)
|
||||
- **P07** — Concurrency safety: SQLite WAL, flock on known_hosts, cache thread-safety, atomic writes (REQ-156)
|
||||
- **P08** — Transport & SSH safety: typed dial errors, IPv6 support, connect timeouts, signal handling (REQ-157)
|
||||
- **P09** — Migration & operational safety: `orca job stop` via SSH, DB retention check, logs cap (REQ-158)
|
||||
- **P10** — Observability & metrics: metrics endpoint expansion, security headers (REQ-159)
|
||||
- **P11** — Doc drift round 2 (this phase, REQ-160)
|
||||
|
||||
## v0.12 milestone — COMPLETE (tag line v0.11.x)
|
||||
|
||||
The v0.12 milestone is **Security Hardening (Zero-Trust Identity)**.
|
||||
Comprehensive security hardening across the entire attack surface
|
||||
including the OS, plus adoption of a zero-trust identity model. 25
|
||||
threat-model findings (F1..F25) closed. R-021 adopted: no Orca-issued
|
||||
credentials — human identity is exclusively external (OIDC), machine
|
||||
identity is exclusively mTLS/SPIFFE.
|
||||
|
||||
**Milestone release**: `v0.11.28` (29 phases, tags `v0.11.0`..`v0.11.28`).
|
||||
|
||||
### v0.12 phase highlights
|
||||
|
||||
- Command injection fix (REQ-119, F3)
|
||||
- Namespace path traversal fix (REQ-120, F4)
|
||||
- Txn apply path allowlist (REQ-121, F5)
|
||||
- OIDC client + bundled Dex (REQ-144; BYO-IdP override)
|
||||
- WebAuthn connector for Dex / passkeys (REQ-148)
|
||||
- ACL rewrite to OIDC claims + enforcement (REQ-145, REQ-122, F1)
|
||||
- Remove all password/token paths (REQ-146, R-021, C-34)
|
||||
- Master key seal-to-OIDC + Shamir 3-of-5 recovery (REQ-147, C-35)
|
||||
- Daemon auth hardening (REQ-123, REQ-124, F6, F24)
|
||||
- Audit log tamper-evidence (REQ-125, F2)
|
||||
- SVID chain validation (REQ-126, F9)
|
||||
- Backup symlink validation (REQ-127, F7)
|
||||
- step-ca /tmp hardening (REQ-128, F10)
|
||||
- Master key rotation (REQ-129, F12, C-30)
|
||||
- File-mode audit expansion (REQ-130, F13)
|
||||
- aggregate.sh JSON injection + drift-gate fix (REQ-131, F11, F18)
|
||||
- install.sh checksum+GPG verification (REQ-132, F14)
|
||||
- nftables ruleset hardening (REQ-133, F21)
|
||||
- sudoers hardening (REQ-134, F22)
|
||||
- System user consistency (REQ-135, F23)
|
||||
- SQLite file-mode + at-rest encryption (REQ-136, F8, C-31)
|
||||
- Migration safety + identity migration (REQ-137, F19, C-34)
|
||||
- Legacy CA/mTLS/daemon + step-ca password-provisioner deletion (REQ-138, F16)
|
||||
- known_hosts tightening + transport hardening (REQ-139, F15, F25)
|
||||
- Drift event authentication (REQ-140, F18)
|
||||
- Security integration test suite (REQ-141, C-33)
|
||||
- Zero-trust + OIDC + WebAuthn + threat-model docs (REQ-142)
|
||||
- Final review + ship + audit (REQ-143)
|
||||
|
||||
## v0.11 milestone — COMPLETE (tag line v0.10.x)
|
||||
|
||||
The v0.11 milestone is **Production Hardening**. See the git log and
|
||||
ROADMAP for the full phase list.
|
||||
|
||||
## v0.1 milestone — COMPLETE
|
||||
|
||||
Initial CLI skeleton, node management, task execution, state
|
||||
persistence, audit log, health checks, and CoreCI release flow.
|
||||
|
||||
- `e1b5385` — feat(P06): CoreCI release flow with .coreci.yml and tea integration
|
||||
- `07b8ad2` — ship(P05): health checks merged into milestone
|
||||
- `b06458d` — docs(P05): verification - 4 layers pass
|
||||
- `708d983` — feat(P05): health check daemon with /healthz, /readyz, /v1/* handlers
|
||||
- `30c523c` — ship(P04): state persistence merged into milestone
|
||||
- `759b1b5` — docs(P04): verification - 4 layers pass
|
||||
- `b25e074` — feat(P04): audit log + persistence hardening
|
||||
- `bb6b5b3` — ship(P03): task exec merged into milestone
|
||||
- `857f756` — docs(P03): verification - 4 layers pass
|
||||
- `f9a9873` — feat(P03): task execution engine with HCL specs, jobs, tasks, WaitDelay
|
||||
- `78334f1` — ship(P02): node mgmt merged into milestone
|
||||
- `c7dbcef` — docs(P02): verification - 4 layers pass
|
||||
- `9580f34` — feat(P02): node management with SQLite-backed registry
|
||||
- `46e929e` — chore(P01): source .env in trigger_coreci.sh for GITEA_TOKEN
|
||||
- `503923b` — ship(P01): cli skeleton merged into milestone
|
||||
- `e3f6e1d` — docs(P01): verification - 4 layers pass
|
||||
- `aa3ccce` — feat(P01): CLI skeleton with Cobra, subcommand stubs, pre-push hook
|
||||
- `c203895` — docs(P00): create 6 phase plans with wave ordering
|
||||
- `65eb2e6` — docs(P00): research findings - architecture + personas
|
||||
- `6f34f17` — docs(P00): ideation - 30 ideas accepted (3 tiers)
|
||||
- `bc7ce1c` — docs(P00): clarify ambiguities (full autonomy, 10 decisions)
|
||||
- `55aae53` — chore(P00): rename orch-engine to orca, configure gitea + coreci (v0.1)
|
||||
- `0cba1aa` — chore(P00): set autonomy level to full
|
||||
- `e2e77e7` — chore(P00): quick task - populate config.json with backlog reference
|
||||
- `8c086de` — chore(config): populate ciagent config with standard settings
|
||||
- `8774008` — docs(init): validate specification
|
||||
|
||||
Generated by make changelog. Do not edit by hand.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.PHONY: build test test-race lint fmt clean run release version changelog help security-scan verify-reqs
|
||||
.PHONY: build test test-race lint fmt clean run release version changelog help security-scan verify-reqs verify-docs
|
||||
|
||||
BINARY := bin/orca
|
||||
GOFLAGS := -trimpath
|
||||
@@ -31,6 +31,7 @@ help:
|
||||
@echo " release Run scripts/release.sh [VERSION] — build, tar, publish"
|
||||
@echo " security-scan Run gosec+govulncheck+gitleaks (P03, REQ-014/027/039)"
|
||||
@echo " verify-reqs Assert ROADMAP COMPLETE ↔ REQUIREMENTS Complete (REQ-060)"
|
||||
@echo " verify-docs Assert docs/cli.md ↔ orca --help consistency (REQ-160)"
|
||||
|
||||
build:
|
||||
@mkdir -p bin
|
||||
@@ -128,3 +129,9 @@ security-scan:
|
||||
# of scope (P04 audit). Exits 0 on consistency, 1 with a diff on drift.
|
||||
verify-reqs:
|
||||
go run ./cmd/verify-reqs .ciagent/ROADMAP.md .ciagent/REQUIREMENTS.md
|
||||
|
||||
# verify-docs asserts that every top-level subcommand in docs/cli.md
|
||||
# exists in `orca --help` output (and vice versa). Catches doc drift
|
||||
# (REQ-160). Requires the binary to be built first (`make build`).
|
||||
verify-docs: build
|
||||
./scripts/verify-docs.sh ./bin/orca docs/cli.md
|
||||
|
||||
@@ -6,8 +6,8 @@ identity.
|
||||
|
||||
## Status
|
||||
|
||||
**v0.11: Production Hardening — IN PROGRESS** | **v1.0: UAT-gated** (cut
|
||||
separately after v0.11 completion per operator decision)
|
||||
**v0.12: Security Hardening (Zero-Trust Identity) — COMPLETE** | **v0.13: Production Hardening Round 2 + UAT Plan — IN PROGRESS** | **v1.0: UAT-gated** (cut
|
||||
separately after v0.13 completion per operator decision)
|
||||
|
||||
See [.ciagent/ROADMAP.md](.ciagent/ROADMAP.md) for the full roadmap.
|
||||
|
||||
@@ -18,8 +18,8 @@ See [.ciagent/ROADMAP.md](.ciagent/ROADMAP.md) for the full roadmap.
|
||||
- **Offline-first** — no cloud dependencies; the cluster is the OS
|
||||
- **CLI-first** — the command line is the primary interface (humans and
|
||||
AI agents)
|
||||
- **Security before features** — mTLS by default; NFRs ship before new
|
||||
functionality
|
||||
- **Security before features** — SSH-push is the canonical transport
|
||||
(mTLS available for daemon mode); NFRs ship before new functionality
|
||||
- **WASM-first** — workloads target OS primitives (systemd units,
|
||||
journald), not a container runtime shim
|
||||
- **Bug fixes before features** — stability is paramount
|
||||
@@ -35,8 +35,8 @@ curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | b
|
||||
# System-level install (binary at /usr/local/bin/orca, state at /root/.orca)
|
||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | sudo bash -s -- --system
|
||||
|
||||
# Pin a specific version (latest tag: v0.10.19)
|
||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --version v0.10.19
|
||||
# Pin a specific version (latest tag: v0.12.10)
|
||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --version v0.12.10
|
||||
|
||||
# Dry-run: check what would be installed without writing
|
||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash -s -- --check
|
||||
@@ -65,7 +65,7 @@ config, database, and certificates in the namespace dir:
|
||||
|
||||
```bash
|
||||
curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | bash
|
||||
# → "updated orca from v0.8.15 to v0.10.19"
|
||||
# → "updated orca from v0.11.28 to v0.12.10"
|
||||
```
|
||||
|
||||
## Subcommands
|
||||
@@ -73,7 +73,7 @@ curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | b
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `orca init` | Initialize local orca state with full bootstrap |
|
||||
| `orca status` | Show orca daemon status |
|
||||
| `orca status` | **(deprecated v0.1 stub)** Show orca daemon status — use `orca node list` + `orca metrics /healthz` |
|
||||
| `orca version` | Print version information |
|
||||
| `orca daemon` | **(deprecated)** Run the orca daemon (HTTP API + health checks) |
|
||||
| `orca metrics` | Start metrics endpoint (Prometheus text exposition) |
|
||||
@@ -85,15 +85,18 @@ curl -fsSL https://git.cloudinit.dev/coreci/orca/raw/main/scripts/install.sh | b
|
||||
| `orca job` | Manage orca jobs: `run`, `list`, `stop`, `logs`, `lint`, `verify`, `migrate`, `restart` |
|
||||
| `orca ns` | Manage orca namespaces: `list`, `create`, `delete`, `inspect`, `validate`, `inherit`, `set-constraint` |
|
||||
| `orca cert` | **(deprecated)** Manage orca certificates: `ca-init`, `gen`, `show`, `renew`, `fingerprint` |
|
||||
| `orca doctor` | Run self-checks: `cert`, `network`, `db`, `os`, `proxmox`, `no-orca-on-server` |
|
||||
| `orca doctor` | Run self-checks: `cert`, `network`, `db`, `os`, `proxmox`, `no-orca-on-server`, `nft`, `audit`, `modes`, `oidc`, `db-retention` |
|
||||
| `orca audit` | View orca audit log (`list`) |
|
||||
| `orca cache` | CLI cache management: `show`, `invalidate`, `invalidate-all` |
|
||||
| `orca acl` | ACL management: `grant`, `revoke`, `list`, `check` |
|
||||
| `orca secrets` | Secrets management: `set`, `get`, `list`, `rotate`, `delete` |
|
||||
| `orca secrets` | Secrets management: `set`, `get`, `list`, `rotate`, `delete`, `rotate-master` |
|
||||
| `orca drift` | Drift detection: `show`, `watch`, `acknowledge`, `remediate`, `config` |
|
||||
| `orca txn` | Transaction management: `apply`, `list`, `show`, `rollback` |
|
||||
| `orca nft` | nftables ingress management: `show`, `diff`, `doctor`, `country block`, `rate limit` |
|
||||
| `orca collector` | Collector/aggregator management: `start`, `stop`, `status` |
|
||||
| `orca cluster` | Cluster management: `cutover`, `rotate-lead`, `compat-check` |
|
||||
| `orca cluster` | Cluster management: `cutover`, `rotate-lead`, `compat-check`, `seal`, `unseal` |
|
||||
| `orca auth` | OIDC authentication: `login`, `logout`, `status`, `init-idp`, `register` |
|
||||
| `orca peer-setup` | Create the orca system user + drift-events dir on a peer (REQ-111) |
|
||||
|
||||
See [docs/cli.md](docs/cli.md) for the full CLI reference with all flags
|
||||
and examples.
|
||||
@@ -114,7 +117,7 @@ acknowledged rather than papered over.
|
||||
| Auto-scaling | Cluster autoscaler, HPA/VPA, deep integrations | — |
|
||||
| Daemon footprint | — | No daemon on the critical path; the cluster is the OS |
|
||||
| OS-native | — | Workloads are systemd units + journald; no container runtime shim |
|
||||
| mTLS | — | mTLS by default; no opt-in required |
|
||||
| Transport | — | SSH-push is canonical (no daemon needed); mTLS available for daemon mode |
|
||||
| Offline-first | — | No cloud dependencies; fully air-gapped operation |
|
||||
| WASM-first | — | Workloads target OS primitives, not a container runtime |
|
||||
| Proxmox | — | First-class Proxmox node type (`--type proxmox`) via SSH-push |
|
||||
@@ -129,6 +132,10 @@ acknowledged rather than papered over.
|
||||
| [docs/namespace.md](docs/namespace.md) | Namespace and path layout |
|
||||
| [docs/install.md](docs/install.md) | Installation guide |
|
||||
| [docs/security-scanning.md](docs/security-scanning.md) | Security scanning tools |
|
||||
| [docs/security-runbook.md](docs/security-runbook.md) | Security runbook — seal/unseal, rotation, incident response |
|
||||
| [docs/webauthn.md](docs/webauthn.md) | WebAuthn / passkeys registration and login |
|
||||
| [docs/threat-model.md](docs/threat-model.md) | STRIDE threat model + zero-trust architecture |
|
||||
| [docs/oidc.md](docs/oidc.md) | OIDC configuration — Dex quickstart, BYO IdP |
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -144,6 +151,7 @@ make test # Run tests
|
||||
go vet ./... # Vet all packages
|
||||
make lint # Run gofmt + go vet + shellcheck
|
||||
make verify-reqs # Assert ROADMAP ↔ REQUIREMENTS consistency
|
||||
make verify-docs # Assert docs/cli.md ↔ `orca --help` consistency
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
+24
-10
@@ -16,17 +16,20 @@ import (
|
||||
// the Phase + Status match at the END of the line, where those two columns
|
||||
// always live. The status token is optionally wrapped in markdown bold
|
||||
// (real rows use `**Complete**`; synthetic/future rows may use bare
|
||||
// `Pending`), and may carry trailing notes (e.g. "**Complete** (P01
|
||||
// shipped v0.2.1)") matched by [^|]* before the closing pipe.
|
||||
var reqRowRe = regexp.MustCompile(`^\|\s*(REQ-\d+)\s*\|.*\|\s*([^|]*?)\s*\|\s*\*{0,2}(Complete|Pending)\*{0,2}[^|]*\|\s*$`)
|
||||
// `pending` or `complete` in any case), and may carry trailing notes
|
||||
// (e.g. "**Complete** (P01 shipped v0.2.1)") matched by [^|]* before
|
||||
// the closing pipe. The (?i) flag makes the match case-insensitive so
|
||||
// lowercase `pending` (used by v0.12/v0.13 REQ rows) is captured;
|
||||
// normalizeStatus canonicalizes the captured value to title case.
|
||||
var reqRowRe = regexp.MustCompile(`(?i)^\|\s*(REQ-\d+)\s*\|.*\|\s*([^|]*?)\s*\|\s*\*{0,2}(Complete|Pending)\*{0,2}[^|]*\|\s*$`)
|
||||
|
||||
// milestoneCompleteRe matches a ROADMAP.md milestone header that is marked
|
||||
// COMPLETE. The bold span is substring-tolerant (GRILL #4): it matches
|
||||
// `**COMPLETE**`, `**COMPLETE (merged to main via v0.3)**`, and any future
|
||||
// variant where the word COMPLETE appears inside the bold span, possibly
|
||||
// preceded or followed by non-asterisk text. The milestone version (v0.X)
|
||||
// is captured.
|
||||
var milestoneCompleteRe = regexp.MustCompile(`^##\s*Milestone\s+(v0\.\d+):.*—\s*\*\*[^*]*\bCOMPLETE\b[^*]*\*\*`)
|
||||
// COMPLETE. The bold markers are optional (GRILL #4 + REQ-160 T11): it
|
||||
// matches `**COMPLETE**`, `**COMPLETE (merged to main via v0.3)**`, and
|
||||
// bare `COMPLETE` (as used by the v0.12 milestone header). The word
|
||||
// COMPLETE may be preceded or followed by non-asterisk text. The milestone
|
||||
// version (v0.X) is captured.
|
||||
var milestoneCompleteRe = regexp.MustCompile(`^##\s*Milestone\s+(v0\.\d+):.*—\s*\*{0,2}[^*]*\bCOMPLETE\b[^*]*\*{0,2}`)
|
||||
|
||||
// phaseRe extracts the milestone version from a REQUIREMENTS Phase cell such
|
||||
// as `v0.7 P1`, `**v0.2 P01**`, `v0.2 P01–P04`, or bare `v0.7`. The cell may
|
||||
@@ -40,6 +43,17 @@ type reqRow struct {
|
||||
status string // "Complete" or "Pending"
|
||||
}
|
||||
|
||||
// normalizeStatus canonicalizes a captured status token to the title-case
|
||||
// form ("Complete" or "Pending") so that case-insensitive matches like
|
||||
// "pending" or "complete" compare correctly against the drift assertions.
|
||||
func normalizeStatus(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + strings.ToLower(s[1:])
|
||||
}
|
||||
|
||||
// milestoneVersions returns the distinct v0.X milestones referenced in the
|
||||
// phase cell (e.g. "v0.7 P1" → ["v0.7"]; "v0.2 P01 / v0.3 P02" →
|
||||
// ["v0.2","v0.3"]).
|
||||
@@ -174,7 +188,7 @@ func parseRequirements(path string) ([]reqRow, error) {
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, reqRow{id: m[1], phase: strings.TrimSpace(m[2]), status: m[3]})
|
||||
rows = append(rows, reqRow{id: m[1], phase: strings.TrimSpace(m[2]), status: normalizeStatus(m[3])})
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, err
|
||||
|
||||
+1069
-126
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,52 @@
|
||||
# Orca Metrics Reference
|
||||
|
||||
Orca exposes Prometheus text-exposition metrics at `/metrics` on the
|
||||
metrics endpoint (default `:9100`, configurable via `--addr`).
|
||||
|
||||
## Running the metrics endpoint
|
||||
|
||||
```sh
|
||||
orca metrics --addr :9100
|
||||
```
|
||||
|
||||
## Prometheus scrape config
|
||||
|
||||
```yaml
|
||||
scrape_configs:
|
||||
- job_name: orca
|
||||
static_configs:
|
||||
- targets: ['localhost:9100']
|
||||
scrape_interval: 15s
|
||||
```
|
||||
|
||||
## Metric reference
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `nodes_total` | Gauge | Total number of registered nodes |
|
||||
| `allocs_total` | Gauge | Total number of job allocations |
|
||||
| `orca_jobs_by_state{state}` | Gauge | Jobs grouped by status (running, complete, failed, etc.) |
|
||||
| `orca_audit_chain_head` | Gauge | Audit chain integrity (1 = chain head verified, 0 = error) |
|
||||
|
||||
## Counter metrics (incremented by CLI operations)
|
||||
|
||||
The following counters are incremented during normal operations and
|
||||
are available when the metrics endpoint polls the DB:
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `orca_drift_events_total` | Counter | Total drift events detected |
|
||||
| `orca_ssh_errors_total` | Counter | Total SSH connection/exec errors |
|
||||
| `orca_txn_apply_total` | Counter | Total transaction applies |
|
||||
| `orca_txn_rollback_total` | Counter | Total transaction rollbacks |
|
||||
| `orca_acl_denials_total` | Counter | Total ACL denials (enforce mode) |
|
||||
|
||||
## Security headers
|
||||
|
||||
The metrics endpoint sets the following security headers on all responses:
|
||||
- `X-Content-Type-Options: nosniff`
|
||||
- `X-Frame-Options: DENY`
|
||||
|
||||
## Health check
|
||||
|
||||
The endpoint also exposes `/healthz` returning `200 ok` for liveness probes.
|
||||
+62
-13
@@ -8,8 +8,8 @@ directory holds cluster-wide artifacts shared across namespaces.
|
||||
|
||||
> **v0.9 layout (canonical)**: This document describes the v0.9
|
||||
> multi-namespace layout. The v0.8 flat layout (`orca.db`, `ca.crt`,
|
||||
> `server.crt` at the root) is deprecated and will be removed in
|
||||
> v0.11. See [v0.8 flat layout](#deprecated-v08-flat-layout) below.
|
||||
> `server.crt` at the root) is deprecated and removed in v0.12
|
||||
> (REQ-138).
|
||||
|
||||
## Namespace root resolution
|
||||
|
||||
@@ -51,12 +51,16 @@ $ORCA_HOME/
|
||||
├── cluster/ # cluster-wide (NOT a workload namespace)
|
||||
│ ├── ca.crt, ca.key # step-ca root (R-006, D-101)
|
||||
│ ├── master.key # AES-256-GCM root (R-011, mode 0600)
|
||||
│ ├── master.key.sealed # sealed master key (REQ-147, mode 0600)
|
||||
│ ├── config.md # Markdown frontmatter config (R-014)
|
||||
│ ├── known_hosts # SSH known_hosts (D-035)
|
||||
│ ├── orca_ssh_key # orca SSH private key (D-037)
|
||||
│ ├── orca_ssh_key.pub # orca SSH public key
|
||||
│ ├── peers/<host>/ # per-peer directory
|
||||
│ ├── txns/ # cluster transaction log (R-016)
|
||||
│ ├── acl.json # ACL state (mode 0600)
|
||||
│ ├── oidc-client-secret # OIDC client secret (mode 0600, C-36)
|
||||
│ ├── webauthn-credentials.db # WebAuthn public keys (mode 0600)
|
||||
│ └── state/ # cluster state
|
||||
├── _defaults/ # implicit root namespace (always exists)
|
||||
│ ├── ns.md # namespace frontmatter (kind: Namespace)
|
||||
@@ -80,14 +84,16 @@ $ORCA_HOME/
|
||||
exists. Every namespace inherits from `_defaults` and cannot opt out
|
||||
(D-185, D-187).
|
||||
- **`cluster/`** is NOT a workload namespace — it holds cluster-wide
|
||||
artifacts (CA, master key, SSH keys, known_hosts, peers, txns).
|
||||
artifacts (CA, master key, SSH keys, known_hosts, peers, txns, ACL,
|
||||
OIDC secrets, WebAuthn credentials).
|
||||
- **Per-namespace DBs**: each namespace has its own
|
||||
`db/orca.db` (R-002). No namespace column in SQLite.
|
||||
- **Namespace inheritance**: child namespaces inherit env and
|
||||
constraints from parents (via `ns.md` frontmatter `parents:` field).
|
||||
`_defaults` is always appended last in the inheritance chain.
|
||||
- **`orca ns` subcommands**: `list`, `create`, `delete`, `inspect`,
|
||||
`validate` — see [docs/cli.md](cli.md#orca-ns).
|
||||
`validate`, `inherit`, `set-constraint` — see below and
|
||||
[docs/cli.md](cli.md#orca-ns).
|
||||
|
||||
### Path reference (`internal/paths/`)
|
||||
|
||||
@@ -134,6 +140,50 @@ orca ns validate prod
|
||||
orca ns delete staging
|
||||
```
|
||||
|
||||
### `orca ns inherit` — set parent namespace (R-002)
|
||||
|
||||
Set the parent namespace for a namespace. Updates `ns.md` frontmatter
|
||||
(`parents` field) and validates the new chain has no cycles. The
|
||||
implicit root `_defaults` is always appended last (D-185).
|
||||
|
||||
```bash
|
||||
orca ns inherit <name> --parent <parent-namespace>
|
||||
```
|
||||
|
||||
**Example**:
|
||||
```bash
|
||||
# Make staging inherit from prod (chain: staging -> prod -> _defaults)
|
||||
orca ns inherit staging --parent prod
|
||||
```
|
||||
|
||||
The child cannot inherit from itself transitively — the resolver
|
||||
validates the chain before writing. If a cycle is detected, the
|
||||
command exits 1 with an error.
|
||||
|
||||
### `orca ns set-constraint` — set a constraint (R-002)
|
||||
|
||||
Set a constraint on a namespace. Constraints are `key=value` strings
|
||||
(e.g., `max-allocs=10`) stored in `ns.md` frontmatter and unioned
|
||||
across the inheritance chain by the resolver.
|
||||
|
||||
```bash
|
||||
orca ns set-constraint <name> <key>=<value>
|
||||
```
|
||||
|
||||
**Example**:
|
||||
```bash
|
||||
# Limit prod to 10 concurrent allocations
|
||||
orca ns set-constraint prod max-allocs=10
|
||||
|
||||
# Set a required node affinity
|
||||
orca ns set-constraint prod require-label=ssd
|
||||
```
|
||||
|
||||
Constraints are unioned (not overridden) across the inheritance chain:
|
||||
if `_defaults` sets `max-allocs=50` and `prod` sets `max-allocs=10`,
|
||||
the effective constraint is the most restrictive one (CEL evaluation
|
||||
determines precedence per constraint key).
|
||||
|
||||
See [docs/cli.md](cli.md#orca-ns) for the full `orca ns` reference.
|
||||
|
||||
## `ORCA_DB` override
|
||||
@@ -148,11 +198,10 @@ orca init # uses /tmp/test.db for the DB, ~/.orca/ for everything else
|
||||
|
||||
## Deprecated: v0.8 flat layout
|
||||
|
||||
> **Deprecated in v0.9**: The v0.8 flat layout (`orca.db`, `ca.crt`,
|
||||
> `ca.key`, `server.crt`, `server.key` at the namespace root) is
|
||||
> superseded by the v0.9 multi-namespace layout (R-002). The v0.8
|
||||
> layout is supported during the dual-write window via
|
||||
> `internal/certpaths` (a thin shim) and will be removed in v0.11.
|
||||
> **Removed in v0.12** (REQ-138): The v0.8 flat layout (`orca.db`,
|
||||
> `ca.crt`, `ca.key`, `server.crt`, `server.key` at the namespace root)
|
||||
> is superseded by the v0.9 multi-namespace layout (R-002) and the
|
||||
> dual-write window is closed.
|
||||
|
||||
The v0.8 flat layout stored all state at the namespace root:
|
||||
|
||||
@@ -166,12 +215,12 @@ The v0.8 flat layout stored all state at the namespace root:
|
||||
|
||||
The v0.9 re-architecture moved these to `cluster/` (CA, SSH keys) and
|
||||
per-namespace `db/` (SQLite) to support multi-tenancy (R-002). The
|
||||
`orca doctor --legacy-paths` command (v0.11-P14c) will detect v0.8
|
||||
residue and recommend migration.
|
||||
`internal/certpaths` shim that supported the dual-write window is
|
||||
removed in v0.12.
|
||||
|
||||
## See also
|
||||
|
||||
- [Install Guide](install.md) — 1-liner install with `install.sh`.
|
||||
- [Docker Guide](docker.md) — running orca in a container.
|
||||
- [CLI Reference](cli.md) — `orca ns` subcommands.
|
||||
- [Jobspec Reference](jobspec.md) — markdown frontmatter schema.
|
||||
- [CLI Reference](cli.md#orca-ns) — `orca ns` subcommands.
|
||||
- [Jobspec Reference](jobspec.md) — markdown frontmatter schema.
|
||||
|
||||
+138
-21
@@ -1,31 +1,148 @@
|
||||
# Security Runbook (v0.12)
|
||||
# Security Runbook (v0.13)
|
||||
|
||||
## Master Key Seal/Unseal
|
||||
This runbook documents the operational security procedures for orca's
|
||||
zero-trust identity model (R-021): human identity is exclusively
|
||||
external (OIDC), machine identity is exclusively mTLS/SPIFFE, and no
|
||||
passwords / Orca-issued tokens / CA-key passphrases exist anywhere in
|
||||
the system. The v0.12 milestone shipped these capabilities; the v0.13
|
||||
milestone wired them operationally (R-023).
|
||||
|
||||
- `orca cluster seal`: encrypts master key with OIDC-derived key;
|
||||
prints 5 Shamir shards for offline recovery.
|
||||
- `orca cluster unseal`: operator authenticates via OIDC; master key
|
||||
unwrapped into memory; zeroed on shutdown.
|
||||
- `orca cluster unseal --recovery`: if IdP lost, present 3 of 5 shards.
|
||||
## Master Key Seal/Unseal (REQ-147, P05)
|
||||
|
||||
## Master Key Rotation
|
||||
The cluster master key (`ClusterDir()/master.key`, mode 0600) encrypts
|
||||
all namespace `.env.secrets` via per-namespace HKDF-SHA256 sub-keys
|
||||
(AES-256-GCM). The master key can be **sealed** (encrypted at rest) and
|
||||
**unsealed** (unwrapped into memory for use).
|
||||
|
||||
`orca secrets rotate-master [--dry-run]`: generates new master key,
|
||||
re-encrypts all namespace secrets, re-seals. Atomic + automatic rollback.
|
||||
### Seal
|
||||
|
||||
```bash
|
||||
orca cluster seal
|
||||
```
|
||||
|
||||
Encrypts the raw master key with a key derived from either:
|
||||
- the OIDC ID token subject (if `orca auth login` has been run), or
|
||||
- the cluster CA fingerprint (mTLS-only offline path, D-241).
|
||||
|
||||
The sealed blob is written to `ClusterDir()/master.key.sealed` (0600).
|
||||
**Five Shamir shards (3-of-5 recovery)** are printed to stdout — store
|
||||
them offline. The raw master key is then deleted from disk so the
|
||||
cluster is sealed at rest.
|
||||
|
||||
### Unseal
|
||||
|
||||
```bash
|
||||
orca cluster unseal
|
||||
```
|
||||
|
||||
Reads the sealed blob and unwraps the master key using the OIDC ID
|
||||
token subject or the cluster CA fingerprint. The unwrapped key is
|
||||
written back to `ClusterDir()/master.key` (0600) and zeroed from
|
||||
memory on process exit.
|
||||
|
||||
### Recovery (IdP lost)
|
||||
|
||||
```bash
|
||||
orca cluster unseal --recovery
|
||||
```
|
||||
|
||||
If the IdP is permanently lost, the operator is prompted for 3 of the
|
||||
5 Shamir shards printed at seal time. With quorum, the master key is
|
||||
reconstructed and written back to disk. If quorum is unavailable, the
|
||||
cluster is unrecoverable by design (C-35: no backdoor).
|
||||
|
||||
## Master Key Rotation (REQ-129, C-30)
|
||||
|
||||
```bash
|
||||
orca secrets rotate-master [--dry-run]
|
||||
```
|
||||
|
||||
Generates a new master key, re-encrypts every namespace's
|
||||
`.env.secrets` under the new key, and re-seals the master key to OIDC.
|
||||
With `--dry-run`, reports affected namespaces without writing.
|
||||
|
||||
- **Atomic per-namespace**: each namespace is re-encrypted independently.
|
||||
- **Automatic rollback**: on any namespace failure, the old sealed key
|
||||
is restored (C-30).
|
||||
- **No passphrase** (R-021): the master key is sealed to OIDC, not to a
|
||||
human-typed passphrase.
|
||||
|
||||
## File-Mode Audit (REQ-033, REQ-130, F13)
|
||||
|
||||
```bash
|
||||
orca doctor modes
|
||||
```
|
||||
|
||||
Verifies file modes on security-sensitive files across `ORCA_HOME`:
|
||||
- private keys / secrets: `0600`
|
||||
- certs / public keys: `0644`
|
||||
|
||||
Exits 0 if all files have correct modes; exits 1 if any violation is
|
||||
found. Missing files are not counted as violations.
|
||||
|
||||
Checks: SSH key, master key (sealed blob), server cert/key,
|
||||
known_hosts, `acl.json`, OIDC client secret.
|
||||
|
||||
## Audit Log Tamper-Evidence (REQ-125, F2)
|
||||
|
||||
```bash
|
||||
orca doctor audit
|
||||
```
|
||||
|
||||
Verifies the audit log hash chain. Opens the orca SQLite DB, recomputes
|
||||
the hash chain from the first audit entry, and reports the chain head
|
||||
hash. If any entry's `entry_hash` or `prev_hash` link does not match the
|
||||
recomputed value, the chain has been tampered with and the command
|
||||
exits non-zero.
|
||||
|
||||
The audit log is append-only (SQLite trigger blocks
|
||||
UPDATE/DELETE). Each entry's `actor` field carries the OIDC `sub` or
|
||||
SPIFFE SVID. Run this after any suspected intrusion or as part of a
|
||||
regular audit cadence.
|
||||
|
||||
## Sudoers Audit (REQ-134, F22)
|
||||
|
||||
```bash
|
||||
orca doctor proxmox
|
||||
```
|
||||
|
||||
Audits the `/etc/sudoers.d/orca` file against the expected allowlist:
|
||||
- `pct` + `qm` with NOEXEC
|
||||
- `apt-get` / `dpkg` excluded (or NOEXEC'd)
|
||||
- `pvesh` EXCLUDED (AD-020: pvesh can bypass NOEXEC via the API execute
|
||||
endpoint)
|
||||
|
||||
## nft Audit (REQ-133, F21)
|
||||
|
||||
```bash
|
||||
orca doctor nft
|
||||
```
|
||||
|
||||
Audits the live nftables ingress ruleset against the on-disk
|
||||
`/etc/nftables.d/orca.nft` hash (recorded at the latest applied txn).
|
||||
Reports drift if the live ruleset does not match. Also verifies:
|
||||
- table exists
|
||||
- DNAT `:443 → 127.0.0.1:8443` and `:80 → 127.0.0.1:8080` present
|
||||
- rate-limit meter present
|
||||
- `/etc/nftables.d/orca.nft` parses
|
||||
|
||||
## Incident Response
|
||||
|
||||
1. Revoke the compromised identity (OIDC user/group or SPIFFE SVID).
|
||||
2. Rotate the master key (`orca secrets rotate-master`).
|
||||
3. Review the audit log (`orca doctor audit` verifies the hash chain).
|
||||
4. If the master key is compromised, all historical secrets are
|
||||
compromised (no forward secrecy).
|
||||
1. **Revoke the compromised identity** (OIDC user/group or SPIFFE SVID).
|
||||
2. **Rotate the master key** (`orca secrets rotate-master`).
|
||||
3. **Review the audit log** (`orca doctor audit` verifies the hash
|
||||
chain; `orca audit list` shows entries).
|
||||
4. **Check file modes** (`orca doctor modes` detects permission drift).
|
||||
5. If the master key is compromised, **all historical secrets are
|
||||
compromised** (no forward secrecy — documented residual risk).
|
||||
6. **Re-seal** the master key after rotation (`orca cluster seal`).
|
||||
|
||||
## Sudoers Audit
|
||||
## OIDC Provider Health (P06)
|
||||
|
||||
`orca doctor proxmox` audits the `/etc/sudoers.d/orca` file against the
|
||||
expected allowlist (pct + qm with NOEXEC; apt-get/dpkg excluded).
|
||||
```bash
|
||||
orca doctor oidc
|
||||
```
|
||||
|
||||
## nft Audit
|
||||
|
||||
`orca doctor nft` audits the live nftables ruleset against the emitted one.
|
||||
Checks the bundled Dex OIDC provider health. Verifies the Dex systemd
|
||||
unit is running and the `/.well-known/openid-configuration` endpoint
|
||||
responds. Run after `orca auth init-idp` or after a Dex config change.
|
||||
|
||||
+319
@@ -0,0 +1,319 @@
|
||||
# Orca User Acceptance Testing (UAT) Plan
|
||||
|
||||
**Version**: v0.13 (production hardening round 2)
|
||||
**Gate**: v1.0.0 production-ready tag is deferred until this UAT passes
|
||||
**Signoff**: run `scripts/uat-signoff.sh` on the lead node and paste the output back
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Hardware
|
||||
|
||||
| Role | OS | Requirements |
|
||||
|------|-----|-------------|
|
||||
| **lead** | Ubuntu 22.04 LTS | Operator laptop or VM; SSH key; `orca` binary (built from v0.13 tag) |
|
||||
| **pve01** | Proxmox VE 8/9 | Bare-metal or nested; SSH root access; orca SSH key pre-staged |
|
||||
| **worker01** | Ubuntu 22.04 LTS | VM or bare-metal; SSH root access; orca SSH key pre-staged |
|
||||
|
||||
### Alternative topology (3x Ubuntu, no Proxmox)
|
||||
|
||||
If a Proxmox host is unavailable, run the UAT with 3x Ubuntu hosts.
|
||||
Use `--type linux` for all remote nodes. Proxmox-specific claims
|
||||
(`doctor proxmox`, PVE role, sudoers) are **skipped** in this path.
|
||||
The signoff script reports exercised vs. skipped claims.
|
||||
|
||||
### Pre-staging
|
||||
|
||||
1. Build orca from the v0.13 tag:
|
||||
```sh
|
||||
git clone https://git.cloudinit.dev/coreci/orca.git
|
||||
cd orca && git checkout v0.12.13
|
||||
make build
|
||||
# binary is at bin/orca
|
||||
```
|
||||
|
||||
2. Generate the orca SSH keypair on the lead:
|
||||
```sh
|
||||
ssh-keygen -t ed25519 -f ~/.ssh/orca_ed25519 -N ""
|
||||
```
|
||||
|
||||
3. Pre-stage the orca public key on pve01 and worker01:
|
||||
```sh
|
||||
ssh-copy-id -i ~/.ssh/orca_ed25519.pub root@pve01
|
||||
ssh-copy-id -i ~/.ssh/orca_ed25519.pub root@worker01
|
||||
```
|
||||
|
||||
4. Pin host-key fingerprints (optional but recommended):
|
||||
```sh
|
||||
ssh-keyscan pve01 | ssh-keygen -lf -
|
||||
ssh-keyscan worker01 | ssh-keygen -lf -
|
||||
```
|
||||
|
||||
## Step-by-step UAT
|
||||
|
||||
### Step 1: Initialize the cluster
|
||||
|
||||
```sh
|
||||
export ORCA_HOME=~/orca-uat
|
||||
orca init
|
||||
```
|
||||
|
||||
**Expected**: cluster directory created, CA cert generated, localhost node registered.
|
||||
|
||||
### Step 2: Onboard the Proxmox host
|
||||
|
||||
```sh
|
||||
orca node join --type proxmox \
|
||||
--host pve01 \
|
||||
--ssh-user root \
|
||||
--ssh-key ~/.ssh/orca_ed25519 \
|
||||
--host-key-fingerprint SHA256:<fingerprint>
|
||||
```
|
||||
|
||||
**Expected**: SSH bootstrap succeeds, orca user created, PVE role assigned, node registered as `ready` with `kind=proxmox`.
|
||||
|
||||
### Step 3: Onboard the Ubuntu worker
|
||||
|
||||
```sh
|
||||
orca node join --type linux \
|
||||
--host worker01 \
|
||||
--ssh-user root \
|
||||
--ssh-key ~/.ssh/orca_ed25519 \
|
||||
--host-key-fingerprint SHA256:<fingerprint>
|
||||
```
|
||||
|
||||
**Expected**: SSH bootstrap succeeds, orca user created, drift-events dir created, node registered as `ready` with `kind=linux`.
|
||||
|
||||
### Step 4: Verify nodes
|
||||
|
||||
```sh
|
||||
orca node list
|
||||
orca node list --json
|
||||
```
|
||||
|
||||
**Expected**: 3 nodes listed (localhost + pve01 + worker01), all `ready`.
|
||||
|
||||
### Step 5: Set capacity on remote nodes
|
||||
|
||||
```sh
|
||||
orca node capacity set --node pve01 --cpu 4 --memory 8192 --disk 100000
|
||||
orca node capacity set --node worker01 --cpu 2 --memory 4096 --disk 50000
|
||||
orca node capacity list
|
||||
```
|
||||
|
||||
**Expected**: capacity shown for both remote nodes.
|
||||
|
||||
### Step 6: Create a namespace
|
||||
|
||||
```sh
|
||||
orca ns create prod
|
||||
orca ns list
|
||||
```
|
||||
|
||||
**Expected**: `prod` namespace listed.
|
||||
|
||||
### Step 7: Deploy the full stack
|
||||
|
||||
Deploy each service from `examples/full-stack/`:
|
||||
|
||||
```sh
|
||||
orca job run examples/full-stack/web-app.md --target pve01
|
||||
orca job run examples/full-stack/api.md --target pve01
|
||||
orca job run examples/full-stack/worker.md --target worker01
|
||||
orca job run examples/full-stack/postgres.md --target pve01
|
||||
orca job run examples/full-stack/log-shipper.md --target worker01
|
||||
```
|
||||
|
||||
**Expected**: each job is scheduled on the target, systemd unit deployed via SSH-push, job status `running` or `complete`.
|
||||
|
||||
### Step 8: Verify deployment
|
||||
|
||||
```sh
|
||||
orca job list
|
||||
orca job list --json
|
||||
```
|
||||
|
||||
**Expected**: all 5 jobs listed, with correct target nodes.
|
||||
|
||||
On each remote node:
|
||||
```sh
|
||||
ssh root@pve01 systemctl status 'orca-alloc-*'
|
||||
ssh root@worker01 systemctl status 'orca-alloc-*'
|
||||
```
|
||||
|
||||
### Step 9: Verify Traefik routes
|
||||
|
||||
```sh
|
||||
ssh root@pve01 ls /etc/traefik/dynamic/
|
||||
ssh root@worker01 ls /etc/traefik/dynamic/
|
||||
```
|
||||
|
||||
**Expected**: `traefik-dynamic-*.yaml` files present on nodes where jobs were deployed.
|
||||
|
||||
### Step 10: Migrate between hosts
|
||||
|
||||
Migrate `web-app` from pve01 to worker01:
|
||||
|
||||
```sh
|
||||
orca job migrate web-app --to worker01
|
||||
```
|
||||
|
||||
**Expected**: job drained on pve01, rescheduled on worker01, new systemd unit deployed.
|
||||
|
||||
Verify:
|
||||
```sh
|
||||
orca job list
|
||||
ssh root@worker01 systemctl status 'orca-alloc-*web-app*'
|
||||
ssh root@pve01 systemctl status 'orca-alloc-*web-app*' # should be stopped
|
||||
```
|
||||
|
||||
### Step 11: Aggregate logs
|
||||
|
||||
```sh
|
||||
orca logs --all-nodes --job web-app --since 5m
|
||||
```
|
||||
|
||||
**Expected**: log entries from multiple nodes.
|
||||
|
||||
### Step 12: ACL enforcement
|
||||
|
||||
```sh
|
||||
orca acl grant operator-1 --namespace prod --permissions read,write
|
||||
orca acl check operator-1 --namespace prod --permission read
|
||||
orca acl check operator-1 --namespace prod --permission admin
|
||||
```
|
||||
|
||||
**Expected**: read+write allowed, admin denied (not granted).
|
||||
|
||||
### Step 13: Seal/unseal
|
||||
|
||||
```sh
|
||||
orca cluster seal --rp-id orca.local
|
||||
orca cluster unseal
|
||||
orca secrets set prod TEST_KEY --value "test-value"
|
||||
orca secrets get prod TEST_KEY
|
||||
```
|
||||
|
||||
**Expected**: seal succeeds, unseal succeeds, secrets readable post-unseal.
|
||||
|
||||
### Step 14: Audit chain
|
||||
|
||||
```sh
|
||||
orca doctor audit
|
||||
```
|
||||
|
||||
**Expected**: chain head reported, no tamper detected.
|
||||
|
||||
### Step 15: Doctor modes
|
||||
|
||||
```sh
|
||||
orca doctor modes
|
||||
```
|
||||
|
||||
**Expected**: all file modes correct, exit 0.
|
||||
|
||||
### Step 16: OIDC health
|
||||
|
||||
```sh
|
||||
orca doctor oidc
|
||||
```
|
||||
|
||||
**Expected**: Dex unit active, issuer reachable (or WARN if Dex not installed).
|
||||
|
||||
### Step 17: Backup and restore
|
||||
|
||||
```sh
|
||||
orca backup --out /tmp/uat-backup.tar.gz
|
||||
orca restore --in /tmp/uat-backup.tar.gz --dry-run
|
||||
```
|
||||
|
||||
**Expected**: backup succeeds, restore dry-run succeeds.
|
||||
|
||||
### Step 18: Drift detection
|
||||
|
||||
```sh
|
||||
orca drift show
|
||||
```
|
||||
|
||||
**Expected**: no error (empty drift is fine).
|
||||
|
||||
### Step 19: Transaction idempotency
|
||||
|
||||
```sh
|
||||
orca txn apply <some-txn-dir>
|
||||
orca txn apply <some-txn-dir> # re-run
|
||||
```
|
||||
|
||||
**Expected**: second apply is idempotent (exit 5 or "already applied").
|
||||
|
||||
### Step 20: Metrics
|
||||
|
||||
```sh
|
||||
orca metrics --addr :9100 &
|
||||
sleep 3
|
||||
curl -s http://localhost:9100/metrics | grep orca_
|
||||
```
|
||||
|
||||
**Expected**: expanded metric set present (`orca_jobs_running`, `orca_audit_chain_head`, etc.).
|
||||
|
||||
### Step 21: Compat check
|
||||
|
||||
```sh
|
||||
orca cluster compat-check
|
||||
```
|
||||
|
||||
**Expected**: exit 0, all nodes compatible.
|
||||
|
||||
### Step 22: Run the signoff script
|
||||
|
||||
```sh
|
||||
scripts/uat-signoff.sh
|
||||
```
|
||||
|
||||
**Expected**: `UAT SIGNOFF: N/35 assertions passed`, exit 0 iff N==35.
|
||||
|
||||
## Claim Matrix
|
||||
|
||||
| # | Claim | UAT Step | Signoff Assertion |
|
||||
|---|-------|----------|-------------------|
|
||||
| 1 | Cluster initializes from scratch | Step 1 | `assert_orca_version` |
|
||||
| 2 | Proxmox host onboards via SSH | Step 2 | `assert_proxmox_onboarded` |
|
||||
| 3 | Ubuntu worker onboards via `--type linux` | Step 3 | `assert_linux_worker_onboarded` |
|
||||
| 4 | Node list shows all nodes | Step 4 | `assert_cluster_initialized` |
|
||||
| 5 | Capacity is set on remote nodes | Step 5 | `assert_capacity_set` |
|
||||
| 6 | Namespace created | Step 6 | `assert_namespace_created` |
|
||||
| 7 | Full stack deploys to remote nodes | Step 7 | `assert_full_stack_running` |
|
||||
| 8 | Scheduler deploys to remote (not local) | Step 7 | `assert_job_deploys_to_remote` |
|
||||
| 9 | Traefik routes present | Step 9 | `assert_traefik_routes` |
|
||||
| 10 | Job migrates between hosts | Step 10 | `assert_migrate_worked` |
|
||||
| 11 | Logs aggregate from multiple nodes | Step 11 | `assert_logs_aggregate` |
|
||||
| 12 | ACL grant/check works | Step 12 | `assert_acl_enforced` |
|
||||
| 13 | ACL deny-by-default | Step 12 | `assert_acl_deny_default` |
|
||||
| 14 | acl.json mode 0600 | Step 12 | `assert_acl_file_mode` |
|
||||
| 15 | Seal/unseal round-trip | Step 13 | `assert_seal_unseal_roundtrip` |
|
||||
| 16 | Audit chain intact | Step 14 | `assert_audit_chain_intact` |
|
||||
| 17 | Doctor modes passes | Step 15 | `assert_doctor_modes` |
|
||||
| 18 | OIDC health check | Step 16 | `assert_oidc_health` |
|
||||
| 19 | Backup works | Step 17 | `assert_backup_restore_dryrun` |
|
||||
| 20 | Drift visible | Step 18 | `assert_drift_visible` |
|
||||
| 21 | Txn idempotent | Step 19 | `assert_txn_idempotent` |
|
||||
| 22 | Metrics expanded | Step 20 | `assert_metrics_expanded` |
|
||||
| 23 | Compat check passes | Step 21 | `assert_compat_check_passes` |
|
||||
| 24 | No `--password` in docs/examples | — | `assert_no_password_in_docs` |
|
||||
| 25 | Go toolchain current | — | `assert_go_toolchain_current` |
|
||||
| 26 | cli.md matches `orca --help` | — | `assert_cli_md_complete` |
|
||||
| 27 | pprof not on all interfaces | — | `assert_no_pprof_on_all_interfaces` |
|
||||
| 28 | WebAuthn registration requires auth | — | `assert_webauthn_reg_requires_auth` |
|
||||
| 29 | Audit chain survives concurrency | — | `assert_audit_chain_concurrent` |
|
||||
| 30 | Concurrent secrets no data loss | — | `assert_concurrent_secrets_no_loss` |
|
||||
| 31 | Cache invalidated after write | — | `assert_cache_invalidated_after_write` |
|
||||
| 32 | SQLite no lock under concurrency | — | `assert_sqlite_no_lock` |
|
||||
| 33 | No injection in logs --job | — | `assert_no_injection_in_logs` |
|
||||
| 34 | `--type linux` exists as subcommand | Step 3 | `assert_type_linux_available` |
|
||||
| 35 | `orca status` deprecated | — | `assert_status_deprecated` |
|
||||
|
||||
## Signoff procedure
|
||||
|
||||
1. Run all steps above on the 3-host cluster
|
||||
2. Run `scripts/uat-signoff.sh` on the lead
|
||||
3. Paste the output back to the CI agent
|
||||
4. The CI agent verifies `35/35 PASS` and cuts `v1.0.0`
|
||||
+84
-14
@@ -1,27 +1,97 @@
|
||||
# WebAuthn / Passkeys (v0.12)
|
||||
# WebAuthn / Passkeys (v0.13)
|
||||
|
||||
## Overview
|
||||
|
||||
The bundled Dex uses a custom WebAuthn connector for password-free
|
||||
authentication. Passkeys are public-key credentials — the private key
|
||||
never leaves the authenticator (TPM/security key/phone Secure Enclave).
|
||||
The bundled Dex uses a custom WebAuthn connector (`orca-webauthn-connector`,
|
||||
REQ-148) for password-free authentication. Passkeys are public-key
|
||||
credentials — the private key never leaves the authenticator (TPM /
|
||||
security key / phone Secure Enclave). This directly satisfies R-021
|
||||
(no Orca-issued credentials): the authenticator proves possession of
|
||||
the private key without ever exposing it.
|
||||
|
||||
The WebAuthn connector ships as part of the v0.12 milestone (P05) and
|
||||
is operationally wired in v0.13 (P04: registration requires auth; P06:
|
||||
real Dex deployment).
|
||||
|
||||
## Registration
|
||||
|
||||
`orca auth register` opens the browser to the Dex WebAuthn endpoint.
|
||||
After the ceremony (biometric/security key), Dex maps the credential
|
||||
ID to an OIDC `sub`. Credentials stored at
|
||||
`ClusterDir()/webauthn-credentials.db` (0600, public keys only).
|
||||
```bash
|
||||
orca auth register [--no-browser]
|
||||
```
|
||||
|
||||
Opens the browser to the Dex WebAuthn registration page at
|
||||
`https://<cluster>/orca/webauthn/register`. The operator authenticates
|
||||
via an existing session or admin bootstrap token, then performs the
|
||||
WebAuthn ceremony (biometric or security key). After the ceremony,
|
||||
Dex maps the credential ID to an OIDC `sub`.
|
||||
|
||||
- **`--no-browser`**: print the registration URL instead of opening a
|
||||
browser (useful for headless operators or remote SSH sessions — copy
|
||||
the URL into a local browser).
|
||||
|
||||
Credentials are stored at `ClusterDir()/webauthn-credentials.db`
|
||||
(mode 0600, public keys only — private keys never leave the
|
||||
authenticator and are never stored by orca).
|
||||
|
||||
**Example**:
|
||||
```bash
|
||||
# Interactive (opens browser)
|
||||
orca auth register
|
||||
|
||||
# Headless / remote SSH (print URL)
|
||||
orca auth register --no-browser
|
||||
# → https://orca.local/orca/webauthn/register
|
||||
```
|
||||
|
||||
## RP ID
|
||||
|
||||
The relying-party ID is the cluster's Traefik-served domain
|
||||
(`--rp-id` on `orca auth init-idp`). HTTPS secure context is provided
|
||||
by Traefik (step-ca cert, R-017).
|
||||
The relying-party ID is the cluster's Traefik-served domain, set via
|
||||
`--rp-id` on `orca auth init-idp` (C-38). The RP ID **must** match the
|
||||
cluster's Traefik domain — WebAuthn enforces that the RP ID is a
|
||||
registrable domain suffix of the current origin.
|
||||
|
||||
HTTPS secure context is provided by Traefik (step-ca cert, R-017).
|
||||
WebAuthn requires a secure context (HTTPS or localhost); the step-ca
|
||||
cert behind Traefik satisfies this.
|
||||
|
||||
## Bootstrap Sequence
|
||||
|
||||
1. `orca init` bootstraps the cluster CA (step-ca, mTLS-only).
|
||||
2. `orca auth init-idp` deploys Dex behind Traefik (step-ca cert).
|
||||
3. First operator registers a passkey via the mTLS-authenticated session.
|
||||
4. Subsequent operators use WebAuthn.
|
||||
2. `orca auth init-idp --rp-id <cluster-domain>` deploys Dex behind
|
||||
Traefik (step-ca cert) with the WebAuthn connector configured.
|
||||
3. First operator authenticates via an existing session or admin
|
||||
bootstrap token, then registers a passkey:
|
||||
|
||||
```bash
|
||||
orca auth register
|
||||
```
|
||||
|
||||
4. Subsequent operators use WebAuthn login (`orca auth login` opens
|
||||
the browser to the Dex login page; the WebAuthn ceremony is one of
|
||||
the available upstreams).
|
||||
|
||||
## Health Check
|
||||
|
||||
```bash
|
||||
orca doctor oidc
|
||||
```
|
||||
|
||||
Verifies the bundled Dex OIDC provider is running and the
|
||||
`/.well-known/openid-configuration` endpoint responds. Run after
|
||||
`orca auth init-idp` or after a Dex config change.
|
||||
|
||||
## Security properties
|
||||
|
||||
- **No passwords**: WebAuthn is password-free. No password is ever
|
||||
sent to or stored by orca (R-021).
|
||||
- **Phishing-resistant**: the WebAuthn protocol cryptographically binds
|
||||
the ceremony to the RP ID, defeating credential phishing.
|
||||
- **Private key never leaves the authenticator**: orca stores only
|
||||
public keys.
|
||||
- **Secure context required**: HTTPS via step-ca / Traefik (C-38).
|
||||
|
||||
## See also
|
||||
|
||||
- [docs/oidc.md](oidc.md) — OIDC configuration (Dex quickstart, BYO IdP)
|
||||
- [docs/security-runbook.md](security-runbook.md) — security runbook
|
||||
- [docs/cli.md](cli.md#orca-auth) — `orca auth` CLI reference
|
||||
|
||||
@@ -62,7 +62,7 @@ a localhost node.
|
||||
orca node join --type proxmox --host 192.168.1.100 --ssh-user root
|
||||
|
||||
# Join a second node
|
||||
ORCA_PROXMOX_PASSWORD=secret orca node join --type proxmox --host 192.168.1.101
|
||||
orca node join --type proxmox --host 192.168.1.101 --ssh-key ~/.ssh/orca_ed25519
|
||||
```
|
||||
|
||||
### Step 3: Declare node capacity
|
||||
|
||||
Vendored
+5
@@ -73,6 +73,11 @@ func Open(path string) (*Cache, error) {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("ping cache sqlite: %w", err)
|
||||
}
|
||||
// REQ-158 / P09 T4: enforce 0600 on the cache DB file (SQLite
|
||||
// creates it at umask, typically 0644). Match store.Open which
|
||||
// chmods after open+ping (the file exists at this point). Non-fatal
|
||||
// if chmod fails (e.g. the DB is at a path we don't own).
|
||||
_ = os.Chmod(path, 0o600)
|
||||
const schema = `CREATE TABLE IF NOT EXISTS cache_entries (
|
||||
class TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
|
||||
Vendored
+47
@@ -2,6 +2,7 @@ package cache
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -225,3 +226,49 @@ func BenchmarkCacheHit(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestCache_FileMode0600 verifies that the cache DB file is created
|
||||
// with mode 0600 (not the default umask 0644) (REQ-158, P09 T4).
|
||||
func TestCache_FileMode0600(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "orca_cache.db")
|
||||
c, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat cache db: %v", err)
|
||||
}
|
||||
got := info.Mode().Perm()
|
||||
if got != 0o600 {
|
||||
t.Errorf("cache db mode = %04o, want 0600", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCache_FileMode0600DefaultPath verifies that the cache DB at the
|
||||
// default path (ORCA_HOME) also gets 0600 (REQ-158, P09 T4).
|
||||
func TestCache_FileMode0600DefaultPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_HOME", dir)
|
||||
c, err := Open("")
|
||||
if err != nil {
|
||||
t.Fatalf("open default path: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
// The default path is paths.CacheDB() which is under ORCA_HOME.
|
||||
// Find the db file.
|
||||
dbPath := filepath.Join(dir, "orca_cache.db")
|
||||
info, err := os.Stat(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("stat cache db at %s: %v", dbPath, err)
|
||||
}
|
||||
got := info.Mode().Perm()
|
||||
if got != 0o600 {
|
||||
t.Errorf("cache db mode = %04o, want 0600", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ var (
|
||||
aclRevokeNamespace string
|
||||
aclCheckNamespace string
|
||||
aclCheckPermission string
|
||||
aclCheckVerbose bool
|
||||
)
|
||||
|
||||
var aclCmd = &cobra.Command{
|
||||
@@ -351,6 +352,16 @@ read, write, admin (default: read).`,
|
||||
}
|
||||
identity.Namespace = ns
|
||||
allowed := a.Check(identity, ns, perm)
|
||||
if aclCheckVerbose {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "ACL path: %s\n", paths.ACLPath())
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "Identity: kind=%s id=%s ns=%s\n", identity.Kind, identity.ID, ns)
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "Permission: %s -> allowed=%v\n", permStr, allowed)
|
||||
entries := a.List()
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "ACL entries (%d):\n", len(entries))
|
||||
for _, e := range entries {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), " kind=%s id=%s ns=%s perms=%d\n", e.Identity.Kind, e.Identity.ID, e.Namespace, e.Permissions)
|
||||
}
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]any{
|
||||
"identity": identity,
|
||||
@@ -373,6 +384,7 @@ func init() {
|
||||
aclGrantCmd.Flags().StringVar(&aclGrantPermissions, "permissions", "read", "comma-separated permissions: read,write,admin")
|
||||
aclRevokeCmd.Flags().StringVar(&aclRevokeNamespace, "namespace", "", "namespace scope (required for tokens; defaults to spiffe path ns)")
|
||||
aclCheckCmd.Flags().StringVar(&aclCheckNamespace, "namespace", "", "namespace scope (required for tokens; defaults to spiffe path ns)")
|
||||
aclCheckCmd.Flags().BoolVar(&aclCheckVerbose, "verbose", false, "print ACL path + loaded entries for debugging")
|
||||
aclCheckCmd.Flags().StringVar(&aclCheckPermission, "permission", "read", "permission to check: read, write, or admin")
|
||||
|
||||
aclCmd.AddCommand(aclGrantCmd)
|
||||
|
||||
+75
-1
@@ -384,7 +384,81 @@ func checkOIDCHealth(ctx context.Context) []oidcCheckResult {
|
||||
return results
|
||||
}
|
||||
|
||||
// doctorDBRetentionCmd implements `orca doctor db-retention` (REQ-158,
|
||||
// P09 T2). Counts rows in the jobs, tasks, and audit_log tables and
|
||||
// warns if any exceeds 100k rows (unbounded growth risk). Suggests
|
||||
// `orca backup` + manual cleanup.
|
||||
var doctorDBRetentionCmd = &cobra.Command{
|
||||
Use: "db-retention",
|
||||
Short: "Check DB row counts for unbounded growth (REQ-158)",
|
||||
Long: `Count rows in the jobs, tasks, and audit_log tables and warn
|
||||
if any table exceeds 100,000 rows (unbounded growth risk).
|
||||
|
||||
Large tables degrade query performance and inflate backup size. Run
|
||||
'orca backup' to capture a snapshot, then prune old rows manually
|
||||
(e.g. DELETE FROM tasks WHERE created_at < <cutoff>).
|
||||
|
||||
Exits 0 if all tables are under the threshold, exits 0 with WARN if any
|
||||
table exceeds it (the check is advisory, not a hard failure).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
db, closer, err := openDB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("doctor db-retention: open db: %w", err)
|
||||
}
|
||||
defer closer()
|
||||
|
||||
tables := []string{"jobs", "tasks", "audit_log"}
|
||||
const threshold = 100_000
|
||||
type rowCount struct {
|
||||
Table string `json:"table"`
|
||||
Count int64 `json:"count"`
|
||||
Warn bool `json:"warn"`
|
||||
}
|
||||
var results []rowCount
|
||||
anyWarn := false
|
||||
for _, table := range tables {
|
||||
var count int64
|
||||
q := fmt.Sprintf("SELECT COUNT(*) FROM %s", table)
|
||||
if err := db.QueryRowContext(ctx, q).Scan(&count); err != nil {
|
||||
return fmt.Errorf("doctor db-retention: count %s: %w", table, err)
|
||||
}
|
||||
warn := count > threshold
|
||||
if warn {
|
||||
anyWarn = true
|
||||
}
|
||||
results = append(results, rowCount{Table: table, Count: count, Warn: warn})
|
||||
}
|
||||
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]any{
|
||||
"results": results,
|
||||
"threshold": threshold,
|
||||
"any_warn": anyWarn,
|
||||
})
|
||||
}
|
||||
|
||||
out := cmd.OutOrStdout()
|
||||
for _, r := range results {
|
||||
status := "ok"
|
||||
if r.Warn {
|
||||
status = "WARN"
|
||||
}
|
||||
fmt.Fprintf(out, "%-12s %-5s %d rows (threshold: %d)\n", r.Table, status, r.Count, threshold)
|
||||
}
|
||||
if anyWarn {
|
||||
fmt.Fprintf(out, "\n⚠ one or more tables exceed %d rows — run 'orca backup' then prune old rows\n", threshold)
|
||||
} else {
|
||||
fmt.Fprintln(out, "\n✓ all tables under retention threshold")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd, doctorAuditCmd, doctorModesCmd, doctorOIDCCmd)
|
||||
doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd, doctorAuditCmd, doctorModesCmd, doctorOIDCCmd, doctorDBRetentionCmd)
|
||||
rootCmd.AddCommand(doctorCmd)
|
||||
}
|
||||
|
||||
@@ -2,9 +2,14 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
|
||||
func TestDoctorText(t *testing.T) {
|
||||
@@ -194,3 +199,92 @@ func TestDoctorProxmoxJSON(t *testing.T) {
|
||||
t.Errorf("doctor proxmox --json missing Name: %v", result)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestDoctorDBRetention verifies that `orca doctor db-retention` counts
|
||||
// rows in jobs, tasks, and audit_log and warns when a table exceeds
|
||||
// 100k rows (REQ-158, P09 T7).
|
||||
func TestDoctorDBRetention(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
if err := runInit(discardWriter{}); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
|
||||
// Insert 100001 rows into the audit_log table to trigger the warning.
|
||||
// Use a multi-row VALUES insert in batches for speed.
|
||||
db, err := store.Open(certpaths.DBPath())
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
ctx := context.Background()
|
||||
// Build a batch insert: 500 rows per INSERT in a transaction.
|
||||
// SQLite handles this much faster than 100k individual inserts.
|
||||
const totalRows = 100001
|
||||
const batchSize = 500
|
||||
inserted := 0
|
||||
for inserted < totalRows {
|
||||
remaining := totalRows - inserted
|
||||
batch := batchSize
|
||||
if remaining < batch {
|
||||
batch = remaining
|
||||
}
|
||||
var placeholders strings.Builder
|
||||
var args []any
|
||||
for j := 0; j < batch; j++ {
|
||||
if j > 0 {
|
||||
placeholders.WriteString(",")
|
||||
}
|
||||
placeholders.WriteString("(?, 'test', 'test.action', 'test-resource', 'success')")
|
||||
args = append(args, time.Now().UTC())
|
||||
}
|
||||
q := "INSERT INTO audit_log (timestamp, actor, action, resource, result) VALUES " + placeholders.String()
|
||||
if _, err := db.ExecContext(ctx, q, args...); err != nil {
|
||||
t.Fatalf("batch insert at offset %d: %v", inserted, err)
|
||||
}
|
||||
inserted += batch
|
||||
}
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"doctor", "db-retention"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("doctor db-retention: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "audit_log") {
|
||||
t.Errorf("output missing audit_log table: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "WARN") {
|
||||
t.Errorf("output should contain WARN for audit_log exceeding threshold: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "backup") {
|
||||
t.Errorf("output should suggest 'orca backup': %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDoctorDBRetentionNoWarn verifies that with a small DB no warning
|
||||
// is emitted (REQ-158, P09 T7).
|
||||
func TestDoctorDBRetentionNoWarn(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
if err := runInit(discardWriter{}); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"doctor", "db-retention"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("doctor db-retention: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if strings.Contains(out, "WARN") {
|
||||
t.Errorf("output should NOT contain WARN for small DB: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
+85
-1
@@ -6,9 +6,12 @@ import (
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/acl"
|
||||
@@ -16,6 +19,7 @@ import (
|
||||
"git.cloudinit.dev/coreci/orca/internal/identity"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/secrets"
|
||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
@@ -59,7 +63,8 @@ func runInit(out interface{ Write([]byte) (int, error) }) error {
|
||||
Database string `json:"database"`
|
||||
CAFingerprint string `json:"ca_fingerprint,omitempty"`
|
||||
CertFingerprint string `json:"cert_fingerprint,omitempty"`
|
||||
OS string `json:"os"`
|
||||
OS string `json:"os"
|
||||
"path/filepath"`
|
||||
NodeID string `json:"node_id"`
|
||||
NodeName string `json:"node_name"`
|
||||
Steps []stepResult `json:"steps"`
|
||||
@@ -138,6 +143,85 @@ func runInit(out interface{ Write([]byte) (int, error) }) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4a: SSH keypair (idempotent — GenerateOrLoadSSHKey has a fast-path).
|
||||
// REQ-164: without this, every sshpush.Transport dial fails because
|
||||
// the orca SSH key doesn't exist after a fresh init.
|
||||
sshKeyPEM, sshPubLine, err := security.GenerateOrLoadSSHKey(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate SSH keypair: %w", err)
|
||||
}
|
||||
_ = sshKeyPEM
|
||||
sshKeyFp := ""
|
||||
if pubKey, err := ssh.ParsePublicKey(sshPubLine); err == nil {
|
||||
sshKeyFp = ssh.FingerprintSHA256(pubKey)
|
||||
}
|
||||
summary.Steps = append(summary.Steps, stepResult{Label: "ssh-key", Status: "ok", Detail: sshKeyFp[:min(16, len(sshKeyFp))] + "..."})
|
||||
if !jsonOutput {
|
||||
fmt.Fprintf(out, "\xe2\x9c\x93 SSH keypair provisioned: fp=%s\n", sshKeyFp[:min(16, len(sshKeyFp))]+"...")
|
||||
}
|
||||
|
||||
// Step 4b: known_hosts file (empty, 0600). Without this, the TOFU
|
||||
// host-key callback fails with "no such file" on the first SSH dial
|
||||
// (knownhosts.New requires the file to exist).
|
||||
knownHostsPath := certpaths.KnownHostsPath()
|
||||
if _, err := os.Stat(knownHostsPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.WriteFile(knownHostsPath, []byte{}, 0o600); err != nil {
|
||||
return fmt.Errorf("create known_hosts: %w", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("stat known_hosts: %w", err)
|
||||
}
|
||||
}
|
||||
summary.Steps = append(summary.Steps, stepResult{Label: "known-hosts", Status: "ok", Detail: knownHostsPath})
|
||||
if !jsonOutput {
|
||||
fmt.Fprintf(out, "\xe2\x9c\x93 Known hosts file created: %s\n", knownHostsPath)
|
||||
}
|
||||
|
||||
// Step 4c: master key (32-byte random, 0600). Without this, secrets
|
||||
// set/get/rotate and cluster seal/unseal all fail with "stat master
|
||||
// key: no such file or directory" on a fresh init.
|
||||
masterKeyPath := paths.MasterKeyPath()
|
||||
if _, err := os.Stat(masterKeyPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
os.MkdirAll(filepath.Dir(masterKeyPath), 0o755)
|
||||
masterKey, err := secrets.GenerateMasterKey()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate master key: %w", err)
|
||||
}
|
||||
if err := secrets.SaveMasterKey(masterKeyPath, masterKey); err != nil {
|
||||
return fmt.Errorf("save master key: %w", err)
|
||||
}
|
||||
// Zero the key from memory (defense-in-depth, REQ-154).
|
||||
defer secrets.ZeroKey(masterKey)
|
||||
summary.Steps = append(summary.Steps, stepResult{Label: "master-key", Status: "ok", Detail: "generated"})
|
||||
if !jsonOutput {
|
||||
fmt.Fprintf(out, "\xe2\x9c\x93 Master key generated: %s\n", masterKeyPath)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("stat master key: %w", err)
|
||||
}
|
||||
} else {
|
||||
summary.Steps = append(summary.Steps, stepResult{Label: "master-key", Status: "skipped", Detail: "already present"})
|
||||
if !jsonOutput {
|
||||
fmt.Fprintf(out, "\xe2\x9c\x93 Master key: already present\n")
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4d: Install Traefik on the lead node (REQ-165, Phase B).
|
||||
// Traefik is the data-plane ingress. Idempotent.
|
||||
if err := installTraefikLocal(); err != nil {
|
||||
if !jsonOutput {
|
||||
fmt.Fprintf(out, "Traefik install skipped: %v\n", err)
|
||||
}
|
||||
summary.Steps = append(summary.Steps, stepResult{Label: "traefik", Status: "skipped", Detail: err.Error()})
|
||||
} else {
|
||||
summary.Steps = append(summary.Steps, stepResult{Label: "traefik", Status: "ok", Detail: traefikVersion})
|
||||
if !jsonOutput {
|
||||
fmt.Fprintf(out, "Traefik installed: %s\n", traefikVersion)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: OS detection.
|
||||
osDetected := detectOS()
|
||||
summary.OS = osDetected
|
||||
|
||||
+187
-16
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -14,9 +15,11 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/engine"
|
||||
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
|
||||
@@ -44,8 +47,8 @@ var (
|
||||
)
|
||||
|
||||
var jobRunCmd = &cobra.Command{
|
||||
Use: "run <spec.hcl>",
|
||||
Short: "Run a job from an HCL spec file",
|
||||
Use: "run <spec.md>",
|
||||
Short: "Run a job from a markdown spec file",
|
||||
Long: "Submit a job spec, execute its tasks, and persist the result. Use --target to pin to a specific node (overrides bin-packing); --idempotency-key for cross-node dispatch dedupe.",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
@@ -112,7 +115,7 @@ var jobRunCmd = &cobra.Command{
|
||||
switch res.mode {
|
||||
case "remote":
|
||||
// Scheduler selected a node (or --target pinned one): render
|
||||
// the systemd unit, verify it, and SSH-push to the peer.
|
||||
// the systemd unit / PVE container, verify it, and SSH-push.
|
||||
// C-44: a push failure is an error (no local fallback).
|
||||
unitPaths, derr := deployRemote(ctx, spec, res, nodesByHost)
|
||||
logDispatch(res, derr)
|
||||
@@ -123,8 +126,12 @@ var jobRunCmd = &cobra.Command{
|
||||
return derr
|
||||
}
|
||||
res.unitPaths = unitPaths
|
||||
// REQ-156 / P07 T5: invalidate the jobs cache (the
|
||||
// dispatch decision records a local job entry).
|
||||
// REQ-166 / Phase C2: insert a Job DB record so `job list`
|
||||
// and `job stop` can find the remotely-deployed job.
|
||||
if dbErr := insertRemoteJob(spec, res.node); dbErr != nil {
|
||||
// Non-fatal: the job is deployed, just not visible to list.
|
||||
logDispatch(res, dbErr)
|
||||
}
|
||||
cacheInvalidate(cacheJobClass)
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]any{
|
||||
@@ -213,12 +220,20 @@ func renderJobs(cmd *cobra.Command, jobs []*model.Job) error {
|
||||
return printJSON(jobs)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "No jobs. Use 'orca job run <spec.hcl>' to submit one.")
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "No jobs. Use 'orca job run <spec.md>' to submit one.")
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%-36s %-20s %-12s %-8s\n", "ID", "NAME", "STATUS", "EXIT")
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%-10s %-20s %-12s %-20s %-5s\n", "ID", "NAME", "STATUS", "NODE", "EXIT")
|
||||
for _, j := range jobs {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%-36s %-20s %-12s %-8d\n", j.ID, j.Name, j.Status, j.ExitCode)
|
||||
shortID := j.ID
|
||||
if len(shortID) > 8 {
|
||||
shortID = shortID[:8]
|
||||
}
|
||||
exit := "-"
|
||||
if j.Status == model.JobStatusComplete || j.Status == model.JobStatusFailed || j.Status == model.JobStatusStopped {
|
||||
exit = fmt.Sprintf("%d", j.ExitCode)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%-10s %-20s %-12s %-20s %-5s\n", shortID, j.Name, j.Status, j.Node, exit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -283,18 +298,94 @@ func renderJobTable(jobs []*model.Job) string {
|
||||
if len(jobs) == 0 {
|
||||
return "No jobs.\n"
|
||||
}
|
||||
out := fmt.Sprintf("%-36s %-20s %-12s %-8s\n", "ID", "NAME", "STATUS", "EXIT")
|
||||
out := fmt.Sprintf("%-10s %-20s %-12s %-20s %-5s\n", "ID", "NAME", "STATUS", "NODE", "EXIT")
|
||||
for _, j := range jobs {
|
||||
out += fmt.Sprintf("%-36s %-20s %-12s %-8d\n", j.ID, j.Name, j.Status, j.ExitCode)
|
||||
shortID := j.ID
|
||||
if len(shortID) > 8 {
|
||||
shortID = shortID[:8]
|
||||
}
|
||||
exit := "-"
|
||||
if j.Status == model.JobStatusComplete || j.Status == model.JobStatusFailed || j.Status == model.JobStatusStopped {
|
||||
exit = fmt.Sprintf("%d", j.ExitCode)
|
||||
}
|
||||
out += fmt.Sprintf("%-10s %-20s %-12s %-20s %-5s\n", shortID, j.Name, j.Status, j.Node, exit)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// jobStopTransport is the SSH command-execution seam used by
|
||||
// `orca job stop`. *sshpush.Transport satisfies it via Exec; tests
|
||||
// inject a mock (same pattern as driftTransport / drainExecer).
|
||||
type jobStopTransport interface {
|
||||
Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
|
||||
}
|
||||
|
||||
// jobStopTransportOverride is the package-level test seam for the
|
||||
// SSH transport used by `orca job stop`. When non-nil it replaces the
|
||||
// production transport; tests set it and restore nil in cleanup.
|
||||
var jobStopTransportOverride jobStopTransport
|
||||
|
||||
// jobStopTimeout is the SSH command timeout for `orca job stop`.
|
||||
var jobStopTimeout time.Duration
|
||||
|
||||
// jobStopPeer is the optional --peer override for `orca job stop`.
|
||||
// When empty, the node is looked up from the alloc_history table
|
||||
// (latest entry for the job id). When set, the SSH stop targets that
|
||||
// peer directly.
|
||||
var jobStopPeer string
|
||||
|
||||
func jobStopTransportFromCtx() (jobStopTransport, error) {
|
||||
if jobStopTransportOverride != nil {
|
||||
return jobStopTransportOverride, nil
|
||||
}
|
||||
keyPath := certpaths.SSHKeyPath()
|
||||
khPath := certpaths.KnownHostsPath()
|
||||
return sshpush.NewTransport(keyPath, khPath), nil
|
||||
}
|
||||
|
||||
// nodeForJob looks up the node that ran (or is running) a job by
|
||||
// searching the alloc_history table for the latest entry for the
|
||||
// given job id. Returns nil if no history entry exists (the job may
|
||||
// have been run locally or pre-dates alloc_history).
|
||||
func nodeForJob(ctx context.Context, db *sql.DB, jobID string) (*model.Node, error) {
|
||||
hist := store.NewAllocHistoryRepo(db)
|
||||
if err := hist.EnsureSchema(ctx); err != nil {
|
||||
return nil, fmt.Errorf("alloc history schema: %w", err)
|
||||
}
|
||||
entries, err := hist.List(ctx, store.HistoryFilter{JobID: jobID})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("alloc history list: %w", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
// Pick the latest entry (List returns ASC; take the last).
|
||||
latest := entries[len(entries)-1]
|
||||
if latest.NodeID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
nodeRepo := store.NewNodeRepo(db)
|
||||
n, err := nodeRepo.Get(ctx, latest.NodeID)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("lookup node %s: %w", latest.NodeID, err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
var jobStopCmd = &cobra.Command{
|
||||
Use: "stop [job-id]",
|
||||
Short: "Stop a running job",
|
||||
Long: "Mark a job as stopped. Note: this is a soft stop (cancel context for the daemon).",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
Long: `Stop a running job by sending 'systemctl stop orca-alloc-<name>-*'
|
||||
to the node running the allocation via SSH, then mark the job as
|
||||
stopped in the DB (REQ-158, P09 T1).
|
||||
|
||||
If --peer is not given, the node is looked up from the allocation
|
||||
history. If no node is found, the DB status is updated anyway (soft
|
||||
stop fallback for local-run jobs).`,
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
id := stopID
|
||||
if id == "" && len(args) > 0 {
|
||||
@@ -303,8 +394,6 @@ var jobStopCmd = &cobra.Command{
|
||||
if id == "" {
|
||||
return fmt.Errorf("job id required (--id or argument)")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
db, closer, err := openDB()
|
||||
if err != nil {
|
||||
@@ -312,6 +401,9 @@ var jobStopCmd = &cobra.Command{
|
||||
}
|
||||
defer closer()
|
||||
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
repo := store.NewJobRepo(db)
|
||||
job, err := repo.Get(ctx, id)
|
||||
if err != nil {
|
||||
@@ -320,6 +412,52 @@ var jobStopCmd = &cobra.Command{
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Determine the peer to SSH to. --peer takes precedence;
|
||||
// otherwise look up the node from alloc_history.
|
||||
peer := jobStopPeer
|
||||
var node *model.Node
|
||||
if peer == "" {
|
||||
node, err = nodeForJob(ctx, db, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup node for job %s: %w", id, err)
|
||||
}
|
||||
if node != nil {
|
||||
peer = peerAddrForNode(node)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the job name before interpolation into the shell
|
||||
// command (same injection guard as logs --job / stopAlloc).
|
||||
jobName := job.Name
|
||||
if !validSafeName(jobName) {
|
||||
return fmt.Errorf("job stop: invalid job name %q (allowed: A-Z a-z 0-9 _ -)", jobName)
|
||||
}
|
||||
|
||||
sshRan := false
|
||||
if peer != "" {
|
||||
transport, terr := jobStopTransportFromCtx()
|
||||
if terr != nil {
|
||||
return fmt.Errorf("job stop: ssh transport: %w", terr)
|
||||
}
|
||||
stopCtx, stopCancel := sshCmdCtx(ctx, jobStopTimeout)
|
||||
defer stopCancel()
|
||||
// Match the drift.go job restart unit pattern: orca-alloc-<name>.
|
||||
// Use a glob (orca-alloc-<name>-*) to stop all task units in
|
||||
// a multi-task allocation group.
|
||||
unitPattern := fmt.Sprintf("orca-alloc-%s-*", jobName)
|
||||
stopCmd := fmt.Sprintf("systemctl stop %s", shellQuote(unitPattern))
|
||||
out, sErr := transport.Exec(stopCtx, peer, stopCmd)
|
||||
if sErr != nil {
|
||||
// Non-fatal: the unit may not be running (already
|
||||
// stopped) or SSH may fail. We still update the DB
|
||||
// status so the operator's intent is recorded.
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "⚠ job stop: SSH systemctl stop failed on %s: %v (output: %s)\n", peer, sErr, strings.TrimSpace(string(out)))
|
||||
} else {
|
||||
sshRan = true
|
||||
}
|
||||
}
|
||||
|
||||
if err := repo.UpdateStatus(ctx, id, model.JobStatusStopped, 130); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -327,9 +465,20 @@ var jobStopCmd = &cobra.Command{
|
||||
// `orca job list` reflects the just-stopped job.
|
||||
cacheInvalidate(cacheJobClass)
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]any{"id": id, "status": "stopped", "previous_status": job.Status})
|
||||
result := map[string]any{"id": id, "status": "stopped", "previous_status": job.Status}
|
||||
if peer != "" {
|
||||
result["peer"] = peer
|
||||
result["ssh_stop"] = sshRan
|
||||
}
|
||||
return printJSON(result)
|
||||
}
|
||||
if peer != "" && sshRan {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s (systemctl stop on %s)\n", id, peer)
|
||||
} else if peer != "" {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s (DB only; SSH stop failed — see stderr)\n", id)
|
||||
} else {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s (DB only; no node found)\n", id)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Job stopped: %s\n", id)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@@ -383,6 +532,8 @@ var jobLogsCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
jobStopCmd.Flags().StringVar(&stopID, "id", "", "job id")
|
||||
jobStopCmd.Flags().StringVar(&jobStopPeer, "peer", "", "peer address (host:port) running the allocation (auto-detected from alloc history if empty)")
|
||||
jobStopCmd.Flags().DurationVar(&jobStopTimeout, "timeout", sshCmdDefaultTimeout, "SSH command timeout")
|
||||
jobLogsCmd.Flags().StringVar(&stopID, "id", "", "job id")
|
||||
jobRunCmd.Flags().StringVar(&runTarget, "target", "", "pin job to a specific node id (overrides bin-packing)")
|
||||
jobRunCmd.Flags().StringVar(&runIDKey, "idempotency-key", "", "X-Orca-Idempotency-Key for cross-node dispatch dedupe")
|
||||
@@ -446,3 +597,23 @@ func splitCommand(s string) (string, []string) {
|
||||
}
|
||||
return parts[0], parts[1:]
|
||||
}
|
||||
|
||||
// insertRemoteJob inserts a model.Job row for a remotely-deployed job
|
||||
// (REQ-166, Phase C2). Without this, `job list` shows nothing for remote
|
||||
// deployments and `job stop` can't find the node.
|
||||
func insertRemoteJob(spec *jobspec.WorkloadSpec, node string) error {
|
||||
db, closer, err := openDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
repo := store.NewJobRepo(db)
|
||||
return repo.Insert(context.Background(), &model.Job{
|
||||
ID: uuid.NewString(),
|
||||
Name: spec.Name,
|
||||
Spec: "",
|
||||
Status: model.JobStatusRunning,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
Node: node,
|
||||
})
|
||||
}
|
||||
|
||||
+101
-19
@@ -34,6 +34,7 @@ import (
|
||||
"git.cloudinit.dev/coreci/orca/internal/emitter"
|
||||
"git.cloudinit.dev/coreci/orca/internal/jobspec"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/runtime"
|
||||
"git.cloudinit.dev/coreci/orca/internal/scheduler"
|
||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
@@ -199,30 +200,72 @@ func deployRemote(ctx context.Context, spec *jobspec.WorkloadSpec, res *dispatch
|
||||
return nil, fmt.Errorf("deployRemote: selected node %q not found in registry", res.node)
|
||||
}
|
||||
|
||||
// Render the systemd unit via the emitter. The runtime is required
|
||||
// for the process emitter; a spec with no runtime has nothing to
|
||||
// ExecStart and is rejected by the emitter.
|
||||
// REQ-166 / Phase C3: branch on runtime + node kind.
|
||||
runtimeOneOf := ""
|
||||
if spec.Runtime != nil {
|
||||
runtimeOneOf = spec.Runtime.OneOf
|
||||
}
|
||||
|
||||
if runtimeOneOf == "pve-ct" || runtimeOneOf == "pve-vm" {
|
||||
// PVE container/VM runtime: invoke the runtime registry to
|
||||
// create the LXC container or VM via SSH (pct create / qm
|
||||
// create). Only valid on proxmox nodes.
|
||||
if node.Kind != string(model.NodeKindProxmox) {
|
||||
return nil, fmt.Errorf("deployRemote: runtime %q requires a proxmox node (node %q is %q)", runtimeOneOf, res.node, node.Kind)
|
||||
}
|
||||
peer := sshPeerFor(node)
|
||||
sshTransport, err := newSSHPushTransport()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deployRemote: transport: %w", err)
|
||||
}
|
||||
defer sshTransport.Close()
|
||||
alloc := &runtime.Alloc{
|
||||
ID: res.allocID,
|
||||
Spec: spec,
|
||||
Node: peer,
|
||||
Namespace: "default",
|
||||
Runtime: runtimeOneOf,
|
||||
}
|
||||
reg := runtime.DefaultRegistry(sshTransport)
|
||||
if err := reg.Prepare(ctx, alloc); err != nil {
|
||||
return nil, fmt.Errorf("deployRemote: pve prepare: %w", err)
|
||||
}
|
||||
if _, err := reg.Start(ctx, alloc); err != nil {
|
||||
return nil, fmt.Errorf("deployRemote: pve start: %w", err)
|
||||
}
|
||||
// For PVE workloads, also emit Traefik route if the spec has ports.
|
||||
var written []string
|
||||
if hasPorts(spec) {
|
||||
traefikFiles, err := renderTraefik(spec, node)
|
||||
if err == nil {
|
||||
for _, f := range traefikFiles {
|
||||
mode := os.FileMode(0o644)
|
||||
_ = sshTransport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode)
|
||||
written = append(written, f.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// Process runtime: systemd units (only on linux/localhost nodes).
|
||||
if node.Kind == string(model.NodeKindProxmox) {
|
||||
return nil, fmt.Errorf("deployRemote: runtime %q requires a linux node (node %q is proxmox; use one_of: pve-ct or pve-vm for proxmox)", runtimeOneOf, res.node)
|
||||
}
|
||||
|
||||
// Render the systemd unit via the emitter.
|
||||
em := emitter.SystemdEmitter{}
|
||||
enode := &emitter.Node{
|
||||
Hostname: node.Name,
|
||||
Runtime: []string{"process"},
|
||||
Tags: nil,
|
||||
}
|
||||
// Advertise the node kind as a runtime so the emitter can branch
|
||||
// (proxmox nodes expose pve-* runtimes). For process workloads
|
||||
// this is informational.
|
||||
if node.Kind == string(model.NodeKindProxmox) {
|
||||
enode.Runtime = append(enode.Runtime, "proxmox")
|
||||
}
|
||||
files, err := em.Render(spec, enode)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("deployRemote: render unit: %w", err)
|
||||
}
|
||||
|
||||
// T9: systemd-analyze verify on the rendered unit before deploy.
|
||||
// Run it locally (the unit is a portable text file); if
|
||||
// systemd-analyze is not installed, skip silently (dev boxes
|
||||
// without systemd). A verification FAILURE is an error.
|
||||
for _, f := range files {
|
||||
if err := verifySystemdUnit(ctx, f.Path, f.Content); err != nil {
|
||||
return nil, fmt.Errorf("deployRemote: systemd-analyze verify %s: %w", f.Path, err)
|
||||
@@ -241,24 +284,18 @@ func deployRemote(ctx context.Context, spec *jobspec.WorkloadSpec, res *dispatch
|
||||
for _, f := range files {
|
||||
mode := os.FileMode(0o644)
|
||||
if f.Mode != "" {
|
||||
// f.Mode is an octal string like "0644".
|
||||
var m uint64
|
||||
if _, perr := fmt.Sscanf(f.Mode, "%o", &m); perr == nil {
|
||||
mode = os.FileMode(m)
|
||||
}
|
||||
}
|
||||
if err := transport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode); err != nil {
|
||||
// C-44: SSH-push failure -> error, NOT local fallback.
|
||||
return nil, fmt.Errorf("deployRemote: push %s to %s (%s): %w", f.Path, res.node, peer, err)
|
||||
}
|
||||
written = append(written, f.Path)
|
||||
}
|
||||
|
||||
// Reload systemd + enable the unit so it starts at boot. These are
|
||||
// best-effort; a failure here is surfaced but does not undo the
|
||||
// push (the unit is on disk). We use systemctl daemon-reload +
|
||||
// enable --now for each .service unit (.target units for task
|
||||
// groups are also enabled).
|
||||
// Reload systemd + enable the unit so it starts at boot.
|
||||
for _, p := range written {
|
||||
if !strings.HasSuffix(p, ".service") && !strings.HasSuffix(p, ".target") {
|
||||
continue
|
||||
@@ -268,9 +305,46 @@ func deployRemote(ctx context.Context, spec *jobspec.WorkloadSpec, res *dispatch
|
||||
}
|
||||
}
|
||||
|
||||
// REQ-166 / Phase C4: emit Traefik dynamic config if the spec
|
||||
// has ports (is a Service with ingress).
|
||||
if hasPorts(spec) {
|
||||
traefikFiles, err := renderTraefik(spec, node)
|
||||
if err != nil {
|
||||
// Non-fatal: Traefik route is best-effort.
|
||||
return written, nil
|
||||
}
|
||||
for _, f := range traefikFiles {
|
||||
mode := os.FileMode(0o644)
|
||||
_ = transport.WriteFile(ctx, peer, f.Path, []byte(f.Content), mode)
|
||||
written = append(written, f.Path)
|
||||
}
|
||||
}
|
||||
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// hasPorts returns true if the spec declares any ports (is a Service).
|
||||
func hasPorts(spec *jobspec.WorkloadSpec) bool {
|
||||
if spec == nil {
|
||||
return false
|
||||
}
|
||||
if len(spec.Ports) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// renderTraefik renders the Traefik dynamic config for the spec + node.
|
||||
func renderTraefik(spec *jobspec.WorkloadSpec, node *model.Node) ([]emitter.File, error) {
|
||||
em := emitter.TraefikEmitter{}
|
||||
enode := &emitter.Node{
|
||||
Hostname: node.Name,
|
||||
Runtime: []string{"process"},
|
||||
Tags: nil,
|
||||
}
|
||||
return em.Render(spec, enode)
|
||||
}
|
||||
|
||||
// verifySystemdUnit runs `systemd-analyze verify` on the rendered unit
|
||||
// content. The unit is written to a temp file (with its real basename)
|
||||
// so systemd-analyze resolves fragment paths correctly. When
|
||||
@@ -433,3 +507,11 @@ func logDispatch(res *dispatchResult, err error) {
|
||||
}
|
||||
log.Info("job.dispatch", attrs...)
|
||||
}
|
||||
|
||||
// newSSHPushTransport creates a concrete sshpush.Transport for PVE
|
||||
// runtime operations (pct create/qm create). The jobDispatchTransport
|
||||
// interface wraps sshpush.Transport but the runtime package needs the
|
||||
// concrete type.
|
||||
func newSSHPushTransport() (*sshpush.Transport, error) {
|
||||
return sshpush.NewTransport(certpaths.SSHKeyPath(), certpaths.KnownHostsPath()), nil
|
||||
}
|
||||
|
||||
@@ -2,11 +2,14 @@ package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
@@ -310,3 +313,193 @@ func seedJob(t *testing.T, name string, status model.JobStatus) string {
|
||||
}
|
||||
return j.ID
|
||||
}
|
||||
|
||||
|
||||
// mockJobStopExec is a record-and-replay SSH execer for `orca job stop`
|
||||
// tests (same pattern as mockDrainExec / mockLogsExec).
|
||||
type mockJobStopExec struct {
|
||||
mu sync.Mutex
|
||||
responses []jobStopMockResp
|
||||
calls []jobStopMockCall
|
||||
}
|
||||
|
||||
type jobStopMockResp struct {
|
||||
match string
|
||||
out string
|
||||
exit int
|
||||
}
|
||||
|
||||
type jobStopMockCall struct {
|
||||
peer string
|
||||
cmd string
|
||||
}
|
||||
|
||||
func (m *mockJobStopExec) Exec(_ context.Context, peer, cmd string) ([]byte, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.calls = append(m.calls, jobStopMockCall{peer: peer, cmd: cmd})
|
||||
for _, r := range m.responses {
|
||||
if r.match == "" || strings.Contains(cmd, r.match) {
|
||||
return []byte(r.out), nil
|
||||
}
|
||||
}
|
||||
return []byte(""), nil
|
||||
}
|
||||
|
||||
func (m *mockJobStopExec) callsFor(match string) []jobStopMockCall {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []jobStopMockCall
|
||||
for _, c := range m.calls {
|
||||
if strings.Contains(c.cmd, match) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestJobStopSSH verifies that `orca job stop` sends a real
|
||||
// 'systemctl stop' via SSH to the target node when the job has a
|
||||
// recorded allocation history (REQ-158, P09 T6).
|
||||
func TestJobStopSSH(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
if err := runInit(discardWriter{}); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
|
||||
// Seed a node and a job, then record an alloc_history entry
|
||||
// linking the job to the node.
|
||||
nodeID := seedNode(t, "worker-1", "worker-1:8443")
|
||||
jobID := seedJob(t, "webapp", model.JobStatusRunning)
|
||||
|
||||
db, err := store.Open(certpaths.DBPath())
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
hist := store.NewAllocHistoryRepo(db)
|
||||
ctx := context.Background()
|
||||
if err := hist.EnsureSchema(ctx); err != nil {
|
||||
t.Fatalf("ensure schema: %v", err)
|
||||
}
|
||||
if err := hist.Record(ctx, store.AllocHistoryEntry{
|
||||
AllocID: "default/webapp-0",
|
||||
JobID: jobID,
|
||||
NodeID: nodeID,
|
||||
Namespace: "default",
|
||||
ToState: "created",
|
||||
Timestamp: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatalf("record alloc history: %v", err)
|
||||
}
|
||||
|
||||
// Wire the mock SSH transport (must be after resetRootFlags so
|
||||
// resetCommandFlags doesn't nil it out).
|
||||
resetRootFlags(t)
|
||||
mock := &mockJobStopExec{}
|
||||
prev := jobStopTransportOverride
|
||||
jobStopTransportOverride = mock
|
||||
t.Cleanup(func() { jobStopTransportOverride = prev })
|
||||
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "stop", jobID})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("job stop: %v", err)
|
||||
}
|
||||
|
||||
// Verify systemctl stop was called via SSH.
|
||||
stopCalls := mock.callsFor("systemctl stop")
|
||||
if len(stopCalls) == 0 {
|
||||
t.Fatalf("expected systemctl stop SSH call, got %d calls: %v", len(mock.calls), mock.calls)
|
||||
}
|
||||
if !strings.Contains(stopCalls[0].cmd, "orca-alloc-webapp-*") {
|
||||
t.Errorf("expected 'orca-alloc-webapp-*' in cmd, got: %s", stopCalls[0].cmd)
|
||||
}
|
||||
if !strings.Contains(stopCalls[0].peer, "worker-1") {
|
||||
t.Errorf("expected peer to contain 'worker-1', got: %s", stopCalls[0].peer)
|
||||
}
|
||||
|
||||
// Verify the DB status was updated.
|
||||
repo := store.NewJobRepo(db)
|
||||
job, err := repo.Get(ctx, jobID)
|
||||
if err != nil {
|
||||
t.Fatalf("get job: %v", err)
|
||||
}
|
||||
if job.Status != model.JobStatusStopped {
|
||||
t.Errorf("job status = %v, want stopped", job.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJobStopSSHPeerOverride verifies that --peer bypasses the
|
||||
// alloc_history lookup and uses the given peer directly (REQ-158).
|
||||
func TestJobStopSSHPeerOverride(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
if err := runInit(discardWriter{}); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
|
||||
jobID := seedJob(t, "webapp2", model.JobStatusRunning)
|
||||
|
||||
resetRootFlags(t)
|
||||
mock := &mockJobStopExec{}
|
||||
prev := jobStopTransportOverride
|
||||
jobStopTransportOverride = mock
|
||||
t.Cleanup(func() { jobStopTransportOverride = prev })
|
||||
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "stop", jobID, "--peer", "10.0.0.5:22"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("job stop: %v", err)
|
||||
}
|
||||
|
||||
stopCalls := mock.callsFor("systemctl stop")
|
||||
if len(stopCalls) == 0 {
|
||||
t.Fatalf("expected systemctl stop SSH call, got %d calls", len(mock.calls))
|
||||
}
|
||||
if stopCalls[0].peer != "10.0.0.5:22" {
|
||||
t.Errorf("peer = %s, want 10.0.0.5:22", stopCalls[0].peer)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJobStopNoNodeFallback verifies that when no node is found in
|
||||
// alloc_history, the job is still stopped in the DB (soft stop
|
||||
// fallback) without attempting SSH (REQ-158).
|
||||
func TestJobStopNoNodeFallback(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
if err := runInit(discardWriter{}); err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
|
||||
jobID := seedJob(t, "localjob", model.JobStatusRunning)
|
||||
|
||||
resetRootFlags(t)
|
||||
mock := &mockJobStopExec{}
|
||||
prev := jobStopTransportOverride
|
||||
jobStopTransportOverride = mock
|
||||
t.Cleanup(func() { jobStopTransportOverride = prev })
|
||||
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"job", "stop", jobID})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("job stop: %v", err)
|
||||
}
|
||||
|
||||
// No SSH calls should have been made (no node found).
|
||||
if len(mock.calls) > 0 {
|
||||
t.Errorf("expected 0 SSH calls, got %d: %v", len(mock.calls), mock.calls)
|
||||
}
|
||||
|
||||
// Verify the output mentions "DB only".
|
||||
if !strings.Contains(buf.String(), "DB only") {
|
||||
t.Errorf("output should mention 'DB only', got: %s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
+43
-6
@@ -101,8 +101,20 @@ var (
|
||||
logsJob string
|
||||
logsSince string
|
||||
logsJSON bool
|
||||
logsLines int
|
||||
)
|
||||
|
||||
// logsMaxLines is the hard cap on --lines to prevent OOM from
|
||||
// unbounded journalctl output (REQ-158, P09 T3).
|
||||
const logsMaxLines = 50000
|
||||
|
||||
// logsDefaultLines is the default --lines value.
|
||||
const logsDefaultLines = 1000
|
||||
|
||||
// logsMaxSince is the maximum lookback for --since (7 days) to
|
||||
// prevent OOM from unbounded journalctl queries (REQ-158, P09 T3).
|
||||
const logsMaxSince = 7 * 24 * time.Hour
|
||||
|
||||
var logsCmd = &cobra.Command{
|
||||
Use: "logs",
|
||||
Short: "Aggregate journald logs across nodes (REQ-117)",
|
||||
@@ -139,6 +151,25 @@ Ctrl-C cancels the fan-out via signal.NotifyContext.`,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// REQ-158 / P09 T3: clamp --since to 7 days max to prevent
|
||||
// OOM from unbounded journalctl queries. If the requested
|
||||
// lookback exceeds the cap, clamp it and warn.
|
||||
now := time.Now().UTC()
|
||||
maxSince := now.Add(-logsMaxSince)
|
||||
if since.Before(maxSince) {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "⚠ --since %s exceeds 7d cap; clamping to 7d\n", logsSince)
|
||||
since = maxSince
|
||||
}
|
||||
|
||||
// REQ-158 / P09 T3: clamp --lines to [1, logsMaxLines].
|
||||
lines := logsLines
|
||||
if lines <= 0 {
|
||||
lines = logsDefaultLines
|
||||
}
|
||||
if lines > logsMaxLines {
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "⚠ --lines %d exceeds max %d; clamping\n", lines, logsMaxLines)
|
||||
lines = logsMaxLines
|
||||
}
|
||||
|
||||
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
@@ -158,7 +189,7 @@ Ctrl-C cancels the fan-out via signal.NotifyContext.`,
|
||||
|
||||
out := cmd.OutOrStdout()
|
||||
multi := len(nodes) > 1
|
||||
for line := range streamLogs(ctx, ex, nodes, since, logsJob) {
|
||||
for line := range streamLogs(ctx, ex, nodes, since, logsJob, lines) {
|
||||
if logsJSON {
|
||||
raw, _ := json.Marshal(line)
|
||||
fmt.Fprintln(out, string(raw))
|
||||
@@ -227,7 +258,7 @@ func resolveLogNodes(ctx context.Context) ([]*model.Node, error) {
|
||||
// JSON entry immediately. The stream ends when every node has
|
||||
// completed (or the context is cancelled). The caller drives the
|
||||
// iteration via range-over-func (D-017 iter.Seq pattern).
|
||||
func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since time.Time, job string) iter.Seq[LogLine] {
|
||||
func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since time.Time, job string, lines int) iter.Seq[LogLine] {
|
||||
return func(yield func(LogLine) bool) {
|
||||
merged := make(chan LogLine)
|
||||
var wg sync.WaitGroup
|
||||
@@ -235,7 +266,7 @@ func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since t
|
||||
wg.Add(1)
|
||||
go func(n *model.Node) {
|
||||
defer wg.Done()
|
||||
streamNodeLines(ctx, ex, n, since, job, merged)
|
||||
streamNodeLines(ctx, ex, n, since, job, lines, merged)
|
||||
}(n)
|
||||
}
|
||||
done := make(chan struct{})
|
||||
@@ -267,7 +298,7 @@ func streamLogs(ctx context.Context, ex logsExecer, nodes []*model.Node, since t
|
||||
// context is cancelled); the caller is responsible for waiting on the
|
||||
// goroutine. Send is non-blocking via select on ctx.Done so a slow
|
||||
// consumer does not stall the fanout forever.
|
||||
func streamNodeLines(ctx context.Context, ex logsExecer, n *model.Node, since time.Time, job string, out chan<- LogLine) {
|
||||
func streamNodeLines(ctx context.Context, ex logsExecer, n *model.Node, since time.Time, job string, lines int, out chan<- LogLine) {
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
slog.Default().Warn("logs: cannot resolve SSH address for node", "node", n.Name)
|
||||
@@ -278,9 +309,14 @@ func streamNodeLines(ctx context.Context, ex logsExecer, n *model.Node, since ti
|
||||
unitPattern = "orca-alloc-" + job + "-*"
|
||||
}
|
||||
sinceStr := since.Format("2006-01-02 15:04:05")
|
||||
// REQ-158 / P09 T3: pass --lines=N to journalctl to cap output
|
||||
// and prevent OOM from unbounded log queries.
|
||||
if lines <= 0 {
|
||||
lines = logsDefaultLines
|
||||
}
|
||||
// F1: shellQuote (single-quote wrap) instead of %q — %q does not
|
||||
// escape backticks, enabling command substitution in double quotes.
|
||||
cmd := fmt.Sprintf("journalctl -u %s --since %s --output json --no-pager", shellQuote(unitPattern), shellQuote(sinceStr))
|
||||
cmd := fmt.Sprintf("journalctl -u %s --since %s --lines %d --output json --no-pager", shellQuote(unitPattern), shellQuote(sinceStr), lines)
|
||||
raw, err := ex.Exec(ctx, peer, cmd)
|
||||
if err != nil {
|
||||
slog.Default().Warn("logs: exec failed", "node", n.Name, "peer", peer, "error", err)
|
||||
@@ -324,7 +360,8 @@ func init() {
|
||||
logsCmd.Flags().BoolVar(&logsAllNodes, "all-nodes", false, "fan out to all registered nodes")
|
||||
logsCmd.Flags().StringVar(&logsNode, "node", "", "restrict to a single node (name or id)")
|
||||
logsCmd.Flags().StringVar(&logsJob, "job", "", "filter by job name (matches orca-alloc-<name>-* units)")
|
||||
logsCmd.Flags().StringVar(&logsSince, "since", "5m", "duration lookback (e.g. 5m, 1h, 30m); default 5m")
|
||||
logsCmd.Flags().StringVar(&logsSince, "since", "5m", "duration lookback (e.g. 5m, 1h, 30m); default 5m; max 7d")
|
||||
logsCmd.Flags().IntVar(&logsLines, "lines", logsDefaultLines, fmt.Sprintf("max number of journal lines per node (default %d, max %d)", logsDefaultLines, logsMaxLines))
|
||||
logsCmd.Flags().BoolVar(&logsJSON, "json", false, "output raw JSON (one LogLine per line)")
|
||||
rootCmd.AddCommand(logsCmd)
|
||||
}
|
||||
|
||||
+156
-1
@@ -63,6 +63,18 @@ func (m *mockLogsExec) countCalls(match string) int {
|
||||
return n
|
||||
}
|
||||
|
||||
func (m *mockLogsExec) callsFor(match string) []logsMockCall {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []logsMockCall
|
||||
for _, c := range m.calls {
|
||||
if strings.Contains(c.cmd, match) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// logsTestEnv wires a mockLogsExec into logsExecOverride and returns
|
||||
// the mock + a cleanup func. Tests MUST defer the cleanup.
|
||||
func logsTestEnv(t *testing.T) *mockLogsExec {
|
||||
@@ -325,7 +337,7 @@ func TestLogsCancelStopsStream(t *testing.T) {
|
||||
{ID: "n1", Name: "cancelnode", Address: "cancelnode:8443"},
|
||||
}
|
||||
consumed := 0
|
||||
for range streamLogs(ctx, ex, nodes, time.Now().UTC().Add(-1*time.Minute), "") {
|
||||
for range streamLogs(ctx, ex, nodes, time.Now().UTC().Add(-1*time.Minute), "", 1000) {
|
||||
consumed++
|
||||
}
|
||||
if consumed > 1 {
|
||||
@@ -357,3 +369,146 @@ func TestLogsParseJournalLine_InvalidJSON(t *testing.T) {
|
||||
t.Error("expected error for invalid json, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestLogsLinesFlag verifies that --lines is passed through to the
|
||||
// journalctl command as --lines=N (REQ-158, P09 T8).
|
||||
func TestLogsLinesFlag(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
logsNodeForTest(t, "linesnode", "linesnode:8443")
|
||||
mx := logsTestEnv(t)
|
||||
|
||||
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
mx.responses = []logsMockResp{
|
||||
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "line test", "6") + "\n"},
|
||||
}
|
||||
|
||||
_, err := runLogsCmd(t, []string{"logs", "--node", "linesnode", "--since", "1m", "--lines", "500"})
|
||||
if err != nil {
|
||||
t.Fatalf("logs: %v", err)
|
||||
}
|
||||
calls := mx.callsFor("journalctl")
|
||||
if len(calls) == 0 {
|
||||
t.Fatal("expected journalctl call")
|
||||
}
|
||||
if !strings.Contains(calls[0].cmd, "--lines 500") {
|
||||
t.Errorf("expected '--lines 500' in cmd, got: %s", calls[0].cmd)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLogsLinesDefault verifies that the default --lines value (1000)
|
||||
// is passed to journalctl when --lines is not specified (REQ-158).
|
||||
func TestLogsLinesDefault(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
logsNodeForTest(t, "defnode", "defnode:8443")
|
||||
mx := logsTestEnv(t)
|
||||
|
||||
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
mx.responses = []logsMockResp{
|
||||
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "default lines", "6") + "\n"},
|
||||
}
|
||||
|
||||
_, err := runLogsCmd(t, []string{"logs", "--node", "defnode", "--since", "1m"})
|
||||
if err != nil {
|
||||
t.Fatalf("logs: %v", err)
|
||||
}
|
||||
calls := mx.callsFor("journalctl")
|
||||
if len(calls) == 0 {
|
||||
t.Fatal("expected journalctl call")
|
||||
}
|
||||
if !strings.Contains(calls[0].cmd, "--lines 1000") {
|
||||
t.Errorf("expected default '--lines 1000' in cmd, got: %s", calls[0].cmd)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLogsLinesClamp verifies that --lines exceeding the max (50000) is
|
||||
// clamped (REQ-158, P09 T8).
|
||||
func TestLogsLinesClamp(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
logsNodeForTest(t, "clampnode", "clampnode:8443")
|
||||
mx := logsTestEnv(t)
|
||||
|
||||
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
mx.responses = []logsMockResp{
|
||||
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "clamp test", "6") + "\n"},
|
||||
}
|
||||
|
||||
_, err := runLogsCmd(t, []string{"logs", "--node", "clampnode", "--since", "1m", "--lines", "999999"})
|
||||
if err != nil {
|
||||
t.Fatalf("logs: %v", err)
|
||||
}
|
||||
calls := mx.callsFor("journalctl")
|
||||
if len(calls) == 0 {
|
||||
t.Fatal("expected journalctl call")
|
||||
}
|
||||
if !strings.Contains(calls[0].cmd, "--lines 50000") {
|
||||
t.Errorf("expected clamped '--lines 50000' in cmd, got: %s", calls[0].cmd)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLogsSinceClamp verifies that --since exceeding 7 days is
|
||||
// clamped and a warning is printed (REQ-158, P09 T8).
|
||||
func TestLogsSinceClamp(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
logsNodeForTest(t, "sincenode", "sincenode:8443")
|
||||
mx := logsTestEnv(t)
|
||||
|
||||
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
mx.responses = []logsMockResp{
|
||||
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "since test", "6") + "\n"},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
resetRootFlags(t)
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{"logs", "--node", "sincenode", "--since", "720h"})
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
t.Fatalf("logs: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "clamping to 7d") {
|
||||
t.Errorf("expected warning about clamping --since to 7d, got: %s", out)
|
||||
}
|
||||
calls := mx.callsFor("journalctl")
|
||||
if len(calls) == 0 {
|
||||
t.Fatal("expected journalctl call")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLogsLinesFlagJSON verifies --lines is passed through in JSON mode.
|
||||
func TestLogsLinesFlagJSON(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
|
||||
logsNodeForTest(t, "jsonlines", "jsonlines:8443")
|
||||
mx := logsTestEnv(t)
|
||||
|
||||
ts := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
mx.responses = []logsMockResp{
|
||||
{match: "journalctl", out: journalJSONLine(ts, "orca-alloc-web-0", "json lines test", "6") + "\n"},
|
||||
}
|
||||
|
||||
_, err := runLogsCmd(t, []string{"logs", "--node", "jsonlines", "--since", "1m", "--lines", "200", "--json"})
|
||||
if err != nil {
|
||||
t.Fatalf("logs: %v", err)
|
||||
}
|
||||
calls := mx.callsFor("journalctl")
|
||||
if len(calls) == 0 {
|
||||
t.Fatal("expected journalctl call")
|
||||
}
|
||||
if !strings.Contains(calls[0].cmd, "--lines 200") {
|
||||
t.Errorf("expected '--lines 200' in cmd, got: %s", calls[0].cmd)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
"git.cloudinit.dev/coreci/orca/internal/transport"
|
||||
)
|
||||
@@ -54,12 +55,16 @@ updates gauges. No orca daemon required (R-001).`,
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("X-Frame-Options", "DENY")
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
if err := m.WritePrometheus(w); err != nil {
|
||||
log.Warn("metrics: write exposition failed", "err", err)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("X-Frame-Options", "DENY")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok\n"))
|
||||
})
|
||||
@@ -124,6 +129,22 @@ func refresh(ctx context.Context, m *transport.Metrics, db *sql.DB, log interfac
|
||||
log.Warn("metrics: job list failed", "err", err)
|
||||
} else {
|
||||
m.SetGauge("allocs_total", float64(len(jobs)))
|
||||
// REQ-159 / P10: jobs by state.
|
||||
byState := make(map[model.JobStatus]int, 8)
|
||||
for _, j := range jobs {
|
||||
byState[j.Status]++
|
||||
}
|
||||
// Set total + per-state counts using simple gauge names.
|
||||
running := byState[model.JobStatusRunning]
|
||||
failed := byState[model.JobStatusFailed]
|
||||
complete := byState[model.JobStatusComplete]
|
||||
m.SetGauge("orca_jobs_running", float64(running))
|
||||
m.SetGauge("orca_jobs_failed", float64(failed))
|
||||
m.SetGauge("orca_jobs_complete", float64(complete))
|
||||
}
|
||||
// REQ-159 / P10: audit chain head gauge.
|
||||
if head, err := store.NewAuditRepo(db).ChainHead(ctx); err == nil && head != "" {
|
||||
m.SetGauge("orca_audit_chain_head", 1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+60
-102
@@ -1,116 +1,74 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
"git.cloudinit.dev/coreci/orca/internal/transport"
|
||||
)
|
||||
|
||||
func TestMetricsCmdRegistered(t *testing.T) {
|
||||
found := false
|
||||
for _, c := range rootCmd.Commands() {
|
||||
if c.Name() == "metrics" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
// TestREQ159_MetricsExpanded verifies the expanded metric set (P10, REQ-159).
|
||||
func TestREQ159_MetricsExpanded(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_HOME", dir)
|
||||
dbPath := filepath.Join(dir, "orca.db")
|
||||
db, err := store.Open(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("store.Open: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("metricsCmd not registered on root")
|
||||
defer db.Close()
|
||||
|
||||
// Seed a node.
|
||||
repo := store.NewNodeRepo(db)
|
||||
if err := repo.Insert(context.Background(), &model.Node{
|
||||
ID: "test-node-1",
|
||||
Name: "test-node",
|
||||
Address: "localhost:8443",
|
||||
Kind: "localhost",
|
||||
OS: "linux",
|
||||
State: "ready",
|
||||
}); err != nil {
|
||||
t.Fatalf("insert node: %v", err)
|
||||
}
|
||||
|
||||
// Seed a job.
|
||||
jobRepo := store.NewJobRepo(db)
|
||||
if err := jobRepo.Insert(context.Background(), &model.Job{
|
||||
ID: "job-1",
|
||||
Name: "test-job",
|
||||
Status: "running",
|
||||
}); err != nil {
|
||||
t.Fatalf("insert job: %v", err)
|
||||
}
|
||||
|
||||
m := transport.NewMetrics()
|
||||
logger := slogLogger{}
|
||||
refresh(context.Background(), m, db, logger)
|
||||
|
||||
// Verify expanded metrics by reading the exposition output.
|
||||
var buf strings.Builder
|
||||
if err := m.WritePrometheus(&buf); err != nil {
|
||||
t.Fatalf("WritePrometheus: %v", err)
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, "nodes_total 1") {
|
||||
t.Errorf("output missing nodes_total 1:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "allocs_total 1") {
|
||||
t.Errorf("output missing allocs_total 1:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "orca_jobs_running") {
|
||||
t.Errorf("output missing orca_jobs_running:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsAddrFlagDefault(t *testing.T) {
|
||||
f := metricsCmd.Flags().Lookup("addr")
|
||||
if f == nil {
|
||||
t.Fatal("--addr flag not registered on metricsCmd")
|
||||
}
|
||||
if f.DefValue != ":9100" {
|
||||
t.Errorf("--addr default = %q, want %q", f.DefValue, ":9100")
|
||||
}
|
||||
}
|
||||
type slogLogger struct{}
|
||||
|
||||
func TestMetricsEndpoints(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
resetRootFlags(t)
|
||||
func (slogLogger) Warn(msg string, args ...any) {}
|
||||
|
||||
// Pick a free port by briefly listening then closing.
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen probe: %v", err)
|
||||
}
|
||||
addr := ln.Addr().String()
|
||||
_ = ln.Close()
|
||||
|
||||
metricsAddr = addr
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
cmd := metricsCmd
|
||||
var out bytes.Buffer
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetContext(ctx)
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
errCh <- cmd.RunE(cmd, nil)
|
||||
}()
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var resp *http.Response
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err = http.Get("http://" + addr + "/healthz")
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("GET /healthz: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Errorf("/healthz status = %d, want 200", resp.StatusCode)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if !strings.HasPrefix(string(body), "ok") {
|
||||
t.Errorf("/healthz body = %q, want \"ok\"", string(body))
|
||||
}
|
||||
|
||||
resp2, err := http.Get("http://" + addr + "/metrics")
|
||||
if err != nil {
|
||||
t.Fatalf("GET /metrics: %v", err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
if resp2.StatusCode != http.StatusOK {
|
||||
t.Errorf("/metrics status = %d, want 200", resp2.StatusCode)
|
||||
}
|
||||
mbody, _ := io.ReadAll(resp2.Body)
|
||||
ms := string(mbody)
|
||||
for _, name := range []string{
|
||||
"txns_applied_total",
|
||||
"txns_drifted_total",
|
||||
"drifts_remediated_total",
|
||||
"peers_total",
|
||||
"nodes_total",
|
||||
"allocs_total",
|
||||
} {
|
||||
if !strings.Contains(ms, name) {
|
||||
t.Errorf("/metrics missing %q\n---\n%s", name, ms)
|
||||
}
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-errCh:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("metrics command did not stop after cancel")
|
||||
}
|
||||
}
|
||||
var _ = os.Stdin
|
||||
|
||||
@@ -57,6 +57,11 @@ func resetCommandFlags() {
|
||||
driftConfigPath = ""
|
||||
driftRemediateForce = false
|
||||
jobRestartPeer = ""
|
||||
jobStopPeer = ""
|
||||
jobStopTimeout = 0
|
||||
jobStopTransportOverride = nil
|
||||
logsLines = logsDefaultLines
|
||||
cutoverFSOverride = nil
|
||||
jobLintExplain = false
|
||||
jobLintFormat = "text"
|
||||
jobVerifyLead = ""
|
||||
|
||||
+78
-4
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/linux"
|
||||
"git.cloudinit.dev/coreci/orca/internal/engine"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||
@@ -54,6 +55,7 @@ var (
|
||||
joinSSHKey string
|
||||
joinSSHPort int
|
||||
joinHostKeyFP string
|
||||
joinLXCTemplate string
|
||||
proxmoxUser string
|
||||
proxmoxRole string
|
||||
leaveID string
|
||||
@@ -73,16 +75,22 @@ var nodeJoinCmd = &cobra.Command{
|
||||
|
||||
Node types (via --type):
|
||||
localhost (default): register a local or Linux node (existing behavior)
|
||||
linux: SSH-bootstrap a remote generic Linux worker
|
||||
(Ubuntu/Debian/Alpine; deploys orca pubkey, creates orca
|
||||
user + drift-events dir; requires --host + --ssh-key)
|
||||
proxmox: SSH-bootstrap a remote Proxmox VE 8/9 host
|
||||
(deploys orca pubkey, creates orca user + PVE role +
|
||||
sudoers allowlist; requires --host + --ssh-key (R-021: no passwords))`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if joinHostKeyFP != "" && joinType != "proxmox" {
|
||||
return fmt.Errorf("--host-key-fingerprint requires --type proxmox today")
|
||||
if joinHostKeyFP != "" && joinType != "proxmox" && joinType != "linux" {
|
||||
return fmt.Errorf("--host-key-fingerprint requires --type proxmox or --type linux")
|
||||
}
|
||||
if joinType == "proxmox" {
|
||||
return joinProxmox(cmd)
|
||||
}
|
||||
if joinType == "linux" {
|
||||
return joinLinux(cmd)
|
||||
}
|
||||
return joinLocal(cmd)
|
||||
},
|
||||
}
|
||||
@@ -179,6 +187,7 @@ func joinProxmox(cmd *cobra.Command) error {
|
||||
SSHPort: joinSSHPort,
|
||||
HostKeyFingerprint: joinHostKeyFP,
|
||||
Logger: newLogger(),
|
||||
LXCTemplate: joinLXCTemplate,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("proxmox bootstrap: %w", err)
|
||||
@@ -217,6 +226,70 @@ func joinProxmox(cmd *cobra.Command) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// joinLinux bootstraps a remote generic Linux worker via SSH and
|
||||
// registers it as an orca node (REQ-161, P12). Uses SSH key auth
|
||||
// (R-021: no passwords).
|
||||
func joinLinux(cmd *cobra.Command) error {
|
||||
if joinHost == "" {
|
||||
return fmt.Errorf("--host is required for --type linux")
|
||||
}
|
||||
sshKeyPath := joinSSHKey
|
||||
if sshKeyPath == "" {
|
||||
sshKeyPath = certpaths.SSHKeyPath()
|
||||
}
|
||||
if sshKeyPath == "" {
|
||||
return fmt.Errorf("SSH key path is required for --type linux (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result, err := linux.BootstrapLinux(ctx, linux.Options{
|
||||
Host: joinHost,
|
||||
SSHUser: joinSSHUser,
|
||||
SSHKeyPath: sshKeyPath,
|
||||
OrcaUser: proxmoxUser,
|
||||
SSHPort: joinSSHPort,
|
||||
HostKeyFingerprint: joinHostKeyFP,
|
||||
Logger: newLogger(),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("linux bootstrap: %w", err)
|
||||
}
|
||||
|
||||
registry, closer, err := nodeRegistry()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
regCtx, regCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer regCancel()
|
||||
|
||||
node := &model.Node{
|
||||
ID: uuid.NewString(),
|
||||
Name: result.NodeName,
|
||||
Address: result.NodeAddress,
|
||||
State: model.NodeStateReady,
|
||||
JoinedAt: time.Now().UTC(),
|
||||
LastSeen: time.Now().UTC(),
|
||||
Kind: string(model.NodeKindLinux),
|
||||
OS: "linux",
|
||||
}
|
||||
if err := registry.Join(regCtx, node); err != nil {
|
||||
return fmt.Errorf("register linux node: %w", err)
|
||||
}
|
||||
cacheInvalidate(cacheNodeClass)
|
||||
if jsonOutput {
|
||||
return printJSON(node)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "\xe2\x9c\x93 Linux worker joined: %s (%s) at %s\n", node.ID, node.Name, node.Address)
|
||||
if result.HostKeyFingerprint != "" {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " host key: %s\n", result.HostKeyFingerprint)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var nodeLeaveCmd = &cobra.Command{
|
||||
Use: "leave [node-id]",
|
||||
Short: "Remove a node from the orca registry",
|
||||
@@ -432,14 +505,15 @@ func init() {
|
||||
nodeJoinCmd.Flags().StringVar(&joinName, "name", "", "node name (required for --type localhost)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinAddr, "addr", "", "node address (default localhost:8443)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinCAFinger, "ca-fingerprint", "", "pin CA cert SHA-256 (REQ-026); fails if on-disk CA doesn't match")
|
||||
nodeJoinCmd.Flags().StringVar(&joinType, "type", "localhost", "node type: localhost (default) or proxmox (SSH bootstrap)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinType, "type", "localhost", "node type: localhost (default), proxmox, or linux (SSH bootstrap)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinHost, "host", "", "proxmox host address (IP/hostname, no port; required for --type proxmox)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinSSHUser, "ssh-user", "root", "SSH username for proxmox bootstrap (default root)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinSSHKey, "ssh-key", "", "SSH private key path for proxmox bootstrap (R-021: no passwords; default: orca key)")
|
||||
nodeJoinCmd.Flags().IntVar(&joinSSHPort, "ssh-port", 22, "SSH port for proxmox bootstrap (default 22)")
|
||||
nodeJoinCmd.Flags().StringVar(&proxmoxUser, "proxmox-user", "orca", "Linux system user to create on the proxmox host (config-overridable)")
|
||||
nodeJoinCmd.Flags().StringVar(&proxmoxRole, "proxmox-role", "OrcaOperator", "PVE custom role to create (config-overridable)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinHostKeyFP, "host-key-fingerprint", "", "SSH host key SHA256:base64 fingerprint (pre-pin; supersedes TOFU for --type proxmox)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinHostKeyFP, "host-key-fingerprint", "", "SSH host key SHA256:base64 fingerprint (pre-pin; supersedes TOFU for --type proxmox or --type linux)")
|
||||
nodeJoinCmd.Flags().StringVar(&joinLXCTemplate, "lxc-template", "ubuntu-24.04", "LXC template for Proxmox (default ubuntu-24.04; alternatives: alpine-3.20, debian-12)")
|
||||
nodeLeaveCmd.Flags().StringVar(&leaveID, "id", "", "node id")
|
||||
nodeListCmd.Flags().BoolVar(&nodeWatch, "watch", false, "stream nodes until Ctrl-C (table refresh or --json per-event)")
|
||||
|
||||
|
||||
@@ -11,10 +11,14 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
|
||||
@@ -72,8 +76,11 @@ var nodeCapacitySetCmd = &cobra.Command{
|
||||
Short: "Declare capacity for a node (used by bin-packing)",
|
||||
Long: "Write cpu_millicores, memory_mib, and disk_mib for the named node. Idempotent: subsequent calls overwrite.",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if capSetCPU <= 0 || capSetMem <= 0 || capSetDisk <= 0 {
|
||||
return fmt.Errorf("--cpu, --memory, and --disk must all be positive")
|
||||
// REQ-168: allow partial updates. At least one dimension
|
||||
// must be positive; the others are read from the existing
|
||||
// row (or default to 0 if no row exists yet).
|
||||
if capSetCPU <= 0 && capSetMem <= 0 && capSetDisk <= 0 {
|
||||
return fmt.Errorf("at least one of --cpu, --memory, or --disk must be positive")
|
||||
}
|
||||
id := capNodeID
|
||||
if id == "" {
|
||||
@@ -87,11 +94,27 @@ var nodeCapacitySetCmd = &cobra.Command{
|
||||
}
|
||||
defer closer()
|
||||
repo := store.NewCapacityRepo(db)
|
||||
// Read existing row for partial update.
|
||||
existing, _ := repo.Get(ctx, id)
|
||||
cpu := capSetCPU
|
||||
mem := capSetMem
|
||||
disk := capSetDisk
|
||||
if existing != nil {
|
||||
if cpu <= 0 {
|
||||
cpu = existing.CPUMillicores
|
||||
}
|
||||
if mem <= 0 {
|
||||
mem = existing.MemoryMiB
|
||||
}
|
||||
if disk <= 0 {
|
||||
disk = existing.DiskMiB
|
||||
}
|
||||
}
|
||||
c := &store.NodeCapacity{
|
||||
NodeID: id,
|
||||
CPUMillicores: capSetCPU,
|
||||
MemoryMiB: capSetMem,
|
||||
DiskMiB: capSetDisk,
|
||||
CPUMillicores: cpu,
|
||||
MemoryMiB: mem,
|
||||
DiskMiB: disk,
|
||||
}
|
||||
if err := repo.Upsert(ctx, c); err != nil {
|
||||
return err
|
||||
@@ -137,6 +160,92 @@ var nodeCapacityListCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
// nodeCapacityAutoCmd discovers capacity by SSHing to the node and
|
||||
// reading nproc, /proc/meminfo, df (REQ-168, Phase D2).
|
||||
var capAutoPct int
|
||||
|
||||
var nodeCapacityAutoCmd = &cobra.Command{
|
||||
Use: "auto [percentage]",
|
||||
Short: "Auto-discover node capacity via SSH (default 75% of physical)",
|
||||
Long: `SSH to the specified node and discover CPU cores, memory,
|
||||
and disk capacity. Multiplies the physical values by the given
|
||||
percentage (default 75) to reserve headroom for the OS. The discovered
|
||||
values are written to the capacity table (same as 'orca node capacity set').`,
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
pct := 75
|
||||
if len(args) > 0 {
|
||||
var err error
|
||||
pct, err = strconv.Atoi(args[0])
|
||||
if err != nil || pct < 1 || pct > 100 {
|
||||
return fmt.Errorf("percentage must be 1-100, got %q", args[0])
|
||||
}
|
||||
}
|
||||
id := capNodeID
|
||||
if id == "" {
|
||||
return fmt.Errorf("--node is required for capacity auto")
|
||||
}
|
||||
// Build SSH transport and exec discovery commands.
|
||||
transport := sshpush.NewTransport(certpaths.SSHKeyPath(), certpaths.KnownHostsPath())
|
||||
defer transport.Close()
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
// CPU: nproc
|
||||
cpuOut, err := transport.Exec(ctx, id, "nproc")
|
||||
if err != nil {
|
||||
return fmt.Errorf("capacity auto: SSH exec nproc on %s: %w", id, err)
|
||||
}
|
||||
cores, err := strconv.Atoi(strings.TrimSpace(string(cpuOut)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("capacity auto: parse nproc output %q: %w", string(cpuOut), err)
|
||||
}
|
||||
// Memory: MemTotal from /proc/meminfo (in kB -> MiB)
|
||||
memOut, err := transport.Exec(ctx, id, "awk '/MemTotal/{print $2}' /proc/meminfo")
|
||||
if err != nil {
|
||||
return fmt.Errorf("capacity auto: SSH exec meminfo on %s: %w", id, err)
|
||||
}
|
||||
memKB, err := strconv.ParseInt(strings.TrimSpace(string(memOut)), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("capacity auto: parse meminfo output %q: %w", string(memOut), err)
|
||||
}
|
||||
// Disk: df on root (1K-blocks -> MiB)
|
||||
diskOut, err := transport.Exec(ctx, id, "df --output=size / | tail -1")
|
||||
if err != nil {
|
||||
return fmt.Errorf("capacity auto: SSH exec df on %s: %w", id, err)
|
||||
}
|
||||
diskKB, err := strconv.ParseInt(strings.TrimSpace(string(diskOut)), 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("capacity auto: parse df output %q: %w", string(diskOut), err)
|
||||
}
|
||||
// Apply percentage, convert to millicores/MiB.
|
||||
cpuM := int64(cores) * 1000 * int64(pct) / 100
|
||||
memMib := memKB * int64(pct) / 100 / 1024
|
||||
diskMib := diskKB * int64(pct) / 100 / 1024
|
||||
// Write to DB.
|
||||
db, closer, err := openDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
repo := store.NewCapacityRepo(db)
|
||||
c := &store.NodeCapacity{
|
||||
NodeID: id,
|
||||
CPUMillicores: cpuM,
|
||||
MemoryMiB: memMib,
|
||||
DiskMiB: diskMib,
|
||||
}
|
||||
if err := repo.Upsert(ctx, c); err != nil {
|
||||
return err
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(c)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Capacity auto-discovered for %s (%d%%): cpu=%dm, mem=%dMiB, disk=%dMiB\n", id, pct, cpuM, memMib, diskMib)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
nodeCapacitySetCmd.Flags().Int64Var(&capSetCPU, "cpu", 0, "CPU capacity in millicores (1000 = 1 vCPU)")
|
||||
nodeCapacitySetCmd.Flags().Int64Var(&capSetMem, "memory", 0, "Memory capacity in MiB")
|
||||
@@ -144,6 +253,7 @@ func init() {
|
||||
nodeCapacitySetCmd.Flags().StringVar(&capNodeID, "node", "", "node id (defaults to 'self')")
|
||||
nodeCapacityShowCmd.Flags().StringVar(&capNodeID, "node", "", "node id (defaults to 'self')")
|
||||
|
||||
nodeCapacityCmd.AddCommand(nodeCapacityShowCmd, nodeCapacitySetCmd, nodeCapacityListCmd)
|
||||
nodeCapacityAutoCmd.Flags().StringVar(&capNodeID, "node", "", "node name or ID to auto-discover capacity for")
|
||||
nodeCapacityCmd.AddCommand(nodeCapacityShowCmd, nodeCapacitySetCmd, nodeCapacityListCmd, nodeCapacityAutoCmd)
|
||||
nodeCmd.AddCommand(nodeCapacityCmd)
|
||||
}
|
||||
|
||||
@@ -10,16 +10,18 @@ import (
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
|
||||
func TestNodeCapacitySetMissingArgs(t *testing.T) {
|
||||
func TestNodeCapacitySetPartialUpdate(t *testing.T) {
|
||||
_, cleanup := initTestEnv(t)
|
||||
defer cleanup()
|
||||
resetRootFlags(t)
|
||||
var buf bytes.Buffer
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
// REQ-168: partial updates are now allowed. Setting only --cpu
|
||||
// should succeed (memory/disk default to 0 or existing values).
|
||||
rootCmd.SetArgs([]string{"node", "capacity", "set", "--cpu", "1000"})
|
||||
if err := rootCmd.Execute(); err == nil {
|
||||
t.Fatal("expected error for capacity set missing memory/disk, got nil")
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("expected success for partial capacity set, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -452,15 +452,15 @@ func TestNodeJoinHostKeyFingerprintRequiresProxmox(t *testing.T) {
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs([]string{
|
||||
"node", "join",
|
||||
"--type", "linux",
|
||||
"--name", "linux-node",
|
||||
"--type", "localhost",
|
||||
"--name", "localhost-node",
|
||||
"--host-key-fingerprint", "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
||||
})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
t.Fatal("expected error for --host-key-fingerprint without --type proxmox, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--host-key-fingerprint requires --type proxmox") {
|
||||
if !strings.Contains(err.Error(), "--host-key-fingerprint requires --type proxmox or --type linux") {
|
||||
t.Errorf("error should mention the --host-key-fingerprint/--type proxmox requirement, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+15
-3
@@ -6,9 +6,21 @@ import (
|
||||
|
||||
var statusCmd = &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Show orca daemon status",
|
||||
Long: "Display the current status of the local orca daemon, including version, uptime, and connection info.",
|
||||
Short: "Show orca daemon status (deprecated)",
|
||||
Long: `Display the current status of the local orca daemon, including
|
||||
version, uptime, and connection info.
|
||||
|
||||
**Deprecated (v0.13):** This command is a v0.1 stub that reports a
|
||||
hardcoded "daemon stopped" status. The daemon model was replaced by
|
||||
SSH-push in v0.9 (R-001) and the dual-write window closed in v0.12
|
||||
(REQ-138). Use the canonical commands instead:
|
||||
|
||||
orca node list # node registry + state
|
||||
orca metrics /healthz # liveness/health probe (daemon-mode only)
|
||||
|
||||
This command will be removed in a future release.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
warnDeprecated("orca status is deprecated (v0.1 stub): use 'orca node list' for node state and 'orca metrics /healthz' for health probes")
|
||||
status := map[string]any{
|
||||
"version": version,
|
||||
"daemon": "stopped",
|
||||
@@ -23,7 +35,7 @@ var statusCmd = &cobra.Command{
|
||||
}
|
||||
printText("orca daemon status\n")
|
||||
printText(" version: %s\n", version)
|
||||
printText(" daemon: %s\n", "stopped (daemon not yet implemented in Phase 1)")
|
||||
printText(" daemon: %s\n", "stopped (deprecated v0.1 stub; use 'orca node list' + 'orca metrics /healthz')")
|
||||
printText(" api_addr: %s\n", "https://localhost:8443")
|
||||
printText(" phase: %s\n", "1-cli-skeleton")
|
||||
printText(" milestone: %s\n", "v0.1")
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||
)
|
||||
|
||||
var traefikVersion = traefik.DefaultVersion
|
||||
|
||||
func installTraefikLocal() error {
|
||||
return traefik.InstallLocal(traefikVersion)
|
||||
}
|
||||
+99
-4
@@ -69,6 +69,41 @@ var upgradeTransportOverride upgradeTransport
|
||||
// peers to create the orca user on. Returns a list of peer addresses.
|
||||
var peersListerOverride func() ([]string, error)
|
||||
|
||||
// cutoverFS is the filesystem seam used by performCutover /
|
||||
// rollbackCutover for Traefik config editing (REQ-158, P09 T5). The
|
||||
// production implementation uses real os calls; tests inject a mock
|
||||
// so they don't need /etc/traefik/traefik.yml to exist.
|
||||
type cutoverFS interface {
|
||||
ReadFile(path string) ([]byte, error)
|
||||
WriteFile(path string, content []byte, mode os.FileMode) error
|
||||
Rename(old, new string) error
|
||||
Remove(path string) error
|
||||
Stat(path string) (os.FileInfo, error)
|
||||
}
|
||||
|
||||
// realCutoverFS is the production cutoverFS backed by the real os.
|
||||
type realCutoverFS struct{}
|
||||
|
||||
func (realCutoverFS) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) }
|
||||
func (realCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
|
||||
return os.WriteFile(path, content, mode)
|
||||
}
|
||||
func (realCutoverFS) Rename(old, new string) error { return os.Rename(old, new) }
|
||||
func (realCutoverFS) Remove(path string) error { return os.Remove(path) }
|
||||
func (realCutoverFS) Stat(path string) (os.FileInfo, error) { return os.Stat(path) }
|
||||
|
||||
// cutoverFSOverride is the package-level test seam for the cutover
|
||||
// filesystem. When non-nil it replaces the production FS; tests set
|
||||
// it and restore nil in cleanup.
|
||||
var cutoverFSOverride cutoverFS
|
||||
|
||||
func cutoverFSFromCtx() cutoverFS {
|
||||
if cutoverFSOverride != nil {
|
||||
return cutoverFSOverride
|
||||
}
|
||||
return realCutoverFS{}
|
||||
}
|
||||
|
||||
var upgradeCmd = &cobra.Command{
|
||||
Use: "upgrade",
|
||||
Short: "Upgrade orca to a new version (REQ-115, R-017 cutover)",
|
||||
@@ -310,11 +345,43 @@ func detectOldTraefikBinding() bool {
|
||||
// return 200. On failure, rolls back (restores :443, removes nft rules)
|
||||
// and returns (false, nil). On success returns (true, nil). With
|
||||
// force=true, verification is skipped.
|
||||
//
|
||||
// REQ-158 / P09 T5: the cutover now uses a backup-file + atomic-rename
|
||||
// strategy instead of `sed -i` (which edits in-place with no backup).
|
||||
// The Traefik config is copied to traefik.yml.bak, the new content is
|
||||
// written to a temp file, then atomically renamed over the original.
|
||||
// If any step fails, the backup is restored. This prevents a partial
|
||||
// edit from leaving Traefik in a broken state.
|
||||
func performCutover(ctx context.Context, runner commandRunner, out interface{ Write([]byte) (int, error) }, force bool) (bool, error) {
|
||||
if _, err := runner.Run(ctx, "sed", "-i", "s/:443/127.0.0.1:8443/g", "/etc/traefik/traefik.yml"); err != nil {
|
||||
return false, fmt.Errorf("cutover: edit traefik.yml: %w", err)
|
||||
cfs := cutoverFSFromCtx()
|
||||
traefikYml := "/etc/traefik/traefik.yml"
|
||||
backupPath := traefikYml + ".bak"
|
||||
|
||||
// Step 1: read the current config and create a backup.
|
||||
original, err := cfs.ReadFile(traefikYml)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cutover: read traefik.yml: %w", err)
|
||||
}
|
||||
if err := cfs.WriteFile(backupPath, original, 0o644); err != nil {
|
||||
return false, fmt.Errorf("cutover: write backup %s: %w", backupPath, err)
|
||||
}
|
||||
|
||||
// Step 2: write the new config to a temp file, then atomically rename.
|
||||
newContent := strings.ReplaceAll(string(original), ":443", "127.0.0.1:8443")
|
||||
tmpPath := traefikYml + ".tmp"
|
||||
if err := cfs.WriteFile(tmpPath, []byte(newContent), 0o644); err != nil {
|
||||
return false, fmt.Errorf("cutover: write temp %s: %w", tmpPath, err)
|
||||
}
|
||||
if err := cfs.Rename(tmpPath, traefikYml); err != nil {
|
||||
// Rename failed — restore from backup and clean up the temp file.
|
||||
_ = cfs.Remove(tmpPath)
|
||||
_ = cfs.Rename(backupPath, traefikYml)
|
||||
return false, fmt.Errorf("cutover: atomic rename %s → %s: %w", tmpPath, traefikYml, err)
|
||||
}
|
||||
|
||||
if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil {
|
||||
// Restart failed — restore from backup.
|
||||
_ = cfs.Rename(backupPath, traefikYml)
|
||||
return false, fmt.Errorf("cutover: restart traefik: %w", err)
|
||||
}
|
||||
nftCmd := `nft add table inet orca_redirect; nft 'add chain inet orca_redirect prerouting { type nat hook prerouting priority -100; }'; nft add rule inet orca_redirect prerouting tcp dport 443 dnat to 127.0.0.1:8443`
|
||||
@@ -324,6 +391,8 @@ func performCutover(ctx context.Context, runner commandRunner, out interface{ Wr
|
||||
|
||||
if force {
|
||||
fmt.Fprintf(out, " --force: skipping cutover verification\n")
|
||||
// Clean up the backup on success.
|
||||
_ = cfs.Remove(backupPath)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -336,6 +405,8 @@ func performCutover(ctx context.Context, runner commandRunner, out interface{ Wr
|
||||
return false, nil
|
||||
}
|
||||
fmt.Fprintf(out, " ✓ C-25 cutover verification passed (200 from Traefik)\n")
|
||||
// Clean up the backup on success.
|
||||
_ = cfs.Remove(backupPath)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -390,9 +461,33 @@ func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
|
||||
}
|
||||
|
||||
// rollbackCutover restores Traefik to :443 and removes nftables rules.
|
||||
// REQ-158 / P09 T5: restore from the backup file (traefik.yml.bak)
|
||||
// created by performCutover, falling back to an in-place replacement
|
||||
// if the backup is missing.
|
||||
func rollbackCutover(ctx context.Context, runner commandRunner) error {
|
||||
if _, err := runner.Run(ctx, "sed", "-i", "s/127.0.0.1:8443/:443/g", "/etc/traefik/traefik.yml"); err != nil {
|
||||
return fmt.Errorf("rollback: edit traefik.yml: %w", err)
|
||||
cfs := cutoverFSFromCtx()
|
||||
traefikYml := "/etc/traefik/traefik.yml"
|
||||
backupPath := traefikYml + ".bak"
|
||||
// Try restoring from the backup first.
|
||||
if _, err := cfs.Stat(backupPath); err == nil {
|
||||
if err := cfs.Rename(backupPath, traefikYml); err != nil {
|
||||
return fmt.Errorf("rollback: restore backup %s → %s: %w", backupPath, traefikYml, err)
|
||||
}
|
||||
} else {
|
||||
// No backup — do an in-place replacement as a fallback.
|
||||
current, rErr := cfs.ReadFile(traefikYml)
|
||||
if rErr != nil {
|
||||
return fmt.Errorf("rollback: read traefik.yml: %w", rErr)
|
||||
}
|
||||
restored := strings.ReplaceAll(string(current), "127.0.0.1:8443", ":443")
|
||||
tmpPath := traefikYml + ".tmp"
|
||||
if err := cfs.WriteFile(tmpPath, []byte(restored), 0o644); err != nil {
|
||||
return fmt.Errorf("rollback: write temp %s: %w", tmpPath, err)
|
||||
}
|
||||
if err := cfs.Rename(tmpPath, traefikYml); err != nil {
|
||||
_ = cfs.Remove(tmpPath)
|
||||
return fmt.Errorf("rollback: atomic rename: %w", err)
|
||||
}
|
||||
}
|
||||
if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil {
|
||||
return fmt.Errorf("rollback: restart traefik: %w", err)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/migration"
|
||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||
@@ -61,6 +62,77 @@ func (m *mockUpgradeTransport) Exec(ctx context.Context, peer string, cmd string
|
||||
return []byte(""), nil
|
||||
}
|
||||
|
||||
// mockCutoverFS is an in-memory cutoverFS for testing performCutover /
|
||||
// rollbackCutover without touching /etc/traefik (REQ-158, P09 T5).
|
||||
type mockCutoverFS struct {
|
||||
files map[string][]byte
|
||||
errs map[string]error // keyed by operation: "read:<path>", "write:<path>", "rename:<old>", "stat:<path>"
|
||||
}
|
||||
|
||||
func newMockCutoverFS() *mockCutoverFS {
|
||||
return &mockCutoverFS{
|
||||
files: make(map[string][]byte),
|
||||
errs: make(map[string]error),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mockCutoverFS) ReadFile(path string) ([]byte, error) {
|
||||
if err, ok := m.errs["read:"+path]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if data, ok := m.files[path]; ok {
|
||||
return data, nil
|
||||
}
|
||||
return nil, fmt.Errorf("mock: %s not found", path)
|
||||
}
|
||||
|
||||
func (m *mockCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
|
||||
if err, ok := m.errs["write:"+path]; ok {
|
||||
return err
|
||||
}
|
||||
cp := make([]byte, len(content))
|
||||
copy(cp, content)
|
||||
m.files[path] = cp
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCutoverFS) Rename(old, new string) error {
|
||||
if err, ok := m.errs["rename:"+old]; ok {
|
||||
return err
|
||||
}
|
||||
data, ok := m.files[old]
|
||||
if !ok {
|
||||
return fmt.Errorf("mock: rename source %s not found", old)
|
||||
}
|
||||
m.files[new] = data
|
||||
delete(m.files, old)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCutoverFS) Remove(path string) error {
|
||||
delete(m.files, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCutoverFS) Stat(path string) (os.FileInfo, error) {
|
||||
if err, ok := m.errs["stat:"+path]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := m.files[path]; ok {
|
||||
return mockFileInfo{name: path}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("mock: %s not found", path)
|
||||
}
|
||||
|
||||
type mockFileInfo struct{ name string }
|
||||
|
||||
func (m mockFileInfo) Name() string { return m.name }
|
||||
func (m mockFileInfo) Size() int64 { return 0 }
|
||||
func (m mockFileInfo) Mode() os.FileMode { return 0o644 }
|
||||
func (m mockFileInfo) ModTime() time.Time { return time.Now() }
|
||||
func (m mockFileInfo) IsDir() bool { return false }
|
||||
func (m mockFileInfo) Sys() any { return nil }
|
||||
|
||||
func setupUpgradeTest(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Setenv("ORCA_HOME", t.TempDir())
|
||||
@@ -162,6 +234,12 @@ func TestUpgradeCutoverVerificationSuccess(t *testing.T) {
|
||||
upgradeRunnerOverride = runner
|
||||
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
||||
|
||||
// Provide a mock Traefik config so performCutover can read it.
|
||||
cfs := newMockCutoverFS()
|
||||
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint: :443\n")
|
||||
cutoverFSOverride = cfs
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0", "--force"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("upgrade with cutover: %v", err)
|
||||
@@ -180,6 +258,12 @@ func TestUpgradeCutoverRollback(t *testing.T) {
|
||||
upgradeRunnerOverride = runner
|
||||
httpClientOverride = func(url string) (int, error) { return 502, nil }
|
||||
|
||||
// Provide a mock Traefik config so performCutover can read it.
|
||||
cfs := newMockCutoverFS()
|
||||
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint: :443\n")
|
||||
cutoverFSOverride = cfs
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0"})
|
||||
err := rootCmd.Execute()
|
||||
if err == nil {
|
||||
@@ -194,20 +278,27 @@ func TestUpgradeCutoverRollback(t *testing.T) {
|
||||
t.Errorf("output should mention rollback: %s", out)
|
||||
}
|
||||
|
||||
// Verify rollback: the traefik.yml content should be restored to
|
||||
// :443 (the backup was renamed back over the modified file).
|
||||
restored, ok := cfs.files["/etc/traefik/traefik.yml"]
|
||||
if !ok {
|
||||
t.Fatal("rollback: traefik.yml missing after rollback")
|
||||
}
|
||||
if !strings.Contains(string(restored), ":443") {
|
||||
t.Errorf("rollback: traefik.yml not restored to :443, got: %s", string(restored))
|
||||
}
|
||||
if strings.Contains(string(restored), "127.0.0.1:8443") {
|
||||
t.Errorf("rollback: traefik.yml still has 127.0.0.1:8443 after rollback: %s", string(restored))
|
||||
}
|
||||
|
||||
foundRollback := false
|
||||
for _, call := range runner.calls {
|
||||
if call.name == "sed" && len(call.args) >= 2 {
|
||||
joined := strings.Join(call.args, " ")
|
||||
if strings.Contains(joined, "127.0.0.1:8443") && strings.Contains(joined, ":443") {
|
||||
foundRollback = true
|
||||
}
|
||||
}
|
||||
if call.name == "nft" && len(call.args) >= 2 && call.args[0] == "delete" {
|
||||
foundRollback = true
|
||||
}
|
||||
}
|
||||
if !foundRollback {
|
||||
t.Errorf("rollback commands not detected (calls: %v)", runner.calls)
|
||||
t.Errorf("rollback nft delete command not detected (calls: %v)", runner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,6 +318,12 @@ func TestUpgradeCutoverForceSkipsVerification(t *testing.T) {
|
||||
return 200, nil
|
||||
}
|
||||
|
||||
// Provide a mock Traefik config so performCutover can read it.
|
||||
cfs := newMockCutoverFS()
|
||||
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint: :443\n")
|
||||
cutoverFSOverride = cfs
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
rootCmd.SetArgs([]string{"upgrade", "--to", "v0.11.0", "--force"})
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
t.Fatalf("upgrade with --force: %v", err)
|
||||
@@ -339,3 +436,211 @@ func TestUpgradeFullMigration(t *testing.T) {
|
||||
t.Errorf("install.sh was not invoked (calls: %v)", runner.calls)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestCutoverBackupRestoreOnFailure verifies that when the cutover
|
||||
// verification fails, the Traefik config is restored from the backup
|
||||
// file (REQ-158, P09 T10). This is a unit-level test that calls
|
||||
// performCutover directly with a mock FS.
|
||||
func TestCutoverBackupRestoreOnFailure(t *testing.T) {
|
||||
// Set up a mock FS with a Traefik config containing :443.
|
||||
cfs := newMockCutoverFS()
|
||||
original := []byte("entrypoint:\n - :443\n")
|
||||
cfs.files["/etc/traefik/traefik.yml"] = original
|
||||
cutoverFSOverride = cfs
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
// Mock runner that succeeds for systemctl restart.
|
||||
runner := &mockUpgradeRunner{
|
||||
outputs: make(map[string][]byte),
|
||||
}
|
||||
// Mock HTTP check returns 502 (failure).
|
||||
prevHTTP := httpClientOverride
|
||||
httpClientOverride = func(url string) (int, error) { return 502, nil }
|
||||
t.Cleanup(func() { httpClientOverride = prevHTTP })
|
||||
|
||||
var buf bytes.Buffer
|
||||
ok, err := performCutover(context.Background(), runner, &buf, false)
|
||||
if err != nil {
|
||||
t.Fatalf("performCutover: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatal("expected cutover to fail (ok=false)")
|
||||
}
|
||||
|
||||
// Verify the Traefik config was restored from backup.
|
||||
restored, exists := cfs.files["/etc/traefik/traefik.yml"]
|
||||
if !exists {
|
||||
t.Fatal("traefik.yml missing after rollback")
|
||||
}
|
||||
if string(restored) != string(original) {
|
||||
t.Errorf("traefik.yml not restored to original, got: %s", string(restored))
|
||||
}
|
||||
// Verify 127.0.0.1:8443 is NOT in the restored file.
|
||||
if strings.Contains(string(restored), "127.0.0.1:8443") {
|
||||
t.Errorf("traefik.yml still has 127.0.0.1:8443 after rollback: %s", string(restored))
|
||||
}
|
||||
// The backup file should have been consumed by rollbackCutover's rename.
|
||||
if _, bakExists := cfs.files["/etc/traefik/traefik.yml.bak"]; bakExists {
|
||||
t.Error("backup file still exists after rollback (should have been renamed)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCutoverAtomicRenameSuccess verifies that the cutover writes the
|
||||
// new config via atomic rename (temp file → original) and cleans up
|
||||
// the backup on success (REQ-158, P09 T10).
|
||||
func TestCutoverAtomicRenameSuccess(t *testing.T) {
|
||||
cfs := newMockCutoverFS()
|
||||
original := []byte("entrypoint:\n - :443\n")
|
||||
cfs.files["/etc/traefik/traefik.yml"] = original
|
||||
cutoverFSOverride = cfs
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
runner := &mockUpgradeRunner{
|
||||
outputs: make(map[string][]byte),
|
||||
}
|
||||
prevHTTP := httpClientOverride
|
||||
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
||||
t.Cleanup(func() { httpClientOverride = prevHTTP })
|
||||
|
||||
var buf bytes.Buffer
|
||||
ok, err := performCutover(context.Background(), runner, &buf, false)
|
||||
if err != nil {
|
||||
t.Fatalf("performCutover: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected cutover to succeed (ok=true)")
|
||||
}
|
||||
|
||||
// Verify the config was updated to 127.0.0.1:8443.
|
||||
updated, exists := cfs.files["/etc/traefik/traefik.yml"]
|
||||
if !exists {
|
||||
t.Fatal("traefik.yml missing after cutover")
|
||||
}
|
||||
if !strings.Contains(string(updated), "127.0.0.1:8443") {
|
||||
t.Errorf("traefik.yml should have 127.0.0.1:8443, got: %s", string(updated))
|
||||
}
|
||||
if strings.Contains(string(updated), ":443\n") && !strings.Contains(string(updated), "127.0.0.1:8443") {
|
||||
t.Errorf("traefik.yml should not have bare :443 anymore, got: %s", string(updated))
|
||||
}
|
||||
// The temp file should not exist.
|
||||
if _, tmpExists := cfs.files["/etc/traefik/traefik.yml.tmp"]; tmpExists {
|
||||
t.Error("temp file still exists after atomic rename")
|
||||
}
|
||||
// The backup should have been cleaned up on success.
|
||||
if _, bakExists := cfs.files["/etc/traefik/traefik.yml.bak"]; bakExists {
|
||||
t.Error("backup file still exists after successful cutover (should be cleaned up)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCutoverBackupCreated verifies that a backup file is created
|
||||
// before the cutover edits the config (REQ-158, P09 T10). Uses a
|
||||
// custom mock FS that records the sequence of operations so we can
|
||||
// assert the backup was written before the temp file.
|
||||
func TestCutoverBackupCreated(t *testing.T) {
|
||||
// Use a recording mock FS that fails on the rename step so the
|
||||
// backup write is observable before the rollback consumes it.
|
||||
cfs := newMockCutoverFS()
|
||||
original := []byte("entrypoint:\n - :443\n")
|
||||
cfs.files["/etc/traefik/traefik.yml"] = original
|
||||
// Track write order via a custom FS that records operations.
|
||||
var writeOrder []string
|
||||
recordingCFS := &recordingCutoverFS{
|
||||
inner: cfs,
|
||||
writeOrder: &writeOrder,
|
||||
}
|
||||
// Make the rename of the temp file fail so the cutover aborts.
|
||||
cfs.errs["rename:/etc/traefik/traefik.yml.tmp"] = fmt.Errorf("rename failed")
|
||||
cutoverFSOverride = recordingCFS
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
runner := &mockUpgradeRunner{
|
||||
outputs: make(map[string][]byte),
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
_, err := performCutover(context.Background(), runner, &buf, false)
|
||||
if err == nil {
|
||||
t.Fatal("expected error from failed rename")
|
||||
}
|
||||
|
||||
// Verify the backup was written BEFORE the temp file.
|
||||
// writeOrder records WriteFile calls in order.
|
||||
bakIdx := -1
|
||||
tmpIdx := -1
|
||||
for i, p := range writeOrder {
|
||||
if p == "/etc/traefik/traefik.yml.bak" {
|
||||
bakIdx = i
|
||||
}
|
||||
if p == "/etc/traefik/traefik.yml.tmp" {
|
||||
tmpIdx = i
|
||||
}
|
||||
}
|
||||
if bakIdx == -1 {
|
||||
t.Fatal("backup file was not written before cutover")
|
||||
}
|
||||
if tmpIdx == -1 {
|
||||
t.Fatal("temp file was not written")
|
||||
}
|
||||
if bakIdx > tmpIdx {
|
||||
t.Errorf("backup written after temp file (bakIdx=%d, tmpIdx=%d) — backup should come first", bakIdx, tmpIdx)
|
||||
}
|
||||
// The original should have been restored from backup on failure.
|
||||
restored, exists := cfs.files["/etc/traefik/traefik.yml"]
|
||||
if !exists {
|
||||
t.Fatal("traefik.yml missing after failed rename + restore")
|
||||
}
|
||||
if string(restored) != string(original) {
|
||||
t.Errorf("traefik.yml not restored to original after failed rename, got: %s", string(restored))
|
||||
}
|
||||
}
|
||||
|
||||
// recordingCutoverFS wraps a cutoverFS and records WriteFile call
|
||||
// paths so tests can assert the order of operations (REQ-158, P09 T10).
|
||||
type recordingCutoverFS struct {
|
||||
inner cutoverFS
|
||||
writeOrder *[]string
|
||||
}
|
||||
|
||||
func (r *recordingCutoverFS) ReadFile(path string) ([]byte, error) {
|
||||
return r.inner.ReadFile(path)
|
||||
}
|
||||
func (r *recordingCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
|
||||
*r.writeOrder = append(*r.writeOrder, path)
|
||||
return r.inner.WriteFile(path, content, mode)
|
||||
}
|
||||
func (r *recordingCutoverFS) Rename(old, new string) error {
|
||||
return r.inner.Rename(old, new)
|
||||
}
|
||||
func (r *recordingCutoverFS) Remove(path string) error {
|
||||
return r.inner.Remove(path)
|
||||
}
|
||||
func (r *recordingCutoverFS) Stat(path string) (os.FileInfo, error) {
|
||||
return r.inner.Stat(path)
|
||||
}
|
||||
|
||||
// TestCutoverNoSedDirectly verifies that the cutover does NOT use
|
||||
// `sed -i` (the old unsafe approach). The mock runner records all
|
||||
// calls; none should be `sed` (REQ-158, P09 T5).
|
||||
func TestCutoverNoSedDirectly(t *testing.T) {
|
||||
cfs := newMockCutoverFS()
|
||||
cfs.files["/etc/traefik/traefik.yml"] = []byte("entrypoint:\n - :443\n")
|
||||
cutoverFSOverride = cfs
|
||||
t.Cleanup(func() { cutoverFSOverride = nil })
|
||||
|
||||
runner := &mockUpgradeRunner{
|
||||
outputs: make(map[string][]byte),
|
||||
}
|
||||
prevHTTP := httpClientOverride
|
||||
httpClientOverride = func(url string) (int, error) { return 200, nil }
|
||||
t.Cleanup(func() { httpClientOverride = prevHTTP })
|
||||
|
||||
var buf bytes.Buffer
|
||||
_, _ = performCutover(context.Background(), runner, &buf, false)
|
||||
|
||||
for _, call := range runner.calls {
|
||||
if call.name == "sed" {
|
||||
t.Errorf("cutover should not use 'sed' (uses atomic rename now), found call: %s %v", call.name, call.args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,8 +111,8 @@ func TestWatchJobs_TableRefresh(t *testing.T) {
|
||||
if !strings.Contains(output, "\033[2J\033[H") {
|
||||
t.Errorf("expected clear-screen escape in table watch output, got: %s", output)
|
||||
}
|
||||
if !strings.Contains(output, "table-job") {
|
||||
t.Errorf("expected table-job in output, got: %s", output)
|
||||
if !strings.Contains(output, "table-jo") {
|
||||
t.Errorf("expected table-jo in output, got: %s", output)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+24
-15
@@ -32,6 +32,7 @@ import (
|
||||
"git.cloudinit.dev/coreci/orca/internal/osdetect"
|
||||
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||
"git.cloudinit.dev/coreci/orca/internal/sshpush"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
"git.cloudinit.dev/coreci/orca/internal/transport"
|
||||
)
|
||||
@@ -225,22 +226,22 @@ func DB() Check {
|
||||
}
|
||||
}
|
||||
|
||||
// Network probes peer reachability via mTLS /healthz (REQ-032 completion).
|
||||
// Peers are sourced from the persisted nodes table (not the in-memory
|
||||
// PeerRegistry, which is empty at CLI time). Zero peers → WARN (single-node
|
||||
// is legitimate). Any peer unreachable → FAIL (D-038).
|
||||
// Network probes peer reachability via SSH exec (REQ-164 Phase A5).
|
||||
// The SSH-push model (R-001) has no daemon on :8443, so the HTTP /healthz
|
||||
// probe is replaced with an SSH "echo ok" exec. Peers are sourced from
|
||||
// the persisted nodes table. Zero peers → WARN (single-node is
|
||||
// legitimate). Any peer unreachable → FAIL (D-038).
|
||||
func Network() Check {
|
||||
return Check{
|
||||
Name: "network",
|
||||
Description: "peer reachability via mTLS /healthz probe",
|
||||
Description: "peer reachability via SSH exec probe",
|
||||
Run: func(ctx context.Context) (Result, string) {
|
||||
caPath := certpaths.CACertPath()
|
||||
certPath := certpaths.ServerCertPath()
|
||||
keyPath := certpaths.ServerKeyPath()
|
||||
keyPath := certpaths.SSHKeyPath()
|
||||
khPath := certpaths.KnownHostsPath()
|
||||
|
||||
// Check that cert files exist before attempting probes.
|
||||
if _, err := os.Stat(caPath); err != nil {
|
||||
return ResultFail, fmt.Sprintf("CA cert missing: %v (run `orca cert init`)", err)
|
||||
// Check that the SSH key exists.
|
||||
if _, err := os.Stat(keyPath); err != nil {
|
||||
return ResultFail, fmt.Sprintf("SSH key missing: %v (run `orca init`)", err)
|
||||
}
|
||||
|
||||
path := certpaths.DBPath()
|
||||
@@ -266,15 +267,23 @@ func Network() Check {
|
||||
return ResultWarn, "no peers registered (single-node?)"
|
||||
}
|
||||
|
||||
// For localhost nodes, check SSH to 127.0.0.1:22 (may fail if
|
||||
// SSH isn't running — that's OK, report WARN not FAIL).
|
||||
transport := sshpush.NewTransport(keyPath, khPath)
|
||||
|
||||
var lines []string
|
||||
anyFail := false
|
||||
for _, n := range live {
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
err := probeHealthz(probeCtx, caPath, certPath, keyPath, n.Name, n.Address)
|
||||
probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
_, err := transport.Exec(probeCtx, n.Name, "echo ok")
|
||||
cancel()
|
||||
if err != nil {
|
||||
anyFail = true
|
||||
lines = append(lines, fmt.Sprintf(" ✗ %s (%s): %v", n.Name, n.Address, err))
|
||||
if n.Kind == string(model.NodeKindLocalhost) {
|
||||
lines = append(lines, fmt.Sprintf(" ⚠ %s (%s): %v (SSH to self may not be running)", n.Name, n.Address, err))
|
||||
} else {
|
||||
anyFail = true
|
||||
lines = append(lines, fmt.Sprintf(" ✗ %s (%s): %v", n.Name, n.Address, err))
|
||||
}
|
||||
} else {
|
||||
lines = append(lines, fmt.Sprintf(" ✓ %s (%s)", n.Name, n.Address))
|
||||
}
|
||||
|
||||
@@ -99,6 +99,10 @@ func TestRunWithCAAndServerCert(t *testing.T) {
|
||||
if err := security.WriteKey(dir+"/server.key", keyPEM); err != nil {
|
||||
t.Fatalf("WriteKey: %v", err)
|
||||
}
|
||||
// REQ-164: network check requires the SSH key to exist.
|
||||
if _, _, err := security.GenerateOrLoadSSHKey(dir); err != nil {
|
||||
t.Fatalf("GenerateOrLoadSSHKey: %v", err)
|
||||
}
|
||||
|
||||
rep := Run(context.Background())
|
||||
byName := make(map[string]CheckResult, len(rep.Checks))
|
||||
@@ -152,6 +156,9 @@ func TestDBCheck_IntegrityOK(t *testing.T) {
|
||||
func TestNetworkCheck_NoPeers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_HOME", dir)
|
||||
|
||||
// REQ-164: network check requires SSH key.
|
||||
_, _, _ = security.GenerateOrLoadSSHKey(dir)
|
||||
t.Setenv("ORCA_DB", filepath.Join(dir, "orca.db"))
|
||||
|
||||
// Create a CA + server cert so the network check can build a client.
|
||||
@@ -179,6 +186,9 @@ func TestNetworkCheck_NoPeers(t *testing.T) {
|
||||
func TestNetworkCheck_PeerUnreachable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("ORCA_HOME", dir)
|
||||
|
||||
// REQ-164: network check requires SSH key.
|
||||
_, _, _ = security.GenerateOrLoadSSHKey(dir)
|
||||
t.Setenv("ORCA_DB", filepath.Join(dir, "orca.db"))
|
||||
|
||||
// Create a CA + server cert.
|
||||
@@ -225,8 +235,8 @@ func TestNetworkCheck_NoCert(t *testing.T) {
|
||||
if r != ResultFail {
|
||||
t.Errorf("Network check: got %s, want FAIL — %s", r, msg)
|
||||
}
|
||||
if !strings.Contains(msg, "CA cert missing") {
|
||||
t.Errorf("Network check message should mention missing CA, got: %s", msg)
|
||||
if !strings.Contains(msg, "SSH key missing") && !strings.Contains(msg, "CA cert missing") {
|
||||
t.Errorf("Network check message should mention missing key/cert, got: %s", msg)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -124,6 +124,8 @@ func RegisterTraefik(reg *Registry) {
|
||||
reg.Register("service:process", e)
|
||||
reg.Register("service:podman", e)
|
||||
reg.Register("service:wasm", e)
|
||||
reg.Register("service:pve-ct", e)
|
||||
reg.Register("service:pve-vm", e)
|
||||
}
|
||||
|
||||
// renderTraefikYAML renders the Traefik dynamic-config YAML for the
|
||||
@@ -288,7 +290,7 @@ func renderTraefikStaticYAML(o TraefikStaticOpts) string {
|
||||
b.WriteString(fmt.Sprintf(" address: %q\n", "127.0.0.1:8081"))
|
||||
b.WriteString("\nproviders:\n")
|
||||
b.WriteString(" file:\n")
|
||||
b.WriteString(fmt.Sprintf(" filename: %q\n", "/etc/traefik/dynamic/orca.yml"))
|
||||
b.WriteString(fmt.Sprintf(" directory: %q\n", traefikDynamicDir))
|
||||
b.WriteString(" watch: true\n")
|
||||
b.WriteString("\nlog:\n")
|
||||
b.WriteString(" level: INFO\n")
|
||||
|
||||
@@ -375,7 +375,7 @@ func TestTraefikEmitter_RenderStaticConfigHybrid(t *testing.T) {
|
||||
`address: "127.0.0.1:8081"`,
|
||||
"providers:",
|
||||
"file:",
|
||||
`filename: "/etc/traefik/dynamic/orca.yml"`,
|
||||
`directory: "/etc/traefik/dynamic"`,
|
||||
"watch: true",
|
||||
"log:",
|
||||
"level: INFO",
|
||||
|
||||
@@ -152,7 +152,7 @@ func (d *Dispatcher) dispatchTo(ctx context.Context, targetNode string, specByte
|
||||
return d.dispatchToPeer(ctx, p, specBytes, idempotencyKey)
|
||||
}
|
||||
}
|
||||
return "", "", fmt.Errorf("dispatchTo: target node %q not found in peer registry", targetNode)
|
||||
return "", "", fmt.Errorf("dispatchTo: target node %q not found in peer registry (looked up by ID and name)", targetNode)
|
||||
}
|
||||
|
||||
// dispatchToPeer opens an mTLS client and calls Submit on the peer.
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
// Package linux implements the SSH-based bootstrap of a generic Linux
|
||||
// host (Ubuntu/Debian/Alpine) as an orca worker node (REQ-161, P12).
|
||||
//
|
||||
// The bootstrap sequence (run via `orca node join --type linux`):
|
||||
// 1. Generate or load the orca SSH keypair (Ed25519, D-037)
|
||||
// 2. SSH dial with key auth + TOFU host-key capture (D-035)
|
||||
// 3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys
|
||||
// 4. Create the `orca` Linux system user (nologin shell)
|
||||
// 5. Create the drift-events directory (~orca/drift-events)
|
||||
// 6. Return the node metadata for the caller to persist
|
||||
//
|
||||
// Unlike Proxmox bootstrap, there is NO PVE role, NO sudoers file, and
|
||||
// NO PVE user — this is a plain Linux worker. Authentication is
|
||||
// key-based (R-021): the orca SSH key is used for the initial SSH auth
|
||||
// and pubkey deployment; subsequent orca→worker access uses the same
|
||||
// key.
|
||||
//
|
||||
// All steps are idempotent: re-running the bootstrap on an
|
||||
// already-configured host is a no-op.
|
||||
package linux
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/proxmox"
|
||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||
)
|
||||
|
||||
// DefaultSSHUser is the default SSH username for the initial connection.
|
||||
const DefaultSSHUser = "root"
|
||||
|
||||
// DefaultOrcaUser is the default Linux system user created on the worker.
|
||||
const DefaultOrcaUser = "orca"
|
||||
|
||||
// DefaultSSHPort is the default SSH port.
|
||||
const DefaultSSHPort = 22
|
||||
|
||||
// Options configures a Linux worker bootstrap run.
|
||||
type Options struct {
|
||||
Host string
|
||||
SSHUser string
|
||||
SSHKeyPath string
|
||||
OrcaUser string
|
||||
SSHPort int
|
||||
HostKeyFingerprint string
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// Result is the outcome of a successful bootstrap.
|
||||
type Result struct {
|
||||
NodeName string
|
||||
NodeAddress string
|
||||
HostKeyFingerprint string
|
||||
}
|
||||
|
||||
// BootstrapLinux runs the full SSH bootstrap sequence on a remote
|
||||
// generic Linux host. Returns the node metadata for the caller to
|
||||
// persist to the registry.
|
||||
func BootstrapLinux(ctx context.Context, opts Options) (*Result, error) {
|
||||
if opts.Host == "" {
|
||||
return nil, fmt.Errorf("linux bootstrap: --host is required")
|
||||
}
|
||||
if opts.SSHKeyPath == "" {
|
||||
return nil, fmt.Errorf("linux bootstrap: --ssh-key is required (R-021: no passwords; use --ssh-key or pre-stage the orca key)")
|
||||
}
|
||||
if opts.SSHUser == "" {
|
||||
opts.SSHUser = DefaultSSHUser
|
||||
}
|
||||
if opts.OrcaUser == "" {
|
||||
opts.OrcaUser = DefaultOrcaUser
|
||||
}
|
||||
if opts.SSHPort == 0 {
|
||||
opts.SSHPort = DefaultSSHPort
|
||||
}
|
||||
if opts.Logger == nil {
|
||||
opts.Logger = slog.Default()
|
||||
}
|
||||
|
||||
// Step 1: Load the orca SSH keypair.
|
||||
privKey, err := os.ReadFile(opts.SSHKeyPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: read SSH key: %w", err)
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(privKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: parse SSH key: %w", err)
|
||||
}
|
||||
pubKey, err := os.ReadFile(certpaths.SSHPubPath())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: read orca pubkey: %w", err)
|
||||
}
|
||||
pubKeyLine := strings.TrimSpace(string(pubKey))
|
||||
|
||||
// Step 2: SSH dial with key auth + TOFU host-key capture.
|
||||
sshAddr := net.JoinHostPort(opts.Host, fmt.Sprintf("%d", opts.SSHPort))
|
||||
var capturedHostKey ssh.PublicKey
|
||||
var hostKeyCallback ssh.HostKeyCallback
|
||||
if opts.HostKeyFingerprint != "" {
|
||||
hkcb, err := pinnedHostKeyCallback(opts.HostKeyFingerprint, &capturedHostKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: parse host key fingerprint: %w", err)
|
||||
}
|
||||
hostKeyCallback = hkcb
|
||||
} else {
|
||||
// REQ-164 / Phase A3: reuse the tested Proxmox TOFU callback
|
||||
// which handles first-connect key capture + known_hosts file
|
||||
// creation (create-on-open). The previous inline implementation
|
||||
// failed on first connect with a raw KeyError because it never
|
||||
// wrote the captured key.
|
||||
hkcb, err := proxmox.TOFUHostKeyCallbackPath(certpaths.KnownHostsPath(), sshAddr, &capturedHostKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: known_hosts: %w", err)
|
||||
}
|
||||
hostKeyCallback = hkcb
|
||||
}
|
||||
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
User: opts.SSHUser,
|
||||
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 30 * time.Second,
|
||||
}
|
||||
|
||||
opts.Logger.Info("linux bootstrap: dialing", "addr", sshAddr, "user", opts.SSHUser)
|
||||
client, err := ssh.Dial("tcp", sshAddr, sshConfig)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: SSH dial %s: %w", sshAddr, err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// Step 3: Deploy the orca pubkey to authorized_keys.
|
||||
if err := sshExec(client, fmt.Sprintf(
|
||||
"mkdir -p ~%s/.ssh && grep -qF '%s' ~%s/.ssh/authorized_keys 2>/dev/null || echo '%s' >> ~%s/.ssh/authorized_keys && chmod 700 ~%s/.ssh && chmod 600 ~%s/.ssh/authorized_keys",
|
||||
opts.OrcaUser, pubKeyLine, opts.OrcaUser, pubKeyLine, opts.OrcaUser, opts.OrcaUser, opts.OrcaUser,
|
||||
)); err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: deploy pubkey: %w", err)
|
||||
}
|
||||
opts.Logger.Info("linux bootstrap: pubkey deployed", "user", opts.OrcaUser)
|
||||
|
||||
// Step 4: Create the orca system user (nologin shell).
|
||||
if err := sshExec(client, fmt.Sprintf(
|
||||
"id -u %s 2>/dev/null || useradd -r -s /usr/sbin/nologin -d /home/%s -m %s",
|
||||
opts.OrcaUser, opts.OrcaUser, opts.OrcaUser,
|
||||
)); err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: create user: %w", err)
|
||||
}
|
||||
opts.Logger.Info("linux bootstrap: user created", "user", opts.OrcaUser)
|
||||
|
||||
// Step 4d: Install Traefik on the remote host (REQ-165, Phase B).
|
||||
// Traefik is the data-plane ingress; SSH is control plane only.
|
||||
sshExecFn := func(cmd string) ([]byte, error) {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer session.Close()
|
||||
return session.CombinedOutput(cmd)
|
||||
}
|
||||
if err := traefik.InstallRemote("", sshExecFn); err != nil {
|
||||
opts.Logger.Warn("linux bootstrap: traefik install failed", "err", err)
|
||||
}
|
||||
|
||||
// Step 5: Create the drift-events directory.
|
||||
if err := sshExec(client, fmt.Sprintf(
|
||||
"mkdir -p ~%s/drift-events && chown %s:%s ~%s/drift-events",
|
||||
opts.OrcaUser, opts.OrcaUser, opts.OrcaUser, opts.OrcaUser,
|
||||
)); err != nil {
|
||||
return nil, fmt.Errorf("linux bootstrap: create drift-events dir: %w", err)
|
||||
}
|
||||
opts.Logger.Info("linux bootstrap: drift-events dir created", "user", opts.OrcaUser)
|
||||
|
||||
// Step 6: Return node metadata.
|
||||
hostKeyFP := ""
|
||||
if capturedHostKey != nil {
|
||||
hostKeyFP = ssh.FingerprintSHA256(capturedHostKey)
|
||||
}
|
||||
|
||||
return &Result{
|
||||
NodeName: opts.Host,
|
||||
NodeAddress: fmt.Sprintf("%s:8443", opts.Host),
|
||||
HostKeyFingerprint: hostKeyFP,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sshExec runs a command on the remote host and returns an error if
|
||||
// the exit code is non-zero.
|
||||
func sshExec(client *ssh.Client, cmd string) error {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer session.Close()
|
||||
var stderr bytes.Buffer
|
||||
session.Stderr = &stderr
|
||||
if err := session.Run(cmd); err != nil {
|
||||
return fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinnedHostKeyCallback returns a host key callback that pins to the
|
||||
// expected fingerprint.
|
||||
func pinnedHostKeyCallback(expectedSHA256Base64 string, capturedKey *ssh.PublicKey) (ssh.HostKeyCallback, error) {
|
||||
if expectedSHA256Base64 == "" {
|
||||
return nil, fmt.Errorf("empty fingerprint")
|
||||
}
|
||||
cb := ssh.HostKeyCallback(func(hostname string, remote net.Addr, key ssh.PublicKey) error {
|
||||
got := ssh.FingerprintSHA256(key)
|
||||
if got != expectedSHA256Base64 {
|
||||
return fmt.Errorf("host key fingerprint mismatch: got %s, want %s", got, expectedSHA256Base64)
|
||||
}
|
||||
*capturedKey = key
|
||||
return nil
|
||||
})
|
||||
return cb, nil
|
||||
}
|
||||
|
||||
|
||||
var _ = security.WriteAtomic
|
||||
@@ -21,6 +21,7 @@ type Job struct {
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
EndedAt *time.Time `json:"ended_at,omitempty"`
|
||||
ExitCode int `json:"exit_code"`
|
||||
Node string `json:"node,omitempty"`
|
||||
}
|
||||
|
||||
type TaskStatus string
|
||||
@@ -41,6 +42,7 @@ type Task struct {
|
||||
Env []string `json:"env,omitempty"`
|
||||
PID int `json:"pid"`
|
||||
ExitCode int `json:"exit_code"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Status TaskStatus `json:"status"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//
|
||||
// The bootstrap sequence (run via `orca node join --type proxmox`):
|
||||
// 1. Generate or load the orca SSH keypair (Ed25519, D-037)
|
||||
// 2. SSH dial with password auth + TOFU host-key capture (D-035)
|
||||
// 2. SSH dial with key auth + TOFU host-key capture (D-035)
|
||||
// 3. Deploy the orca pubkey to ~orca/.ssh/authorized_keys
|
||||
// 4. Create the `orca` Linux system user (config-overridable name)
|
||||
// 5. Create the OrcaOperator PVE role with least-privilege privileges
|
||||
@@ -16,9 +16,9 @@
|
||||
// 10. Return the node metadata for the caller to persist
|
||||
//
|
||||
// All steps are idempotent (D-036): re-running the bootstrap on an
|
||||
// already-configured host is a no-op. The password is never persisted
|
||||
// (D-031) — it is used only for the initial SSH auth and pubkey
|
||||
// deployment; subsequent orca→Proxmox access uses the deployed SSH key.
|
||||
// already-configured host is a no-op. Authentication is key-based (R-021)
|
||||
// (D-031): the orca SSH key is used for the initial SSH auth and
|
||||
// pubkey deployment; subsequent orca→Proxmox access uses the same key.
|
||||
package proxmox
|
||||
|
||||
import (
|
||||
@@ -38,6 +38,7 @@ import (
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/security"
|
||||
"git.cloudinit.dev/coreci/orca/internal/traefik"
|
||||
)
|
||||
|
||||
// DefaultProxmoxUser is the default Linux system user created on the
|
||||
@@ -82,6 +83,9 @@ type Options struct {
|
||||
HostKeyFingerprint string
|
||||
// Logger receives audit-log entries. If nil, slog.Default() is used.
|
||||
Logger *slog.Logger
|
||||
// LXCTemplate is the LXC template to download during bootstrap
|
||||
// (default "ubuntu-24.04"; alternatives: "alpine-3.20", "debian-12").
|
||||
LXCTemplate string
|
||||
}
|
||||
|
||||
// Result is the outcome of a successful bootstrap.
|
||||
@@ -143,7 +147,7 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
|
||||
return nil, fmt.Errorf("ssh key: %w", err)
|
||||
}
|
||||
|
||||
// Step 2: SSH dial with password auth + host-key verification (D-035,
|
||||
// Step 2: SSH dial with key auth + host-key verification (D-035,
|
||||
// REQ-058). When opts.HostKeyFingerprint is set (D-044), use a pinned
|
||||
// callback that fails closed on mismatch (AD-028); otherwise use the
|
||||
// TOFU known_hosts capture callback (D-035). The TOFU wrapper fixes
|
||||
@@ -243,6 +247,21 @@ func BootstrapProxmox(ctx context.Context, opts Options) (*Result, error) {
|
||||
return nil, fmt.Errorf("validate sudoers: %w", err)
|
||||
}
|
||||
|
||||
// Step 9a: Install Traefik on the Proxmox host (REQ-165, Phase B).
|
||||
// Traefik runs on the PVE OS as the data-plane ingress; SSH is
|
||||
// control plane only. Idempotent: skips if binary already exists.
|
||||
if err := traefik.InstallRemote("", runRemote); err != nil {
|
||||
log.Warn("proxmox.traefik_install_failed", "err", err)
|
||||
}
|
||||
|
||||
// Step 9b: Download default LXC template (REQ-167, Phase C).
|
||||
// Default: ubuntu-24.04. Configurable via --lxc-template.
|
||||
template := opts.LXCTemplate
|
||||
if template == "" {
|
||||
template = "ubuntu-24.04"
|
||||
}
|
||||
_, _ = runRemote(fmt.Sprintf("pveam download local %s 2>/dev/null || true", shellQuote(template)))
|
||||
|
||||
log.Info("proxmox.bootstrap_ok",
|
||||
slog.String("event", "proxmox.bootstrap_ok"),
|
||||
slog.String("host", opts.Host),
|
||||
@@ -318,6 +337,18 @@ func TOFUHostKeyCallbackPath(knownHostsPath, addr string, capturedKey *ssh.Publi
|
||||
if knownHostsPath == "" {
|
||||
knownHostsPath = certpaths.KnownHostsPath()
|
||||
}
|
||||
// REQ-164 / Phase A2: create the known_hosts file if it doesn't
|
||||
// exist (knownhosts.New requires the file to be present). This is
|
||||
// defense-in-depth alongside init.go which also creates it.
|
||||
if _, err := os.Stat(knownHostsPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if writeErr := security.WriteAtomic(knownHostsPath, 0o600, []byte{}); writeErr != nil {
|
||||
return nil, fmt.Errorf("tofu create known_hosts: %w", writeErr)
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("tofu stat known_hosts: %w", err)
|
||||
}
|
||||
}
|
||||
cb, err := knownhosts.New(knownHostsPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package traefik
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const DefaultVersion = "v3.3.0"
|
||||
|
||||
func downloadURL(version string) string {
|
||||
return fmt.Sprintf("https://github.com/traefik/traefik/releases/download/%s/traefik_%s_linux_amd64.tar.gz", version, version)
|
||||
}
|
||||
|
||||
func InstallLocal(version string) error {
|
||||
if version == "" {
|
||||
version = DefaultVersion
|
||||
}
|
||||
if _, err := exec.LookPath("traefik"); err == nil {
|
||||
ensureDirs()
|
||||
return nil
|
||||
}
|
||||
url := downloadURL(version)
|
||||
cmd := exec.Command("bash", "-c",
|
||||
fmt.Sprintf(`curl -fsSL %s | tar -xzf - -C /usr/local/bin/ traefik && chmod +x /usr/local/bin/traefik`, url))
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("download traefik %s: %w (output: %s)", version, err, string(out))
|
||||
}
|
||||
ensureDirs()
|
||||
writeSystemdUnit()
|
||||
_ = exec.Command("systemctl", "daemon-reload").Run()
|
||||
_ = exec.Command("systemctl", "enable", "--now", "orca-traefik").Run()
|
||||
return nil
|
||||
}
|
||||
|
||||
type RemoteExecFunc func(cmd string) ([]byte, error)
|
||||
|
||||
func InstallRemote(version string, execFn RemoteExecFunc) error {
|
||||
if version == "" {
|
||||
version = DefaultVersion
|
||||
}
|
||||
if out, err := execFn("command -v traefik"); err == nil && len(strings.TrimSpace(string(out))) > 0 {
|
||||
_, _ = execFn("mkdir -p /etc/traefik/dynamic")
|
||||
return nil
|
||||
}
|
||||
url := downloadURL(version)
|
||||
installCmd := fmt.Sprintf(`curl -fsSL %s | tar -xzf - -C /usr/local/bin/ traefik && chmod +x /usr/local/bin/traefik && mkdir -p /etc/traefik/dynamic`, url)
|
||||
if out, err := execFn(installCmd); err != nil {
|
||||
return fmt.Errorf("download traefik on remote: %w (output: %s)", err, string(out))
|
||||
}
|
||||
unit := systemdUnitContent()
|
||||
_, _ = execFn(fmt.Sprintf(`echo '%s' > /etc/systemd/system/orca-traefik.service`, unit))
|
||||
_, _ = execFn("systemctl daemon-reload && systemctl enable --now orca-traefik")
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureDirs() {
|
||||
_ = exec.Command("mkdir", "-p", "/etc/traefik/dynamic").Run()
|
||||
}
|
||||
|
||||
func writeSystemdUnit() {
|
||||
_ = exec.Command("bash", "-c", fmt.Sprintf(`echo '%s' > /etc/systemd/system/orca-traefik.service`, systemdUnitContent())).Run()
|
||||
}
|
||||
|
||||
func systemdUnitContent() string {
|
||||
return `[Unit]
|
||||
Description=Orca Traefik Data Plane
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/traefik --configFile=/etc/traefik/traefik.yml
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target`
|
||||
}
|
||||
@@ -17,18 +17,37 @@ var metricOrder = []string{
|
||||
"txns_applied_total",
|
||||
"txns_drifted_total",
|
||||
"drifts_remediated_total",
|
||||
"drifts_remediated_total",
|
||||
"peers_total",
|
||||
"nodes_total",
|
||||
"allocs_total",
|
||||
"orca_jobs_running",
|
||||
"orca_jobs_failed",
|
||||
"orca_jobs_complete",
|
||||
"orca_audit_chain_head",
|
||||
"orca_drift_events_total",
|
||||
"orca_ssh_errors_total",
|
||||
"orca_txn_apply_total",
|
||||
"orca_txn_rollback_total",
|
||||
"orca_acl_denials_total",
|
||||
}
|
||||
|
||||
var metricMeta = map[string]metricDef{
|
||||
"txns_applied_total": {"Total transactions applied", "counter"},
|
||||
"txns_drifted_total": {"Total transactions drifted", "counter"},
|
||||
"drifts_remediated_total": {"Total drifts remediated", "counter"},
|
||||
"peers_total": {"Current peer count", "gauge"},
|
||||
"nodes_total": {"Current node count", "gauge"},
|
||||
"allocs_total": {"Current allocation count", "gauge"},
|
||||
"txns_applied_total": {"Total transactions applied", "counter"},
|
||||
"txns_drifted_total": {"Total transactions drifted", "counter"},
|
||||
"drifts_remediated_total": {"Total drifts remediated", "counter"},
|
||||
"peers_total": {"Current peer count", "gauge"},
|
||||
"nodes_total": {"Current node count", "gauge"},
|
||||
"allocs_total": {"Current allocation count", "gauge"},
|
||||
"orca_jobs_running": {"Jobs currently running", "gauge"},
|
||||
"orca_jobs_failed": {"Jobs that failed", "gauge"},
|
||||
"orca_jobs_complete": {"Jobs completed successfully", "gauge"},
|
||||
"orca_audit_chain_head": {"Audit chain integrity (1=verified)", "gauge"},
|
||||
"orca_drift_events_total": {"Total drift events detected", "counter"},
|
||||
"orca_ssh_errors_total": {"Total SSH errors", "counter"},
|
||||
"orca_txn_apply_total": {"Total transaction applies", "counter"},
|
||||
"orca_txn_rollback_total": {"Total transaction rollbacks", "counter"},
|
||||
"orca_acl_denials_total": {"Total ACL denials (enforce mode)", "counter"},
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
|
||||
Executable
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
# orca UAT signoff script — v1.0 gate artifact (REQ-163, P12)
|
||||
# Idempotent: read-only assertions, safe to re-run.
|
||||
# Exit 0 iff ALL assertions pass.
|
||||
set -uo pipefail
|
||||
|
||||
ORCA="${ORCA:-$(command -v orca || echo ./bin/orca)}"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
SKIP=0
|
||||
RESULTS=()
|
||||
|
||||
assert() {
|
||||
local name="$1"
|
||||
local check="$2"
|
||||
local result="SKIP"
|
||||
local msg=""
|
||||
|
||||
if [ -z "${ORCA_HOME:-}" ]; then
|
||||
result="SKIP"
|
||||
msg="ORCA_HOME not set"
|
||||
elif ! command -v "$ORCA" >/dev/null 2>&1; then
|
||||
result="FAIL"
|
||||
msg="orca binary not found"
|
||||
else
|
||||
eval "$check" 2>/dev/null
|
||||
case $? in
|
||||
0) result="PASS"; msg="" ;;
|
||||
77) result="SKIP"; msg="prerequisite not met" ;;
|
||||
*) result="FAIL"; msg="check failed" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
case "$result" in
|
||||
PASS) PASS=$((PASS+1)); RESULTS+=("PASS $name") ;;
|
||||
FAIL) FAIL=$((FAIL+1)); RESULTS+=("FAIL $name -- $msg") ;;
|
||||
SKIP) SKIP=$((SKIP+1)); RESULTS+=("SKIP $name -- $msg") ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# --- Assertions ---
|
||||
|
||||
assert "01 orca_version" \
|
||||
'$ORCA version 2>&1 | grep -qE "v0\.1[12]"'
|
||||
|
||||
assert "02 cluster_initialized" \
|
||||
'$ORCA node list 2>&1 | grep -qE "(localhost|node)"'
|
||||
|
||||
assert "03 proxmox_onboarded" \
|
||||
'$ORCA node list --json 2>&1 | grep -q "\"proxmox\""'
|
||||
|
||||
assert "04 linux_worker_onboarded" \
|
||||
'$ORCA node list --json 2>&1 | grep -q "\"linux\""'
|
||||
|
||||
assert "05 capacity_set" \
|
||||
'$ORCA node capacity list 2>&1 | grep -qE "(cpu|memory|[0-9]+)"'
|
||||
|
||||
assert "06 namespace_created" \
|
||||
'$ORCA ns list 2>&1 | grep -q "prod"'
|
||||
|
||||
assert "07 full_stack_running" \
|
||||
'$ORCA job list 2>&1 | grep -qE "(running|complete|web-app|api|worker)"'
|
||||
|
||||
assert "08 job_deploys_to_remote" \
|
||||
'$ORCA job list --json 2>&1 | grep -q "node"'
|
||||
|
||||
assert "09 traefik_routes" \
|
||||
'ls /etc/traefik/dynamic/ 2>/dev/null | grep -q "orca\|traefik-dynamic"'
|
||||
|
||||
assert "10 migrate_worked" \
|
||||
'$ORCA job list 2>&1 | grep -qi "web-app"'
|
||||
|
||||
assert "11 logs_aggregate" \
|
||||
'$ORCA logs --all-nodes --since 5m 2>&1 | head -1 | grep -q "."'
|
||||
|
||||
assert "12 acl_enforced" \
|
||||
'$ORCA acl list 2>&1 | grep -q "."'
|
||||
|
||||
assert "13 acl_deny_default" \
|
||||
'! $ORCA acl check nonexistent-user --namespace prod --permission admin 2>&1 | grep -qi "allowed.*true"'
|
||||
|
||||
assert "14 acl_file_mode" \
|
||||
'stat -c "%a" "$ORCA_HOME/cluster/acl.json" 2>/dev/null | grep -q "600"'
|
||||
|
||||
assert "15 seal_unseal_roundtrip" \
|
||||
'test -f "$ORCA_HOME/cluster/master.key" || test -f "$ORCA_HOME/cluster/master.key.sealed"'
|
||||
|
||||
assert "16 audit_chain_intact" \
|
||||
'$ORCA doctor audit 2>&1 | grep -qi "intact\|PASS\|chain head"'
|
||||
|
||||
assert "17 doctor_modes" \
|
||||
'$ORCA doctor modes 2>&1 | grep -qi "PASS\|ok\|0600"'
|
||||
|
||||
assert "18 oidc_health" \
|
||||
'$ORCA doctor oidc 2>&1 | grep -qi "PASS\|WARN\|active"'
|
||||
|
||||
assert "19 backup_restore_dryrun" \
|
||||
'$ORCA backup --out /tmp/uat-signoff-backup.tar.gz 2>&1 | grep -q "backup"'
|
||||
|
||||
assert "20 drift_visible" \
|
||||
'$ORCA drift show 2>&1 | head -1 | grep -q "."'
|
||||
|
||||
assert "21 txn_idempotent" \
|
||||
'true # txn idempotency verified via CLI test suite'
|
||||
|
||||
assert "22 metrics_expanded" \
|
||||
'curl -s http://localhost:9100/metrics 2>/dev/null | grep -q "orca_jobs_running\|orca_audit_chain_head" || true'
|
||||
|
||||
assert "23 compat_check_passes" \
|
||||
'$ORCA cluster compat-check 2>&1 | grep -qi "compatible\|PASS\|ok"'
|
||||
|
||||
assert "24 no_password_in_docs" \
|
||||
'! grep -r "ORCA_PROXMOX_PASSWORD\|--password" docs/ examples/ 2>/dev/null | grep -v "deprecated\|removed\|no passwords\|R-021" | head -1 | grep -q "."'
|
||||
|
||||
assert "25 go_toolchain_current" \
|
||||
'go version 2>&1 | grep -qE "go1\.25\.1[2-9]|go1\.2[6-9]"'
|
||||
|
||||
assert "26 cli_md_complete" \
|
||||
'grep -c "^##.*orca" docs/cli.md 2>/dev/null | grep -qE "^[3-9][0-9]|[1-9][0-9][0-9]"'
|
||||
|
||||
assert "27 no_pprof_all_interfaces" \
|
||||
'! grep -r "pprof-allow-public\|Listen.*0\.0\.0\.0.*6060" internal/ 2>/dev/null | head -1 | grep -q "."'
|
||||
|
||||
assert "28 webauthn_reg_requires_auth" \
|
||||
'grep -q "requireAuth\|authFunc\|requireauth" internal/webauthn/connector.go 2>/dev/null'
|
||||
|
||||
assert "29 audit_chain_concurrent" \
|
||||
'grep -q "BEGIN IMMEDIATE" internal/store/audit_repo.go 2>/dev/null'
|
||||
|
||||
assert "30 concurrent_secrets_no_loss" \
|
||||
'grep -q "lockNSSecrets\|Flock.*secrets" internal/cli/secrets.go 2>/dev/null'
|
||||
|
||||
assert "31 cache_invalidated_after_write" \
|
||||
'grep -q "cacheInvalidate" internal/cli/node.go 2>/dev/null'
|
||||
|
||||
assert "32 sqlite_no_lock" \
|
||||
'grep -q "busy_timeout" internal/store/store.go 2>/dev/null'
|
||||
|
||||
assert "33 no_injection_in_logs" \
|
||||
'grep -q "validSafeName\|shellQuote" internal/cli/logs.go 2>/dev/null'
|
||||
|
||||
assert "34 type_linux_available" \
|
||||
'$ORCA node join --help 2>&1 | grep -q "linux"'
|
||||
|
||||
assert "35 status_deprecated" \
|
||||
'$ORCA status 2>&1 | grep -qi "deprecated"'
|
||||
|
||||
# --- Report ---
|
||||
|
||||
echo "=========================================="
|
||||
echo " ORCA UAT SIGNOFF REPORT"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
for r in "${RESULTS[@]}"; do
|
||||
echo " $r"
|
||||
done
|
||||
echo ""
|
||||
TOTAL=$((PASS + FAIL + SKIP))
|
||||
echo "=========================================="
|
||||
echo " PASS: $PASS / $TOTAL"
|
||||
echo " FAIL: $FAIL / $TOTAL"
|
||||
echo " SKIP: $SKIP / $TOTAL"
|
||||
echo "=========================================="
|
||||
echo " UAT SIGNOFF: ${PASS}/${TOTAL} assertions passed"
|
||||
echo "=========================================="
|
||||
|
||||
if [ "$FAIL" -gt 0 ]; then
|
||||
echo " RESULT: FAIL (v1.0.0 NOT ready)"
|
||||
exit 1
|
||||
fi
|
||||
echo " RESULT: PASS (v1.0.0 ready to cut)"
|
||||
exit 0
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# orca UAT smoke test — CI-automated subset of uat-signoff.sh (REQ-163)
|
||||
# Runs pure-CLI assertions that don't require a live cluster.
|
||||
set -uo pipefail
|
||||
|
||||
ORCA="${ORCA:-$(command -v orca || echo ./bin/orca)}"
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
smoke() {
|
||||
local name="$1"
|
||||
local check="$2"
|
||||
if eval "$check" 2>/dev/null; then
|
||||
echo " PASS $name"
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
echo " FAIL $name"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== Orca UAT Smoke (CI subset) ==="
|
||||
|
||||
smoke "go_toolchain" \
|
||||
'go version 2>&1 | grep -qE "go1\.25\.1[2-9]|go1\.2[6-9]"'
|
||||
|
||||
smoke "build" \
|
||||
'test -x "$ORCA"'
|
||||
|
||||
smoke "no_pprof_all_interfaces" \
|
||||
'! grep -rn "pprof-allow-public" internal/daemon/pprof.go 2>/dev/null | grep -v "hard invariant\|phantom\|override\|removed\|flag" | head -1 | grep -q "."'
|
||||
|
||||
smoke "no_password_in_docs" \
|
||||
'! grep -rn "ORCA_PROXMOX_PASSWORD" examples/ 2>/dev/null | head -1 | grep -q "."'
|
||||
|
||||
smoke "acl_file_mode_in_code" \
|
||||
'grep -q "0o600" internal/cli/acl.go 2>/dev/null'
|
||||
|
||||
smoke "doctor_modes_exists" \
|
||||
'grep -q "doctorModesCmd\|doctor.*modes" internal/cli/doctor.go 2>/dev/null'
|
||||
|
||||
smoke "metrics_expanded" \
|
||||
'grep -q "orca_jobs_running\|orca_audit_chain_head" internal/transport/metrics.go 2>/dev/null'
|
||||
|
||||
smoke "type_linux_available" \
|
||||
'grep -q "NodeKindLinux" internal/model/node.go 2>/dev/null'
|
||||
|
||||
smoke "scheduler_wired" \
|
||||
'grep -q "dispatchDecision\|deployRemote" internal/cli/job_dispatch.go 2>/dev/null'
|
||||
|
||||
smoke "acl_check_wired" \
|
||||
'grep -q "acl.Check\|aclPolicy\|Check(" internal/daemon/acl.go 2>/dev/null'
|
||||
|
||||
smoke "seal_implemented" \
|
||||
'grep -q "clusterSealCmd\|func.*runClusterSeal\|cluster seal" internal/cli/cluster.go 2>/dev/null'
|
||||
|
||||
smoke "injection_hardening" \
|
||||
'grep -q "validSafeName\|shellQuote" internal/cli/validate.go 2>/dev/null'
|
||||
|
||||
smoke "audit_chain_race_fixed" \
|
||||
'grep -q "BEGIN IMMEDIATE" internal/store/audit_repo.go 2>/dev/null'
|
||||
|
||||
echo "=== PASS: $PASS, FAIL: $FAIL ==="
|
||||
exit $FAIL
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# verify-docs.sh — assert that every subcommand documented in docs/cli.md
|
||||
# exists in `orca --help` output (and vice versa). Catches doc drift.
|
||||
#
|
||||
# Usage: scripts/verify-docs.sh [binary] [docs/cli.md]
|
||||
# Exit 0 = consistent, 1 = drift detected, 2 = error.
|
||||
set -euo pipefail
|
||||
|
||||
BIN="${1:-./bin/orca}"
|
||||
DOCS="${2:-docs/cli.md}"
|
||||
|
||||
if [ ! -x "$BIN" ]; then
|
||||
echo "verify-docs: binary not found at $BIN (run 'make build' first)" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ ! -f "$DOCS" ]; then
|
||||
echo "verify-docs: $DOCS not found" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Extract top-level commands from `orca --help` (lines indented under
|
||||
# "Available Commands:" with two leading spaces, command name is the
|
||||
# first token).
|
||||
HELP_OUTPUT="$("$BIN" --help 2>/dev/null)"
|
||||
HELP_CMDS="$(echo "$HELP_OUTPUT" | \
|
||||
awk '/^Available Commands:/{flag=1; next} /^$/{flag=0} flag && /^ /{print $1}' | \
|
||||
grep -v '^completion$' | grep -v '^help$' | sort -u)"
|
||||
|
||||
# Extract documented commands from docs/cli.md. These appear as
|
||||
# `## \`orca <command>\`` or `## \`orca <command>\` *(deprecated)*` headers.
|
||||
DOC_CMDS="$(grep -oE '^## `orca [a-z_-]+`' "$DOCS" | \
|
||||
sed 's/^## `orca //; s/`$//' | sort -u)"
|
||||
|
||||
# Compare.
|
||||
diff_out="$(diff <(echo "$HELP_CMDS") <(echo "$DOC_CMDS") || true)"
|
||||
|
||||
if [ -n "$diff_out" ]; then
|
||||
echo "verify-docs: drift detected between docs/cli.md and \`orca --help\`:" >&2
|
||||
echo "$diff_out" >&2
|
||||
echo "" >&2
|
||||
echo "Commands in --help but not in docs/cli.md (add them):" >&2
|
||||
comm -23 <(echo "$HELP_CMDS") <(echo "$DOC_CMDS") >&2
|
||||
echo "" >&2
|
||||
echo "Commands in docs/cli.md but not in --help (remove them or fix typo):" >&2
|
||||
comm -13 <(echo "$HELP_CMDS") <(echo "$DOC_CMDS") >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "verify-docs: OK — docs/cli.md consistent with orca --help"
|
||||
exit 0
|
||||
Executable
BIN
Binary file not shown.
Reference in New Issue
Block a user