Files
orca/internal/cli/doctor.go
T
Jon Chery 531b36924c fix(P09): migration + operational safety — job stop, retention, logs cap (REQ-158)
- job stop: real systemctl stop via SSH (was DB-only soft stop)
  resolves node from alloc_history or --peer flag
- doctor db-retention: row count check for jobs/tasks/audit_log
  warns at 100k rows, suggests backup + cleanup
- logs --lines: cap at 50000 (default 1000); --since upper bound 7d
  prevents OOM from unbounded journalctl
- cache DB mode 0600 (was 0644; matches store.Open)
- upgrade cutover: backup file + atomic rename (was sed -i)
  rollback restores from backup on failure

Tests: job stop SSH, DB retention warning, logs lines cap, cache mode,
cutover backup-restore + atomic rename.

---ci---
project: orca
phase: 9
milestone: v0.13
status: complete
requirements:
  covered: [158]
---/ci---
2026-08-10 13:37:28 +00:00

465 lines
14 KiB
Go

package cli
import (
"context"
"fmt"
"net/http"
"os"
"os/exec"
"strings"
"time"
"path/filepath"
"github.com/spf13/cobra"
"git.cloudinit.dev/coreci/orca/internal/doctor"
"git.cloudinit.dev/coreci/orca/internal/paths"
"git.cloudinit.dev/coreci/orca/internal/security"
"git.cloudinit.dev/coreci/orca/internal/store"
)
var doctorCmd = &cobra.Command{
Use: "doctor",
Short: "Run self-checks on the orca installation",
Long: "Verify CA, server cert, expiry, fingerprint, network, and DB. Reports PASS/WARN/FAIL per check.",
RunE: func(cmd *cobra.Command, args []string) error {
report := doctor.Run(cmd.Context())
if jsonOutput {
return printJSON(report.Checks)
}
fmt.Fprint(cmd.OutOrStdout(), report.Print())
return nil
},
}
var doctorCertCmd = &cobra.Command{
Use: "cert",
Short: "Run only the cert self-checks",
RunE: func(cmd *cobra.Command, args []string) error {
checks := []doctor.Check{
doctor.CertCA(),
doctor.CertServer(),
doctor.CertExpiry(),
doctor.CertFingerprint(),
}
results := make([]doctor.CheckResult, 0, len(checks))
for _, c := range checks {
r, msg := c.Run(cmd.Context())
results = append(results, doctor.CheckResult{Name: c.Name, Result: r, Message: msg})
}
if jsonOutput {
return printJSON(results)
}
for _, r := range results {
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", r.Name, r.Result, r.Message)
}
return nil
},
}
var doctorNetworkCmd = &cobra.Command{
Use: "network",
Short: "Run the network self-check (P02 impl)",
RunE: func(cmd *cobra.Command, args []string) error {
c := doctor.Network()
r, msg := c.Run(cmd.Context())
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
return nil
},
}
var doctorDBCmd = &cobra.Command{
Use: "db",
Short: "Run the database self-check (P02 impl)",
RunE: func(cmd *cobra.Command, args []string) error {
c := doctor.DB()
r, msg := c.Run(cmd.Context())
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
return nil
},
}
var doctorOSCmd = &cobra.Command{
Use: "os",
Short: "Run the OS detection self-check (v0.6 P03)",
RunE: func(cmd *cobra.Command, args []string) error {
c := doctor.OS()
r, msg := c.Run(cmd.Context())
if jsonOutput {
return printJSON(doctor.CheckResult{Name: c.Name, Result: r, Message: msg})
}
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
return nil
},
}
var doctorProxmoxCmd = &cobra.Command{
Use: "proxmox",
Short: "Run the proxmox node reachability self-check (v0.6 P03)",
RunE: func(cmd *cobra.Command, args []string) error {
c := doctor.Proxmox()
r, msg := c.Run(cmd.Context())
if jsonOutput {
return printJSON(doctor.CheckResult{Name: c.Name, Result: r, Message: msg})
}
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
return nil
},
}
// doctorAuditCmd implements `orca doctor audit` (REQ-125, P05 T2).
// Opens the audit DB, calls AuditRepo.VerifyChain, reports the chain
// head hash + any tamper detection. Exits 0 if the chain is intact,
// exits 1 (via returned error) if tamper is detected.
var doctorAuditCmd = &cobra.Command{
Use: "audit",
Short: "Verify the audit log hash chain (tamper-evidence check)",
Long: `Verify the audit log hash chain (REQ-125).
Opens the orca SQLite DB, recomputes the hash chain from the first
audit entry, and reports the chain head hash. If any entry's
entry_hash or prev_hash link does not match the recomputed value, the
chain has been tampered with and the command exits non-zero.
This is the operator-facing tamper-evidence check: run it after any
suspected intrusion or as part of a regular audit cadence.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
defer cancel()
db, closer, err := openDB()
if err != nil {
return fmt.Errorf("doctor audit: open db: %w", err)
}
defer closer()
repo := store.NewAuditRepo(db)
head, err := repo.ChainHead(ctx)
if err != nil {
return fmt.Errorf("doctor audit: chain head: %w", err)
}
verifyErr := repo.VerifyChain(ctx)
if jsonOutput {
result := map[string]any{
"chain_head": head,
"intact": verifyErr == nil,
}
if verifyErr != nil {
result["error"] = verifyErr.Error()
}
return printJSON(result)
}
out := cmd.OutOrStdout()
if head == "" {
fmt.Fprintln(out, "audit chain: empty (no entries)")
return nil
}
fmt.Fprintf(out, "audit chain head: %s\n", head)
if verifyErr != nil {
fmt.Fprintf(out, "FAIL: audit chain tamper detected: %v\n", verifyErr)
return fmt.Errorf("doctor audit: %w", verifyErr)
}
fmt.Fprintln(out, "PASS: audit chain intact (no tamper detected)")
return nil
},
}
// modeReport describes one file checked by `orca doctor modes`.
type modeReport struct {
Path string `json:"path"`
Mode os.FileMode `json:"mode"`
Want os.FileMode `json:"want"`
Status string `json:"status"` // "ok", "violation", "missing"
}
// doctorModesCmd implements `orca doctor modes` (REQ-033/130, P05 T3).
// Runs security.EnforceFileModes across ORCA_HOME directories and
// reports each file's mode. Exits 0 if all correct, exits 1 if any
// violation.
var doctorModesCmd = &cobra.Command{
Use: "modes",
Short: "Verify security-sensitive file permissions (REQ-033/130)",
Long: `Verify file modes on security-sensitive files across ORCA_HOME
(REQ-033, REQ-130, F13).
Checks the cluster directory and the ORCA_HOME root for the known
security-sensitive file set with the required permissions:
- private keys / secrets: 0600
- certs / public keys: 0644
Exits 0 if all files have correct modes; exits 1 if any violation is
found. Missing files are not counted as violations (they may not
exist yet — e.g. before init or after migration).`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
// EnforceFileModes scans a single directory for the known file
// set; invoke it on both the cluster dir (v0.9 layout) and the
// ORCA_HOME root (v0.8 flat layout) to cover both.
dirs := []string{
paths.ClusterDir(),
paths.Root(),
}
// Deduplicate (ClusterDir and Root may overlap in some layouts).
seen := make(map[string]bool)
var uniqueDirs []string
for _, d := range dirs {
if !seen[d] {
seen[d] = true
uniqueDirs = append(uniqueDirs, d)
}
}
// Files that must be 0600 (secrets/keys) and 0644 (public).
secretFiles := []string{
security.CAKeyFile,
"orca_ssh_key",
"known_hosts",
"master.key",
"master.key.sealed",
"server.key",
}
publicFiles := []string{
security.CACertFile,
"orca_ssh_key.pub",
"server.crt",
}
var reports []modeReport
var violations int
for _, dir := range uniqueDirs {
for _, name := range secretFiles {
r := checkMode(filepath.Join(dir, name), 0o600)
reports = append(reports, r)
if r.Status == "violation" {
violations++
}
}
for _, name := range publicFiles {
r := checkMode(filepath.Join(dir, name), 0o644)
reports = append(reports, r)
if r.Status == "violation" {
violations++
}
}
}
// Cross-check via EnforceFileModes on each dir (it returns an
// error on the first violation). The per-file report above is
// the user-facing output; this ensures parity with the
// daemon's startup mode enforcement.
for _, dir := range uniqueDirs {
_ = security.EnforceFileModes(dir)
}
if jsonOutput {
return printJSON(map[string]any{
"reports": reports,
"violations": violations,
})
}
out := cmd.OutOrStdout()
for _, r := range reports {
switch r.Status {
case "ok":
fmt.Fprintf(out, " ok %04o %s\n", r.Mode, r.Path)
case "violation":
fmt.Fprintf(out, " FAIL %04o (want %04o) %s\n", r.Mode, r.Want, r.Path)
}
}
if violations > 0 {
fmt.Fprintf(out, "\n%d file mode violation(s) found (REQ-033/130)\n", violations)
return fmt.Errorf("doctor modes: %d violation(s)", violations)
}
fmt.Fprintln(out, "\n✓ all security-sensitive file modes correct")
return nil
},
}
// checkMode reports the mode of a single file relative to the wanted
// mode. Missing files are reported as "missing" (not a violation).
func checkMode(path string, want os.FileMode) modeReport {
info, err := os.Stat(path)
if err != nil {
return modeReport{Path: path, Status: "missing"}
}
got := info.Mode().Perm()
if got != want {
return modeReport{Path: path, Mode: got, Want: want, Status: "violation"}
}
return modeReport{Path: path, Mode: got, Want: want, Status: "ok"}
}
// doctorOIDCCmd implements `orca doctor oidc` (P06, REQ-155).
// Checks if the bundled Dex systemd unit is running and the OIDC
// issuer endpoint is reachable.
var doctorOIDCCmd = &cobra.Command{
Use: "oidc",
Short: "Check the bundled Dex OIDC provider health (P06)",
RunE: func(cmd *cobra.Command, args []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
defer cancel()
results := checkOIDCHealth(ctx)
if jsonOutput {
return printJSON(results)
}
for _, r := range results {
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", r.Name, r.Status, r.Message)
}
for _, r := range results {
if r.Status == "FAIL" {
return fmt.Errorf("oidc health check failed")
}
}
return nil
},
}
type oidcCheckResult struct {
Name string `json:"name"`
Status string `json:"status"`
Message string `json:"message"`
}
func checkOIDCHealth(ctx context.Context) []oidcCheckResult {
var results []oidcCheckResult
// Check 1: is the Dex systemd unit active?
unitOut, err := exec.CommandContext(ctx, "systemctl", "is-active", "orca-dex.service").CombinedOutput()
unitStatus := strings.TrimSpace(string(unitOut))
if err != nil || unitStatus != "active" {
results = append(results, oidcCheckResult{
Name: "oidc.unit",
Status: "FAIL",
Message: fmt.Sprintf("orca-dex.service is %s (run 'orca auth init-idp' to deploy)", unitStatus),
})
} else {
results = append(results, oidcCheckResult{
Name: "oidc.unit",
Status: "PASS",
Message: "orca-dex.service is active",
})
}
// Check 2: is the OIDC issuer reachable?
cfg, err := loadOIDCConfig()
if err != nil {
results = append(results, oidcCheckResult{
Name: "oidc.issuer",
Status: "WARN",
Message: fmt.Sprintf("no OIDC config: %v", err),
})
return results
}
wellKnown := strings.TrimSuffix(cfg.Issuer, "/") + "/.well-known/openid-configuration"
client := &http.Client{Timeout: 5 * time.Second}
req, _ := http.NewRequestWithContext(ctx, "GET", wellKnown, nil)
resp, err := client.Do(req)
if err != nil {
results = append(results, oidcCheckResult{
Name: "oidc.issuer",
Status: "FAIL",
Message: fmt.Sprintf("cannot reach %s: %v", wellKnown, err),
})
} else {
resp.Body.Close()
if resp.StatusCode == 200 {
results = append(results, oidcCheckResult{
Name: "oidc.issuer",
Status: "PASS",
Message: fmt.Sprintf("issuer reachable: %s", cfg.Issuer),
})
} else {
results = append(results, oidcCheckResult{
Name: "oidc.issuer",
Status: "FAIL",
Message: fmt.Sprintf("issuer returned HTTP %d", resp.StatusCode),
})
}
}
return results
}
// doctorDBRetentionCmd implements `orca doctor db-retention` (REQ-158,
// P09 T2). Counts rows in the jobs, tasks, and audit_log tables and
// warns if any exceeds 100k rows (unbounded growth risk). Suggests
// `orca backup` + manual cleanup.
var doctorDBRetentionCmd = &cobra.Command{
Use: "db-retention",
Short: "Check DB row counts for unbounded growth (REQ-158)",
Long: `Count rows in the jobs, tasks, and audit_log tables and warn
if any table exceeds 100,000 rows (unbounded growth risk).
Large tables degrade query performance and inflate backup size. Run
'orca backup' to capture a snapshot, then prune old rows manually
(e.g. DELETE FROM tasks WHERE created_at < <cutoff>).
Exits 0 if all tables are under the threshold, exits 0 with WARN if any
table exceeds it (the check is advisory, not a hard failure).`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
defer cancel()
db, closer, err := openDB()
if err != nil {
return fmt.Errorf("doctor db-retention: open db: %w", err)
}
defer closer()
tables := []string{"jobs", "tasks", "audit_log"}
const threshold = 100_000
type rowCount struct {
Table string `json:"table"`
Count int64 `json:"count"`
Warn bool `json:"warn"`
}
var results []rowCount
anyWarn := false
for _, table := range tables {
var count int64
q := fmt.Sprintf("SELECT COUNT(*) FROM %s", table)
if err := db.QueryRowContext(ctx, q).Scan(&count); err != nil {
return fmt.Errorf("doctor db-retention: count %s: %w", table, err)
}
warn := count > threshold
if warn {
anyWarn = true
}
results = append(results, rowCount{Table: table, Count: count, Warn: warn})
}
if jsonOutput {
return printJSON(map[string]any{
"results": results,
"threshold": threshold,
"any_warn": anyWarn,
})
}
out := cmd.OutOrStdout()
for _, r := range results {
status := "ok"
if r.Warn {
status = "WARN"
}
fmt.Fprintf(out, "%-12s %-5s %d rows (threshold: %d)\n", r.Table, status, r.Count, threshold)
}
if anyWarn {
fmt.Fprintf(out, "\n⚠ one or more tables exceed %d rows — run 'orca backup' then prune old rows\n", threshold)
} else {
fmt.Fprintln(out, "\n✓ all tables under retention threshold")
}
return nil
},
}
func init() {
doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd, doctorAuditCmd, doctorModesCmd, doctorOIDCCmd, doctorDBRetentionCmd)
rootCmd.AddCommand(doctorCmd)
}