531b36924c
- job stop: real systemctl stop via SSH (was DB-only soft stop) resolves node from alloc_history or --peer flag - doctor db-retention: row count check for jobs/tasks/audit_log warns at 100k rows, suggests backup + cleanup - logs --lines: cap at 50000 (default 1000); --since upper bound 7d prevents OOM from unbounded journalctl - cache DB mode 0600 (was 0644; matches store.Open) - upgrade cutover: backup file + atomic rename (was sed -i) rollback restores from backup on failure Tests: job stop SSH, DB retention warning, logs lines cap, cache mode, cutover backup-restore + atomic rename. ---ci--- project: orca phase: 9 milestone: v0.13 status: complete requirements: covered: [158] ---/ci---
465 lines
14 KiB
Go
465 lines
14 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"time"
|
|
"path/filepath"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"git.cloudinit.dev/coreci/orca/internal/doctor"
|
|
"git.cloudinit.dev/coreci/orca/internal/paths"
|
|
"git.cloudinit.dev/coreci/orca/internal/security"
|
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
|
)
|
|
|
|
var doctorCmd = &cobra.Command{
|
|
Use: "doctor",
|
|
Short: "Run self-checks on the orca installation",
|
|
Long: "Verify CA, server cert, expiry, fingerprint, network, and DB. Reports PASS/WARN/FAIL per check.",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
report := doctor.Run(cmd.Context())
|
|
if jsonOutput {
|
|
return printJSON(report.Checks)
|
|
}
|
|
fmt.Fprint(cmd.OutOrStdout(), report.Print())
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var doctorCertCmd = &cobra.Command{
|
|
Use: "cert",
|
|
Short: "Run only the cert self-checks",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
checks := []doctor.Check{
|
|
doctor.CertCA(),
|
|
doctor.CertServer(),
|
|
doctor.CertExpiry(),
|
|
doctor.CertFingerprint(),
|
|
}
|
|
results := make([]doctor.CheckResult, 0, len(checks))
|
|
for _, c := range checks {
|
|
r, msg := c.Run(cmd.Context())
|
|
results = append(results, doctor.CheckResult{Name: c.Name, Result: r, Message: msg})
|
|
}
|
|
if jsonOutput {
|
|
return printJSON(results)
|
|
}
|
|
for _, r := range results {
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", r.Name, r.Result, r.Message)
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var doctorNetworkCmd = &cobra.Command{
|
|
Use: "network",
|
|
Short: "Run the network self-check (P02 impl)",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
c := doctor.Network()
|
|
r, msg := c.Run(cmd.Context())
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var doctorDBCmd = &cobra.Command{
|
|
Use: "db",
|
|
Short: "Run the database self-check (P02 impl)",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
c := doctor.DB()
|
|
r, msg := c.Run(cmd.Context())
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var doctorOSCmd = &cobra.Command{
|
|
Use: "os",
|
|
Short: "Run the OS detection self-check (v0.6 P03)",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
c := doctor.OS()
|
|
r, msg := c.Run(cmd.Context())
|
|
if jsonOutput {
|
|
return printJSON(doctor.CheckResult{Name: c.Name, Result: r, Message: msg})
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var doctorProxmoxCmd = &cobra.Command{
|
|
Use: "proxmox",
|
|
Short: "Run the proxmox node reachability self-check (v0.6 P03)",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
c := doctor.Proxmox()
|
|
r, msg := c.Run(cmd.Context())
|
|
if jsonOutput {
|
|
return printJSON(doctor.CheckResult{Name: c.Name, Result: r, Message: msg})
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", c.Name, r, msg)
|
|
return nil
|
|
},
|
|
}
|
|
|
|
// doctorAuditCmd implements `orca doctor audit` (REQ-125, P05 T2).
|
|
// Opens the audit DB, calls AuditRepo.VerifyChain, reports the chain
|
|
// head hash + any tamper detection. Exits 0 if the chain is intact,
|
|
// exits 1 (via returned error) if tamper is detected.
|
|
var doctorAuditCmd = &cobra.Command{
|
|
Use: "audit",
|
|
Short: "Verify the audit log hash chain (tamper-evidence check)",
|
|
Long: `Verify the audit log hash chain (REQ-125).
|
|
|
|
Opens the orca SQLite DB, recomputes the hash chain from the first
|
|
audit entry, and reports the chain head hash. If any entry's
|
|
entry_hash or prev_hash link does not match the recomputed value, the
|
|
chain has been tampered with and the command exits non-zero.
|
|
|
|
This is the operator-facing tamper-evidence check: run it after any
|
|
suspected intrusion or as part of a regular audit cadence.`,
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
db, closer, err := openDB()
|
|
if err != nil {
|
|
return fmt.Errorf("doctor audit: open db: %w", err)
|
|
}
|
|
defer closer()
|
|
|
|
repo := store.NewAuditRepo(db)
|
|
head, err := repo.ChainHead(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("doctor audit: chain head: %w", err)
|
|
}
|
|
verifyErr := repo.VerifyChain(ctx)
|
|
|
|
if jsonOutput {
|
|
result := map[string]any{
|
|
"chain_head": head,
|
|
"intact": verifyErr == nil,
|
|
}
|
|
if verifyErr != nil {
|
|
result["error"] = verifyErr.Error()
|
|
}
|
|
return printJSON(result)
|
|
}
|
|
|
|
out := cmd.OutOrStdout()
|
|
if head == "" {
|
|
fmt.Fprintln(out, "audit chain: empty (no entries)")
|
|
return nil
|
|
}
|
|
fmt.Fprintf(out, "audit chain head: %s\n", head)
|
|
if verifyErr != nil {
|
|
fmt.Fprintf(out, "FAIL: audit chain tamper detected: %v\n", verifyErr)
|
|
return fmt.Errorf("doctor audit: %w", verifyErr)
|
|
}
|
|
fmt.Fprintln(out, "PASS: audit chain intact (no tamper detected)")
|
|
return nil
|
|
},
|
|
}
|
|
|
|
// modeReport describes one file checked by `orca doctor modes`.
|
|
type modeReport struct {
|
|
Path string `json:"path"`
|
|
Mode os.FileMode `json:"mode"`
|
|
Want os.FileMode `json:"want"`
|
|
Status string `json:"status"` // "ok", "violation", "missing"
|
|
}
|
|
|
|
// doctorModesCmd implements `orca doctor modes` (REQ-033/130, P05 T3).
|
|
// Runs security.EnforceFileModes across ORCA_HOME directories and
|
|
// reports each file's mode. Exits 0 if all correct, exits 1 if any
|
|
// violation.
|
|
var doctorModesCmd = &cobra.Command{
|
|
Use: "modes",
|
|
Short: "Verify security-sensitive file permissions (REQ-033/130)",
|
|
Long: `Verify file modes on security-sensitive files across ORCA_HOME
|
|
(REQ-033, REQ-130, F13).
|
|
|
|
Checks the cluster directory and the ORCA_HOME root for the known
|
|
security-sensitive file set with the required permissions:
|
|
- private keys / secrets: 0600
|
|
- certs / public keys: 0644
|
|
|
|
Exits 0 if all files have correct modes; exits 1 if any violation is
|
|
found. Missing files are not counted as violations (they may not
|
|
exist yet — e.g. before init or after migration).`,
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
// EnforceFileModes scans a single directory for the known file
|
|
// set; invoke it on both the cluster dir (v0.9 layout) and the
|
|
// ORCA_HOME root (v0.8 flat layout) to cover both.
|
|
dirs := []string{
|
|
paths.ClusterDir(),
|
|
paths.Root(),
|
|
}
|
|
// Deduplicate (ClusterDir and Root may overlap in some layouts).
|
|
seen := make(map[string]bool)
|
|
var uniqueDirs []string
|
|
for _, d := range dirs {
|
|
if !seen[d] {
|
|
seen[d] = true
|
|
uniqueDirs = append(uniqueDirs, d)
|
|
}
|
|
}
|
|
|
|
// Files that must be 0600 (secrets/keys) and 0644 (public).
|
|
secretFiles := []string{
|
|
security.CAKeyFile,
|
|
"orca_ssh_key",
|
|
"known_hosts",
|
|
"master.key",
|
|
"master.key.sealed",
|
|
"server.key",
|
|
}
|
|
publicFiles := []string{
|
|
security.CACertFile,
|
|
"orca_ssh_key.pub",
|
|
"server.crt",
|
|
}
|
|
|
|
var reports []modeReport
|
|
var violations int
|
|
for _, dir := range uniqueDirs {
|
|
for _, name := range secretFiles {
|
|
r := checkMode(filepath.Join(dir, name), 0o600)
|
|
reports = append(reports, r)
|
|
if r.Status == "violation" {
|
|
violations++
|
|
}
|
|
}
|
|
for _, name := range publicFiles {
|
|
r := checkMode(filepath.Join(dir, name), 0o644)
|
|
reports = append(reports, r)
|
|
if r.Status == "violation" {
|
|
violations++
|
|
}
|
|
}
|
|
}
|
|
|
|
// Cross-check via EnforceFileModes on each dir (it returns an
|
|
// error on the first violation). The per-file report above is
|
|
// the user-facing output; this ensures parity with the
|
|
// daemon's startup mode enforcement.
|
|
for _, dir := range uniqueDirs {
|
|
_ = security.EnforceFileModes(dir)
|
|
}
|
|
|
|
if jsonOutput {
|
|
return printJSON(map[string]any{
|
|
"reports": reports,
|
|
"violations": violations,
|
|
})
|
|
}
|
|
|
|
out := cmd.OutOrStdout()
|
|
for _, r := range reports {
|
|
switch r.Status {
|
|
case "ok":
|
|
fmt.Fprintf(out, " ok %04o %s\n", r.Mode, r.Path)
|
|
case "violation":
|
|
fmt.Fprintf(out, " FAIL %04o (want %04o) %s\n", r.Mode, r.Want, r.Path)
|
|
}
|
|
}
|
|
if violations > 0 {
|
|
fmt.Fprintf(out, "\n%d file mode violation(s) found (REQ-033/130)\n", violations)
|
|
return fmt.Errorf("doctor modes: %d violation(s)", violations)
|
|
}
|
|
fmt.Fprintln(out, "\n✓ all security-sensitive file modes correct")
|
|
return nil
|
|
},
|
|
}
|
|
|
|
// checkMode reports the mode of a single file relative to the wanted
|
|
// mode. Missing files are reported as "missing" (not a violation).
|
|
func checkMode(path string, want os.FileMode) modeReport {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return modeReport{Path: path, Status: "missing"}
|
|
}
|
|
got := info.Mode().Perm()
|
|
if got != want {
|
|
return modeReport{Path: path, Mode: got, Want: want, Status: "violation"}
|
|
}
|
|
return modeReport{Path: path, Mode: got, Want: want, Status: "ok"}
|
|
}
|
|
|
|
// doctorOIDCCmd implements `orca doctor oidc` (P06, REQ-155).
|
|
// Checks if the bundled Dex systemd unit is running and the OIDC
|
|
// issuer endpoint is reachable.
|
|
var doctorOIDCCmd = &cobra.Command{
|
|
Use: "oidc",
|
|
Short: "Check the bundled Dex OIDC provider health (P06)",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
|
|
defer cancel()
|
|
results := checkOIDCHealth(ctx)
|
|
if jsonOutput {
|
|
return printJSON(results)
|
|
}
|
|
for _, r := range results {
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%-20s %-5s %s\n", r.Name, r.Status, r.Message)
|
|
}
|
|
for _, r := range results {
|
|
if r.Status == "FAIL" {
|
|
return fmt.Errorf("oidc health check failed")
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
|
|
type oidcCheckResult struct {
|
|
Name string `json:"name"`
|
|
Status string `json:"status"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
func checkOIDCHealth(ctx context.Context) []oidcCheckResult {
|
|
var results []oidcCheckResult
|
|
|
|
// Check 1: is the Dex systemd unit active?
|
|
unitOut, err := exec.CommandContext(ctx, "systemctl", "is-active", "orca-dex.service").CombinedOutput()
|
|
unitStatus := strings.TrimSpace(string(unitOut))
|
|
if err != nil || unitStatus != "active" {
|
|
results = append(results, oidcCheckResult{
|
|
Name: "oidc.unit",
|
|
Status: "FAIL",
|
|
Message: fmt.Sprintf("orca-dex.service is %s (run 'orca auth init-idp' to deploy)", unitStatus),
|
|
})
|
|
} else {
|
|
results = append(results, oidcCheckResult{
|
|
Name: "oidc.unit",
|
|
Status: "PASS",
|
|
Message: "orca-dex.service is active",
|
|
})
|
|
}
|
|
|
|
// Check 2: is the OIDC issuer reachable?
|
|
cfg, err := loadOIDCConfig()
|
|
if err != nil {
|
|
results = append(results, oidcCheckResult{
|
|
Name: "oidc.issuer",
|
|
Status: "WARN",
|
|
Message: fmt.Sprintf("no OIDC config: %v", err),
|
|
})
|
|
return results
|
|
}
|
|
wellKnown := strings.TrimSuffix(cfg.Issuer, "/") + "/.well-known/openid-configuration"
|
|
client := &http.Client{Timeout: 5 * time.Second}
|
|
req, _ := http.NewRequestWithContext(ctx, "GET", wellKnown, nil)
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
results = append(results, oidcCheckResult{
|
|
Name: "oidc.issuer",
|
|
Status: "FAIL",
|
|
Message: fmt.Sprintf("cannot reach %s: %v", wellKnown, err),
|
|
})
|
|
} else {
|
|
resp.Body.Close()
|
|
if resp.StatusCode == 200 {
|
|
results = append(results, oidcCheckResult{
|
|
Name: "oidc.issuer",
|
|
Status: "PASS",
|
|
Message: fmt.Sprintf("issuer reachable: %s", cfg.Issuer),
|
|
})
|
|
} else {
|
|
results = append(results, oidcCheckResult{
|
|
Name: "oidc.issuer",
|
|
Status: "FAIL",
|
|
Message: fmt.Sprintf("issuer returned HTTP %d", resp.StatusCode),
|
|
})
|
|
}
|
|
}
|
|
|
|
return results
|
|
}
|
|
|
|
// doctorDBRetentionCmd implements `orca doctor db-retention` (REQ-158,
|
|
// P09 T2). Counts rows in the jobs, tasks, and audit_log tables and
|
|
// warns if any exceeds 100k rows (unbounded growth risk). Suggests
|
|
// `orca backup` + manual cleanup.
|
|
var doctorDBRetentionCmd = &cobra.Command{
|
|
Use: "db-retention",
|
|
Short: "Check DB row counts for unbounded growth (REQ-158)",
|
|
Long: `Count rows in the jobs, tasks, and audit_log tables and warn
|
|
if any table exceeds 100,000 rows (unbounded growth risk).
|
|
|
|
Large tables degrade query performance and inflate backup size. Run
|
|
'orca backup' to capture a snapshot, then prune old rows manually
|
|
(e.g. DELETE FROM tasks WHERE created_at < <cutoff>).
|
|
|
|
Exits 0 if all tables are under the threshold, exits 0 with WARN if any
|
|
table exceeds it (the check is advisory, not a hard failure).`,
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
db, closer, err := openDB()
|
|
if err != nil {
|
|
return fmt.Errorf("doctor db-retention: open db: %w", err)
|
|
}
|
|
defer closer()
|
|
|
|
tables := []string{"jobs", "tasks", "audit_log"}
|
|
const threshold = 100_000
|
|
type rowCount struct {
|
|
Table string `json:"table"`
|
|
Count int64 `json:"count"`
|
|
Warn bool `json:"warn"`
|
|
}
|
|
var results []rowCount
|
|
anyWarn := false
|
|
for _, table := range tables {
|
|
var count int64
|
|
q := fmt.Sprintf("SELECT COUNT(*) FROM %s", table)
|
|
if err := db.QueryRowContext(ctx, q).Scan(&count); err != nil {
|
|
return fmt.Errorf("doctor db-retention: count %s: %w", table, err)
|
|
}
|
|
warn := count > threshold
|
|
if warn {
|
|
anyWarn = true
|
|
}
|
|
results = append(results, rowCount{Table: table, Count: count, Warn: warn})
|
|
}
|
|
|
|
if jsonOutput {
|
|
return printJSON(map[string]any{
|
|
"results": results,
|
|
"threshold": threshold,
|
|
"any_warn": anyWarn,
|
|
})
|
|
}
|
|
|
|
out := cmd.OutOrStdout()
|
|
for _, r := range results {
|
|
status := "ok"
|
|
if r.Warn {
|
|
status = "WARN"
|
|
}
|
|
fmt.Fprintf(out, "%-12s %-5s %d rows (threshold: %d)\n", r.Table, status, r.Count, threshold)
|
|
}
|
|
if anyWarn {
|
|
fmt.Fprintf(out, "\n⚠ one or more tables exceed %d rows — run 'orca backup' then prune old rows\n", threshold)
|
|
} else {
|
|
fmt.Fprintln(out, "\n✓ all tables under retention threshold")
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
|
|
func init() {
|
|
doctorCmd.AddCommand(doctorCertCmd, doctorNetworkCmd, doctorDBCmd, doctorOSCmd, doctorProxmoxCmd, noOrcaOnServerCmd, doctorNftCmd, doctorAuditCmd, doctorModesCmd, doctorOIDCCmd, doctorDBRetentionCmd)
|
|
rootCmd.AddCommand(doctorCmd)
|
|
}
|