// Package doctor implements `orca doctor`, a small battery of self-checks // for the orca installation. The cert, network, and db checks surface // common configuration errors before they become runtime failures. // // REQ-032: `orca doctor` is a first-class subcommand in v0.2 P01. // Per-phase subcommands: // // orca doctor — runs all checks, prints a summary // orca doctor cert — CA, server cert, expiry, fingerprint pin // orca doctor network — TCP reachability + mTLS handshake (stub in P01) // orca doctor db — SQLite open + migration apply (stub in P01) // // Each check returns a Result of PASS, WARN, or FAIL with a free-form // message. The aggregator prints one line per check. package doctor import ( "context" "crypto/x509" "encoding/pem" "fmt" "net/http" "os" "sort" "strings" "time" "golang.org/x/crypto/ssh" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/model" "git.cloudinit.dev/coreci/orca/internal/osdetect" "git.cloudinit.dev/coreci/orca/internal/proxmox" "git.cloudinit.dev/coreci/orca/internal/security" "git.cloudinit.dev/coreci/orca/internal/store" "git.cloudinit.dev/coreci/orca/internal/transport" ) // Result is the outcome of a single check. type Result string const ( ResultPass Result = "PASS" ResultWarn Result = "WARN" ResultFail Result = "FAIL" ) // Check is a single self-check. type Check struct { Name string Description string Run func(ctx context.Context) (Result, string) } // Report is the aggregated result of running all checks. type Report struct { Time time.Time Checks []CheckResult } // CheckResult is the outcome of one Check. type CheckResult struct { Name string Result Result Message string } // All returns the full battery of checks. func All() []Check { return []Check{ CertCA(), CertServer(), CertExpiry(), CertFingerprint(), OS(), Network(), Proxmox(), DB(), } } // Run executes every check and returns a Report. func Run(ctx context.Context) *Report { checks := All() results := make([]CheckResult, 0, len(checks)) for _, c := range checks { r, msg := c.Run(ctx) results = append(results, CheckResult{ Name: c.Name, Result: r, Message: msg, }) } return &Report{Time: time.Now(), Checks: results} } // Print renders the Report. func (r *Report) Print() string { out := fmt.Sprintf("orca doctor — %s\n\n", r.Time.UTC().Format(time.RFC3339)) pass, warn, fail := 0, 0, 0 sort.Slice(r.Checks, func(i, j int) bool { return r.Checks[i].Name < r.Checks[j].Name }) for _, c := range r.Checks { out += fmt.Sprintf("%-20s %-5s %s\n", c.Name, c.Result, c.Message) switch c.Result { case ResultPass: pass++ case ResultWarn: warn++ case ResultFail: fail++ } } out += fmt.Sprintf("\n%d PASS, %d WARN, %d FAIL\n", pass, warn, fail) return out } // CertCA checks the on-disk CA exists with the right file modes (REQ-033). func CertCA() Check { return Check{ Name: "cert.ca", Description: "CA at ~/.orca with mode 0600/0644 (REQ-033)", Run: func(_ context.Context) (Result, string) { dir := certpaths.Dir() if err := security.EnforceFileModes(dir); err != nil { return ResultFail, err.Error() } return ResultPass, fmt.Sprintf("CA at %s with mode 0644/0600", dir) }, } } // CertServer checks the server cert is present and parseable. func CertServer() Check { return Check{ Name: "cert.server", Description: "server.crt exists, signed by local CA", Run: func(_ context.Context) (Result, string) { certPath := certpaths.ServerCertPath() if _, err := os.Stat(certPath); err != nil { return ResultFail, fmt.Sprintf("server cert missing: %v", err) } fp, err := security.Fingerprint(certPath) if err != nil { return ResultFail, err.Error() } return ResultPass, fmt.Sprintf("server cert at %s, fp=%s", certPath, fp[:16]+"...") }, } } // CertExpiry returns WARN if the server cert is within 30 days of expiry // (REQ-034). Otherwise PASS. func CertExpiry() Check { return Check{ Name: "cert.expiry", Description: "server cert validity window (> 30d = PASS, ≤ 30d = WARN)", Run: func(_ context.Context) (Result, string) { cert, err := loadCert(certpaths.ServerCertPath()) if err != nil { return ResultFail, err.Error() } remaining := time.Until(cert.NotAfter) days := int(remaining.Hours() / 24) if days < 0 { return ResultFail, fmt.Sprintf("server cert EXPIRED %dd ago", -days) } if days <= 30 { return ResultWarn, fmt.Sprintf("server cert expires in %dd — run `orca cert renew`", days) } return ResultPass, fmt.Sprintf("server cert valid for %dd more", days) }, } } // CertFingerprint prints the CA fingerprint so the operator can copy // it to peers. Always PASS (or FAIL if the cert is missing). func CertFingerprint() Check { return Check{ Name: "cert.fingerprint", Description: "CA fingerprint (for cross-node pinning)", Run: func(_ context.Context) (Result, string) { fp, err := security.Fingerprint(certpaths.CACertPath()) if err != nil { return ResultFail, err.Error() } return ResultPass, fmt.Sprintf("CA fp=%s (use at `orca node join --ca-fingerprint`)", fp) }, } } // DB checks SQLite integrity and migration version (REQ-032 completion). func DB() Check { return Check{ Name: "db", Description: "SQLite integrity_check + migration version", Run: func(ctx context.Context) (Result, string) { path := certpaths.DBPath() db, err := store.Open(path) if err != nil { return ResultFail, fmt.Sprintf("open db: %v", err) } defer db.Close() var integrity string if err := db.QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&integrity); err != nil { return ResultFail, fmt.Sprintf("integrity_check: %v", err) } if !strings.EqualFold(integrity, "ok") { return ResultFail, fmt.Sprintf("integrity_check: %s", integrity) } version, err := store.MigrationVersion(ctx, db) if err != nil { return ResultFail, fmt.Sprintf("migration version: %v", err) } if version == "" { return ResultWarn, "integrity OK but no migrations applied (fresh db)" } return ResultPass, fmt.Sprintf("integrity OK, migrations up to %s", version) }, } } // Network probes peer reachability via mTLS /healthz (REQ-032 completion). // Peers are sourced from the persisted nodes table (not the in-memory // PeerRegistry, which is empty at CLI time). Zero peers → WARN (single-node // is legitimate). Any peer unreachable → FAIL (D-038). func Network() Check { return Check{ Name: "network", Description: "peer reachability via mTLS /healthz probe", Run: func(ctx context.Context) (Result, string) { caPath := certpaths.CACertPath() certPath := certpaths.ServerCertPath() keyPath := certpaths.ServerKeyPath() // Check that cert files exist before attempting probes. if _, err := os.Stat(caPath); err != nil { return ResultFail, fmt.Sprintf("CA cert missing: %v (run `orca cert init`)", err) } path := certpaths.DBPath() db, err := store.Open(path) if err != nil { return ResultFail, fmt.Sprintf("open db: %v", err) } defer db.Close() nodes, err := store.NewNodeRepo(db).List(ctx) if err != nil { return ResultFail, fmt.Sprintf("list nodes: %v", err) } live := make([]*model.Node, 0, len(nodes)) for _, n := range nodes { if n.State != model.NodeStateLeft { live = append(live, n) } } if len(live) == 0 { return ResultWarn, "no peers registered (single-node?)" } var lines []string anyFail := false for _, n := range live { probeCtx, cancel := context.WithTimeout(ctx, 3*time.Second) err := probeHealthz(probeCtx, caPath, certPath, keyPath, n.Name, n.Address) cancel() if err != nil { anyFail = true lines = append(lines, fmt.Sprintf(" ✗ %s (%s): %v", n.Name, n.Address, err)) } else { lines = append(lines, fmt.Sprintf(" ✓ %s (%s)", n.Name, n.Address)) } } result := ResultPass if anyFail { result = ResultFail } return result, strings.Join(lines, "\n") }, } } // probeHealthz opens an mTLS connection to the peer and GETs /healthz. func probeHealthz(ctx context.Context, caPath, certPath, keyPath, serverName, addr string) error { client, err := transport.NewMTLSClient(caPath, serverName, certPath, keyPath) if err != nil { return fmt.Errorf("mTLS client: %w", err) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+addr+"/healthz", nil) if err != nil { return fmt.Errorf("request: %w", err) } resp, err := client.Do(req) if err != nil { return fmt.Errorf("probe: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("healthz returned %d", resp.StatusCode) } return nil } // OS checks that the auto-detected OS matches the stored localhost // node's os field (REQ-052). Drift (e.g., OS upgraded since init) // returns WARN; match returns PASS; missing localhost node returns FAIL. func OS() Check { return Check{ Name: "os", Description: "localhost OS detection vs stored node row", Run: func(ctx context.Context) (Result, string) { detected := osdetect.Detect() db, err := store.Open(certpaths.DBPath()) if err != nil { return ResultFail, fmt.Sprintf("open db: %v", err) } defer db.Close() node, err := store.NewNodeRepo(db).GetByName(ctx, "localhost") if err == store.ErrNotFound { return ResultFail, "no localhost node registered — run `orca init`" } if err != nil { return ResultFail, fmt.Sprintf("lookup localhost node: %v", err) } if node.OS == "" { return ResultWarn, fmt.Sprintf("localhost node has no os field (pre-0006 row?); detected=%s — re-run `orca init` to refresh", detected) } if node.OS != detected { return ResultWarn, fmt.Sprintf("OS drift: init=%s, now=%s — re-run `orca init` to refresh", node.OS, detected) } return ResultPass, fmt.Sprintf("localhost os=%s (matches /etc/os-release)", detected) }, } } // Proxmox probes each kind=proxmox node via SSH with `pveversion` // (REQ-052). Clones the Network() pattern: list nodes, filter by kind, // 3s timeout per peer, PASS/WARN/FAIL per node. Zero proxmox nodes // returns WARN (single-node cluster is legitimate). func Proxmox() Check { return Check{ Name: "proxmox", Description: "proxmox node reachability via SSH pveversion probe", Run: func(ctx context.Context) (Result, string) { db, err := store.Open(certpaths.DBPath()) if err != nil { return ResultFail, fmt.Sprintf("open db: %v", err) } defer db.Close() nodes, err := store.NewNodeRepo(db).List(ctx) if err != nil { return ResultFail, fmt.Sprintf("list nodes: %v", err) } proxmoxNodes := make([]*model.Node, 0, len(nodes)) for _, n := range nodes { if n.Kind == string(model.NodeKindProxmox) && n.State != model.NodeStateLeft { proxmoxNodes = append(proxmoxNodes, n) } } if len(proxmoxNodes) == 0 { return ResultWarn, "no proxmox nodes registered (single-node?)" } var lines []string anyFail := false for _, n := range proxmoxNodes { probeCtx, cancel := context.WithTimeout(ctx, 3*time.Second) err := probeProxmoxPVEVersion(probeCtx, n.Name) cancel() if err != nil { anyFail = true lines = append(lines, fmt.Sprintf(" ✗ %s: %v", n.Name, err)) } else { lines = append(lines, fmt.Sprintf(" ✓ %s", n.Name)) } } result := ResultPass if anyFail { result = ResultFail } return result, strings.Join(lines, "\n") }, } } // probeProxmoxPVEVersion SSHes into the proxmox host and runs // `pveversion` to verify reachability + PVE installation. Uses the // orca SSH key for auth (deployed during `orca node join --type proxmox`) // and the known_hosts TOFU store for host-key verification (D-035). func probeProxmoxPVEVersion(ctx context.Context, host string) error { // Load the orca SSH key for public-key auth. keyPEM, err := os.ReadFile(certpaths.SSHKeyPath()) if err != nil { return fmt.Errorf("read SSH key: %w (run `orca node join --type proxmox` first)", err) } signer, err := ssh.ParsePrivateKey(keyPEM) if err != nil { return fmt.Errorf("parse SSH key: %w", err) } // Extract host from the node address (orca stores host:8443; // SSH needs host:22). We dial the SSH port, not the orca daemon port. sshHost := host if strings.Contains(host, ":") { sshHost = strings.SplitN(host, ":", 2)[0] } sshAddr := sshHost + ":22" // Use the shared TOFU capture-fix wrapper (T02.9 — GRILL condition // #2: doctor parity with bootstrap). Without this, a first-connect // proxmox node (entry missing from known_hosts) fails the doctor // probe even though it joined fine — the v0.6 ship-defect. hostKeyCallback, err := proxmox.TOFUHostKeyCallback(sshAddr, nil) if err != nil { return fmt.Errorf("known_hosts: %w", err) } config := &ssh.ClientConfig{ User: "orca", Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, HostKeyCallback: hostKeyCallback, Timeout: 3 * time.Second, } dialer := &netDialer{} conn, err := dialer.DialContext(ctx, "tcp", sshAddr, config) if err != nil { return fmt.Errorf("ssh dial: %w", err) } defer conn.Close() session, err := conn.NewSession() if err != nil { return fmt.Errorf("new session: %w", err) } defer session.Close() out, err := session.CombinedOutput("pveversion") if err != nil { return fmt.Errorf("pveversion: %w (output: %s)", err, strings.TrimSpace(string(out))) } return nil } // netDialer wraps ssh.Dial with context support. The ssh package's // Dial doesn't accept a context directly, so we use a dialer that // respects ctx cancellation via a goroutine + channel. type netDialer struct{} func (d *netDialer) DialContext(ctx context.Context, network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { type result struct { client *ssh.Client err error } ch := make(chan result, 1) go func() { client, err := ssh.Dial(network, addr, config) ch <- result{client, err} }() select { case <-ctx.Done(): // Best-effort: if the dial succeeds after ctx cancellation, // the goroutine will close the client. We return the ctx error. go func() { if r := <-ch; r.client != nil { _ = r.client.Close() } }() return nil, ctx.Err() case r := <-ch: return r.client, r.err } } // loadCert reads a PEM cert from path and parses the first CERTIFICATE // block. func loadCert(path string) (*x509.Certificate, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read %s: %w", path, err) } block, _ := pem.Decode(data) if block == nil { return nil, fmt.Errorf("no PEM block in %s", path) } if block.Type != "CERTIFICATE" { return nil, fmt.Errorf("PEM type %q in %s, want CERTIFICATE", block.Type, path) } return x509.ParseCertificate(block.Bytes) }