// Package cli: recovery.go implements the full-recovery semantics for // `orca restore` (P07, v0.11 milestone). // // On top of P04's signed-restore, restore now: // // 1. Verifies the signature (P04). // 2. --dry-run: extracts to a temp staging dir, verifies, reports, // cleans up, and exits — never touching the real ORCA_HOME. // 3. Without --force: scans every peer for running orca-alloc-* // systemd units; if any are running, refuses the restore (data // loss protection) and reports which allocs are running. // 4. With --force: stops the running allocs (SSH systemctl stop), // extracts the tarball, then restarts the allocs from the restored // state (SSH systemctl start). // 5. Post-restore verification: master key present + valid, namespace // dirs exist, SQLite DBs openable; reports discrepancies. // 6. Records the restore in the audit log (internal/store/audit_repo). package cli import ( "context" "database/sql" "errors" "fmt" "log/slog" "os" "path/filepath" "strings" "github.com/spf13/cobra" "git.cloudinit.dev/coreci/orca/internal/backup" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/engine" "git.cloudinit.dev/coreci/orca/internal/paths" "git.cloudinit.dev/coreci/orca/internal/secrets" "git.cloudinit.dev/coreci/orca/internal/store" ) // RestoreOptions configures the P07 full-recovery restore. It mirrors // backup.RestoreOptions but adds DryRun and is consumed by runRestore // (the CLI layer), which performs the alloc reconciliation and audit // logging that the lower-level backup.Restore does not. type RestoreOptions struct { InputPath string TargetDir string MasterKey []byte Force bool DryRun bool } // ErrRunningAllocs is returned when restore refuses to clobber a live // cluster that has running allocations and --force was not given. var ErrRunningAllocs = errors.New("restore: running allocations present (use --force to stop and restart)") // runRestore is the entry point invoked by restoreCmd.RunE. It performs // the full P07 recovery flow. The cmd is used only for output; ctx comes // from cmd.Context(). func runRestore(cmd *cobra.Command, opts RestoreOptions) error { ctx := cmd.Context() if opts.InputPath == "" { return fmt.Errorf("restore: InputPath is empty") } if opts.TargetDir == "" { return fmt.Errorf("restore: TargetDir is empty") } if len(opts.MasterKey) == 0 { return fmt.Errorf("restore: MasterKey is empty") } log := newLogger() sigPath := opts.InputPath + ".sig" if err := backup.VerifySignature(opts.InputPath, sigPath, opts.MasterKey); err != nil { auditRestore(ctx, "failure", err, map[string]any{"path": opts.InputPath, "target": opts.TargetDir}) return fmt.Errorf("restore: verify signature: %w", err) } if opts.DryRun { return runRestoreDryRun(cmd, opts) } ex, exErr := drainExecFromCtx(ctx) if exErr != nil { return fmt.Errorf("ssh transport: %w", exErr) } running, err := scanRunningAllocs(ctx, ex) if err != nil { log.Warn("restore: scan running allocs failed", "error", err) } if len(running) > 0 && !opts.Force { allocList := formatRunningAllocs(running) err := fmt.Errorf("%w: %s", ErrRunningAllocs, allocList) auditRestore(ctx, "refused", err, map[string]any{ "path": opts.InputPath, "target": opts.TargetDir, "running": running, }) return err } if opts.Force && len(running) > 0 { stopped, stopErr := stopRunningAllocs(ctx, ex, running) if stopErr != nil { auditRestore(ctx, "partial", stopErr, map[string]any{ "path": opts.InputPath, "target": opts.TargetDir, "stopped": stopped, "running": running, }) return fmt.Errorf("restore: stop running allocs: %w", stopErr) } log.Info("restore: stopped running allocs", "count", len(stopped)) } ropts := backup.RestoreOptions{ InputPath: opts.InputPath, TargetDir: opts.TargetDir, MasterKey: opts.MasterKey, Force: true, } if err := backup.Restore(ropts); err != nil { auditRestore(ctx, "failure", err, map[string]any{ "path": opts.InputPath, "target": opts.TargetDir, }) return err } if opts.Force && len(running) > 0 { if rerr := restartAllocs(ctx, ex, running); rerr != nil { log.Warn("restore: failed to restart some allocs", "error", rerr) } } verifications := verifyRestoredState(opts.TargetDir, opts.MasterKey) report := buildVerificationReport(verifications) result := "success" var verifyErr error if len(verifications) > 0 { result = "partial" verifyErr = fmt.Errorf("post-restore verification: %d issue(s)", len(verifications)) } auditRestore(ctx, result, verifyErr, map[string]any{ "path": opts.InputPath, "target": opts.TargetDir, "force": opts.Force, "running": running, "verification": report, }) if jsonOutput { return printJSON(map[string]any{ "path": opts.InputPath, "target": opts.TargetDir, "force": opts.Force, "running": running, "verification": report, }) } fmt.Fprintf(cmd.OutOrStdout(), "✓ Restored to: %s\n", opts.TargetDir) if opts.Force && len(running) > 0 { fmt.Fprintf(cmd.OutOrStdout(), " stopped+restarted %d running alloc(s)\n", len(running)) } for _, v := range verifications { fmt.Fprintf(cmd.OutOrStdout(), " ⚠ verify: %s\n", v) } return nil } // runRestoreDryRun extracts the tarball to a temp dir, runs // post-restore verification against that temp dir, reports what WOULD // be restored, and cleans up. The real ORCA_HOME is never touched. func runRestoreDryRun(cmd *cobra.Command, opts RestoreOptions) error { tmp, err := os.MkdirTemp("", "orca-restore-dryrun-*") if err != nil { return fmt.Errorf("restore: dry-run temp dir: %w", err) } defer os.RemoveAll(tmp) log := newLogger() log.Info("restore: dry-run staging dir", "dir", tmp) ropts := backup.RestoreOptions{ InputPath: opts.InputPath, TargetDir: tmp, MasterKey: opts.MasterKey, Force: true, } if err := backup.Restore(ropts); err != nil { return fmt.Errorf("restore: dry-run extract: %w", err) } entries := listExtractedFiles(tmp) verifications := verifyRestoredState(tmp, opts.MasterKey) report := buildVerificationReport(verifications) auditRestore(cmd.Context(), "dry-run", nil, map[string]any{ "path": opts.InputPath, "target": opts.TargetDir, "staging": tmp, "files": len(entries), "verification": report, }) if jsonOutput { return printJSON(map[string]any{ "dry_run": true, "path": opts.InputPath, "target": opts.TargetDir, "files": len(entries), "verification": report, }) } out := cmd.OutOrStdout() fmt.Fprintf(out, "✓ Dry-run: would restore %d file(s) to %s\n", len(entries), opts.TargetDir) if len(entries) > 0 && len(entries) <= 20 { for _, e := range entries { fmt.Fprintf(out, " %s\n", e) } } for _, v := range verifications { fmt.Fprintf(out, " ⚠ verify: %s\n", v) } return nil } // runningAlloc is a running allocation discovered on a peer node. type runningAlloc struct { Node string `json:"node"` Peer string `json:"peer"` AllocID string `json:"alloc_id"` } // scanRunningAllocs lists every peer node in the registry and queries // each via SSH for currently-running orca-alloc-*.service units. A // registry failure or empty registry yields an empty (not error) // slice; per-node failures are logged and skipped so a single // unreachable peer does not block the restore. func scanRunningAllocs(ctx context.Context, ex drainExecer) ([]runningAlloc, error) { reg, closer, err := nodeRegistry() if err != nil { return nil, nil } defer closer() nodes, err := reg.List(ctx) if err != nil { return nil, nil } var out []runningAlloc log := newLogger() for i := range nodes { n := nodes[i] peer := peerAddrForNode(n) if peer == "" { continue } ids, err := listRunningAllocs(ctx, ex, peer) if err != nil { log.Warn("restore: cannot list allocs on node", slog.String("node", n.Name), slog.String("peer", peer), "error", err) continue } for _, id := range ids { out = append(out, runningAlloc{Node: n.Name, Peer: peer, AllocID: id}) } } return out, nil } // formatRunningAllocs renders a list of running allocs for an error // message (one per line: "node/peer: alloc-id"). func formatRunningAllocs(rs []runningAlloc) string { var b strings.Builder for i, a := range rs { if i > 0 { b.WriteString("; ") } fmt.Fprintf(&b, "%s/%s: %s", a.Node, a.Peer, a.AllocID) } return b.String() } // stopRunningAllocs stops every allocation in rs via SSH (systemctl // stop). It returns the ids that were successfully stopped and an // aggregate error if any failed. func stopRunningAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) ([]string, error) { var stopped []string var failed []string for _, a := range rs { if err := stopAlloc(ctx, ex, a.Peer, a.AllocID); err != nil { failed = append(failed, a.AllocID) continue } stopped = append(stopped, a.AllocID) } if len(failed) > 0 { return stopped, fmt.Errorf("failed to stop %d alloc(s): %s", len(failed), strings.Join(failed, ", ")) } return stopped, nil } // restartAllocs restarts every allocation in rs via SSH (systemctl // start). A per-alloc failure is collected and returned as an aggregate // error; the caller treats restart failures as non-fatal (the restore // itself succeeded). func restartAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) error { var failed []string for _, a := range rs { startCmd := fmt.Sprintf("systemctl start %s", allocUnit(a.AllocID)) if _, err := ex.Exec(ctx, a.Peer, startCmd); err != nil { failed = append(failed, a.AllocID) } } if len(failed) > 0 { return fmt.Errorf("failed to restart %d alloc(s): %s", len(failed), strings.Join(failed, ", ")) } return nil } // verifyRestoredState checks the restored tree for the master key // (present + valid length/mode), namespace directories, and openable // SQLite databases. It returns a slice of human-readable discrepancy // strings (empty if everything checks out). func verifyRestoredState(targetDir string, masterKey []byte) []string { var issues []string mkPath := filepath.Join(targetDir, relFromRoot(paths.MasterKeyPath())) info, err := os.Stat(mkPath) if err != nil { issues = append(issues, fmt.Sprintf("master key missing: %s", mkPath)) } else { if info.Mode().Perm() != secrets.MasterKeyMode { issues = append(issues, fmt.Sprintf("master key mode %04o (want %04o)", info.Mode().Perm(), secrets.MasterKeyMode)) } key, rerr := os.ReadFile(mkPath) if rerr != nil { issues = append(issues, fmt.Sprintf("master key unreadable: %v", rerr)) } else if len(key) != secrets.MasterKeyLen { issues = append(issues, fmt.Sprintf("master key length %d (want %d)", len(key), secrets.MasterKeyLen)) } else if masterKey != nil && string(key) != string(masterKey) { issues = append(issues, "master key differs from the key used to verify the signature") } } nsDirs := findNamespaceDirs(targetDir) if len(nsDirs) == 0 { issues = append(issues, "no namespace directories found") } for _, nsDir := range nsDirs { dbPath := filepath.Join(nsDir, "db", "orca.db") if _, err := os.Stat(dbPath); err == nil { if err := dbOpenable(dbPath); err != nil { issues = append(issues, fmt.Sprintf("db not openable %s: %v", dbPath, err)) } } } clusterDB := filepath.Join(targetDir, "orca.db") if _, err := os.Stat(clusterDB); err == nil { if err := dbOpenable(clusterDB); err != nil { issues = append(issues, fmt.Sprintf("cluster db not openable %s: %v", clusterDB, err)) } } return issues } // relFromRoot strips the ORCA_HOME prefix off an absolute path so it // can be rejoined to an arbitrary target dir (used by --dry-run and // non-default --target restores). If the path does not start with the // current ORCA_HOME root, the path's base is returned. func relFromRoot(p string) string { root := filepath.Clean(paths.Root()) clean := filepath.Clean(p) if rel, err := filepath.Rel(root, clean); err == nil && !strings.HasPrefix(rel, "..") { return rel } return filepath.Base(clean) } // findNamespaceDirs returns the top-level directories under targetDir // that look like namespace dirs (excluding the "cluster" dir and dot- // files). A namespace dir is any immediate child of ORCA_HOME that is // a directory and not "cluster". func findNamespaceDirs(targetDir string) []string { entries, err := os.ReadDir(targetDir) if err != nil { return nil } var out []string for _, e := range entries { if !e.IsDir() { continue } name := e.Name() if strings.HasPrefix(name, ".") { continue } if name == "cluster" { continue } out = append(out, filepath.Join(targetDir, name)) } return out } // dbOpenable reports whether the SQLite file at path can be opened // read-only. It uses the same driver as the rest of the codebase // (modernc.org/sqlite via store.Open, but with a read-only pragma). func dbOpenable(path string) error { dsn := "file:" + path + "?mode=ro&_pragma=journal_mode(WAL)" db, err := sql.Open("sqlite", dsn) if err != nil { return err } defer db.Close() if err := db.Ping(); err != nil { return err } return nil } // listExtractedFiles walks a staging dir and returns the relative // paths of all regular files (capped at 1000 for reporting). func listExtractedFiles(root string) []string { var out []string _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return nil } if !info.Mode().IsRegular() { return nil } rel, rerr := filepath.Rel(root, path) if rerr != nil { return nil } out = append(out, filepath.ToSlash(rel)) return nil }) if len(out) > 1000 { out = out[:1000] } return out } // buildVerificationReport converts the discrepancy slice into a // JSON-friendly []string (nil-safe for empty input). func buildVerificationReport(issues []string) []string { if len(issues) == 0 { return []string{} } return issues } // auditRestore records a restore event in the audit log (same pattern // as auditDrain in drain.go). It opens the cluster DB at // certpaths.DBPath(); a failure to open the DB is logged and silently // dropped so a restore is never blocked by the audit log itself. func auditRestore(ctx context.Context, result string, err error, meta map[string]any) { db, dbErr := store.Open(certpaths.DBPath()) if dbErr != nil { newLogger().Warn("restore: audit log unavailable", "error", dbErr) return } defer db.Close() engine.NewAudit(store.NewAuditRepo(db), newLogger()).Record(ctx, "cli", "restore", certpaths.Dir(), result, err, meta) }