feat(P07): recovery (orca restore) — verified restore + alloc protection
Extend restore with --dry-run (extract to temp, report, no write), running-alloc protection (refuse without --force; stop+restart with --force), post-restore verification (master key, namespaces, DBs), audit log entry. ---ci--- project: orca phase: 07 milestone: v0.11 status: execute ---/ci---
This commit is contained in:
Jon Chery
+14
-16
@@ -26,6 +26,7 @@ var (
|
||||
restoreInPath string
|
||||
restoreTargetDir string
|
||||
restoreForce bool
|
||||
restoreDryRun bool
|
||||
)
|
||||
|
||||
var backupCmd = &cobra.Command{
|
||||
@@ -70,11 +71,16 @@ written to --out; the hex-encoded signature to --out + ".sig".`,
|
||||
var restoreCmd = &cobra.Command{
|
||||
Use: "restore",
|
||||
Short: "Restore ORCA_HOME from a verified signed backup",
|
||||
Long: `Restore ORCA_HOME from a verified signed backup (P04).
|
||||
Long: `Restore ORCA_HOME from a verified signed backup (P04/P07).
|
||||
|
||||
Verifies the HMAC-SHA256 signature on --in (using the cluster master
|
||||
key) before extracting. With --force, overwrites a non-empty target;
|
||||
without it, refuses to clobber an existing ORCA_HOME.`,
|
||||
key) before extracting. Reconciles with live state: refuses to clobber
|
||||
running allocations unless --force is given (with --force, stops the
|
||||
running allocs, extracts, then restarts them from the restored state).
|
||||
With --dry-run, extracts to a temp dir and reports what WOULD be
|
||||
restored without touching the real ORCA_HOME. Performs post-restore
|
||||
verification (master key, namespace dirs, SQLite DBs) and records the
|
||||
restore in the audit log.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if restoreInPath == "" {
|
||||
@@ -88,23 +94,14 @@ without it, refuses to clobber an existing ORCA_HOME.`,
|
||||
if target == "" {
|
||||
target = paths.Root()
|
||||
}
|
||||
opts := backup.RestoreOptions{
|
||||
opts := RestoreOptions{
|
||||
InputPath: restoreInPath,
|
||||
TargetDir: target,
|
||||
MasterKey: mk,
|
||||
Force: restoreForce,
|
||||
DryRun: restoreDryRun,
|
||||
}
|
||||
if err := backup.Restore(opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]string{
|
||||
"path": restoreInPath,
|
||||
"target": target,
|
||||
})
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Restored to: %s\n", target)
|
||||
return nil
|
||||
return runRestore(cmd, opts)
|
||||
},
|
||||
}
|
||||
|
||||
@@ -112,7 +109,8 @@ func init() {
|
||||
backupCmd.Flags().StringVar(&backupOutPath, "out", "", "output tarball path (default: orca-backup-<timestamp>.tar.gz in CWD)")
|
||||
restoreCmd.Flags().StringVar(&restoreInPath, "in", "", "input tarball path (required)")
|
||||
restoreCmd.Flags().StringVar(&restoreTargetDir, "target", "", "restore target dir (default: ORCA_HOME)")
|
||||
restoreCmd.Flags().BoolVar(&restoreForce, "force", false, "overwrite a non-empty target directory")
|
||||
restoreCmd.Flags().BoolVar(&restoreForce, "force", false, "overwrite a non-empty target directory and stop+restart running allocs")
|
||||
restoreCmd.Flags().BoolVar(&restoreDryRun, "dry-run", false, "extract to a temp dir and report what would be restored without touching ORCA_HOME")
|
||||
rootCmd.AddCommand(backupCmd)
|
||||
rootCmd.AddCommand(restoreCmd)
|
||||
}
|
||||
|
||||
@@ -42,6 +42,7 @@ func resetCommandFlags() {
|
||||
auditLimit = 50
|
||||
backupOutPath, restoreInPath, restoreTargetDir = "", "", ""
|
||||
restoreForce = false
|
||||
restoreDryRun = false
|
||||
resetNSFlags()
|
||||
// Reset per-command output writers so tests that polluted them
|
||||
// (e.g. daemon tests calling cmd.SetOut(&buf)) don't leak into
|
||||
|
||||
@@ -0,0 +1,466 @@
|
||||
// Package cli: recovery.go implements the full-recovery semantics for
|
||||
// `orca restore` (P07, v0.11 milestone).
|
||||
//
|
||||
// On top of P04's signed-restore, restore now:
|
||||
//
|
||||
// 1. Verifies the signature (P04).
|
||||
// 2. --dry-run: extracts to a temp staging dir, verifies, reports,
|
||||
// cleans up, and exits — never touching the real ORCA_HOME.
|
||||
// 3. Without --force: scans every peer for running orca-alloc-*
|
||||
// systemd units; if any are running, refuses the restore (data
|
||||
// loss protection) and reports which allocs are running.
|
||||
// 4. With --force: stops the running allocs (SSH systemctl stop),
|
||||
// extracts the tarball, then restarts the allocs from the restored
|
||||
// state (SSH systemctl start).
|
||||
// 5. Post-restore verification: master key present + valid, namespace
|
||||
// dirs exist, SQLite DBs openable; reports discrepancies.
|
||||
// 6. Records the restore in the audit log (internal/store/audit_repo).
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/backup"
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/engine"
|
||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/secrets"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
|
||||
// RestoreOptions configures the P07 full-recovery restore. It mirrors
|
||||
// backup.RestoreOptions but adds DryRun and is consumed by runRestore
|
||||
// (the CLI layer), which performs the alloc reconciliation and audit
|
||||
// logging that the lower-level backup.Restore does not.
|
||||
type RestoreOptions struct {
|
||||
InputPath string
|
||||
TargetDir string
|
||||
MasterKey []byte
|
||||
Force bool
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// ErrRunningAllocs is returned when restore refuses to clobber a live
|
||||
// cluster that has running allocations and --force was not given.
|
||||
var ErrRunningAllocs = errors.New("restore: running allocations present (use --force to stop and restart)")
|
||||
|
||||
// runRestore is the entry point invoked by restoreCmd.RunE. It performs
|
||||
// the full P07 recovery flow. The cmd is used only for output; ctx comes
|
||||
// from cmd.Context().
|
||||
func runRestore(cmd *cobra.Command, opts RestoreOptions) error {
|
||||
ctx := cmd.Context()
|
||||
if opts.InputPath == "" {
|
||||
return fmt.Errorf("restore: InputPath is empty")
|
||||
}
|
||||
if opts.TargetDir == "" {
|
||||
return fmt.Errorf("restore: TargetDir is empty")
|
||||
}
|
||||
if len(opts.MasterKey) == 0 {
|
||||
return fmt.Errorf("restore: MasterKey is empty")
|
||||
}
|
||||
log := newLogger()
|
||||
|
||||
sigPath := opts.InputPath + ".sig"
|
||||
if err := backup.VerifySignature(opts.InputPath, sigPath, opts.MasterKey); err != nil {
|
||||
auditRestore(ctx, "failure", err, map[string]any{"path": opts.InputPath, "target": opts.TargetDir})
|
||||
return fmt.Errorf("restore: verify signature: %w", err)
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
return runRestoreDryRun(cmd, opts)
|
||||
}
|
||||
|
||||
ex, exErr := drainExecFromCtx(ctx)
|
||||
if exErr != nil {
|
||||
return fmt.Errorf("ssh transport: %w", exErr)
|
||||
}
|
||||
|
||||
running, err := scanRunningAllocs(ctx, ex)
|
||||
if err != nil {
|
||||
log.Warn("restore: scan running allocs failed", "error", err)
|
||||
}
|
||||
if len(running) > 0 && !opts.Force {
|
||||
allocList := formatRunningAllocs(running)
|
||||
err := fmt.Errorf("%w: %s", ErrRunningAllocs, allocList)
|
||||
auditRestore(ctx, "refused", err, map[string]any{
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
"running": running,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.Force && len(running) > 0 {
|
||||
stopped, stopErr := stopRunningAllocs(ctx, ex, running)
|
||||
if stopErr != nil {
|
||||
auditRestore(ctx, "partial", stopErr, map[string]any{
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
"stopped": stopped,
|
||||
"running": running,
|
||||
})
|
||||
return fmt.Errorf("restore: stop running allocs: %w", stopErr)
|
||||
}
|
||||
log.Info("restore: stopped running allocs", "count", len(stopped))
|
||||
}
|
||||
|
||||
ropts := backup.RestoreOptions{
|
||||
InputPath: opts.InputPath,
|
||||
TargetDir: opts.TargetDir,
|
||||
MasterKey: opts.MasterKey,
|
||||
Force: true,
|
||||
}
|
||||
if err := backup.Restore(ropts); err != nil {
|
||||
auditRestore(ctx, "failure", err, map[string]any{
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.Force && len(running) > 0 {
|
||||
if rerr := restartAllocs(ctx, ex, running); rerr != nil {
|
||||
log.Warn("restore: failed to restart some allocs", "error", rerr)
|
||||
}
|
||||
}
|
||||
|
||||
verifications := verifyRestoredState(opts.TargetDir, opts.MasterKey)
|
||||
report := buildVerificationReport(verifications)
|
||||
result := "success"
|
||||
var verifyErr error
|
||||
if len(verifications) > 0 {
|
||||
result = "partial"
|
||||
verifyErr = fmt.Errorf("post-restore verification: %d issue(s)", len(verifications))
|
||||
}
|
||||
auditRestore(ctx, result, verifyErr, map[string]any{
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
"force": opts.Force,
|
||||
"running": running,
|
||||
"verification": report,
|
||||
})
|
||||
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]any{
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
"force": opts.Force,
|
||||
"running": running,
|
||||
"verification": report,
|
||||
})
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "✓ Restored to: %s\n", opts.TargetDir)
|
||||
if opts.Force && len(running) > 0 {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " stopped+restarted %d running alloc(s)\n", len(running))
|
||||
}
|
||||
for _, v := range verifications {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " ⚠ verify: %s\n", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runRestoreDryRun extracts the tarball to a temp dir, runs
|
||||
// post-restore verification against that temp dir, reports what WOULD
|
||||
// be restored, and cleans up. The real ORCA_HOME is never touched.
|
||||
func runRestoreDryRun(cmd *cobra.Command, opts RestoreOptions) error {
|
||||
tmp, err := os.MkdirTemp("", "orca-restore-dryrun-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("restore: dry-run temp dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
log := newLogger()
|
||||
log.Info("restore: dry-run staging dir", "dir", tmp)
|
||||
|
||||
ropts := backup.RestoreOptions{
|
||||
InputPath: opts.InputPath,
|
||||
TargetDir: tmp,
|
||||
MasterKey: opts.MasterKey,
|
||||
Force: true,
|
||||
}
|
||||
if err := backup.Restore(ropts); err != nil {
|
||||
return fmt.Errorf("restore: dry-run extract: %w", err)
|
||||
}
|
||||
|
||||
entries := listExtractedFiles(tmp)
|
||||
verifications := verifyRestoredState(tmp, opts.MasterKey)
|
||||
report := buildVerificationReport(verifications)
|
||||
auditRestore(cmd.Context(), "dry-run", nil, map[string]any{
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
"staging": tmp,
|
||||
"files": len(entries),
|
||||
"verification": report,
|
||||
})
|
||||
|
||||
if jsonOutput {
|
||||
return printJSON(map[string]any{
|
||||
"dry_run": true,
|
||||
"path": opts.InputPath,
|
||||
"target": opts.TargetDir,
|
||||
"files": len(entries),
|
||||
"verification": report,
|
||||
})
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "✓ Dry-run: would restore %d file(s) to %s\n", len(entries), opts.TargetDir)
|
||||
if len(entries) > 0 && len(entries) <= 20 {
|
||||
for _, e := range entries {
|
||||
fmt.Fprintf(out, " %s\n", e)
|
||||
}
|
||||
}
|
||||
for _, v := range verifications {
|
||||
fmt.Fprintf(out, " ⚠ verify: %s\n", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runningAlloc is a running allocation discovered on a peer node.
|
||||
type runningAlloc struct {
|
||||
Node string `json:"node"`
|
||||
Peer string `json:"peer"`
|
||||
AllocID string `json:"alloc_id"`
|
||||
}
|
||||
|
||||
// scanRunningAllocs lists every peer node in the registry and queries
|
||||
// each via SSH for currently-running orca-alloc-*.service units. A
|
||||
// registry failure or empty registry yields an empty (not error)
|
||||
// slice; per-node failures are logged and skipped so a single
|
||||
// unreachable peer does not block the restore.
|
||||
func scanRunningAllocs(ctx context.Context, ex drainExecer) ([]runningAlloc, error) {
|
||||
reg, closer, err := nodeRegistry()
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
defer closer()
|
||||
nodes, err := reg.List(ctx)
|
||||
if err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
var out []runningAlloc
|
||||
log := newLogger()
|
||||
for i := range nodes {
|
||||
n := nodes[i]
|
||||
peer := peerAddrForNode(n)
|
||||
if peer == "" {
|
||||
continue
|
||||
}
|
||||
ids, err := listRunningAllocs(ctx, ex, peer)
|
||||
if err != nil {
|
||||
log.Warn("restore: cannot list allocs on node",
|
||||
slog.String("node", n.Name), slog.String("peer", peer), "error", err)
|
||||
continue
|
||||
}
|
||||
for _, id := range ids {
|
||||
out = append(out, runningAlloc{Node: n.Name, Peer: peer, AllocID: id})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// formatRunningAllocs renders a list of running allocs for an error
|
||||
// message (one per line: "node/peer: alloc-id").
|
||||
func formatRunningAllocs(rs []runningAlloc) string {
|
||||
var b strings.Builder
|
||||
for i, a := range rs {
|
||||
if i > 0 {
|
||||
b.WriteString("; ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%s/%s: %s", a.Node, a.Peer, a.AllocID)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// stopRunningAllocs stops every allocation in rs via SSH (systemctl
|
||||
// stop). It returns the ids that were successfully stopped and an
|
||||
// aggregate error if any failed.
|
||||
func stopRunningAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) ([]string, error) {
|
||||
var stopped []string
|
||||
var failed []string
|
||||
for _, a := range rs {
|
||||
if err := stopAlloc(ctx, ex, a.Peer, a.AllocID); err != nil {
|
||||
failed = append(failed, a.AllocID)
|
||||
continue
|
||||
}
|
||||
stopped = append(stopped, a.AllocID)
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return stopped, fmt.Errorf("failed to stop %d alloc(s): %s", len(failed), strings.Join(failed, ", "))
|
||||
}
|
||||
return stopped, nil
|
||||
}
|
||||
|
||||
// restartAllocs restarts every allocation in rs via SSH (systemctl
|
||||
// start). A per-alloc failure is collected and returned as an aggregate
|
||||
// error; the caller treats restart failures as non-fatal (the restore
|
||||
// itself succeeded).
|
||||
func restartAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) error {
|
||||
var failed []string
|
||||
for _, a := range rs {
|
||||
startCmd := fmt.Sprintf("systemctl start %s", allocUnit(a.AllocID))
|
||||
if _, err := ex.Exec(ctx, a.Peer, startCmd); err != nil {
|
||||
failed = append(failed, a.AllocID)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("failed to restart %d alloc(s): %s", len(failed), strings.Join(failed, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyRestoredState checks the restored tree for the master key
|
||||
// (present + valid length/mode), namespace directories, and openable
|
||||
// SQLite databases. It returns a slice of human-readable discrepancy
|
||||
// strings (empty if everything checks out).
|
||||
func verifyRestoredState(targetDir string, masterKey []byte) []string {
|
||||
var issues []string
|
||||
|
||||
mkPath := filepath.Join(targetDir, relFromRoot(paths.MasterKeyPath()))
|
||||
info, err := os.Stat(mkPath)
|
||||
if err != nil {
|
||||
issues = append(issues, fmt.Sprintf("master key missing: %s", mkPath))
|
||||
} else {
|
||||
if info.Mode().Perm() != secrets.MasterKeyMode {
|
||||
issues = append(issues, fmt.Sprintf("master key mode %04o (want %04o)", info.Mode().Perm(), secrets.MasterKeyMode))
|
||||
}
|
||||
key, rerr := os.ReadFile(mkPath)
|
||||
if rerr != nil {
|
||||
issues = append(issues, fmt.Sprintf("master key unreadable: %v", rerr))
|
||||
} else if len(key) != secrets.MasterKeyLen {
|
||||
issues = append(issues, fmt.Sprintf("master key length %d (want %d)", len(key), secrets.MasterKeyLen))
|
||||
} else if masterKey != nil && string(key) != string(masterKey) {
|
||||
issues = append(issues, "master key differs from the key used to verify the signature")
|
||||
}
|
||||
}
|
||||
|
||||
nsDirs := findNamespaceDirs(targetDir)
|
||||
if len(nsDirs) == 0 {
|
||||
issues = append(issues, "no namespace directories found")
|
||||
}
|
||||
for _, nsDir := range nsDirs {
|
||||
dbPath := filepath.Join(nsDir, "db", "orca.db")
|
||||
if _, err := os.Stat(dbPath); err == nil {
|
||||
if err := dbOpenable(dbPath); err != nil {
|
||||
issues = append(issues, fmt.Sprintf("db not openable %s: %v", dbPath, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clusterDB := filepath.Join(targetDir, "orca.db")
|
||||
if _, err := os.Stat(clusterDB); err == nil {
|
||||
if err := dbOpenable(clusterDB); err != nil {
|
||||
issues = append(issues, fmt.Sprintf("cluster db not openable %s: %v", clusterDB, err))
|
||||
}
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
// relFromRoot strips the ORCA_HOME prefix off an absolute path so it
|
||||
// can be rejoined to an arbitrary target dir (used by --dry-run and
|
||||
// non-default --target restores). If the path does not start with the
|
||||
// current ORCA_HOME root, the path's base is returned.
|
||||
func relFromRoot(p string) string {
|
||||
root := filepath.Clean(paths.Root())
|
||||
clean := filepath.Clean(p)
|
||||
if rel, err := filepath.Rel(root, clean); err == nil && !strings.HasPrefix(rel, "..") {
|
||||
return rel
|
||||
}
|
||||
return filepath.Base(clean)
|
||||
}
|
||||
|
||||
// findNamespaceDirs returns the top-level directories under targetDir
|
||||
// that look like namespace dirs (excluding the "cluster" dir and dot-
|
||||
// files). A namespace dir is any immediate child of ORCA_HOME that is
|
||||
// a directory and not "cluster".
|
||||
func findNamespaceDirs(targetDir string) []string {
|
||||
entries, err := os.ReadDir(targetDir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
if strings.HasPrefix(name, ".") {
|
||||
continue
|
||||
}
|
||||
if name == "cluster" {
|
||||
continue
|
||||
}
|
||||
out = append(out, filepath.Join(targetDir, name))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dbOpenable reports whether the SQLite file at path can be opened
|
||||
// read-only. It uses the same driver as the rest of the codebase
|
||||
// (modernc.org/sqlite via store.Open, but with a read-only pragma).
|
||||
func dbOpenable(path string) error {
|
||||
dsn := "file:" + path + "?mode=ro&_pragma=journal_mode(WAL)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
if err := db.Ping(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// listExtractedFiles walks a staging dir and returns the relative
|
||||
// paths of all regular files (capped at 1000 for reporting).
|
||||
func listExtractedFiles(root string) []string {
|
||||
var out []string
|
||||
_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
rel, rerr := filepath.Rel(root, path)
|
||||
if rerr != nil {
|
||||
return nil
|
||||
}
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
return nil
|
||||
})
|
||||
if len(out) > 1000 {
|
||||
out = out[:1000]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildVerificationReport converts the discrepancy slice into a
|
||||
// JSON-friendly []string (nil-safe for empty input).
|
||||
func buildVerificationReport(issues []string) []string {
|
||||
if len(issues) == 0 {
|
||||
return []string{}
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
// auditRestore records a restore event in the audit log (same pattern
|
||||
// as auditDrain in drain.go). It opens the cluster DB at
|
||||
// certpaths.DBPath(); a failure to open the DB is logged and silently
|
||||
// dropped so a restore is never blocked by the audit log itself.
|
||||
func auditRestore(ctx context.Context, result string, err error, meta map[string]any) {
|
||||
db, dbErr := store.Open(certpaths.DBPath())
|
||||
if dbErr != nil {
|
||||
newLogger().Warn("restore: audit log unavailable", "error", dbErr)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
engine.NewAudit(store.NewAuditRepo(db), newLogger()).Record(ctx, "cli", "restore", certpaths.Dir(), result, err, meta)
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.cloudinit.dev/coreci/orca/internal/backup"
|
||||
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/model"
|
||||
"git.cloudinit.dev/coreci/orca/internal/paths"
|
||||
"git.cloudinit.dev/coreci/orca/internal/secrets"
|
||||
"git.cloudinit.dev/coreci/orca/internal/store"
|
||||
)
|
||||
|
||||
// recoveryTestEnv sets up an ORCA_HOME with a master key, an empty
|
||||
// cluster DB (so nodeRegistry works), and returns the home dir + a
|
||||
// cleanup. It does NOT install a mock drain execer (callers that need
|
||||
// one install scriptedDrainExec themselves).
|
||||
func recoveryTestEnv(t *testing.T) (string, func()) {
|
||||
t.Helper()
|
||||
dir, cleanup := initTestEnv(t)
|
||||
if err := os.MkdirAll(paths.ClusterDir(), 0o755); err != nil {
|
||||
t.Fatalf("mkdir cluster dir: %v", err)
|
||||
}
|
||||
mk, err := secrets.GenerateMasterKey()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateMasterKey: %v", err)
|
||||
}
|
||||
if err := secrets.SaveMasterKey(paths.MasterKeyPath(), mk); err != nil {
|
||||
t.Fatalf("SaveMasterKey: %v", err)
|
||||
}
|
||||
// Ensure the cluster DB exists (migrations run on Open) so
|
||||
// auditRestore and nodeRegistry work.
|
||||
db, err := store.Open(certpaths.DBPath())
|
||||
if err != nil {
|
||||
t.Fatalf("open cluster db: %v", err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close cluster db: %v", err)
|
||||
}
|
||||
return dir, cleanup
|
||||
}
|
||||
|
||||
// makeBackup creates a signed tarball of homeDir containing the given
|
||||
// relative file payloads (map[relPath]content) and returns the tarball
|
||||
// path. It reuses the real `orca backup` command path for realism.
|
||||
func makeBackup(t *testing.T, homeDir string, files map[string]string) string {
|
||||
t.Helper()
|
||||
for rel, body := range files {
|
||||
p := filepath.Join(homeDir, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", rel, err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", rel, err)
|
||||
}
|
||||
}
|
||||
outDir := t.TempDir()
|
||||
out := filepath.Join(outDir, "orca-backup.tar.gz")
|
||||
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMasterKey: %v", err)
|
||||
}
|
||||
if err := backup.Backup(backup.BackupOptions{
|
||||
SourceDir: homeDir,
|
||||
OutputPath: out,
|
||||
MasterKey: mk,
|
||||
}); err != nil {
|
||||
t.Fatalf("Backup: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// restoreNodeForTest inserts a node with a fixed id+name so scanRunningAllocs
|
||||
// can find it. Uses the test ORCA_HOME cluster DB.
|
||||
func restoreNodeForTest(t *testing.T, name, addr string) *model.Node {
|
||||
t.Helper()
|
||||
db, err := store.Open(certpaths.DBPath())
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
n := &model.Node{
|
||||
ID: "node-" + name,
|
||||
Name: name,
|
||||
Address: addr,
|
||||
State: model.NodeStateReady,
|
||||
JoinedAt: time.Now().UTC(),
|
||||
LastSeen: time.Now().UTC(),
|
||||
Kind: string(model.NodeKindLinux),
|
||||
}
|
||||
if err := store.NewNodeRepo(db).Insert(context.Background(), n); err != nil {
|
||||
t.Fatalf("insert node: %v", err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// runRestoreArgs invokes the restore command with the given flags and
|
||||
// returns (output, error).
|
||||
func runRestoreArgs(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
resetRootFlags(t)
|
||||
rootCmd.SetOut(&buf)
|
||||
rootCmd.SetErr(&buf)
|
||||
rootCmd.SetArgs(append([]string{"restore"}, args...))
|
||||
err := rootCmd.Execute()
|
||||
return buf.String(), err
|
||||
}
|
||||
|
||||
func TestRestore_DryRun_DoesNotTouchORCAHome(t *testing.T) {
|
||||
home, cleanup := recoveryTestEnv(t)
|
||||
defer cleanup()
|
||||
// Put a marker file in ORCA_HOME that must survive the dry-run.
|
||||
marker := filepath.Join(home, "marker.txt")
|
||||
if err := os.WriteFile(marker, []byte("original"), 0o644); err != nil {
|
||||
t.Fatalf("write marker: %v", err)
|
||||
}
|
||||
// Build the backup from a SEPARATE source dir (not ORCA_HOME) so we
|
||||
// can prove the dry-run never writes into ORCA_HOME.
|
||||
srcDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(srcDir, "keep.txt"), []byte("payload"), 0o644); err != nil {
|
||||
t.Fatalf("write keep.txt: %v", err)
|
||||
}
|
||||
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMasterKey: %v", err)
|
||||
}
|
||||
out := filepath.Join(t.TempDir(), "orca-backup.tar.gz")
|
||||
if err := backup.Backup(backup.BackupOptions{
|
||||
SourceDir: srcDir,
|
||||
OutputPath: out,
|
||||
MasterKey: mk,
|
||||
}); err != nil {
|
||||
t.Fatalf("Backup: %v", err)
|
||||
}
|
||||
|
||||
before, _ := os.ReadDir(home)
|
||||
out2, err := runRestoreArgs(t, "--in", out, "--dry-run")
|
||||
if err != nil {
|
||||
t.Fatalf("restore --dry-run: %v", err)
|
||||
}
|
||||
if !strings.Contains(out2, "would restore") {
|
||||
t.Errorf("dry-run output missing 'would restore': %s", out2)
|
||||
}
|
||||
// ORCA_HOME must be untouched: marker intact, no keep.txt written.
|
||||
got, err := os.ReadFile(marker)
|
||||
if err != nil {
|
||||
t.Fatalf("marker missing after dry-run: %v", err)
|
||||
}
|
||||
if string(got) != "original" {
|
||||
t.Errorf("marker changed by dry-run: %q", string(got))
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(home, "keep.txt")); err == nil {
|
||||
t.Errorf("dry-run wrote keep.txt into ORCA_HOME")
|
||||
}
|
||||
after, _ := os.ReadDir(home)
|
||||
if len(before) != len(after) {
|
||||
t.Errorf("ORCA_HOME entry count changed by dry-run: before=%d after=%d", len(before), len(after))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestore_RefusesRunningAllocsWithoutForce(t *testing.T) {
|
||||
home, cleanup := recoveryTestEnv(t)
|
||||
defer cleanup()
|
||||
out := makeBackup(t, home, map[string]string{"keep.txt": "payload"})
|
||||
|
||||
restoreNodeForTest(t, "runnode", "runnode:8443")
|
||||
mx := &scriptedDrainExec{}
|
||||
mx.queueAlways("list-units", "orca-alloc-web-0.service loaded active running\n", 0)
|
||||
drainExecOverride = mx
|
||||
|
||||
_, err := runRestoreArgs(t, "--in", out, "--target", filepath.Join(t.TempDir(), "restored"))
|
||||
if err == nil {
|
||||
t.Fatal("restore should refuse when allocs are running")
|
||||
}
|
||||
if !errors.Is(err, ErrRunningAllocs) {
|
||||
t.Errorf("expected ErrRunningAllocs, got: %v", err)
|
||||
}
|
||||
// Must NOT have attempted to stop or start anything.
|
||||
if mx.countCalls("systemctl stop") != 0 {
|
||||
t.Errorf("restore without --force issued stop commands: %+v", mx.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestore_Force_StopsAndRestartsAllocs(t *testing.T) {
|
||||
home, cleanup := recoveryTestEnv(t)
|
||||
defer cleanup()
|
||||
out := makeBackup(t, home, map[string]string{"keep.txt": "payload"})
|
||||
|
||||
restoreNodeForTest(t, "forcenode", "forcenode:8443")
|
||||
mx := &scriptedDrainExec{}
|
||||
// list-units always returns the running alloc; stop/start succeed.
|
||||
mx.queueAlways("list-units", "orca-alloc-web-0.service loaded active running\n", 0)
|
||||
mx.queueAlways("systemctl stop", "", 0)
|
||||
mx.queueAlways("systemctl start", "", 0)
|
||||
drainExecOverride = mx
|
||||
|
||||
target := filepath.Join(t.TempDir(), "restored")
|
||||
out2, err := runRestoreArgs(t, "--in", out, "--target", target, "--force")
|
||||
if err != nil {
|
||||
t.Fatalf("restore --force: %v\n%s", err, out2)
|
||||
}
|
||||
if mx.countCalls("systemctl stop orca-alloc-web-0") == 0 {
|
||||
t.Errorf("expected a stop command for web-0, calls: %+v", mx.calls)
|
||||
}
|
||||
if mx.countCalls("systemctl start orca-alloc-web-0") == 0 {
|
||||
t.Errorf("expected a start command for web-0, calls: %+v", mx.calls)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(target, "keep.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("restored keep.txt missing: %v", err)
|
||||
}
|
||||
if string(got) != "payload" {
|
||||
t.Errorf("restored keep.txt = %q, want %q", string(got), "payload")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestore_PostVerify_MasterKeyMissing(t *testing.T) {
|
||||
_, cleanup := recoveryTestEnv(t)
|
||||
defer cleanup()
|
||||
// Build a backup whose payload LACKS the master key. We back up a
|
||||
// different source dir so the cluster/master.key is not included.
|
||||
srcDir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(srcDir, "only.txt"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatalf("write only.txt: %v", err)
|
||||
}
|
||||
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
||||
if err != nil {
|
||||
t.Fatalf("LoadMasterKey: %v", err)
|
||||
}
|
||||
out := filepath.Join(t.TempDir(), "nb.tar.gz")
|
||||
if err := backup.Backup(backup.BackupOptions{
|
||||
SourceDir: srcDir,
|
||||
OutputPath: out,
|
||||
MasterKey: mk,
|
||||
}); err != nil {
|
||||
t.Fatalf("Backup: %v", err)
|
||||
}
|
||||
|
||||
// No mock drain execer → no nodes → no running allocs → restore
|
||||
// proceeds. The target is a fresh dir; verify will report the
|
||||
// missing master key.
|
||||
target := filepath.Join(t.TempDir(), "restored")
|
||||
out2, err := runRestoreArgs(t, "--in", out, "--target", target)
|
||||
if err != nil {
|
||||
t.Fatalf("restore returned error: %v\n%s", err, out2)
|
||||
}
|
||||
if !strings.Contains(out2, "master key missing") {
|
||||
t.Errorf("expected 'master key missing' in output, got: %s", out2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestore_AuditLogEntry(t *testing.T) {
|
||||
home, cleanup := recoveryTestEnv(t)
|
||||
defer cleanup()
|
||||
out := makeBackup(t, home, map[string]string{"keep.txt": "payload"})
|
||||
|
||||
// No nodes → no running allocs → restore succeeds and records.
|
||||
target := filepath.Join(t.TempDir(), "restored")
|
||||
if _, err := runRestoreArgs(t, "--in", out, "--target", target); err != nil {
|
||||
t.Fatalf("restore: %v", err)
|
||||
}
|
||||
|
||||
db, err := store.Open(certpaths.DBPath())
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
entries, err := store.NewAuditRepo(db).List(context.Background(), 50)
|
||||
if err != nil {
|
||||
t.Fatalf("list audit: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, e := range entries {
|
||||
if e.Action == "restore" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no 'restore' entry in audit log; entries: %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestore_SignatureFailure_Refuses(t *testing.T) {
|
||||
_, cleanup := recoveryTestEnv(t)
|
||||
defer cleanup()
|
||||
out := filepath.Join(t.TempDir(), "bad.tar.gz")
|
||||
if err := os.WriteFile(out, []byte("not a tarball"), 0o644); err != nil {
|
||||
t.Fatalf("write fake tarball: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(out+".sig", []byte("deadbeef"), 0o644); err != nil {
|
||||
t.Fatalf("write fake sig: %v", err)
|
||||
}
|
||||
_, err := runRestoreArgs(t, "--in", out, "--target", filepath.Join(t.TempDir(), "r"))
|
||||
if err == nil {
|
||||
t.Fatal("restore with bad signature should fail")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user