diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 592856d..687221a 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -26,6 +26,7 @@ var ( restoreInPath string restoreTargetDir string restoreForce bool + restoreDryRun bool ) var backupCmd = &cobra.Command{ @@ -70,11 +71,16 @@ written to --out; the hex-encoded signature to --out + ".sig".`, var restoreCmd = &cobra.Command{ Use: "restore", Short: "Restore ORCA_HOME from a verified signed backup", - Long: `Restore ORCA_HOME from a verified signed backup (P04). + Long: `Restore ORCA_HOME from a verified signed backup (P04/P07). Verifies the HMAC-SHA256 signature on --in (using the cluster master -key) before extracting. With --force, overwrites a non-empty target; -without it, refuses to clobber an existing ORCA_HOME.`, +key) before extracting. Reconciles with live state: refuses to clobber +running allocations unless --force is given (with --force, stops the +running allocs, extracts, then restarts them from the restored state). +With --dry-run, extracts to a temp dir and reports what WOULD be +restored without touching the real ORCA_HOME. Performs post-restore +verification (master key, namespace dirs, SQLite DBs) and records the +restore in the audit log.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { if restoreInPath == "" { @@ -88,23 +94,14 @@ without it, refuses to clobber an existing ORCA_HOME.`, if target == "" { target = paths.Root() } - opts := backup.RestoreOptions{ + opts := RestoreOptions{ InputPath: restoreInPath, TargetDir: target, MasterKey: mk, Force: restoreForce, + DryRun: restoreDryRun, } - if err := backup.Restore(opts); err != nil { - return err - } - if jsonOutput { - return printJSON(map[string]string{ - "path": restoreInPath, - "target": target, - }) - } - fmt.Fprintf(cmd.OutOrStdout(), "✓ Restored to: %s\n", target) - return nil + return runRestore(cmd, opts) }, } @@ -112,7 +109,8 @@ func init() { backupCmd.Flags().StringVar(&backupOutPath, "out", "", "output tarball path (default: orca-backup-.tar.gz in CWD)") restoreCmd.Flags().StringVar(&restoreInPath, "in", "", "input tarball path (required)") restoreCmd.Flags().StringVar(&restoreTargetDir, "target", "", "restore target dir (default: ORCA_HOME)") - restoreCmd.Flags().BoolVar(&restoreForce, "force", false, "overwrite a non-empty target directory") + restoreCmd.Flags().BoolVar(&restoreForce, "force", false, "overwrite a non-empty target directory and stop+restart running allocs") + restoreCmd.Flags().BoolVar(&restoreDryRun, "dry-run", false, "extract to a temp dir and report what would be restored without touching ORCA_HOME") rootCmd.AddCommand(backupCmd) rootCmd.AddCommand(restoreCmd) } diff --git a/internal/cli/namespace_test.go b/internal/cli/namespace_test.go index 9bba178..2bed420 100644 --- a/internal/cli/namespace_test.go +++ b/internal/cli/namespace_test.go @@ -42,6 +42,7 @@ func resetCommandFlags() { auditLimit = 50 backupOutPath, restoreInPath, restoreTargetDir = "", "", "" restoreForce = false + restoreDryRun = false resetNSFlags() // Reset per-command output writers so tests that polluted them // (e.g. daemon tests calling cmd.SetOut(&buf)) don't leak into diff --git a/internal/cli/recovery.go b/internal/cli/recovery.go new file mode 100644 index 0000000..9aa0be1 --- /dev/null +++ b/internal/cli/recovery.go @@ -0,0 +1,466 @@ +// Package cli: recovery.go implements the full-recovery semantics for +// `orca restore` (P07, v0.11 milestone). +// +// On top of P04's signed-restore, restore now: +// +// 1. Verifies the signature (P04). +// 2. --dry-run: extracts to a temp staging dir, verifies, reports, +// cleans up, and exits — never touching the real ORCA_HOME. +// 3. Without --force: scans every peer for running orca-alloc-* +// systemd units; if any are running, refuses the restore (data +// loss protection) and reports which allocs are running. +// 4. With --force: stops the running allocs (SSH systemctl stop), +// extracts the tarball, then restarts the allocs from the restored +// state (SSH systemctl start). +// 5. Post-restore verification: master key present + valid, namespace +// dirs exist, SQLite DBs openable; reports discrepancies. +// 6. Records the restore in the audit log (internal/store/audit_repo). +package cli + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "git.cloudinit.dev/coreci/orca/internal/backup" + "git.cloudinit.dev/coreci/orca/internal/certpaths" + "git.cloudinit.dev/coreci/orca/internal/engine" + "git.cloudinit.dev/coreci/orca/internal/paths" + "git.cloudinit.dev/coreci/orca/internal/secrets" + "git.cloudinit.dev/coreci/orca/internal/store" +) + +// RestoreOptions configures the P07 full-recovery restore. It mirrors +// backup.RestoreOptions but adds DryRun and is consumed by runRestore +// (the CLI layer), which performs the alloc reconciliation and audit +// logging that the lower-level backup.Restore does not. +type RestoreOptions struct { + InputPath string + TargetDir string + MasterKey []byte + Force bool + DryRun bool +} + +// ErrRunningAllocs is returned when restore refuses to clobber a live +// cluster that has running allocations and --force was not given. +var ErrRunningAllocs = errors.New("restore: running allocations present (use --force to stop and restart)") + +// runRestore is the entry point invoked by restoreCmd.RunE. It performs +// the full P07 recovery flow. The cmd is used only for output; ctx comes +// from cmd.Context(). +func runRestore(cmd *cobra.Command, opts RestoreOptions) error { + ctx := cmd.Context() + if opts.InputPath == "" { + return fmt.Errorf("restore: InputPath is empty") + } + if opts.TargetDir == "" { + return fmt.Errorf("restore: TargetDir is empty") + } + if len(opts.MasterKey) == 0 { + return fmt.Errorf("restore: MasterKey is empty") + } + log := newLogger() + + sigPath := opts.InputPath + ".sig" + if err := backup.VerifySignature(opts.InputPath, sigPath, opts.MasterKey); err != nil { + auditRestore(ctx, "failure", err, map[string]any{"path": opts.InputPath, "target": opts.TargetDir}) + return fmt.Errorf("restore: verify signature: %w", err) + } + + if opts.DryRun { + return runRestoreDryRun(cmd, opts) + } + + ex, exErr := drainExecFromCtx(ctx) + if exErr != nil { + return fmt.Errorf("ssh transport: %w", exErr) + } + + running, err := scanRunningAllocs(ctx, ex) + if err != nil { + log.Warn("restore: scan running allocs failed", "error", err) + } + if len(running) > 0 && !opts.Force { + allocList := formatRunningAllocs(running) + err := fmt.Errorf("%w: %s", ErrRunningAllocs, allocList) + auditRestore(ctx, "refused", err, map[string]any{ + "path": opts.InputPath, + "target": opts.TargetDir, + "running": running, + }) + return err + } + + if opts.Force && len(running) > 0 { + stopped, stopErr := stopRunningAllocs(ctx, ex, running) + if stopErr != nil { + auditRestore(ctx, "partial", stopErr, map[string]any{ + "path": opts.InputPath, + "target": opts.TargetDir, + "stopped": stopped, + "running": running, + }) + return fmt.Errorf("restore: stop running allocs: %w", stopErr) + } + log.Info("restore: stopped running allocs", "count", len(stopped)) + } + + ropts := backup.RestoreOptions{ + InputPath: opts.InputPath, + TargetDir: opts.TargetDir, + MasterKey: opts.MasterKey, + Force: true, + } + if err := backup.Restore(ropts); err != nil { + auditRestore(ctx, "failure", err, map[string]any{ + "path": opts.InputPath, + "target": opts.TargetDir, + }) + return err + } + + if opts.Force && len(running) > 0 { + if rerr := restartAllocs(ctx, ex, running); rerr != nil { + log.Warn("restore: failed to restart some allocs", "error", rerr) + } + } + + verifications := verifyRestoredState(opts.TargetDir, opts.MasterKey) + report := buildVerificationReport(verifications) + result := "success" + var verifyErr error + if len(verifications) > 0 { + result = "partial" + verifyErr = fmt.Errorf("post-restore verification: %d issue(s)", len(verifications)) + } + auditRestore(ctx, result, verifyErr, map[string]any{ + "path": opts.InputPath, + "target": opts.TargetDir, + "force": opts.Force, + "running": running, + "verification": report, + }) + + if jsonOutput { + return printJSON(map[string]any{ + "path": opts.InputPath, + "target": opts.TargetDir, + "force": opts.Force, + "running": running, + "verification": report, + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "✓ Restored to: %s\n", opts.TargetDir) + if opts.Force && len(running) > 0 { + fmt.Fprintf(cmd.OutOrStdout(), " stopped+restarted %d running alloc(s)\n", len(running)) + } + for _, v := range verifications { + fmt.Fprintf(cmd.OutOrStdout(), " ⚠ verify: %s\n", v) + } + return nil +} + +// runRestoreDryRun extracts the tarball to a temp dir, runs +// post-restore verification against that temp dir, reports what WOULD +// be restored, and cleans up. The real ORCA_HOME is never touched. +func runRestoreDryRun(cmd *cobra.Command, opts RestoreOptions) error { + tmp, err := os.MkdirTemp("", "orca-restore-dryrun-*") + if err != nil { + return fmt.Errorf("restore: dry-run temp dir: %w", err) + } + defer os.RemoveAll(tmp) + log := newLogger() + log.Info("restore: dry-run staging dir", "dir", tmp) + + ropts := backup.RestoreOptions{ + InputPath: opts.InputPath, + TargetDir: tmp, + MasterKey: opts.MasterKey, + Force: true, + } + if err := backup.Restore(ropts); err != nil { + return fmt.Errorf("restore: dry-run extract: %w", err) + } + + entries := listExtractedFiles(tmp) + verifications := verifyRestoredState(tmp, opts.MasterKey) + report := buildVerificationReport(verifications) + auditRestore(cmd.Context(), "dry-run", nil, map[string]any{ + "path": opts.InputPath, + "target": opts.TargetDir, + "staging": tmp, + "files": len(entries), + "verification": report, + }) + + if jsonOutput { + return printJSON(map[string]any{ + "dry_run": true, + "path": opts.InputPath, + "target": opts.TargetDir, + "files": len(entries), + "verification": report, + }) + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "✓ Dry-run: would restore %d file(s) to %s\n", len(entries), opts.TargetDir) + if len(entries) > 0 && len(entries) <= 20 { + for _, e := range entries { + fmt.Fprintf(out, " %s\n", e) + } + } + for _, v := range verifications { + fmt.Fprintf(out, " ⚠ verify: %s\n", v) + } + return nil +} + +// runningAlloc is a running allocation discovered on a peer node. +type runningAlloc struct { + Node string `json:"node"` + Peer string `json:"peer"` + AllocID string `json:"alloc_id"` +} + +// scanRunningAllocs lists every peer node in the registry and queries +// each via SSH for currently-running orca-alloc-*.service units. A +// registry failure or empty registry yields an empty (not error) +// slice; per-node failures are logged and skipped so a single +// unreachable peer does not block the restore. +func scanRunningAllocs(ctx context.Context, ex drainExecer) ([]runningAlloc, error) { + reg, closer, err := nodeRegistry() + if err != nil { + return nil, nil + } + defer closer() + nodes, err := reg.List(ctx) + if err != nil { + return nil, nil + } + var out []runningAlloc + log := newLogger() + for i := range nodes { + n := nodes[i] + peer := peerAddrForNode(n) + if peer == "" { + continue + } + ids, err := listRunningAllocs(ctx, ex, peer) + if err != nil { + log.Warn("restore: cannot list allocs on node", + slog.String("node", n.Name), slog.String("peer", peer), "error", err) + continue + } + for _, id := range ids { + out = append(out, runningAlloc{Node: n.Name, Peer: peer, AllocID: id}) + } + } + return out, nil +} + +// formatRunningAllocs renders a list of running allocs for an error +// message (one per line: "node/peer: alloc-id"). +func formatRunningAllocs(rs []runningAlloc) string { + var b strings.Builder + for i, a := range rs { + if i > 0 { + b.WriteString("; ") + } + fmt.Fprintf(&b, "%s/%s: %s", a.Node, a.Peer, a.AllocID) + } + return b.String() +} + +// stopRunningAllocs stops every allocation in rs via SSH (systemctl +// stop). It returns the ids that were successfully stopped and an +// aggregate error if any failed. +func stopRunningAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) ([]string, error) { + var stopped []string + var failed []string + for _, a := range rs { + if err := stopAlloc(ctx, ex, a.Peer, a.AllocID); err != nil { + failed = append(failed, a.AllocID) + continue + } + stopped = append(stopped, a.AllocID) + } + if len(failed) > 0 { + return stopped, fmt.Errorf("failed to stop %d alloc(s): %s", len(failed), strings.Join(failed, ", ")) + } + return stopped, nil +} + +// restartAllocs restarts every allocation in rs via SSH (systemctl +// start). A per-alloc failure is collected and returned as an aggregate +// error; the caller treats restart failures as non-fatal (the restore +// itself succeeded). +func restartAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) error { + var failed []string + for _, a := range rs { + startCmd := fmt.Sprintf("systemctl start %s", allocUnit(a.AllocID)) + if _, err := ex.Exec(ctx, a.Peer, startCmd); err != nil { + failed = append(failed, a.AllocID) + } + } + if len(failed) > 0 { + return fmt.Errorf("failed to restart %d alloc(s): %s", len(failed), strings.Join(failed, ", ")) + } + return nil +} + +// verifyRestoredState checks the restored tree for the master key +// (present + valid length/mode), namespace directories, and openable +// SQLite databases. It returns a slice of human-readable discrepancy +// strings (empty if everything checks out). +func verifyRestoredState(targetDir string, masterKey []byte) []string { + var issues []string + + mkPath := filepath.Join(targetDir, relFromRoot(paths.MasterKeyPath())) + info, err := os.Stat(mkPath) + if err != nil { + issues = append(issues, fmt.Sprintf("master key missing: %s", mkPath)) + } else { + if info.Mode().Perm() != secrets.MasterKeyMode { + issues = append(issues, fmt.Sprintf("master key mode %04o (want %04o)", info.Mode().Perm(), secrets.MasterKeyMode)) + } + key, rerr := os.ReadFile(mkPath) + if rerr != nil { + issues = append(issues, fmt.Sprintf("master key unreadable: %v", rerr)) + } else if len(key) != secrets.MasterKeyLen { + issues = append(issues, fmt.Sprintf("master key length %d (want %d)", len(key), secrets.MasterKeyLen)) + } else if masterKey != nil && string(key) != string(masterKey) { + issues = append(issues, "master key differs from the key used to verify the signature") + } + } + + nsDirs := findNamespaceDirs(targetDir) + if len(nsDirs) == 0 { + issues = append(issues, "no namespace directories found") + } + for _, nsDir := range nsDirs { + dbPath := filepath.Join(nsDir, "db", "orca.db") + if _, err := os.Stat(dbPath); err == nil { + if err := dbOpenable(dbPath); err != nil { + issues = append(issues, fmt.Sprintf("db not openable %s: %v", dbPath, err)) + } + } + } + + clusterDB := filepath.Join(targetDir, "orca.db") + if _, err := os.Stat(clusterDB); err == nil { + if err := dbOpenable(clusterDB); err != nil { + issues = append(issues, fmt.Sprintf("cluster db not openable %s: %v", clusterDB, err)) + } + } + return issues +} + +// relFromRoot strips the ORCA_HOME prefix off an absolute path so it +// can be rejoined to an arbitrary target dir (used by --dry-run and +// non-default --target restores). If the path does not start with the +// current ORCA_HOME root, the path's base is returned. +func relFromRoot(p string) string { + root := filepath.Clean(paths.Root()) + clean := filepath.Clean(p) + if rel, err := filepath.Rel(root, clean); err == nil && !strings.HasPrefix(rel, "..") { + return rel + } + return filepath.Base(clean) +} + +// findNamespaceDirs returns the top-level directories under targetDir +// that look like namespace dirs (excluding the "cluster" dir and dot- +// files). A namespace dir is any immediate child of ORCA_HOME that is +// a directory and not "cluster". +func findNamespaceDirs(targetDir string) []string { + entries, err := os.ReadDir(targetDir) + if err != nil { + return nil + } + var out []string + for _, e := range entries { + if !e.IsDir() { + continue + } + name := e.Name() + if strings.HasPrefix(name, ".") { + continue + } + if name == "cluster" { + continue + } + out = append(out, filepath.Join(targetDir, name)) + } + return out +} + +// dbOpenable reports whether the SQLite file at path can be opened +// read-only. It uses the same driver as the rest of the codebase +// (modernc.org/sqlite via store.Open, but with a read-only pragma). +func dbOpenable(path string) error { + dsn := "file:" + path + "?mode=ro&_pragma=journal_mode(WAL)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return err + } + defer db.Close() + if err := db.Ping(); err != nil { + return err + } + return nil +} + +// listExtractedFiles walks a staging dir and returns the relative +// paths of all regular files (capped at 1000 for reporting). +func listExtractedFiles(root string) []string { + var out []string + _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if !info.Mode().IsRegular() { + return nil + } + rel, rerr := filepath.Rel(root, path) + if rerr != nil { + return nil + } + out = append(out, filepath.ToSlash(rel)) + return nil + }) + if len(out) > 1000 { + out = out[:1000] + } + return out +} + +// buildVerificationReport converts the discrepancy slice into a +// JSON-friendly []string (nil-safe for empty input). +func buildVerificationReport(issues []string) []string { + if len(issues) == 0 { + return []string{} + } + return issues +} + +// auditRestore records a restore event in the audit log (same pattern +// as auditDrain in drain.go). It opens the cluster DB at +// certpaths.DBPath(); a failure to open the DB is logged and silently +// dropped so a restore is never blocked by the audit log itself. +func auditRestore(ctx context.Context, result string, err error, meta map[string]any) { + db, dbErr := store.Open(certpaths.DBPath()) + if dbErr != nil { + newLogger().Warn("restore: audit log unavailable", "error", dbErr) + return + } + defer db.Close() + engine.NewAudit(store.NewAuditRepo(db), newLogger()).Record(ctx, "cli", "restore", certpaths.Dir(), result, err, meta) +} diff --git a/internal/cli/recovery_test.go b/internal/cli/recovery_test.go new file mode 100644 index 0000000..7338ceb --- /dev/null +++ b/internal/cli/recovery_test.go @@ -0,0 +1,306 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.cloudinit.dev/coreci/orca/internal/backup" + "git.cloudinit.dev/coreci/orca/internal/certpaths" + "git.cloudinit.dev/coreci/orca/internal/model" + "git.cloudinit.dev/coreci/orca/internal/paths" + "git.cloudinit.dev/coreci/orca/internal/secrets" + "git.cloudinit.dev/coreci/orca/internal/store" +) + +// recoveryTestEnv sets up an ORCA_HOME with a master key, an empty +// cluster DB (so nodeRegistry works), and returns the home dir + a +// cleanup. It does NOT install a mock drain execer (callers that need +// one install scriptedDrainExec themselves). +func recoveryTestEnv(t *testing.T) (string, func()) { + t.Helper() + dir, cleanup := initTestEnv(t) + if err := os.MkdirAll(paths.ClusterDir(), 0o755); err != nil { + t.Fatalf("mkdir cluster dir: %v", err) + } + mk, err := secrets.GenerateMasterKey() + if err != nil { + t.Fatalf("GenerateMasterKey: %v", err) + } + if err := secrets.SaveMasterKey(paths.MasterKeyPath(), mk); err != nil { + t.Fatalf("SaveMasterKey: %v", err) + } + // Ensure the cluster DB exists (migrations run on Open) so + // auditRestore and nodeRegistry work. + db, err := store.Open(certpaths.DBPath()) + if err != nil { + t.Fatalf("open cluster db: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close cluster db: %v", err) + } + return dir, cleanup +} + +// makeBackup creates a signed tarball of homeDir containing the given +// relative file payloads (map[relPath]content) and returns the tarball +// path. It reuses the real `orca backup` command path for realism. +func makeBackup(t *testing.T, homeDir string, files map[string]string) string { + t.Helper() + for rel, body := range files { + p := filepath.Join(homeDir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", rel, err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + outDir := t.TempDir() + out := filepath.Join(outDir, "orca-backup.tar.gz") + mk, err := secrets.LoadMasterKey(paths.MasterKeyPath()) + if err != nil { + t.Fatalf("LoadMasterKey: %v", err) + } + if err := backup.Backup(backup.BackupOptions{ + SourceDir: homeDir, + OutputPath: out, + MasterKey: mk, + }); err != nil { + t.Fatalf("Backup: %v", err) + } + return out +} + +// restoreNodeForTest inserts a node with a fixed id+name so scanRunningAllocs +// can find it. Uses the test ORCA_HOME cluster DB. +func restoreNodeForTest(t *testing.T, name, addr string) *model.Node { + t.Helper() + db, err := store.Open(certpaths.DBPath()) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer db.Close() + n := &model.Node{ + ID: "node-" + name, + Name: name, + Address: addr, + State: model.NodeStateReady, + JoinedAt: time.Now().UTC(), + LastSeen: time.Now().UTC(), + Kind: string(model.NodeKindLinux), + } + if err := store.NewNodeRepo(db).Insert(context.Background(), n); err != nil { + t.Fatalf("insert node: %v", err) + } + return n +} + +// runRestoreArgs invokes the restore command with the given flags and +// returns (output, error). +func runRestoreArgs(t *testing.T, args ...string) (string, error) { + t.Helper() + var buf bytes.Buffer + resetRootFlags(t) + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + rootCmd.SetArgs(append([]string{"restore"}, args...)) + err := rootCmd.Execute() + return buf.String(), err +} + +func TestRestore_DryRun_DoesNotTouchORCAHome(t *testing.T) { + home, cleanup := recoveryTestEnv(t) + defer cleanup() + // Put a marker file in ORCA_HOME that must survive the dry-run. + marker := filepath.Join(home, "marker.txt") + if err := os.WriteFile(marker, []byte("original"), 0o644); err != nil { + t.Fatalf("write marker: %v", err) + } + // Build the backup from a SEPARATE source dir (not ORCA_HOME) so we + // can prove the dry-run never writes into ORCA_HOME. + srcDir := t.TempDir() + if err := os.WriteFile(filepath.Join(srcDir, "keep.txt"), []byte("payload"), 0o644); err != nil { + t.Fatalf("write keep.txt: %v", err) + } + mk, err := secrets.LoadMasterKey(paths.MasterKeyPath()) + if err != nil { + t.Fatalf("LoadMasterKey: %v", err) + } + out := filepath.Join(t.TempDir(), "orca-backup.tar.gz") + if err := backup.Backup(backup.BackupOptions{ + SourceDir: srcDir, + OutputPath: out, + MasterKey: mk, + }); err != nil { + t.Fatalf("Backup: %v", err) + } + + before, _ := os.ReadDir(home) + out2, err := runRestoreArgs(t, "--in", out, "--dry-run") + if err != nil { + t.Fatalf("restore --dry-run: %v", err) + } + if !strings.Contains(out2, "would restore") { + t.Errorf("dry-run output missing 'would restore': %s", out2) + } + // ORCA_HOME must be untouched: marker intact, no keep.txt written. + got, err := os.ReadFile(marker) + if err != nil { + t.Fatalf("marker missing after dry-run: %v", err) + } + if string(got) != "original" { + t.Errorf("marker changed by dry-run: %q", string(got)) + } + if _, err := os.Stat(filepath.Join(home, "keep.txt")); err == nil { + t.Errorf("dry-run wrote keep.txt into ORCA_HOME") + } + after, _ := os.ReadDir(home) + if len(before) != len(after) { + t.Errorf("ORCA_HOME entry count changed by dry-run: before=%d after=%d", len(before), len(after)) + } +} + +func TestRestore_RefusesRunningAllocsWithoutForce(t *testing.T) { + home, cleanup := recoveryTestEnv(t) + defer cleanup() + out := makeBackup(t, home, map[string]string{"keep.txt": "payload"}) + + restoreNodeForTest(t, "runnode", "runnode:8443") + mx := &scriptedDrainExec{} + mx.queueAlways("list-units", "orca-alloc-web-0.service loaded active running\n", 0) + drainExecOverride = mx + + _, err := runRestoreArgs(t, "--in", out, "--target", filepath.Join(t.TempDir(), "restored")) + if err == nil { + t.Fatal("restore should refuse when allocs are running") + } + if !errors.Is(err, ErrRunningAllocs) { + t.Errorf("expected ErrRunningAllocs, got: %v", err) + } + // Must NOT have attempted to stop or start anything. + if mx.countCalls("systemctl stop") != 0 { + t.Errorf("restore without --force issued stop commands: %+v", mx.calls) + } +} + +func TestRestore_Force_StopsAndRestartsAllocs(t *testing.T) { + home, cleanup := recoveryTestEnv(t) + defer cleanup() + out := makeBackup(t, home, map[string]string{"keep.txt": "payload"}) + + restoreNodeForTest(t, "forcenode", "forcenode:8443") + mx := &scriptedDrainExec{} + // list-units always returns the running alloc; stop/start succeed. + mx.queueAlways("list-units", "orca-alloc-web-0.service loaded active running\n", 0) + mx.queueAlways("systemctl stop", "", 0) + mx.queueAlways("systemctl start", "", 0) + drainExecOverride = mx + + target := filepath.Join(t.TempDir(), "restored") + out2, err := runRestoreArgs(t, "--in", out, "--target", target, "--force") + if err != nil { + t.Fatalf("restore --force: %v\n%s", err, out2) + } + if mx.countCalls("systemctl stop orca-alloc-web-0") == 0 { + t.Errorf("expected a stop command for web-0, calls: %+v", mx.calls) + } + if mx.countCalls("systemctl start orca-alloc-web-0") == 0 { + t.Errorf("expected a start command for web-0, calls: %+v", mx.calls) + } + got, err := os.ReadFile(filepath.Join(target, "keep.txt")) + if err != nil { + t.Fatalf("restored keep.txt missing: %v", err) + } + if string(got) != "payload" { + t.Errorf("restored keep.txt = %q, want %q", string(got), "payload") + } +} + +func TestRestore_PostVerify_MasterKeyMissing(t *testing.T) { + _, cleanup := recoveryTestEnv(t) + defer cleanup() + // Build a backup whose payload LACKS the master key. We back up a + // different source dir so the cluster/master.key is not included. + srcDir := t.TempDir() + if err := os.WriteFile(filepath.Join(srcDir, "only.txt"), []byte("x"), 0o644); err != nil { + t.Fatalf("write only.txt: %v", err) + } + mk, err := secrets.LoadMasterKey(paths.MasterKeyPath()) + if err != nil { + t.Fatalf("LoadMasterKey: %v", err) + } + out := filepath.Join(t.TempDir(), "nb.tar.gz") + if err := backup.Backup(backup.BackupOptions{ + SourceDir: srcDir, + OutputPath: out, + MasterKey: mk, + }); err != nil { + t.Fatalf("Backup: %v", err) + } + + // No mock drain execer → no nodes → no running allocs → restore + // proceeds. The target is a fresh dir; verify will report the + // missing master key. + target := filepath.Join(t.TempDir(), "restored") + out2, err := runRestoreArgs(t, "--in", out, "--target", target) + if err != nil { + t.Fatalf("restore returned error: %v\n%s", err, out2) + } + if !strings.Contains(out2, "master key missing") { + t.Errorf("expected 'master key missing' in output, got: %s", out2) + } +} + +func TestRestore_AuditLogEntry(t *testing.T) { + home, cleanup := recoveryTestEnv(t) + defer cleanup() + out := makeBackup(t, home, map[string]string{"keep.txt": "payload"}) + + // No nodes → no running allocs → restore succeeds and records. + target := filepath.Join(t.TempDir(), "restored") + if _, err := runRestoreArgs(t, "--in", out, "--target", target); err != nil { + t.Fatalf("restore: %v", err) + } + + db, err := store.Open(certpaths.DBPath()) + if err != nil { + t.Fatalf("open db: %v", err) + } + defer db.Close() + entries, err := store.NewAuditRepo(db).List(context.Background(), 50) + if err != nil { + t.Fatalf("list audit: %v", err) + } + var found bool + for _, e := range entries { + if e.Action == "restore" { + found = true + break + } + } + if !found { + t.Errorf("no 'restore' entry in audit log; entries: %+v", entries) + } +} + +func TestRestore_SignatureFailure_Refuses(t *testing.T) { + _, cleanup := recoveryTestEnv(t) + defer cleanup() + out := filepath.Join(t.TempDir(), "bad.tar.gz") + if err := os.WriteFile(out, []byte("not a tarball"), 0o644); err != nil { + t.Fatalf("write fake tarball: %v", err) + } + if err := os.WriteFile(out+".sig", []byte("deadbeef"), 0o644); err != nil { + t.Fatalf("write fake sig: %v", err) + } + _, err := runRestoreArgs(t, "--in", out, "--target", filepath.Join(t.TempDir(), "r")) + if err == nil { + t.Fatal("restore with bad signature should fail") + } +}