orca/internal/cli/recovery.go

// Package cli: recovery.go implements the full-recovery semantics for
// `orca restore` (P07, v0.11 milestone).
//
// On top of P04's signed-restore, restore now:
//
//  1. Verifies the signature (P04).
//  2. --dry-run: extracts to a temp staging dir, verifies, reports,
//     cleans up, and exits — never touching the real ORCA_HOME.
//  3. Without --force: scans every peer for running orca-alloc-*
//     systemd units; if any are running, refuses the restore (data
//     loss protection) and reports which allocs are running.
//  4. With --force: stops the running allocs (SSH systemctl stop),
//     extracts the tarball, then restarts the allocs from the restored
//     state (SSH systemctl start).
//  5. Post-restore verification: master key present + valid, namespace
//     dirs exist, SQLite DBs openable; reports discrepancies.
//  6. Records the restore in the audit log (internal/store/audit_repo).
package cli

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"

	"git.cloudinit.dev/coreci/orca/internal/backup"
	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/engine"
	"git.cloudinit.dev/coreci/orca/internal/paths"
	"git.cloudinit.dev/coreci/orca/internal/secrets"
	"git.cloudinit.dev/coreci/orca/internal/store"
)

// RestoreOptions configures the P07 full-recovery restore. It mirrors
// backup.RestoreOptions but adds DryRun and is consumed by runRestore
// (the CLI layer), which performs the alloc reconciliation and audit
// logging that the lower-level backup.Restore does not.
type RestoreOptions struct {
	InputPath string
	TargetDir string
	MasterKey []byte
	Force     bool
	DryRun    bool
}

// ErrRunningAllocs is returned when restore refuses to clobber a live
// cluster that has running allocations and --force was not given.
var ErrRunningAllocs = errors.New("restore: running allocations present (use --force to stop and restart)")

// runRestore is the entry point invoked by restoreCmd.RunE. It performs
// the full P07 recovery flow. The cmd is used only for output; ctx comes
// from cmd.Context().
func runRestore(cmd *cobra.Command, opts RestoreOptions) error {
	ctx := cmd.Context()
	if opts.InputPath == "" {
		return fmt.Errorf("restore: InputPath is empty")
	}
	if opts.TargetDir == "" {
		return fmt.Errorf("restore: TargetDir is empty")
	}
	if len(opts.MasterKey) == 0 {
		return fmt.Errorf("restore: MasterKey is empty")
	}
	log := newLogger()

	sigPath := opts.InputPath + ".sig"
	if err := backup.VerifySignature(opts.InputPath, sigPath, opts.MasterKey); err != nil {
		auditRestore(ctx, "failure", err, map[string]any{"path": opts.InputPath, "target": opts.TargetDir})
		return fmt.Errorf("restore: verify signature: %w", err)
	}

	if opts.DryRun {
		return runRestoreDryRun(cmd, opts)
	}

	ex, exErr := drainExecFromCtx(ctx)
	if exErr != nil {
		return fmt.Errorf("ssh transport: %w", exErr)
	}

	running, err := scanRunningAllocs(ctx, ex)
	if err != nil {
		log.Warn("restore: scan running allocs failed", "error", err)
	}
	if len(running) > 0 && !opts.Force {
		allocList := formatRunningAllocs(running)
		err := fmt.Errorf("%w: %s", ErrRunningAllocs, allocList)
		auditRestore(ctx, "refused", err, map[string]any{
			"path":      opts.InputPath,
			"target":     opts.TargetDir,
			"running":    running,
		})
		return err
	}

	if opts.Force && len(running) > 0 {
		stopped, stopErr := stopRunningAllocs(ctx, ex, running)
		if stopErr != nil {
			auditRestore(ctx, "partial", stopErr, map[string]any{
				"path":    opts.InputPath,
				"target":  opts.TargetDir,
				"stopped": stopped,
				"running": running,
			})
			return fmt.Errorf("restore: stop running allocs: %w", stopErr)
		}
		log.Info("restore: stopped running allocs", "count", len(stopped))
	}

	ropts := backup.RestoreOptions{
		InputPath: opts.InputPath,
		TargetDir: opts.TargetDir,
		MasterKey: opts.MasterKey,
		Force:     true,
	}
	if err := backup.Restore(ropts); err != nil {
		auditRestore(ctx, "failure", err, map[string]any{
			"path":   opts.InputPath,
			"target": opts.TargetDir,
		})
		return err
	}

	if opts.Force && len(running) > 0 {
		if rerr := restartAllocs(ctx, ex, running); rerr != nil {
			log.Warn("restore: failed to restart some allocs", "error", rerr)
		}
	}

	verifications := verifyRestoredState(opts.TargetDir, opts.MasterKey)
	report := buildVerificationReport(verifications)
	result := "success"
	var verifyErr error
	if len(verifications) > 0 {
		result = "partial"
		verifyErr = fmt.Errorf("post-restore verification: %d issue(s)", len(verifications))
	}
	auditRestore(ctx, result, verifyErr, map[string]any{
		"path":         opts.InputPath,
		"target":       opts.TargetDir,
		"force":        opts.Force,
		"running":      running,
		"verification": report,
	})

	if jsonOutput {
		return printJSON(map[string]any{
			"path":         opts.InputPath,
			"target":       opts.TargetDir,
			"force":        opts.Force,
			"running":      running,
			"verification": report,
		})
	}
	fmt.Fprintf(cmd.OutOrStdout(), "✓ Restored to: %s\n", opts.TargetDir)
	if opts.Force && len(running) > 0 {
		fmt.Fprintf(cmd.OutOrStdout(), "  stopped+restarted %d running alloc(s)\n", len(running))
	}
	for _, v := range verifications {
		fmt.Fprintf(cmd.OutOrStdout(), "  ⚠ verify: %s\n", v)
	}
	return nil
}

// runRestoreDryRun extracts the tarball to a temp dir, runs
// post-restore verification against that temp dir, reports what WOULD
// be restored, and cleans up. The real ORCA_HOME is never touched.
func runRestoreDryRun(cmd *cobra.Command, opts RestoreOptions) error {
	tmp, err := os.MkdirTemp("", "orca-restore-dryrun-*")
	if err != nil {
		return fmt.Errorf("restore: dry-run temp dir: %w", err)
	}
	defer os.RemoveAll(tmp)
	log := newLogger()
	log.Info("restore: dry-run staging dir", "dir", tmp)

	ropts := backup.RestoreOptions{
		InputPath: opts.InputPath,
		TargetDir: tmp,
		MasterKey: opts.MasterKey,
		Force:     true,
	}
	if err := backup.Restore(ropts); err != nil {
		return fmt.Errorf("restore: dry-run extract: %w", err)
	}

	entries := listExtractedFiles(tmp)
	verifications := verifyRestoredState(tmp, opts.MasterKey)
	report := buildVerificationReport(verifications)
	auditRestore(cmd.Context(), "dry-run", nil, map[string]any{
		"path":         opts.InputPath,
		"target":       opts.TargetDir,
		"staging":      tmp,
		"files":        len(entries),
		"verification": report,
	})

	if jsonOutput {
		return printJSON(map[string]any{
			"dry_run":      true,
			"path":         opts.InputPath,
			"target":       opts.TargetDir,
			"files":        len(entries),
			"verification": report,
		})
	}
	out := cmd.OutOrStdout()
	fmt.Fprintf(out, "✓ Dry-run: would restore %d file(s) to %s\n", len(entries), opts.TargetDir)
	if len(entries) > 0 && len(entries) <= 20 {
		for _, e := range entries {
			fmt.Fprintf(out, "  %s\n", e)
		}
	}
	for _, v := range verifications {
		fmt.Fprintf(out, "  ⚠ verify: %s\n", v)
	}
	return nil
}

// runningAlloc is a running allocation discovered on a peer node.
type runningAlloc struct {
	Node    string `json:"node"`
	Peer    string `json:"peer"`
	AllocID string `json:"alloc_id"`
}

// scanRunningAllocs lists every peer node in the registry and queries
// each via SSH for currently-running orca-alloc-*.service units. A
// registry failure or empty registry yields an empty (not error)
// slice; per-node failures are logged and skipped so a single
// unreachable peer does not block the restore.
func scanRunningAllocs(ctx context.Context, ex drainExecer) ([]runningAlloc, error) {
	reg, closer, err := nodeRegistry()
	if err != nil {
		return nil, nil
	}
	defer closer()
	nodes, err := reg.List(ctx)
	if err != nil {
		return nil, nil
	}
	var out []runningAlloc
	log := newLogger()
	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		ids, err := listRunningAllocs(ctx, ex, peer)
		if err != nil {
			log.Warn("restore: cannot list allocs on node",
				slog.String("node", n.Name), slog.String("peer", peer), "error", err)
			continue
		}
		for _, id := range ids {
			out = append(out, runningAlloc{Node: n.Name, Peer: peer, AllocID: id})
		}
	}
	return out, nil
}

// formatRunningAllocs renders a list of running allocs for an error
// message (one per line: "node/peer: alloc-id").
func formatRunningAllocs(rs []runningAlloc) string {
	var b strings.Builder
	for i, a := range rs {
		if i > 0 {
			b.WriteString("; ")
		}
		fmt.Fprintf(&b, "%s/%s: %s", a.Node, a.Peer, a.AllocID)
	}
	return b.String()
}

// stopRunningAllocs stops every allocation in rs via SSH (systemctl
// stop). It returns the ids that were successfully stopped and an
// aggregate error if any failed.
func stopRunningAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) ([]string, error) {
	var stopped []string
	var failed []string
	for _, a := range rs {
		if err := stopAlloc(ctx, ex, a.Peer, a.AllocID); err != nil {
			failed = append(failed, a.AllocID)
			continue
		}
		stopped = append(stopped, a.AllocID)
	}
	if len(failed) > 0 {
		return stopped, fmt.Errorf("failed to stop %d alloc(s): %s", len(failed), strings.Join(failed, ", "))
	}
	return stopped, nil
}

// restartAllocs restarts every allocation in rs via SSH (systemctl
// start). A per-alloc failure is collected and returned as an aggregate
// error; the caller treats restart failures as non-fatal (the restore
// itself succeeded).
func restartAllocs(ctx context.Context, ex drainExecer, rs []runningAlloc) error {
	var failed []string
	for _, a := range rs {
		startCmd := fmt.Sprintf("systemctl start %s", allocUnit(a.AllocID))
		if _, err := ex.Exec(ctx, a.Peer, startCmd); err != nil {
			failed = append(failed, a.AllocID)
		}
	}
	if len(failed) > 0 {
		return fmt.Errorf("failed to restart %d alloc(s): %s", len(failed), strings.Join(failed, ", "))
	}
	return nil
}

// verifyRestoredState checks the restored tree for the master key
// (present + valid length/mode), namespace directories, and openable
// SQLite databases. It returns a slice of human-readable discrepancy
// strings (empty if everything checks out).
func verifyRestoredState(targetDir string, masterKey []byte) []string {
	var issues []string

	mkPath := filepath.Join(targetDir, relFromRoot(paths.MasterKeyPath()))
	info, err := os.Stat(mkPath)
	if err != nil {
		issues = append(issues, fmt.Sprintf("master key missing: %s", mkPath))
	} else {
		if info.Mode().Perm() != secrets.MasterKeyMode {
			issues = append(issues, fmt.Sprintf("master key mode %04o (want %04o)", info.Mode().Perm(), secrets.MasterKeyMode))
		}
		key, rerr := os.ReadFile(mkPath)
		if rerr != nil {
			issues = append(issues, fmt.Sprintf("master key unreadable: %v", rerr))
		} else if len(key) != secrets.MasterKeyLen {
			issues = append(issues, fmt.Sprintf("master key length %d (want %d)", len(key), secrets.MasterKeyLen))
		} else if masterKey != nil && string(key) != string(masterKey) {
			issues = append(issues, "master key differs from the key used to verify the signature")
		}
	}

	nsDirs := findNamespaceDirs(targetDir)
	if len(nsDirs) == 0 {
		issues = append(issues, "no namespace directories found")
	}
	for _, nsDir := range nsDirs {
		dbPath := filepath.Join(nsDir, "db", "orca.db")
		if _, err := os.Stat(dbPath); err == nil {
			if err := dbOpenable(dbPath); err != nil {
				issues = append(issues, fmt.Sprintf("db not openable %s: %v", dbPath, err))
			}
		}
	}

	clusterDB := filepath.Join(targetDir, "orca.db")
	if _, err := os.Stat(clusterDB); err == nil {
		if err := dbOpenable(clusterDB); err != nil {
			issues = append(issues, fmt.Sprintf("cluster db not openable %s: %v", clusterDB, err))
		}
	}
	return issues
}

// relFromRoot strips the ORCA_HOME prefix off an absolute path so it
// can be rejoined to an arbitrary target dir (used by --dry-run and
// non-default --target restores). If the path does not start with the
// current ORCA_HOME root, the path's base is returned.
func relFromRoot(p string) string {
	root := filepath.Clean(paths.Root())
	clean := filepath.Clean(p)
	if rel, err := filepath.Rel(root, clean); err == nil && !strings.HasPrefix(rel, "..") {
		return rel
	}
	return filepath.Base(clean)
}

// findNamespaceDirs returns the top-level directories under targetDir
// that look like namespace dirs (excluding the "cluster" dir and dot-
// files). A namespace dir is any immediate child of ORCA_HOME that is
// a directory and not "cluster".
func findNamespaceDirs(targetDir string) []string {
	entries, err := os.ReadDir(targetDir)
	if err != nil {
		return nil
	}
	var out []string
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		name := e.Name()
		if strings.HasPrefix(name, ".") {
			continue
		}
		if name == "cluster" {
			continue
		}
		out = append(out, filepath.Join(targetDir, name))
	}
	return out
}

// dbOpenable reports whether the SQLite file at path can be opened
// read-only. It uses the same driver as the rest of the codebase
// (modernc.org/sqlite via store.Open, but with a read-only pragma).
func dbOpenable(path string) error {
	dsn := "file:" + path + "?mode=ro&_pragma=journal_mode(WAL)"
	db, err := sql.Open("sqlite", dsn)
	if err != nil {
		return err
	}
	defer db.Close()
	if err := db.Ping(); err != nil {
		return err
	}
	return nil
}

// listExtractedFiles walks a staging dir and returns the relative
// paths of all regular files (capped at 1000 for reporting).
func listExtractedFiles(root string) []string {
	var out []string
	_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if !info.Mode().IsRegular() {
			return nil
		}
		rel, rerr := filepath.Rel(root, path)
		if rerr != nil {
			return nil
		}
		out = append(out, filepath.ToSlash(rel))
		return nil
	})
	if len(out) > 1000 {
		out = out[:1000]
	}
	return out
}

// buildVerificationReport converts the discrepancy slice into a
// JSON-friendly []string (nil-safe for empty input).
func buildVerificationReport(issues []string) []string {
	if len(issues) == 0 {
		return []string{}
	}
	return issues
}

// auditRestore records a restore event in the audit log (same pattern
// as auditDrain in drain.go). It opens the cluster DB at
// certpaths.DBPath(); a failure to open the DB is logged and silently
// dropped so a restore is never blocked by the audit log itself.
func auditRestore(ctx context.Context, result string, err error, meta map[string]any) {
	db, dbErr := store.Open(certpaths.DBPath())
	if dbErr != nil {
		newLogger().Warn("restore: audit log unavailable", "error", dbErr)
		return
	}
	defer db.Close()
	engine.NewAudit(store.NewAuditRepo(db), newLogger()).Record(ctx, "cli", "restore", certpaths.Dir(), result, err, meta)
}