Jon Chery
f530c9a3f7
Extend restore with --dry-run (extract to temp, report, no write), running-alloc protection (refuse without --force; stop+restart with --force), post-restore verification (master key, namespaces, DBs), audit log entry. ---ci--- project: orca phase: 07 milestone: v0.11 status: execute ---/ci---
307 lines
9.6 KiB
Go
307 lines
9.6 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.cloudinit.dev/coreci/orca/internal/backup"
|
|
"git.cloudinit.dev/coreci/orca/internal/certpaths"
|
|
"git.cloudinit.dev/coreci/orca/internal/model"
|
|
"git.cloudinit.dev/coreci/orca/internal/paths"
|
|
"git.cloudinit.dev/coreci/orca/internal/secrets"
|
|
"git.cloudinit.dev/coreci/orca/internal/store"
|
|
)
|
|
|
|
// recoveryTestEnv sets up an ORCA_HOME with a master key, an empty
|
|
// cluster DB (so nodeRegistry works), and returns the home dir + a
|
|
// cleanup. It does NOT install a mock drain execer (callers that need
|
|
// one install scriptedDrainExec themselves).
|
|
func recoveryTestEnv(t *testing.T) (string, func()) {
|
|
t.Helper()
|
|
dir, cleanup := initTestEnv(t)
|
|
if err := os.MkdirAll(paths.ClusterDir(), 0o755); err != nil {
|
|
t.Fatalf("mkdir cluster dir: %v", err)
|
|
}
|
|
mk, err := secrets.GenerateMasterKey()
|
|
if err != nil {
|
|
t.Fatalf("GenerateMasterKey: %v", err)
|
|
}
|
|
if err := secrets.SaveMasterKey(paths.MasterKeyPath(), mk); err != nil {
|
|
t.Fatalf("SaveMasterKey: %v", err)
|
|
}
|
|
// Ensure the cluster DB exists (migrations run on Open) so
|
|
// auditRestore and nodeRegistry work.
|
|
db, err := store.Open(certpaths.DBPath())
|
|
if err != nil {
|
|
t.Fatalf("open cluster db: %v", err)
|
|
}
|
|
if err := db.Close(); err != nil {
|
|
t.Fatalf("close cluster db: %v", err)
|
|
}
|
|
return dir, cleanup
|
|
}
|
|
|
|
// makeBackup creates a signed tarball of homeDir containing the given
|
|
// relative file payloads (map[relPath]content) and returns the tarball
|
|
// path. It reuses the real `orca backup` command path for realism.
|
|
func makeBackup(t *testing.T, homeDir string, files map[string]string) string {
|
|
t.Helper()
|
|
for rel, body := range files {
|
|
p := filepath.Join(homeDir, rel)
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
t.Fatalf("mkdir %s: %v", rel, err)
|
|
}
|
|
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
|
t.Fatalf("write %s: %v", rel, err)
|
|
}
|
|
}
|
|
outDir := t.TempDir()
|
|
out := filepath.Join(outDir, "orca-backup.tar.gz")
|
|
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
|
if err != nil {
|
|
t.Fatalf("LoadMasterKey: %v", err)
|
|
}
|
|
if err := backup.Backup(backup.BackupOptions{
|
|
SourceDir: homeDir,
|
|
OutputPath: out,
|
|
MasterKey: mk,
|
|
}); err != nil {
|
|
t.Fatalf("Backup: %v", err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// restoreNodeForTest inserts a node with a fixed id+name so scanRunningAllocs
|
|
// can find it. Uses the test ORCA_HOME cluster DB.
|
|
func restoreNodeForTest(t *testing.T, name, addr string) *model.Node {
|
|
t.Helper()
|
|
db, err := store.Open(certpaths.DBPath())
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
defer db.Close()
|
|
n := &model.Node{
|
|
ID: "node-" + name,
|
|
Name: name,
|
|
Address: addr,
|
|
State: model.NodeStateReady,
|
|
JoinedAt: time.Now().UTC(),
|
|
LastSeen: time.Now().UTC(),
|
|
Kind: string(model.NodeKindLinux),
|
|
}
|
|
if err := store.NewNodeRepo(db).Insert(context.Background(), n); err != nil {
|
|
t.Fatalf("insert node: %v", err)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// runRestoreArgs invokes the restore command with the given flags and
|
|
// returns (output, error).
|
|
func runRestoreArgs(t *testing.T, args ...string) (string, error) {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
resetRootFlags(t)
|
|
rootCmd.SetOut(&buf)
|
|
rootCmd.SetErr(&buf)
|
|
rootCmd.SetArgs(append([]string{"restore"}, args...))
|
|
err := rootCmd.Execute()
|
|
return buf.String(), err
|
|
}
|
|
|
|
func TestRestore_DryRun_DoesNotTouchORCAHome(t *testing.T) {
|
|
home, cleanup := recoveryTestEnv(t)
|
|
defer cleanup()
|
|
// Put a marker file in ORCA_HOME that must survive the dry-run.
|
|
marker := filepath.Join(home, "marker.txt")
|
|
if err := os.WriteFile(marker, []byte("original"), 0o644); err != nil {
|
|
t.Fatalf("write marker: %v", err)
|
|
}
|
|
// Build the backup from a SEPARATE source dir (not ORCA_HOME) so we
|
|
// can prove the dry-run never writes into ORCA_HOME.
|
|
srcDir := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(srcDir, "keep.txt"), []byte("payload"), 0o644); err != nil {
|
|
t.Fatalf("write keep.txt: %v", err)
|
|
}
|
|
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
|
if err != nil {
|
|
t.Fatalf("LoadMasterKey: %v", err)
|
|
}
|
|
out := filepath.Join(t.TempDir(), "orca-backup.tar.gz")
|
|
if err := backup.Backup(backup.BackupOptions{
|
|
SourceDir: srcDir,
|
|
OutputPath: out,
|
|
MasterKey: mk,
|
|
}); err != nil {
|
|
t.Fatalf("Backup: %v", err)
|
|
}
|
|
|
|
before, _ := os.ReadDir(home)
|
|
out2, err := runRestoreArgs(t, "--in", out, "--dry-run")
|
|
if err != nil {
|
|
t.Fatalf("restore --dry-run: %v", err)
|
|
}
|
|
if !strings.Contains(out2, "would restore") {
|
|
t.Errorf("dry-run output missing 'would restore': %s", out2)
|
|
}
|
|
// ORCA_HOME must be untouched: marker intact, no keep.txt written.
|
|
got, err := os.ReadFile(marker)
|
|
if err != nil {
|
|
t.Fatalf("marker missing after dry-run: %v", err)
|
|
}
|
|
if string(got) != "original" {
|
|
t.Errorf("marker changed by dry-run: %q", string(got))
|
|
}
|
|
if _, err := os.Stat(filepath.Join(home, "keep.txt")); err == nil {
|
|
t.Errorf("dry-run wrote keep.txt into ORCA_HOME")
|
|
}
|
|
after, _ := os.ReadDir(home)
|
|
if len(before) != len(after) {
|
|
t.Errorf("ORCA_HOME entry count changed by dry-run: before=%d after=%d", len(before), len(after))
|
|
}
|
|
}
|
|
|
|
func TestRestore_RefusesRunningAllocsWithoutForce(t *testing.T) {
|
|
home, cleanup := recoveryTestEnv(t)
|
|
defer cleanup()
|
|
out := makeBackup(t, home, map[string]string{"keep.txt": "payload"})
|
|
|
|
restoreNodeForTest(t, "runnode", "runnode:8443")
|
|
mx := &scriptedDrainExec{}
|
|
mx.queueAlways("list-units", "orca-alloc-web-0.service loaded active running\n", 0)
|
|
drainExecOverride = mx
|
|
|
|
_, err := runRestoreArgs(t, "--in", out, "--target", filepath.Join(t.TempDir(), "restored"))
|
|
if err == nil {
|
|
t.Fatal("restore should refuse when allocs are running")
|
|
}
|
|
if !errors.Is(err, ErrRunningAllocs) {
|
|
t.Errorf("expected ErrRunningAllocs, got: %v", err)
|
|
}
|
|
// Must NOT have attempted to stop or start anything.
|
|
if mx.countCalls("systemctl stop") != 0 {
|
|
t.Errorf("restore without --force issued stop commands: %+v", mx.calls)
|
|
}
|
|
}
|
|
|
|
func TestRestore_Force_StopsAndRestartsAllocs(t *testing.T) {
|
|
home, cleanup := recoveryTestEnv(t)
|
|
defer cleanup()
|
|
out := makeBackup(t, home, map[string]string{"keep.txt": "payload"})
|
|
|
|
restoreNodeForTest(t, "forcenode", "forcenode:8443")
|
|
mx := &scriptedDrainExec{}
|
|
// list-units always returns the running alloc; stop/start succeed.
|
|
mx.queueAlways("list-units", "orca-alloc-web-0.service loaded active running\n", 0)
|
|
mx.queueAlways("systemctl stop", "", 0)
|
|
mx.queueAlways("systemctl start", "", 0)
|
|
drainExecOverride = mx
|
|
|
|
target := filepath.Join(t.TempDir(), "restored")
|
|
out2, err := runRestoreArgs(t, "--in", out, "--target", target, "--force")
|
|
if err != nil {
|
|
t.Fatalf("restore --force: %v\n%s", err, out2)
|
|
}
|
|
if mx.countCalls("systemctl stop orca-alloc-web-0") == 0 {
|
|
t.Errorf("expected a stop command for web-0, calls: %+v", mx.calls)
|
|
}
|
|
if mx.countCalls("systemctl start orca-alloc-web-0") == 0 {
|
|
t.Errorf("expected a start command for web-0, calls: %+v", mx.calls)
|
|
}
|
|
got, err := os.ReadFile(filepath.Join(target, "keep.txt"))
|
|
if err != nil {
|
|
t.Fatalf("restored keep.txt missing: %v", err)
|
|
}
|
|
if string(got) != "payload" {
|
|
t.Errorf("restored keep.txt = %q, want %q", string(got), "payload")
|
|
}
|
|
}
|
|
|
|
func TestRestore_PostVerify_MasterKeyMissing(t *testing.T) {
|
|
_, cleanup := recoveryTestEnv(t)
|
|
defer cleanup()
|
|
// Build a backup whose payload LACKS the master key. We back up a
|
|
// different source dir so the cluster/master.key is not included.
|
|
srcDir := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(srcDir, "only.txt"), []byte("x"), 0o644); err != nil {
|
|
t.Fatalf("write only.txt: %v", err)
|
|
}
|
|
mk, err := secrets.LoadMasterKey(paths.MasterKeyPath())
|
|
if err != nil {
|
|
t.Fatalf("LoadMasterKey: %v", err)
|
|
}
|
|
out := filepath.Join(t.TempDir(), "nb.tar.gz")
|
|
if err := backup.Backup(backup.BackupOptions{
|
|
SourceDir: srcDir,
|
|
OutputPath: out,
|
|
MasterKey: mk,
|
|
}); err != nil {
|
|
t.Fatalf("Backup: %v", err)
|
|
}
|
|
|
|
// No mock drain execer → no nodes → no running allocs → restore
|
|
// proceeds. The target is a fresh dir; verify will report the
|
|
// missing master key.
|
|
target := filepath.Join(t.TempDir(), "restored")
|
|
out2, err := runRestoreArgs(t, "--in", out, "--target", target)
|
|
if err != nil {
|
|
t.Fatalf("restore returned error: %v\n%s", err, out2)
|
|
}
|
|
if !strings.Contains(out2, "master key missing") {
|
|
t.Errorf("expected 'master key missing' in output, got: %s", out2)
|
|
}
|
|
}
|
|
|
|
func TestRestore_AuditLogEntry(t *testing.T) {
|
|
home, cleanup := recoveryTestEnv(t)
|
|
defer cleanup()
|
|
out := makeBackup(t, home, map[string]string{"keep.txt": "payload"})
|
|
|
|
// No nodes → no running allocs → restore succeeds and records.
|
|
target := filepath.Join(t.TempDir(), "restored")
|
|
if _, err := runRestoreArgs(t, "--in", out, "--target", target); err != nil {
|
|
t.Fatalf("restore: %v", err)
|
|
}
|
|
|
|
db, err := store.Open(certpaths.DBPath())
|
|
if err != nil {
|
|
t.Fatalf("open db: %v", err)
|
|
}
|
|
defer db.Close()
|
|
entries, err := store.NewAuditRepo(db).List(context.Background(), 50)
|
|
if err != nil {
|
|
t.Fatalf("list audit: %v", err)
|
|
}
|
|
var found bool
|
|
for _, e := range entries {
|
|
if e.Action == "restore" {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
t.Errorf("no 'restore' entry in audit log; entries: %+v", entries)
|
|
}
|
|
}
|
|
|
|
func TestRestore_SignatureFailure_Refuses(t *testing.T) {
|
|
_, cleanup := recoveryTestEnv(t)
|
|
defer cleanup()
|
|
out := filepath.Join(t.TempDir(), "bad.tar.gz")
|
|
if err := os.WriteFile(out, []byte("not a tarball"), 0o644); err != nil {
|
|
t.Fatalf("write fake tarball: %v", err)
|
|
}
|
|
if err := os.WriteFile(out+".sig", []byte("deadbeef"), 0o644); err != nil {
|
|
t.Fatalf("write fake sig: %v", err)
|
|
}
|
|
_, err := runRestoreArgs(t, "--in", out, "--target", filepath.Join(t.TempDir(), "r"))
|
|
if err == nil {
|
|
t.Fatal("restore with bad signature should fail")
|
|
}
|
|
}
|