orca/internal/cli/upgrade.go

// Package cli: upgrade.go implements the `orca upgrade --to <version>`
// subcommand (REQ-115, gates C-25 and C-27). It is a thin wrapper
// around scripts/install.sh + orca restore that also handles the
// R-017 binding cutover (Traefik :443 → 127.0.0.1:8443 + nftables) for
// existing v0.9/v0.10 clusters, creates the `orca` system user on
// existing peers (C-27, needed before P10b drift detection works),
// and optionally imports the v0.8 internal CA into step-ca.
package cli

import (
	"context"
	"fmt"
	"log/slog"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/migration"
	"git.cloudinit.dev/coreci/orca/internal/paths"
	"git.cloudinit.dev/coreci/orca/internal/security"
)

var (
	upgradeTo       string
	upgradeImportCA bool
	upgradeForce    bool
	upgradeDryRun   bool
)

// commandRunner is the seam for running shell commands (install.sh,
// nft, useradd). Tests inject a mock; production uses execRunner.
type commandRunner interface {
	Run(ctx context.Context, name string, args ...string) ([]byte, error)
}

// execRunner runs commands via os/exec.
type execRunner struct{}

func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	return cmd.CombinedOutput()
}

// upgradeRunnerOverride is the package-level test seam for the
// command runner.
var upgradeRunnerOverride commandRunner

// httpClientOverride is the package-level test seam for the cutover
// verification HTTP check (C-25). Tests inject a mock.
var httpClientOverride func(url string) (int, error)

// upgradeTransport is the SSH surface the upgrade command needs for
// C-27 orca user creation on peers. Mirrors peerSetupTransport.
type upgradeTransport interface {
	Exec(ctx context.Context, peer string, cmd string) ([]byte, error)
}

// upgradeTransportOverride is the package-level test seam for the
// SSH transport.
var upgradeTransportOverride upgradeTransport

// peersListerOverride is the package-level test seam for listing
// peers to create the orca user on. Returns a list of peer addresses.
var peersListerOverride func() ([]string, error)

// cutoverFS is the filesystem seam used by performCutover /
// rollbackCutover for Traefik config editing (REQ-158, P09 T5). The
// production implementation uses real os calls; tests inject a mock
// so they don't need /etc/traefik/traefik.yml to exist.
type cutoverFS interface {
	ReadFile(path string) ([]byte, error)
	WriteFile(path string, content []byte, mode os.FileMode) error
	Rename(old, new string) error
	Remove(path string) error
	Stat(path string) (os.FileInfo, error)
}

// realCutoverFS is the production cutoverFS backed by the real os.
type realCutoverFS struct{}

func (realCutoverFS) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) }
func (realCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error {
	return os.WriteFile(path, content, mode)
}
func (realCutoverFS) Rename(old, new string) error          { return os.Rename(old, new) }
func (realCutoverFS) Remove(path string) error              { return os.Remove(path) }
func (realCutoverFS) Stat(path string) (os.FileInfo, error) { return os.Stat(path) }

// cutoverFSOverride is the package-level test seam for the cutover
// filesystem. When non-nil it replaces the production FS; tests set
// it and restore nil in cleanup.
var cutoverFSOverride cutoverFS

func cutoverFSFromCtx() cutoverFS {
	if cutoverFSOverride != nil {
		return cutoverFSOverride
	}
	return realCutoverFS{}
}

var upgradeCmd = &cobra.Command{
	Use:   "upgrade",
	Short: "Upgrade orca to a new version (REQ-115, R-017 cutover)",
	Long: `orca upgrade --to <version> is a thin wrapper around
install.sh + orca restore. It handles:
  - R-017 binding cutover: Traefik :443 → 127.0.0.1:8443 + nftables
    for existing v0.9/v0.10 clusters (with C-25 post-cutover
    verification and rollback on failure)
  - C-27: creates the 'orca' system user on existing peers
    (useradd -r orca, idempotent) — needed before P10b drift detection
  - Optional --import-ca: imports the v0.8 internal CA into step-ca
  - If a v0.8 layout is detected, runs Migratev08tov11 first
  - Idempotent: if already at the target version, no-op`,
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runUpgrade(cmd, cmd.OutOrStdout())
	},
}

func init() {
	upgradeCmd.Flags().StringVar(&upgradeTo, "to", "", "target version (e.g. v0.11.0) (required)")
	upgradeCmd.Flags().BoolVar(&upgradeImportCA, "import-ca", false, "import the v0.8 internal CA into step-ca during upgrade")
	upgradeCmd.Flags().BoolVar(&upgradeForce, "force", false, "skip cutover verification (use with caution)")
	upgradeCmd.Flags().BoolVar(&upgradeDryRun, "dry-run", false, "report what would be done without making changes")
	rootCmd.AddCommand(upgradeCmd)
}

// acquireUpgradeLock atomically creates an exclusive lock file at
// paths.ClusterDir()/upgrade.lock (REQ-156, P07 T3). Returns a release
// function that MUST be deferred (it removes the lock file). If the
// lock file already exists, returns an error "upgrade already in
// progress" — preventing two concurrent `orca upgrade` invocations
// from racing on the same cluster state (cutover, install.sh, peer
// user creation). O_CREATE|O_EXCL is atomic under POSIX: only one of
// two racing callers succeeds; the other gets EEXIST.
func acquireUpgradeLock() (func(), error) {
	lockPath := filepath.Join(paths.ClusterDir(), "upgrade.lock")
	if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil {
		return nil, fmt.Errorf("create cluster dir for upgrade lock: %w", err)
	}
	f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
	if err != nil {
		if os.IsExist(err) {
			return nil, fmt.Errorf("upgrade already in progress (lock file %s exists; remove it if stale)", lockPath)
		}
		return nil, fmt.Errorf("acquire upgrade lock: %w", err)
	}
	// Write the current PID + timestamp for diagnostics (best-effort;
	// a stale lock from a crashed process is the operator's signal).
	_, _ = f.WriteString(fmt.Sprintf("pid=%d started=%s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339)))
	_ = f.Close()
	return func() { _ = os.Remove(lockPath) }, nil
}

// UpgradeResult is the JSON-serializable summary of an upgrade run.
type UpgradeResult struct {
	TargetVersion  string   `json:"target_version"`
	CurrentVersion string   `json:"current_version"`
	DryRun         bool     `json:"dry_run"`
	MigratedV08    bool     `json:"migrated_v08"`
	CutoverNeeded  bool     `json:"cutover_needed"`
	CutoverOK      bool     `json:"cutover_ok,omitempty"`
	CutoverRolled  bool     `json:"cutover_rolled_back,omitempty"`
	UsersCreated   []string `json:"users_created,omitempty"`
	CAImported     bool     `json:"ca_imported,omitempty"`
	BinaryUpdated  bool     `json:"binary_updated"`
}

func runUpgrade(cmd *cobra.Command, out interface{ Write([]byte) (int, error) }) error {
	if upgradeTo == "" {
		return fmt.Errorf("--to is required (e.g. --to v0.11.0)")
	}

	res := UpgradeResult{
		TargetVersion:  upgradeTo,
		CurrentVersion: version,
		DryRun:         upgradeDryRun,
	}

	if !jsonOutput {
		fmt.Fprintf(out, "orca upgrade --to %s (current: %s)\n", upgradeTo, version)
	}

	if version == strings.TrimPrefix(upgradeTo, "v") && !upgradeDryRun {
		if !jsonOutput {
			fmt.Fprintf(out, "✓ Already at target version %s; no-op\n", upgradeTo)
		}
		res.BinaryUpdated = false
		if jsonOutput {
			return printJSON(res)
		}
		return nil
	}

	// REQ-156 / P07 T3: v0.8 layout detection is read-only and MUST
	// run BEFORE the upgrade lock is acquired — the lock creates the
	// cluster/ dir (for the lock file), and Detectv08 treats the
	// presence of a cluster/ dir as "already v0.11" (no migration
	// needed). Detecting first avoids a false negative that would
	// skip the migration on a genuine v0.8 layout.
	home := paths.Root()
	needV08Migration := migration.Detectv08(home)

	// Acquire an exclusive upgrade lock for the rest of the run so
	// two concurrent `orca upgrade` invocations cannot race on the
	// cutover / install.sh / peer user creation. The lock is released
	// on return (including error paths).
	upgradeRelease, err := acquireUpgradeLock()
	if err != nil {
		return err
	}
	defer upgradeRelease()

	if needV08Migration {
		if !jsonOutput {
			fmt.Fprintf(out, "• v0.8 layout detected; running data migration first\n")
		}
		if upgradeDryRun {
			fmt.Fprintf(out, "  [dry-run] would run Migratev08tov11(source=%s)\n", home)
		} else {
			if err := migration.Migratev08tov11(migration.MigrateOptions{
				SourceDir: home,
				TargetDir: home,
				ImportCA:  upgradeImportCA,
				DryRun:    false,
			}); err != nil {
				return fmt.Errorf("v0.8 migration: %w", err)
			}
		}
		res.MigratedV08 = true
	}

	runner := upgradeRunnerOverride
	if runner == nil {
		runner = execRunner{}
	}

	cutoverNeeded := detectOldTraefikBinding()
	res.CutoverNeeded = cutoverNeeded
	if cutoverNeeded {
		if !jsonOutput {
			fmt.Fprintf(out, "• R-017 cutover: Traefik on :443 detected; cutting over to 127.0.0.1:8443 + nftables\n")
		}
		if upgradeDryRun {
			fmt.Fprintf(out, "  [dry-run] would reconfigure Traefik to 127.0.0.1:8443 + add nftables redirect\n")
		} else {
			ok, err := performCutover(cmd.Context(), runner, out, upgradeForce)
			if err != nil {
				return err
			}
			res.CutoverOK = ok
			if !ok {
				res.CutoverRolled = true
				if jsonOutput {
					return printJSON(res)
				}
				return fmt.Errorf("cutover verification failed; rolled back to :443")
			}
		}
	}

	peers, err := listPeers()
	if err != nil {
		slog.Warn("upgrade: list peers failed", "err", err)
	}
	if len(peers) > 0 {
		if !jsonOutput {
			fmt.Fprintf(out, "• C-27: creating orca system user on %d peer(s)\n", len(peers))
		}
		created, err := createOrcaUserOnPeers(cmd.Context(), peers)
		if err != nil {
			slog.Warn("upgrade: orca user creation on peers failed", "err", err)
		}
		res.UsersCreated = created
	}

	if upgradeImportCA && !res.MigratedV08 {
		if !jsonOutput {
			fmt.Fprintf(out, "• Importing v0.8 internal CA into step-ca\n")
		}
		if upgradeDryRun {
			fmt.Fprintf(out, "  [dry-run] would import ca.crt/ca.key into step-ca\n")
		} else {
			if err := importCA(cmd.Context()); err != nil {
				slog.Warn("upgrade: CA import failed", "err", err)
			} else {
				res.CAImported = true
			}
		}
	}

	if upgradeDryRun {
		fmt.Fprintf(out, "  [dry-run] would download and install orca %s via install.sh\n", upgradeTo)
	} else {
		if !jsonOutput {
			fmt.Fprintf(out, "• Downloading and installing orca %s via install.sh\n", upgradeTo)
		}
		installSh := findInstallScript()
		if _, err := runner.Run(cmd.Context(), "bash", installSh, "--version", upgradeTo); err != nil {
			return fmt.Errorf("install.sh: %w", err)
		}
		res.BinaryUpdated = true
	}

	if jsonOutput {
		return printJSON(res)
	}
	fmt.Fprintf(out, "\n✓ upgrade to %s complete\n", upgradeTo)
	if cutoverNeeded && !upgradeDryRun {
		fmt.Fprintf(out, "\nRollback procedure (if cutover failed):\n")
		fmt.Fprintf(out, "  1. Restore Traefik entrypoint to :443\n")
		fmt.Fprintf(out, "  2. Remove nftables rules: nft delete table inet orca_redirect\n")
		fmt.Fprintf(out, "  3. Restart Traefik: systemctl restart traefik\n")
		fmt.Fprintf(out, "  4. Verify: curl -k https://localhost:443/\n")
	}
	return nil
}

// detectOldTraefikBinding returns true if Traefik is listening on :443
// (the v0.9/v0.10 default that R-017 cuts over from). Best-effort: if
// the check fails, returns false (no cutover needed).
func detectOldTraefikBinding() bool {
	runner := upgradeRunnerOverride
	if runner == nil {
		runner = execRunner{}
	}
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()
	out, err := runner.Run(ctx, "ss", "-tlnp")
	if err != nil {
		return false
	}
	return strings.Contains(string(out), ":443")
}

// performCutover reconfigures Traefik to listen on 127.0.0.1:8443 and
// adds nftables DNAT redirect from :443 → 127.0.0.1:8443. Then runs
// C-25 post-cutover verification: curl -k https://localhost:443/ must
// return 200. On failure, rolls back (restores :443, removes nft rules)
// and returns (false, nil). On success returns (true, nil). With
// force=true, verification is skipped.
//
// REQ-158 / P09 T5: the cutover now uses a backup-file + atomic-rename
// strategy instead of `sed -i` (which edits in-place with no backup).
// The Traefik config is copied to traefik.yml.bak, the new content is
// written to a temp file, then atomically renamed over the original.
// If any step fails, the backup is restored. This prevents a partial
// edit from leaving Traefik in a broken state.
func performCutover(ctx context.Context, runner commandRunner, out interface{ Write([]byte) (int, error) }, force bool) (bool, error) {
	cfs := cutoverFSFromCtx()
	traefikYml := "/etc/traefik/traefik.yml"
	backupPath := traefikYml + ".bak"

	// Step 1: read the current config and create a backup.
	original, err := cfs.ReadFile(traefikYml)
	if err != nil {
		return false, fmt.Errorf("cutover: read traefik.yml: %w", err)
	}
	if err := cfs.WriteFile(backupPath, original, 0o644); err != nil {
		return false, fmt.Errorf("cutover: write backup %s: %w", backupPath, err)
	}

	// Step 2: write the new config to a temp file, then atomically rename.
	newContent := strings.ReplaceAll(string(original), ":443", "127.0.0.1:8443")
	tmpPath := traefikYml + ".tmp"
	if err := cfs.WriteFile(tmpPath, []byte(newContent), 0o644); err != nil {
		return false, fmt.Errorf("cutover: write temp %s: %w", tmpPath, err)
	}
	if err := cfs.Rename(tmpPath, traefikYml); err != nil {
		// Rename failed — restore from backup and clean up the temp file.
		_ = cfs.Remove(tmpPath)
		_ = cfs.Rename(backupPath, traefikYml)
		return false, fmt.Errorf("cutover: atomic rename %s → %s: %w", tmpPath, traefikYml, err)
	}

	if _, err := runner.Run(ctx, "bash", "-c", "systemctl stop orca-traefik.service 2>/dev/null; systemctl disable orca-traefik.service 2>/dev/null; rm -f /etc/systemd/system/orca-traefik.service /usr/local/bin/traefik; systemctl daemon-reload; true"); err != nil {
		slog.Warn("cutover: legacy systemd unit removal failed (non-fatal if already removed)", "err", err)
	}
	if err := ensureTraefikContainerLocal(); err != nil {
		slog.Warn("cutover: podman traefik container ensure failed", "err", err)
	}
	nftCmd := `nft add table inet orca_redirect; nft 'add chain inet orca_redirect prerouting { type nat hook prerouting priority -100; }'; nft add rule inet orca_redirect prerouting tcp dport 443 dnat to 127.0.0.1:8443`
	if _, err := runner.Run(ctx, "bash", "-c", nftCmd); err != nil {
		slog.Warn("cutover: nftables rule add failed (non-fatal if already present)", "err", err)
	}

	if force {
		fmt.Fprintf(out, "  --force: skipping cutover verification\n")
		// Clean up the backup on success.
		_ = cfs.Remove(backupPath)
		return true, nil
	}

	if err := verifyCutover(out); err != nil {
		fmt.Fprintf(out, "  ✗ C-25 cutover verification failed: %v\n", err)
		fmt.Fprintf(out, "  Rolling back to :443...\n")
		if rbErr := rollbackCutover(ctx, runner); rbErr != nil {
			slog.Error("cutover: rollback failed", "err", rbErr)
		}
		return false, nil
	}
	fmt.Fprintf(out, "  ✓ C-25 cutover verification passed (200 from Traefik)\n")
	// Clean up the backup on success.
	_ = cfs.Remove(backupPath)
	return true, nil
}

// verifyCutover runs the C-25 post-cutover check: an HTTPS GET to
// https://localhost:443/ must return HTTP 200.
//
// REQ-157 / P08 T7: previously this used the default http.Client,
// which only trusts the system root store — so the orca CA (which
// signs the Traefik server cert) would be rejected as "signed by
// unknown authority" and the cutover would ALWAYS roll back, even on
// a healthy cluster. Now it builds a *tls.Config from the orca CA
// pool (security.ClientTLSConfig against certpaths.CACertPath()) so
// the server cert validates. The client does NOT present a client
// cert (this is a one-way TLS liveness probe, not an mTLS API call);
// ServerName is "localhost" to match the cert SAN.
func verifyCutover(out interface{ Write([]byte) (int, error) }) error {
	if httpClientOverride != nil {
		code, err := httpClientOverride("https://localhost:443/")
		if err != nil {
			return err
		}
		if code != 200 {
			return fmt.Errorf("HTTP %d (want 200)", code)
		}
		return nil
	}

	caPath := certpaths.CACertPath()
	tlsCfg, err := security.ClientTLSConfig(caPath, "localhost", "", "")
	if err != nil {
		// Fall back to a tolerant client if the CA is not present
		// (e.g. running verifyCutover in a test harness without a
		// cluster). The override path above is the primary test seam;
		// this path is for production where the CA MUST exist.
		return fmt.Errorf("verifyCutover: load orca CA %s: %w", caPath, err)
	}
	client := &http.Client{
		Timeout: 10 * time.Second,
		Transport: &http.Transport{
			TLSClientConfig: tlsCfg,
		},
	}
	resp, err := client.Get("https://localhost:443/")
	if err != nil {
		return fmt.Errorf("curl: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf("HTTP %d (want 200)", resp.StatusCode)
	}
	return nil
}

// rollbackCutover restores Traefik to :443 and removes nftables rules.
// REQ-158 / P09 T5: restore from the backup file (traefik.yml.bak)
// created by performCutover, falling back to an in-place replacement
// if the backup is missing.
func rollbackCutover(ctx context.Context, runner commandRunner) error {
	cfs := cutoverFSFromCtx()
	traefikYml := "/etc/traefik/traefik.yml"
	backupPath := traefikYml + ".bak"
	// Try restoring from the backup first.
	if _, err := cfs.Stat(backupPath); err == nil {
		if err := cfs.Rename(backupPath, traefikYml); err != nil {
			return fmt.Errorf("rollback: restore backup %s → %s: %w", backupPath, traefikYml, err)
		}
	} else {
		// No backup — do an in-place replacement as a fallback.
		current, rErr := cfs.ReadFile(traefikYml)
		if rErr != nil {
			return fmt.Errorf("rollback: read traefik.yml: %w", rErr)
		}
		restored := strings.ReplaceAll(string(current), "127.0.0.1:8443", ":443")
		tmpPath := traefikYml + ".tmp"
		if err := cfs.WriteFile(tmpPath, []byte(restored), 0o644); err != nil {
			return fmt.Errorf("rollback: write temp %s: %w", tmpPath, err)
		}
		if err := cfs.Rename(tmpPath, traefikYml); err != nil {
			_ = cfs.Remove(tmpPath)
			return fmt.Errorf("rollback: atomic rename: %w", err)
		}
	}
	if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil {
		return fmt.Errorf("rollback: restart traefik: %w", err)
	}
	if _, err := runner.Run(ctx, "nft", "delete", "table", "inet", "orca_redirect"); err != nil {
		slog.Warn("rollback: nft delete table failed (non-fatal if not present)", "err", err)
	}
	return nil
}

// createOrcaUserOnPeers runs the idempotent `useradd -r orca` on each
// peer via SSH (C-27). Returns the list of peers where the user was
// created (or already existed).
func createOrcaUserOnPeers(ctx context.Context, peers []string) ([]string, error) {
	transport := upgradeTransportOverride
	if transport == nil {
		return nil, fmt.Errorf("no SSH transport available for peer user creation")
	}
	var created []string
	for _, peer := range peers {
		cmd := "useradd -r orca -s /usr/sbin/nologin 2>/dev/null || true"
		if _, err := transport.Exec(ctx, peer, cmd); err != nil {
			slog.Warn("upgrade: useradd on peer failed", "peer", peer, "err", err)
			continue
		}
		created = append(created, peer)
	}
	return created, nil
}

// listPeers returns the list of peer addresses for C-27 orca user
// creation. Uses the peersListerOverride test seam if set; otherwise
// returns an empty list (no peers configured).
func listPeers() ([]string, error) {
	if peersListerOverride != nil {
		return peersListerOverride()
	}
	return nil, nil
}

// importCA imports the v0.8 internal CA into step-ca. This is the
// C-07 gate. Production wires a real step-ca client; tests use the
// migration.caImporterOverride seam.
func importCA(ctx context.Context) error {
	caCrt := filepath.Join(paths.Root(), "ca.crt")
	caKey := filepath.Join(paths.Root(), "ca.key")
	if _, err := os.Stat(caCrt); err != nil {
		return fmt.Errorf("import CA: ca.crt not found: %w", err)
	}
	if _, err := os.Stat(caKey); err != nil {
		return fmt.Errorf("import CA: ca.key not found: %w", err)
	}
	importer := migration.GetCAImporter()
	if importer == nil {
		return fmt.Errorf("import CA: no CA importer available")
	}
	return importer.ImportCA(ctx, caCrt, caKey)
}

// findInstallScript returns the path to scripts/install.sh. Checks
// the repo-local path first; falls back to downloading via curl
// (handled by the caller).
func findInstallScript() string {
	candidates := []string{
		"scripts/install.sh",
		"/usr/local/share/orca/scripts/install.sh",
	}
	for _, c := range candidates {
		if _, err := os.Stat(c); err == nil {
			return c
		}
	}
	return "scripts/install.sh"
}