// Package cli: upgrade.go implements the `orca upgrade --to ` // subcommand (REQ-115, gates C-25 and C-27). It is a thin wrapper // around scripts/install.sh + orca restore that also handles the // R-017 binding cutover (Traefik :443 → 127.0.0.1:8443 + nftables) for // existing v0.9/v0.10 clusters, creates the `orca` system user on // existing peers (C-27, needed before P10b drift detection works), // and optionally imports the v0.8 internal CA into step-ca. package cli import ( "context" "fmt" "log/slog" "net/http" "os" "os/exec" "path/filepath" "strings" "time" "github.com/spf13/cobra" "git.cloudinit.dev/coreci/orca/internal/certpaths" "git.cloudinit.dev/coreci/orca/internal/migration" "git.cloudinit.dev/coreci/orca/internal/paths" "git.cloudinit.dev/coreci/orca/internal/security" ) var ( upgradeTo string upgradeImportCA bool upgradeForce bool upgradeDryRun bool ) // commandRunner is the seam for running shell commands (install.sh, // nft, useradd). Tests inject a mock; production uses execRunner. type commandRunner interface { Run(ctx context.Context, name string, args ...string) ([]byte, error) } // execRunner runs commands via os/exec. type execRunner struct{} func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) return cmd.CombinedOutput() } // upgradeRunnerOverride is the package-level test seam for the // command runner. var upgradeRunnerOverride commandRunner // httpClientOverride is the package-level test seam for the cutover // verification HTTP check (C-25). Tests inject a mock. var httpClientOverride func(url string) (int, error) // upgradeTransport is the SSH surface the upgrade command needs for // C-27 orca user creation on peers. Mirrors peerSetupTransport. type upgradeTransport interface { Exec(ctx context.Context, peer string, cmd string) ([]byte, error) } // upgradeTransportOverride is the package-level test seam for the // SSH transport. var upgradeTransportOverride upgradeTransport // peersListerOverride is the package-level test seam for listing // peers to create the orca user on. Returns a list of peer addresses. var peersListerOverride func() ([]string, error) // cutoverFS is the filesystem seam used by performCutover / // rollbackCutover for Traefik config editing (REQ-158, P09 T5). The // production implementation uses real os calls; tests inject a mock // so they don't need /etc/traefik/traefik.yml to exist. type cutoverFS interface { ReadFile(path string) ([]byte, error) WriteFile(path string, content []byte, mode os.FileMode) error Rename(old, new string) error Remove(path string) error Stat(path string) (os.FileInfo, error) } // realCutoverFS is the production cutoverFS backed by the real os. type realCutoverFS struct{} func (realCutoverFS) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } func (realCutoverFS) WriteFile(path string, content []byte, mode os.FileMode) error { return os.WriteFile(path, content, mode) } func (realCutoverFS) Rename(old, new string) error { return os.Rename(old, new) } func (realCutoverFS) Remove(path string) error { return os.Remove(path) } func (realCutoverFS) Stat(path string) (os.FileInfo, error) { return os.Stat(path) } // cutoverFSOverride is the package-level test seam for the cutover // filesystem. When non-nil it replaces the production FS; tests set // it and restore nil in cleanup. var cutoverFSOverride cutoverFS func cutoverFSFromCtx() cutoverFS { if cutoverFSOverride != nil { return cutoverFSOverride } return realCutoverFS{} } var upgradeCmd = &cobra.Command{ Use: "upgrade", Short: "Upgrade orca to a new version (REQ-115, R-017 cutover)", Long: `orca upgrade --to is a thin wrapper around install.sh + orca restore. It handles: - R-017 binding cutover: Traefik :443 → 127.0.0.1:8443 + nftables for existing v0.9/v0.10 clusters (with C-25 post-cutover verification and rollback on failure) - C-27: creates the 'orca' system user on existing peers (useradd -r orca, idempotent) — needed before P10b drift detection - Optional --import-ca: imports the v0.8 internal CA into step-ca - If a v0.8 layout is detected, runs Migratev08tov11 first - Idempotent: if already at the target version, no-op`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { return runUpgrade(cmd, cmd.OutOrStdout()) }, } func init() { upgradeCmd.Flags().StringVar(&upgradeTo, "to", "", "target version (e.g. v0.11.0) (required)") upgradeCmd.Flags().BoolVar(&upgradeImportCA, "import-ca", false, "import the v0.8 internal CA into step-ca during upgrade") upgradeCmd.Flags().BoolVar(&upgradeForce, "force", false, "skip cutover verification (use with caution)") upgradeCmd.Flags().BoolVar(&upgradeDryRun, "dry-run", false, "report what would be done without making changes") rootCmd.AddCommand(upgradeCmd) } // acquireUpgradeLock atomically creates an exclusive lock file at // paths.ClusterDir()/upgrade.lock (REQ-156, P07 T3). Returns a release // function that MUST be deferred (it removes the lock file). If the // lock file already exists, returns an error "upgrade already in // progress" — preventing two concurrent `orca upgrade` invocations // from racing on the same cluster state (cutover, install.sh, peer // user creation). O_CREATE|O_EXCL is atomic under POSIX: only one of // two racing callers succeeds; the other gets EEXIST. func acquireUpgradeLock() (func(), error) { lockPath := filepath.Join(paths.ClusterDir(), "upgrade.lock") if err := os.MkdirAll(filepath.Dir(lockPath), 0o755); err != nil { return nil, fmt.Errorf("create cluster dir for upgrade lock: %w", err) } f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) if err != nil { if os.IsExist(err) { return nil, fmt.Errorf("upgrade already in progress (lock file %s exists; remove it if stale)", lockPath) } return nil, fmt.Errorf("acquire upgrade lock: %w", err) } // Write the current PID + timestamp for diagnostics (best-effort; // a stale lock from a crashed process is the operator's signal). _, _ = f.WriteString(fmt.Sprintf("pid=%d started=%s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339))) _ = f.Close() return func() { _ = os.Remove(lockPath) }, nil } // UpgradeResult is the JSON-serializable summary of an upgrade run. type UpgradeResult struct { TargetVersion string `json:"target_version"` CurrentVersion string `json:"current_version"` DryRun bool `json:"dry_run"` MigratedV08 bool `json:"migrated_v08"` CutoverNeeded bool `json:"cutover_needed"` CutoverOK bool `json:"cutover_ok,omitempty"` CutoverRolled bool `json:"cutover_rolled_back,omitempty"` UsersCreated []string `json:"users_created,omitempty"` CAImported bool `json:"ca_imported,omitempty"` BinaryUpdated bool `json:"binary_updated"` } func runUpgrade(cmd *cobra.Command, out interface{ Write([]byte) (int, error) }) error { if upgradeTo == "" { return fmt.Errorf("--to is required (e.g. --to v0.11.0)") } res := UpgradeResult{ TargetVersion: upgradeTo, CurrentVersion: version, DryRun: upgradeDryRun, } if !jsonOutput { fmt.Fprintf(out, "orca upgrade --to %s (current: %s)\n", upgradeTo, version) } if version == strings.TrimPrefix(upgradeTo, "v") && !upgradeDryRun { if !jsonOutput { fmt.Fprintf(out, "✓ Already at target version %s; no-op\n", upgradeTo) } res.BinaryUpdated = false if jsonOutput { return printJSON(res) } return nil } // REQ-156 / P07 T3: v0.8 layout detection is read-only and MUST // run BEFORE the upgrade lock is acquired — the lock creates the // cluster/ dir (for the lock file), and Detectv08 treats the // presence of a cluster/ dir as "already v0.11" (no migration // needed). Detecting first avoids a false negative that would // skip the migration on a genuine v0.8 layout. home := paths.Root() needV08Migration := migration.Detectv08(home) // Acquire an exclusive upgrade lock for the rest of the run so // two concurrent `orca upgrade` invocations cannot race on the // cutover / install.sh / peer user creation. The lock is released // on return (including error paths). upgradeRelease, err := acquireUpgradeLock() if err != nil { return err } defer upgradeRelease() if needV08Migration { if !jsonOutput { fmt.Fprintf(out, "• v0.8 layout detected; running data migration first\n") } if upgradeDryRun { fmt.Fprintf(out, " [dry-run] would run Migratev08tov11(source=%s)\n", home) } else { if err := migration.Migratev08tov11(migration.MigrateOptions{ SourceDir: home, TargetDir: home, ImportCA: upgradeImportCA, DryRun: false, }); err != nil { return fmt.Errorf("v0.8 migration: %w", err) } } res.MigratedV08 = true } runner := upgradeRunnerOverride if runner == nil { runner = execRunner{} } cutoverNeeded := detectOldTraefikBinding() res.CutoverNeeded = cutoverNeeded if cutoverNeeded { if !jsonOutput { fmt.Fprintf(out, "• R-017 cutover: Traefik on :443 detected; cutting over to 127.0.0.1:8443 + nftables\n") } if upgradeDryRun { fmt.Fprintf(out, " [dry-run] would reconfigure Traefik to 127.0.0.1:8443 + add nftables redirect\n") } else { ok, err := performCutover(cmd.Context(), runner, out, upgradeForce) if err != nil { return err } res.CutoverOK = ok if !ok { res.CutoverRolled = true if jsonOutput { return printJSON(res) } return fmt.Errorf("cutover verification failed; rolled back to :443") } } } peers, err := listPeers() if err != nil { slog.Warn("upgrade: list peers failed", "err", err) } if len(peers) > 0 { if !jsonOutput { fmt.Fprintf(out, "• C-27: creating orca system user on %d peer(s)\n", len(peers)) } created, err := createOrcaUserOnPeers(cmd.Context(), peers) if err != nil { slog.Warn("upgrade: orca user creation on peers failed", "err", err) } res.UsersCreated = created } if upgradeImportCA && !res.MigratedV08 { if !jsonOutput { fmt.Fprintf(out, "• Importing v0.8 internal CA into step-ca\n") } if upgradeDryRun { fmt.Fprintf(out, " [dry-run] would import ca.crt/ca.key into step-ca\n") } else { if err := importCA(cmd.Context()); err != nil { slog.Warn("upgrade: CA import failed", "err", err) } else { res.CAImported = true } } } if upgradeDryRun { fmt.Fprintf(out, " [dry-run] would download and install orca %s via install.sh\n", upgradeTo) } else { if !jsonOutput { fmt.Fprintf(out, "• Downloading and installing orca %s via install.sh\n", upgradeTo) } installSh := findInstallScript() if _, err := runner.Run(cmd.Context(), "bash", installSh, "--version", upgradeTo); err != nil { return fmt.Errorf("install.sh: %w", err) } res.BinaryUpdated = true } if jsonOutput { return printJSON(res) } fmt.Fprintf(out, "\n✓ upgrade to %s complete\n", upgradeTo) if cutoverNeeded && !upgradeDryRun { fmt.Fprintf(out, "\nRollback procedure (if cutover failed):\n") fmt.Fprintf(out, " 1. Restore Traefik entrypoint to :443\n") fmt.Fprintf(out, " 2. Remove nftables rules: nft delete table inet orca_redirect\n") fmt.Fprintf(out, " 3. Restart Traefik: systemctl restart traefik\n") fmt.Fprintf(out, " 4. Verify: curl -k https://localhost:443/\n") } return nil } // detectOldTraefikBinding returns true if Traefik is listening on :443 // (the v0.9/v0.10 default that R-017 cuts over from). Best-effort: if // the check fails, returns false (no cutover needed). func detectOldTraefikBinding() bool { runner := upgradeRunnerOverride if runner == nil { runner = execRunner{} } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() out, err := runner.Run(ctx, "ss", "-tlnp") if err != nil { return false } return strings.Contains(string(out), ":443") } // performCutover reconfigures Traefik to listen on 127.0.0.1:8443 and // adds nftables DNAT redirect from :443 → 127.0.0.1:8443. Then runs // C-25 post-cutover verification: curl -k https://localhost:443/ must // return 200. On failure, rolls back (restores :443, removes nft rules) // and returns (false, nil). On success returns (true, nil). With // force=true, verification is skipped. // // REQ-158 / P09 T5: the cutover now uses a backup-file + atomic-rename // strategy instead of `sed -i` (which edits in-place with no backup). // The Traefik config is copied to traefik.yml.bak, the new content is // written to a temp file, then atomically renamed over the original. // If any step fails, the backup is restored. This prevents a partial // edit from leaving Traefik in a broken state. func performCutover(ctx context.Context, runner commandRunner, out interface{ Write([]byte) (int, error) }, force bool) (bool, error) { cfs := cutoverFSFromCtx() traefikYml := "/etc/traefik/traefik.yml" backupPath := traefikYml + ".bak" // Step 1: read the current config and create a backup. original, err := cfs.ReadFile(traefikYml) if err != nil { return false, fmt.Errorf("cutover: read traefik.yml: %w", err) } if err := cfs.WriteFile(backupPath, original, 0o644); err != nil { return false, fmt.Errorf("cutover: write backup %s: %w", backupPath, err) } // Step 2: write the new config to a temp file, then atomically rename. newContent := strings.ReplaceAll(string(original), ":443", "127.0.0.1:8443") tmpPath := traefikYml + ".tmp" if err := cfs.WriteFile(tmpPath, []byte(newContent), 0o644); err != nil { return false, fmt.Errorf("cutover: write temp %s: %w", tmpPath, err) } if err := cfs.Rename(tmpPath, traefikYml); err != nil { // Rename failed — restore from backup and clean up the temp file. _ = cfs.Remove(tmpPath) _ = cfs.Rename(backupPath, traefikYml) return false, fmt.Errorf("cutover: atomic rename %s → %s: %w", tmpPath, traefikYml, err) } if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil { // Restart failed — restore from backup. _ = cfs.Rename(backupPath, traefikYml) return false, fmt.Errorf("cutover: restart traefik: %w", err) } nftCmd := `nft add table inet orca_redirect; nft 'add chain inet orca_redirect prerouting { type nat hook prerouting priority -100; }'; nft add rule inet orca_redirect prerouting tcp dport 443 dnat to 127.0.0.1:8443` if _, err := runner.Run(ctx, "bash", "-c", nftCmd); err != nil { slog.Warn("cutover: nftables rule add failed (non-fatal if already present)", "err", err) } if force { fmt.Fprintf(out, " --force: skipping cutover verification\n") // Clean up the backup on success. _ = cfs.Remove(backupPath) return true, nil } if err := verifyCutover(out); err != nil { fmt.Fprintf(out, " ✗ C-25 cutover verification failed: %v\n", err) fmt.Fprintf(out, " Rolling back to :443...\n") if rbErr := rollbackCutover(ctx, runner); rbErr != nil { slog.Error("cutover: rollback failed", "err", rbErr) } return false, nil } fmt.Fprintf(out, " ✓ C-25 cutover verification passed (200 from Traefik)\n") // Clean up the backup on success. _ = cfs.Remove(backupPath) return true, nil } // verifyCutover runs the C-25 post-cutover check: an HTTPS GET to // https://localhost:443/ must return HTTP 200. // // REQ-157 / P08 T7: previously this used the default http.Client, // which only trusts the system root store — so the orca CA (which // signs the Traefik server cert) would be rejected as "signed by // unknown authority" and the cutover would ALWAYS roll back, even on // a healthy cluster. Now it builds a *tls.Config from the orca CA // pool (security.ClientTLSConfig against certpaths.CACertPath()) so // the server cert validates. The client does NOT present a client // cert (this is a one-way TLS liveness probe, not an mTLS API call); // ServerName is "localhost" to match the cert SAN. func verifyCutover(out interface{ Write([]byte) (int, error) }) error { if httpClientOverride != nil { code, err := httpClientOverride("https://localhost:443/") if err != nil { return err } if code != 200 { return fmt.Errorf("HTTP %d (want 200)", code) } return nil } caPath := certpaths.CACertPath() tlsCfg, err := security.ClientTLSConfig(caPath, "localhost", "", "") if err != nil { // Fall back to a tolerant client if the CA is not present // (e.g. running verifyCutover in a test harness without a // cluster). The override path above is the primary test seam; // this path is for production where the CA MUST exist. return fmt.Errorf("verifyCutover: load orca CA %s: %w", caPath, err) } client := &http.Client{ Timeout: 10 * time.Second, Transport: &http.Transport{ TLSClientConfig: tlsCfg, }, } resp, err := client.Get("https://localhost:443/") if err != nil { return fmt.Errorf("curl: %w", err) } defer resp.Body.Close() if resp.StatusCode != 200 { return fmt.Errorf("HTTP %d (want 200)", resp.StatusCode) } return nil } // rollbackCutover restores Traefik to :443 and removes nftables rules. // REQ-158 / P09 T5: restore from the backup file (traefik.yml.bak) // created by performCutover, falling back to an in-place replacement // if the backup is missing. func rollbackCutover(ctx context.Context, runner commandRunner) error { cfs := cutoverFSFromCtx() traefikYml := "/etc/traefik/traefik.yml" backupPath := traefikYml + ".bak" // Try restoring from the backup first. if _, err := cfs.Stat(backupPath); err == nil { if err := cfs.Rename(backupPath, traefikYml); err != nil { return fmt.Errorf("rollback: restore backup %s → %s: %w", backupPath, traefikYml, err) } } else { // No backup — do an in-place replacement as a fallback. current, rErr := cfs.ReadFile(traefikYml) if rErr != nil { return fmt.Errorf("rollback: read traefik.yml: %w", rErr) } restored := strings.ReplaceAll(string(current), "127.0.0.1:8443", ":443") tmpPath := traefikYml + ".tmp" if err := cfs.WriteFile(tmpPath, []byte(restored), 0o644); err != nil { return fmt.Errorf("rollback: write temp %s: %w", tmpPath, err) } if err := cfs.Rename(tmpPath, traefikYml); err != nil { _ = cfs.Remove(tmpPath) return fmt.Errorf("rollback: atomic rename: %w", err) } } if _, err := runner.Run(ctx, "systemctl", "restart", "traefik"); err != nil { return fmt.Errorf("rollback: restart traefik: %w", err) } if _, err := runner.Run(ctx, "nft", "delete", "table", "inet", "orca_redirect"); err != nil { slog.Warn("rollback: nft delete table failed (non-fatal if not present)", "err", err) } return nil } // createOrcaUserOnPeers runs the idempotent `useradd -r orca` on each // peer via SSH (C-27). Returns the list of peers where the user was // created (or already existed). func createOrcaUserOnPeers(ctx context.Context, peers []string) ([]string, error) { transport := upgradeTransportOverride if transport == nil { return nil, fmt.Errorf("no SSH transport available for peer user creation") } var created []string for _, peer := range peers { cmd := "useradd -r orca -s /usr/sbin/nologin 2>/dev/null || true" if _, err := transport.Exec(ctx, peer, cmd); err != nil { slog.Warn("upgrade: useradd on peer failed", "peer", peer, "err", err) continue } created = append(created, peer) } return created, nil } // listPeers returns the list of peer addresses for C-27 orca user // creation. Uses the peersListerOverride test seam if set; otherwise // returns an empty list (no peers configured). func listPeers() ([]string, error) { if peersListerOverride != nil { return peersListerOverride() } return nil, nil } // importCA imports the v0.8 internal CA into step-ca. This is the // C-07 gate. Production wires a real step-ca client; tests use the // migration.caImporterOverride seam. func importCA(ctx context.Context) error { caCrt := filepath.Join(paths.Root(), "ca.crt") caKey := filepath.Join(paths.Root(), "ca.key") if _, err := os.Stat(caCrt); err != nil { return fmt.Errorf("import CA: ca.crt not found: %w", err) } if _, err := os.Stat(caKey); err != nil { return fmt.Errorf("import CA: ca.key not found: %w", err) } importer := migration.GetCAImporter() if importer == nil { return fmt.Errorf("import CA: no CA importer available") } return importer.ImportCA(ctx, caCrt, caKey) } // findInstallScript returns the path to scripts/install.sh. Checks // the repo-local path first; falls back to downloading via curl // (handled by the caller). func findInstallScript() string { candidates := []string{ "scripts/install.sh", "/usr/local/share/orca/scripts/install.sh", } for _, c := range candidates { if _, err := os.Stat(c); err == nil { return c } } return "scripts/install.sh" }