orca/internal/cli/cutover.go

package cli

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"time"

	"github.com/spf13/cobra"

	"git.cloudinit.dev/coreci/orca/internal/certpaths"
	"git.cloudinit.dev/coreci/orca/internal/engine"
	"git.cloudinit.dev/coreci/orca/internal/store"
)

var cutoverTimeout time.Duration

var clusterCutoverCmd = &cobra.Command{
	Use:   "cutover",
	Short: "Stop v0.8 orca daemons and adopt running allocs (P14b)",
	Long: `Stop the v0.8 orca-daemon on every peer that still runs one,
discover its running allocations (orca-alloc-*.service), and adopt each
into the SSH-push path (mark it managed by the CLI-side scheduler).

The allocation's systemd unit keeps running independently of the
daemon; the cutover only re-records ownership in the cluster store
and stops the daemon.

Idempotent: a peer whose daemon is already stopped is a no-op for that
peer. Re-adopting an already-adopted alloc is a no-op.`,
	Args: cobra.NoArgs,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runCutover(cmd)
	},
}

func runCutover(cmd *cobra.Command) error {
	ctx, cancel := context.WithTimeout(cmd.Context(), cutoverTimeout)
	defer cancel()

	reg, closer, err := nodeRegistry()
	if err != nil {
		return err
	}
	defer closer()

	nodes, err := reg.List(ctx)
	if err != nil {
		return fmt.Errorf("list nodes: %w", err)
	}

	ex, err := drainExecFromCtx(cmd.Context())
	if err != nil {
		return fmt.Errorf("ssh transport: %w", err)
	}

	log := newLogger()

	type peerResult struct {
		Node           string   `json:"node"`
		Peer           string   `json:"peer"`
		DaemonStopped  bool     `json:"daemon_stopped"`
		AlreadyStopped bool     `json:"already_stopped"`
		Adopted        []string `json:"adopted"`
		Failed         string   `json:"failed,omitempty"`
	}

	results := make([]peerResult, 0, len(nodes))
	var stopped, already, failed, adopted []string

	for i := range nodes {
		n := nodes[i]
		peer := peerAddrForNode(n)
		if peer == "" {
			continue
		}
		pr := peerResult{Node: n.Name, Peer: peer, Adopted: []string{}}

		stopCmd := "systemctl stop orca-daemon.service"
		_, stopErr := ex.Exec(ctx, peer, stopCmd)
		switch {
		case stopErr == nil:
			pr.DaemonStopped = true
			stopped = append(stopped, n.Name)
		default:
			var exitErr *sshExitErr
			if errors.As(stopErr, &exitErr) && exitErr.code == 5 {
				pr.AlreadyStopped = true
				already = append(already, n.Name)
			} else {
				pr.Failed = stopErr.Error()
				failed = append(failed, n.Name)
				results = append(results, pr)
				log.Warn("cutover: stop daemon failed",
					slog.String("node", n.Name), slog.String("peer", peer), "error", stopErr)
				continue
			}
		}

		ids, listErr := listRunningAllocs(ctx, ex, peer)
		if listErr != nil {
			pr.Failed = listErr.Error()
			failed = append(failed, n.Name)
			results = append(results, pr)
			log.Warn("cutover: list allocs failed",
				slog.String("node", n.Name), slog.String("peer", peer), "error", listErr)
			continue
		}
		for _, id := range ids {
			if err := adoptAlloc(ctx, n.ID, id); err != nil {
				log.Warn("cutover: adopt alloc failed",
					slog.String("alloc", id), slog.String("node", n.Name), "error", err)
				pr.Failed = fmt.Sprintf("%sadopt %s: %v", pr.Failed, id, err)
				continue
			}
			pr.Adopted = append(pr.Adopted, id)
			adopted = append(adopted, n.Name+"/"+id)
		}
		results = append(results, pr)
	}

	summary := map[string]any{
		"stopped":         stopped,
		"already_stopped": already,
		"failed":          failed,
		"adopted":         adopted,
		"per_node":        results,
	}

	if db, dbErr := store.Open(certpaths.DBPath()); dbErr == nil {
		engine.NewAudit(store.NewAuditRepo(db), log).Record(ctx, "cli", "cluster.cutover", "cluster", "success", nil, summary)
		db.Close()
	}

	if jsonOutput {
		return printJSON(summary)
	}
	out := cmd.OutOrStdout()
	fmt.Fprintf(out, "✓ cutover complete (%d stopped, %d already stopped, %d failed, %d adopted)\n",
		len(stopped), len(already), len(failed), len(adopted))
	for _, n := range stopped {
		fmt.Fprintf(out, "  stopped %s\n", n)
	}
	for _, n := range already {
		fmt.Fprintf(out, "  already-stopped %s\n", n)
	}
	for _, a := range adopted {
		fmt.Fprintf(out, "  adopted %s\n", a)
	}
	for _, n := range failed {
		fmt.Fprintf(out, "  failed %s\n", n)
	}
	if len(failed) > 0 {
		return fmt.Errorf("cutover: %d peer(s) failed", len(failed))
	}
	return nil
}

func adoptAlloc(ctx context.Context, nodeID, allocID string) error {
	db, err := store.Open(certpaths.DBPath())
	if err != nil {
		return fmt.Errorf("open db: %w", err)
	}
	defer db.Close()
	hist := store.NewAllocHistoryRepo(db)
	if err := hist.EnsureSchema(ctx); err != nil {
		return fmt.Errorf("alloc history schema: %w", err)
	}
	entry := store.AllocHistoryEntry{
		AllocID:   allocID,
		NodeID:    nodeID,
		FromState: "daemon-managed",
		ToState:   "ssh-push-managed",
		Timestamp: time.Now().UTC(),
		Reason:    "p14b-cutover",
	}
	return hist.Record(ctx, entry)
}

func init() {
	clusterCutoverCmd.Flags().DurationVar(&cutoverTimeout, "timeout", 5*time.Minute,
		"max time for the full cutover across all peers")
}