orca/internal/cli/metrics.go

package cli

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/spf13/cobra"

	"git.cloudinit.dev/coreci/orca/internal/model"
	"git.cloudinit.dev/coreci/orca/internal/store"
	"git.cloudinit.dev/coreci/orca/internal/transport"
)

var metricsAddr string

var metricsCmd = &cobra.Command{
	Use:   "metrics",
	Short: "Run the orca metrics endpoint (Prometheus text exposition)",
	Long: `Start a standalone HTTP server exposing Prometheus text-exposition
metrics at /metrics and a liveness probe at /healthz. Designed to run on the
CLI host or a designated metrics host; polls cluster state periodically and
updates gauges. No orca daemon required (R-001).`,
	RunE: func(cmd *cobra.Command, args []string) error {
		log := newLogger()
		m := transport.NewMetrics()

		db, closer, err := openDB()
		if err != nil {
			log.Warn("metrics: open db failed, gauges will stay 0", "err", err)
			db = nil
			closer = func() error { return nil }
		}
		defer closer()

		var pollWG sync.WaitGroup
		pollCtx, pollCancel := context.WithCancel(cmd.Context())
		defer func() {
			pollCancel()
			pollWG.Wait()
		}()

		pollWG.Add(1)
		go func() {
			defer pollWG.Done()
			pollLoop(pollCtx, m, db, log)
		}()

		mux := http.NewServeMux()
		mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("X-Content-Type-Options", "nosniff")
			w.Header().Set("X-Frame-Options", "DENY")
			w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
			if err := m.WritePrometheus(w); err != nil {
				log.Warn("metrics: write exposition failed", "err", err)
			}
		})
		mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("X-Content-Type-Options", "nosniff")
			w.Header().Set("X-Frame-Options", "DENY")
			w.WriteHeader(http.StatusOK)
			_, _ = w.Write([]byte("ok\n"))
		})

		srv := &http.Server{
			Addr:    metricsAddr,
			Handler: mux,
		}

		errCh := make(chan error, 1)
		go func() {
			err := srv.ListenAndServe()
			if err != nil && !errors.Is(err, http.ErrServerClosed) {
				errCh <- err
			}
		}()

		fmt.Fprintf(cmd.OutOrStdout(), "✓ orca metrics listening on %s\n", metricsAddr)
		fmt.Fprintln(cmd.OutOrStdout(), "  /metrics - Prometheus text exposition")
		fmt.Fprintln(cmd.OutOrStdout(), "  /healthz - liveness probe")
		fmt.Fprintln(cmd.OutOrStdout(), "  press Ctrl+C to stop")

		ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM)
		defer stop()

		select {
		case <-ctx.Done():
			fmt.Fprintln(cmd.OutOrStdout(), "\nshutting down...")
			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			return srv.Shutdown(shutdownCtx)
		case err := <-errCh:
			return fmt.Errorf("metrics server: %w", err)
		}
	},
}

func pollLoop(ctx context.Context, m *transport.Metrics, db *sql.DB, log interface{ Warn(string, ...any) }) {
	if db == nil {
		return
	}
	ticker := time.NewTicker(15 * time.Second)
	defer ticker.Stop()
	refresh(ctx, m, db, log)
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			refresh(ctx, m, db, log)
		}
	}
}

func refresh(ctx context.Context, m *transport.Metrics, db *sql.DB, log interface{ Warn(string, ...any) }) {
	if nodes, err := store.NewNodeRepo(db).List(ctx); err != nil {
		log.Warn("metrics: node list failed", "err", err)
	} else {
		m.SetGauge("nodes_total", float64(len(nodes)))
	}
	if jobs, err := store.NewJobRepo(db).List(ctx); err != nil {
		log.Warn("metrics: job list failed", "err", err)
	} else {
		m.SetGauge("allocs_total", float64(len(jobs)))
		// REQ-159 / P10: jobs by state.
		byState := make(map[model.JobStatus]int, 8)
		for _, j := range jobs {
			byState[j.Status]++
		}
		// Set total + per-state counts using simple gauge names.
		running := byState[model.JobStatusRunning]
		failed := byState[model.JobStatusFailed]
		complete := byState[model.JobStatusComplete]
		m.SetGauge("orca_jobs_running", float64(running))
		m.SetGauge("orca_jobs_failed", float64(failed))
		m.SetGauge("orca_jobs_complete", float64(complete))
	}
	// REQ-159 / P10: audit chain head gauge.
	if head, err := store.NewAuditRepo(db).ChainHead(ctx); err == nil && head != "" {
		m.SetGauge("orca_audit_chain_head", 1)
	}
}

func init() {
	metricsCmd.Flags().StringVar(&metricsAddr, "addr", ":9100", "listen address for the metrics HTTP server")
	rootCmd.AddCommand(metricsCmd)
}