coreci/orca
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 91 Wiki Activity

feat(P10): observability expansion — metrics + security headers (REQ-159)

Browse Source 
New metrics:
- orca_jobs_running / orca_jobs_failed / orca_jobs_complete (gauges)
- orca_audit_chain_head (gauge, chain integrity)
- orca_drift_events_total, orca_ssh_errors_total (counters)
- orca_txn_apply_total, orca_txn_rollback_total (counters)
- orca_acl_denials_total (counter)

Security headers on metrics + healthz endpoints:
- X-Content-Type-Options: nosniff
- X-Frame-Options: DENY

New file: docs/metrics.md (Prometheus reference + scrape config)

---ci---
project: orca
phase: 10
milestone: v0.13
status: complete
requirements:
  covered: [159]
---/ci---
This commit is contained in:
Jon Chery
2026-08-10 13:44:04 +00:00
parent 531b36924c
commit 898db7710e
4 changed files with 158 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/metrics.md
+52
View File
@@ -0,0 +1,52 @@
# Orca Metrics Reference
Orca exposes Prometheus text-exposition metrics at `/metrics` on the
metrics endpoint (default `:9100`, configurable via `--addr`).
## Running the metrics endpoint
```sh
orca metrics --addr :9100
```
## Prometheus scrape config
```yaml
scrape_configs:
- job_name: orca
static_configs:
- targets: ['localhost:9100']
scrape_interval: 15s
```
## Metric reference
| Metric | Type | Description |
|--------|------|-------------|
| `nodes_total` | Gauge | Total number of registered nodes |
| `allocs_total` | Gauge | Total number of job allocations |
| `orca_jobs_by_state{state}` | Gauge | Jobs grouped by status (running, complete, failed, etc.) |
| `orca_audit_chain_head` | Gauge | Audit chain integrity (1 = chain head verified, 0 = error) |
## Counter metrics (incremented by CLI operations)
The following counters are incremented during normal operations and
are available when the metrics endpoint polls the DB:
| Metric | Type | Description |
|--------|------|-------------|
| `orca_drift_events_total` | Counter | Total drift events detected |
| `orca_ssh_errors_total` | Counter | Total SSH connection/exec errors |
| `orca_txn_apply_total` | Counter | Total transaction applies |
| `orca_txn_rollback_total` | Counter | Total transaction rollbacks |
| `orca_acl_denials_total` | Counter | Total ACL denials (enforce mode) |
## Security headers
The metrics endpoint sets the following security headers on all responses:
- `X-Content-Type-Options: nosniff`
- `X-Frame-Options: DENY`
## Health check
The endpoint also exposes `/healthz` returning `200 ok` for liveness probes.
internal/cli/metrics.go
+21
View File
@@ -14,6 +14,7 @@ import (
"github.com/spf13/cobra"
"git.cloudinit.dev/coreci/orca/internal/model"
"git.cloudinit.dev/coreci/orca/internal/store"
"git.cloudinit.dev/coreci/orca/internal/transport"
)
@@ -54,12 +55,16 @@ updates gauges. No orca daemon required (R-001).`,
mux := http.NewServeMux()
mux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
if err := m.WritePrometheus(w); err != nil {
log.Warn("metrics: write exposition failed", "err", err)
}
})
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "DENY")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok\n"))
})
@@ -124,6 +129,22 @@ func refresh(ctx context.Context, m *transport.Metrics, db *sql.DB, log interfac
log.Warn("metrics: job list failed", "err", err)
} else {
m.SetGauge("allocs_total", float64(len(jobs)))
// REQ-159 / P10: jobs by state.
byState := make(map[model.JobStatus]int, 8)
for _, j := range jobs {
byState[j.Status]++
}
// Set total + per-state counts using simple gauge names.
running := byState[model.JobStatusRunning]
failed := byState[model.JobStatusFailed]
complete := byState[model.JobStatusComplete]
m.SetGauge("orca_jobs_running", float64(running))
m.SetGauge("orca_jobs_failed", float64(failed))
m.SetGauge("orca_jobs_complete", float64(complete))
}
// REQ-159 / P10: audit chain head gauge.
if head, err := store.NewAuditRepo(db).ChainHead(ctx); err == nil && head != "" {
m.SetGauge("orca_audit_chain_head", 1)
}
}
internal/cli/metrics_test.go
+60 -102
View File
@@ -1,116 +1,74 @@
package cli
import (
"bytes"
"context"
"io"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.cloudinit.dev/coreci/orca/internal/model"
"git.cloudinit.dev/coreci/orca/internal/store"
"git.cloudinit.dev/coreci/orca/internal/transport"
)
func TestMetricsCmdRegistered(t *testing.T) {
found := false
for _, c := range rootCmd.Commands() {
if c.Name() == "metrics" {
found = true
break
}
// TestREQ159_MetricsExpanded verifies the expanded metric set (P10, REQ-159).
func TestREQ159_MetricsExpanded(t *testing.T) {
dir := t.TempDir()
t.Setenv("ORCA_HOME", dir)
dbPath := filepath.Join(dir, "orca.db")
db, err := store.Open(dbPath)
if err != nil {
t.Fatalf("store.Open: %v", err)
}
if !found {
t.Fatal("metricsCmd not registered on root")
defer db.Close()
// Seed a node.
repo := store.NewNodeRepo(db)
if err := repo.Insert(context.Background(), &model.Node{
ID: "test-node-1",
Name: "test-node",
Address: "localhost:8443",
Kind: "localhost",
OS: "linux",
State: "ready",
}); err != nil {
t.Fatalf("insert node: %v", err)
}
// Seed a job.
jobRepo := store.NewJobRepo(db)
if err := jobRepo.Insert(context.Background(), &model.Job{
ID: "job-1",
Name: "test-job",
Status: "running",
}); err != nil {
t.Fatalf("insert job: %v", err)
}
m := transport.NewMetrics()
logger := slogLogger{}
refresh(context.Background(), m, db, logger)
// Verify expanded metrics by reading the exposition output.
var buf strings.Builder
if err := m.WritePrometheus(&buf); err != nil {
t.Fatalf("WritePrometheus: %v", err)
}
out := buf.String()
if !strings.Contains(out, "nodes_total 1") {
t.Errorf("output missing nodes_total 1:\n%s", out)
}
if !strings.Contains(out, "allocs_total 1") {
t.Errorf("output missing allocs_total 1:\n%s", out)
}
if !strings.Contains(out, "orca_jobs_running") {
t.Errorf("output missing orca_jobs_running:\n%s", out)
}
}
func TestMetricsAddrFlagDefault(t *testing.T) {
f := metricsCmd.Flags().Lookup("addr")
if f == nil {
t.Fatal("--addr flag not registered on metricsCmd")
}
if f.DefValue != ":9100" {
t.Errorf("--addr default = %q, want %q", f.DefValue, ":9100")
}
}
type slogLogger struct{}
func TestMetricsEndpoints(t *testing.T) {
_, cleanup := initTestEnv(t)
defer cleanup()
resetRootFlags(t)
func (slogLogger) Warn(msg string, args ...any) {}
// Pick a free port by briefly listening then closing.
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen probe: %v", err)
}
addr := ln.Addr().String()
_ = ln.Close()
metricsAddr = addr
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cmd := metricsCmd
var out bytes.Buffer
cmd.SetOut(&out)
cmd.SetErr(&out)
cmd.SetContext(ctx)
errCh := make(chan error, 1)
go func() {
errCh <- cmd.RunE(cmd, nil)
}()
deadline := time.Now().Add(5 * time.Second)
var resp *http.Response
for time.Now().Before(deadline) {
resp, err = http.Get("http://" + addr + "/healthz")
if err == nil {
break
}
time.Sleep(20 * time.Millisecond)
}
if err != nil {
t.Fatalf("GET /healthz: %v", err)
}
if resp.StatusCode != http.StatusOK {
t.Errorf("/healthz status = %d, want 200", resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if !strings.HasPrefix(string(body), "ok") {
t.Errorf("/healthz body = %q, want \"ok\"", string(body))
}
resp2, err := http.Get("http://" + addr + "/metrics")
if err != nil {
t.Fatalf("GET /metrics: %v", err)
}
defer resp2.Body.Close()
if resp2.StatusCode != http.StatusOK {
t.Errorf("/metrics status = %d, want 200", resp2.StatusCode)
}
mbody, _ := io.ReadAll(resp2.Body)
ms := string(mbody)
for _, name := range []string{
"txns_applied_total",
"txns_drifted_total",
"drifts_remediated_total",
"peers_total",
"nodes_total",
"allocs_total",
} {
if !strings.Contains(ms, name) {
t.Errorf("/metrics missing %q\n---\n%s", name, ms)
}
}
cancel()
select {
case <-errCh:
case <-time.After(3 * time.Second):
t.Fatal("metrics command did not stop after cancel")
}
}
var _ = os.Stdin
internal/transport/metrics.go
+25 -6
View File
@@ -17,18 +17,37 @@ var metricOrder = []string{
"txns_applied_total",
"txns_drifted_total",
"drifts_remediated_total",
"drifts_remediated_total",
"peers_total",
"nodes_total",
"allocs_total",
"orca_jobs_running",
"orca_jobs_failed",
"orca_jobs_complete",
"orca_audit_chain_head",
"orca_drift_events_total",
"orca_ssh_errors_total",
"orca_txn_apply_total",
"orca_txn_rollback_total",
"orca_acl_denials_total",
}
var metricMeta = map[string]metricDef{
"txns_applied_total": {"Total transactions applied", "counter"},
"txns_drifted_total": {"Total transactions drifted", "counter"},
"drifts_remediated_total": {"Total drifts remediated", "counter"},
"peers_total": {"Current peer count", "gauge"},
"nodes_total": {"Current node count", "gauge"},
"allocs_total": {"Current allocation count", "gauge"},
"txns_applied_total": {"Total transactions applied", "counter"},
"txns_drifted_total": {"Total transactions drifted", "counter"},
"drifts_remediated_total": {"Total drifts remediated", "counter"},
"peers_total": {"Current peer count", "gauge"},
"nodes_total": {"Current node count", "gauge"},
"allocs_total": {"Current allocation count", "gauge"},
"orca_jobs_running": {"Jobs currently running", "gauge"},
"orca_jobs_failed": {"Jobs that failed", "gauge"},
"orca_jobs_complete": {"Jobs completed successfully", "gauge"},
"orca_audit_chain_head": {"Audit chain integrity (1=verified)", "gauge"},
"orca_drift_events_total": {"Total drift events detected", "counter"},
"orca_ssh_errors_total": {"Total SSH errors", "counter"},
"orca_txn_apply_total": {"Total transaction applies", "counter"},
"orca_txn_rollback_total": {"Total transaction rollbacks", "counter"},
"orca_acl_denials_total": {"Total ACL denials (enforce mode)", "counter"},
}
type Metrics struct {