From bb17615f411f0f9aa8ac7ef8ff1d483761c2a68f Mon Sep 17 00:00:00 2001 From: Praxis CI Date: Sat, 1 Aug 2026 14:18:23 +0000 Subject: [PATCH] =?UTF-8?q?feat(P01):=20SLICE-03+04=20=E2=80=94=20PVE=20AP?= =?UTF-8?q?I=20layer=20(8=20scripts)=20+=20health-check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SLICE-03 (devops-engineer): ported from coreci/scripts/proxmox/: api.sh (verbatim), ct-exists.sh, lxc-clone.sh (4GB/16GB, hostname=praxis), lxc-config.sh (praxis env vars, praxis-firstboot.sh snippet), lxc-start.sh, stage-snippet.sh (G-101 FIX: bakes GITEA_TOKEN into snippet via sed), timing.sh (verbatim), rollback.sh (proxy code removed) SLICE-04 (devops-engineer): health-check.sh adapted for /health:8789 (G-104 FIX: timeout bumped 300s→600s for Docker build margin) REQ-DEPLOY-03, 04, 05, 07, 08 covered. ---ci--- project: praxis phase: 1 milestone: v0.2 status: execute slice: 03-04 wave: 2 ---/ci--- --- scripts/proxmox/api.sh | 175 +++++++++++++++++++++++++++++++ scripts/proxmox/ct-exists.sh | 59 +++++++++++ scripts/proxmox/health-check.sh | 70 +++++++++++++ scripts/proxmox/lxc-clone.sh | 57 ++++++++++ scripts/proxmox/lxc-config.sh | 127 ++++++++++++++++++++++ scripts/proxmox/lxc-start.sh | 32 ++++++ scripts/proxmox/rollback.sh | 58 ++++++++++ scripts/proxmox/stage-snippet.sh | 127 ++++++++++++++++++++++ scripts/proxmox/timing.sh | 102 ++++++++++++++++++ 9 files changed, 807 insertions(+) create mode 100755 scripts/proxmox/api.sh create mode 100755 scripts/proxmox/ct-exists.sh create mode 100755 scripts/proxmox/health-check.sh create mode 100755 scripts/proxmox/lxc-clone.sh create mode 100755 scripts/proxmox/lxc-config.sh create mode 100755 scripts/proxmox/lxc-start.sh create mode 100755 scripts/proxmox/rollback.sh create mode 100755 scripts/proxmox/stage-snippet.sh create mode 100755 scripts/proxmox/timing.sh diff --git a/scripts/proxmox/api.sh b/scripts/proxmox/api.sh new file mode 100755 index 0000000..ded1e83 --- /dev/null +++ b/scripts/proxmox/api.sh @@ -0,0 +1,175 @@ +#!/bin/sh +# CoreCI — Proxmox VE REST API shared helpers. +# +# Sourced by the other scripts/proxmox/*.sh scripts. Provides: +# pve_curl — authenticated curl wrapper (PVEAPIToken header, TLS opt) +# pve_poll — poll an async UPID until status == "stopped" +# pve_nextid — fetch the next free VMID +# pve_get — GET with 503 bounded retry (idempotent reads only) +# pve_env — validate required env vars are set +# +# All helpers use `set -eu` semantics (fail fast). The caller is +# expected to `set -eu` and `source` this file. + +# ── TLS handling ────────────────────────────────────────────── +# PROXMOX_TLS_SKIP_VERIFY=true → curl --insecure (self-signed certs). +# Default is false (secure; operator opts in for self-signed). +pve_tls_insecure() { + case "${PROXMOX_TLS_SKIP_VERIFY:-false}" in + true|1|yes|TRUE) echo "--insecure" ;; + *) echo "" ;; + esac +} + +# ── Auth header ──────────────────────────────────────────────── +# PVEAPIToken=USER@REALM!TOKENID=SECRET (no ticket step, no CSRF) +pve_auth_header() { + printf '%s' "PVEAPIToken=${PROXMOX_API_TOKEN:?PROXMOX_API_TOKEN is required}" +} + +# ── Core curl wrapper ────────────────────────────────────────── +# Usage: pve_curl [form-data-args...] +# Returns the raw JSON `data` field on stdout (jq -r .data). +# Exits non-zero on HTTP >= 300 or curl failure. +pve_curl() { + method="$1"; path="$2"; shift 2 + url="${PROXMOX_API_URL:?PROXMOX_API_URL is required}${path}" + insecure="$(pve_tls_insecure)" + + if [ "$#" -gt 0 ]; then + # Form-encoded body for POST/PUT (key=value pairs) + data_args="" + for pair in "$@"; do + data_args="${data_args} --data-urlencode ${pair}" + done + # shellcheck disable=SC2086 + response=$(curl -sS $insecure \ + -X "$method" \ + -H "Authorization: $(pve_auth_header)" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + $data_args \ + "$url") + else + # shellcheck disable=SC2086 + response=$(curl -sS $insecure \ + -X "$method" \ + -H "Authorization: $(pve_auth_header)" \ + "$url") + fi + + # Proxmox always wraps responses in {"data": ...}. Check for errors. + status=$(printf '%s' "$response" | jq -r '.errors // empty') + if [ -n "$status" ]; then + echo "pve_curl: API error for $method $path: $status" >&2 + printf '%s' "$response" >&2 + return 1 + fi + + printf '%s' "$response" | jq -r '.data' +} + +# ── GET with 503 bounded retry (idempotent reads only) ──────── +# IDEATE-19: transient 503s (node busy/restarting) retried 3× / 2s backoff. +# NOT used for mutating calls (clone/start/stop) — those are UPID-polled. +pve_get() { + path="$1" + url="${PROXMOX_API_URL:?}${path}" + insecure="$(pve_tls_insecure)" + attempt=0 + max=3 + while [ "$attempt" -lt "$max" ]; do + # shellcheck disable=SC2086 + response=$(curl -sS -w '\n%{http_code}' $insecure \ + -X GET \ + -H "Authorization: $(pve_auth_header)" \ + "$url") + http_code=$(printf '%s' "$response" | tail -1) + body=$(printf '%s' "$response" | sed '$d') + if [ "$http_code" = "503" ] && [ "$((attempt + 1))" -lt "$max" ]; then + attempt=$((attempt + 1)) + echo "pve_get: 503 from $path, retry $attempt/$max in 2s..." >&2 + sleep 2 + continue + fi + if [ "$http_code" != "200" ]; then + echo "pve_get: HTTP $http_code for $path" >&2 + printf '%s' "$body" >&2 + return 1 + fi + printf '%s' "$body" | jq -r '.data' + return 0 + done + # Exhausted all 503 retries. + echo "pve_get: 503 from $path after $max attempts" >&2 + return 1 +} + +# ── UPID polling ─────────────────────────────────────────────── +# Mutating Proxmox calls return a UPID string. Poll until done. +# Usage: pve_poll +# Exits non-zero if the task exitstatus != "OK". +pve_poll() { + upid="$1" + node="${PROXMOX_NODE:?PROXMOX_NODE is required}" + path="/nodes/${node}/tasks/${upid}/status" + attempt=0 + max_attempts=120 # 120 × 2s = 4 min max + while [ "$attempt" -lt "$max_attempts" ]; do + status=$(pve_curl GET "$path") + running=$(printf '%s' "$status" | jq -r '.status') + if [ "$running" = "stopped" ]; then + exitstatus=$(printf '%s' "$status" | jq -r '.exitstatus') + # "OK" is the clean success. "WARNINGS: N" is a successful + # completion with non-fatal warnings (e.g. systemd 255 + # nesting hint on CT create). Both are acceptable. + case "$exitstatus" in + OK|WARNINGS\ *) + return 0 + ;; + *) + echo "pve_poll: task $upid failed with exitstatus: $exitstatus" >&2 + return 1 + ;; + esac + fi + attempt=$((attempt + 1)) + sleep 2 + done + echo "pve_poll: timeout waiting for task $upid" >&2 + return 1 +} + +# ── Next free VMID ──────────────────────────────────────────── +pve_nextid() { + pve_curl GET "/cluster/nextid" | jq -r '. | tonumber' +} + +# ── Env validation ──────────────────────────────────────────── +# Usage: pve_env VAR1 VAR2 ... — exits 1 if any is unset/empty +pve_env() { + missing=0 + for var in "$@"; do + eval "val=\"\${${var}:-}\"" + if [ -z "$val" ]; then + echo "pve_env: $var is required but not set" >&2 + missing=1 + fi + done + return "$missing" +} + +# ── lxc.environment form-encoding helper ────────────────────── +# Proxmox PUT /config accepts repeated lxc.environment=KEY=value. +# This builds the curl data args from KEY=value pairs. +# Usage: pve_lxc_env_args KEY1=VAL1 KEY2=VAL2 ... +# Emits one "lxc.environment=KEY=VAL" token per arg, newline-separated, +# so the caller can pass each line to curl --data-urlencode. (Prior +# version concatenated all args into a single malformed blob.) +pve_lxc_env_args() { + first=1 + for pair in "$@"; do + [ "$first" -eq 0 ] && printf '\n' + printf '%s' "lxc.environment=${pair}" + first=0 + done +} \ No newline at end of file diff --git a/scripts/proxmox/ct-exists.sh b/scripts/proxmox/ct-exists.sh new file mode 100755 index 0000000..59d6619 --- /dev/null +++ b/scripts/proxmox/ct-exists.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Praxis — CT existence + running-state helpers (P16 — deploy idempotency). +# +# Sourced by the deploy orchestrator (lxc-deploy.sh) to detect an +# existing CT before clone. Idempotent re-deploy: +# - healthy + running → skip clone/config/start (exit 0 / continue) +# - exists but unhealthy → error with guidance (--recreate / --reconfigure) +# - not exists → proceed with clone (current path) +# +# These helpers wrap pve_get against GET /nodes/{node}/lxc/{vmid}/status/current. +# A 404 (CT not found) returns HTTP non-200 → pve_get exits non-zero; the +# helpers translate that into the 0/1 return codes the orchestrators branch on. +# `set -eu` is NOT used here (the caller is set -eu; this file defines +# functions that intentionally swallow non-zero pve_get returns). +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE (via api.sh) +# Functions: +# ct_exists → 0 if the CT exists (200), 1 if not (404/other) +# ct_running → 0 if the CT exists AND status == "running", +# 1 otherwise (not exists, or not running) +# ct_status → echoes the raw status string (e.g. "running", +# "stopped") on stdout; empty if not exists +# +# Source this file AFTER api.sh: +# . "${SCRIPT_DIR}/ct-exists.sh" + +# ct_exists → 0 if the CT exists, 1 if not. +# Uses pve_get against /status/current; a non-200 (404) is "not found". +# Under `set -eu` in the caller, the `|| true` prevents an exit on the +# pve_get failure path. +ct_exists() { + vmid="$1" + node="${PROXMOX_NODE:?PROXMOX_NODE is required}" + status_json=$(pve_get "/nodes/${node}/lxc/${vmid}/status/current" 2>/dev/null || true) + [ -n "$status_json" ] && [ "$status_json" != "null" ] +} + +# ct_running → 0 if the CT exists AND status == "running", else 1. +ct_running() { + vmid="$1" + node="${PROXMOX_NODE:?PROXMOX_NODE is required}" + status_json=$(pve_get "/nodes/${node}/lxc/${vmid}/status/current" 2>/dev/null || true) + if [ -z "$status_json" ] || [ "$status_json" = "null" ]; then + return 1 + fi + running=$(printf '%s' "$status_json" | jq -r '.status // empty' 2>/dev/null || true) + [ "$running" = "running" ] +} + +# ct_status → echoes the status string on stdout; empty if not exists. +ct_status() { + vmid="$1" + node="${PROXMOX_NODE:?PROXMOX_NODE is required}" + status_json=$(pve_get "/nodes/${node}/lxc/${vmid}/status/current" 2>/dev/null || true) + if [ -z "$status_json" ] || [ "$status_json" = "null" ]; then + return 0 + fi + printf '%s' "$(printf '%s' "$status_json" | jq -r '.status // empty' 2>/dev/null || true)" +} \ No newline at end of file diff --git a/scripts/proxmox/health-check.sh b/scripts/proxmox/health-check.sh new file mode 100755 index 0000000..d5c0590 --- /dev/null +++ b/scripts/proxmox/health-check.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# Praxis — Poll a deployed LXC container's /health endpoint. +# +# Adapted from coreci/scripts/proxmox/health-check.sh. +# Coreci polls /healthz:18080; praxis polls /health:8789. +# +# If PRAXIS_HEALTH_URL is set, use it directly. Otherwise, query +# the Proxmox /interfaces endpoint for the CT's bridge IP and +# construct http://:/health. +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE, +# PRAXIS_HEALTH_URL (optional override), PRAXIS_PORT (default 8789), +# PRAXIS_HEALTH_TIMEOUT (default 600 — first-boot Docker build + +# compose up may take up to 5 min; G-104 FIX bumped from 300s to +# give margin vs the 5-min worst-case build time per RESEARCH.md Q7) +# Args: $1 = VMID +# Exit: 0 if healthy within timeout, 1 otherwise + +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=api.sh disable=SC1091 +. "${SCRIPT_DIR}/api.sh" + +pve_env PROXMOX_API_URL PROXMOX_API_TOKEN PROXMOX_NODE + +vmid="${1:?usage: health-check.sh }" +http_port="${PRAXIS_PORT:-8789}" +timeout_s="${PRAXIS_HEALTH_TIMEOUT:-600}" + +# Resolve health URL +if [ -n "${PRAXIS_HEALTH_URL:-}" ]; then + health_url="${PRAXIS_HEALTH_URL}" +else + # Query the CT's network interfaces for the bridge IP. + node="${PROXMOX_NODE}" + ifaces=$(pve_get "/nodes/${node}/lxc/${vmid}/interfaces" 2>/dev/null || true) + if [ -z "$ifaces" ] || [ "$ifaces" = "null" ]; then + echo "health-check: cannot resolve bridge IP for VMID ${vmid} (set PRAXIS_HEALTH_URL)" >&2 + exit 1 + fi + # Pick the first non-loopback IPv4 address. Emit only the IP fields + # (not hwaddr — it precedes .inet/.ip in PVE's response and head -1 + # would pick the MAC — a bug fixed in coreci v3.6 P18 review). + ip=$(printf '%s' "$ifaces" | jq -r \ + '.[] | select(.name != "lo") | (.inet? // .ip? // empty)' 2>/dev/null | grep -v '^$' | head -1) + if [ -z "$ip" ] || [ "$ip" = "null" ]; then + echo "health-check: no bridge IP found for VMID ${vmid} (set PRAXIS_HEALTH_URL)" >&2 + exit 1 + fi + health_url="http://${ip}:${http_port}/health" +fi + +echo "health-check: polling ${health_url} for up to ${timeout_s}s..." >&2 +ok=0 +# shellcheck disable=SC2034 +for i in $(seq 1 "$timeout_s"); do + if curl -fsS --connect-timeout 2 "$health_url" >/dev/null 2>&1; then + ok=1 + break + fi + sleep 1 +done + +if [ "$ok" -ne 1 ]; then + echo "health-check: praxis did not become healthy within ${timeout_s}s at ${health_url}" >&2 + exit 1 +fi + +echo "health-check: praxis healthy at ${health_url}" >&2 \ No newline at end of file diff --git a/scripts/proxmox/lxc-clone.sh b/scripts/proxmox/lxc-clone.sh new file mode 100755 index 0000000..9b3beb6 --- /dev/null +++ b/scripts/proxmox/lxc-clone.sh @@ -0,0 +1,57 @@ +#!/bin/sh +# Praxis — Create a Proxmox LXC container from a template via REST API. +# +# Uses the POST /nodes/{node}/lxc endpoint with ostemplate= +# (create-from-template) instead of the storage clone endpoint. The +# clone endpoint rejects API tokens (`user != root@pam` guard), but +# the create endpoint accepts them — so this path works end-to-end +# with a PVEAPIToken. Pure REST, no SSH. +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE, +# PROXMOX_STORAGE, PROXMOX_TEMPLATE_VOLID +# Args: $1 = target VMID (from pve_nextid) +# Stdout: the new VMID (integer) +# Exit: 0 on success, 1 on failure + +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=api.sh disable=SC1091 +. "${SCRIPT_DIR}/api.sh" + +pve_env PROXMOX_API_URL PROXMOX_API_TOKEN PROXMOX_NODE \ + PROXMOX_STORAGE PROXMOX_TEMPLATE_VOLID + +newid="${1:?usage: lxc-clone.sh }" +node="${PROXMOX_NODE}" +storage="${PROXMOX_STORAGE}" +template_volid="${PROXMOX_TEMPLATE_VOLID}" + +# POST /nodes/{node}/lxc — create a CT from a template. +# Body (form-encoded): vmid, ostemplate, hostname, storage, rootfs, ... +# Returns: UPID (async task). Poll until done. +create_path="/nodes/${node}/lxc" +hostname="${PRAXIS_HOSTNAME:-praxis}" + +echo "lxc-clone: creating VMID ${newid} from ${template_volid}" >&2 +upid=$(pve_curl POST "$create_path" \ + "vmid=${newid}" \ + "ostemplate=${template_volid}" \ + "hostname=${hostname}" \ + "storage=${storage}" \ + "rootfs=${storage}:16" \ + "memory=${PROXMOX_MEMORY_MB:-4096}" \ + "net0=name=eth0,bridge=vmbr0,ip=dhcp" \ + "arch=amd64" \ + "features=nesting=1") + +if [ -z "$upid" ] || [ "$upid" = "null" ]; then + echo "lxc-clone: failed to start create (empty UPID)" >&2 + exit 1 +fi + +echo "lxc-clone: polling create task ${upid}" >&2 +pve_poll "$upid" + +echo "lxc-clone: CT ${newid} created from ${template_volid}" >&2 +printf '%s\n' "$newid" \ No newline at end of file diff --git a/scripts/proxmox/lxc-config.sh b/scripts/proxmox/lxc-config.sh new file mode 100755 index 0000000..b923df2 --- /dev/null +++ b/scripts/proxmox/lxc-config.sh @@ -0,0 +1,127 @@ +#!/bin/sh +# Praxis — Configure a created LXC container. +# +# Sets memory + onboot via the REST PUT /config (API-token-accepted), +# then sets hookscript + lxc.environment via SSH to the PVE host (these +# are root-only via REST: `hookscript` rejects API tokens, and +# `lxc.environment` is not in the REST schema). The hookscript points +# at the snippet staged by stage-snippet.sh (local:snippets/praxis- +# firstboot.sh). +# +# G-101: The GITEA_TOKEN must be available to the hookscript which runs +# on the PVE HOST (lxc.environment is NOT visible to the host-side +# hookscript). The token is baked into the snippet by stage-snippet.sh. +# The lxc.environment lines here put GITEA_TOKEN into the CT for the +# CT's own use (docker-compose env_file reads it), but the hookscript +# relies on the baked-in value. +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE, +# PRAXIS_VERSION (git clone tag/branch, default latest), +# GITEA_TOKEN (for the private repo fetch inside the CT), +# DEEPGRAM_API_KEY, CARTESIA_API_KEY, OLLAMA_API_KEY (secrets, +# may be empty in v0.2 infrastructure-only), +# PRAXIS_DB_PATH (default /app/data/praxis.db), +# PRAXIS_TTS, PRAXIS_SCENARIO (optional, with defaults), +# OLLAMA_BASE_URL, OLLAMA_CHAT_URL, OLLAMA_ROLEPLAY_MODEL, +# OLLAMA_DEBRIEF_MODEL, +# DEEPGRAM_MODEL, DEEPGRAM_LANGUAGE, DEEPGRAM_REGION, +# CARTESIA_VOICE_ID, +# PRAXIS_PORT (default 8789), +# PROXMOX_MEMORY_MB (optional, default 4096), +# PROXMOX_STORAGE (for the hookscript volid prefix), +# PROXMOX_SSH_HOST (optional; defaults to PROXMOX_NODE) +# Args: $1 = VMID +# Exit: 0 on success, 1 on failure + +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=api.sh disable=SC1091 +. "${SCRIPT_DIR}/api.sh" + +pve_env PROXMOX_API_URL PROXMOX_API_TOKEN PROXMOX_NODE + +vmid="${1:?usage: lxc-config.sh }" +node="${PROXMOX_NODE}" +memory="${PROXMOX_MEMORY_MB:-4096}" +version="${PRAXIS_VERSION:-latest}" +port="${PRAXIS_PORT:-8789}" +db_path="${PRAXIS_DB_PATH:-/app/data/praxis.db}" +storage="${PROXMOX_STORAGE:-local}" +hookscript_volid="${storage}:snippets/praxis-firstboot.sh" +ssh_host="${PROXMOX_SSH_HOST:-${node}}" + +# Optional praxis config (with defaults; empty is valid for v0.2). +praxis_tts="${PRAXIS_TTS:-deepgram}" +praxis_scenario="${PRAXIS_SCENARIO:-default}" + +# Secret keys (may be empty in v0.2 infrastructure-only slice). +deepgram_key="${DEEPGRAM_API_KEY:-}" +cartesia_key="${CARTESIA_API_KEY:-}" +ollama_key="${OLLAMA_API_KEY:-}" + +# Ollama config (with defaults). +ollama_base="${OLLAMA_BASE_URL:-http://ollama.cloudinit.dev:11434}" +ollama_chat="${OLLAMA_CHAT_URL:-${ollama_base}/v1/chat/completions}" +ollama_roleplay="${OLLAMA_ROLEPLAY_MODEL:-gemma4:cloud}" +ollama_debrief="${OLLAMA_DEBRIEF_MODEL:-deepseek-v4-flash:cloud}" + +# Deepgram config (with defaults). +deepgram_model="${DEEPGRAM_MODEL:-nova-3}" +deepgram_lang="${DEEPGRAM_LANGUAGE:-en-US}" +deepgram_region="${DEEPGRAM_REGION:-us-east-1}" + +# Cartesia config (with defaults; empty in v0.2). +cartesia_voice="${CARTESIA_VOICE_ID:-}" + +config_path="/nodes/${node}/lxc/${vmid}/config" + +echo "lxc-config: configuring VMID ${vmid} (memory=${memory}MB, onboot=1, hookscript=${hookscript_volid})" >&2 + +# Step 1: REST-accepted fields (memory, onboot). PUT /config is +# synchronous (no UPID), returns null on success. +pve_curl PUT "$config_path" "onboot=1" "memory=${memory}" + +# Step 2: root-only fields (hookscript, lxc.environment) via SSH to the +# PVE host config file. These are rejected by the REST API for API +# tokens and lxc.environment is not in the REST schema at all. +conf_file="/etc/pve/lxc/${vmid}.conf" +ssh_opts="-o StrictHostKeyChecking=no" +# Build the lines to append (remove any prior hookscript/onboot/lxc.environment +# lines first to keep the config idempotent). +append_lines() { + printf 'onboot: 1\n' + printf 'hookscript: %s\n' "$hookscript_volid" + printf 'lxc.environment: PRAXIS_VERSION=%s\n' "$version" + printf 'lxc.environment: PRAXIS_PORT=%s\n' "$port" + printf 'lxc.environment: PRAXIS_DB_PATH=%s\n' "$db_path" + printf 'lxc.environment: PRAXIS_TTS=%s\n' "$praxis_tts" + printf 'lxc.environment: PRAXIS_SCENARIO=%s\n' "$praxis_scenario" + if [ -n "${GITEA_TOKEN:-}" ]; then + printf 'lxc.environment: GITEA_TOKEN=%s\n' "$GITEA_TOKEN" + fi + printf 'lxc.environment: DEEPGRAM_API_KEY=%s\n' "$deepgram_key" + printf 'lxc.environment: CARTESIA_API_KEY=%s\n' "$cartesia_key" + printf 'lxc.environment: OLLAMA_API_KEY=%s\n' "$ollama_key" + printf 'lxc.environment: OLLAMA_BASE_URL=%s\n' "$ollama_base" + printf 'lxc.environment: OLLAMA_CHAT_URL=%s\n' "$ollama_chat" + printf 'lxc.environment: OLLAMA_ROLEPLAY_MODEL=%s\n' "$ollama_roleplay" + printf 'lxc.environment: OLLAMA_DEBRIEF_MODEL=%s\n' "$ollama_debrief" + printf 'lxc.environment: DEEPGRAM_MODEL=%s\n' "$deepgram_model" + printf 'lxc.environment: DEEPGRAM_LANGUAGE=%s\n' "$deepgram_lang" + printf 'lxc.environment: DEEPGRAM_REGION=%s\n' "$deepgram_region" + printf 'lxc.environment: CARTESIA_VOICE_ID=%s\n' "$cartesia_voice" +} +# shellcheck disable=SC2029 +# SC2029: conf='${conf_file}' intentionally expands on the client side — +# the script builds the remote /etc/pve/lxc/.conf path from the +# local variable and ships the literal path to the remote host. +append_lines | ssh "$ssh_opts" "root@${ssh_host}" " + conf='${conf_file}' + # Remove prior hookscript/onboot/lxc.environment lines. + sed -i '/^hookscript:/d;/^onboot:/d;/^lxc\.environment: PRAXIS/d;/^lxc\.environment: GITEA_TOKEN/d;/^lxc\.environment: DEEPGRAM/d;/^lxc\.environment: CARTESIA/d;/^lxc\.environment: OLLAMA/d' \"\$conf\" 2>/dev/null || true + cat >> \"\$conf\" + echo 'lxc-config: SSH config updated' >&2 +" + +echo "lxc-config: VMID ${vmid} configured" >&2 \ No newline at end of file diff --git a/scripts/proxmox/lxc-start.sh b/scripts/proxmox/lxc-start.sh new file mode 100755 index 0000000..25587b8 --- /dev/null +++ b/scripts/proxmox/lxc-start.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Praxis — Start a Proxmox LXC container and poll the async task. +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE +# Args: $1 = VMID +# Exit: 0 on success, 1 on failure + +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=api.sh disable=SC1091 +. "${SCRIPT_DIR}/api.sh" + +pve_env PROXMOX_API_URL PROXMOX_API_TOKEN PROXMOX_NODE + +vmid="${1:?usage: lxc-start.sh }" +node="${PROXMOX_NODE}" + +start_path="/nodes/${node}/lxc/${vmid}/status/start" + +echo "lxc-start: starting VMID ${vmid}" >&2 +upid=$(pve_curl POST "$start_path") + +if [ -z "$upid" ] || [ "$upid" = "null" ]; then + echo "lxc-start: failed to start (empty UPID)" >&2 + exit 1 +fi + +echo "lxc-start: polling start task ${upid}" >&2 +pve_poll "$upid" + +echo "lxc-start: VMID ${vmid} is running" >&2 \ No newline at end of file diff --git a/scripts/proxmox/rollback.sh b/scripts/proxmox/rollback.sh new file mode 100755 index 0000000..fba95d8 --- /dev/null +++ b/scripts/proxmox/rollback.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# Praxis — Rollback a failed LXC deployment. +# +# Stops (graceful, then force) and destroys the CT. Idempotent: +# a 404 (CT already gone) is not an error. +# +# Praxis v0.2 has no proxy/traefik tier, so there is no backend-route +# removal step here (unlike the coreci rollback which referenced +# PROXY_VMID and proxy/backend-remove.sh). If a proxy tier is added in +# a later slice, restore that step from coreci/scripts/proxmox/rollback.sh. +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE +# Args: $1 = VMID +# Exit: 0 on success (including already-gone), 1 on failure + +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=api.sh disable=SC1091 +. "${SCRIPT_DIR}/api.sh" + +pve_env PROXMOX_API_URL PROXMOX_API_TOKEN PROXMOX_NODE + +vmid="${1:?usage: rollback.sh }" +node="${PROXMOX_NODE}" + +echo "rollback: cleaning up VMID ${vmid}" >&2 + +# Graceful shutdown +shutdown_path="/nodes/${node}/lxc/${vmid}/status/shutdown" +upid=$(pve_curl POST "$shutdown_path" "timeoutStop=30" 2>/dev/null || true) +if [ -n "$upid" ] && [ "$upid" != "null" ]; then + pve_poll "$upid" 2>/dev/null || true +fi + +# Check if still running; force stop if so +status=$(pve_get "/nodes/${node}/lxc/${vmid}/status/current" 2>/dev/null || true) +if [ -n "$status" ] && [ "$status" != "null" ]; then + running=$(printf '%s' "$status" | jq -r '.status' 2>/dev/null || true) + if [ "$running" = "running" ]; then + echo "rollback: force-stopping VMID ${vmid}" >&2 + stop_path="/nodes/${node}/lxc/${vmid}/status/stop" + upid=$(pve_curl POST "$stop_path" 2>/dev/null || true) + if [ -n "$upid" ] && [ "$upid" != "null" ]; then + pve_poll "$upid" 2>/dev/null || true + fi + fi +fi + +# Destroy (idempotent — 404 is fine) +echo "rollback: destroying VMID ${vmid}" >&2 +destroy_path="/nodes/${node}/lxc/${vmid}" +upid=$(pve_curl DELETE "$destroy_path" 2>/dev/null || true) +if [ -n "$upid" ] && [ "$upid" != "null" ]; then + pve_poll "$upid" 2>/dev/null || true +fi + +echo "rollback: VMID ${vmid} cleaned up" >&2 \ No newline at end of file diff --git a/scripts/proxmox/stage-snippet.sh b/scripts/proxmox/stage-snippet.sh new file mode 100755 index 0000000..00b228a --- /dev/null +++ b/scripts/proxmox/stage-snippet.sh @@ -0,0 +1,127 @@ +#!/bin/sh +# Praxis — Stage the first-boot hookscript to Proxmox snippet storage. +# +# Uploads scripts/proxmox/firstboot-hook.sh to local:snippets/ via the +# Proxmox `download-url` endpoint, fetching it from the Gitea raw URL +# (the repo is private, so the token is passed in the query string — +# acceptable for an automated deploy pipeline). +# +# G-101 FIX: The hookscript runs on the PVE HOST where lxc.environment +# is NOT available. The GITEA_TOKEN (needed to clone the private repo +# during first-boot) must be BAKED INTO the snippet itself. This script: +# a) Fetches the raw firstboot-hook.sh from Gitea +# b) Uses sed to replace the ${GITEA_TOKEN} placeholder with the +# actual token value (baking the secret into the snippet) +# c) Serves the modified snippet over a local HTTP one-shot server +# so the Proxmox download-url endpoint can fetch it +# d) Polls the upload task and verifies the snippet is staged +# +# Idempotent: re-running overwrites the snippet (download-url replaces +# the file). Run this before lxc-deploy.sh creates the CT, since +# lxc-config.sh references the snippet via hookscript=. +# +# Env: PROXMOX_API_URL, PROXMOX_API_TOKEN, PROXMOX_NODE, +# PROXMOX_STORAGE, GITEA_TOKEN (for the private repo raw URL and +# to bake into the snippet — REQUIRED for G-101), +# GITEA_HOST (optional; default git.cloudinit.dev), +# PRAXIS_VERSION (optional; git ref for the raw URL, default main) +# Args: none +# Exit: 0 on success, 1 on failure + +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=api.sh disable=SC1091 +. "${SCRIPT_DIR}/api.sh" + +pve_env PROXMOX_API_URL PROXMOX_API_TOKEN PROXMOX_NODE PROXMOX_STORAGE GITEA_TOKEN + +GITEA_HOST="${GITEA_HOST:-git.cloudinit.dev}" +PRAXIS_REF="${PRAXIS_VERSION:-main}" +SNIPPET_NAME="praxis-firstboot.sh" + +# Gitea raw URL with token in the query string. Gitea accepts ?token= +# for raw file access on private repos. The repo is coreci/praxis +# (org=coreci, repo=praxis) on the same Gitea host as coreci/coreci. +RAW_URL="https://${GITEA_HOST}/coreci/praxis/raw/branch/${PRAXIS_REF}/scripts/proxmox/firstboot-hook.sh?token=${GITEA_TOKEN}" + +# Fetch the raw snippet to a temp file. +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT +raw_snippet="${tmp_dir}/${SNIPPET_NAME}" +echo "stage-snippet: fetching firstboot-hook.sh from Gitea" >&2 +insecure="$(pve_tls_insecure)" +# shellcheck disable=SC2086 +curl -sS -f $insecure -o "$raw_snippet" "$RAW_URL" + +# G-101: Bake the GITEA_TOKEN into the snippet. The hookscript runs on +# the PVE host where lxc.environment is not visible, so the token must +# be embedded in the snippet itself. The firstboot-hook.sh uses a +# literal `${GITEA_TOKEN}` placeholder that we substitute here. +# Using a sed delimiter unlikely to appear in a token (= would break on +# base64 padding; | is safe for typical token charsets). +echo "stage-snippet: baking GITEA_TOKEN into snippet (G-101 fix)" >&2 +sed -i "s|\${GITEA_TOKEN}|${GITEA_TOKEN}|g" "$raw_snippet" + +# Serve the modified snippet over a local one-shot HTTP server so the +# Proxmox download-url endpoint can fetch it. Proxmox runs on the PVE +# host; this script runs on the deploy host which may be the PVE host +# itself (loopback) or a remote box. Use a high port and bind to +# loopback; tell Proxmox to fetch from 127.0.0.1 only if this deploy +# host IS the PVE host. For the remote case, PROXMOX_DOWNLOAD_URL must +# be set to a URL the PVE host can reach this host by. +# +# Simplest robust path: use python3's http.server bound to loopback, +# run it in the background, point Proxmox at the loopback URL. This +# works when the deploy host and PVE host are the same machine (the +# common praxis case — single-node PVE). +listen_port="${STAGE_SNIPPET_PORT:-18099}" +listen_host="${STAGE_SNIPPET_HOST:-127.0.0.1}" +# The URL Proxmox will fetch from. If PROXMOX_DOWNLOAD_URL_BASE is set, +# use it (operator override for remote-deploy-host cases); otherwise +# default to the loopback URL (deploy-host == PVE-host). +download_url_base="${PROXMOX_DOWNLOAD_URL_BASE:-http://${listen_host}:${listen_port}}" +fetch_url="${download_url_base}/${SNIPPET_NAME}" + +# Start a one-shot HTTP server (serve the temp dir, then exit after one +# download). python3 is available on the PVE host by default. +( cd "$tmp_dir" && python3 -m http.server --bind "$listen_host" "$listen_port" >/dev/null 2>&1 & + http_pid=$! + # Kill the server after 60s as a safety net (download-url is fast). + ( sleep 60 && kill "$http_pid" 2>/dev/null ) & + wait "$http_pid" 2>/dev/null || true +) & +server_pid=$! +# Give the server a moment to bind. +sleep 1 + +dl_path="/nodes/${PROXMOX_NODE}/storage/${PROXMOX_STORAGE}/download-url" + +echo "stage-snippet: uploading ${SNIPPET_NAME} to ${PROXMOX_STORAGE}:snippets/ (via ${fetch_url})" >&2 +# download-url params: url=, content=snippets, filename= +upid=$(pve_curl POST "$dl_path" \ + "url=${fetch_url}" \ + "content=snippets" \ + "filename=${SNIPPET_NAME}") + +if [ -z "$upid" ] || [ "$upid" = "null" ]; then + echo "stage-snippet: failed to start download (empty UPID)" >&2 + kill "$server_pid" 2>/dev/null || true + exit 1 +fi + +echo "stage-snippet: polling upload task ${upid}" >&2 +pve_poll "$upid" + +# Stop the HTTP server (download-url is done). +kill "$server_pid" 2>/dev/null || true + +# Verify the snippet is now present in storage. +content=$(pve_get "/nodes/${PROXMOX_NODE}/storage/${PROXMOX_STORAGE}/content") +volid="${PROXMOX_STORAGE}:snippets/${SNIPPET_NAME}" +if ! printf '%s' "$content" | jq -e --arg v "$volid" '.[] | select(.volid==$v)' >/dev/null 2>&1; then + echo "stage-snippet: snippet ${volid} not found after upload" >&2 + exit 1 +fi + +echo "stage-snippet: ${volid} staged" >&2 \ No newline at end of file diff --git a/scripts/proxmox/timing.sh b/scripts/proxmox/timing.sh new file mode 100755 index 0000000..5b7a8ec --- /dev/null +++ b/scripts/proxmox/timing.sh @@ -0,0 +1,102 @@ +# CoreCI — deploy-stage timing helper (P11 — IDEATE-39). +# +# Sourced (not executed) by the deploy orchestrators +# (proxy-deploy.sh, lxc-deploy.sh) to emit structured slog-style +# JSON timing lines for each deploy stage to stderr, where a log +# aggregator (or `2>>timing.log`) can pick them up. +# +# Usage: +# . /path/to/timing.sh +# timing_start clone +# ... clone work ... +# timing_end clone +# +# Emits one JSON line per timing_end to stderr: +# {"event":"deploy_timing","stage":"clone","duration_s":3} +# +# Optional node_exporter textfile collector: if the env var +# NODE_TEXTFILE_COLLECTOR_DIR points to a writable directory, the +# latest per-stage duration is ALSO written there as +# `coreci_deploy_timing_.prom` so a node_exporter textfile +# collector scrapes it. If the dir is unset or unwritable, only the +# JSON log is emitted (the structured-log-first decision, PLAN v3.6 +# P11 Wave 2). +# +# Dependencies: date (POSIX epoch via +%s). jq is NOT required (the +# JSON line is constructed with printf so there is no external dep +# on the slow path). Idempotent: re-sourcing is harmless (the +# _TIMING_STARTS associative state is reset on source, but the +# orchestrator sources exactly once at startup). +# +# NOTE: Ported verbatim from coreci. The metric/prefix names retain +# the `coreci_` origin identifier for compatibility with existing +# node_exporter dashboards; rename in a follow-up if desired. +# +# shellcheck shell=sh + +# _TIMING_STARTS is a flat file-backed map (stage → epoch seconds). +# POSIX sh has no associative arrays, so we use a single newline- +# separated string of "stage=epoch" records and scan it. Stages are +# short identifiers (clone/config/start/health/smoke) so the linear +# scan is trivially cheap. +_TIMING_STARTS="" + +# timing_start — record the current epoch for . +# Overwrites a prior start for the same stage (idempotent re-entry). +timing_start() { + _stage="$1" + _now=$(date +%s) + # Drop any prior record for this stage, then append the fresh one. + _TIMING_STARTS="$(printf '%s\n' "$_TIMING_STARTS" \ + | while IFS= read -r _line; do + case "$_line" in + "${_stage}="*) ;; + *) [ -n "$_line" ] && printf '%s\n' "$_line" ;; + esac + done)" + _TIMING_STARTS="${_TIMING_STARTS:+${_TIMING_STARTS} +}${_stage}=${_now}" +} + +# timing_end — compute duration since timing_start , +# emit the JSON line to stderr, and optionally write the textfile +# collector entry. If no start was recorded for , emit nothing +# (defensive — a stray timing_end with no start is a no-op). +timing_end() { + _stage="$1" + _now=$(date +%s) + _start="" + # Scan the records for the matching stage. + _rest="" + while IFS= read -r _line; do + [ -n "$_line" ] || continue + case "$_line" in + "${_stage}="*) + _start="${_line#*=}" + ;; + *) + _rest="${_rest:+${_rest} +}${_line}" + ;; + esac + done <&2 + # Optional node_exporter textfile collector. + if [ -n "${NODE_TEXTFILE_COLLECTOR_DIR:-}" ] && \ + [ -d "$NODE_TEXTFILE_COLLECTOR_DIR" ] && \ + [ -w "$NODE_TEXTFILE_COLLECTOR_DIR" ]; then + _tf="${NODE_TEXTFILE_COLLECTOR_DIR}/coreci_deploy_timing_${_stage}.prom" + { + printf '# HELP coreci_deploy_timing_seconds Duration of the %s deploy stage.\n' "$_stage" + printf '# TYPE coreci_deploy_timing_seconds gauge\n' + printf 'coreci_deploy_timing_seconds{stage="%s"} %s\n' "$_stage" "$_duration" + } > "$_tf" 2>/dev/null || true + fi +} \ No newline at end of file