diff --git a/infra/run-observability-check.sh b/infra/run-observability-check.sh index a8d01fa..b340f64 100755 --- a/infra/run-observability-check.sh +++ b/infra/run-observability-check.sh @@ -1,41 +1,45 @@ #!/usr/bin/env bash # -# S-16a (#122): assert the observability backplane is live against the running stack. -# Not just "the containers started" — it asks Grafana to health-check its provisioned -# Tempo and Prometheus datasources, which proves Grafana can actually reach both over -# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana. +# S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING +# stack. Runs curl INSIDE the compose network (like the other verify checks) because +# the stack's published ports aren't on the CI runner's localhost — the stack is a set +# of sibling containers on the host daemon. It asks Grafana to reach its provisioned +# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's +# Grafana plugin implements no health method) — so it proves the datasources are wired, +# not merely that the containers started. Polls, so it tolerates a cold Grafana. # -# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT) +# Does NOT manage the stack lifecycle (the caller owns bring-up + teardown). set -euo pipefail -GRAFANA="${GRAFANA_URL:-http://localhost:3000}" -AUTH="${GRAFANA_AUTH:-admin:admin}" TIMEOUT="${OBS_TIMEOUT:-60}" +AUTH="${GRAFANA_AUTH:-admin:admin}" -# poll -# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin -# doesn't implement the datasource /health method, so instead of the Prometheus- -# style health probe we prove reachability through Grafana's datasource proxy. +gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)" +[ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)" +gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")" +base="http://$gf_ip:3000" +echo ">> grafana=$gf_ip network=$net" + +# Run curl inside a throwaway container on the stack network (reaches services by IP). +net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; } + +# poll poll() { - local desc="$1" url="$2" expr="$3" got deadline - deadline=$(( $(date +%s) + TIMEOUT )) + local desc="$1" pat="$2"; shift 2 + local deadline=$(( $(date +%s) + TIMEOUT )) while :; do - got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \ - | python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)" - [ "$got" = "True" ] && { echo " ✓ $desc"; return 0; } - if [ "$(date +%s)" -ge "$deadline" ]; then - echo " ✗ $desc — check failed ($url)" >&2 - return 1 - fi - sleep 2 + if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo " ✓ $desc"; return 0; fi + if [ "$(date +%s)" -ge "$deadline" ]; then echo " ✗ $desc ($*)" >&2; return 1; fi + sleep 3 done } -echo "Checking observability backplane at $GRAFANA ..." +echo "Checking observability backplane at $base ..." poll "Grafana is healthy" \ - "$GRAFANA/api/health" "d['database'] == 'ok'" + '"database":[[:space:]]*"ok"' "$base/api/health" poll "Prometheus datasource reachable" \ - "$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'" + '"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health" poll "Tempo datasource reachable (via Grafana proxy)" \ - "$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))" + '"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo" echo "Observability backplane OK."