#!/usr/bin/env bash
#
# S-16a (#122): assert the observability backplane is live against the running stack.
# Not just "the containers started" — it asks Grafana to health-check its provisioned
# Tempo and Prometheus datasources, which proves Grafana can actually reach both over
# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana.
#
# Usage: run-observability-check.sh   (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT)
set -euo pipefail

GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
AUTH="${GRAFANA_AUTH:-admin:admin}"
TIMEOUT="${OBS_TIMEOUT:-60}"

# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
# doesn't implement the datasource /health method, so instead of the Prometheus-
# style health probe we prove reachability through Grafana's datasource proxy.
poll() {
  local desc="$1" url="$2" expr="$3" got deadline
  deadline=$(( $(date +%s) + TIMEOUT ))
  while :; do
    got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
      | python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
    [ "$got" = "True" ] && { echo "  ✓ $desc"; return 0; }
    if [ "$(date +%s)" -ge "$deadline" ]; then
      echo "  ✗ $desc — check failed ($url)" >&2
      return 1
    fi
    sleep 2
  done
}

echo "Checking observability backplane at $GRAFANA ..."
poll "Grafana is healthy" \
  "$GRAFANA/api/health" "d['database'] == 'ok'"
poll "Prometheus datasource reachable" \
  "$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
poll "Tempo datasource reachable (via Grafana proxy)" \
  "$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
echo "Observability backplane OK."