From dda4c58e1c3342913c31f0170b4996c5a17cb1ff Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Thu, 25 Jun 2026 10:57:52 +0200 Subject: [PATCH] fix(infra): portable health poll instead of compose --wait (refs #30) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make smoke` errored locally because podman-compose doesn't implement `docker compose up --wait` (`unrecognized arguments: --wait`). Replace the `--wait` step with infra/wait-healthy.sh, which polls each durable health-checked service ($(WAIT_SVCS)) via `docker ps` + `docker inspect '{{.State.Health.Status}}'`. This: - works on both docker compose (CI) and podman-compose (local) — only plain docker primitives, no `--wait`; - reads the in-container healthcheck, so it needs no host port access (the CI runner can't reach published ports); - ignores the one-shot init jobs, sidestepping the "--wait fails when a consumer-less one-shot exits 0" issue (flowable-init). Verified on podman-compose: wait-healthy.sh reports bff healthy (rc=0); podman exposes .State.Health.Status (starting -> healthy) and the name filter matches both `_` and `-` container naming. Docs: gitea-actions-gotchas.md updated (the two `--wait` sections folded into one "portable health poll" section). Co-Authored-By: Claude Opus 4.8 (1M context) --- Makefile | 21 ++++++------ docs/runbooks/gitea-actions-gotchas.md | 44 ++++++++++---------------- infra/wait-healthy.sh | 36 +++++++++++++++++++++ 3 files changed, 63 insertions(+), 38 deletions(-) create mode 100755 infra/wait-healthy.sh diff --git a/Makefile b/Makefile index b3908f3..b6cdec2 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,9 @@ SLN := register-referentie.slnx COMPOSE := infra/docker-compose.yml -# Long-running services with a healthcheck — the smoke waits on these. One-shot -# init jobs (oz-init, nrc-init, flowable-init) are NOT listed: `--wait` fails when -# a one-shot with no `service_completed_successfully` dependant exits (flowable-init), -# so we wait on the durable services instead. See docs/runbooks/gitea-actions-gotchas.md. +# Long-running services with a healthcheck — the smoke polls these for readiness +# (infra/wait-healthy.sh). One-shot init jobs (oz-init, nrc-init, flowable-init) +# are not polled; they only need to have run. See docs/runbooks/gitea-actions-gotchas.md. WAIT_SVCS := openzaak nrc-web acl bff # Config files (OpenZaak data.yaml, Keycloak realms, Flowable BPMN) are streamed # into external named volumes via `docker cp` (infra/seed-config.sh) instead of @@ -58,16 +57,16 @@ unit: dotnet test $(SLN) -c Release ## smoke: seed config, bring the whole stack up, wait for health-checked services, tear down -# SEED populates the external config volumes first (the upstream images are used -# verbatim — no build for them). `up -d --build` then starts EVERYTHING (building -# only our acl/bff). The second `up --wait` waits for the durable, health-checked -# services ($(WAIT_SVCS)); the one-shots are excluded because `--wait` fails when a -# one-shot with no `service_completed_successfully` dependant exits (flowable-init). -# Healthchecks run inside the containers, so no host port access is needed. +# SEED populates the external config volumes first (upstream images used verbatim; +# only our acl/bff are built). `up -d --build` starts EVERYTHING. Readiness is +# checked by infra/wait-healthy.sh polling the durable, health-checked services +# ($(WAIT_SVCS)) via `docker inspect` — portable across docker compose and +# podman-compose, and needing no `--wait` flag or host port access. The one-shots +# (oz-init, flowable-init) aren't polled; they just need to have run. smoke: $(SEED) oz kc fl docker compose -f $(COMPOSE) up -d --build - bash -c 'docker compose -f $(COMPOSE) up -d --wait --wait-timeout 420 $(WAIT_SVCS); rc=$$?; docker compose -f $(COMPOSE) down --volumes; docker volume rm -f $(CFG_VOLS) >/dev/null 2>&1; exit $$rc' + bash -c 'WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS); rc=$$?; docker compose -f $(COMPOSE) down --volumes; docker volume rm -f $(CFG_VOLS) >/dev/null 2>&1; exit $$rc' ## up: seed config volumes and start the full stack (use instead of bare ## `docker compose up`, which can't self-seed the external config volumes) diff --git a/docs/runbooks/gitea-actions-gotchas.md b/docs/runbooks/gitea-actions-gotchas.md index 42e5bc0..52a88cc 100644 --- a/docs/runbooks/gitea-actions-gotchas.md +++ b/docs/runbooks/gitea-actions-gotchas.md @@ -83,36 +83,26 @@ seeds then starts. CI uses `make smoke`, which does the same. No bind mounts of these config files remain, so the SELinux `:z`/`:Z` relabel flag is no longer needed anywhere in `infra/` (named volumes don't need relabeling). -## `--wait` fails on one-shot containers with no dependant +## Readiness: a portable health poll, not `docker compose up --wait` -`docker compose up --wait` treats a service that **exits** as a failure of the -"stay up" condition — **unless** another service depends on it with -`condition: service_completed_successfully`. Our init jobs `oz-init` and -`nrc-init` are fine (`openzaak`/`nrc-web` depend on their completion), but -`flowable-init` deploys the BPMN and exits 0 with **no dependant**, so a -whole-project `--wait` fails the moment it exits — even with everything else -healthy. The symptom is a `compose-smoke` failure whose last compose line is: +The smoke does **not** use `docker compose up --wait`, for three reasons: -``` -container infra-flowable-init-1 exited (0) -``` +- **podman-compose doesn't implement `--wait`** (`unrecognized arguments: + --wait`), so it would break local dev. +- A whole-project `--wait` **fails when a one-shot with no + `service_completed_successfully` dependant exits** — `flowable-init` deploys + the BPMN and exits 0, which `--wait` treats as the project failing (symptom: + last compose line `container infra-flowable-init-1 exited (0)`). +- The containerized CI runner **can't reach published host ports**, so an + external `curl localhost:8080/health` doesn't work either. -**Fix.** The smoke does **not** `--wait` on the whole project. It starts -everything with `up -d`, then `up -d --wait ` only for the durable, -health-checked services (`openzaak nrc-web acl bff` — see `WAIT_SVCS` in the -`Makefile`). One-shots still run (and deploy), they just don't gate `--wait`. - -This also removed the old external `curl http://localhost:8080/health` check: -the CI job runs in a container and **can't reach published host ports** at -`localhost`, and the per-service healthchecks (which run *inside* the -containers) already prove readiness, so `--wait` succeeding *is* the smoke. - -## `--wait` needs an explicit timeout - -`docker compose up --wait` defaults to a 60-second timeout in some Compose v2 -releases. A cold OpenZaak migrate alone takes ~50 s, so the smoke target passes -`--wait-timeout 300` (see `Makefile`). The 3-minute Definition-of-Done budget -still holds — this just stops `--wait` giving up before the stack is healthy. +**Fix.** `infra/wait-healthy.sh` polls each durable, health-checked service +(`openzaak nrc-web acl bff` — `WAIT_SVCS` in the `Makefile`) with `docker ps` + +`docker inspect '{{.State.Health.Status}}'`, waiting for `healthy`. That uses +only primitives both docker compose and podman-compose support, reads the +in-container healthcheck (no host port needed), and ignores the one-shots (they +only need to have run). `WAIT_TIMEOUT` (default 420 s) covers the cold +OpenZaak migrate (~90 s) plus app start. ## PostGIS readiness vs. `pg_isready` diff --git a/infra/wait-healthy.sh b/infra/wait-healthy.sh new file mode 100755 index 0000000..a054f70 --- /dev/null +++ b/infra/wait-healthy.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Wait until the named compose services report a healthy healthcheck. +# +# Portable across `docker compose` (CI) and `podman-compose` (local dev): it uses +# plain `docker ps` + `docker inspect`, so it needs neither `docker compose +# up --wait` (podman-compose doesn't implement that flag) nor host port access +# (the containerized CI runner can't reach published ports). It also sidesteps the +# `--wait`-fails-when-a-one-shot-exits issue, since we only poll long-running +# services that declare a healthcheck. See docs/runbooks/gitea-actions-gotchas.md. +# +# Usage: WAIT_TIMEOUT=420 wait-healthy.sh [ ...] +set -euo pipefail + +timeout="${WAIT_TIMEOUT:-420}" +deadline=$(( $(date +%s) + timeout )) + +# compose service name -> container id. The name filter matches both docker +# compose ("infra-openzaak-1") and podman-compose ("infra_openzaak_1") naming. +cid_for() { docker ps -aq --filter "name=$1" | head -1; } + +for svc in "$@"; do + echo "waiting for '$svc' to be healthy (timeout ${timeout}s)..." + while :; do + cid="$(cid_for "$svc")" + status="" + [ -n "$cid" ] && status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null || true)" + [ "$status" = "healthy" ] && { echo " '$svc' is healthy"; break; } + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "TIMEOUT: '$svc' not healthy (status=${status:-no-container})" >&2 + docker ps -a --filter "name=$svc" >&2 || true + exit 1 + fi + sleep 3 + done +done