Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
43c6c8e0d2 | ||
|
|
f29a8598fd | ||
|
|
f12e9ce746 |
@@ -9,12 +9,6 @@ on:
|
|||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
# Supersede stale runs: a new push to the same branch/PR cancels the previous run, so the runner's
|
|
||||||
# concurrency slots aren't spent on commits nobody is waiting for (refs #127).
|
|
||||||
concurrency:
|
|
||||||
group: ci-${{ github.workflow }}-${{ github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
# Self-hosted runner — see docs/runbooks/ci.md for the runner setup.
|
# Self-hosted runner — see docs/runbooks/ci.md for the runner setup.
|
||||||
# `uses:` are absolute, tag-pinned URLs (CLAUDE.md §8.7 / §15).
|
# `uses:` are absolute, tag-pinned URLs (CLAUDE.md §8.7 / §15).
|
||||||
|
|
||||||
@@ -135,20 +129,12 @@ jobs:
|
|||||||
path: services/bff/StrykerOutput/**/reports/mutation-report.html
|
path: services/bff/StrykerOutput/**/reports/mutation-report.html
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
|
|
||||||
# One stage for every check that needs the live stack. Booting OpenZaak once (instead
|
# One stage for every check that needs the live stack. On the single self-hosted
|
||||||
# of once per job) is the cheapest layout (issue #58). No setup-dotnet: the ACL test runs
|
# runner jobs run sequentially, so booting OpenZaak once (instead of once per job)
|
||||||
# in a built image and everything reaches services by container IP. Needs Docker + egress
|
# is the cheapest layout (issue #58). No setup-dotnet: the ACL test runs in a built
|
||||||
|
# image and everything reaches services by container IP. Needs Docker + egress
|
||||||
# (base images, nuget, selectielijst.openzaak.nl).
|
# (base images, nuget, selectielijst.openzaak.nl).
|
||||||
#
|
|
||||||
# `needs: [mutation]` is NOT a data dependency — it serialises the two memory-heavy jobs so
|
|
||||||
# they never co-schedule now the runner has capacity >1. A concurrent Stryker run + full-stack
|
|
||||||
# bring-up + Playwright browser on one host is what OOMs the e2e (commit d5e5fa2, #126). The
|
|
||||||
# light .NET/frontend jobs have no `needs`, so they still parallelise up to runner capacity.
|
|
||||||
# `if: !cancelled()` keeps verify-stack running even when the mutation ratchet fails (so we don't
|
|
||||||
# lose its signal) while still honouring run cancellation from the concurrency group above.
|
|
||||||
verify-stack:
|
verify-stack:
|
||||||
needs: [mutation]
|
|
||||||
if: ${{ !cancelled() }}
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: https://github.com/actions/checkout@v4
|
- uses: https://github.com/actions/checkout@v4
|
||||||
@@ -168,8 +154,6 @@ jobs:
|
|||||||
run: make verify-domain
|
run: make verify-domain
|
||||||
- name: BFF → Keycloak + domain + projection
|
- name: BFF → Keycloak + domain + projection
|
||||||
run: make verify-bff
|
run: make verify-bff
|
||||||
- name: Distributed traces reach Tempo (one connected trace across services)
|
|
||||||
run: TRACING_TIMEOUT=120 make verify-tracing
|
|
||||||
- name: Self-service e2e (Playwright, login → submit → success)
|
- name: Self-service e2e (Playwright, login → submit → success)
|
||||||
run: make verify-e2e
|
run: make verify-e2e
|
||||||
# Log dump must precede teardown (which removes the containers).
|
# Log dump must precede teardown (which removes the containers).
|
||||||
|
|||||||
+1
-1
@@ -260,7 +260,7 @@ Split (issue #11 closed) into two independently-demoable slices per §13 — the
|
|||||||
Split into independently deployable sub-slices (CLAUDE.md §13):
|
Split into independently deployable sub-slices (CLAUDE.md §13):
|
||||||
|
|
||||||
- **S-16a** (#122) · Observability backplane — Grafana Tempo + Prometheus + Grafana in compose, datasources auto-provisioned (ADR-0023). No collector; config baked into built images.
|
- **S-16a** (#122) · Observability backplane — Grafana Tempo + Prometheus + Grafana in compose, datasources auto-provisioned (ADR-0023). No collector; config baked into built images.
|
||||||
- **S-16b** (#123) · Distributed traces across the five .NET services (OTLP → Tempo; traceparent propagates via the typed HttpClients). Depends on S-16a. ✅
|
- **S-16b** (#123) · Distributed traces across the five .NET services (OTLP → Tempo; nginx `traceparent` passthrough). Depends on S-16a.
|
||||||
- **S-16c** (#124) · Prometheus metrics + golden-signal Grafana dashboards. Depends on S-16a.
|
- **S-16c** (#124) · Prometheus metrics + golden-signal Grafana dashboards. Depends on S-16a.
|
||||||
|
|
||||||
### S-17 · Quartz.NET scheduler — herregistratie reminder sweep ✅
|
### S-17 · Quartz.NET scheduler — herregistratie reminder sweep ✅
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK)
|
|||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
|
|
||||||
.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
|
.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
|
||||||
|
|
||||||
## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions)
|
## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions)
|
||||||
## `verify` is the live-stack stage (full stack up once → ACL + notification checks).
|
## `verify` is the live-stack stage (full stack up once → ACL + notification checks).
|
||||||
@@ -175,11 +175,6 @@ verify-e2e:
|
|||||||
verify-observability:
|
verify-observability:
|
||||||
bash infra/run-observability-check.sh
|
bash infra/run-observability-check.sh
|
||||||
|
|
||||||
## verify-tracing: assert one connected distributed trace spans the .NET services in Tempo
|
|
||||||
## (S-16b), against the already-running stack.
|
|
||||||
verify-tracing:
|
|
||||||
bash infra/run-tracing-check.sh
|
|
||||||
|
|
||||||
## verify: local mirror of the CI verify-stack job — full stack up once, all checks,
|
## verify: local mirror of the CI verify-stack job — full stack up once, all checks,
|
||||||
## tear down (always). For fast single-concern local iteration use `integration`
|
## tear down (always). For fast single-concern local iteration use `integration`
|
||||||
## (oz-only) or `verify-notifications` (oz+nrc) instead.
|
## (oz-only) or `verify-notifications` (oz+nrc) instead.
|
||||||
|
|||||||
@@ -5,33 +5,6 @@ copy-pasteable walkthrough against a local `make up` stack.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## S-16b — distributed traces across the .NET services (#123, ADR-0023)
|
|
||||||
|
|
||||||
**Outcome:** the five .NET services (BFF, Domain, ACL, projection-api, event-subscriber) now emit
|
|
||||||
OpenTelemetry traces — ASP.NET Core + `HttpClient` auto-instrumentation, exported over OTLP to Tempo.
|
|
||||||
Because every cross-service call goes through a typed `HttpClient`, the W3C `traceparent` propagates for
|
|
||||||
free, so a request is **one connected trace** across the services (bff → domain → acl → openzaak;
|
|
||||||
bff → projection-api). `/health` is filtered out. No browser-side instrumentation yet, so the trace
|
|
||||||
begins at the BFF; the async Flowable-poll boundary is a separate trace (ADR-0023).
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. Automated (a CI verify-stack step): generate BFF traffic and assert Tempo holds one trace
|
|
||||||
# spanning multiple services.
|
|
||||||
make verify-tracing # → OK — trace <id> spans ['bff', 'projection-api']
|
|
||||||
|
|
||||||
# 2. By hand: drive the stack, then explore traces in Grafana.
|
|
||||||
make up
|
|
||||||
curl -s localhost:8080/openbaar/register >/dev/null # BFF → projection-api
|
|
||||||
open http://localhost:3000 # Grafana → Explore → Tempo → Search → service.name = bff → open a trace
|
|
||||||
```
|
|
||||||
|
|
||||||
**The path:** each host wires `AddOpenTelemetry().WithTracing(AddAspNetCoreInstrumentation +
|
|
||||||
AddHttpClientInstrumentation + AddOtlpExporter)`; `OTEL_SERVICE_NAME` / `OTEL_EXPORTER_OTLP_ENDPOINT`
|
|
||||||
come from compose; spans export to **tempo:4317** and render in Grafana against the provisioned Tempo
|
|
||||||
datasource.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## S-16a — observability backplane: Tempo + Prometheus + Grafana (#122, ADR-0023)
|
## S-16a — observability backplane: Tempo + Prometheus + Grafana (#122, ADR-0023)
|
||||||
|
|
||||||
**Outcome:** the compose stack now includes a Grafana-native observability backplane — **Tempo** (OTLP
|
**Outcome:** the compose stack now includes a Grafana-native observability backplane — **Tempo** (OTLP
|
||||||
|
|||||||
@@ -296,10 +296,6 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
image: register-referentie/acl:dev
|
image: register-referentie/acl:dev
|
||||||
environment:
|
environment:
|
||||||
# OpenTelemetry traces → Tempo (S-16b, ADR-0023).
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
|
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
|
||||||
OTEL_SERVICE_NAME: acl
|
|
||||||
# Overridable so verify-domain can point the ACL at the same OpenZaak host that
|
# Overridable so verify-domain can point the ACL at the same OpenZaak host that
|
||||||
# owns the seeded zaaktype URL (host-consistent zaak creation, ADR-0009).
|
# owns the seeded zaaktype URL (host-consistent zaak creation, ADR-0009).
|
||||||
Acl__OpenZaak__BaseUrl: ${ACL_OPENZAAK_BASEURL:-http://openzaak:8000/}
|
Acl__OpenZaak__BaseUrl: ${ACL_OPENZAAK_BASEURL:-http://openzaak:8000/}
|
||||||
@@ -338,10 +334,6 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
image: register-referentie/domain:dev
|
image: register-referentie/domain:dev
|
||||||
environment:
|
environment:
|
||||||
# OpenTelemetry traces → Tempo (S-16b, ADR-0023).
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
|
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
|
||||||
OTEL_SERVICE_NAME: domain
|
|
||||||
Flowable__BaseUrl: http://flowable-rest:8080/flowable-rest/
|
Flowable__BaseUrl: http://flowable-rest:8080/flowable-rest/
|
||||||
Flowable__Username: rest-admin
|
Flowable__Username: rest-admin
|
||||||
Flowable__Password: test
|
Flowable__Password: test
|
||||||
@@ -368,10 +360,6 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
image: register-referentie/bff:dev
|
image: register-referentie/bff:dev
|
||||||
environment:
|
environment:
|
||||||
# OpenTelemetry traces → Tempo (S-16b, ADR-0023).
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
|
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
|
||||||
OTEL_SERVICE_NAME: bff
|
|
||||||
# The BFF is the portals' only backend; it validates digid tokens and fans out (ADR-0010).
|
# The BFF is the portals' only backend; it validates digid tokens and fans out (ADR-0010).
|
||||||
# Keycloak (start-dev) derives the issuer from the request host, so the BFF authority and the
|
# Keycloak (start-dev) derives the issuer from the request host, so the BFF authority and the
|
||||||
# verify token request both use keycloak:8080 to keep the issuer consistent.
|
# verify token request both use keycloak:8080 to keep the issuer consistent.
|
||||||
@@ -424,10 +412,6 @@ services:
|
|||||||
dockerfile: services/event-subscriber/Dockerfile
|
dockerfile: services/event-subscriber/Dockerfile
|
||||||
image: register-referentie/event-subscriber:dev
|
image: register-referentie/event-subscriber:dev
|
||||||
environment:
|
environment:
|
||||||
# OpenTelemetry traces → Tempo (S-16b, ADR-0023).
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
|
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
|
||||||
OTEL_SERVICE_NAME: event-subscriber
|
|
||||||
ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
|
ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
|
||||||
# The subscriber enriches the projection with each zaak's reference (identificatie) by asking
|
# The subscriber enriches the projection with each zaak's reference (identificatie) by asking
|
||||||
# the ACL — the only code allowed to read ZGW (§8.1, #78).
|
# the ACL — the only code allowed to read ZGW (§8.1, #78).
|
||||||
@@ -457,10 +441,6 @@ services:
|
|||||||
dockerfile: services/projection-api/Dockerfile
|
dockerfile: services/projection-api/Dockerfile
|
||||||
image: register-referentie/projection-api:dev
|
image: register-referentie/projection-api:dev
|
||||||
environment:
|
environment:
|
||||||
# OpenTelemetry traces → Tempo (S-16b, ADR-0023).
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
|
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
|
||||||
OTEL_SERVICE_NAME: projection-api
|
|
||||||
ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
|
ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
|
||||||
ports:
|
ports:
|
||||||
- "8120:8080"
|
- "8120:8080"
|
||||||
@@ -558,16 +538,12 @@ services:
|
|||||||
context: ./observability/tempo
|
context: ./observability/tempo
|
||||||
image: register-referentie/tempo:dev
|
image: register-referentie/tempo:dev
|
||||||
command: ["-config.file=/etc/tempo.yaml"]
|
command: ["-config.file=/etc/tempo.yaml"]
|
||||||
# Cap the backplane's footprint so it can't starve the app stack + the Playwright browser on the
|
|
||||||
# memory-tight CI runner (verify-e2e OOM history, commit d5e5fa2). Generous vs idle (~150M).
|
|
||||||
mem_limit: 400m
|
|
||||||
networks: [cg]
|
networks: [cg]
|
||||||
|
|
||||||
prometheus:
|
prometheus:
|
||||||
build:
|
build:
|
||||||
context: ./observability/prometheus
|
context: ./observability/prometheus
|
||||||
image: register-referentie/prometheus:dev
|
image: register-referentie/prometheus:dev
|
||||||
mem_limit: 400m
|
|
||||||
ports:
|
ports:
|
||||||
- "9090:9090"
|
- "9090:9090"
|
||||||
networks: [cg]
|
networks: [cg]
|
||||||
@@ -576,7 +552,6 @@ services:
|
|||||||
build:
|
build:
|
||||||
context: ./observability/grafana
|
context: ./observability/grafana
|
||||||
image: register-referentie/grafana:dev
|
image: register-referentie/grafana:dev
|
||||||
mem_limit: 512m
|
|
||||||
environment:
|
environment:
|
||||||
GF_SECURITY_ADMIN_USER: admin
|
GF_SECURITY_ADMIN_USER: admin
|
||||||
GF_SECURITY_ADMIN_PASSWORD: admin
|
GF_SECURITY_ADMIN_PASSWORD: admin
|
||||||
|
|||||||
@@ -1,45 +1,41 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
#
|
#
|
||||||
# S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING
|
# S-16a (#122): assert the observability backplane is live against the running stack.
|
||||||
# stack. Runs curl INSIDE the compose network (like the other verify checks) because
|
# Not just "the containers started" — it asks Grafana to health-check its provisioned
|
||||||
# the stack's published ports aren't on the CI runner's localhost — the stack is a set
|
# Tempo and Prometheus datasources, which proves Grafana can actually reach both over
|
||||||
# of sibling containers on the host daemon. It asks Grafana to reach its provisioned
|
# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana.
|
||||||
# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's
|
|
||||||
# Grafana plugin implements no health method) — so it proves the datasources are wired,
|
|
||||||
# not merely that the containers started. Polls, so it tolerates a cold Grafana.
|
|
||||||
#
|
#
|
||||||
# Does NOT manage the stack lifecycle (the caller owns bring-up + teardown).
|
# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT)
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
TIMEOUT="${OBS_TIMEOUT:-60}"
|
GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
|
||||||
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
||||||
|
TIMEOUT="${OBS_TIMEOUT:-60}"
|
||||||
|
|
||||||
gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)"
|
# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
|
||||||
[ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; }
|
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
|
||||||
net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)"
|
# doesn't implement the datasource /health method, so instead of the Prometheus-
|
||||||
gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")"
|
# style health probe we prove reachability through Grafana's datasource proxy.
|
||||||
base="http://$gf_ip:3000"
|
|
||||||
echo ">> grafana=$gf_ip network=$net"
|
|
||||||
|
|
||||||
# Run curl inside a throwaway container on the stack network (reaches services by IP).
|
|
||||||
net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; }
|
|
||||||
|
|
||||||
# poll <description> <grep -E pattern> <curl args...>
|
|
||||||
poll() {
|
poll() {
|
||||||
local desc="$1" pat="$2"; shift 2
|
local desc="$1" url="$2" expr="$3" got deadline
|
||||||
local deadline=$(( $(date +%s) + TIMEOUT ))
|
deadline=$(( $(date +%s) + TIMEOUT ))
|
||||||
while :; do
|
while :; do
|
||||||
if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo " ✓ $desc"; return 0; fi
|
got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
|
||||||
if [ "$(date +%s)" -ge "$deadline" ]; then echo " ✗ $desc ($*)" >&2; return 1; fi
|
| python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
|
||||||
sleep 3
|
[ "$got" = "True" ] && { echo " ✓ $desc"; return 0; }
|
||||||
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||||
|
echo " ✗ $desc — check failed ($url)" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
echo "Checking observability backplane at $base ..."
|
echo "Checking observability backplane at $GRAFANA ..."
|
||||||
poll "Grafana is healthy" \
|
poll "Grafana is healthy" \
|
||||||
'"database":[[:space:]]*"ok"' "$base/api/health"
|
"$GRAFANA/api/health" "d['database'] == 'ok'"
|
||||||
poll "Prometheus datasource reachable" \
|
poll "Prometheus datasource reachable" \
|
||||||
'"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health"
|
"$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
|
||||||
poll "Tempo datasource reachable (via Grafana proxy)" \
|
poll "Tempo datasource reachable (via Grafana proxy)" \
|
||||||
'"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo"
|
"$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
|
||||||
echo "Observability backplane OK."
|
echo "Observability backplane OK."
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
#
|
|
||||||
# S-16b (#123): assert one connected distributed trace spans the .NET services in Tempo,
|
|
||||||
# against an ALREADY-RUNNING full stack. Runs the driver in a python:3-slim container on the
|
|
||||||
# stack network (services reached by container IP; the runner can't reach published ports —
|
|
||||||
# gitea-actions-gotchas.md §5/§6). Does NOT manage the stack lifecycle.
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
|
|
||||||
ip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"; }
|
|
||||||
|
|
||||||
bff="$(docker ps -q --filter 'name=[-_]bff[-_]' | head -1)"
|
|
||||||
tempo="$(docker ps -q --filter 'name=[-_]tempo[-_]' | head -1)"
|
|
||||||
[ -n "$bff" ] && [ -n "$tempo" ] || { echo "ERROR: bff and/or tempo not running — bring the stack up first" >&2; exit 1; }
|
|
||||||
net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$bff" | head -1)"
|
|
||||||
bff_ip="$(ip "$bff")"; tempo_ip="$(ip "$tempo")"
|
|
||||||
echo ">> network=$net bff=$bff_ip tempo=$tempo_ip"
|
|
||||||
|
|
||||||
cid="$(docker create --network "$net" \
|
|
||||||
-e "BFF=http://$bff_ip:8080" -e "TEMPO=http://$tempo_ip:3200" \
|
|
||||||
-e "TRACING_TIMEOUT=${TRACING_TIMEOUT:-90}" \
|
|
||||||
python:3-slim python /tracing-check.py)"
|
|
||||||
docker cp "$here/tracing-check.py" "$cid:/tracing-check.py" >/dev/null
|
|
||||||
rc=0; docker start -a "$cid" || rc=$?
|
|
||||||
docker rm -f "$cid" >/dev/null
|
|
||||||
exit $rc
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""S-16b (#123): prove distributed tracing works end to end.
|
|
||||||
|
|
||||||
Generate anonymous BFF traffic (GET /openbaar/register, which the BFF serves by
|
|
||||||
calling projection-api — no auth, no OpenZaak egress), then query Tempo and assert
|
|
||||||
that ONE trace contains spans from both `bff` and `projection-api`. That proves the
|
|
||||||
services export OTLP to Tempo AND that the W3C traceparent propagates across the
|
|
||||||
HttpClient hop, stitching the request into a single connected trace.
|
|
||||||
|
|
||||||
Stdlib only (urllib/json) so it runs in a bare python:3-slim container in-network.
|
|
||||||
"""
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.error
|
|
||||||
import urllib.parse
|
|
||||||
import urllib.request
|
|
||||||
|
|
||||||
BFF = os.environ["BFF"] # http://<bff-ip>:8080
|
|
||||||
TEMPO = os.environ["TEMPO"] # http://<tempo-ip>:3200
|
|
||||||
TIMEOUT = int(os.environ.get("TRACING_TIMEOUT", "90"))
|
|
||||||
WANT = {"bff", "projection-api"} # the two services that must share one trace
|
|
||||||
|
|
||||||
|
|
||||||
def _get(url):
|
|
||||||
with urllib.request.urlopen(url, timeout=10) as r:
|
|
||||||
return r.read()
|
|
||||||
|
|
||||||
|
|
||||||
def generate_traffic():
|
|
||||||
# A non-2xx still produces spans; only total unreachability of the BFF is fatal.
|
|
||||||
for _ in range(3):
|
|
||||||
try:
|
|
||||||
_get(f"{BFF}/openbaar/register")
|
|
||||||
except urllib.error.HTTPError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def search_trace_ids():
|
|
||||||
q = urllib.parse.quote('{ resource.service.name = "bff" }')
|
|
||||||
try:
|
|
||||||
data = json.loads(_get(f"{TEMPO}/api/search?q={q}&limit=50"))
|
|
||||||
except Exception:
|
|
||||||
return []
|
|
||||||
return [t["traceID"] for t in data.get("traces", [])]
|
|
||||||
|
|
||||||
|
|
||||||
def services_in_trace(trace_id):
|
|
||||||
try:
|
|
||||||
data = json.loads(_get(f"{TEMPO}/api/traces/{trace_id}"))
|
|
||||||
except Exception:
|
|
||||||
return set()
|
|
||||||
names = set()
|
|
||||||
for batch in data.get("batches", []):
|
|
||||||
for attr in batch.get("resource", {}).get("attributes", []):
|
|
||||||
if attr.get("key") == "service.name":
|
|
||||||
names.add(attr.get("value", {}).get("stringValue"))
|
|
||||||
return names
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
deadline = time.time() + TIMEOUT
|
|
||||||
generate_traffic()
|
|
||||||
seen = set()
|
|
||||||
while time.time() < deadline:
|
|
||||||
for tid in search_trace_ids():
|
|
||||||
names = services_in_trace(tid)
|
|
||||||
seen |= names
|
|
||||||
if WANT.issubset(names):
|
|
||||||
print(f"OK — trace {tid} spans {sorted(names)}")
|
|
||||||
return 0
|
|
||||||
time.sleep(3)
|
|
||||||
generate_traffic()
|
|
||||||
print(f"FAIL — no single trace spanned {sorted(WANT)}; services seen: {sorted(seen)}",
|
|
||||||
file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -5,13 +5,6 @@
|
|||||||
<ProjectReference Include="..\Acl.Infrastructure\Acl.Infrastructure.csproj" />
|
<ProjectReference Include="..\Acl.Infrastructure\Acl.Infrastructure.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|||||||
@@ -1,21 +1,8 @@
|
|||||||
using Acl.Application;
|
using Acl.Application;
|
||||||
using Acl.Infrastructure;
|
using Acl.Infrastructure;
|
||||||
using OpenTelemetry.Resources;
|
|
||||||
using OpenTelemetry.Trace;
|
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and
|
|
||||||
// outgoing HttpClient calls (the ACL → OpenZaak hop), exported over OTLP to Tempo. Service name +
|
|
||||||
// OTLP endpoint come from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable.
|
|
||||||
builder.Services.AddOpenTelemetry()
|
|
||||||
.ConfigureResource(r => r.AddService(
|
|
||||||
builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
|
|
||||||
.WithTracing(tracing => tracing
|
|
||||||
.AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddOtlpExporter());
|
|
||||||
|
|
||||||
builder.Services.AddSingleton<IClock, SystemClock>();
|
builder.Services.AddSingleton<IClock, SystemClock>();
|
||||||
builder.Services.AddSingleton(sp => sp.GetRequiredService<IConfiguration>()
|
builder.Services.AddSingleton(sp => sp.GetRequiredService<IConfiguration>()
|
||||||
.GetSection("Acl:Defaults").Get<AclDefaults>()
|
.GetSection("Acl:Defaults").Get<AclDefaults>()
|
||||||
|
|||||||
@@ -10,10 +10,6 @@
|
|||||||
<!-- OIDC/JWT validation of Keycloak-issued tokens (ADR-0010) and OpenAPI generation. -->
|
<!-- OIDC/JWT validation of Keycloak-issued tokens (ADR-0010) and OpenAPI generation. -->
|
||||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.8" />
|
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.8" />
|
||||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.8" />
|
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.8" />
|
||||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -3,23 +3,9 @@ using System.Text.Json;
|
|||||||
using System.Text.Json.Serialization;
|
using System.Text.Json.Serialization;
|
||||||
using Bff.Api;
|
using Bff.Api;
|
||||||
using Microsoft.AspNetCore.Authentication.JwtBearer;
|
using Microsoft.AspNetCore.Authentication.JwtBearer;
|
||||||
using OpenTelemetry.Resources;
|
|
||||||
using OpenTelemetry.Trace;
|
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and
|
|
||||||
// outgoing HttpClient calls (BFF → Domain, BFF → projection-api), exported over OTLP to Tempo, so a
|
|
||||||
// portal request is one connected trace across the services. Service name + OTLP endpoint come from
|
|
||||||
// OTEL_* env (compose); the exporter no-ops when Tempo is unreachable. /health is filtered out.
|
|
||||||
builder.Services.AddOpenTelemetry()
|
|
||||||
.ConfigureResource(r => r.AddService(
|
|
||||||
builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
|
|
||||||
.WithTracing(tracing => tracing
|
|
||||||
.AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddOtlpExporter());
|
|
||||||
|
|
||||||
var keycloakAuthority = builder.Configuration["Keycloak:Authority"]
|
var keycloakAuthority = builder.Configuration["Keycloak:Authority"]
|
||||||
?? throw new InvalidOperationException("Missing configuration 'Keycloak:Authority'");
|
?? throw new InvalidOperationException("Missing configuration 'Keycloak:Authority'");
|
||||||
// Behandelaars authenticate against a *different* Keycloak realm (medewerker) than citizens (digid),
|
// Behandelaars authenticate against a *different* Keycloak realm (medewerker) than citizens (digid),
|
||||||
|
|||||||
@@ -6,10 +6,6 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
|
|
||||||
<PackageReference Include="Quartz.Extensions.Hosting" Version="3.18.2" />
|
<PackageReference Include="Quartz.Extensions.Hosting" Version="3.18.2" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
|||||||
@@ -1,25 +1,10 @@
|
|||||||
using Big.Application;
|
using Big.Application;
|
||||||
using Big.Domain;
|
using Big.Domain;
|
||||||
using Big.Infrastructure;
|
using Big.Infrastructure;
|
||||||
using OpenTelemetry.Resources;
|
|
||||||
using OpenTelemetry.Trace;
|
|
||||||
using Quartz;
|
using Quartz;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and
|
|
||||||
// outgoing HttpClient calls, exported over OTLP to Tempo, so a request is one connected trace across
|
|
||||||
// the services. Service name + OTLP endpoint come from OTEL_* env (compose); the exporter no-ops
|
|
||||||
// harmlessly when Tempo is unreachable (e.g. a service run standalone). /health is filtered out so
|
|
||||||
// liveness polls don't flood the traces.
|
|
||||||
builder.Services.AddOpenTelemetry()
|
|
||||||
.ConfigureResource(r => r.AddService(
|
|
||||||
builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
|
|
||||||
.WithTracing(tracing => tracing
|
|
||||||
.AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddOtlpExporter());
|
|
||||||
|
|
||||||
// Options bound from configuration (compose sets Flowable__* and Acl__* env vars).
|
// Options bound from configuration (compose sets Flowable__* and Acl__* env vars).
|
||||||
builder.Services.AddSingleton(sp => sp.GetRequiredService<IConfiguration>()
|
builder.Services.AddSingleton(sp => sp.GetRequiredService<IConfiguration>()
|
||||||
.GetSection("Flowable").Get<FlowableOptions>()
|
.GetSection("Flowable").Get<FlowableOptions>()
|
||||||
|
|||||||
@@ -5,13 +5,6 @@
|
|||||||
<ProjectReference Include="..\..\projection-api\Projection.ReadModel\Projection.ReadModel.csproj" />
|
<ProjectReference Include="..\..\projection-api\Projection.ReadModel\Projection.ReadModel.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|||||||
@@ -1,22 +1,9 @@
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using EventSubscriber.Application;
|
using EventSubscriber.Application;
|
||||||
using OpenTelemetry.Resources;
|
|
||||||
using OpenTelemetry.Trace;
|
|
||||||
using Projection.ReadModel;
|
using Projection.ReadModel;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument the incoming NRC notification callback and
|
|
||||||
// the outgoing ACL enrichment call, exported over OTLP to Tempo. Service name + OTLP endpoint come
|
|
||||||
// from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable.
|
|
||||||
builder.Services.AddOpenTelemetry()
|
|
||||||
.ConfigureResource(r => r.AddService(
|
|
||||||
builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
|
|
||||||
.WithTracing(tracing => tracing
|
|
||||||
.AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddOtlpExporter());
|
|
||||||
|
|
||||||
var connectionString = builder.Configuration.GetConnectionString("Projection")
|
var connectionString = builder.Configuration.GetConnectionString("Projection")
|
||||||
?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'");
|
?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'");
|
||||||
// The exact Authorization header value Open Notificaties sends on each abonnement callback.
|
// The exact Authorization header value Open Notificaties sends on each abonnement callback.
|
||||||
|
|||||||
@@ -1,21 +1,8 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using OpenTelemetry.Resources;
|
|
||||||
using OpenTelemetry.Trace;
|
|
||||||
using Projection.ReadModel;
|
using Projection.ReadModel;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests, exported
|
|
||||||
// over OTLP to Tempo, so a BFF → projection-api read is one connected trace. Service name + OTLP
|
|
||||||
// endpoint come from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable.
|
|
||||||
builder.Services.AddOpenTelemetry()
|
|
||||||
.ConfigureResource(r => r.AddService(
|
|
||||||
builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
|
|
||||||
.WithTracing(tracing => tracing
|
|
||||||
.AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
|
|
||||||
.AddHttpClientInstrumentation()
|
|
||||||
.AddOtlpExporter());
|
|
||||||
|
|
||||||
var connectionString = builder.Configuration.GetConnectionString("Projection")
|
var connectionString = builder.Configuration.GetConnectionString("Projection")
|
||||||
?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'");
|
?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'");
|
||||||
|
|
||||||
|
|||||||
@@ -4,13 +4,6 @@
|
|||||||
<ProjectReference Include="..\Projection.ReadModel\Projection.ReadModel.csproj" />
|
<ProjectReference Include="..\Projection.ReadModel\Projection.ReadModel.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
|
|
||||||
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
|||||||
Reference in New Issue
Block a user