diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 0000000..4cc6928 --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "dotnet-stryker": { + "version": "4.15.0", + "commands": [ + "dotnet-stryker" + ], + "rollForward": false + } + } +} \ No newline at end of file diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 6774463..e090e7f 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -43,6 +43,26 @@ jobs: dotnet-version: '10.0.x' - run: make unit + mutation: + runs-on: ubuntu-latest + steps: + - uses: https://github.com/actions/checkout@v4 + - uses: https://github.com/actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - run: make mutation + # Publish the Stryker HTML report. `if: always()` uploads it even when the + # ratchet fails — that is exactly when you want to inspect the survivors. + # Glob handles Stryker's non-deterministic StrykerOutput// dir. + # Pinned to @v3 deliberately: @v4 refuses to run on Gitea (GHES guard) — + # see docs/runbooks/gitea-actions-gotchas.md §4. + - uses: https://github.com/actions/upload-artifact@v3 + if: always() + with: + name: acl-mutation-report + path: services/acl/StrykerOutput/**/reports/mutation-report.html + if-no-files-found: warn + compose-smoke: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 91aea34..1053420 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ coverage*.json coverage*.xml *.coverage +# Stryker.NET mutation-testing reports (regenerated by `make mutation`) +StrykerOutput/ + # Rider / VS / VS Code .idea/ .vs/ diff --git a/Makefile b/Makefile index 403475b..50a6049 100644 --- a/Makefile +++ b/Makefile @@ -43,10 +43,10 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit smoke up down local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation smoke up down local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help -## ci: run the full pipeline — lint, build, unit, smoke (mirrors Gitea Actions) -ci: lint build unit smoke +## ci: run the full pipeline — lint, build, unit, mutation, smoke (mirrors Gitea Actions) +ci: lint build unit mutation smoke ## lint: verify formatting (no changes) lint: @@ -60,6 +60,15 @@ build: unit: dotnet test $(SLN) -c Release +## mutation: run the Stryker.NET ratchet on the ACL (fails below the recorded baseline) +# Stryker is pinned as a local dotnet tool (.config/dotnet-tools.json); `tool restore` +# makes `make mutation` work from a fresh clone. Config + break threshold (the ratchet, +# CLAUDE.md §5) live in services/acl/stryker-config.json. The ACL is the first service +# with branching logic, so it sets the repo-wide baseline; later slices ratchet it up. +mutation: + dotnet tool restore + cd services/acl && dotnet stryker + ## smoke: seed config, bring the whole stack up, wait for health-checked services, tear down # SEED populates the external config volumes first (upstream images used verbatim; # only our acl/bff are built). `up -d --build` starts EVERYTHING. Readiness is diff --git a/docs/architecture/adr-0005-mutation-testing.md b/docs/architecture/adr-0005-mutation-testing.md new file mode 100644 index 0000000..23f11e9 --- /dev/null +++ b/docs/architecture/adr-0005-mutation-testing.md @@ -0,0 +1,68 @@ +# ADR-0005: Stryker.NET for mutation testing, baseline on the ACL + +- **Status:** Accepted +- **Date:** 2026-06-25 +- **Deciders:** Respellion engineering +- **Relates to:** S-04b (#47); proposed in #51; supports CLAUDE.md §5 (mutation ratchet) and §3 (Definition of Done) + +## Context + +CLAUDE.md §5 mandates Stryker on every PR with a **ratchet**: CI fails on a regression +below the established baseline, and the baseline only ever moves up. §3 lists "mutation +(ratchet)" as a Definition-of-Done gate for **every** slice. Yet no baseline existed — so, +strictly, no slice could satisfy that gate. S-04b establishes it. + +The ACL is the natural place to set the first baseline: it is the first service with real +branching logic — `OpenZaakGateway` (HTTP contract, geo CRS headers, error handling), +`ZgwToken` (HS256 JWT minting), and the `AclService` default-fill mapping. We need a tool +that: + +- mutates C# and runs the existing xUnit suite per mutant, +- is reproducible (same version locally and in CI, no global install), +- understands this repo's `.slnx` solution format (used repo-wide), +- emits a break threshold CI can gate on. + +## Decision + +**Use [Stryker.NET](https://stryker-mutator.io/docs/stryker-net/) (`dotnet-stryker`), +pinned as a local dotnet tool**, configured in solution mode against `Acl.slnx`. + +- Pinned in `.config/dotnet-tools.json` (v4.15.0); `dotnet tool restore` makes + `make mutation` reproducible from a fresh clone, locally and in CI — no global install. +- **Solution mode** (`stryker-config.json` → `solution: Acl.slnx`) mutates the two projects + under test (`Acl.Application`, `Acl.Infrastructure`); `Acl.Api` is untested and skipped. + Stryker 4.15 reads `.slnx` directly, so no throwaway `.sln` shim is needed. +- A `mutation` make target runs it; it is wired into `make ci` and a parallel Gitea Actions + `mutation` job, keeping `make ci` an exact mirror of the pipeline. + +**Baseline:** writing S-04b's tests surfaced that the ACL suite was thin — the initial +score was **35%** (survivors: unasserted CRS headers, null guards, error paths, and JWT +claims). Those tests were strengthened (killing the mutants honestly rather than lowering +the bar), raising the score to **95%**. The enforced `break` threshold is set to **90%** — +one-mutant headroom over the ~20-mutant surface, since a single mutant is ≈5%. + +## Consequences + +- **Positive:** test *strength* is gated, not just coverage; the ratchet protects the ACL's + ZGW contract logic; the baseline is repo-wide and ratchets upward per §5. +- **Cost:** a new dependency (`dotnet-stryker`) and a slower CI job than unit tests (~25 s on + the small ACL). Pinned + tool-restored, so reproducible. +- **One accepted survivor:** a mutation of the empty-response *exception message string*. + Asserting exception message text is brittle and the behaviour (type + control flow) is + unchanged — treated as an equivalent mutant, not a test gap. +- **Commitment:** later slices ratchet the threshold up deliberately, never down (§5). New + services add their own mutation run as they gain branching logic (BFF, Domain, …). +- **Replaceable by:** no realistic .NET alternative — Stryker.NET is the tool §5 already + names; the fallback is no mutation testing, which §5 forbids. + +## Alternatives considered + +- **Global `dotnet tool install -g`** — rejected: not reproducible/pinned per clone; the + local manifest gives every checkout and the CI runner the same version. +- **Mutate the whole `register-referentie.slnx`** — rejected for this slice: scopes the + baseline to services with no logic yet (BFF skeleton), diluting the signal. Each service + opts in as it gains logic. +- **Application-only scope** — rejected: would leave `Acl.Infrastructure`'s HTTP/JWT logic — + the riskiest code — unguarded by the ratchet. +- **Coverage gate instead of mutation** — rejected: line coverage does not measure whether + tests would *catch* a regression; that is the whole point of §5. diff --git a/docs/runbooks/ci.md b/docs/runbooks/ci.md index 704a6c8..155fe30 100644 --- a/docs/runbooks/ci.md +++ b/docs/runbooks/ci.md @@ -16,6 +16,7 @@ and CI cannot drift: | `lint` | `make lint` → `dotnet format … --verify-no-changes` | .NET 10 SDK | | `build` | `make build` → `dotnet build … -c Release` | .NET 10 SDK | | `unit` | `make unit` → `dotnet test … -c Release` | .NET 10 SDK | +| `mutation` | `make mutation` → `dotnet tool restore` → `dotnet stryker` (ACL); uploads the HTML report as an artifact | .NET 10 SDK | | `compose-smoke` | `make smoke` → seed config volumes → `up -d` (full stack) → `up --wait` durable services → `down` | container engine + compose v2 | All `uses:` references are absolute, tag-pinned URLs (`https://github.com/actions/checkout@v4`, @@ -30,6 +31,38 @@ Actions resolves them from GitHub. > bare `docker compose up` no longer self-seeds; use `make up`. See > [gitea-actions-gotchas.md](gitea-actions-gotchas.md). +## Mutation testing (the ratchet) + +The `mutation` job enforces test *strength*, not just coverage (CLAUDE.md §5). +[Stryker.NET](https://stryker-mutator.io/docs/stryker-net/) is pinned as a local +dotnet tool (`.config/dotnet-tools.json`), so it runs identically locally and in CI: + +```bash +make mutation # dotnet tool restore + dotnet stryker on the ACL +``` + +Config lives in [`services/acl/stryker-config.json`](../../services/acl/stryker-config.json). +It runs in **solution mode** against `Acl.slnx`, mutating the two projects under test +(`Acl.Application`, `Acl.Infrastructure`); `Acl.Api` has no tests and is skipped. + +**Baseline (the ratchet):** the ACL is the first service with branching logic, so it +sets the repo-wide baseline. Observed score **95%**; enforced `break` threshold **90%** +(one-mutant headroom over the ~20-mutant surface). Stryker exits non-zero — failing the +job — when the score drops below `break`. Per §5 the baseline only moves **up**, and only +as a slice's stated outcome; never lower it. New services add their own mutation run as +they gain logic. + +The HTML report is written to `services/acl/StrykerOutput//reports/` (git-ignored); +open it to see survived vs. killed mutants. + +In CI the `mutation` job publishes that report as the **`acl-mutation-report`** artifact +(download it from the run's summary page). The upload step uses `if: always()`, so the +report is available even when the ratchet *fails* — which is exactly when you want to inspect +the survivors. It is the repo's first use of `actions/upload-artifact`, pinned to **`@v3`**: +`@v4` refuses to run on Gitea (its `@actions/artifact` v2 library blocks any non-github.com +server as "GHES"), while `@v3` speaks the artifact protocol Gitea implements. See +[gitea-actions-gotchas.md §4](gitea-actions-gotchas.md) (§15). + ## Running the stack locally without `make` (Windows / Docker Desktop) `make` and the bash helpers assume a Unix shell. To bring the whole stack up on a @@ -55,8 +88,9 @@ the containerized CI runner). Keep the two files in sync. Until the runner exists, run the full pipeline yourself before pushing: ```bash -make ci # lint + build + unit + smoke — what the pipeline runs +make ci # lint + build + unit + mutation + smoke — what the pipeline runs make lint # or a single stage +make mutation # Stryker.NET ratchet on the ACL make smoke # compose up --wait, curl /health, tear down ``` diff --git a/docs/runbooks/gitea-actions-gotchas.md b/docs/runbooks/gitea-actions-gotchas.md index 52da86d..0c7408b 100644 --- a/docs/runbooks/gitea-actions-gotchas.md +++ b/docs/runbooks/gitea-actions-gotchas.md @@ -13,6 +13,7 @@ those containers share a filesystem — or a `localhost` — breaks. | Bind-mounted config arrives empty | `docker cp` config into external volumes | `infra/seed-config.sh` | | `docker compose up --wait` is unsupported / flaky | poll health with `docker inspect` | `infra/wait-healthy.sh` | | `pg_isready` passes before PostGIS is ready | add a `PostGIS_Version()` probe | the db healthchecks | +| `upload-artifact@v4` fails ("not supported on GHES") | pin `@v3` | `.gitea/workflows/ci.yaml` (`mutation` job) | --- @@ -98,3 +99,28 @@ plus app start. container that starts migrating in that window can fail on a missing PostGIS. So the db healthchecks add a `SELECT PostGIS_Version()` probe, making dependents wait for the extension, not just the port. + +--- + +## 4. `actions/upload-artifact@v4` refuses to run on Gitea + +**Symptom** — the `mutation` job's `make mutation` step passes (95% score), but the +upload step right after it fails the job: + +``` +::error::@actions/artifact v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ +are not currently supported on GHES. +❌ Failure - Main https://github.com/actions/upload-artifact@v4 +``` + +**Why** — `upload-artifact@v4` bundles `@actions/artifact` v2, which inspects the +server URL and **hard-aborts on anything that isn't `github.com`**, treating Gitea +as an unsupported GitHub Enterprise Server. The check fires regardless of whether +the Gitea server can actually store artifacts (1.24+ can). It is the *action*, not +the server, that refuses. + +**Fix** — pin **`actions/upload-artifact@v3`** (and `download-artifact@v3` if ever +needed). v3 uses the older artifact protocol that Gitea implements, and has no GHES +guard. Inputs are the same (`name`, `path`, `if-no-files-found`), so it is a drop-in +swap. Do **not** bump to `@v4` until act_runner advertises github.com-compatible +artifact support. diff --git a/mkdocs.yml b/mkdocs.yml index 92e4c5b..33f4935 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,6 +26,7 @@ nav: - "ADR-0002: Catalogus design": architecture/adr-0002-catalogus-design.md - "ADR-0003: ACL default-fill": architecture/adr-0003-default-fill.md - "ADR-0004: BDD framework": architecture/adr-0004-bdd-framework.md + - "ADR-0005: Mutation testing": architecture/adr-0005-mutation-testing.md - Working in Gitea: gitea-workflow.md - Runbooks: - CI: runbooks/ci.md diff --git a/services/acl/Acl.Tests/AclServiceTests.cs b/services/acl/Acl.Tests/AclServiceTests.cs index e1348e1..d1bc38b 100644 --- a/services/acl/Acl.Tests/AclServiceTests.cs +++ b/services/acl/Acl.Tests/AclServiceTests.cs @@ -44,4 +44,21 @@ public class AclServiceTests Assert.Equal(defaults.ZaaktypeUrl, req.Zaaktype); Assert.Equal(new DateOnly(2026, 6, 4), req.Startdatum); } + + [Fact] + public async Task Rejects_a_null_registration_without_calling_the_gateway() + { + var gateway = new FakeGateway(); + var defaults = new AclDefaults + { + Bronorganisatie = "517439943", + VerantwoordelijkeOrganisatie = "517439943", + Vertrouwelijkheidaanduiding = "openbaar", + ZaaktypeUrl = new("http://openzaak/catalogi/api/v1/zaaktypen/big"), + }; + var service = new AclService(gateway, defaults, new FixedClock(new DateOnly(2026, 6, 4))); + + await Assert.ThrowsAsync(() => service.OpenZaakAsync(null!)); + Assert.Null(gateway.Captured); + } } diff --git a/services/acl/Acl.Tests/OpenZaakGatewayTests.cs b/services/acl/Acl.Tests/OpenZaakGatewayTests.cs index b6aa129..5110660 100644 --- a/services/acl/Acl.Tests/OpenZaakGatewayTests.cs +++ b/services/acl/Acl.Tests/OpenZaakGatewayTests.cs @@ -1,5 +1,7 @@ using System.Net; using System.Net.Http.Json; +using System.Text; +using System.Text.Json; using Acl.Application; using Acl.Infrastructure; @@ -14,38 +16,120 @@ public class OpenZaakGatewayTests => onSend(request); } - [Fact] - public async Task Posts_zaak_to_openzaak_with_bearer_and_default_fields_and_returns_url() + private static OpenZaakGateway Gateway(StubHandler handler) => new( + new HttpClient(handler), + new OpenZaakOptions { BaseUrl = new("http://openzaak"), ClientId = "cid", Secret = "sec" }); + + private static ZaakRequest SampleRequest() => new( + "517439943", "517439943", "openbaar", + new("http://openzaak/catalogi/api/v1/zaaktypen/big"), new DateOnly(2026, 6, 4)); + + private static StubHandler Created(out RequestCapture capture) { - HttpRequestMessage? seen = null; - string? body = null; - var handler = new StubHandler(async req => + var c = new RequestCapture(); + capture = c; + return new StubHandler(async req => { - seen = req; - body = await req.Content!.ReadAsStringAsync(); + c.Seen = req; + c.Body = req.Content is null ? null : await req.Content.ReadAsStringAsync(); return new HttpResponseMessage(HttpStatusCode.Created) { Content = JsonContent.Create(new { url = "http://openzaak/zaken/api/v1/zaken/xyz" }), }; }); - var gateway = new OpenZaakGateway( - new HttpClient(handler), - new OpenZaakOptions { BaseUrl = new("http://openzaak"), ClientId = "cid", Secret = "sec" }); - var request = new ZaakRequest( - "517439943", "517439943", "openbaar", - new("http://openzaak/catalogi/api/v1/zaaktypen/big"), new DateOnly(2026, 6, 4)); + } - var url = await gateway.OpenZaakAsync(request); + private sealed class RequestCapture + { + public HttpRequestMessage? Seen; + public string? Body; + } + + [Fact] + public async Task Posts_zaak_to_openzaak_with_bearer_and_default_fields_and_returns_url() + { + var handler = Created(out var capture); + + var url = await Gateway(handler).OpenZaakAsync(SampleRequest()); Assert.Equal("http://openzaak/zaken/api/v1/zaken/xyz", url.ToString()); - Assert.Equal(HttpMethod.Post, seen!.Method); - Assert.Equal("http://openzaak/zaken/api/v1/zaken", seen.RequestUri!.ToString()); - Assert.Equal("Bearer", seen.Headers.Authorization!.Scheme); - Assert.False(string.IsNullOrWhiteSpace(seen.Headers.Authorization.Parameter)); - Assert.Contains("\"bronorganisatie\":\"517439943\"", body); - Assert.Contains("\"verantwoordelijkeOrganisatie\":\"517439943\"", body); - Assert.Contains("\"vertrouwelijkheidaanduiding\":\"openbaar\"", body); - Assert.Contains("\"startdatum\":\"2026-06-04\"", body); - Assert.Contains("\"zaaktype\":\"http://openzaak/catalogi/api/v1/zaaktypen/big\"", body); + Assert.Equal(HttpMethod.Post, capture.Seen!.Method); + Assert.Equal("http://openzaak/zaken/api/v1/zaken", capture.Seen.RequestUri!.ToString()); + Assert.Equal("Bearer", capture.Seen.Headers.Authorization!.Scheme); + Assert.False(string.IsNullOrWhiteSpace(capture.Seen.Headers.Authorization.Parameter)); + Assert.Contains("\"bronorganisatie\":\"517439943\"", capture.Body); + Assert.Contains("\"verantwoordelijkeOrganisatie\":\"517439943\"", capture.Body); + Assert.Contains("\"vertrouwelijkheidaanduiding\":\"openbaar\"", capture.Body); + Assert.Contains("\"startdatum\":\"2026-06-04\"", capture.Body); + Assert.Contains("\"zaaktype\":\"http://openzaak/catalogi/api/v1/zaaktypen/big\"", capture.Body); + } + + [Fact] + public async Task Sends_the_geo_crs_headers_required_by_the_zaken_api() + { + var handler = Created(out var capture); + + await Gateway(handler).OpenZaakAsync(SampleRequest()); + + Assert.Equal("EPSG:4326", Assert.Single(capture.Seen!.Headers.GetValues("Accept-Crs"))); + Assert.Equal("EPSG:4326", Assert.Single(capture.Seen.Content!.Headers.GetValues("Content-Crs"))); + } + + [Fact] + public async Task Mints_a_hs256_jwt_carrying_the_acl_identity_claims() + { + var handler = Created(out var capture); + + await Gateway(handler).OpenZaakAsync(SampleRequest()); + + var parts = capture.Seen!.Headers.Authorization!.Parameter!.Split('.'); + Assert.Equal(3, parts.Length); + using var header = JsonDocument.Parse(DecodeSegment(parts[0])); + Assert.Equal("HS256", header.RootElement.GetProperty("alg").GetString()); + Assert.Equal("JWT", header.RootElement.GetProperty("typ").GetString()); + using var payload = JsonDocument.Parse(DecodeSegment(parts[1])); + Assert.Equal("cid", payload.RootElement.GetProperty("client_id").GetString()); + Assert.Equal("acl", payload.RootElement.GetProperty("user_id").GetString()); + Assert.Equal("acl", payload.RootElement.GetProperty("user_representation").GetString()); + } + + [Fact] + public async Task Throws_when_openzaak_rejects_the_request() + { + var handler = new StubHandler(_ => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest))); + + await Assert.ThrowsAsync( + () => Gateway(handler).OpenZaakAsync(SampleRequest())); + } + + [Fact] + public async Task Throws_when_openzaak_returns_an_empty_body() + { + var handler = new StubHandler(_ => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.Created) + { + Content = new StringContent("null", Encoding.UTF8, "application/json"), + })); + + await Assert.ThrowsAsync( + () => Gateway(handler).OpenZaakAsync(SampleRequest())); + } + + [Fact] + public async Task Rejects_a_null_request() + { + var handler = new StubHandler(_ => throw new InvalidOperationException("should not be sent")); + + await Assert.ThrowsAsync( + () => Gateway(handler).OpenZaakAsync(null!)); + } + + // ZGW tokens are base64url with padding stripped (ZgwToken.B64Url); restore it to decode. + private static string DecodeSegment(string segment) + { + var b64 = segment.Replace('-', '+').Replace('_', '/'); + b64 = (b64.Length % 4) switch { 2 => b64 + "==", 3 => b64 + "=", _ => b64 }; + return Encoding.UTF8.GetString(Convert.FromBase64String(b64)); } } diff --git a/services/acl/stryker-config.json b/services/acl/stryker-config.json new file mode 100644 index 0000000..4321e06 --- /dev/null +++ b/services/acl/stryker-config.json @@ -0,0 +1,11 @@ +{ + "stryker-config": { + "solution": "Acl.slnx", + "reporters": ["progress", "html"], + "thresholds": { + "high": 95, + "low": 90, + "break": 90 + } + } +}