diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index ea39123..2790001 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -70,6 +70,12 @@ jobs: restore-keys: | nuget-${{ runner.os }}- - run: make unit + # Job summary (#136): a per-service pass/fail table from the TRX `make unit` wrote. + - name: Unit test summary + if: always() + run: | + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || exit 0 + python3 infra/trx-summary.py TestResults >> "$GITHUB_STEP_SUMMARY" # Frontend (Nx/Angular) lane: install with pnpm, then Nx lint + test + build. frontend: @@ -84,6 +90,12 @@ jobs: node-version: '24' cache: 'pnpm' - run: make frontend + # Job summary (#136): a per-frontend (app) pass/fail table from the vitest JSON each app wrote. + - name: Frontend test summary + if: always() + run: | + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || exit 0 + python3 infra/vitest-summary.py test-output >> "$GITHUB_STEP_SUMMARY" mutation: runs-on: ubuntu-latest @@ -99,6 +111,29 @@ jobs: restore-keys: | nuget-${{ runner.os }}- - run: make mutation + # Job summary (#136): render each service's Stryker Markdown report on the run page (Gitea + # 1.27 $GITHUB_STEP_SUMMARY). `if: always()` so a ratchet break still reports โ€” and because + # `make mutation` stops at the first break, the summary also shows exactly where it stopped. + # Guarded so it no-ops on a runner/server without summary support. Strips the report's UTF-8 BOM. + - name: Mutation score summary + if: always() + run: | + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || exit 0 + { + echo "## ๐Ÿงฌ Mutation testing" + echo + for svc in acl event-subscriber domain bff; do + echo "### $svc" + echo + report=$(ls services/"$svc"/StrykerOutput/*/reports/mutation-report.md 2>/dev/null | sort | tail -1) + if [ -n "$report" ]; then + sed '1s/^\xef\xbb\xbf//' "$report" + else + echo "_No report โ€” \`make mutation\` stopped before \`$svc\` (earlier ratchet break)._" + fi + echo + done + } >> "$GITHUB_STEP_SUMMARY" # Publish the Stryker HTML reports. `if: always()` uploads them even when the # ratchet fails โ€” that is exactly when you want to inspect the survivors. # `continue-on-error` keeps the upload best-effort: the mutation *gate* is the @@ -158,26 +193,80 @@ jobs: - uses: https://github.com/actions/checkout@v4 # Bring the full stack up + wait for health โ€” this also is the DoD "compose up # reaches green health" smoke (it replaces the old compose-smoke job). + # Each check carries an `id` so the summary step below can report its per-check outcome (#136). + # A failed check skips the rest (no step `if:`), so the table shows exactly where it stopped. - name: Bring up the full stack & wait for health + id: up run: make verify-up - name: Observability backplane (Grafana + Tempo + Prometheus datasources) + id: obs run: OBS_TIMEOUT=180 make verify-observability - name: ACL โ†” OpenZaak integration tests + id: acl run: make verify-acl - name: OpenZaak โ†’ NRC notification delivery + id: nrc run: make verify-nrc - name: OpenZaak โ†’ NRC โ†’ Event Subscriber โ†’ projection-api + id: projection run: make verify-projection - name: Domain โ†’ Flowable โ†’ ACL โ†’ OpenZaak + id: domain run: make verify-domain - name: BFF โ†’ Keycloak + domain + projection + id: bff run: make verify-bff - name: Distributed traces reach Tempo (one connected trace across services) + id: tracing run: TRACING_TIMEOUT=120 make verify-tracing - name: Golden-signal metrics scraped by Prometheus (/metrics on every service) + id: metrics run: METRICS_TIMEOUT=120 make verify-metrics - name: Self-service e2e (Playwright, login โ†’ submit โ†’ success) + id: e2e run: make verify-e2e + # Job summary (#136): a pass/fail table of every live-stack check, so a red verify-stack shows + # which check failed at a glance. `if: always()` (step-level โ€” safe on runner 2.0.0, unlike the + # job-level status-function `if` of #134) so it renders even after a check fails. + - name: verify-stack check summary + if: always() + env: + UP: ${{ steps.up.outcome }} + OBS: ${{ steps.obs.outcome }} + ACL: ${{ steps.acl.outcome }} + NRC: ${{ steps.nrc.outcome }} + PROJECTION: ${{ steps.projection.outcome }} + DOMAIN: ${{ steps.domain.outcome }} + BFF: ${{ steps.bff.outcome }} + TRACING: ${{ steps.tracing.outcome }} + METRICS: ${{ steps.metrics.outcome }} + E2E: ${{ steps.e2e.outcome }} + run: | + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || exit 0 + icon() { case "$1" in success) echo "โœ…";; failure) echo "โŒ";; skipped) echo "โญ๏ธ";; cancelled) echo "๐Ÿšซ";; *) echo "โ” ${1:-โ€”}";; esac; } + { + echo "## ๐Ÿ”Œ verify-stack checks" + echo + echo "| Check | Result |" + echo "| ----- | :----: |" + echo "| Bring up + health | $(icon "$UP") |" + echo "| Observability backplane | $(icon "$OBS") |" + echo "| ACL โ†” OpenZaak | $(icon "$ACL") |" + echo "| OpenZaak โ†’ NRC | $(icon "$NRC") |" + echo "| NRC โ†’ Event Subscriber โ†’ projection | $(icon "$PROJECTION") |" + echo "| Domain โ†’ Flowable โ†’ ACL โ†’ OpenZaak | $(icon "$DOMAIN") |" + echo "| BFF โ†’ Keycloak + domain + projection | $(icon "$BFF") |" + echo "| Distributed traces (Tempo) | $(icon "$TRACING") |" + echo "| Golden-signal metrics (Prometheus) | $(icon "$METRICS") |" + echo "| Self-service e2e (Playwright) | $(icon "$E2E") |" + } >> "$GITHUB_STEP_SUMMARY" + # Job summary (#136): per-spec Playwright results, from the JSON report run-e2e-check.sh copied + # out of the e2e container. Turns a red e2e into a one-glance "which spec" instead of a log dive. + - name: e2e spec summary + if: always() + run: | + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || exit 0 + python3 infra/playwright-summary.py tests/e2e/playwright-report.json >> "$GITHUB_STEP_SUMMARY" # Log dump must precede teardown (which removes the containers). - name: Dump container logs on failure if: failure() diff --git a/.gitignore b/.gitignore index ae54b38..b8d2203 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,6 @@ tests/e2e/node_modules/ tests/e2e/test-results/ tests/e2e/playwright-report/ __pycache__/ +TestResults/ +test-output/ +tests/e2e/playwright-report.json diff --git a/Makefile b/Makefile index 80975da..69a5ade 100644 --- a/Makefile +++ b/Makefile @@ -70,8 +70,9 @@ build: dotnet build $(SLN) -c Release ## unit: run unit tests (excludes the container-backed Integration lane) +# TRX per test project (โ†’ TestResults/) feeds the CI per-service summary (#136); harmless locally. unit: - dotnet test $(SLN) -c Release --filter "Category!=Integration" + dotnet test $(SLN) -c Release --filter "Category!=Integration" --logger trx --results-directory TestResults ## mutation: run the Stryker.NET ratchet on each service with branching logic (fails below baseline) # Stryker is pinned as a local dotnet tool (.config/dotnet-tools.json); `tool restore` diff --git a/apps/behandel/project.json b/apps/behandel/project.json index 4c7a5e2..32d87f4 100644 --- a/apps/behandel/project.json +++ b/apps/behandel/project.json @@ -64,7 +64,9 @@ "test": { "executor": "@angular/build:unit-test", "options": { - "watch": false + "watch": false, + "reporters": ["default", "json"], + "outputFile": "{workspaceRoot}/test-output/{projectName}.json" } }, "serve-static": { diff --git a/apps/beheer/project.json b/apps/beheer/project.json index 890a071..d42393d 100644 --- a/apps/beheer/project.json +++ b/apps/beheer/project.json @@ -64,7 +64,9 @@ "test": { "executor": "@angular/build:unit-test", "options": { - "watch": false + "watch": false, + "reporters": ["default", "json"], + "outputFile": "{workspaceRoot}/test-output/{projectName}.json" } }, "serve-static": { diff --git a/apps/openbaar/project.json b/apps/openbaar/project.json index a85dfbd..b80161e 100644 --- a/apps/openbaar/project.json +++ b/apps/openbaar/project.json @@ -64,7 +64,9 @@ "test": { "executor": "@angular/build:unit-test", "options": { - "watch": false + "watch": false, + "reporters": ["default", "json"], + "outputFile": "{workspaceRoot}/test-output/{projectName}.json" } }, "serve-static": { diff --git a/apps/self-service/project.json b/apps/self-service/project.json index d856b4f..c4e2e38 100644 --- a/apps/self-service/project.json +++ b/apps/self-service/project.json @@ -64,7 +64,9 @@ "test": { "executor": "@angular/build:unit-test", "options": { - "watch": false + "watch": false, + "reporters": ["default", "json"], + "outputFile": "{workspaceRoot}/test-output/{projectName}.json" } }, "serve-static": { diff --git a/docs/runbooks/gitea-actions-gotchas.md b/docs/runbooks/gitea-actions-gotchas.md index 0be41d3..8ecdfc0 100644 --- a/docs/runbooks/gitea-actions-gotchas.md +++ b/docs/runbooks/gitea-actions-gotchas.md @@ -221,3 +221,27 @@ fails", prefer serialising with a `concurrency` group over `needs` + `always()`. **Also** โ€” a run already stuck this way will **not** clear itself; force-cancel it from the Actions UI (plain cancel can also stall on this version, #35782). Push the workflow fix to produce a fresh run. + +--- + +## 8. Job summaries (`$GITHUB_STEP_SUMMARY`) need Gitea โ‰ฅ1.27 + runner โ‰ฅ2.0 + +Markdown a step appends to the `$GITHUB_STEP_SUMMARY` file renders on the run page +(no artifact download). We use it for per-run reports (#136): mutation scores +(Stryker `markdown` reporter), per-service unit results (`infra/trx-summary.py` over +TRX), per-frontend results (`infra/vitest-summary.py` over each app's vitest JSON), +the verify-stack check table, and per-spec e2e results (`infra/playwright-summary.py`). + +**Requirements / conventions:** + +- Requires **Gitea โ‰ฅ 1.27** (stores/renders summaries) and **act_runner โ‰ฅ 2.0.0** + (uploads them). Older pairings silently skip the upload. +- **Guard every write:** `[ -n "${GITHUB_STEP_SUMMARY:-}" ] || exit 0` โ€” on a runner + without support the var is unset and `>> "$GITHUB_STEP_SUMMARY"` would be an + ambiguous-redirect error. The guard makes the step a no-op locally / on old runners. +- Use `if: always()` (step-level) on summary steps so they render even when the thing + they report on failed. Step-level `always()` is fine on 2.0.0 โ€” unlike the *job*-level + status-function `if` of ยง7. +- Getting a report out of the e2e container: Playwright writes `playwright-report.json` + inside the container; `infra/run-e2e-check.sh` `docker cp`s it back to the host + (capturing the test exit code first) so the summary step can read it. diff --git a/infra/playwright-summary.py b/infra/playwright-summary.py new file mode 100644 index 0000000..25e1490 --- /dev/null +++ b/infra/playwright-summary.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Render a per-spec table from a Playwright JSON report for a Gitea job summary (#136). + +Reads the JSON report (default: tests/e2e/playwright-report.json) that run-e2e-check.sh copies out +of the e2e container, and prints a markdown table (one row per spec) to stdout. The CI step +redirects it into $GITHUB_STEP_SUMMARY. Stdlib only. +""" +import json +import os +import sys + +STATUS_ICON = {"expected": "โœ…", "unexpected": "โŒ", "skipped": "โญ๏ธ", "flaky": "โš ๏ธ"} + + +def walk(suite, out): + for spec in suite.get("specs", []): + # A spec's status is carried on its test(s): expected/unexpected/skipped/flaky. + statuses = [t.get("status") for t in spec.get("tests", [])] + status = ("unexpected" if "unexpected" in statuses + else "flaky" if "flaky" in statuses + else "skipped" if statuses and all(s == "skipped" for s in statuses) + else "expected" if spec.get("ok", False) + else "unexpected") + out.append({"file": spec.get("file") or suite.get("file") or suite.get("title", ""), + "title": spec.get("title", ""), "status": status}) + for child in suite.get("suites", []): + walk(child, out) + + +def main(path): + if not os.path.exists(path): + print("## ๐ŸŽญ e2e (Playwright)\n\n_No e2e report โ€” the run did not reach the e2e step._") + return 0 + with open(path) as fh: + report = json.load(fh) + specs = [] + for suite in report.get("suites", []): + walk(suite, specs) + + print("## ๐ŸŽญ e2e (Playwright)\n") + stats = report.get("stats", {}) + if stats: + print(f"**{stats.get('expected', 0)} passed ยท {stats.get('unexpected', 0)} failed ยท " + f"{stats.get('flaky', 0)} flaky ยท {stats.get('skipped', 0)} skipped** " + f"({round(stats.get('duration', 0) / 1000)}s)\n") + if not specs: + print("_No specs ran._") + return 0 + print("| Spec | Result |") + print("| ---- | :----: |") + for s in specs: + print(f"| {s['file']} โ€บ {s['title']} | {STATUS_ICON.get(s['status'], 'โ”')} |") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "tests/e2e/playwright-report.json")) diff --git a/infra/run-e2e-check.sh b/infra/run-e2e-check.sh index e781a9d..65f0197 100755 --- a/infra/run-e2e-check.sh +++ b/infra/run-e2e-check.sh @@ -26,4 +26,8 @@ cid="$(docker create --network "$net" -w /e2e --ipc=host \ mcr.microsoft.com/playwright:v1.61.1-noble sh -c 'npm install --no-audit --no-fund && npx playwright test')" trap 'docker rm -f "$cid" >/dev/null 2>&1 || true' EXIT docker cp "$root/tests/e2e/." "$cid:/e2e" >/dev/null -docker start -a "$cid" +rc=0 +docker start -a "$cid" || rc=$? +# Copy the Playwright JSON report out โ€” regardless of pass/fail โ€” for the CI job summary (#136). +docker cp "$cid:/e2e/playwright-report.json" "$root/tests/e2e/playwright-report.json" 2>/dev/null || true +exit $rc diff --git a/infra/trx-summary.py b/infra/trx-summary.py new file mode 100644 index 0000000..ca67f67 --- /dev/null +++ b/infra/trx-summary.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Render a per-test-project table from .trx files for a Gitea job summary (#136). + +Reads every *.trx in the given directory (default: TestResults), pulls each project's +counters + assembly name, and prints a GitHub/Gitea-flavoured markdown table to stdout. +The CI step redirects that into $GITHUB_STEP_SUMMARY. Stdlib only. +""" +import glob +import os +import sys +import xml.etree.ElementTree as ET + +NS = {"t": "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"} + + +def project_name(root): + # The test assembly path, e.g. โ€ฆ/services/domain/Big.Tests/bin/โ€ฆ/big.tests.dll. Prefer the + # owning service folder (services/) so "domain" shows rather than the opaque "big.tests"; + # fall back to the assembly basename for projects outside services/ (e.g. tests/acceptance). + ut = root.find(".//t:TestDefinitions/t:UnitTest", NS) + storage = ut.get("storage") if ut is not None else None + if not storage: + return None + parts = storage.replace("\\", "/").split("/") + if "services" in parts: + return parts[parts.index("services") + 1] + base = os.path.basename(parts[-1]) + return base[:-4] if base.lower().endswith(".dll") else base + + +def parse(path): + root = ET.parse(path).getroot() + c = root.find(".//t:ResultSummary/t:Counters", NS) + if c is None: + return None + total = int(c.get("total", 0)) + if total == 0: # e.g. the Integration project, filtered out of the unit run + return None + executed = int(c.get("executed", 0)) + passed = int(c.get("passed", 0)) + failed = int(c.get("failed", 0)) + int(c.get("error", 0)) + skipped = total - executed + return { + "name": project_name(root) or os.path.basename(path), + "passed": passed, "failed": failed, "skipped": skipped, "total": total, + } + + +def main(results_dir): + rows = [r for r in (parse(p) for p in sorted(glob.glob(os.path.join(results_dir, "*.trx")))) if r] + if not rows: + print("_No test results found._") + return 0 + rows.sort(key=lambda r: r["name"]) + print("## โœ… Unit tests\n") + print("| Project | Result | Passed | Failed | Skipped | Total |") + print("| ------- | :----: | -----: | -----: | ------: | ----: |") + for r in rows: + status = "โŒ" if r["failed"] else "โœ…" + print(f"| {r['name']} | {status} | {r['passed']} | {r['failed']} | {r['skipped']} | {r['total']} |") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "TestResults")) diff --git a/infra/vitest-summary.py b/infra/vitest-summary.py new file mode 100644 index 0000000..d74cd6a --- /dev/null +++ b/infra/vitest-summary.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Render a per-frontend test table from vitest JSON reports for a Gitea job summary (#136). + +Reads every *.json in the given directory (default: test-output), each written by an app's +`test` target (reporters: json, outputFile: {workspaceRoot}/test-output/{projectName}.json), and +prints a markdown table to stdout โ€” one row per frontend app. The CI step redirects it into +$GITHUB_STEP_SUMMARY. Stdlib only. +""" +import glob +import json +import os +import sys + + +def main(results_dir): + rows = [] + for path in sorted(glob.glob(os.path.join(results_dir, "*.json"))): + try: + with open(path) as fh: + d = json.load(fh) + except (OSError, ValueError): + continue + rows.append({ + "name": os.path.splitext(os.path.basename(path))[0], + "passed": d.get("numPassedTests", 0), + "failed": d.get("numFailedTests", 0), + "skipped": d.get("numPendingTests", 0) + d.get("numTodoTests", 0), + "total": d.get("numTotalTests", 0), + "ok": d.get("success", False), + }) + if not rows: + print("_No frontend test results found._") + return 0 + print("## ๐Ÿ…ฐ๏ธ Frontend tests\n") + print("| Frontend | Result | Passed | Failed | Skipped | Total |") + print("| -------- | :----: | -----: | -----: | ------: | ----: |") + for r in rows: + status = "โœ…" if r["ok"] and not r["failed"] else "โŒ" + print(f"| {r['name']} | {status} | {r['passed']} | {r['failed']} | {r['skipped']} | {r['total']} |") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "test-output")) diff --git a/services/acl/stryker-config.json b/services/acl/stryker-config.json index 0412cc4..c64e3d3 100644 --- a/services/acl/stryker-config.json +++ b/services/acl/stryker-config.json @@ -2,7 +2,7 @@ "stryker-config": { "solution": "Acl.slnx", "test-projects": ["Acl.Tests/Acl.Tests.csproj"], - "reporters": ["progress", "html"], + "reporters": ["progress", "html", "markdown"], "thresholds": { "high": 95, "low": 90, diff --git a/services/bff/stryker-config.json b/services/bff/stryker-config.json index fc3fe3c..46c8815 100644 --- a/services/bff/stryker-config.json +++ b/services/bff/stryker-config.json @@ -2,7 +2,7 @@ "stryker-config": { "solution": "Bff.slnx", "test-projects": ["Bff.Tests/Bff.Tests.csproj"], - "reporters": ["progress", "html"], + "reporters": ["progress", "html", "markdown"], "mutate": [ "!**/Program.cs", "!**/DownstreamClients.cs" diff --git a/services/domain/stryker-config.json b/services/domain/stryker-config.json index 3b51fae..c7ee5c3 100644 --- a/services/domain/stryker-config.json +++ b/services/domain/stryker-config.json @@ -2,7 +2,7 @@ "stryker-config": { "solution": "Big.slnx", "test-projects": ["Big.Tests/Big.Tests.csproj"], - "reporters": ["progress", "html"], + "reporters": ["progress", "html", "markdown"], "mutate": [ "!**/OpenZaakJobPump.cs", "!**/BeoordelingEscalatiePump.cs", diff --git a/services/event-subscriber/stryker-config.json b/services/event-subscriber/stryker-config.json index c99d4e4..00ea73e 100644 --- a/services/event-subscriber/stryker-config.json +++ b/services/event-subscriber/stryker-config.json @@ -2,7 +2,7 @@ "stryker-config": { "solution": "EventSubscriber.slnx", "test-projects": ["EventSubscriber.Tests/EventSubscriber.Tests.csproj"], - "reporters": ["progress", "html"], + "reporters": ["progress", "html", "markdown"], "thresholds": { "high": 95, "low": 90, diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index 43298c0..f0106f6 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -21,7 +21,9 @@ export default defineConfig({ // OOM-killed mid-action ("Page crashed") โ€” fixing the flakiness at its source rather than leaning // on `retries` (CLAUDE.md ยง15). Only two long-running happy-path specs, so serial costs little. workers: 1, - reporter: [['list']], + // `list` for the live log; `json` (โ†’ /e2e/playwright-report.json in the container) is copied out + // by run-e2e-check.sh and rendered as a per-spec table in the CI job summary (#136). + reporter: [['list'], ['json', { outputFile: 'playwright-report.json' }]], use: { baseURL, trace: 'on-first-retry',