fix(ci): carry the failing spec's error into the e2e job summary (refs #161)
The per-spec table now has a "Why" column holding the spec's first error, flattened for a markdown cell: ANSI stripped, newlines collapsed, `|` escaped (a real report's message is multi-line, coloured, and embeds source-snippet gutters), clipped to 300 chars. The column only appears when something failed. So a red e2e names its cause in the summary even when the log is truncated or the run is killed mid-stream — which is the state #161 was filed from. Shape verified against an actual @playwright/test 1.61 failing report, not just the fixture. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,20 @@
|
|||||||
|
# Overlay: make the CI compose stack usable from a HOST browser.
|
||||||
|
# Same two mechanisms infra/docker-compose.local.yml already uses — pin Keycloak's issuer to the
|
||||||
|
# host-published address, and point each portal's runtime config.json at it. The BFF needs no
|
||||||
|
# change: it discovers metadata over keycloak:8080 and the discovered issuer is the pinned
|
||||||
|
# localhost:8180, which is what browser tokens carry.
|
||||||
|
services:
|
||||||
|
keycloak:
|
||||||
|
environment:
|
||||||
|
KC_HOSTNAME: http://localhost:8180
|
||||||
|
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
|
||||||
|
self-service:
|
||||||
|
volumes:
|
||||||
|
- ./local-config/self-service.config.json:/usr/share/nginx/html/config.json:ro,z
|
||||||
|
behandel:
|
||||||
|
volumes:
|
||||||
|
- ./local-config/behandel.config.json:/usr/share/nginx/html/config.json:ro,z
|
||||||
|
# beheer is the same medewerker realm as behandel, so it reuses behandel's config verbatim.
|
||||||
|
beheer:
|
||||||
|
volumes:
|
||||||
|
- ./local-config/behandel.config.json:/usr/share/nginx/html/config.json:ro,z
|
||||||
@@ -7,10 +7,34 @@ redirects it into $GITHUB_STEP_SUMMARY. Stdlib only.
|
|||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
STATUS_ICON = {"expected": "✅", "unexpected": "❌", "skipped": "⏭️", "flaky": "⚠️"}
|
STATUS_ICON = {"expected": "✅", "unexpected": "❌", "skipped": "⏭️", "flaky": "⚠️"}
|
||||||
|
|
||||||
|
# A verdict alone still costs a log dive, and a killed or truncated job leaves no log to dive into
|
||||||
|
# (#161) — so a failing spec carries its first error into the table. Playwright errors are multi-line
|
||||||
|
# with a "Call log:", which a markdown table cell cannot hold, so they are flattened and clipped.
|
||||||
|
ERROR_CLIP = 300
|
||||||
|
|
||||||
|
|
||||||
|
def first_error(spec):
|
||||||
|
"""The first error message across a spec's test results, flattened for one table cell."""
|
||||||
|
for test in spec.get("tests", []):
|
||||||
|
for result in test.get("results", []):
|
||||||
|
for error in result.get("errors", []):
|
||||||
|
message = (error.get("message") or "").strip()
|
||||||
|
if not message:
|
||||||
|
continue
|
||||||
|
# Strip ANSI colour, collapse to one line, and keep it inside the cell.
|
||||||
|
message = re.sub(r"\x1b\[[0-9;]*m", "", message)
|
||||||
|
message = " ".join(message.split())
|
||||||
|
if len(message) > ERROR_CLIP:
|
||||||
|
message = message[:ERROR_CLIP - 1].rstrip() + "…"
|
||||||
|
# `|` would end the cell early.
|
||||||
|
return message.replace("|", "\\|")
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def walk(suite, out):
|
def walk(suite, out):
|
||||||
for spec in suite.get("specs", []):
|
for spec in suite.get("specs", []):
|
||||||
@@ -22,7 +46,8 @@ def walk(suite, out):
|
|||||||
else "expected" if spec.get("ok", False)
|
else "expected" if spec.get("ok", False)
|
||||||
else "unexpected")
|
else "unexpected")
|
||||||
out.append({"file": spec.get("file") or suite.get("file") or suite.get("title", ""),
|
out.append({"file": spec.get("file") or suite.get("file") or suite.get("title", ""),
|
||||||
"title": spec.get("title", ""), "status": status})
|
"title": spec.get("title", ""), "status": status,
|
||||||
|
"error": first_error(spec) if status in ("unexpected", "flaky") else ""})
|
||||||
for child in suite.get("suites", []):
|
for child in suite.get("suites", []):
|
||||||
walk(child, out)
|
walk(child, out)
|
||||||
|
|
||||||
@@ -46,6 +71,13 @@ def main(path):
|
|||||||
if not specs:
|
if not specs:
|
||||||
print("_No specs ran._")
|
print("_No specs ran._")
|
||||||
return 0
|
return 0
|
||||||
|
# The failure column only earns its width when something failed.
|
||||||
|
if any(s["error"] for s in specs):
|
||||||
|
print("| Spec | Result | Why |")
|
||||||
|
print("| ---- | :----: | --- |")
|
||||||
|
for s in specs:
|
||||||
|
print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} | {s['error']} |")
|
||||||
|
else:
|
||||||
print("| Spec | Result |")
|
print("| Spec | Result |")
|
||||||
print("| ---- | :----: |")
|
print("| ---- | :----: |")
|
||||||
for s in specs:
|
for s in specs:
|
||||||
|
|||||||
@@ -62,6 +62,26 @@ def test_failing_spec_reports_why():
|
|||||||
assert not any(line.startswith("Call log:") for line in md.splitlines()), md
|
assert not any(line.startswith("Call log:") for line in md.splitlines()), md
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_playwright_error_is_flattened():
|
||||||
|
# A real report's message is multi-line and ANSI-coloured, and embeds the source snippet with
|
||||||
|
# `|` gutters — all three would break the table cell. Shape verified against an actual
|
||||||
|
# @playwright/test 1.61 JSON report.
|
||||||
|
md = render({
|
||||||
|
"stats": {"expected": 0, "unexpected": 1, "flaky": 0, "skipped": 0, "duration": 1_000},
|
||||||
|
"suites": [{"file": "catalogus.spec.ts", "specs": [
|
||||||
|
spec_entry("a beheerder sees the catalogus", "unexpected",
|
||||||
|
["Error: expect(locator).toBeVisible() failed\n\n"
|
||||||
|
"\x1b[2mLocator: \x1b[22mgetByRole('heading')\n"
|
||||||
|
" 12 | await login(page);\n> 13 | await expect(heading).toBeVisible();\n"]),
|
||||||
|
]}],
|
||||||
|
})
|
||||||
|
row = [line for line in md.splitlines() if line.startswith("| catalogus.spec.ts")][0]
|
||||||
|
assert "\x1b" not in row, row
|
||||||
|
assert "Locator: getByRole('heading')" in row, row
|
||||||
|
# Every literal `|` from the snippet gutters is escaped, so the row keeps exactly 3 cells.
|
||||||
|
assert row.count("|") - row.count("\\|") == 4, row
|
||||||
|
|
||||||
|
|
||||||
def test_passing_run_stays_quiet():
|
def test_passing_run_stays_quiet():
|
||||||
md = render({
|
md = render({
|
||||||
"stats": {"expected": 1, "unexpected": 0, "flaky": 0, "skipped": 0, "duration": 5_000},
|
"stats": {"expected": 1, "unexpected": 0, "flaky": 0, "skipped": 0, "duration": 5_000},
|
||||||
|
|||||||
Reference in New Issue
Block a user