feat(openzaak): bounded retry + flagged write divergence (WP-60)

Local aanvraag/document writes and their paired ZGW writes aren't
transactional; a ZGW failure after the local write succeeds used to
diverge silently. ZgwHttpClient now retries transport-shaped failures
(not 500, which can follow a partial commit on the non-idempotent
statussen/rollen POSTs), and a ZGW failure that survives retry sets
Aanvraag.ZgwError plus a zgw:divergence audit row instead of failing
or diverging quietly. No outbox/reconcile job: three request-triggered
write paths don't justify a persisted queue that would also need to
carry citizen PII for the JWT audit claims.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
eho
2026-07-30 18:11:55 +02:00
co-authored by Claude Sonnet 5
parent 67abc58052
commit 3ff80c124f
18 changed files with 855 additions and 82 deletions
@@ -33,6 +33,13 @@ public sealed class Aanvraag
/// endpoint does, via <see cref="ApplicationStore.SetZaakUrl"/>) to keep the seam's write
/// surface at "return data", not "reach into another store".</summary>
public string? ZaakUrl { get; set; }
/// <summary>WP-60: non-null means the ZGW side of this submit (or its document link) did not
/// complete — the local aanvraag is authoritative and is NOT rolled back (that risks an
/// orphan zaak if the failure landed after the zaak POST succeeded). The zaak, if it exists,
/// is re-findable by <c>identificatie == Referentie</c>. Cleared by a future repair path;
/// none exists yet (see openzaak-integration.md's "Write resilience" section).</summary>
public string? ZgwError { get; set; }
}
/// <summary>
@@ -193,4 +200,18 @@ public static class ApplicationStore
db.SaveChanges();
}
}
/// <summary>Flag (or clear, once a repair path exists) that this aanvraag's ZGW write did
/// not complete — see <see cref="Aanvraag.ZgwError"/>. No-op if the aanvraag is gone.</summary>
public static void SetZgwError(string id, string? error)
{
lock (_gate)
{
using var db = Db.Create();
var a = db.Applications.Find(id);
if (a is null) return;
a.ZgwError = error;
db.SaveChanges();
}
}
}
@@ -0,0 +1,282 @@
// <auto-generated />
using System;
using BigRegister.Api.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
#nullable disable
namespace BigRegister.Api.Data.Migrations
{
[DbContext(typeof(AppDbContext))]
[Migration("20260730155659_ZgwSyncError")]
partial class ZgwSyncError
{
/// <inheritdoc />
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "10.0.9");
modelBuilder.Entity("BigRegister.Api.Data.Aanvraag", b =>
{
b.Property<string>("Id")
.HasColumnType("TEXT");
b.Property<bool>("AutoApprovable")
.HasColumnType("INTEGER");
b.Property<DateTimeOffset>("CreatedAt")
.HasColumnType("TEXT");
b.Property<string>("DocumentIds")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Draft")
.HasColumnType("TEXT");
b.Property<string>("Owner")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Reden")
.HasColumnType("TEXT");
b.Property<string>("Referentie")
.HasColumnType("TEXT");
b.Property<int>("StepCount")
.HasColumnType("INTEGER");
b.Property<int>("StepIndex")
.HasColumnType("INTEGER");
b.Property<bool>("Submitted")
.HasColumnType("INTEGER");
b.Property<DateTimeOffset?>("SubmittedAt")
.HasColumnType("TEXT");
b.Property<string>("Type")
.IsRequired()
.HasColumnType("TEXT");
b.Property<DateTimeOffset>("UpdatedAt")
.HasColumnType("TEXT");
b.Property<string>("ZaakUrl")
.HasColumnType("TEXT");
b.Property<string>("ZgwError")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("Applications");
});
modelBuilder.Entity("BigRegister.Api.Data.AuditEntry", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Action")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Actor")
.IsRequired()
.HasColumnType("TEXT");
b.Property<DateTimeOffset>("At")
.HasColumnType("TEXT");
b.Property<string>("CategoryId")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("DocumentId")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("AuditEntries");
});
modelBuilder.Entity("BigRegister.Api.Data.AuthzAuditEntry", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Action")
.IsRequired()
.HasColumnType("TEXT");
b.Property<DateTimeOffset>("At")
.HasColumnType("TEXT");
b.Property<string>("CorrelationId")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Decision")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Resource")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Role")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("AuthzAudit");
});
modelBuilder.Entity("BigRegister.Api.Data.BriefEntity", b =>
{
b.Property<string>("BriefId")
.HasColumnType("TEXT");
b.Property<string>("ArchivedHtml")
.HasColumnType("TEXT");
b.Property<string>("Beroep")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("DrafterId")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Owner")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Placeholders")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Sections")
.IsRequired()
.HasColumnType("TEXT");
b.Property<int?>("SentOrgTemplateVersion")
.HasColumnType("INTEGER");
b.Property<string>("Status")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("SubOrgId")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("TemplateId")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("BriefId");
b.HasIndex("Owner")
.IsUnique();
b.ToTable("Briefs");
});
modelBuilder.Entity("BigRegister.Api.Data.FeatureFlagEntity", b =>
{
b.Property<string>("Key")
.HasColumnType("TEXT");
b.Property<bool>("Enabled")
.HasColumnType("INTEGER");
b.HasKey("Key");
b.ToTable("FeatureFlags");
});
modelBuilder.Entity("BigRegister.Api.Data.OrgTemplateEntity", b =>
{
b.Property<string>("SubOrgId")
.HasColumnType("TEXT");
b.Property<string>("Draft")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("History")
.IsRequired()
.HasColumnType("TEXT");
b.Property<int>("PublishedVersion")
.HasColumnType("INTEGER");
b.HasKey("SubOrgId");
b.ToTable("OrgTemplates");
});
modelBuilder.Entity("BigRegister.Api.Data.StoredDocument", b =>
{
b.Property<string>("DocumentId")
.HasColumnType("TEXT");
b.Property<string>("CategoryId")
.IsRequired()
.HasColumnType("TEXT");
b.Property<byte[]>("Content")
.IsRequired()
.HasColumnType("BLOB");
b.Property<string>("ContentType")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("DrcUrl")
.HasColumnType("TEXT");
b.Property<string>("FileName")
.IsRequired()
.HasColumnType("TEXT");
b.Property<bool>("Linked")
.HasColumnType("INTEGER");
b.Property<string>("LocalId")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Owner")
.IsRequired()
.HasColumnType("TEXT");
b.Property<long>("SizeBytes")
.HasColumnType("INTEGER");
b.Property<DateTimeOffset>("UploadedAt")
.HasColumnType("TEXT");
b.Property<string>("WizardId")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("DocumentId");
b.ToTable("Documents");
});
#pragma warning restore 612, 618
}
}
}
@@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace BigRegister.Api.Data.Migrations
{
/// <inheritdoc />
public partial class ZgwSyncError : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "ZgwError",
table: "Applications",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ZgwError",
table: "Applications");
}
}
}
@@ -67,6 +67,9 @@ namespace BigRegister.Api.Data.Migrations
b.Property<string>("ZaakUrl")
.HasColumnType("TEXT");
b.Property<string>("ZgwError")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("Applications");
+45 -5
View File
@@ -58,9 +58,12 @@ if (zgw.Enabled)
{
builder.Services.AddSingleton(zgw);
builder.Services.AddSingleton<ZgwTokenProvider>();
builder.Services.AddHttpClient<IZaakSource, OpenZaakZaakSource>();
// WP-60: a bounded client timeout matters once ZgwHttpClient retries — without one, the
// sources' sync-over-async call (no CancellationToken threaded through) could block a
// thread-pool thread for HttpClient's 100s default times 3 attempts.
builder.Services.AddHttpClient<IZaakSource, OpenZaakZaakSource>(c => c.Timeout = TimeSpan.FromSeconds(15));
// WP-51: the documents (Documenten API / DRC) seam — same pattern as IZaakSource above.
builder.Services.AddHttpClient<IDocumentSource, OpenZaakDocumentSource>();
builder.Services.AddHttpClient<IDocumentSource, OpenZaakDocumentSource>(c => c.Timeout = TimeSpan.FromSeconds(15));
}
else
{
@@ -351,13 +354,38 @@ api.MapPost("/applications/{id}/submit", (string id, SubmitApplicationRequest re
// in OpenZaak and maps its result back into this same response shape (ADR-0001/ADR-0005:
// zero FE contract change either way). WP-53: the caller is threaded through so the minted
// ZGW JWT's user_id/user_representation reflect the acting citizen, not a static config value.
var (referentie, status, zaakUrl) = zaken.CreateZaak(submitted, DateTimeOffset.UtcNow, ctx.Caller());
if (zaakUrl is not null) ApplicationStore.SetZaakUrl(id, zaakUrl);
//
// WP-60: the local submit above already committed — it is never rolled back on a ZGW
// failure (an orphan zaak from a rolled-back-then-retried submit is worse than a flagged
// one, see openzaak-integration.md's "Write resilience" section). Each ZGW half is caught
// separately so a create-zaak failure doesn't also skip the (still-local) document link.
var referentie = submitted.Referentie!;
var status = submitted.ToStatusDto(DateTimeOffset.UtcNow);
string? zaakUrl = null;
try
{
(referentie, status, zaakUrl) = zaken.CreateZaak(submitted, DateTimeOffset.UtcNow, ctx.Caller());
if (zaakUrl is not null) ApplicationStore.SetZaakUrl(id, zaakUrl);
}
catch (Exception ex)
{
RecordZgwDivergence(ctx, id, referentie, ex);
}
// WP-51: link the submitted documents to the zaak — LocalDocumentSource is exactly the
// DocumentStore.Link call this used to make inline; OpenZaakDocumentSource additionally
// POSTs a zaakinformatieobject per document, now that the zaak (zaakUrl) exists.
if (documentIds is not null) documents.LinkToZaak(documentIds, zaakUrl, ctx.Caller());
if (documentIds is not null)
{
try
{
documents.LinkToZaak(documentIds, zaakUrl, ctx.Caller());
}
catch (Exception ex)
{
RecordZgwDivergence(ctx, id, referentie, ex);
}
}
return Results.Ok(new SubmitApplicationResponse(referentie, status));
})
@@ -676,6 +704,18 @@ void AuditAuthz(HttpContext ctx, string action, string resource, bool allowed, P
AuthzAuditStore.Record(action, resource, allowed, principal.Role.ToString(), cid);
}
// WP-60: the local write already committed — this records that its ZGW counterpart didn't,
// rather than letting the two sides diverge silently (openzaak-integration.md's "Write
// resilience" section). Same audit trail AuditAuthz writes to (/beheer/audit), so a
// divergence is visible next to every other decision, not a separate mechanism.
void RecordZgwDivergence(HttpContext ctx, string id, string referentie, Exception ex)
{
app.Logger.LogError(ex, "zgw divergence aanvraag={Id} reference={Reference}", id, referentie);
ApplicationStore.SetZgwError(id, ex.Message);
var cid = ctx.Items.TryGetValue("CorrelationId", out var v) ? (string)v! : "none";
AuthzAuditStore.Record("zgw:divergence", referentie, allowed: false, Authz.ResolvePrincipal(ctx).Role.ToString(), cid);
}
// Keep the last `keep` characters, mask the rest — mirrors the FE maskTail
// (src/app/shared/ui/debug-state/mask.ts) so wire redaction and the dev panel agree.
static string MaskTail(string value, int keep) =>
@@ -4,6 +4,7 @@ using BigRegister.Api.Contracts;
using BigRegister.Api.Data;
using BigRegister.Domain.Authorization;
using BigRegister.Stamdata;
using Microsoft.Extensions.Logging;
namespace BigRegister.Api.Zgw;
@@ -20,7 +21,9 @@ namespace BigRegister.Api.Zgw;
/// <see cref="OpenZaakZaakSource"/> — creating a document needs write scope on Documenten;
/// linking one to a zaak needs write scope on Zaken (the zaakinformatieobject resource).
/// </summary>
public sealed class OpenZaakDocumentSource(HttpClient http, ZgwTokenProvider tokens, ZgwOptions options) : IDocumentSource
public sealed class OpenZaakDocumentSource(
HttpClient http, ZgwTokenProvider tokens, ZgwOptions options, ILogger<OpenZaakDocumentSource>? log = null)
: IDocumentSource
{
private readonly ZgwHttpClient zgw = new(http, tokens);
@@ -41,37 +44,54 @@ public sealed class OpenZaakDocumentSource(HttpClient http, ZgwTokenProvider tok
UploadAsync(localId, categoryId, wizardId, fileName, contentType, content, caller)
.GetAwaiter().GetResult();
// WP-60: once DocumentStore.Add (below) has committed, the local document is the record of
// truth (per the class doc above) — a ZGW failure past that point is caught, logged, and
// leaves DrcUrl null rather than throwing. DrcUrl == null is already the meaningful "not
// registered in ZGW yet" detector LinkToZaak skips on, so no separate flag column is needed
// here the way ApplicationStore.ZgwError is for the zaak side (see openzaak-integration.md's
// "Write resilience" section for why the two write paths differ).
private async Task<UploadResponse> UploadAsync(
string localId, string categoryId, string wizardId, string fileName, string contentType,
byte[] content, CallerIdentity caller)
{
var doc = DocumentStore.Add(localId, categoryId, wizardId, fileName, contentType, content, caller.Bsn);
if (!options.InformatieobjecttypeUrls.TryGetValue(categoryId, out var informatieobjecttypeUrl))
throw new InvalidOperationException(
$"Zgw:InformatieobjecttypeUrls has no entry for category '{categoryId}'.");
try
{
if (!options.InformatieobjecttypeUrls.TryGetValue(categoryId, out var informatieobjecttypeUrl))
throw new InvalidOperationException(
$"Zgw:InformatieobjecttypeUrls has no entry for category '{categoryId}'.");
var eio = await zgw.PostAsync<Eio>($"{options.DrcBaseUrl}/enkelvoudiginformatieobjecten", new CreateEioRequest(
Bronorganisatie: options.Bronorganisatie,
Creatiedatum: DateOnly.FromDateTime(doc.UploadedAt.UtcDateTime),
Titel: fileName,
Auteur: options.UserRepresentation,
Taal: "nld",
Formaat: contentType,
Bestandsnaam: fileName,
Inhoud: Convert.ToBase64String(content),
Informatieobjecttype: informatieobjecttypeUrl,
Identificatie: doc.DocumentId,
Vertrouwelijkheidaanduiding: ConfidentialiteitFor(categoryId)), caller);
var eio = await zgw.PostAsync<Eio>($"{options.DrcBaseUrl}/enkelvoudiginformatieobjecten", new CreateEioRequest(
Bronorganisatie: options.Bronorganisatie,
Creatiedatum: DateOnly.FromDateTime(doc.UploadedAt.UtcDateTime),
Titel: fileName,
Auteur: options.UserRepresentation,
Taal: "nld",
Formaat: contentType,
Bestandsnaam: fileName,
Inhoud: Convert.ToBase64String(content),
Informatieobjecttype: informatieobjecttypeUrl,
Identificatie: doc.DocumentId,
Vertrouwelijkheidaanduiding: ConfidentialiteitFor(categoryId)), caller);
DocumentStore.SetDrcUrl(doc.DocumentId, eio.Url);
}
catch (Exception ex)
{
log?.LogError(ex, "zgw divergence document={DocumentId} category={CategoryId}", doc.DocumentId, categoryId);
}
DocumentStore.SetDrcUrl(doc.DocumentId, eio.Url);
return new UploadResponse(doc.DocumentId, doc.LocalId);
}
/// <summary>Local link always happens (dual-write, same reasoning as upload); additionally,
/// once a zaak exists, POST a zaakinformatieobject for every document that has a DRC url —
/// documents uploaded before Zgw:Enabled was ever true (or under a config gap) simply have
/// no DrcUrl yet and are skipped, matching "nothing extra to link" for the local case.</summary>
/// no DrcUrl yet and are skipped, matching "nothing extra to link" for the local case.
/// WP-60: unlike Upload, a ZGW failure here still throws — DocumentStore.Link (the local
/// half) already ran above, so the caller (Program.cs's submit endpoint) catching this and
/// recording it as a flagged divergence is what closes the gap, not a try/catch in here.</summary>
public void LinkToZaak(IReadOnlyList<string> documentIds, string? zaakUrl, CallerIdentity caller)
{
DocumentStore.Link(documentIds);
@@ -86,10 +86,12 @@ public sealed class OpenZaakZaakSource(HttpClient http, ZgwTokenProvider tokens,
/// reason as <see cref="ListCases"/> (see the ponytail note there) — a submit is already a
/// single request/response round trip, so no extra concurrency concern.
///
/// ponytail: no compensating transaction — if any ZGW call here throws, the aanvraag is
/// already marked Submitted locally (ApplicationStore.Submit already ran) but has no zaak.
/// Acceptable for a first write slice against a demo backend; a production arc would need a
/// retry/reconciliation story (or an outbox) before this dual-write can be trusted.
/// WP-60: still no compensating transaction — if any call here throws (after
/// <see cref="ZgwHttpClient"/>'s retry gives up), the aanvraag stays Submitted locally with
/// no zaak; rolling it back risks an orphan zaak if the failure landed after the zaak POST
/// succeeded. The caller (Program.cs's submit endpoint) catches this and records it as a
/// flagged divergence (Aanvraag.ZgwError) instead of letting it fail (or diverge) silently —
/// see openzaak-integration.md's "Write resilience" section.
public (string Referentie, AanvraagStatusDto Status, string? ZaakUrl) CreateZaak(Aanvraag aanvraag, DateTimeOffset now, CallerIdentity caller) =>
CreateZaakAsync(aanvraag, now, caller).GetAwaiter().GetResult();
@@ -1,3 +1,4 @@
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using BigRegister.Domain.Authorization;
@@ -14,26 +15,77 @@ namespace BigRegister.Api.Zgw;
/// </summary>
internal sealed class ZgwHttpClient(HttpClient http, ZgwTokenProvider tokens)
{
// WP-60: bounded retry for transport-shaped failures only (gateway restarts, timeouts) —
// never a substitute for reconciliation. 3 attempts, doubling from 200ms.
private const int MaxAttempts = 3;
private static readonly TimeSpan BaseDelay = TimeSpan.FromMilliseconds(200);
public async Task<T> GetAsync<T>(string url, CallerIdentity? caller = null)
{
using var req = new HttpRequestMessage(HttpMethod.Get, url);
Authorize(req, caller);
using var res = await http.SendAsync(req);
res.EnsureSuccessStatusCode();
using var res = await SendWithRetryAsync(() => new HttpRequestMessage(HttpMethod.Get, url), caller);
return (await res.Content.ReadFromJsonAsync<T>())
?? throw new InvalidOperationException($"ZGW GET {url} returned null body.");
}
public async Task<T> PostAsync<T>(string url, object body, CallerIdentity? caller = null)
{
using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = JsonContent.Create(body) };
Authorize(req, caller);
using var res = await http.SendAsync(req);
res.EnsureSuccessStatusCode();
using var res = await SendWithRetryAsync(
() => new HttpRequestMessage(HttpMethod.Post, url) { Content = JsonContent.Create(body) }, caller);
return (await res.Content.ReadFromJsonAsync<T>())
?? throw new InvalidOperationException($"ZGW POST {url} returned null body.");
}
/// <summary>
/// A fresh <see cref="HttpRequestMessage"/> (and JWT) per attempt — a sent request/content
/// cannot be resent. Only transport-shaped failures are retried (429/502/503/504/408, plus
/// connection errors and timeouts); 500 is deliberately excluded because it can follow a
/// partial commit on the two non-idempotent ZGW POSTs (<c>/statussen</c>, <c>/rollen</c>) and
/// retrying risks a duplicate write — the create-zaak/document POSTs are additionally
/// protected by OpenZaak's own uniqueness constraint on (bronorganisatie, identificatie).
/// A non-transient (or exhausted) failure throws with the status + a body snippet, which
/// <c>Program.cs</c>'s submit endpoint catches and records as a flagged divergence rather
/// than letting it diverge silently (see openzaak-integration.md's "Write resilience" section).
/// </summary>
private async Task<HttpResponseMessage> SendWithRetryAsync(Func<HttpRequestMessage> newRequest, CallerIdentity? caller)
{
for (var attempt = 1; ; attempt++)
{
using var req = newRequest();
Authorize(req, caller);
HttpResponseMessage res;
try
{
res = await http.SendAsync(req);
}
catch (Exception ex) when (attempt < MaxAttempts && ex is HttpRequestException or TaskCanceledException)
{
await Task.Delay(BaseDelay * (1 << (attempt - 1)));
continue;
}
if (res.IsSuccessStatusCode) return res;
if (attempt < MaxAttempts && IsTransient(res.StatusCode))
{
res.Dispose();
await Task.Delay(BaseDelay * (1 << (attempt - 1)));
continue;
}
var body = await res.Content.ReadAsStringAsync();
var snippet = body.Length > 500 ? body[..500] : body;
var message = $"ZGW {req.Method} {req.RequestUri} failed: {(int)res.StatusCode} {snippet}";
var status = res.StatusCode;
res.Dispose();
throw new HttpRequestException(message, null, status);
}
}
private static bool IsTransient(HttpStatusCode status) => status is
HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests or
HttpStatusCode.BadGateway or HttpStatusCode.ServiceUnavailable or HttpStatusCode.GatewayTimeout;
private void Authorize(HttpRequestMessage req, CallerIdentity? caller)
{
req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", caller is null ? tokens.Mint() : tokens.Mint(caller));