feat(openzaak): bounded retry + flagged write divergence (WP-60)

Local aanvraag/document writes and their paired ZGW writes aren't
transactional; a ZGW failure after the local write succeeds used to
diverge silently. ZgwHttpClient now retries transport-shaped failures
(not 500, which can follow a partial commit on the non-idempotent
statussen/rollen POSTs), and a ZGW failure that survives retry sets
Aanvraag.ZgwError plus a zgw:divergence audit row instead of failing
or diverging quietly. No outbox/reconcile job: three request-triggered
write paths don't justify a persisted queue that would also need to
carry citizen PII for the JWT audit claims.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

backend/tests/BigRegister.Tests/OpenZaakDocumentSourceTests.cs

@@ -1,3 +1,4 @@
+using System.Net;
 using BigRegister.Api.Data;
 using BigRegister.Api.Zgw;
 using BigRegister.Domain.Authorization;
@@ -77,15 +78,38 @@ public class OpenZaakDocumentSourceTests
     Assert.Contains("\"vertrouwelijkheidaanduiding\":\"openbaar\"", body);
   }
 
+  // WP-60: once DocumentStore.Add has committed, a ZGW-side failure (config gap or transport)
+  // no longer throws — the local document is authoritative and DrcUrl stays null (the same
+  // detector LinkToZaak already skips on for pre-Zgw documents).
+
   [Fact]
-  public void Upload_throws_when_the_category_has_no_configured_informatieobjecttype()
+  public void Upload_keeps_the_local_document_when_the_category_has_no_configured_informatieobjecttype()
   {
     var options = Options();
     var handler = new ZgwStubHandler(url => throw new InvalidOperationException($"no HTTP call expected, got {url}"));
     var source = new OpenZaakDocumentSource(new HttpClient(handler), new ZgwTokenProvider(options), options);
 
-    Assert.Throws<InvalidOperationException>(() =>
-        source.Upload("local-1", "unknown-category", "registratie", "f.pdf", "application/pdf", [1, 2, 3], Caller));
+    var response = source.Upload("local-1", "unknown-category", "registratie", "f.pdf", "application/pdf", [1, 2, 3], Caller);
+
+    Assert.Equal("local-1", response.LocalId);
+    Assert.Empty(handler.Requests);
+    Assert.Null(DocumentStore.Get(response.DocumentId)!.DrcUrl);
+  }
+
+  [Fact]
+  public void Upload_keeps_the_local_document_and_does_not_throw_when_drc_rejects_it()
+  {
+    var options = Options();
+    var handler = new ZgwStubHandler(
+        url => throw new InvalidOperationException($"unexpected success body requested for {url}"),
+        (_, _) => HttpStatusCode.BadRequest);
+    var source = new OpenZaakDocumentSource(new HttpClient(handler), new ZgwTokenProvider(options), options);
+
+    var response = source.Upload("local-1", "identiteit", "registratie", "paspoort.pdf", "application/pdf",
+        "%PDF-1.4 fake"u8.ToArray(), Caller);
+
+    Assert.Equal("local-1", response.LocalId);
+    Assert.Null(DocumentStore.Get(response.DocumentId)!.DrcUrl);
   }
 
   [Fact]

backend/tests/BigRegister.Tests/OpenZaakZaakSourceTests.cs

@@ -1,3 +1,4 @@
+using System.Net;
 using BigRegister.Api.Data;
 using BigRegister.Api.Zgw;
 using BigRegister.Domain.Authorization;
@@ -159,4 +160,95 @@ public class OpenZaakZaakSourceTests
 
     Assert.Throws<InvalidOperationException>(() => source.CreateZaak(aanvraag, DateTimeOffset.UtcNow, caller));
   }
+
+  // --- WP-60: bounded retry in ZgwHttpClient, exercised through the create-zaak write path ---
+
+  private static (ZgwOptions options, Aanvraag aanvraag, CallerIdentity caller) CreateZaakFixture()
+  {
+    const string zaaktypeUrl = $"{ZtBase}/zaaktypen/zt-registratie";
+    var options = new ZgwOptions
+    {
+      ZrcBaseUrl = ZrcBase,
+      ZtcBaseUrl = ZtBase,
+      ClientId = "c",
+      Secret = "s",
+      Bronorganisatie = "123443210",
+      VerantwoordelijkeOrganisatie = "123443210",
+      ZaaktypeUrls = new() { ["registratie"] = zaaktypeUrl },
+    };
+    var aanvraag = new Aanvraag { Id = "a1", Type = "registratie", Owner = "111222333", Referentie = "BIG-2026-000123" };
+    var caller = new CallerIdentity(aanvraag.Owner, "Dr. Test", PrincipalRole.Drafter);
+    return (options, aanvraag, caller);
+  }
+
+  private static string RespondFor(string zaaktypeUrl, string url) => url switch
+  {
+    _ when url == $"{ZrcBase}/zaken" => $$"""
+      { "url": "{{ZrcBase}}/zaken/uuid-new", "identificatie": "BIG-2026-000123",
+        "zaaktype": "{{zaaktypeUrl}}", "startdatum": "2026-07-28",
+        "einddatum": null, "registratiedatum": "2026-07-28" }
+      """,
+    _ when url.StartsWith($"{ZtBase}/statustypen") => """
+      { "count": 1, "next": null,
+        "results": [ { "url": "https://oz.example/catalogi/api/v1/statustypen/st-1", "volgnummer": 1 } ] }
+      """,
+    _ when url.StartsWith($"{ZtBase}/roltypen") => """
+      { "count": 1, "next": null,
+        "results": [ { "url": "https://oz.example/catalogi/api/v1/roltypen/rt-initiator" } ] }
+      """,
+    _ when url == $"{ZrcBase}/statussen" => "{}",
+    _ when url == $"{ZrcBase}/rollen" => "{}",
+    _ => throw new InvalidOperationException($"unexpected ZGW call {url}"),
+  };
+
+  [Fact]
+  public void CreateZaak_retries_a_transient_failure_and_then_succeeds()
+  {
+    var (options, aanvraag, caller) = CreateZaakFixture();
+    var zaaktypeUrl = options.ZaaktypeUrls["registratie"];
+    var handler = new ZgwStubHandler(
+        url => RespondFor(zaaktypeUrl, url),
+        (url, attempt) => url == $"{ZrcBase}/zaken" && attempt == 0 ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK);
+    var source = new OpenZaakZaakSource(new HttpClient(handler), new ZgwTokenProvider(options), options);
+
+    var (referentie, _, zaakUrl) = source.CreateZaak(aanvraag, new DateTimeOffset(2026, 7, 28, 12, 0, 0, TimeSpan.Zero), caller);
+
+    Assert.Equal("BIG-2026-000123", referentie);
+    Assert.Equal($"{ZrcBase}/zaken/uuid-new", zaakUrl);
+    Assert.Equal(2, handler.Requests.Count(r => r == $"{ZrcBase}/zaken"));
+  }
+
+  [Fact]
+  public void CreateZaak_gives_up_after_three_attempts_on_a_persistent_transient_failure()
+  {
+    var (options, aanvraag, caller) = CreateZaakFixture();
+    var zaaktypeUrl = options.ZaaktypeUrls["registratie"];
+    var handler = new ZgwStubHandler(
+        url => RespondFor(zaaktypeUrl, url),
+        (url, _) => url == $"{ZrcBase}/zaken" ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK);
+    var source = new OpenZaakZaakSource(new HttpClient(handler), new ZgwTokenProvider(options), options);
+
+    var ex = Assert.Throws<HttpRequestException>(() => source.CreateZaak(aanvraag, DateTimeOffset.UtcNow, caller));
+
+    Assert.Contains("503", ex.Message);
+    Assert.Equal(3, handler.Requests.Count(r => r == $"{ZrcBase}/zaken"));
+  }
+
+  [Fact]
+  public void CreateZaak_does_not_retry_a_permanent_rejection()
+  {
+    var (options, aanvraag, caller) = CreateZaakFixture();
+    var zaaktypeUrl = options.ZaaktypeUrls["registratie"];
+    var handler = new ZgwStubHandler(
+        url => RespondFor(zaaktypeUrl, url),
+        (url, _) => url == $"{ZrcBase}/statussen" ? HttpStatusCode.BadRequest : HttpStatusCode.OK);
+    var source = new OpenZaakZaakSource(new HttpClient(handler), new ZgwTokenProvider(options), options);
+
+    Assert.Throws<HttpRequestException>(() => source.CreateZaak(aanvraag, DateTimeOffset.UtcNow, caller));
+
+    // No retry on 400, and — the property that makes the whole design safe — no duplicate
+    // zaak was created by a retry that never should have happened.
+    Assert.Equal(1, handler.Requests.Count(r => r == $"{ZrcBase}/statussen"));
+    Assert.Equal(1, handler.Requests.Count(r => r == $"{ZrcBase}/zaken"));
+  }
 }

backend/tests/BigRegister.Tests/ZgwDivergenceTests.cs

@@ -0,0 +1,120 @@
+using System.Net;
+using System.Net.Http.Json;
+using BigRegister.Api.Contracts;
+using BigRegister.Api.Data;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace BigRegister.Tests;
+
+/// <summary>
+/// WP-60's required verification: a ZGW failure mid-submit must not leave the two write sides
+/// silently diverged — it's flagged (<see cref="Aanvraag.ZgwError"/>, an audit row) instead.
+/// Not an <see cref="IClassFixture{TFixture}"/> off <see cref="TestWebApplicationFactory"/>: that
+/// fixture hardcodes <c>Zgw:Enabled=false</c>, so this builds its own factory the same way
+/// <see cref="OpenZaakIntegrationTests"/> does, but with a stub primary handler
+/// (<see cref="ZgwStubHandler"/>) instead of a live OpenZaak.
+/// </summary>
+public class ZgwDivergenceTests
+{
+  private const string ZrcBase = "https://oz.example/zaken/api/v1";
+  private const string ZtBase = "https://oz.example/catalogi/api/v1";
+  private const string ZaaktypeUrl = $"{ZtBase}/zaaktypen/zt-1";
+
+  private static WebApplicationFactory<Program> Factory(ZgwStubHandler stub)
+  {
+    var dbPath = Path.Combine(Path.GetTempPath(), $"bigregister-zgw-divergence-{Guid.NewGuid():N}.db");
+    return new WebApplicationFactory<Program>().WithWebHostBuilder(builder => builder
+        .UseSetting("ConnectionStrings:AppDb", $"Data Source={dbPath}")
+        .UseSetting("Zgw:Enabled", "true")
+        .UseSetting("Zgw:ZrcBaseUrl", ZrcBase)
+        .UseSetting("Zgw:ZtcBaseUrl", ZtBase)
+        .UseSetting("Zgw:ClientId", "c")
+        .UseSetting("Zgw:Secret", "s")
+        .UseSetting("Zgw:Bronorganisatie", "123443210")
+        .UseSetting("Zgw:VerantwoordelijkeOrganisatie", "123443210")
+        .UseSetting("Zgw:ZaaktypeUrls:registratie", ZaaktypeUrl)
+        .ConfigureServices(services => services.ConfigureHttpClientDefaults(b =>
+            b.ConfigurePrimaryHttpMessageHandler(() => stub))));
+  }
+
+  /// <summary>Doesn't call GET /applications first (unlike ApplicationTests.Create) — under
+  /// Zgw:Enabled=true that route goes through IZaakSource too, which this test's stub doesn't
+  /// need to answer since every test here uses a fresh db and creates exactly one aanvraag.</summary>
+  private static async Task<string> CreateConcept(HttpClient client, string type = "registratie")
+  {
+    var res = await client.PostAsJsonAsync("/api/v1/applications", new { type });
+    res.EnsureSuccessStatusCode();
+    var body = (await res.Content.ReadFromJsonAsync<ApplicationDetailDto>())!;
+    return body.Id;
+  }
+
+  private static string SuccessBody(string url) => url switch
+  {
+    _ when url == $"{ZrcBase}/zaken" => $$"""
+      { "url": "{{ZrcBase}}/zaken/uuid-new", "identificatie": "BIG-2026-000123",
+        "zaaktype": "{{ZaaktypeUrl}}", "startdatum": "2026-07-30",
+        "einddatum": null, "registratiedatum": "2026-07-30" }
+      """,
+    _ when url.StartsWith($"{ZtBase}/statustypen") => """
+      { "count": 1, "next": null,
+        "results": [ { "url": "https://oz.example/catalogi/api/v1/statustypen/st-1", "volgnummer": 1 } ] }
+      """,
+    _ when url.StartsWith($"{ZtBase}/roltypen") => """
+      { "count": 1, "next": null,
+        "results": [ { "url": "https://oz.example/catalogi/api/v1/roltypen/rt-initiator" } ] }
+      """,
+    _ when url == $"{ZrcBase}/statussen" => "{}",
+    _ when url == $"{ZrcBase}/rollen" => "{}",
+    _ => throw new InvalidOperationException($"unexpected ZGW call {url}"),
+  };
+
+  [Fact]
+  public async Task Submit_with_a_failing_zgw_flags_the_divergence_instead_of_diverging_silently()
+  {
+    var stub = new ZgwStubHandler(SuccessBody, (url, _) => url == $"{ZrcBase}/zaken" ? HttpStatusCode.ServiceUnavailable : HttpStatusCode.OK);
+    using var factory = Factory(stub);
+    using var client = factory.CreateClient();
+
+    var id = await CreateConcept(client);
+    var res = await client.PostAsJsonAsync($"/api/v1/applications/{id}/submit", new { diplomaHerkomst = "duo" });
+
+    // The local write is still authoritative: 200 with a real reference, not a 500.
+    res.EnsureSuccessStatusCode();
+    var body = (await res.Content.ReadFromJsonAsync<SubmitApplicationResponse>())!;
+    Assert.NotEmpty(body.Referentie);
+
+    var stored = ApplicationStore.ListAll().Single(a => a.Id == id);
+    Assert.Null(stored.ZaakUrl);
+    Assert.NotNull(stored.ZgwError);
+
+    var audit = await client.SendAsync(AdminRequest(HttpMethod.Get, "/api/v1/admin/audit"));
+    audit.EnsureSuccessStatusCode();
+    var entries = (await audit.Content.ReadFromJsonAsync<List<AuthzAuditDto>>())!;
+    Assert.Contains(entries, e => e.Action == "zgw:divergence" && e.Decision == "deny" && e.Resource == body.Referentie);
+  }
+
+  [Fact]
+  public async Task Submit_with_a_healthy_zgw_leaves_no_divergence_flag()
+  {
+    var stub = new ZgwStubHandler(SuccessBody);
+    using var factory = Factory(stub);
+    using var client = factory.CreateClient();
+
+    var id = await CreateConcept(client);
+    var res = await client.PostAsJsonAsync($"/api/v1/applications/{id}/submit", new { diplomaHerkomst = "duo" });
+    res.EnsureSuccessStatusCode();
+
+    var stored = ApplicationStore.ListAll().Single(a => a.Id == id);
+    Assert.Equal($"{ZrcBase}/zaken/uuid-new", stored.ZaakUrl);
+    Assert.Null(stored.ZgwError);
+  }
+
+  private static HttpRequestMessage AdminRequest(HttpMethod method, string path)
+  {
+    var req = new HttpRequestMessage(method, path);
+    req.Headers.Add("X-Role", "admin");
+    return req;
+  }
+}

backend/tests/BigRegister.Tests/ZgwStubHandler.cs

@@ -9,8 +9,15 @@ namespace BigRegister.Tests;
 /// URL across GET/POST). Records every request's url/body/auth-scheme for assertion.
 /// Factored out of OpenZaakZaakSourceTests once OpenZaakDocumentSourceTests needed the
 /// identical stub.
+///
+/// WP-60: an optional <paramref name="status"/> callback lets a test inject a failing status
+/// for a given url on a given (0-based) attempt — e.g. "503 on the first call to /zaken, then
+/// let it through" — to exercise ZgwHttpClient's retry without a live server. When it returns
+/// a non-2xx code, <paramref name="respond"/> is not called for that attempt (so a test that
+/// models an "always fails" url never has to also teach `respond` a success body it never
+/// reaches).
 /// </summary>
-internal sealed class ZgwStubHandler(Func<string, string> respond) : HttpMessageHandler
+internal sealed class ZgwStubHandler(Func<string, string> respond, Func<string, int, HttpStatusCode>? status = null) : HttpMessageHandler
 {
   public List<string> Requests { get; } = new();
   public List<string?> AuthSchemes { get; } = new();
@@ -21,9 +28,18 @@ internal sealed class ZgwStubHandler(Func<string, string> respond) : HttpMessage
   protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
   {
     var url = request.RequestUri!.ToString();
+    var attempt = Requests.Count(r => r == url);
     Requests.Add(url);
     AuthSchemes.Add(request.Headers.Authorization?.Scheme);
     Bodies.Add(request.Content?.ReadAsStringAsync(cancellationToken).GetAwaiter().GetResult() ?? "");
+
+    var code = status?.Invoke(url, attempt) ?? HttpStatusCode.OK;
+    if (!((int)code >= 200 && (int)code < 300))
+      return Task.FromResult(new HttpResponseMessage(code)
+      {
+        Content = new StringContent("{\"detail\":\"stub failure\"}", Encoding.UTF8, "application/json"),
+      });
+
     return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)
     {
       Content = new StringContent(respond(url), Encoding.UTF8, "application/json"),