fix(db): classify transient vs permanent SQL errors in Database.CachedWrite (#7)

CachedWrite buffered ALL write failures and retried forever, never returning a synchronous failure to the script — permanent SQL errors (constraint/syntax/ permission) were treated as transient. Mirror the External-System API path: attempt immediately, return Failed synchronously on permanent SQL errors (no buffering), buffer only transient errors; the S&F retry path parks permanent failures instead of retrying forever. New SqlErrorClassifier + PermanentDatabaseException.
2026-06-15 13:53:15 -04:00
parent 198770f578
commit d05270640d
7 changed files with 907 additions and 29 deletions
@@ -77,7 +77,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder);
@@ -118,7 +123,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder);
@@ -147,7 +157,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder, parentExecutionId);
@@ -169,7 +184,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder);
@@ -207,7 +227,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder);
@@ -248,7 +273,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder, parentExecutionId);
@@ -281,7 +311,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder);
@@ -310,7 +345,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder
        {
            ThrowOnForward = new InvalidOperationException("simulated forwarder outage"),
@@ -348,7 +388,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = new ScriptRuntimeContext.DatabaseHelper(
@@ -384,7 +429,12 @@ public class DatabaseCachedWriteEmissionTests
                It.IsAny<CancellationToken>(),
                It.IsAny<TrackedOperationId?>(),
                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
-            .Returns(Task.CompletedTask);
+            // M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
+            // buffered result (WasBuffered=true) models the transient-failure
+            // path these enqueue-telemetry tests exercise — only the CachedSubmit
+            // packet is emitted; the S&F retry loop (not the helper) owns the
+            // terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
+            .ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
        var forwarder = new CapturingForwarder();

        var helper = CreateHelper(gateway.Object, forwarder);
@@ -393,4 +443,97 @@ public class DatabaseCachedWriteEmissionTests
        var packet = Assert.Single(forwarder.Telemetry);
        Assert.Null(packet.Operational.SourceNode);
    }
+
+    // ── M2.3 (#7): immediate-completion lifecycle (WasBuffered=false) ──
+
+    private static Mock<IDatabaseGateway> GatewayReturning(ExternalCallResult result)
+    {
+        var gateway = new Mock<IDatabaseGateway>();
+        gateway
+            .Setup(g => g.CachedWriteAsync(
+                It.IsAny<string>(), It.IsAny<string>(),
+                It.IsAny<IReadOnlyDictionary<string, object?>?>(),
+                It.IsAny<string?>(),
+                It.IsAny<CancellationToken>(),
+                It.IsAny<TrackedOperationId?>(),
+                It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
+            .ReturnsAsync(result);
+        return gateway;
+    }
+
+    [Fact]
+    public async Task CachedWrite_ImmediateSuccess_EmitsSubmitAttemptedThenDeliveredResolve()
+    {
+        // An immediate success (WasBuffered=false) bypasses the S&F retry loop,
+        // so the helper itself must emit the Attempted + terminal CachedResolve
+        // rows — mirroring ExternalSystem.CachedCall's immediate-success path.
+        var gateway = GatewayReturning(
+            new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: false));
+        var forwarder = new CapturingForwarder();
+
+        var helper = CreateHelper(gateway.Object, forwarder);
+        var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
+
+        Assert.Equal(3, forwarder.Telemetry.Count);
+
+        var submit = forwarder.Telemetry[0].Audit.AsRow();
+        Assert.Equal(AuditKind.CachedSubmit, submit.Kind);
+        Assert.Equal(AuditStatus.Submitted, submit.Status);
+
+        var attempted = forwarder.Telemetry[1].Audit.AsRow();
+        Assert.Equal(AuditChannel.DbOutbound, attempted.Channel);
+        Assert.Equal(AuditKind.DbWriteCached, attempted.Kind);
+        Assert.Equal(AuditStatus.Attempted, attempted.Status);
+
+        var resolve = forwarder.Telemetry[2];
+        Assert.Equal(AuditChannel.DbOutbound, resolve.Audit.AsRow().Channel);
+        Assert.Equal(AuditKind.CachedResolve, resolve.Audit.AsRow().Kind);
+        Assert.Equal(AuditStatus.Delivered, resolve.Audit.AsRow().Status);
+        Assert.Equal(trackedId.Value, resolve.Audit.AsRow().CorrelationId);
+        Assert.Equal("Delivered", resolve.Operational.Status);
+        Assert.NotNull(resolve.Operational.TerminalAtUtc);
+    }
+
+    [Fact]
+    public async Task CachedWrite_ImmediatePermanentFailure_EmitsSubmitAttemptedThenFailedResolve()
+    {
+        // A synchronous permanent SQL failure (Success=false, WasBuffered=false)
+        // also bypasses S&F; the terminal CachedResolve must be Failed and the
+        // error message must propagate onto the row.
+        const string error = "Permanent database error: Permanent SQL error 2627 on myDb: ...";
+        var gateway = GatewayReturning(
+            new ExternalCallResult(Success: false, ResponseJson: null, ErrorMessage: error, WasBuffered: false));
+        var forwarder = new CapturingForwarder();
+
+        var helper = CreateHelper(gateway.Object, forwarder);
+        var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
+
+        Assert.Equal(3, forwarder.Telemetry.Count);
+
+        var resolve = forwarder.Telemetry[2];
+        Assert.Equal(AuditKind.CachedResolve, resolve.Audit.AsRow().Kind);
+        Assert.Equal(AuditStatus.Failed, resolve.Audit.AsRow().Status);
+        Assert.Equal(error, resolve.Audit.AsRow().ErrorMessage);
+        Assert.Equal("Failed", resolve.Operational.Status);
+        Assert.Equal(error, resolve.Operational.LastError);
+        Assert.NotNull(resolve.Operational.TerminalAtUtc);
+        Assert.NotEqual(default, trackedId);
+    }
+
+    [Fact]
+    public async Task CachedWrite_BufferedTransient_EmitsOnlySubmit_NoTerminalRows()
+    {
+        // The WasBuffered=true path must NOT emit Attempted / CachedResolve — the
+        // S&F retry loop owns those. Only the CachedSubmit row is emitted by the
+        // helper.
+        var gateway = GatewayReturning(
+            new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
+        var forwarder = new CapturingForwarder();
+
+        var helper = CreateHelper(gateway.Object, forwarder);
+        await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
+
+        var packet = Assert.Single(forwarder.Telemetry);
+        Assert.Equal(AuditKind.CachedSubmit, packet.Audit.AsRow().Kind);
+    }
 }