fix(db): classify transient vs permanent SQL errors in Database.CachedWrite (#7)

CachedWrite buffered ALL write failures and retried forever, never returning a
synchronous failure to the script — permanent SQL errors (constraint/syntax/
permission) were treated as transient. Mirror the External-System API path:
attempt immediately, return Failed synchronously on permanent SQL errors (no
buffering), buffer only transient errors; the S&F retry path parks permanent
failures instead of retrying forever. New SqlErrorClassifier + PermanentDatabaseException.
This commit is contained in:
Joseph Doherty
2026-06-15 13:53:15 -04:00
parent 198770f578
commit d05270640d
7 changed files with 907 additions and 29 deletions
@@ -100,7 +100,14 @@ public class DatabaseGatewayTests
var sf = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService(
storage, sfOptions, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService>.Instance);
var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance, storeAndForward: sf);
// M2.3 (#7): CachedWriteAsync now attempts the write immediately and
// only buffers on a TRANSIENT failure. The stub forces a transient
// outcome so this test exercises the buffering path deterministically
// without a real SQL Server.
var gateway = new ExecuteStubGateway(
_repository,
sf,
onExecute: () => throw new TransientDatabaseException("deadlock", errorNumber: 1205));
// Audit Log #23 (ExecutionId Task 4): a known execution id / source
// script so the gateway -> EnqueueAsync hop can be asserted below.
@@ -157,7 +164,11 @@ public class DatabaseGatewayTests
var sf = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService(
storage, sfOptions, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService>.Instance);
var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance, storeAndForward: sf);
// M2.3 (#7): force a transient outcome so the write reaches S&F.
var gateway = new ExecuteStubGateway(
_repository,
sf,
onExecute: () => throw new TransientDatabaseException("deadlock", errorNumber: 1205));
await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
@@ -167,6 +178,219 @@ public class DatabaseGatewayTests
Assert.NotEqual(0, maxRetries);
}
// ── M2.3 (#7): transient-vs-permanent SQL classification on the immediate
// cached-write attempt + the buffered retry path ──
/// <summary>
/// Builds a real, initialised in-memory store-and-forward service plus a
/// keep-alive connection (the SQLite shared-cache DB lives only while a
/// connection is open). The caller disposes <paramref name="keepAlive"/>.
/// </summary>
private static (ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService Sf, string ConnStr, Microsoft.Data.Sqlite.SqliteConnection KeepAlive)
NewStoreAndForward()
{
var dbName = $"EsgCachedWriteClassify_{Guid.NewGuid():N}";
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
var keepAlive = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
keepAlive.Open();
var storage = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardStorage(
connStr, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardStorage>.Instance);
storage.InitializeAsync().GetAwaiter().GetResult();
var sfOptions = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardOptions
{
DefaultMaxRetries = 99,
DefaultRetryInterval = TimeSpan.FromMinutes(10),
RetryTimerInterval = TimeSpan.FromMinutes(10),
};
var sf = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService(
storage, sfOptions, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService>.Instance);
return (sf, connStr, keepAlive);
}
[Fact]
public async Task CachedWrite_PermanentSqlError_ReturnsFailedSynchronously_NotBuffered()
{
// A constraint/syntax/permission failure on the IMMEDIATE attempt must
// be returned to the script as Failed and must NOT be buffered — mirrors
// ExternalSystemClient.CachedCallAsync's PermanentExternalSystemException
// path.
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
StubConnection(conn);
var (sf, connStr, keepAlive) = NewStoreAndForward();
using var _ = keepAlive;
var gateway = new ExecuteStubGateway(
_repository,
sf,
onExecute: () => throw new PermanentDatabaseException(
"Violation of PRIMARY KEY constraint", errorNumber: 2627));
var result = await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
Assert.False(result.Success);
Assert.False(result.WasBuffered);
Assert.NotNull(result.ErrorMessage);
// Nothing buffered — the permanent failure short-circuited S&F.
Assert.Equal(0, ReadBufferDepth(connStr));
}
[Fact]
public async Task CachedWrite_TransientSqlError_BuffersToStoreAndForward()
{
// A deadlock / timeout on the IMMEDIATE attempt is transient — the write
// is handed to S&F (WasBuffered=true), not returned as Failed.
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test")
{
Id = 1,
MaxRetries = 5,
RetryDelay = TimeSpan.FromSeconds(12),
};
StubConnection(conn);
var (sf, connStr, keepAlive) = NewStoreAndForward();
using var _ = keepAlive;
var gateway = new ExecuteStubGateway(
_repository,
sf,
onExecute: () => throw new TransientDatabaseException(
"Transaction was deadlocked", errorNumber: 1205));
var result = await gateway.CachedWriteAsync(
"testDb", "UPDATE t SET v = 1", new Dictionary<string, object?> { ["x"] = 1 });
Assert.True(result.Success); // accepted for delivery
Assert.True(result.WasBuffered); // handed to S&F, not synchronously failed
Assert.Null(result.ErrorMessage);
Assert.Equal(1, ReadBufferDepth(connStr));
}
[Fact]
public async Task CachedWrite_ImmediateSuccess_NotBuffered_ReturnsDelivered()
{
// A write that succeeds immediately is done — it must NOT be buffered,
// and the result reports success (WasBuffered=false), mirroring the API
// path's immediate-success behaviour.
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
StubConnection(conn);
var (sf, connStr, keepAlive) = NewStoreAndForward();
using var _ = keepAlive;
var gateway = new ExecuteStubGateway(_repository, sf, onExecute: () => { /* succeeds */ });
var result = await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
Assert.True(result.Success);
Assert.False(result.WasBuffered);
Assert.Null(result.ErrorMessage);
Assert.Equal(0, ReadBufferDepth(connStr));
}
[Fact]
public async Task DeliverBuffered_TransientSqlError_RethrowsSoEngineRetries()
{
// On the retry path a transient failure must propagate so the S&F engine
// schedules another retry — mirrors ExternalSystemClient.DeliverBuffered
// letting TransientExternalSystemException escape.
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
StubConnection(conn);
var gateway = new ExecuteStubGateway(
_repository,
storeAndForward: null,
onExecute: () => throw new TransientDatabaseException("timeout", errorNumber: -2));
var message = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardMessage
{
Id = Guid.NewGuid().ToString("N"),
Category = ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite,
Target = "testDb",
PayloadJson =
"""{"ConnectionName":"testDb","Sql":"INSERT INTO t VALUES (1)","Parameters":null}""",
};
await Assert.ThrowsAsync<TransientDatabaseException>(
() => gateway.DeliverBufferedAsync(message));
}
[Fact]
public async Task DeliverBuffered_PermanentSqlError_ReturnsFalseSoMessageParks()
{
// On the retry path a permanent failure must park the message (return
// false) rather than retry forever — mirrors ExternalSystemClient.
// DeliverBuffered returning false on PermanentExternalSystemException.
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
StubConnection(conn);
var gateway = new ExecuteStubGateway(
_repository,
storeAndForward: null,
onExecute: () => throw new PermanentDatabaseException(
"Invalid column name", errorNumber: 207));
var message = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardMessage
{
Id = Guid.NewGuid().ToString("N"),
Category = ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite,
Target = "testDb",
PayloadJson =
"""{"ConnectionName":"testDb","Sql":"INSERT INTO t VALUES (1)","Parameters":null}""",
};
var delivered = await gateway.DeliverBufferedAsync(message);
Assert.False(delivered); // permanent — the S&F engine parks the message
}
/// <summary>
/// Reads the current buffered-message count off the S&amp;F SQLite DB by
/// counting <c>sf_messages</c> rows (the engine's persistence table).
/// </summary>
private static int ReadBufferDepth(string connStr)
{
using var conn = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
conn.Open();
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM sf_messages";
return Convert.ToInt32(cmd.ExecuteScalar());
}
/// <summary>
/// Test gateway that substitutes the SQL-execution seam so a test can drive
/// success / transient / permanent outcomes without a real SQL Server (and
/// without fabricating a <see cref="Microsoft.Data.SqlClient.SqlException"/>,
/// which has no public constructor). Production classifies a real
/// <c>SqlException</c> into <see cref="TransientDatabaseException"/> /
/// <see cref="PermanentDatabaseException"/> at this same seam.
/// </summary>
private sealed class ExecuteStubGateway : DatabaseGateway
{
private readonly Action _onExecute;
public ExecuteStubGateway(
IExternalSystemRepository repository,
ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService? storeAndForward,
Action onExecute)
: base(repository, NullLogger<DatabaseGateway>.Instance, storeAndForward)
=> _onExecute = onExecute;
internal override Task ExecuteWriteAsync(
string connectionName,
string connectionString,
string sql,
IReadOnlyDictionary<string, object?> parameters,
CancellationToken cancellationToken)
{
_onExecute();
return Task.CompletedTask;
}
}
private static (int MaxRetries, long RetryIntervalMs, Guid? ExecutionId, string? SourceScript)
ReadBufferedRetrySettings(string connStr)
{
@@ -0,0 +1,65 @@
namespace ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.Tests;
/// <summary>
/// M2.3 (#7): unit tests for the transient-vs-permanent SQL error-number
/// classifier that <c>DatabaseGateway</c> uses to decide whether a failed
/// cached write should be buffered (transient) or returned to the script
/// synchronously / parked (permanent).
/// </summary>
public class SqlErrorClassifierTests
{
// The full transient set documented on SqlErrorClassifier — connection,
// timeout, deadlock, and Azure throttle error numbers. A retry can plausibly
// succeed for any of these, so they are buffered to store-and-forward.
[Theory]
[InlineData(-2)] // timeout expired
[InlineData(-1)] // connection error
[InlineData(2)] // network / instance not found
[InlineData(53)] // network path not found
[InlineData(64)] // connection terminated mid-session
[InlineData(233)] // no process on the other end of the pipe
[InlineData(1205)] // deadlock victim
[InlineData(10053)] // transport-level abort
[InlineData(10054)] // connection reset by peer
[InlineData(10060)] // connection timed out
[InlineData(40197)] // Azure SQL service error, retry
[InlineData(40501)] // Azure SQL service busy
[InlineData(40613)] // Azure SQL database unavailable
[InlineData(49918)] // Azure SQL cannot process request (throttle)
[InlineData(49919)] // Azure SQL too many create/update operations
[InlineData(49920)] // Azure SQL too many operations (throttle)
public void IsTransient_KnownTransientNumber_ReturnsTrue(int errorNumber)
{
Assert.True(SqlErrorClassifier.IsTransient(errorNumber));
}
// Constraint, syntax, and permission errors are permanent — retrying the
// identical statement cannot succeed and may cause duplicate side effects.
[Theory]
[InlineData(547)] // constraint violation (FK/CHECK)
[InlineData(2627)] // primary-key / unique constraint violation
[InlineData(2601)] // duplicate key in a unique index
[InlineData(102)] // incorrect syntax
[InlineData(156)] // incorrect syntax near a keyword
[InlineData(207)] // invalid column name
[InlineData(208)] // invalid object name
[InlineData(229)] // permission denied on object
[InlineData(230)] // permission denied on column
[InlineData(262)] // permission denied (CREATE etc.)
public void IsTransient_KnownPermanentNumber_ReturnsFalse(int errorNumber)
{
Assert.False(SqlErrorClassifier.IsTransient(errorNumber));
}
[Theory]
[InlineData(0)] // no error number captured
[InlineData(99999)] // unknown / undocumented number
[InlineData(12345)]
[InlineData(int.MaxValue)]
public void IsTransient_UnknownNumber_DefaultsToPermanent(int errorNumber)
{
// Fail-fast is the safer default: an unrecognised error number must NOT
// be silently retried forever. Unknown => permanent => false.
Assert.False(SqlErrorClassifier.IsTransient(errorNumber));
}
}
@@ -77,7 +77,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
@@ -118,7 +123,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
@@ -147,7 +157,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder, parentExecutionId);
@@ -169,7 +184,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
@@ -207,7 +227,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
@@ -248,7 +273,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder, parentExecutionId);
@@ -281,7 +311,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
@@ -310,7 +345,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder
{
ThrowOnForward = new InvalidOperationException("simulated forwarder outage"),
@@ -348,7 +388,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = new ScriptRuntimeContext.DatabaseHelper(
@@ -384,7 +429,12 @@ public class DatabaseCachedWriteEmissionTests
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.Returns(Task.CompletedTask);
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
// buffered result (WasBuffered=true) models the transient-failure
// path these enqueue-telemetry tests exercise — only the CachedSubmit
// packet is emitted; the S&F retry loop (not the helper) owns the
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
@@ -393,4 +443,97 @@ public class DatabaseCachedWriteEmissionTests
var packet = Assert.Single(forwarder.Telemetry);
Assert.Null(packet.Operational.SourceNode);
}
// ── M2.3 (#7): immediate-completion lifecycle (WasBuffered=false) ──
private static Mock<IDatabaseGateway> GatewayReturning(ExternalCallResult result)
{
var gateway = new Mock<IDatabaseGateway>();
gateway
.Setup(g => g.CachedWriteAsync(
It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
It.IsAny<string?>(),
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.ReturnsAsync(result);
return gateway;
}
[Fact]
public async Task CachedWrite_ImmediateSuccess_EmitsSubmitAttemptedThenDeliveredResolve()
{
// An immediate success (WasBuffered=false) bypasses the S&F retry loop,
// so the helper itself must emit the Attempted + terminal CachedResolve
// rows — mirroring ExternalSystem.CachedCall's immediate-success path.
var gateway = GatewayReturning(
new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
Assert.Equal(3, forwarder.Telemetry.Count);
var submit = forwarder.Telemetry[0].Audit.AsRow();
Assert.Equal(AuditKind.CachedSubmit, submit.Kind);
Assert.Equal(AuditStatus.Submitted, submit.Status);
var attempted = forwarder.Telemetry[1].Audit.AsRow();
Assert.Equal(AuditChannel.DbOutbound, attempted.Channel);
Assert.Equal(AuditKind.DbWriteCached, attempted.Kind);
Assert.Equal(AuditStatus.Attempted, attempted.Status);
var resolve = forwarder.Telemetry[2];
Assert.Equal(AuditChannel.DbOutbound, resolve.Audit.AsRow().Channel);
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.AsRow().Kind);
Assert.Equal(AuditStatus.Delivered, resolve.Audit.AsRow().Status);
Assert.Equal(trackedId.Value, resolve.Audit.AsRow().CorrelationId);
Assert.Equal("Delivered", resolve.Operational.Status);
Assert.NotNull(resolve.Operational.TerminalAtUtc);
}
[Fact]
public async Task CachedWrite_ImmediatePermanentFailure_EmitsSubmitAttemptedThenFailedResolve()
{
// A synchronous permanent SQL failure (Success=false, WasBuffered=false)
// also bypasses S&F; the terminal CachedResolve must be Failed and the
// error message must propagate onto the row.
const string error = "Permanent database error: Permanent SQL error 2627 on myDb: ...";
var gateway = GatewayReturning(
new ExternalCallResult(Success: false, ResponseJson: null, ErrorMessage: error, WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
Assert.Equal(3, forwarder.Telemetry.Count);
var resolve = forwarder.Telemetry[2];
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.AsRow().Kind);
Assert.Equal(AuditStatus.Failed, resolve.Audit.AsRow().Status);
Assert.Equal(error, resolve.Audit.AsRow().ErrorMessage);
Assert.Equal("Failed", resolve.Operational.Status);
Assert.Equal(error, resolve.Operational.LastError);
Assert.NotNull(resolve.Operational.TerminalAtUtc);
Assert.NotEqual(default, trackedId);
}
[Fact]
public async Task CachedWrite_BufferedTransient_EmitsOnlySubmit_NoTerminalRows()
{
// The WasBuffered=true path must NOT emit Attempted / CachedResolve — the
// S&F retry loop owns those. Only the CachedSubmit row is emitted by the
// helper.
var gateway = GatewayReturning(
new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(gateway.Object, forwarder);
await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
var packet = Assert.Single(forwarder.Telemetry);
Assert.Equal(AuditKind.CachedSubmit, packet.Audit.AsRow().Kind);
}
}