fix(db): classify transient vs permanent SQL errors in Database.CachedWrite (#7)
CachedWrite buffered ALL write failures and retried forever, never returning a synchronous failure to the script — permanent SQL errors (constraint/syntax/ permission) were treated as transient. Mirror the External-System API path: attempt immediately, return Failed synchronously on permanent SQL errors (no buffering), buffer only transient errors; the S&F retry path parks permanent failures instead of retrying forever. New SqlErrorClassifier + PermanentDatabaseException.
This commit is contained in:
@@ -100,7 +100,14 @@ public class DatabaseGatewayTests
|
||||
var sf = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService(
|
||||
storage, sfOptions, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService>.Instance);
|
||||
|
||||
var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance, storeAndForward: sf);
|
||||
// M2.3 (#7): CachedWriteAsync now attempts the write immediately and
|
||||
// only buffers on a TRANSIENT failure. The stub forces a transient
|
||||
// outcome so this test exercises the buffering path deterministically
|
||||
// without a real SQL Server.
|
||||
var gateway = new ExecuteStubGateway(
|
||||
_repository,
|
||||
sf,
|
||||
onExecute: () => throw new TransientDatabaseException("deadlock", errorNumber: 1205));
|
||||
|
||||
// Audit Log #23 (ExecutionId Task 4): a known execution id / source
|
||||
// script so the gateway -> EnqueueAsync hop can be asserted below.
|
||||
@@ -157,7 +164,11 @@ public class DatabaseGatewayTests
|
||||
var sf = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService(
|
||||
storage, sfOptions, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService>.Instance);
|
||||
|
||||
var gateway = new DatabaseGateway(_repository, NullLogger<DatabaseGateway>.Instance, storeAndForward: sf);
|
||||
// M2.3 (#7): force a transient outcome so the write reaches S&F.
|
||||
var gateway = new ExecuteStubGateway(
|
||||
_repository,
|
||||
sf,
|
||||
onExecute: () => throw new TransientDatabaseException("deadlock", errorNumber: 1205));
|
||||
|
||||
await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
|
||||
|
||||
@@ -167,6 +178,219 @@ public class DatabaseGatewayTests
|
||||
Assert.NotEqual(0, maxRetries);
|
||||
}
|
||||
|
||||
// ── M2.3 (#7): transient-vs-permanent SQL classification on the immediate
|
||||
// cached-write attempt + the buffered retry path ──
|
||||
|
||||
/// <summary>
|
||||
/// Builds a real, initialised in-memory store-and-forward service plus a
|
||||
/// keep-alive connection (the SQLite shared-cache DB lives only while a
|
||||
/// connection is open). The caller disposes <paramref name="keepAlive"/>.
|
||||
/// </summary>
|
||||
private static (ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService Sf, string ConnStr, Microsoft.Data.Sqlite.SqliteConnection KeepAlive)
|
||||
NewStoreAndForward()
|
||||
{
|
||||
var dbName = $"EsgCachedWriteClassify_{Guid.NewGuid():N}";
|
||||
var connStr = $"Data Source={dbName};Mode=Memory;Cache=Shared";
|
||||
var keepAlive = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
|
||||
keepAlive.Open();
|
||||
var storage = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardStorage(
|
||||
connStr, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardStorage>.Instance);
|
||||
storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
var sfOptions = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardOptions
|
||||
{
|
||||
DefaultMaxRetries = 99,
|
||||
DefaultRetryInterval = TimeSpan.FromMinutes(10),
|
||||
RetryTimerInterval = TimeSpan.FromMinutes(10),
|
||||
};
|
||||
var sf = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService(
|
||||
storage, sfOptions, NullLogger<ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService>.Instance);
|
||||
return (sf, connStr, keepAlive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CachedWrite_PermanentSqlError_ReturnsFailedSynchronously_NotBuffered()
|
||||
{
|
||||
// A constraint/syntax/permission failure on the IMMEDIATE attempt must
|
||||
// be returned to the script as Failed and must NOT be buffered — mirrors
|
||||
// ExternalSystemClient.CachedCallAsync's PermanentExternalSystemException
|
||||
// path.
|
||||
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
|
||||
StubConnection(conn);
|
||||
|
||||
var (sf, connStr, keepAlive) = NewStoreAndForward();
|
||||
using var _ = keepAlive;
|
||||
|
||||
var gateway = new ExecuteStubGateway(
|
||||
_repository,
|
||||
sf,
|
||||
onExecute: () => throw new PermanentDatabaseException(
|
||||
"Violation of PRIMARY KEY constraint", errorNumber: 2627));
|
||||
|
||||
var result = await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.False(result.WasBuffered);
|
||||
Assert.NotNull(result.ErrorMessage);
|
||||
|
||||
// Nothing buffered — the permanent failure short-circuited S&F.
|
||||
Assert.Equal(0, ReadBufferDepth(connStr));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CachedWrite_TransientSqlError_BuffersToStoreAndForward()
|
||||
{
|
||||
// A deadlock / timeout on the IMMEDIATE attempt is transient — the write
|
||||
// is handed to S&F (WasBuffered=true), not returned as Failed.
|
||||
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test")
|
||||
{
|
||||
Id = 1,
|
||||
MaxRetries = 5,
|
||||
RetryDelay = TimeSpan.FromSeconds(12),
|
||||
};
|
||||
StubConnection(conn);
|
||||
|
||||
var (sf, connStr, keepAlive) = NewStoreAndForward();
|
||||
using var _ = keepAlive;
|
||||
|
||||
var gateway = new ExecuteStubGateway(
|
||||
_repository,
|
||||
sf,
|
||||
onExecute: () => throw new TransientDatabaseException(
|
||||
"Transaction was deadlocked", errorNumber: 1205));
|
||||
|
||||
var result = await gateway.CachedWriteAsync(
|
||||
"testDb", "UPDATE t SET v = 1", new Dictionary<string, object?> { ["x"] = 1 });
|
||||
|
||||
Assert.True(result.Success); // accepted for delivery
|
||||
Assert.True(result.WasBuffered); // handed to S&F, not synchronously failed
|
||||
Assert.Null(result.ErrorMessage);
|
||||
|
||||
Assert.Equal(1, ReadBufferDepth(connStr));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CachedWrite_ImmediateSuccess_NotBuffered_ReturnsDelivered()
|
||||
{
|
||||
// A write that succeeds immediately is done — it must NOT be buffered,
|
||||
// and the result reports success (WasBuffered=false), mirroring the API
|
||||
// path's immediate-success behaviour.
|
||||
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
|
||||
StubConnection(conn);
|
||||
|
||||
var (sf, connStr, keepAlive) = NewStoreAndForward();
|
||||
using var _ = keepAlive;
|
||||
|
||||
var gateway = new ExecuteStubGateway(_repository, sf, onExecute: () => { /* succeeds */ });
|
||||
|
||||
var result = await gateway.CachedWriteAsync("testDb", "INSERT INTO t VALUES (1)");
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.False(result.WasBuffered);
|
||||
Assert.Null(result.ErrorMessage);
|
||||
|
||||
Assert.Equal(0, ReadBufferDepth(connStr));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeliverBuffered_TransientSqlError_RethrowsSoEngineRetries()
|
||||
{
|
||||
// On the retry path a transient failure must propagate so the S&F engine
|
||||
// schedules another retry — mirrors ExternalSystemClient.DeliverBuffered
|
||||
// letting TransientExternalSystemException escape.
|
||||
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
|
||||
StubConnection(conn);
|
||||
|
||||
var gateway = new ExecuteStubGateway(
|
||||
_repository,
|
||||
storeAndForward: null,
|
||||
onExecute: () => throw new TransientDatabaseException("timeout", errorNumber: -2));
|
||||
|
||||
var message = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite,
|
||||
Target = "testDb",
|
||||
PayloadJson =
|
||||
"""{"ConnectionName":"testDb","Sql":"INSERT INTO t VALUES (1)","Parameters":null}""",
|
||||
};
|
||||
|
||||
await Assert.ThrowsAsync<TransientDatabaseException>(
|
||||
() => gateway.DeliverBufferedAsync(message));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeliverBuffered_PermanentSqlError_ReturnsFalseSoMessageParks()
|
||||
{
|
||||
// On the retry path a permanent failure must park the message (return
|
||||
// false) rather than retry forever — mirrors ExternalSystemClient.
|
||||
// DeliverBuffered returning false on PermanentExternalSystemException.
|
||||
var conn = new DatabaseConnectionDefinition("testDb", "Server=localhost;Database=test") { Id = 1 };
|
||||
StubConnection(conn);
|
||||
|
||||
var gateway = new ExecuteStubGateway(
|
||||
_repository,
|
||||
storeAndForward: null,
|
||||
onExecute: () => throw new PermanentDatabaseException(
|
||||
"Invalid column name", errorNumber: 207));
|
||||
|
||||
var message = new ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardMessage
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.StoreAndForwardCategory.CachedDbWrite,
|
||||
Target = "testDb",
|
||||
PayloadJson =
|
||||
"""{"ConnectionName":"testDb","Sql":"INSERT INTO t VALUES (1)","Parameters":null}""",
|
||||
};
|
||||
|
||||
var delivered = await gateway.DeliverBufferedAsync(message);
|
||||
|
||||
Assert.False(delivered); // permanent — the S&F engine parks the message
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the current buffered-message count off the S&F SQLite DB by
|
||||
/// counting <c>sf_messages</c> rows (the engine's persistence table).
|
||||
/// </summary>
|
||||
private static int ReadBufferDepth(string connStr)
|
||||
{
|
||||
using var conn = new Microsoft.Data.Sqlite.SqliteConnection(connStr);
|
||||
conn.Open();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM sf_messages";
|
||||
return Convert.ToInt32(cmd.ExecuteScalar());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test gateway that substitutes the SQL-execution seam so a test can drive
|
||||
/// success / transient / permanent outcomes without a real SQL Server (and
|
||||
/// without fabricating a <see cref="Microsoft.Data.SqlClient.SqlException"/>,
|
||||
/// which has no public constructor). Production classifies a real
|
||||
/// <c>SqlException</c> into <see cref="TransientDatabaseException"/> /
|
||||
/// <see cref="PermanentDatabaseException"/> at this same seam.
|
||||
/// </summary>
|
||||
private sealed class ExecuteStubGateway : DatabaseGateway
|
||||
{
|
||||
private readonly Action _onExecute;
|
||||
|
||||
public ExecuteStubGateway(
|
||||
IExternalSystemRepository repository,
|
||||
ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardService? storeAndForward,
|
||||
Action onExecute)
|
||||
: base(repository, NullLogger<DatabaseGateway>.Instance, storeAndForward)
|
||||
=> _onExecute = onExecute;
|
||||
|
||||
internal override Task ExecuteWriteAsync(
|
||||
string connectionName,
|
||||
string connectionString,
|
||||
string sql,
|
||||
IReadOnlyDictionary<string, object?> parameters,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
_onExecute();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private static (int MaxRetries, long RetryIntervalMs, Guid? ExecutionId, string? SourceScript)
|
||||
ReadBufferedRetrySettings(string connStr)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.ExternalSystemGateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// M2.3 (#7): unit tests for the transient-vs-permanent SQL error-number
|
||||
/// classifier that <c>DatabaseGateway</c> uses to decide whether a failed
|
||||
/// cached write should be buffered (transient) or returned to the script
|
||||
/// synchronously / parked (permanent).
|
||||
/// </summary>
|
||||
public class SqlErrorClassifierTests
|
||||
{
|
||||
// The full transient set documented on SqlErrorClassifier — connection,
|
||||
// timeout, deadlock, and Azure throttle error numbers. A retry can plausibly
|
||||
// succeed for any of these, so they are buffered to store-and-forward.
|
||||
[Theory]
|
||||
[InlineData(-2)] // timeout expired
|
||||
[InlineData(-1)] // connection error
|
||||
[InlineData(2)] // network / instance not found
|
||||
[InlineData(53)] // network path not found
|
||||
[InlineData(64)] // connection terminated mid-session
|
||||
[InlineData(233)] // no process on the other end of the pipe
|
||||
[InlineData(1205)] // deadlock victim
|
||||
[InlineData(10053)] // transport-level abort
|
||||
[InlineData(10054)] // connection reset by peer
|
||||
[InlineData(10060)] // connection timed out
|
||||
[InlineData(40197)] // Azure SQL service error, retry
|
||||
[InlineData(40501)] // Azure SQL service busy
|
||||
[InlineData(40613)] // Azure SQL database unavailable
|
||||
[InlineData(49918)] // Azure SQL cannot process request (throttle)
|
||||
[InlineData(49919)] // Azure SQL too many create/update operations
|
||||
[InlineData(49920)] // Azure SQL too many operations (throttle)
|
||||
public void IsTransient_KnownTransientNumber_ReturnsTrue(int errorNumber)
|
||||
{
|
||||
Assert.True(SqlErrorClassifier.IsTransient(errorNumber));
|
||||
}
|
||||
|
||||
// Constraint, syntax, and permission errors are permanent — retrying the
|
||||
// identical statement cannot succeed and may cause duplicate side effects.
|
||||
[Theory]
|
||||
[InlineData(547)] // constraint violation (FK/CHECK)
|
||||
[InlineData(2627)] // primary-key / unique constraint violation
|
||||
[InlineData(2601)] // duplicate key in a unique index
|
||||
[InlineData(102)] // incorrect syntax
|
||||
[InlineData(156)] // incorrect syntax near a keyword
|
||||
[InlineData(207)] // invalid column name
|
||||
[InlineData(208)] // invalid object name
|
||||
[InlineData(229)] // permission denied on object
|
||||
[InlineData(230)] // permission denied on column
|
||||
[InlineData(262)] // permission denied (CREATE etc.)
|
||||
public void IsTransient_KnownPermanentNumber_ReturnsFalse(int errorNumber)
|
||||
{
|
||||
Assert.False(SqlErrorClassifier.IsTransient(errorNumber));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)] // no error number captured
|
||||
[InlineData(99999)] // unknown / undocumented number
|
||||
[InlineData(12345)]
|
||||
[InlineData(int.MaxValue)]
|
||||
public void IsTransient_UnknownNumber_DefaultsToPermanent(int errorNumber)
|
||||
{
|
||||
// Fail-fast is the safer default: an unrecognised error number must NOT
|
||||
// be silently retried forever. Unknown => permanent => false.
|
||||
Assert.False(SqlErrorClassifier.IsTransient(errorNumber));
|
||||
}
|
||||
}
|
||||
+153
-10
@@ -77,7 +77,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
@@ -118,7 +123,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
@@ -147,7 +157,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder, parentExecutionId);
|
||||
@@ -169,7 +184,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
@@ -207,7 +227,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
@@ -248,7 +273,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder, parentExecutionId);
|
||||
@@ -281,7 +311,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
@@ -310,7 +345,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder
|
||||
{
|
||||
ThrowOnForward = new InvalidOperationException("simulated forwarder outage"),
|
||||
@@ -348,7 +388,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = new ScriptRuntimeContext.DatabaseHelper(
|
||||
@@ -384,7 +429,12 @@ public class DatabaseCachedWriteEmissionTests
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
// M2.3 (#7): CachedWriteAsync now returns an ExternalCallResult. The
|
||||
// buffered result (WasBuffered=true) models the transient-failure
|
||||
// path these enqueue-telemetry tests exercise — only the CachedSubmit
|
||||
// packet is emitted; the S&F retry loop (not the helper) owns the
|
||||
// terminal rows, so Assert.Single(forwarder.Telemetry) still holds.
|
||||
.ReturnsAsync(new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
@@ -393,4 +443,97 @@ public class DatabaseCachedWriteEmissionTests
|
||||
var packet = Assert.Single(forwarder.Telemetry);
|
||||
Assert.Null(packet.Operational.SourceNode);
|
||||
}
|
||||
|
||||
// ── M2.3 (#7): immediate-completion lifecycle (WasBuffered=false) ──
|
||||
|
||||
private static Mock<IDatabaseGateway> GatewayReturning(ExternalCallResult result)
|
||||
{
|
||||
var gateway = new Mock<IDatabaseGateway>();
|
||||
gateway
|
||||
.Setup(g => g.CachedWriteAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
|
||||
It.IsAny<string?>(),
|
||||
It.IsAny<CancellationToken>(),
|
||||
It.IsAny<TrackedOperationId?>(),
|
||||
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
|
||||
.ReturnsAsync(result);
|
||||
return gateway;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CachedWrite_ImmediateSuccess_EmitsSubmitAttemptedThenDeliveredResolve()
|
||||
{
|
||||
// An immediate success (WasBuffered=false) bypasses the S&F retry loop,
|
||||
// so the helper itself must emit the Attempted + terminal CachedResolve
|
||||
// rows — mirroring ExternalSystem.CachedCall's immediate-success path.
|
||||
var gateway = GatewayReturning(
|
||||
new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: false));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
|
||||
|
||||
Assert.Equal(3, forwarder.Telemetry.Count);
|
||||
|
||||
var submit = forwarder.Telemetry[0].Audit.AsRow();
|
||||
Assert.Equal(AuditKind.CachedSubmit, submit.Kind);
|
||||
Assert.Equal(AuditStatus.Submitted, submit.Status);
|
||||
|
||||
var attempted = forwarder.Telemetry[1].Audit.AsRow();
|
||||
Assert.Equal(AuditChannel.DbOutbound, attempted.Channel);
|
||||
Assert.Equal(AuditKind.DbWriteCached, attempted.Kind);
|
||||
Assert.Equal(AuditStatus.Attempted, attempted.Status);
|
||||
|
||||
var resolve = forwarder.Telemetry[2];
|
||||
Assert.Equal(AuditChannel.DbOutbound, resolve.Audit.AsRow().Channel);
|
||||
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.AsRow().Kind);
|
||||
Assert.Equal(AuditStatus.Delivered, resolve.Audit.AsRow().Status);
|
||||
Assert.Equal(trackedId.Value, resolve.Audit.AsRow().CorrelationId);
|
||||
Assert.Equal("Delivered", resolve.Operational.Status);
|
||||
Assert.NotNull(resolve.Operational.TerminalAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CachedWrite_ImmediatePermanentFailure_EmitsSubmitAttemptedThenFailedResolve()
|
||||
{
|
||||
// A synchronous permanent SQL failure (Success=false, WasBuffered=false)
|
||||
// also bypasses S&F; the terminal CachedResolve must be Failed and the
|
||||
// error message must propagate onto the row.
|
||||
const string error = "Permanent database error: Permanent SQL error 2627 on myDb: ...";
|
||||
var gateway = GatewayReturning(
|
||||
new ExternalCallResult(Success: false, ResponseJson: null, ErrorMessage: error, WasBuffered: false));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
var trackedId = await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
|
||||
|
||||
Assert.Equal(3, forwarder.Telemetry.Count);
|
||||
|
||||
var resolve = forwarder.Telemetry[2];
|
||||
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.AsRow().Kind);
|
||||
Assert.Equal(AuditStatus.Failed, resolve.Audit.AsRow().Status);
|
||||
Assert.Equal(error, resolve.Audit.AsRow().ErrorMessage);
|
||||
Assert.Equal("Failed", resolve.Operational.Status);
|
||||
Assert.Equal(error, resolve.Operational.LastError);
|
||||
Assert.NotNull(resolve.Operational.TerminalAtUtc);
|
||||
Assert.NotEqual(default, trackedId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CachedWrite_BufferedTransient_EmitsOnlySubmit_NoTerminalRows()
|
||||
{
|
||||
// The WasBuffered=true path must NOT emit Attempted / CachedResolve — the
|
||||
// S&F retry loop owns those. Only the CachedSubmit row is emitted by the
|
||||
// helper.
|
||||
var gateway = GatewayReturning(
|
||||
new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true));
|
||||
var forwarder = new CapturingForwarder();
|
||||
|
||||
var helper = CreateHelper(gateway.Object, forwarder);
|
||||
await helper.CachedWrite("myDb", "INSERT INTO t VALUES (1)");
|
||||
|
||||
var packet = Assert.Single(forwarder.Telemetry);
|
||||
Assert.Equal(AuditKind.CachedSubmit, packet.Audit.AsRow().Kind);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user