dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(db): classify transient vs permanent SQL errors in Database.CachedWrite (#7)

Browse Source 
CachedWrite buffered ALL write failures and retried forever, never returning a
synchronous failure to the script — permanent SQL errors (constraint/syntax/
permission) were treated as transient. Mirror the External-System API path:
attempt immediately, return Failed synchronously on permanent SQL errors (no
buffering), buffer only transient errors; the S&F retry path parks permanent
failures instead of retrying forever. New SqlErrorClassifier + PermanentDatabaseException.
This commit is contained in:
Joseph Doherty
2026-06-15 13:53:15 -04:00
parent 198770f578
commit d05270640d
7 changed files with 907 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.ExternalSystemGateway/DatabaseGateway.cs
+141 -14
View File
@@ -75,7 +75,7 @@ public class DatabaseGateway : IDatabaseGateway
new SqlConnection(connectionString);
/// <inheritdoc />
public async Task CachedWriteAsync(
public async Task<ExternalCallResult> CachedWriteAsync(
string connectionName,
string sql,
IReadOnlyDictionary<string, object?>? parameters = null,
@@ -97,6 +97,44 @@ public class DatabaseGateway : IDatabaseGateway
throw new InvalidOperationException("Store-and-forward service not available for cached writes");
}
// M2.3 (#7): attempt the write IMMEDIATELY and classify the outcome,
// mirroring ExternalSystemClient.CachedCallAsync. The pre-M2.3 behaviour
// enqueued every write unconditionally and the S&F retry sweep then
// retried ALL failures forever — a permanent SQL error (constraint,
// syntax, permission) was never returned to the script and spun in the
// buffer indefinitely. Now:
// * success -> Delivered, NOT buffered;
// * PermanentDatabaseException -> Failed synchronously, NOT buffered;
// * TransientDatabaseException -> buffered to S&F for retry.
try
{
await ExecuteWriteAsync(
connectionName, definition.ConnectionString, sql, parameters ?? EmptyParameters, cancellationToken)
.ConfigureAwait(false);
// Immediate success — the write is done; do not buffer.
return new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: false);
}
catch (PermanentDatabaseException ex)
{
// Permanent failures are returned to the script and never buffered —
// mirrors the PermanentExternalSystemException branch on the API path.
_logger.LogWarning(
ex,
"CachedWrite to '{Connection}' failed permanently (SQL error {Number}); returning Failed without buffering.",
connectionName, ex.SqlErrorNumber);
return new ExternalCallResult(
Success: false, ResponseJson: null, ErrorMessage: $"Permanent database error: {ex.Message}", WasBuffered: false);
}
catch (TransientDatabaseException ex)
{
// Transient failure — hand to S&F so the retry sweep delivers it.
_logger.LogDebug(
ex,
"CachedWrite to '{Connection}' failed transiently (SQL error {Number}); buffering for retry.",
connectionName, ex.SqlErrorNumber);
}
var payload = JsonSerializer.Serialize(new
{
ConnectionName = connectionName,
@@ -119,6 +157,12 @@ public class DatabaseGateway : IDatabaseGateway
originInstanceName,
definition.MaxRetries > 0 ? definition.MaxRetries : null,
definition.RetryDelay > TimeSpan.Zero ? definition.RetryDelay : null,
// M2.3 (#7): attemptImmediateDelivery: false — this method already
// made the write attempt above (the transient-classified failure is
// exactly why we are buffering). Letting EnqueueAsync re-invoke the
// delivery handler would execute the same write a second time —
// mirrors ExternalSystemClient.CachedCallAsync.
attemptImmediateDelivery: false,
// Audit Log #23 (M3): pin the S&F message id to the
// TrackedOperationId so the retry loop (Bundle E Tasks E4/E5) can
// read it back via StoreAndForwardMessage.Id and emit per-attempt +
@@ -136,17 +180,29 @@ public class DatabaseGateway : IDatabaseGateway
// retry-loop cached-write audit rows correlate back to the
// cross-execution chain. Null for a non-routed run.
parentExecutionId: parentExecutionId);
// Buffered for retry — mirrors the API path's WasBuffered=true result.
return new ExternalCallResult(Success: true, ResponseJson: null, ErrorMessage: null, WasBuffered: true);
}
/// <summary>
/// WP-9/10: Delivers a buffered CachedDbWrite during a store-and-forward retry
/// sweep — executes the SQL against the named connection. Returns true on
/// success, false if the connection no longer exists (the message is parked);
/// throws on any execution error so the engine retries.
/// sweep — executes the SQL against the named connection.
/// </summary>
/// <remarks>
/// M2.3 (#7): the outcome is classified, mirroring
/// <see cref="ExternalSystemClient.DeliverBufferedAsync"/>. Returns
/// <c>false</c> — so the S&amp;F engine PARKS the message — when the
/// connection no longer exists, the payload is unreadable, or the SQL fails
/// with a PERMANENT error (constraint / syntax / permission). A TRANSIENT SQL
/// error (<see cref="TransientDatabaseException"/>) propagates so the engine
/// retries. The pre-M2.3 code rethrew on ANY SQL error, so a permanent
/// failure on the retry path looped forever.
/// </remarks>
/// <param name="message">The buffered store-and-forward message to deliver.</param>
/// <param name="cancellationToken">Cancellation token for the delivery operation.</param>
/// <returns>A task that resolves to <c>true</c> on success, or <c>false</c> if the connection no longer exists.</returns>
/// <returns>A task that resolves to <c>true</c> on success, or <c>false</c> when the message must be parked.</returns>
/// <exception cref="TransientDatabaseException">Thrown on a transient SQL failure so the engine retries.</exception>
public async Task<bool> DeliverBufferedAsync(
StoreAndForwardMessage message, CancellationToken cancellationToken = default)
{
@@ -185,22 +241,93 @@ public class DatabaseGateway : IDatabaseGateway
return false;
}
await using var connection = new SqlConnection(definition.ConnectionString);
await connection.OpenAsync(cancellationToken);
using var command = connection.CreateCommand();
command.CommandText = payload.Sql;
if (payload.Parameters != null)
// Materialise the buffered JsonElement parameters into CLR values once,
// then run through the shared ExecuteWriteAsync seam so both the
// immediate-attempt path and this retry path classify SqlException the
// same way.
IReadOnlyDictionary<string, object?> materialisedParameters =
payload.Parameters == null
? EmptyParameters
: payload.Parameters.ToDictionary(
kv => kv.Key, kv => (object?)JsonElementToParameterValue(kv.Value));
try
{
foreach (var (key, value) in payload.Parameters)
await ExecuteWriteAsync(
payload.ConnectionName, definition.ConnectionString, payload.Sql, materialisedParameters, cancellationToken)
.ConfigureAwait(false);
return true;
}
catch (PermanentDatabaseException ex)
{
// Permanent — parking is correct; retrying the identical statement
// cannot succeed. Mirrors ExternalSystemClient.DeliverBufferedAsync
// returning false on PermanentExternalSystemException.
_logger.LogError(
ex,
"Buffered DB write to '{Connection}' failed permanently (SQL error {Number}); parking.",
payload.ConnectionName, ex.SqlErrorNumber);
return false;
}
// TransientDatabaseException propagates — the S&F engine retries.
}
/// <summary>
/// Reusable empty parameter map so the no-parameter paths do not allocate a
/// fresh dictionary each call.
/// </summary>
private static readonly IReadOnlyDictionary<string, object?> EmptyParameters =
new Dictionary<string, object?>();
/// <summary>
/// M2.3 (#7): executes a parameterised SQL write against the given connection
/// string and classifies any <see cref="SqlException"/> into
/// <see cref="TransientDatabaseException"/> / <see cref="PermanentDatabaseException"/>
/// via <see cref="SqlErrorClassifier"/>. This is the single SQL-execution seam
/// shared by the immediate <see cref="CachedWriteAsync"/> attempt and the
/// <see cref="DeliverBufferedAsync"/> retry path. Marked <c>internal virtual</c>
/// so tests can substitute success / transient / permanent outcomes without a
/// real SQL Server (and without fabricating a <see cref="SqlException"/>, which
/// has no public constructor). Mirrors the role of
/// <see cref="ExternalSystemClient.InvokeHttpAsync"/> on the API path.
/// </summary>
/// <param name="connectionName">The human-readable connection name, used only for the classified error message (never the connection string — that would leak credentials into logs / script-visible errors).</param>
/// <param name="connectionString">The ADO.NET connection string to write through.</param>
/// <param name="sql">The SQL statement to execute.</param>
/// <param name="parameters">Materialised CLR parameter values (may be empty).</param>
/// <param name="cancellationToken">Cancellation token for the write.</param>
/// <returns>A task that completes when the write succeeds.</returns>
/// <exception cref="TransientDatabaseException">Thrown for a transient SQL error number.</exception>
/// <exception cref="PermanentDatabaseException">Thrown for a permanent (or unknown) SQL error number.</exception>
internal virtual async Task ExecuteWriteAsync(
string connectionName,
string connectionString,
string sql,
IReadOnlyDictionary<string, object?> parameters,
CancellationToken cancellationToken)
{
try
{
await using var connection = new SqlConnection(connectionString);
await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
using var command = connection.CreateCommand();
command.CommandText = sql;
foreach (var (key, value) in parameters)
{
var parameter = command.CreateParameter();
parameter.ParameterName = key.StartsWith('@') ? key : "@" + key;
parameter.Value = JsonElementToParameterValue(value);
parameter.Value = value ?? DBNull.Value;
command.Parameters.Add(parameter);
}
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
}
catch (SqlException ex)
{
// Classify by SqlException.Number and rethrow as the strongly-typed
// transient / permanent failure the callers branch on. The context
// is the connection NAME, never the connection string.
throw SqlErrorClassifier.Throw(connectionName, ex);
}
await command.ExecuteNonQueryAsync(cancellationToken);
return true;
}
// ExternalSystemGateway-020: a JSON number that does not fit in Int64 must