fix(store-and-forward): inline ordered replication dispatch; Warning + counter on replication failures

This commit is contained in:
Joseph Doherty
2026-07-08 17:35:29 -04:00
parent ca5e79e922
commit 70e7dfd7b9
4 changed files with 92 additions and 11 deletions
@@ -1,4 +1,5 @@
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward;
@@ -133,19 +134,32 @@ public class ReplicationService
private void FireAndForget(ReplicationOperation operation)
{
Task.Run(async () =>
// Invoked inline, NOT via Task.Run: the handler is a non-blocking Akka
// Tell, and thread-pool hand-off destroyed Add/Remove ordering for the
// same message id (arch review 02). Inline invocation preserves issue
// order; Akka's per-sender/receiver guarantee preserves it on the wire.
try
{
try
var task = _replicationHandler!.Invoke(operation);
if (!task.IsCompletedSuccessfully)
{
await _replicationHandler!.Invoke(operation);
task.ContinueWith(t =>
{
ScadaBridgeTelemetry.RecordReplicationFailure();
_logger.LogWarning(t.Exception,
"Replication of {OpType} for message {MessageId} failed (best-effort); standby buffer may be diverging",
operation.OperationType, operation.MessageId);
},
TaskContinuationOptions.OnlyOnFaulted);
}
catch (Exception ex)
{
_logger.LogDebug(ex,
"Replication of {OpType} for message {MessageId} failed (best-effort)",
operation.OperationType, operation.MessageId);
}
});
}
catch (Exception ex)
{
ScadaBridgeTelemetry.RecordReplicationFailure();
_logger.LogWarning(ex,
"Replication of {OpType} for message {MessageId} failed (best-effort); standby buffer may be diverging",
operation.OperationType, operation.MessageId);
}
}
}