fix(health): acked SendAsync transport — report-loss counter restore is live, not dead code

This commit is contained in:
Joseph Doherty
2026-07-08 16:12:48 -04:00
parent 17af376a8e
commit 6300d6d399
5 changed files with 142 additions and 16 deletions
@@ -140,10 +140,13 @@ public class HealthReportSender : BackgroundService
var seq = Interlocked.Increment(ref _sequenceNumber);
// CollectReport atomically read-and-resets
// the per-interval error counters via Interlocked.Exchange. If
// the Send below throws, those counts are otherwise lost
// forever — neither in the un-sent report nor in the now-zeroed
// collector. Snapshot the freshly-collected report so that on a
// the per-interval error counters via Interlocked.Exchange. The
// transport is now an ACKED round-trip (SendAsync throws on
// timeout/nack — review 01 [Medium]), so this restore path is live
// rather than dead code. If the SendAsync below throws, those
// counts are otherwise lost forever — neither in the un-sent
// report nor in the now-zeroed collector. Snapshot the
// freshly-collected report so that on a
// transport failure we can atomically restore the counts back
// into the collector via Interlocked.Add, so the next
// successful report includes them. Concurrent increments
@@ -157,7 +160,7 @@ public class HealthReportSender : BackgroundService
try
{
_transport.Send(reportWithSeq);
await _transport.SendAsync(reportWithSeq, stoppingToken).ConfigureAwait(false);
}
catch
{
@@ -4,13 +4,20 @@ namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
/// <summary>
/// Abstraction for sending health reports to central.
/// In production, implemented via Akka remoting (Tell, fire-and-forget).
/// In production, implemented over Akka ClusterClient with an acked round-trip
/// (review 01 [Medium]): the transport Asks and awaits a
/// <see cref="SiteHealthReportAck"/>, so a lost/rejected report surfaces as a
/// fault the sender can observe (and its per-interval counter-restore fires).
/// </summary>
public interface IHealthReportTransport
{
/// <summary>
/// Sends a health report to central (fire-and-forget).
/// Sends a health report to central and completes only once central has
/// acknowledged processing. MUST THROW on non-delivery (Ask timeout, actor
/// system not started, or a not-accepted ack) — the sender's counter-restore
/// path in <c>HealthReportSender</c> depends on that exception.
/// </summary>
/// <param name="report">The site health report to send.</param>
void Send(SiteHealthReport report);
/// <param name="cancellationToken">Cancels the in-flight send.</param>
Task SendAsync(SiteHealthReport report, CancellationToken cancellationToken);
}
@@ -6,13 +6,17 @@ using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.Host;
/// <summary>
/// Sends SiteHealthReport to the local SiteCommunicationActor via Akka ActorSelection.
/// The SiteCommunicationActor forwards it to central.
/// Sends SiteHealthReport to the local SiteCommunicationActor via Akka ActorSelection,
/// which forwards it to central. The send is an acked round-trip (review 01 [Medium]):
/// it Asks and awaits the central <see cref="SiteHealthReportAck"/>, so a lost or
/// rejected report faults the sender's task (driving its counter-restore path).
/// </summary>
public class AkkaHealthReportTransport : IHealthReportTransport
{
private readonly AkkaHostedService _akkaService;
private static readonly TimeSpan AckTimeout = TimeSpan.FromSeconds(10);
/// <summary>
/// Initializes a new <see cref="AkkaHealthReportTransport"/> backed by the given Akka hosted service.
/// </summary>
@@ -23,12 +27,15 @@ public class AkkaHealthReportTransport : IHealthReportTransport
}
/// <inheritdoc />
public void Send(SiteHealthReport report)
public async Task SendAsync(SiteHealthReport report, CancellationToken cancellationToken)
{
var actorSystem = _akkaService.ActorSystem;
if (actorSystem == null) return;
var actorSystem = _akkaService.ActorSystem
?? throw new InvalidOperationException("Actor system not started — health report not sent");
var siteComm = actorSystem.ActorSelection("/user/site-communication");
siteComm.Tell(report, ActorRefs.NoSender);
var ack = await siteComm.Ask<SiteHealthReportAck>(report, AckTimeout, cancellationToken)
.ConfigureAwait(false);
if (!ack.Accepted)
throw new InvalidOperationException($"Health report #{report.SequenceNumber} rejected: {ack.Error}");
}
}