feat(sitestream): validate live-alarm-cache options + active-aggregator/reconnect telemetry (plan #10 T6)

Extend CommunicationOptionsValidator with eager bounds for the four T4
live-alarm-cache options (linger >= 0, reconcile > 0, seed concurrency 1..64,
subscribers-per-site >= 1). Enforce the per-site viewer cap fail-safe in
SiteAlarmLiveCacheService.Subscribe (reject excess viewers with a no-op
disposable rather than growing the list or throwing into the Blazor render
path). Surface two telemetry instruments on the existing ScadaBridgeTelemetry
meter: an active-aggregator observable gauge and a reconnect counter, wired from
the aggregator actor's PreStart/PostStop and its NodeA<->NodeB flip /
reconcile-driven reopen.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
2026-07-10 12:33:40 -04:00
parent b91ed3c840
commit c4f97d0e87
8 changed files with 263 additions and 5 deletions
@@ -1,6 +1,7 @@
using Akka.Actor;
using Akka.Event;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
@@ -182,6 +183,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
protected override void PreStart()
{
_log.Info("Starting site-alarm aggregator for site {0}", _siteIdentifier);
// Telemetry: this aggregator is now a running per-site live cache (gauge +1). Balanced
// in PostStop, which Akka always runs on termination for any reason.
ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
_lifetimeCts = new CancellationTokenSource();
// Stream-first: open the site-wide alarm stream BEFORE the first seed so deltas
@@ -204,6 +208,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
_lifetimeCts?.Cancel();
_lifetimeCts?.Dispose();
_lifetimeCts = null;
// Telemetry: this aggregator is no longer running (gauge -1). Balances PreStart.
ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
base.PostStop();
}
@@ -223,6 +229,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
{
_log.Info("Site-alarm gRPC stream for {0} was down; reopening on reconcile tick", _siteIdentifier);
_retryCount = 0;
// Telemetry: a reconcile-driven reopen after the stream was given up is a reconnect.
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
OpenGrpcStream();
}
}
@@ -455,6 +463,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
// Flip to the other node.
_useNodeA = !_useNodeA;
// Telemetry: a NodeA↔NodeB failover flip is a reconnect + re-seed.
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
// A failover flip must RE-SEED (never silently serve stale) — kick a reconcile
// fan-out alongside the reconnect. Buffering during the fan-out keeps the new
// stream's deltas coherent with the fresh snapshot.