dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(sitestream): validate live-alarm-cache options + active-aggregator/reconnect telemetry (plan #10 T6)

Browse Source 
Extend CommunicationOptionsValidator with eager bounds for the four T4
live-alarm-cache options (linger >= 0, reconcile > 0, seed concurrency 1..64,
subscribers-per-site >= 1). Enforce the per-site viewer cap fail-safe in
SiteAlarmLiveCacheService.Subscribe (reject excess viewers with a no-op
disposable rather than growing the list or throwing into the Blazor render
path). Surface two telemetry instruments on the existing ScadaBridgeTelemetry
meter: an active-aggregator observable gauge and a reconnect counter, wired from
the aggregator actor's PreStart/PostStop and its NodeA<->NodeB flip /
reconcile-driven reopen.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
2026-07-10 12:33:40 -04:00
parent b91ed3c840
commit c4f97d0e87
8 changed files with 263 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs
+28
View File
@@ -47,11 +47,23 @@ public static class ScadaBridgeTelemetry
Meter.CreateCounter<long>("scadabridge.store_and_forward.replication.failures", unit: "1",
description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node");
/// <summary>
/// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide
/// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the
/// stream was given up (plan #10, Task 6). A sustained climb signals a flapping site link.
/// </summary>
private static readonly Counter<long> _liveAlarmStreamReconnects =
Meter.CreateCounter<long>("scadabridge.site.alarm_cache.reconnects", unit: "1",
description: "Live-alarm aggregator site-wide gRPC stream reconnects (NodeA↔NodeB flip or reconcile-driven reopen).");
// ---------------- Observable gauges ----------------
/// <summary>Current count of open site connections, mutated via <see cref="Interlocked"/>.</summary>
private static long _siteConnectionsUp;
/// <summary>Current count of running per-site live-alarm aggregators, mutated via <see cref="Interlocked"/>.</summary>
private static long _liveAlarmAggregatorsActive;
/// <summary>Provider that yields the live StoreAndForward queue depth; set by a later task.</summary>
private static Func<long>? _queueDepthProvider;
@@ -68,6 +80,13 @@ public static class ScadaBridgeTelemetry
() => Volatile.Read(ref _queueDepthProvider) is { } p ? p() : 0L,
unit: "items",
description: "Current StoreAndForward queue depth.");
/// <summary>Gauge reporting the number of currently running per-site live-alarm aggregators.</summary>
private static readonly ObservableGauge<long> _liveAlarmAggregatorsActiveGauge =
Meter.CreateObservableGauge<long>("scadabridge.site.alarm_cache.aggregators.active",
() => Interlocked.Read(ref _liveAlarmAggregatorsActive),
unit: "1",
description: "Number of per-site live-alarm aggregators currently running on the active central node.");
#pragma warning restore IDE0052
// ---------------- Emit helpers ----------------
@@ -94,6 +113,15 @@ public static class ScadaBridgeTelemetry
/// <summary>Records that a site connection closed (decrements the up-count gauge).</summary>
public static void SiteConnectionClosed() => Interlocked.Decrement(ref _siteConnectionsUp);
/// <summary>Records that a per-site live-alarm aggregator started (increments the active-aggregator gauge).</summary>
public static void LiveAlarmAggregatorStarted() => Interlocked.Increment(ref _liveAlarmAggregatorsActive);
/// <summary>Records that a per-site live-alarm aggregator stopped (decrements the active-aggregator gauge).</summary>
public static void LiveAlarmAggregatorStopped() => Interlocked.Decrement(ref _liveAlarmAggregatorsActive);
/// <summary>Records that a per-site live-alarm aggregator re-established its site-wide gRPC stream.</summary>
public static void RecordLiveAlarmStreamReconnect() => _liveAlarmStreamReconnects.Add(1);
/// <summary>
/// Registers the provider the StoreAndForward queue-depth gauge reads on each observation.
/// A later task supplies a provider that reads the real StoreAndForward depth. A null
src/ZB.MOM.WW.ScadaBridge.Communication/Actors/SiteAlarmAggregatorActor.cs
+11
View File
@@ -1,6 +1,7 @@
using Akka.Actor;
using Akka.Event;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
@@ -182,6 +183,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
protected override void PreStart()
{
_log.Info("Starting site-alarm aggregator for site {0}", _siteIdentifier);
// Telemetry: this aggregator is now a running per-site live cache (gauge +1). Balanced
// in PostStop, which Akka always runs on termination for any reason.
ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
_lifetimeCts = new CancellationTokenSource();
// Stream-first: open the site-wide alarm stream BEFORE the first seed so deltas
@@ -204,6 +208,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
_lifetimeCts?.Cancel();
_lifetimeCts?.Dispose();
_lifetimeCts = null;
// Telemetry: this aggregator is no longer running (gauge -1). Balances PreStart.
ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
base.PostStop();
}
@@ -223,6 +229,8 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
{
_log.Info("Site-alarm gRPC stream for {0} was down; reopening on reconcile tick", _siteIdentifier);
_retryCount = 0;
// Telemetry: a reconcile-driven reopen after the stream was given up is a reconnect.
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
OpenGrpcStream();
}
}
@@ -455,6 +463,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
// Flip to the other node.
_useNodeA = !_useNodeA;
// Telemetry: a NodeA↔NodeB failover flip is a reconnect + re-seed.
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
// A failover flip must RE-SEED (never silently serve stale) — kick a reconcile
// fan-out alongside the reconnect. Buffering during the fan-out keeps the new
// stream's deltas coherent with the fresh snapshot.
src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationOptionsValidator.cs
+20
View File
@@ -65,5 +65,25 @@ public sealed class CommunicationOptionsValidator : OptionsValidatorBase<Communi
builder.RequireThat(options.GrpcMaxConcurrentStreams > 0,
$"Communication:GrpcMaxConcurrentStreams must be positive (was {options.GrpcMaxConcurrentStreams}).");
// ── Aggregated live alarm cache (plan #10, Task 6) ───────────────────────
// Linger drives a Timer dueTime; TimeSpan.Zero is valid (stop the aggregator
// immediately when the last viewer leaves), only a negative value is invalid.
builder.RequireThat(options.LiveAlarmCacheLinger >= TimeSpan.Zero,
$"Communication:LiveAlarmCacheLinger must be zero or a positive duration (was {options.LiveAlarmCacheLinger}).");
// Reconcile interval is a periodic timer cadence AND the start-retry cadence; a
// zero/negative value would spin or never fire.
builder.RequireThat(options.LiveAlarmCacheReconcileInterval > TimeSpan.Zero,
$"Communication:LiveAlarmCacheReconcileInterval must be a positive duration (was {options.LiveAlarmCacheReconcileInterval}).");
// Seed fan-out concurrency gates a SemaphoreSlim; at least 1, capped so a
// fat-fingered config can't open an unbounded burst of cross-cluster Asks.
builder.RequireThat(options.LiveAlarmCacheSeedConcurrency is >= 1 and <= 64,
$"Communication:LiveAlarmCacheSeedConcurrency must be between 1 and 64 (was {options.LiveAlarmCacheSeedConcurrency}).");
// Per-site viewer cap must admit at least one viewer, else the page could never go live.
builder.RequireThat(options.LiveAlarmCacheMaxSubscribersPerSite >= 1,
$"Communication:LiveAlarmCacheMaxSubscribersPerSite must be at least 1 (was {options.LiveAlarmCacheMaxSubscribersPerSite}).");
}
}
src/ZB.MOM.WW.ScadaBridge.Communication/SiteAlarmLiveCacheService.cs
+21 -3
View File
@@ -92,12 +92,18 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
// A pending linger stop is now moot — a viewer arrived. Invalidate it.
entry.CancelLinger();
// Enforce the per-site viewer cap — fail SAFE: reject the excess viewer with a
// no-op disposable rather than throwing (this runs on a Blazor render path) or
// growing the subscriber list unbounded. The shared aggregator keeps serving the
// already-registered viewers; the rejected circuit just keeps its 15s poll
// fallback. Reaching the cap almost always means a Dispose leak or a runaway page.
if (entry.Subscribers.Count >= _options.LiveAlarmCacheMaxSubscribersPerSite)
{
_logger.LogWarning(
"Site {SiteId} live alarm cache already has {Count} viewers (cap {Cap}); " +
"new viewer still registered but this indicates a leak or a very busy page",
siteId, entry.Subscribers.Count, _options.LiveAlarmCacheMaxSubscribersPerSite);
"Site {SiteId} live alarm cache is at its viewer cap ({Cap}); rejecting the new " +
"viewer (it will keep polling). This indicates a subscription leak or a very busy page.",
siteId, _options.LiveAlarmCacheMaxSubscribersPerSite);
return NoOpSubscription.Instance;
}
entry.Subscribers.Add(subscription);
@@ -446,6 +452,18 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
}
}
/// <summary>
/// The handle returned when a Subscribe is rejected at the per-site viewer cap. It was
/// never registered, so <see cref="Dispose"/> is a genuine no-op — safe to dispose any
/// number of times and it never touches the site's subscriber list.
/// </summary>
private sealed class NoOpSubscription : IDisposable
{
public static readonly NoOpSubscription Instance = new();
private NoOpSubscription() { }
public void Dispose() { }
}
/// <summary>The disposable handed to a viewer; idempotent <see cref="Dispose"/> unregisters it.</summary>
private sealed class Subscription : IDisposable
{