feat(sitestream): validate live-alarm-cache options + active-aggregator/reconnect telemetry (plan #10 T6)

Extend CommunicationOptionsValidator with eager bounds for the four T4
live-alarm-cache options (linger >= 0, reconcile > 0, seed concurrency 1..64,
subscribers-per-site >= 1). Enforce the per-site viewer cap fail-safe in
SiteAlarmLiveCacheService.Subscribe (reject excess viewers with a no-op
disposable rather than growing the list or throwing into the Blazor render
path). Surface two telemetry instruments on the existing ScadaBridgeTelemetry
meter: an active-aggregator observable gauge and a reconnect counter, wired from
the aggregator actor's PreStart/PostStop and its NodeA<->NodeB flip /
reconcile-driven reopen.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
2026-07-10 12:33:40 -04:00
parent b91ed3c840
commit c4f97d0e87
8 changed files with 263 additions and 5 deletions
@@ -54,7 +54,8 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
}
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory)
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
int maxSubscribersPerSite = 200)
{
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
// empty immediately (so IsLive flips true fast without any snapshot Asks).
@@ -80,7 +81,8 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
var options = Options.Create(new CommunicationOptions
{
LiveAlarmCacheLinger = linger,
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10)
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite
});
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
@@ -214,6 +216,38 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
}
[Fact]
public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted()
{
// Cap of 2: the first two viewers register; a third is rejected fail-safe with a
// no-op handle (never throws into the render path, never grows the list). Proof it
// was NOT counted: disposing only the two real viewers tears the aggregator down
// after the linger — if the rejected viewer had been registered, one subscriber
// would remain and the site would stay live forever.
var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2);
var sub1 = service.Subscribe(SiteId, () => { });
var sub2 = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
// Third viewer is over the cap.
var overflow = service.Subscribe(SiteId, () => { });
Assert.NotNull(overflow);
// Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing.
overflow.Dispose();
overflow.Dispose();
// Still live with the two real viewers present.
Assert.True(service.IsLive(SiteId));
sub1.Dispose();
sub2.Dispose();
// Both real viewers gone → after the linger the aggregator stops. This only holds
// if the overflow viewer was never added to the subscriber list.
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
}
[Fact]
public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
{