Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs
T
Joseph Doherty c4f97d0e87 feat(sitestream): validate live-alarm-cache options + active-aggregator/reconnect telemetry (plan #10 T6)
Extend CommunicationOptionsValidator with eager bounds for the four T4
live-alarm-cache options (linger >= 0, reconcile > 0, seed concurrency 1..64,
subscribers-per-site >= 1). Enforce the per-site viewer cap fail-safe in
SiteAlarmLiveCacheService.Subscribe (reject excess viewers with a no-op
disposable rather than growing the list or throwing into the Blazor render
path). Surface two telemetry instruments on the existing ScadaBridgeTelemetry
meter: an active-aggregator observable gauge and a reconnect counter, wired from
the aggregator actor's PreStart/PostStop and its NodeA<->NodeB flip /
reconcile-driven reopen.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 12:33:40 -04:00

262 lines
11 KiB
C#

using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
/// <summary>
/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
/// whose site-wide stream never faults, so no reconnect noise.
/// </summary>
public class SiteAlarmLiveCacheServiceTests : TestKit
{
private const int SiteId = 7;
// A mock site-wide alarm stream client whose subscription hangs until cancelled.
private sealed class HangingClient : SiteStreamGrpcClient
{
public HangingClient() : base() { }
public override Task SubscribeSiteAsync(
string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
Action<Exception> onError, CancellationToken ct)
{
var tcs = new TaskCompletionSource();
ct.Register(() => tcs.TrySetResult());
return tcs.Task;
}
public override void Unsubscribe(string correlationId) { }
}
private sealed class CountingFactory : SiteStreamGrpcClientFactory
{
private readonly HangingClient _client = new();
public int GetOrCreateCount;
public CountingFactory() : base(NullLoggerFactory.Instance) { }
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
{
Interlocked.Increment(ref GetOrCreateCount);
return _client;
}
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
}
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
int maxSubscribersPerSite = 200)
{
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
// empty immediately (so IsLive flips true fast without any snapshot Asks).
var site = new Site("Alpha", "site-alpha")
{
Id = SiteId,
GrpcNodeAAddress = "http://localhost:5100",
GrpcNodeBAddress = "http://localhost:5200"
};
var siteRepo = Substitute.For<ISiteRepository>();
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
.Returns(new List<Instance>());
var services = new ServiceCollection();
services.AddScoped(_ => siteRepo);
services.AddScoped(_ => instanceRepo);
var provider = services.BuildServiceProvider();
var options = Options.Create(new CommunicationOptions
{
LiveAlarmCacheLinger = linger,
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite
});
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
factory = new CountingFactory();
var service = new SiteAlarmLiveCacheService(
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
service.SetActorSystem(Sys);
return service;
}
[Fact]
public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
var changes1 = 0;
var changes2 = 0;
using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));
// The (empty) seed completes → aggregator publishes → live.
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
// Exactly ONE gRPC stream opened for the site regardless of viewer count.
Assert.Equal(1, factory.GetOrCreateCount);
// Empty seed → empty current snapshot.
Assert.Empty(service.GetCurrentAlarms(SiteId));
// Both viewers were notified of the initial publish.
AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
}
[Fact]
public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
var sub1 = service.Subscribe(SiteId, () => { });
var sub2 = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
sub1.Dispose();
// One viewer remains — still live, not stopped.
Thread.Sleep(300);
Assert.True(service.IsLive(SiteId));
sub2.Dispose();
// Last viewer left → after the linger the aggregator is torn down and the entry cleared.
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
Assert.Empty(service.GetCurrentAlarms(SiteId));
}
[Fact]
public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
{
var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory);
var sub1 = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
sub1.Dispose(); // schedules linger stop
var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it
// Wait comfortably past the original linger; the aggregator must still be alive.
Thread.Sleep(700);
Assert.True(service.IsLive(SiteId));
// No second aggregator/stream was opened — the same one was kept warm.
Assert.Equal(1, factory.GetOrCreateCount);
sub2.Dispose();
}
[Fact]
public void Dispose_Is_Idempotent()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
var sub = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
sub.Dispose();
sub.Dispose(); // must not throw or double-decrement
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
}
[Fact]
public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
{
// A transient failure resolving the site at first-subscribe must NOT leave the
// viewer cohort polling forever — a bounded retry re-attempts and self-heals.
var site = new Site("Alpha", "site-alpha")
{
Id = SiteId,
GrpcNodeAAddress = "http://localhost:5100",
GrpcNodeBAddress = "http://localhost:5200"
};
var siteRepo = Substitute.For<ISiteRepository>();
// First resolve returns null (transient blip); every resolve thereafter succeeds.
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
.Returns(new List<Instance>());
var services = new ServiceCollection();
services.AddScoped(_ => siteRepo);
services.AddScoped(_ => instanceRepo);
var provider = services.BuildServiceProvider();
var options = Options.Create(new CommunicationOptions
{
LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
// Retry cadence reuses the reconcile interval — keep it short for the test.
LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
});
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
var factory = new CountingFactory();
var service = new SiteAlarmLiveCacheService(
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
service.SetActorSystem(Sys);
using var sub = service.Subscribe(SiteId, () => { });
// First start failed (site not found); a stable single viewer stays subscribed.
// The retry eventually resolves the site and the aggregator goes live — without
// any further Subscribe call.
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
}
[Fact]
public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted()
{
// Cap of 2: the first two viewers register; a third is rejected fail-safe with a
// no-op handle (never throws into the render path, never grows the list). Proof it
// was NOT counted: disposing only the two real viewers tears the aggregator down
// after the linger — if the rejected viewer had been registered, one subscriber
// would remain and the site would stay live forever.
var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2);
var sub1 = service.Subscribe(SiteId, () => { });
var sub2 = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
// Third viewer is over the cap.
var overflow = service.Subscribe(SiteId, () => { });
Assert.NotNull(overflow);
// Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing.
overflow.Dispose();
overflow.Dispose();
// Still live with the two real viewers present.
Assert.True(service.IsLive(SiteId));
sub1.Dispose();
sub2.Dispose();
// Both real viewers gone → after the linger the aggregator stops. This only holds
// if the overflow viewer was never added to the subscriber list.
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
}
[Fact]
public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
using var sub = service.Subscribe(999, () => { }); // no such site in the repo
Thread.Sleep(300);
Assert.False(service.IsLive(999));
Assert.Empty(service.GetCurrentAlarms(999));
}
}