c4f97d0e87
Extend CommunicationOptionsValidator with eager bounds for the four T4 live-alarm-cache options (linger >= 0, reconcile > 0, seed concurrency 1..64, subscribers-per-site >= 1). Enforce the per-site viewer cap fail-safe in SiteAlarmLiveCacheService.Subscribe (reject excess viewers with a no-op disposable rather than growing the list or throwing into the Blazor render path). Surface two telemetry instruments on the existing ScadaBridgeTelemetry meter: an active-aggregator observable gauge and a reconnect counter, wired from the aggregator actor's PreStart/PostStop and its NodeA<->NodeB flip / reconcile-driven reopen. Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
262 lines
11 KiB
C#
262 lines
11 KiB
C#
using Akka.TestKit.Xunit2;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using NSubstitute;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.Communication;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
|
|
|
/// <summary>
|
|
/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
|
|
/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
|
|
/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
|
|
/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
|
|
/// whose site-wide stream never faults, so no reconnect noise.
|
|
/// </summary>
|
|
public class SiteAlarmLiveCacheServiceTests : TestKit
|
|
{
|
|
private const int SiteId = 7;
|
|
|
|
// A mock site-wide alarm stream client whose subscription hangs until cancelled.
|
|
private sealed class HangingClient : SiteStreamGrpcClient
|
|
{
|
|
public HangingClient() : base() { }
|
|
public override Task SubscribeSiteAsync(
|
|
string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
|
|
Action<Exception> onError, CancellationToken ct)
|
|
{
|
|
var tcs = new TaskCompletionSource();
|
|
ct.Register(() => tcs.TrySetResult());
|
|
return tcs.Task;
|
|
}
|
|
public override void Unsubscribe(string correlationId) { }
|
|
}
|
|
|
|
private sealed class CountingFactory : SiteStreamGrpcClientFactory
|
|
{
|
|
private readonly HangingClient _client = new();
|
|
public int GetOrCreateCount;
|
|
|
|
public CountingFactory() : base(NullLoggerFactory.Instance) { }
|
|
|
|
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
|
{
|
|
Interlocked.Increment(ref GetOrCreateCount);
|
|
return _client;
|
|
}
|
|
|
|
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
|
|
}
|
|
|
|
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
|
|
int maxSubscribersPerSite = 200)
|
|
{
|
|
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
|
|
// empty immediately (so IsLive flips true fast without any snapshot Asks).
|
|
var site = new Site("Alpha", "site-alpha")
|
|
{
|
|
Id = SiteId,
|
|
GrpcNodeAAddress = "http://localhost:5100",
|
|
GrpcNodeBAddress = "http://localhost:5200"
|
|
};
|
|
|
|
var siteRepo = Substitute.For<ISiteRepository>();
|
|
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);
|
|
|
|
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
|
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
|
.Returns(new List<Instance>());
|
|
|
|
var services = new ServiceCollection();
|
|
services.AddScoped(_ => siteRepo);
|
|
services.AddScoped(_ => instanceRepo);
|
|
var provider = services.BuildServiceProvider();
|
|
|
|
var options = Options.Create(new CommunicationOptions
|
|
{
|
|
LiveAlarmCacheLinger = linger,
|
|
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
|
|
LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite
|
|
});
|
|
|
|
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
|
|
NullLogger<CommunicationService>.Instance);
|
|
|
|
factory = new CountingFactory();
|
|
var service = new SiteAlarmLiveCacheService(
|
|
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
|
|
service.SetActorSystem(Sys);
|
|
return service;
|
|
}
|
|
|
|
[Fact]
|
|
public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
|
|
{
|
|
var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
|
|
|
|
var changes1 = 0;
|
|
var changes2 = 0;
|
|
using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
|
|
using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));
|
|
|
|
// The (empty) seed completes → aggregator publishes → live.
|
|
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
|
|
|
// Exactly ONE gRPC stream opened for the site regardless of viewer count.
|
|
Assert.Equal(1, factory.GetOrCreateCount);
|
|
// Empty seed → empty current snapshot.
|
|
Assert.Empty(service.GetCurrentAlarms(SiteId));
|
|
// Both viewers were notified of the initial publish.
|
|
AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
[Fact]
|
|
public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
|
|
{
|
|
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
|
|
|
var sub1 = service.Subscribe(SiteId, () => { });
|
|
var sub2 = service.Subscribe(SiteId, () => { });
|
|
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
|
|
|
sub1.Dispose();
|
|
// One viewer remains — still live, not stopped.
|
|
Thread.Sleep(300);
|
|
Assert.True(service.IsLive(SiteId));
|
|
|
|
sub2.Dispose();
|
|
// Last viewer left → after the linger the aggregator is torn down and the entry cleared.
|
|
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
|
Assert.Empty(service.GetCurrentAlarms(SiteId));
|
|
}
|
|
|
|
[Fact]
|
|
public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
|
|
{
|
|
var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory);
|
|
|
|
var sub1 = service.Subscribe(SiteId, () => { });
|
|
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
|
|
|
sub1.Dispose(); // schedules linger stop
|
|
var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it
|
|
|
|
// Wait comfortably past the original linger; the aggregator must still be alive.
|
|
Thread.Sleep(700);
|
|
Assert.True(service.IsLive(SiteId));
|
|
// No second aggregator/stream was opened — the same one was kept warm.
|
|
Assert.Equal(1, factory.GetOrCreateCount);
|
|
|
|
sub2.Dispose();
|
|
}
|
|
|
|
[Fact]
|
|
public void Dispose_Is_Idempotent()
|
|
{
|
|
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
|
var sub = service.Subscribe(SiteId, () => { });
|
|
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
|
|
|
sub.Dispose();
|
|
sub.Dispose(); // must not throw or double-decrement
|
|
|
|
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
[Fact]
|
|
public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
|
|
{
|
|
// A transient failure resolving the site at first-subscribe must NOT leave the
|
|
// viewer cohort polling forever — a bounded retry re-attempts and self-heals.
|
|
var site = new Site("Alpha", "site-alpha")
|
|
{
|
|
Id = SiteId,
|
|
GrpcNodeAAddress = "http://localhost:5100",
|
|
GrpcNodeBAddress = "http://localhost:5200"
|
|
};
|
|
|
|
var siteRepo = Substitute.For<ISiteRepository>();
|
|
// First resolve returns null (transient blip); every resolve thereafter succeeds.
|
|
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);
|
|
|
|
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
|
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
|
.Returns(new List<Instance>());
|
|
|
|
var services = new ServiceCollection();
|
|
services.AddScoped(_ => siteRepo);
|
|
services.AddScoped(_ => instanceRepo);
|
|
var provider = services.BuildServiceProvider();
|
|
|
|
var options = Options.Create(new CommunicationOptions
|
|
{
|
|
LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
|
|
// Retry cadence reuses the reconcile interval — keep it short for the test.
|
|
LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
|
|
});
|
|
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
|
|
NullLogger<CommunicationService>.Instance);
|
|
var factory = new CountingFactory();
|
|
|
|
var service = new SiteAlarmLiveCacheService(
|
|
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
|
|
service.SetActorSystem(Sys);
|
|
|
|
using var sub = service.Subscribe(SiteId, () => { });
|
|
|
|
// First start failed (site not found); a stable single viewer stays subscribed.
|
|
// The retry eventually resolves the site and the aggregator goes live — without
|
|
// any further Subscribe call.
|
|
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
[Fact]
|
|
public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted()
|
|
{
|
|
// Cap of 2: the first two viewers register; a third is rejected fail-safe with a
|
|
// no-op handle (never throws into the render path, never grows the list). Proof it
|
|
// was NOT counted: disposing only the two real viewers tears the aggregator down
|
|
// after the linger — if the rejected viewer had been registered, one subscriber
|
|
// would remain and the site would stay live forever.
|
|
var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2);
|
|
|
|
var sub1 = service.Subscribe(SiteId, () => { });
|
|
var sub2 = service.Subscribe(SiteId, () => { });
|
|
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
|
|
|
// Third viewer is over the cap.
|
|
var overflow = service.Subscribe(SiteId, () => { });
|
|
Assert.NotNull(overflow);
|
|
// Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing.
|
|
overflow.Dispose();
|
|
overflow.Dispose();
|
|
|
|
// Still live with the two real viewers present.
|
|
Assert.True(service.IsLive(SiteId));
|
|
|
|
sub1.Dispose();
|
|
sub2.Dispose();
|
|
|
|
// Both real viewers gone → after the linger the aggregator stops. This only holds
|
|
// if the overflow viewer was never added to the subscriber list.
|
|
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
[Fact]
|
|
public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
|
|
{
|
|
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
|
|
|
using var sub = service.Subscribe(999, () => { }); // no such site in the repo
|
|
Thread.Sleep(300);
|
|
Assert.False(service.IsLive(999));
|
|
Assert.Empty(service.GetCurrentAlarms(999));
|
|
}
|
|
}
|