Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs
T
Joseph Doherty 696a4ffea2 feat(sitestream): central transient per-site live alarm cache w/ seed-then-stream + failover (plan #10 T4)
Adds the active-central-node, in-memory, reference-counted per-site live alarm
cache backing the operator Alarm Summary page. No persisted central alarm store
([PERM]) — no EF entity/table/migration/DbSet.

- ISiteAlarmLiveCache (new): singleton seam — Subscribe(siteId, onChanged) ->
  IDisposable (ref-counted, linger stop on last-out), GetCurrentAlarms, IsLive.
- SiteAlarmAggregatorActor (new): one per site. Seed-then-stream ordering copied
  from DebugStreamBridgeActor — open the site-wide alarm-only gRPC stream first,
  buffer live deltas while the snapshot fan-out runs, flush with per-key dedup
  (AlarmKey = InstanceUniqueName|AlarmName|SourceReference), then live pass-through
  into an in-memory dict. Placeholder rows seeded from the snapshot and never
  expected on the stream; a real-alarm delta (distinct key) never wipes them.
  NodeA<->NodeB reconnect (retry budget + stability window), reconnect re-seeds,
  periodic reconcile (authoritative clear-and-rebuild) corrects instance-set drift,
  and a budget-exhausted stream self-heals on the next reconcile tick.
- SiteAlarmLiveCacheService (new): DI singleton facade — viewer reference-counting,
  linger-delayed last-out stop (version + TryRemove(ref) race guards), the
  snapshot fan-out seed (RequestDebugSnapshotAsync per Enabled instance, capped),
  bounded start-retry self-heal on transient start failure, and the immutable
  published-snapshot store the page reads. Cache mutated only on the actor thread;
  viewer callbacks invoked outside the lock.
- CommunicationOptions: LiveAlarmCacheLinger (30s), LiveAlarmCacheReconcileInterval
  (60s), LiveAlarmCacheSeedConcurrency (8), LiveAlarmCacheMaxSubscribersPerSite
  (200). Task 6 formalizes eager validation + telemetry.
- DI registration + AkkaHostedService SetActorSystem wiring on the active central node.
- Tests: 14 actor (seed/stream ordering, dedup, native-alarm parity, placeholder
  coherence, reconnect re-seed, reconcile replace, self-heal, stop) + 6 service
  (shared start, linger stop, resubscribe cancels stop, idempotent dispose, unknown
  site, transient-start self-heal). Code-reviewer pass: no persistence, no Critical;
  both Important findings addressed.

Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
2026-07-10 12:18:02 -04:00

228 lines
9.3 KiB
C#

using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
/// <summary>
/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
/// whose site-wide stream never faults, so no reconnect noise.
/// </summary>
public class SiteAlarmLiveCacheServiceTests : TestKit
{
private const int SiteId = 7;
// A mock site-wide alarm stream client whose subscription hangs until cancelled.
private sealed class HangingClient : SiteStreamGrpcClient
{
public HangingClient() : base() { }
public override Task SubscribeSiteAsync(
string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
Action<Exception> onError, CancellationToken ct)
{
var tcs = new TaskCompletionSource();
ct.Register(() => tcs.TrySetResult());
return tcs.Task;
}
public override void Unsubscribe(string correlationId) { }
}
private sealed class CountingFactory : SiteStreamGrpcClientFactory
{
private readonly HangingClient _client = new();
public int GetOrCreateCount;
public CountingFactory() : base(NullLoggerFactory.Instance) { }
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
{
Interlocked.Increment(ref GetOrCreateCount);
return _client;
}
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
}
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory)
{
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
// empty immediately (so IsLive flips true fast without any snapshot Asks).
var site = new Site("Alpha", "site-alpha")
{
Id = SiteId,
GrpcNodeAAddress = "http://localhost:5100",
GrpcNodeBAddress = "http://localhost:5200"
};
var siteRepo = Substitute.For<ISiteRepository>();
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
.Returns(new List<Instance>());
var services = new ServiceCollection();
services.AddScoped(_ => siteRepo);
services.AddScoped(_ => instanceRepo);
var provider = services.BuildServiceProvider();
var options = Options.Create(new CommunicationOptions
{
LiveAlarmCacheLinger = linger,
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10)
});
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
factory = new CountingFactory();
var service = new SiteAlarmLiveCacheService(
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
service.SetActorSystem(Sys);
return service;
}
[Fact]
public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
var changes1 = 0;
var changes2 = 0;
using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));
// The (empty) seed completes → aggregator publishes → live.
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
// Exactly ONE gRPC stream opened for the site regardless of viewer count.
Assert.Equal(1, factory.GetOrCreateCount);
// Empty seed → empty current snapshot.
Assert.Empty(service.GetCurrentAlarms(SiteId));
// Both viewers were notified of the initial publish.
AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
}
[Fact]
public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
var sub1 = service.Subscribe(SiteId, () => { });
var sub2 = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
sub1.Dispose();
// One viewer remains — still live, not stopped.
Thread.Sleep(300);
Assert.True(service.IsLive(SiteId));
sub2.Dispose();
// Last viewer left → after the linger the aggregator is torn down and the entry cleared.
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
Assert.Empty(service.GetCurrentAlarms(SiteId));
}
[Fact]
public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
{
var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory);
var sub1 = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
sub1.Dispose(); // schedules linger stop
var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it
// Wait comfortably past the original linger; the aggregator must still be alive.
Thread.Sleep(700);
Assert.True(service.IsLive(SiteId));
// No second aggregator/stream was opened — the same one was kept warm.
Assert.Equal(1, factory.GetOrCreateCount);
sub2.Dispose();
}
[Fact]
public void Dispose_Is_Idempotent()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
var sub = service.Subscribe(SiteId, () => { });
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
sub.Dispose();
sub.Dispose(); // must not throw or double-decrement
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
}
[Fact]
public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
{
// A transient failure resolving the site at first-subscribe must NOT leave the
// viewer cohort polling forever — a bounded retry re-attempts and self-heals.
var site = new Site("Alpha", "site-alpha")
{
Id = SiteId,
GrpcNodeAAddress = "http://localhost:5100",
GrpcNodeBAddress = "http://localhost:5200"
};
var siteRepo = Substitute.For<ISiteRepository>();
// First resolve returns null (transient blip); every resolve thereafter succeeds.
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
.Returns(new List<Instance>());
var services = new ServiceCollection();
services.AddScoped(_ => siteRepo);
services.AddScoped(_ => instanceRepo);
var provider = services.BuildServiceProvider();
var options = Options.Create(new CommunicationOptions
{
LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
// Retry cadence reuses the reconcile interval — keep it short for the test.
LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
});
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
var factory = new CountingFactory();
var service = new SiteAlarmLiveCacheService(
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
service.SetActorSystem(Sys);
using var sub = service.Subscribe(SiteId, () => { });
// First start failed (site not found); a stable single viewer stays subscribed.
// The retry eventually resolves the site and the aggregator goes live — without
// any further Subscribe call.
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
}
[Fact]
public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
{
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
using var sub = service.Subscribe(999, () => { }); // no such site in the repo
Thread.Sleep(300);
Assert.False(service.IsLive(999));
Assert.Empty(service.GetCurrentAlarms(999));
}
}