feat(sitestream): central transient per-site live alarm cache w/ seed-then-stream + failover (plan #10 T4)
Adds the active-central-node, in-memory, reference-counted per-site live alarm cache backing the operator Alarm Summary page. No persisted central alarm store ([PERM]) — no EF entity/table/migration/DbSet. - ISiteAlarmLiveCache (new): singleton seam — Subscribe(siteId, onChanged) -> IDisposable (ref-counted, linger stop on last-out), GetCurrentAlarms, IsLive. - SiteAlarmAggregatorActor (new): one per site. Seed-then-stream ordering copied from DebugStreamBridgeActor — open the site-wide alarm-only gRPC stream first, buffer live deltas while the snapshot fan-out runs, flush with per-key dedup (AlarmKey = InstanceUniqueName|AlarmName|SourceReference), then live pass-through into an in-memory dict. Placeholder rows seeded from the snapshot and never expected on the stream; a real-alarm delta (distinct key) never wipes them. NodeA<->NodeB reconnect (retry budget + stability window), reconnect re-seeds, periodic reconcile (authoritative clear-and-rebuild) corrects instance-set drift, and a budget-exhausted stream self-heals on the next reconcile tick. - SiteAlarmLiveCacheService (new): DI singleton facade — viewer reference-counting, linger-delayed last-out stop (version + TryRemove(ref) race guards), the snapshot fan-out seed (RequestDebugSnapshotAsync per Enabled instance, capped), bounded start-retry self-heal on transient start failure, and the immutable published-snapshot store the page reads. Cache mutated only on the actor thread; viewer callbacks invoked outside the lock. - CommunicationOptions: LiveAlarmCacheLinger (30s), LiveAlarmCacheReconcileInterval (60s), LiveAlarmCacheSeedConcurrency (8), LiveAlarmCacheMaxSubscribersPerSite (200). Task 6 formalizes eager validation + telemetry. - DI registration + AkkaHostedService SetActorSystem wiring on the active central node. - Tests: 14 actor (seed/stream ordering, dedup, native-alarm parity, placeholder coherence, reconnect re-seed, reconcile replace, self-heal, stop) + 6 service (shared start, linger stop, resubscribe cancels stop, idempotent dispose, unknown site, transient-start self-heal). Code-reviewer pass: no persistence, no Critical; both Important findings addressed. Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
+414
@@ -0,0 +1,414 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteAlarmAggregatorActor"/> (plan #10, Task 4) — the transient,
|
||||
/// in-memory, per-site live alarm aggregator. Covers the tricky bits: seed-then-stream
|
||||
/// ordering (a delta arriving before the seed completes is neither lost nor double-applied),
|
||||
/// dedup by AlarmKey, placeholder seeded-but-not-on-stream coherence, NodeA↔NodeB reconnect
|
||||
/// re-seed, and periodic reconcile authoritative-replace (drift correction).
|
||||
/// </summary>
|
||||
public class SiteAlarmAggregatorActorTests : TestKit
|
||||
{
|
||||
private const string SiteId = "site-alpha";
|
||||
private const string GrpcNodeA = "http://localhost:5100";
|
||||
private const string GrpcNodeB = "http://localhost:5200";
|
||||
private const string Instance = "Site1.Pump01";
|
||||
|
||||
public SiteAlarmAggregatorActorTests() : base(@"akka.loglevel = WARNING")
|
||||
{
|
||||
SiteAlarmAggregatorActor.ReconnectDelay = TimeSpan.FromMilliseconds(50);
|
||||
SiteAlarmAggregatorActor.StabilityWindow = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
// ── Test doubles ────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Controllable seed fan-out: each invocation returns a fresh Task the test completes
|
||||
/// on demand (so seed/reconcile timing relative to buffered deltas is deterministic).
|
||||
/// </summary>
|
||||
private sealed class SeedStub
|
||||
{
|
||||
private readonly ConcurrentQueue<TaskCompletionSource<IReadOnlyList<AlarmStateChanged>>> _pending = new();
|
||||
private int _callCount;
|
||||
|
||||
public int CallCount => Volatile.Read(ref _callCount);
|
||||
|
||||
public Task<IReadOnlyList<AlarmStateChanged>> Seed(CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _callCount);
|
||||
var tcs = new TaskCompletionSource<IReadOnlyList<AlarmStateChanged>>(
|
||||
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
ct.Register(() => tcs.TrySetCanceled());
|
||||
_pending.Enqueue(tcs);
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
/// <summary>Completes the oldest not-yet-completed seed call with the given rows.</summary>
|
||||
public void CompleteNext(params AlarmStateChanged[] rows)
|
||||
{
|
||||
SpinWait.SpinUntil(() => _pending.TryPeek(out _), TimeSpan.FromSeconds(3));
|
||||
if (_pending.TryDequeue(out var tcs))
|
||||
tcs.TrySetResult(rows);
|
||||
else
|
||||
throw new InvalidOperationException("No pending seed to complete.");
|
||||
}
|
||||
|
||||
/// <summary>Faults the oldest not-yet-completed seed call (simulates a whole-fan-out failure).</summary>
|
||||
public void FaultNext()
|
||||
{
|
||||
SpinWait.SpinUntil(() => _pending.TryPeek(out _), TimeSpan.FromSeconds(3));
|
||||
if (_pending.TryDequeue(out var tcs))
|
||||
tcs.TrySetException(new InvalidOperationException("seed boom"));
|
||||
else
|
||||
throw new InvalidOperationException("No pending seed to fault.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PublishSink
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
public List<IReadOnlyList<AlarmStateChanged>> Snapshots { get; } = new();
|
||||
|
||||
public void Publish(IReadOnlyList<AlarmStateChanged> snapshot)
|
||||
{
|
||||
lock (_lock) { Snapshots.Add(snapshot); }
|
||||
}
|
||||
|
||||
public IReadOnlyList<AlarmStateChanged>? Latest
|
||||
{
|
||||
get { lock (_lock) { return Snapshots.Count == 0 ? null : Snapshots[^1]; } }
|
||||
}
|
||||
|
||||
public int Count { get { lock (_lock) { return Snapshots.Count; } } }
|
||||
}
|
||||
|
||||
private sealed record SiteSub(
|
||||
string CorrelationId, Action<AlarmStateChanged> OnAlarm, Action<Exception> OnError, CancellationToken Ct);
|
||||
|
||||
private sealed class MockSiteAlarmStreamClient : SiteStreamGrpcClient
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private readonly List<SiteSub> _subs = new();
|
||||
private readonly List<string> _unsubscribed = new();
|
||||
|
||||
public List<SiteSub> Subs { get { lock (_lock) { return _subs.ToList(); } } }
|
||||
public List<string> Unsubscribed { get { lock (_lock) { return _unsubscribed.ToList(); } } }
|
||||
|
||||
public MockSiteAlarmStreamClient() : base() { }
|
||||
|
||||
public override Task SubscribeSiteAsync(
|
||||
string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError, CancellationToken ct)
|
||||
{
|
||||
lock (_lock) { _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, ct)); }
|
||||
var tcs = new TaskCompletionSource();
|
||||
ct.Register(() => tcs.TrySetResult());
|
||||
return tcs.Task; // never completes until cancelled (simulates a live stream)
|
||||
}
|
||||
|
||||
public override void Unsubscribe(string correlationId)
|
||||
{
|
||||
lock (_lock) { _unsubscribed.Add(correlationId); }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class MockSiteAlarmStreamClientFactory : SiteStreamGrpcClientFactory
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, MockSiteAlarmStreamClient> _byEndpoint = new();
|
||||
|
||||
public MockSiteAlarmStreamClientFactory()
|
||||
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance) { }
|
||||
|
||||
public MockSiteAlarmStreamClient ClientFor(string endpoint) =>
|
||||
_byEndpoint.GetOrAdd(endpoint, _ => new MockSiteAlarmStreamClient());
|
||||
|
||||
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
||||
=> ClientFor(grpcEndpoint);
|
||||
|
||||
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint)
|
||||
=> _byEndpoint.TryGetValue(grpcEndpoint, out var c) ? c : null;
|
||||
}
|
||||
|
||||
private (IActorRef Actor, SeedStub Seed, PublishSink Sink, MockSiteAlarmStreamClientFactory Factory) CreateActor(
|
||||
TimeSpan? reconcileInterval = null)
|
||||
{
|
||||
var seed = new SeedStub();
|
||||
var sink = new PublishSink();
|
||||
var factory = new MockSiteAlarmStreamClientFactory();
|
||||
|
||||
var props = Props.Create(() => new SiteAlarmAggregatorActor(
|
||||
SiteId, "corr-1", seed.Seed, sink.Publish, factory, GrpcNodeA, GrpcNodeB,
|
||||
reconcileInterval ?? TimeSpan.FromMinutes(10)));
|
||||
|
||||
var actor = Sys.ActorOf(props);
|
||||
return (actor, seed, sink, factory);
|
||||
}
|
||||
|
||||
private static AlarmStateChanged Alarm(
|
||||
string alarmName, string sourceRef, int priority, DateTimeOffset ts,
|
||||
bool placeholder = false, string instance = Instance) =>
|
||||
new(instance, alarmName, AlarmState.Active, priority, ts)
|
||||
{
|
||||
SourceReference = sourceRef,
|
||||
IsConfiguredPlaceholder = placeholder,
|
||||
Kind = placeholder ? AlarmKind.NativeOpcUa : AlarmKind.Computed
|
||||
};
|
||||
|
||||
private static AlarmStateChanged? Find(IReadOnlyList<AlarmStateChanged>? rows, string alarmName, string sourceRef) =>
|
||||
rows?.FirstOrDefault(r => r.AlarmName == alarmName && r.SourceReference == sourceRef);
|
||||
|
||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Delta_Arriving_Before_Seed_Completes_Is_Buffered_And_Applied_Exactly_Once()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Live delta arrives DURING the seed window (new key, not in the seed).
|
||||
var t1 = DateTimeOffset.UtcNow;
|
||||
actor.Tell(Alarm("Overheat", "", 900, t1));
|
||||
|
||||
// Still buffering — nothing published yet.
|
||||
Assert.Equal(0, sink.Count);
|
||||
|
||||
// Seed completes with a DIFFERENT alarm.
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, t1.AddSeconds(-1)));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 2, TimeSpan.FromSeconds(3));
|
||||
var latest = sink.Latest!;
|
||||
Assert.NotNull(Find(latest, "PumpFault", "")); // seeded
|
||||
var overheat = Find(latest, "Overheat", ""); // buffered gap-window delta survived
|
||||
Assert.NotNull(overheat);
|
||||
Assert.Single(latest, r => r.AlarmName == "Overheat"); // exactly once
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Buffered_Delta_Older_Than_Seed_Entry_Is_Dropped()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var tSeed = DateTimeOffset.UtcNow;
|
||||
// Buffered delta for the SAME key but OLDER than the seed entry.
|
||||
actor.Tell(Alarm("PumpFault", "", 100, tSeed.AddSeconds(-1)));
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
|
||||
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
// Give a beat to ensure no late (dropped) delta sneaks in.
|
||||
Thread.Sleep(150);
|
||||
var latest = sink.Latest!;
|
||||
var pumpFault = Find(latest, "PumpFault", "");
|
||||
Assert.NotNull(pumpFault);
|
||||
Assert.Equal(500, pumpFault!.Priority); // seed value kept; older buffered delta dropped
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Buffered_Delta_Newer_Than_Seed_Entry_Is_Applied()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var tSeed = DateTimeOffset.UtcNow;
|
||||
actor.Tell(Alarm("PumpFault", "", 900, tSeed.AddSeconds(1))); // newer than seed
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && Find(l, "PumpFault", "")?.Priority == 900,
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dedup_By_AlarmKey_Distinguishes_SourceReference()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // empty seed
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
// Same instance + alarm name, different source reference → two distinct rows.
|
||||
actor.Tell(Alarm("LevelAlarm", "Tank01.Level.HiHi", 700, t));
|
||||
actor.Tell(Alarm("LevelAlarm", "Tank01.Level.LoLo", 300, t));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 2, TimeSpan.FromSeconds(3));
|
||||
var latest = sink.Latest!;
|
||||
Assert.NotNull(Find(latest, "LevelAlarm", "Tank01.Level.HiHi"));
|
||||
Assert.NotNull(Find(latest, "LevelAlarm", "Tank01.Level.LoLo"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Same_Key_Live_Update_Replaces_In_Place()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
actor.Tell(Alarm("PumpFault", "", 500, t));
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "")?.Priority == 500, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Newer update for the same key replaces (not duplicates).
|
||||
actor.Tell(Alarm("PumpFault", "", 800, t.AddSeconds(1)));
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "")?.Priority == 800, TimeSpan.FromSeconds(3));
|
||||
Assert.Single(sink.Latest!); // still exactly one row for the key
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Seeded_Native_Alarm_And_Its_Live_Delta_Collapse_Onto_One_Key()
|
||||
{
|
||||
// Identity parity across the two paths (snapshot seed vs live stream): a native
|
||||
// alarm seeded with a non-empty SourceReference and a later live delta for the
|
||||
// SAME (instance, alarmName, sourceRef) must collapse to ONE in-place-updated row,
|
||||
// not a ghost duplicate. Both paths key on SourceReference, so they must agree.
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
seed.CompleteNext(Alarm("Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi", 300, t));
|
||||
AwaitCondition(() => Find(sink.Latest, "Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi") is not null,
|
||||
TimeSpan.FromSeconds(3));
|
||||
|
||||
// Live delta for the exact same identity, newer → updates in place.
|
||||
actor.Tell(Alarm("Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi", 800, t.AddSeconds(1)));
|
||||
|
||||
AwaitCondition(
|
||||
() => Find(sink.Latest, "Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi")?.Priority == 800,
|
||||
TimeSpan.FromSeconds(3));
|
||||
Assert.Single(sink.Latest!); // exactly one row — no seed/live ghost duplicate
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Placeholder_Seeded_Coexists_With_Live_Real_Alarm_And_Is_Not_Wiped()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
// Seed carries a configured-placeholder row (binding with no active conditions).
|
||||
var placeholder = Alarm("Motor1.MotorAlarms", "", 0, t, placeholder: true);
|
||||
seed.CompleteNext(placeholder);
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// A live REAL condition for the same instance arrives (distinct AlarmKey).
|
||||
actor.Tell(Alarm("Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi", 800, t.AddSeconds(1)));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 2, TimeSpan.FromSeconds(3));
|
||||
var latest = sink.Latest!;
|
||||
// Placeholder still present (not wiped by the real alarm under a different key).
|
||||
var ph = Find(latest, "Motor1.MotorAlarms", "");
|
||||
Assert.NotNull(ph);
|
||||
Assert.True(ph!.IsConfiguredPlaceholder);
|
||||
Assert.NotNull(Find(latest, "Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GrpcError_Flips_Node_Reconnects_And_ReSeeds()
|
||||
{
|
||||
var (_, seed, sink, factory) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // initial seed done
|
||||
// Wait for the seed to actually be APPLIED (published) so the re-seed on error is
|
||||
// not skipped as "fan-out already in flight".
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Fail the NodeA stream.
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("NodeA down"));
|
||||
|
||||
// Reconnect must reach NodeB, AND a re-seed fan-out must have been triggered.
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
|
||||
// The failed NodeA stream was unsubscribed (relay released, not left zombie).
|
||||
Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).Unsubscribed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Periodic_Reconcile_Authoritatively_Replaces_Stale_Rows()
|
||||
{
|
||||
var (_, seed, sink, _) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(250));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // seed has a row
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "") is not null, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Reconcile tick fires → seed #2 (e.g. instance was disabled) returns EMPTY.
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // empty
|
||||
|
||||
// The stale PumpFault row must disappear (authoritative replace, not merge).
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 0, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_Initial_Seed_Does_Not_Publish_But_A_Later_Reconcile_Recovers()
|
||||
{
|
||||
var (_, seed, sink, _) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(250));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// The initial seed fan-out throws as a whole.
|
||||
seed.FaultNext();
|
||||
|
||||
// A failed INITIAL seed must not publish (page keeps its poll fallback — not live yet).
|
||||
Thread.Sleep(200);
|
||||
Assert.Equal(0, sink.Count);
|
||||
|
||||
// The periodic reconcile is the backstop: it re-runs the fan-out (flag was cleared
|
||||
// on failure), completes, and the actor publishes → now live.
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, DateTimeOffset.UtcNow));
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "") is not null, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stream_GivenUp_After_MaxRetries_Reopens_On_Reconcile_Tick()
|
||||
{
|
||||
// After the retry budget is exhausted the live stream is left down, but the
|
||||
// aggregator is NOT stopped — the next reconcile tick self-heals it (resets the
|
||||
// budget + reopens) so a sustained site outage never permanently kills the feed.
|
||||
var (_, seed, _, factory) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(300));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Drive initial + 3 retries (each flips node) to exhaust the budget.
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("1"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeB).Subs[0].OnError(new Exception("2"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
|
||||
|
||||
int TotalSubs() => factory.ClientFor(GrpcNodeA).Subs.Count + factory.ClientFor(GrpcNodeB).Subs.Count;
|
||||
var before = TotalSubs(); // 4
|
||||
|
||||
// Fourth error exceeds MaxRetries → stream given up (no immediate reopen).
|
||||
factory.ClientFor(GrpcNodeB).Subs[1].OnError(new Exception("4"));
|
||||
|
||||
// The reconcile tick reopens the stream (self-heal).
|
||||
AwaitCondition(() => TotalSubs() > before, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stop_Message_TearsDown_Grpc_And_Stops_Actor()
|
||||
{
|
||||
var (actor, seed, _, factory) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
|
||||
Watch(actor);
|
||||
actor.Tell(new StopSiteAlarmAggregator());
|
||||
|
||||
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Unsubscribed.Contains("corr-1"),
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
|
||||
/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
|
||||
/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
|
||||
/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
|
||||
/// whose site-wide stream never faults, so no reconnect noise.
|
||||
/// </summary>
|
||||
public class SiteAlarmLiveCacheServiceTests : TestKit
|
||||
{
|
||||
private const int SiteId = 7;
|
||||
|
||||
// A mock site-wide alarm stream client whose subscription hangs until cancelled.
|
||||
private sealed class HangingClient : SiteStreamGrpcClient
|
||||
{
|
||||
public HangingClient() : base() { }
|
||||
public override Task SubscribeSiteAsync(
|
||||
string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
|
||||
Action<Exception> onError, CancellationToken ct)
|
||||
{
|
||||
var tcs = new TaskCompletionSource();
|
||||
ct.Register(() => tcs.TrySetResult());
|
||||
return tcs.Task;
|
||||
}
|
||||
public override void Unsubscribe(string correlationId) { }
|
||||
}
|
||||
|
||||
private sealed class CountingFactory : SiteStreamGrpcClientFactory
|
||||
{
|
||||
private readonly HangingClient _client = new();
|
||||
public int GetOrCreateCount;
|
||||
|
||||
public CountingFactory() : base(NullLoggerFactory.Instance) { }
|
||||
|
||||
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
||||
{
|
||||
Interlocked.Increment(ref GetOrCreateCount);
|
||||
return _client;
|
||||
}
|
||||
|
||||
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
|
||||
}
|
||||
|
||||
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory)
|
||||
{
|
||||
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
|
||||
// empty immediately (so IsLive flips true fast without any snapshot Asks).
|
||||
var site = new Site("Alpha", "site-alpha")
|
||||
{
|
||||
Id = SiteId,
|
||||
GrpcNodeAAddress = "http://localhost:5100",
|
||||
GrpcNodeBAddress = "http://localhost:5200"
|
||||
};
|
||||
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);
|
||||
|
||||
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Instance>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
services.AddScoped(_ => instanceRepo);
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
var options = Options.Create(new CommunicationOptions
|
||||
{
|
||||
LiveAlarmCacheLinger = linger,
|
||||
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10)
|
||||
});
|
||||
|
||||
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
|
||||
factory = new CountingFactory();
|
||||
var service = new SiteAlarmLiveCacheService(
|
||||
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
|
||||
service.SetActorSystem(Sys);
|
||||
return service;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
|
||||
|
||||
var changes1 = 0;
|
||||
var changes2 = 0;
|
||||
using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
|
||||
using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));
|
||||
|
||||
// The (empty) seed completes → aggregator publishes → live.
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
// Exactly ONE gRPC stream opened for the site regardless of viewer count.
|
||||
Assert.Equal(1, factory.GetOrCreateCount);
|
||||
// Empty seed → empty current snapshot.
|
||||
Assert.Empty(service.GetCurrentAlarms(SiteId));
|
||||
// Both viewers were notified of the initial publish.
|
||||
AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
|
||||
var sub1 = service.Subscribe(SiteId, () => { });
|
||||
var sub2 = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
sub1.Dispose();
|
||||
// One viewer remains — still live, not stopped.
|
||||
Thread.Sleep(300);
|
||||
Assert.True(service.IsLive(SiteId));
|
||||
|
||||
sub2.Dispose();
|
||||
// Last viewer left → after the linger the aggregator is torn down and the entry cleared.
|
||||
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
||||
Assert.Empty(service.GetCurrentAlarms(SiteId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory);
|
||||
|
||||
var sub1 = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
sub1.Dispose(); // schedules linger stop
|
||||
var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it
|
||||
|
||||
// Wait comfortably past the original linger; the aggregator must still be alive.
|
||||
Thread.Sleep(700);
|
||||
Assert.True(service.IsLive(SiteId));
|
||||
// No second aggregator/stream was opened — the same one was kept warm.
|
||||
Assert.Equal(1, factory.GetOrCreateCount);
|
||||
|
||||
sub2.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dispose_Is_Idempotent()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
var sub = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
sub.Dispose();
|
||||
sub.Dispose(); // must not throw or double-decrement
|
||||
|
||||
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
|
||||
{
|
||||
// A transient failure resolving the site at first-subscribe must NOT leave the
|
||||
// viewer cohort polling forever — a bounded retry re-attempts and self-heals.
|
||||
var site = new Site("Alpha", "site-alpha")
|
||||
{
|
||||
Id = SiteId,
|
||||
GrpcNodeAAddress = "http://localhost:5100",
|
||||
GrpcNodeBAddress = "http://localhost:5200"
|
||||
};
|
||||
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
// First resolve returns null (transient blip); every resolve thereafter succeeds.
|
||||
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);
|
||||
|
||||
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Instance>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
services.AddScoped(_ => instanceRepo);
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
var options = Options.Create(new CommunicationOptions
|
||||
{
|
||||
LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
|
||||
// Retry cadence reuses the reconcile interval — keep it short for the test.
|
||||
LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
|
||||
});
|
||||
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
var factory = new CountingFactory();
|
||||
|
||||
var service = new SiteAlarmLiveCacheService(
|
||||
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
|
||||
service.SetActorSystem(Sys);
|
||||
|
||||
using var sub = service.Subscribe(SiteId, () => { });
|
||||
|
||||
// First start failed (site not found); a stable single viewer stays subscribed.
|
||||
// The retry eventually resolves the site and the aggregator goes live — without
|
||||
// any further Subscribe call.
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
|
||||
using var sub = service.Subscribe(999, () => { }); // no such site in the repo
|
||||
Thread.Sleep(300);
|
||||
Assert.False(service.IsLive(999));
|
||||
Assert.Empty(service.GetCurrentAlarms(999));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user