merge(deferred-10): aggregated live alarm stream for Alarm Summary
Merges plan #10 (feat/live-alarm-stream) into main alongside plan #22. Claude-Session: https://claude.ai/code/session_01MtdgwpEeCUn6cUA5f1LMPj
This commit is contained in:
Joseph Doherty
@@ -9,6 +9,7 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Security;
|
||||
using AlarmSummaryPage = ZB.MOM.WW.ScadaBridge.CentralUI.Components.Pages.Monitoring.AlarmSummary;
|
||||
|
||||
@@ -26,6 +27,7 @@ public class AlarmSummaryRenderTests : BunitContext
|
||||
{
|
||||
private readonly IAlarmSummaryService _summary = Substitute.For<IAlarmSummaryService>();
|
||||
private readonly ISiteRepository _siteRepo = Substitute.For<ISiteRepository>();
|
||||
private readonly FakeSiteAlarmLiveCache _liveCache = new();
|
||||
|
||||
public AlarmSummaryRenderTests()
|
||||
{
|
||||
@@ -39,14 +41,33 @@ public class AlarmSummaryRenderTests : BunitContext
|
||||
|
||||
_summary.GetSiteAlarmsAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult(new AlarmSummaryResult(rows, Array.Empty<string>())));
|
||||
// BuildFromLiveAlarms (Task 5): flatten the pushed live snapshot the same way
|
||||
// the real service does, so the live-path render tests exercise real behavior.
|
||||
_summary.BuildFromLiveAlarms(Arg.Any<IReadOnlyList<AlarmStateChanged>>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
var alarms = ci.Arg<IReadOnlyList<AlarmStateChanged>>();
|
||||
var built = alarms
|
||||
.Select(a => new AlarmSummaryRow(a.InstanceUniqueName, a))
|
||||
.OrderBy(r => r.InstanceUniqueName, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(r => r.Alarm.AlarmName, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
return new AlarmSummaryResult(built, Array.Empty<string>());
|
||||
});
|
||||
_summary.ComputeRollup(Arg.Any<IReadOnlyList<AlarmSummaryRow>>())
|
||||
.Returns(new AlarmRollup(2, 900, 0, new Dictionary<AlarmKind, int>()));
|
||||
Services.AddSingleton(_summary);
|
||||
|
||||
_siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<IReadOnlyList<Site>>(new List<Site> { new("Site 1", "site1") { Id = 1 } }));
|
||||
.Returns(Task.FromResult<IReadOnlyList<Site>>(new List<Site>
|
||||
{
|
||||
new("Site 1", "site1") { Id = 1 },
|
||||
new("Site 2", "site2") { Id = 2 },
|
||||
}));
|
||||
Services.AddSingleton(_siteRepo);
|
||||
|
||||
Services.AddSingleton<ISiteAlarmLiveCache>(_liveCache);
|
||||
|
||||
var claims = new[]
|
||||
{
|
||||
new Claim(JwtTokenService.UsernameClaimType, "tester"),
|
||||
@@ -109,4 +130,133 @@ public class AlarmSummaryRenderTests : BunitContext
|
||||
private static string FirstRowInstance(IRenderedComponent<AlarmSummaryPage> cut) =>
|
||||
cut.FindAll("tr[data-test='alarm-summary-row']")[0]
|
||||
.QuerySelector("td")!.TextContent.Trim();
|
||||
|
||||
// ── plan #10 Task 5: live-cache-first with poll fallback ───────────────────
|
||||
|
||||
[Fact]
|
||||
public void SelectingSite_SubscribesToTheLiveCache()
|
||||
{
|
||||
RenderWithSiteSelected();
|
||||
|
||||
Assert.Equal(1, _liveCache.SubscribeCount);
|
||||
Assert.Equal(1, _liveCache.LastSubscribedSite);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PollFallbackRenders_WhenCacheNotLiveYet()
|
||||
{
|
||||
// Cache starts not-live, so the site select falls back to the poll's
|
||||
// GetSiteAlarmsAsync snapshot (Zeta + Alpha).
|
||||
Assert.False(_liveCache.IsLive(1));
|
||||
|
||||
var cut = RenderWithSiteSelected();
|
||||
|
||||
Assert.Equal(2, cut.FindAll("tr[data-test='alarm-summary-row']").Count);
|
||||
Assert.Equal("Zeta", FirstRowInstance(cut)); // severity-desc default
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OnChangedDelta_RebuildsRenderedRowsFromLiveSnapshot()
|
||||
{
|
||||
var cut = RenderWithSiteSelected();
|
||||
|
||||
// Push a fresh live snapshot: a single new alarm on a new instance.
|
||||
_liveCache.PushAlarms(new List<AlarmStateChanged>
|
||||
{
|
||||
new("Gamma", "G-alarm", AlarmState.Active, 300, DateTimeOffset.UtcNow),
|
||||
});
|
||||
|
||||
cut.WaitForAssertion(() =>
|
||||
{
|
||||
var rows = cut.FindAll("tr[data-test='alarm-summary-row']");
|
||||
Assert.Single(rows);
|
||||
Assert.Equal("Gamma", FirstRowInstance(cut));
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LeavingSite_UnsubscribesFromTheLiveCache()
|
||||
{
|
||||
var cut = RenderWithSiteSelected();
|
||||
Assert.Equal(0, _liveCache.DisposeCount);
|
||||
|
||||
// Clearing the site picker tears the subscription down.
|
||||
cut.Find("[data-test='alarm-summary-site']").Change("");
|
||||
|
||||
Assert.Equal(1, _liveCache.DisposeCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChangingSite_ResubscribesToTheNewSite()
|
||||
{
|
||||
var cut = RenderWithSiteSelected(); // site 1
|
||||
|
||||
cut.Find("[data-test='alarm-summary-site']").Change("2");
|
||||
|
||||
Assert.Equal(2, _liveCache.SubscribeCount); // one per site
|
||||
Assert.Equal(1, _liveCache.DisposeCount); // old subscription disposed
|
||||
Assert.Equal(2, _liveCache.LastSubscribedSite);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DisposingComponent_UnsubscribesFromTheLiveCache()
|
||||
{
|
||||
var cut = RenderWithSiteSelected();
|
||||
|
||||
cut.Instance.Dispose();
|
||||
|
||||
Assert.True(_liveCache.DisposeCount >= 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Controllable in-memory fake of <see cref="ISiteAlarmLiveCache"/>: records
|
||||
/// subscribe/dispose counts and lets a test push a live snapshot (which flips
|
||||
/// <see cref="IsLive"/> true and fires the stored onChanged), so the page's
|
||||
/// live-delta path can be driven deterministically.
|
||||
/// </summary>
|
||||
private sealed class FakeSiteAlarmLiveCache : ISiteAlarmLiveCache
|
||||
{
|
||||
private Action? _onChanged;
|
||||
private IReadOnlyList<AlarmStateChanged> _current = Array.Empty<AlarmStateChanged>();
|
||||
private bool _live;
|
||||
|
||||
public int SubscribeCount { get; private set; }
|
||||
public int DisposeCount { get; private set; }
|
||||
public int? LastSubscribedSite { get; private set; }
|
||||
|
||||
public IDisposable Subscribe(int siteId, Action onChanged)
|
||||
{
|
||||
SubscribeCount++;
|
||||
LastSubscribedSite = siteId;
|
||||
_onChanged = onChanged;
|
||||
return new Subscription(this);
|
||||
}
|
||||
|
||||
public IReadOnlyList<AlarmStateChanged> GetCurrentAlarms(int siteId) => _current;
|
||||
|
||||
public bool IsLive(int siteId) => _live;
|
||||
|
||||
public void PushAlarms(IReadOnlyList<AlarmStateChanged> alarms)
|
||||
{
|
||||
_current = alarms;
|
||||
_live = true;
|
||||
_onChanged?.Invoke();
|
||||
}
|
||||
|
||||
private sealed class Subscription : IDisposable
|
||||
{
|
||||
private readonly FakeSiteAlarmLiveCache _owner;
|
||||
private bool _disposed;
|
||||
|
||||
public Subscription(FakeSiteAlarmLiveCache owner) => _owner = owner;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
_owner.DisposeCount++;
|
||||
_owner._onChanged = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,4 +167,40 @@ public class AlarmSummaryServiceTests
|
||||
Assert.Equal(0, rollup.WorstSeverity);
|
||||
Assert.Equal(0, rollup.UnackedCount);
|
||||
}
|
||||
|
||||
// ── BuildFromLiveAlarms (plan #10, Task 5) ────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void BuildFromLiveAlarms_FlattensAndSorts_SameAsPollPath_WithEmptyNotReporting()
|
||||
{
|
||||
// Intentionally out-of-order input to prove the deterministic
|
||||
// instance-then-alarm-name sort matches GetSiteAlarmsAsync.
|
||||
var alarms = new List<AlarmStateChanged>
|
||||
{
|
||||
NativeAlarm("zeta", "Zulu", AlarmState.Active, 900, active: true, acked: false),
|
||||
NativeAlarm("alpha", "Bravo", AlarmState.Active, 500, active: true, acked: true),
|
||||
NativeAlarm("alpha", "Alpha", AlarmState.Active, 700, active: true, acked: false),
|
||||
};
|
||||
|
||||
var result = CreateSut().BuildFromLiveAlarms(alarms);
|
||||
|
||||
// One row per alarm, instance taken from the alarm itself.
|
||||
Assert.Equal(3, result.Alarms.Count);
|
||||
Assert.Collection(result.Alarms,
|
||||
r => Assert.Equal(("alpha", "Alpha"), (r.InstanceUniqueName, r.Alarm.AlarmName)),
|
||||
r => Assert.Equal(("alpha", "Bravo"), (r.InstanceUniqueName, r.Alarm.AlarmName)),
|
||||
r => Assert.Equal(("zeta", "Zulu"), (r.InstanceUniqueName, r.Alarm.AlarmName)));
|
||||
|
||||
// The live path can't compute "not reporting" — always empty.
|
||||
Assert.Empty(result.NotReportingInstances);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildFromLiveAlarms_EmptySnapshot_YieldsNoRows()
|
||||
{
|
||||
var result = CreateSut().BuildFromLiveAlarms(Array.Empty<AlarmStateChanged>());
|
||||
|
||||
Assert.Empty(result.Alarms);
|
||||
Assert.Empty(result.NotReportingInstances);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,4 +36,54 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcMaxConcurrentStreams", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── Aggregated live alarm cache options (plan #10, Task 6) ───────────────────
|
||||
|
||||
[Fact]
|
||||
public void ZeroLiveAlarmCacheLinger_IsValid()
|
||||
{
|
||||
// Zero linger = stop the aggregator immediately when the last viewer leaves.
|
||||
var result = Validate(new CommunicationOptions { LiveAlarmCacheLinger = TimeSpan.Zero });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegativeLiveAlarmCacheLinger_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { LiveAlarmCacheLinger = TimeSpan.FromSeconds(-1) });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCacheLinger", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroLiveAlarmCacheReconcileInterval_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { LiveAlarmCacheReconcileInterval = TimeSpan.Zero });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCacheReconcileInterval", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroLiveAlarmCacheSeedConcurrency_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { LiveAlarmCacheSeedConcurrency = 0 });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCacheSeedConcurrency", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExcessiveLiveAlarmCacheSeedConcurrency_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { LiveAlarmCacheSeedConcurrency = 65 });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCacheSeedConcurrency", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroLiveAlarmCacheMaxSubscribersPerSite_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { LiveAlarmCacheMaxSubscribersPerSite = 0 });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("LiveAlarmCacheMaxSubscribersPerSite", result.FailureMessage);
|
||||
}
|
||||
}
|
||||
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteAlarmAggregatorActor"/> (plan #10, Task 4) — the transient,
|
||||
/// in-memory, per-site live alarm aggregator. Covers the tricky bits: seed-then-stream
|
||||
/// ordering (a delta arriving before the seed completes is neither lost nor double-applied),
|
||||
/// dedup by AlarmKey, placeholder seeded-but-not-on-stream coherence, NodeA↔NodeB reconnect
|
||||
/// re-seed, and periodic reconcile authoritative-replace (drift correction).
|
||||
/// </summary>
|
||||
public class SiteAlarmAggregatorActorTests : TestKit
|
||||
{
|
||||
private const string SiteId = "site-alpha";
|
||||
private const string GrpcNodeA = "http://localhost:5100";
|
||||
private const string GrpcNodeB = "http://localhost:5200";
|
||||
private const string Instance = "Site1.Pump01";
|
||||
|
||||
public SiteAlarmAggregatorActorTests() : base(@"akka.loglevel = WARNING")
|
||||
{
|
||||
SiteAlarmAggregatorActor.ReconnectDelay = TimeSpan.FromMilliseconds(50);
|
||||
SiteAlarmAggregatorActor.StabilityWindow = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
// ── Test doubles ────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Controllable seed fan-out: each invocation returns a fresh Task the test completes
|
||||
/// on demand (so seed/reconcile timing relative to buffered deltas is deterministic).
|
||||
/// </summary>
|
||||
private sealed class SeedStub
|
||||
{
|
||||
private readonly ConcurrentQueue<TaskCompletionSource<IReadOnlyList<AlarmStateChanged>>> _pending = new();
|
||||
private int _callCount;
|
||||
|
||||
public int CallCount => Volatile.Read(ref _callCount);
|
||||
|
||||
public Task<IReadOnlyList<AlarmStateChanged>> Seed(CancellationToken ct)
|
||||
{
|
||||
Interlocked.Increment(ref _callCount);
|
||||
var tcs = new TaskCompletionSource<IReadOnlyList<AlarmStateChanged>>(
|
||||
TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
ct.Register(() => tcs.TrySetCanceled());
|
||||
_pending.Enqueue(tcs);
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
/// <summary>Completes the oldest not-yet-completed seed call with the given rows.</summary>
|
||||
public void CompleteNext(params AlarmStateChanged[] rows)
|
||||
{
|
||||
SpinWait.SpinUntil(() => _pending.TryPeek(out _), TimeSpan.FromSeconds(3));
|
||||
if (_pending.TryDequeue(out var tcs))
|
||||
tcs.TrySetResult(rows);
|
||||
else
|
||||
throw new InvalidOperationException("No pending seed to complete.");
|
||||
}
|
||||
|
||||
/// <summary>Faults the oldest not-yet-completed seed call (simulates a whole-fan-out failure).</summary>
|
||||
public void FaultNext()
|
||||
{
|
||||
SpinWait.SpinUntil(() => _pending.TryPeek(out _), TimeSpan.FromSeconds(3));
|
||||
if (_pending.TryDequeue(out var tcs))
|
||||
tcs.TrySetException(new InvalidOperationException("seed boom"));
|
||||
else
|
||||
throw new InvalidOperationException("No pending seed to fault.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PublishSink
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
public List<IReadOnlyList<AlarmStateChanged>> Snapshots { get; } = new();
|
||||
|
||||
public void Publish(IReadOnlyList<AlarmStateChanged> snapshot)
|
||||
{
|
||||
lock (_lock) { Snapshots.Add(snapshot); }
|
||||
}
|
||||
|
||||
public IReadOnlyList<AlarmStateChanged>? Latest
|
||||
{
|
||||
get { lock (_lock) { return Snapshots.Count == 0 ? null : Snapshots[^1]; } }
|
||||
}
|
||||
|
||||
public int Count { get { lock (_lock) { return Snapshots.Count; } } }
|
||||
}
|
||||
|
||||
private sealed record SiteSub(
|
||||
string CorrelationId, Action<AlarmStateChanged> OnAlarm, Action<Exception> OnError, CancellationToken Ct);
|
||||
|
||||
private sealed class MockSiteAlarmStreamClient : SiteStreamGrpcClient
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private readonly List<SiteSub> _subs = new();
|
||||
private readonly List<string> _unsubscribed = new();
|
||||
|
||||
public List<SiteSub> Subs { get { lock (_lock) { return _subs.ToList(); } } }
|
||||
public List<string> Unsubscribed { get { lock (_lock) { return _unsubscribed.ToList(); } } }
|
||||
|
||||
public MockSiteAlarmStreamClient() : base() { }
|
||||
|
||||
public override Task SubscribeSiteAsync(
|
||||
string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError, CancellationToken ct)
|
||||
{
|
||||
lock (_lock) { _subs.Add(new SiteSub(correlationId, onAlarmEvent, onError, ct)); }
|
||||
var tcs = new TaskCompletionSource();
|
||||
ct.Register(() => tcs.TrySetResult());
|
||||
return tcs.Task; // never completes until cancelled (simulates a live stream)
|
||||
}
|
||||
|
||||
public override void Unsubscribe(string correlationId)
|
||||
{
|
||||
lock (_lock) { _unsubscribed.Add(correlationId); }
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class MockSiteAlarmStreamClientFactory : SiteStreamGrpcClientFactory
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, MockSiteAlarmStreamClient> _byEndpoint = new();
|
||||
|
||||
public MockSiteAlarmStreamClientFactory()
|
||||
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance) { }
|
||||
|
||||
public MockSiteAlarmStreamClient ClientFor(string endpoint) =>
|
||||
_byEndpoint.GetOrAdd(endpoint, _ => new MockSiteAlarmStreamClient());
|
||||
|
||||
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
||||
=> ClientFor(grpcEndpoint);
|
||||
|
||||
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint)
|
||||
=> _byEndpoint.TryGetValue(grpcEndpoint, out var c) ? c : null;
|
||||
}
|
||||
|
||||
private (IActorRef Actor, SeedStub Seed, PublishSink Sink, MockSiteAlarmStreamClientFactory Factory) CreateActor(
|
||||
TimeSpan? reconcileInterval = null)
|
||||
{
|
||||
var seed = new SeedStub();
|
||||
var sink = new PublishSink();
|
||||
var factory = new MockSiteAlarmStreamClientFactory();
|
||||
|
||||
var props = Props.Create(() => new SiteAlarmAggregatorActor(
|
||||
SiteId, "corr-1", seed.Seed, sink.Publish, factory, GrpcNodeA, GrpcNodeB,
|
||||
reconcileInterval ?? TimeSpan.FromMinutes(10)));
|
||||
|
||||
var actor = Sys.ActorOf(props);
|
||||
return (actor, seed, sink, factory);
|
||||
}
|
||||
|
||||
private static AlarmStateChanged Alarm(
|
||||
string alarmName, string sourceRef, int priority, DateTimeOffset ts,
|
||||
bool placeholder = false, string instance = Instance) =>
|
||||
new(instance, alarmName, AlarmState.Active, priority, ts)
|
||||
{
|
||||
SourceReference = sourceRef,
|
||||
IsConfiguredPlaceholder = placeholder,
|
||||
Kind = placeholder ? AlarmKind.NativeOpcUa : AlarmKind.Computed
|
||||
};
|
||||
|
||||
private static AlarmStateChanged? Find(IReadOnlyList<AlarmStateChanged>? rows, string alarmName, string sourceRef) =>
|
||||
rows?.FirstOrDefault(r => r.AlarmName == alarmName && r.SourceReference == sourceRef);
|
||||
|
||||
// ── Tests ───────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Delta_Arriving_Before_Seed_Completes_Is_Buffered_And_Applied_Exactly_Once()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Live delta arrives DURING the seed window (new key, not in the seed).
|
||||
var t1 = DateTimeOffset.UtcNow;
|
||||
actor.Tell(Alarm("Overheat", "", 900, t1));
|
||||
|
||||
// Still buffering — nothing published yet.
|
||||
Assert.Equal(0, sink.Count);
|
||||
|
||||
// Seed completes with a DIFFERENT alarm.
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, t1.AddSeconds(-1)));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 2, TimeSpan.FromSeconds(3));
|
||||
var latest = sink.Latest!;
|
||||
Assert.NotNull(Find(latest, "PumpFault", "")); // seeded
|
||||
var overheat = Find(latest, "Overheat", ""); // buffered gap-window delta survived
|
||||
Assert.NotNull(overheat);
|
||||
Assert.Single(latest, r => r.AlarmName == "Overheat"); // exactly once
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Buffered_Delta_Older_Than_Seed_Entry_Is_Dropped()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var tSeed = DateTimeOffset.UtcNow;
|
||||
// Buffered delta for the SAME key but OLDER than the seed entry.
|
||||
actor.Tell(Alarm("PumpFault", "", 100, tSeed.AddSeconds(-1)));
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
|
||||
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
// Give a beat to ensure no late (dropped) delta sneaks in.
|
||||
Thread.Sleep(150);
|
||||
var latest = sink.Latest!;
|
||||
var pumpFault = Find(latest, "PumpFault", "");
|
||||
Assert.NotNull(pumpFault);
|
||||
Assert.Equal(500, pumpFault!.Priority); // seed value kept; older buffered delta dropped
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Buffered_Delta_Newer_Than_Seed_Entry_Is_Applied()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var tSeed = DateTimeOffset.UtcNow;
|
||||
actor.Tell(Alarm("PumpFault", "", 900, tSeed.AddSeconds(1))); // newer than seed
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, tSeed));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && Find(l, "PumpFault", "")?.Priority == 900,
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dedup_By_AlarmKey_Distinguishes_SourceReference()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // empty seed
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
// Same instance + alarm name, different source reference → two distinct rows.
|
||||
actor.Tell(Alarm("LevelAlarm", "Tank01.Level.HiHi", 700, t));
|
||||
actor.Tell(Alarm("LevelAlarm", "Tank01.Level.LoLo", 300, t));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 2, TimeSpan.FromSeconds(3));
|
||||
var latest = sink.Latest!;
|
||||
Assert.NotNull(Find(latest, "LevelAlarm", "Tank01.Level.HiHi"));
|
||||
Assert.NotNull(Find(latest, "LevelAlarm", "Tank01.Level.LoLo"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Same_Key_Live_Update_Replaces_In_Place()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
actor.Tell(Alarm("PumpFault", "", 500, t));
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "")?.Priority == 500, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Newer update for the same key replaces (not duplicates).
|
||||
actor.Tell(Alarm("PumpFault", "", 800, t.AddSeconds(1)));
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "")?.Priority == 800, TimeSpan.FromSeconds(3));
|
||||
Assert.Single(sink.Latest!); // still exactly one row for the key
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Seeded_Native_Alarm_And_Its_Live_Delta_Collapse_Onto_One_Key()
|
||||
{
|
||||
// Identity parity across the two paths (snapshot seed vs live stream): a native
|
||||
// alarm seeded with a non-empty SourceReference and a later live delta for the
|
||||
// SAME (instance, alarmName, sourceRef) must collapse to ONE in-place-updated row,
|
||||
// not a ghost duplicate. Both paths key on SourceReference, so they must agree.
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
seed.CompleteNext(Alarm("Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi", 300, t));
|
||||
AwaitCondition(() => Find(sink.Latest, "Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi") is not null,
|
||||
TimeSpan.FromSeconds(3));
|
||||
|
||||
// Live delta for the exact same identity, newer → updates in place.
|
||||
actor.Tell(Alarm("Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi", 800, t.AddSeconds(1)));
|
||||
|
||||
AwaitCondition(
|
||||
() => Find(sink.Latest, "Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi")?.Priority == 800,
|
||||
TimeSpan.FromSeconds(3));
|
||||
Assert.Single(sink.Latest!); // exactly one row — no seed/live ghost duplicate
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Placeholder_Seeded_Coexists_With_Live_Real_Alarm_And_Is_Not_Wiped()
|
||||
{
|
||||
var (actor, seed, sink, _) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
// Seed carries a configured-placeholder row (binding with no active conditions).
|
||||
var placeholder = Alarm("Motor1.MotorAlarms", "", 0, t, placeholder: true);
|
||||
seed.CompleteNext(placeholder);
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// A live REAL condition for the same instance arrives (distinct AlarmKey).
|
||||
actor.Tell(Alarm("Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi", 800, t.AddSeconds(1)));
|
||||
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 2, TimeSpan.FromSeconds(3));
|
||||
var latest = sink.Latest!;
|
||||
// Placeholder still present (not wiped by the real alarm under a different key).
|
||||
var ph = Find(latest, "Motor1.MotorAlarms", "");
|
||||
Assert.NotNull(ph);
|
||||
Assert.True(ph!.IsConfiguredPlaceholder);
|
||||
Assert.NotNull(Find(latest, "Motor1.MotorAlarms", "Motor1.MotorAlarms.HiHi"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GrpcError_Flips_Node_Reconnects_And_ReSeeds()
|
||||
{
|
||||
var (_, seed, sink, factory) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // initial seed done
|
||||
// Wait for the seed to actually be APPLIED (published) so the re-seed on error is
|
||||
// not skipped as "fan-out already in flight".
|
||||
AwaitCondition(() => sink.Count >= 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Fail the NodeA stream.
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("NodeA down"));
|
||||
|
||||
// Reconnect must reach NodeB, AND a re-seed fan-out must have been triggered.
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(5));
|
||||
// The failed NodeA stream was unsubscribed (relay released, not left zombie).
|
||||
Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).Unsubscribed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Periodic_Reconcile_Authoritatively_Replaces_Stale_Rows()
|
||||
{
|
||||
var (_, seed, sink, _) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(250));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
var t = DateTimeOffset.UtcNow;
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, t)); // seed has a row
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "") is not null, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Reconcile tick fires → seed #2 (e.g. instance was disabled) returns EMPTY.
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(); // empty
|
||||
|
||||
// The stale PumpFault row must disappear (authoritative replace, not merge).
|
||||
AwaitCondition(() => sink.Latest is { } l && l.Count == 0, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failed_Initial_Seed_Does_Not_Publish_But_A_Later_Reconcile_Recovers()
|
||||
{
|
||||
var (_, seed, sink, _) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(250));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// The initial seed fan-out throws as a whole.
|
||||
seed.FaultNext();
|
||||
|
||||
// A failed INITIAL seed must not publish (page keeps its poll fallback — not live yet).
|
||||
Thread.Sleep(200);
|
||||
Assert.Equal(0, sink.Count);
|
||||
|
||||
// The periodic reconcile is the backstop: it re-runs the fan-out (flag was cleared
|
||||
// on failure), completes, and the actor publishes → now live.
|
||||
AwaitCondition(() => seed.CallCount == 2, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext(Alarm("PumpFault", "", 500, DateTimeOffset.UtcNow));
|
||||
AwaitCondition(() => Find(sink.Latest, "PumpFault", "") is not null, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stream_GivenUp_After_MaxRetries_Reopens_On_Reconcile_Tick()
|
||||
{
|
||||
// After the retry budget is exhausted the live stream is left down, but the
|
||||
// aggregator is NOT stopped — the next reconcile tick self-heals it (resets the
|
||||
// budget + reopens) so a sustained site outage never permanently kills the feed.
|
||||
var (_, seed, _, factory) = CreateActor(reconcileInterval: TimeSpan.FromMilliseconds(300));
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
|
||||
// Drive initial + 3 retries (each flips node) to exhaust the budget.
|
||||
factory.ClientFor(GrpcNodeA).Subs[0].OnError(new Exception("1"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 1, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeB).Subs[0].OnError(new Exception("2"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 2, TimeSpan.FromSeconds(5));
|
||||
factory.ClientFor(GrpcNodeA).Subs[1].OnError(new Exception("3"));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeB).Subs.Count == 2, TimeSpan.FromSeconds(5));
|
||||
|
||||
int TotalSubs() => factory.ClientFor(GrpcNodeA).Subs.Count + factory.ClientFor(GrpcNodeB).Subs.Count;
|
||||
var before = TotalSubs(); // 4
|
||||
|
||||
// Fourth error exceeds MaxRetries → stream given up (no immediate reopen).
|
||||
factory.ClientFor(GrpcNodeB).Subs[1].OnError(new Exception("4"));
|
||||
|
||||
// The reconcile tick reopens the stream (self-heal).
|
||||
AwaitCondition(() => TotalSubs() > before, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stop_Message_TearsDown_Grpc_And_Stops_Actor()
|
||||
{
|
||||
var (actor, seed, _, factory) = CreateActor();
|
||||
AwaitCondition(() => seed.CallCount == 1, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Subs.Count == 1, TimeSpan.FromSeconds(3));
|
||||
seed.CompleteNext();
|
||||
|
||||
Watch(actor);
|
||||
actor.Tell(new StopSiteAlarmAggregator());
|
||||
|
||||
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
||||
AwaitCondition(() => factory.ClientFor(GrpcNodeA).Unsubscribed.Contains("corr-1"),
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
}
|
||||
@@ -208,6 +208,78 @@ public class SiteStreamGrpcClientTests
|
||||
Assert.True(cts2.IsCancellationRequested);
|
||||
}
|
||||
|
||||
// --- Site-wide (SubscribeSite) alarm-only stream (plan #10 T3) ---
|
||||
|
||||
[Fact]
|
||||
public void ConvertToAlarmEvent_AlarmChanged_ReturnsMappedAlarm()
|
||||
{
|
||||
// An alarm event on the site-wide stream is delivered to onAlarmEvent with full enrichment.
|
||||
var ts = DateTimeOffset.UtcNow;
|
||||
var evt = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "site-corr",
|
||||
AlarmChanged = new AlarmStateUpdate
|
||||
{
|
||||
InstanceUniqueName = "Site1.Motor01",
|
||||
AlarmName = "T01.Hi",
|
||||
State = AlarmStateEnum.AlarmStateActive,
|
||||
Priority = 700,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(ts),
|
||||
Kind = "NativeOpcUa",
|
||||
Active = true,
|
||||
SourceReference = "T01.Hi"
|
||||
}
|
||||
};
|
||||
|
||||
var alarm = SiteStreamGrpcClient.ConvertToAlarmEvent(evt);
|
||||
|
||||
Assert.NotNull(alarm);
|
||||
Assert.Equal("Site1.Motor01", alarm!.InstanceUniqueName);
|
||||
Assert.Equal("T01.Hi", alarm.AlarmName);
|
||||
Assert.Equal(AlarmState.Active, alarm.State);
|
||||
Assert.Equal(AlarmKind.NativeOpcUa, alarm.Kind);
|
||||
Assert.Equal("T01.Hi", alarm.SourceReference);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertToAlarmEvent_AttributeChanged_ReturnsNull()
|
||||
{
|
||||
// Attribute events must never appear on the alarm-only site-wide stream; if one
|
||||
// somehow arrives it is defensively filtered out rather than delivered or thrown.
|
||||
var evt = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "site-corr",
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "Site1.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Temperature",
|
||||
Value = "42.5",
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow)
|
||||
}
|
||||
};
|
||||
|
||||
Assert.Null(SiteStreamGrpcClient.ConvertToAlarmEvent(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ConvertToAlarmEvent_UnknownEvent_ReturnsNull()
|
||||
{
|
||||
var evt = new SiteStreamEvent { CorrelationId = "site-corr" };
|
||||
Assert.Null(SiteStreamGrpcClient.ConvertToAlarmEvent(evt));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeSiteAsync_OnTestOnlyClient_Throws()
|
||||
{
|
||||
// Guards against subscribing on a channel-less test double, mirroring SubscribeAsync.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
client.SubscribeSiteAsync("corr", _ => { }, _ => { }, CancellationToken.None));
|
||||
}
|
||||
|
||||
// --- Communication-003 regression tests ---
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -21,6 +21,8 @@ public class SiteStreamGrpcServerTests : TestKit
|
||||
_subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
_subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns("sub-1");
|
||||
_subscriber.SubscribeSiteAlarms(Arg.Any<IActorRef>())
|
||||
.Returns("site-sub-1");
|
||||
_logger = NullLogger<SiteStreamGrpcServer>.Instance;
|
||||
}
|
||||
|
||||
@@ -266,6 +268,102 @@ public class SiteStreamGrpcServerTests : TestKit
|
||||
await streamTask;
|
||||
}
|
||||
|
||||
// --- SubscribeSite (site-wide, alarm-only aggregated stream, plan #10 T2) ---
|
||||
|
||||
private static SiteStreamRequest MakeSiteRequest(string correlationId = "site-corr-1")
|
||||
=> new() { CorrelationId = correlationId };
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeSite_SubscribesSiteAlarmsAndRemovesOnCancel()
|
||||
{
|
||||
var server = CreateServer();
|
||||
server.SetReady(Sys);
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeSite(
|
||||
MakeSiteRequest("site-corr-sub"), writer, context));
|
||||
|
||||
await WaitForConditionAsync(() => server.ActiveStreamCount == 1);
|
||||
|
||||
// Site-wide handler must call SubscribeSiteAlarms (no instance filter),
|
||||
// never the per-instance Subscribe.
|
||||
_subscriber.Received(1).SubscribeSiteAlarms(Arg.Any<IActorRef>());
|
||||
_subscriber.DidNotReceive().Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>());
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
_subscriber.Received(1).RemoveSubscriber(Arg.Any<IActorRef>());
|
||||
Assert.Equal(0, server.ActiveStreamCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeSite_RejectsUnsafeCorrelationId()
|
||||
{
|
||||
var server = CreateServer();
|
||||
server.SetReady(Sys);
|
||||
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
var context = CreateMockContext();
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
() => server.SubscribeSite(MakeSiteRequest("bad/id"), writer, context));
|
||||
|
||||
Assert.Equal(StatusCode.InvalidArgument, ex.StatusCode);
|
||||
Assert.Equal(0, server.ActiveStreamCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SubscribeSite_RelaysAlarmStateChangedAsAlarmStateUpdate()
|
||||
{
|
||||
var server = CreateServer();
|
||||
server.SetReady(Sys);
|
||||
|
||||
// Capture the relay actor spawned for the site-wide subscription.
|
||||
IActorRef? capturedActor = null;
|
||||
_subscriber.SubscribeSiteAlarms(Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
capturedActor = ci.Arg<IActorRef>();
|
||||
return "site-sub-relay";
|
||||
});
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
var writtenEvents = new List<SiteStreamEvent>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => writtenEvents.Add(ci.Arg<SiteStreamEvent>()));
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeSite(
|
||||
MakeSiteRequest("site-corr-write"), writer, context));
|
||||
|
||||
await WaitForConditionAsync(() => capturedActor != null);
|
||||
|
||||
// A real alarm transition for ANY instance must arrive as an AlarmStateUpdate.
|
||||
capturedActor!.Tell(new Commons.Messages.Streaming.AlarmStateChanged(
|
||||
"Site1.Pump01",
|
||||
"HighPressure",
|
||||
Commons.Types.Enums.AlarmState.Active,
|
||||
700,
|
||||
DateTimeOffset.UtcNow));
|
||||
|
||||
await WaitForConditionAsync(() => writtenEvents.Count >= 1);
|
||||
|
||||
Assert.Single(writtenEvents);
|
||||
Assert.Equal("site-corr-write", writtenEvents[0].CorrelationId);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AlarmChanged, writtenEvents[0].EventCase);
|
||||
Assert.Equal("Site1.Pump01", writtenEvents[0].AlarmChanged.InstanceUniqueName);
|
||||
Assert.Equal("HighPressure", writtenEvents[0].AlarmChanged.AlarmName);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("corr/with/slash")]
|
||||
[InlineData("corr with space")]
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteAlarmLiveCacheService"/> (plan #10, Task 4) — the DI singleton
|
||||
/// that reference-counts Alarm Summary viewers over one shared per-site aggregator. Covers
|
||||
/// shared start (one aggregator for many viewers), last-out linger stop, and re-subscribe
|
||||
/// cancelling a pending stop. Uses a real (in-memory) ActorSystem and a mock gRPC factory
|
||||
/// whose site-wide stream never faults, so no reconnect noise.
|
||||
/// </summary>
|
||||
public class SiteAlarmLiveCacheServiceTests : TestKit
|
||||
{
|
||||
private const int SiteId = 7;
|
||||
|
||||
// A mock site-wide alarm stream client whose subscription hangs until cancelled.
|
||||
private sealed class HangingClient : SiteStreamGrpcClient
|
||||
{
|
||||
public HangingClient() : base() { }
|
||||
public override Task SubscribeSiteAsync(
|
||||
string correlationId, Action<Commons.Messages.Streaming.AlarmStateChanged> onAlarmEvent,
|
||||
Action<Exception> onError, CancellationToken ct)
|
||||
{
|
||||
var tcs = new TaskCompletionSource();
|
||||
ct.Register(() => tcs.TrySetResult());
|
||||
return tcs.Task;
|
||||
}
|
||||
public override void Unsubscribe(string correlationId) { }
|
||||
}
|
||||
|
||||
private sealed class CountingFactory : SiteStreamGrpcClientFactory
|
||||
{
|
||||
private readonly HangingClient _client = new();
|
||||
public int GetOrCreateCount;
|
||||
|
||||
public CountingFactory() : base(NullLoggerFactory.Instance) { }
|
||||
|
||||
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
||||
{
|
||||
Interlocked.Increment(ref GetOrCreateCount);
|
||||
return _client;
|
||||
}
|
||||
|
||||
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
|
||||
}
|
||||
|
||||
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
|
||||
int maxSubscribersPerSite = 200)
|
||||
{
|
||||
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
|
||||
// empty immediately (so IsLive flips true fast without any snapshot Asks).
|
||||
var site = new Site("Alpha", "site-alpha")
|
||||
{
|
||||
Id = SiteId,
|
||||
GrpcNodeAAddress = "http://localhost:5100",
|
||||
GrpcNodeBAddress = "http://localhost:5200"
|
||||
};
|
||||
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns(site);
|
||||
|
||||
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Instance>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
services.AddScoped(_ => instanceRepo);
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
var options = Options.Create(new CommunicationOptions
|
||||
{
|
||||
LiveAlarmCacheLinger = linger,
|
||||
LiveAlarmCacheReconcileInterval = TimeSpan.FromMinutes(10),
|
||||
LiveAlarmCacheMaxSubscribersPerSite = maxSubscribersPerSite
|
||||
});
|
||||
|
||||
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
|
||||
factory = new CountingFactory();
|
||||
var service = new SiteAlarmLiveCacheService(
|
||||
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
|
||||
service.SetActorSystem(Sys);
|
||||
return service;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out var factory);
|
||||
|
||||
var changes1 = 0;
|
||||
var changes2 = 0;
|
||||
using var sub1 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes1));
|
||||
using var sub2 = service.Subscribe(SiteId, () => Interlocked.Increment(ref changes2));
|
||||
|
||||
// The (empty) seed completes → aggregator publishes → live.
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
// Exactly ONE gRPC stream opened for the site regardless of viewer count.
|
||||
Assert.Equal(1, factory.GetOrCreateCount);
|
||||
// Empty seed → empty current snapshot.
|
||||
Assert.Empty(service.GetCurrentAlarms(SiteId));
|
||||
// Both viewers were notified of the initial publish.
|
||||
AwaitCondition(() => changes1 >= 1 && changes2 >= 1, TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Last_Viewer_Leaving_Stops_Aggregator_After_Linger()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
|
||||
var sub1 = service.Subscribe(SiteId, () => { });
|
||||
var sub2 = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
sub1.Dispose();
|
||||
// One viewer remains — still live, not stopped.
|
||||
Thread.Sleep(300);
|
||||
Assert.True(service.IsLive(SiteId));
|
||||
|
||||
sub2.Dispose();
|
||||
// Last viewer left → after the linger the aggregator is torn down and the entry cleared.
|
||||
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
||||
Assert.Empty(service.GetCurrentAlarms(SiteId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Resubscribe_Within_Linger_Cancels_The_Pending_Stop()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(400), out var factory);
|
||||
|
||||
var sub1 = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
sub1.Dispose(); // schedules linger stop
|
||||
var sub2 = service.Subscribe(SiteId, () => { }); // returns within the linger window → cancels it
|
||||
|
||||
// Wait comfortably past the original linger; the aggregator must still be alive.
|
||||
Thread.Sleep(700);
|
||||
Assert.True(service.IsLive(SiteId));
|
||||
// No second aggregator/stream was opened — the same one was kept warm.
|
||||
Assert.Equal(1, factory.GetOrCreateCount);
|
||||
|
||||
sub2.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dispose_Is_Idempotent()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
var sub = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
sub.Dispose();
|
||||
sub.Dispose(); // must not throw or double-decrement
|
||||
|
||||
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Transient_Start_Failure_Self_Heals_On_Retry_For_A_Stable_Viewer_Cohort()
|
||||
{
|
||||
// A transient failure resolving the site at first-subscribe must NOT leave the
|
||||
// viewer cohort polling forever — a bounded retry re-attempts and self-heals.
|
||||
var site = new Site("Alpha", "site-alpha")
|
||||
{
|
||||
Id = SiteId,
|
||||
GrpcNodeAAddress = "http://localhost:5100",
|
||||
GrpcNodeBAddress = "http://localhost:5200"
|
||||
};
|
||||
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
// First resolve returns null (transient blip); every resolve thereafter succeeds.
|
||||
siteRepo.GetSiteByIdAsync(SiteId, Arg.Any<CancellationToken>()).Returns((Site?)null, site);
|
||||
|
||||
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Instance>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
services.AddScoped(_ => instanceRepo);
|
||||
var provider = services.BuildServiceProvider();
|
||||
|
||||
var options = Options.Create(new CommunicationOptions
|
||||
{
|
||||
LiveAlarmCacheLinger = TimeSpan.FromSeconds(30),
|
||||
// Retry cadence reuses the reconcile interval — keep it short for the test.
|
||||
LiveAlarmCacheReconcileInterval = TimeSpan.FromMilliseconds(300)
|
||||
});
|
||||
var comm = new CommunicationService(Options.Create(new CommunicationOptions()),
|
||||
NullLogger<CommunicationService>.Instance);
|
||||
var factory = new CountingFactory();
|
||||
|
||||
var service = new SiteAlarmLiveCacheService(
|
||||
provider, comm, factory, options, NullLogger<SiteAlarmLiveCacheService>.Instance);
|
||||
service.SetActorSystem(Sys);
|
||||
|
||||
using var sub = service.Subscribe(SiteId, () => { });
|
||||
|
||||
// First start failed (site not found); a stable single viewer stays subscribed.
|
||||
// The retry eventually resolves the site and the aggregator goes live — without
|
||||
// any further Subscribe call.
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Viewer_Over_The_Per_Site_Cap_Is_Rejected_Safely_And_Not_Counted()
|
||||
{
|
||||
// Cap of 2: the first two viewers register; a third is rejected fail-safe with a
|
||||
// no-op handle (never throws into the render path, never grows the list). Proof it
|
||||
// was NOT counted: disposing only the two real viewers tears the aggregator down
|
||||
// after the linger — if the rejected viewer had been registered, one subscriber
|
||||
// would remain and the site would stay live forever.
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _, maxSubscribersPerSite: 2);
|
||||
|
||||
var sub1 = service.Subscribe(SiteId, () => { });
|
||||
var sub2 = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
// Third viewer is over the cap.
|
||||
var overflow = service.Subscribe(SiteId, () => { });
|
||||
Assert.NotNull(overflow);
|
||||
// Disposing the rejected handle is a genuine no-op — safe, idempotent, touches nothing.
|
||||
overflow.Dispose();
|
||||
overflow.Dispose();
|
||||
|
||||
// Still live with the two real viewers present.
|
||||
Assert.True(service.IsLive(SiteId));
|
||||
|
||||
sub1.Dispose();
|
||||
sub2.Dispose();
|
||||
|
||||
// Both real viewers gone → after the linger the aggregator stops. This only holds
|
||||
// if the overflow viewer was never added to the subscriber list.
|
||||
AwaitCondition(() => !service.IsLive(SiteId), TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_Site_Registers_Viewer_But_Never_Goes_Live()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
|
||||
using var sub = service.Subscribe(999, () => { }); // no such site in the repo
|
||||
Thread.Sleep(300);
|
||||
Assert.False(service.IsLive(999));
|
||||
Assert.Empty(service.GetCurrentAlarms(999));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Telemetry guards for the aggregated live alarm cache (plan #10, Task 6): the
|
||||
/// active-aggregator observable gauge reflects Started/Stopped, and the reconnect
|
||||
/// counter increments on each recorded reconnect. Mirrors the MeterListener read
|
||||
/// pattern used by <c>SiteStreamGrpcServerTests.SiteConnectionUpGauge_*</c>; reads are
|
||||
/// relative to a baseline so the process-wide static instruments are robust to any
|
||||
/// parallel test interleaving.
|
||||
/// </summary>
|
||||
public class SiteAlarmLiveCacheTelemetryTests
|
||||
{
|
||||
private static long ReadGauge(string instrumentName)
|
||||
{
|
||||
long observed = 0;
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
|
||||
instrument.Name == instrumentName)
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => observed = measurement);
|
||||
listener.Start();
|
||||
listener.RecordObservableInstruments();
|
||||
return observed;
|
||||
}
|
||||
|
||||
private static long ReadCounter(string instrumentName, Action act)
|
||||
{
|
||||
long delta = 0;
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
|
||||
instrument.Name == instrumentName)
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((_, measurement, _, _) => delta += measurement);
|
||||
listener.Start();
|
||||
act();
|
||||
return delta;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActiveAggregatorGauge_ReflectsStartAndStop()
|
||||
{
|
||||
const string gauge = "scadabridge.site.alarm_cache.aggregators.active";
|
||||
var baseline = ReadGauge(gauge);
|
||||
|
||||
ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
|
||||
try
|
||||
{
|
||||
Assert.Equal(baseline + 1, ReadGauge(gauge));
|
||||
|
||||
ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
|
||||
try
|
||||
{
|
||||
Assert.Equal(baseline + 2, ReadGauge(gauge));
|
||||
}
|
||||
finally
|
||||
{
|
||||
ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
|
||||
}
|
||||
|
||||
Assert.Equal(baseline + 1, ReadGauge(gauge));
|
||||
}
|
||||
finally
|
||||
{
|
||||
ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
|
||||
}
|
||||
|
||||
Assert.Equal(baseline, ReadGauge(gauge));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReconnectCounter_IncrementsOnEachRecordedReconnect()
|
||||
{
|
||||
var delta = ReadCounter("scadabridge.site.alarm_cache.reconnects", () =>
|
||||
{
|
||||
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
|
||||
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
|
||||
});
|
||||
|
||||
Assert.Equal(2, delta);
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,9 @@ public class ScadaBridgeTelemetryTests
|
||||
ScadaBridgeTelemetry.RecordInboundApiRequest("X");
|
||||
ScadaBridgeTelemetry.SiteConnectionOpened();
|
||||
ScadaBridgeTelemetry.SiteConnectionClosed();
|
||||
ScadaBridgeTelemetry.LiveAlarmAggregatorStarted();
|
||||
ScadaBridgeTelemetry.LiveAlarmAggregatorStopped();
|
||||
ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect();
|
||||
ScadaBridgeTelemetry.SetQueueDepthProvider(() => 5);
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end trace for the aggregated live alarm stream (plan #10 deferred item, Task 7).
|
||||
///
|
||||
/// Every per-task unit test (T1–T6) exercises exactly one layer with its neighbour mocked:
|
||||
/// T1 the site broadcast filter (subscriber = TestKit probe, domain events), T2/the existing
|
||||
/// <see cref="GrpcStreamIntegrationTests"/> the server+relay+channel with a MOCKED
|
||||
/// <see cref="ISiteStreamSubscriber"/> stopping at the proto event, T3 the proto→domain client
|
||||
/// mapping in isolation, and T4 the aggregator fed hand-built domain events. Nothing wires the
|
||||
/// whole pipe together.
|
||||
///
|
||||
/// This test assembles the REAL chain end to end — mocking only the gRPC HTTP/2 transport:
|
||||
///
|
||||
/// domain AlarmStateChanged
|
||||
/// → SiteStreamManager.SubscribeSiteAlarms (real site-wide broadcast + alarm-only filter)
|
||||
/// → SiteStreamGrpcServer.SubscribeSite (real server handler)
|
||||
/// → StreamRelayActor (real domain→proto mapping + placeholder drop)
|
||||
/// → SiteStreamEvent proto → wire round-trip (ToByteArray/ParseFrom, simulates HTTP/2)
|
||||
/// → SiteStreamGrpcClient.ConvertToAlarmEvent (real proto→domain mapping)
|
||||
/// → SiteAlarmAggregatorActor cache (real dedup/seed/live)
|
||||
///
|
||||
/// and asserts the invariants the seams can't prove alone: AlarmKey identity
|
||||
/// (instance, name, sourceRef) + native enrichment survive every boundary; attributes and
|
||||
/// placeholder rows never reach the live cache; the single site-wide stream carries alarms
|
||||
/// for MULTIPLE instances (no per-instance filter); and a snapshot-seed row and a live delta
|
||||
/// for the same native alarm — both mapped through the real pipe — collapse onto ONE cache row.
|
||||
///
|
||||
/// Full gRPC-over-HTTP/2 remains a manual docker-cluster smoke (plan §4 Task 7).
|
||||
/// </summary>
|
||||
public class SiteAlarmStreamEndToEndTests : TestKit
|
||||
{
|
||||
private const string InstanceA = "SiteA.Pump01";
|
||||
private const string InstanceB = "SiteA.Motor07";
|
||||
|
||||
// ── The full site→proto→client trace ────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task SiteWideAlarm_TraversesManagerToRelayToClient_PreservingIdentityAndEnrichment_DroppingAttributesAndPlaceholders()
|
||||
{
|
||||
// Site side: a REAL broadcast hub (no mock subscriber) wired straight into the
|
||||
// REAL SubscribeSite server handler.
|
||||
var manager = new SiteStreamManager(
|
||||
new SiteRuntimeOptions { StreamBufferSize = 256 },
|
||||
NullLogger<SiteStreamManager>.Instance);
|
||||
manager.Initialize(Sys);
|
||||
|
||||
var server = new SiteStreamGrpcServer(manager, NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
var written = new ConcurrentQueue<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => written.Enqueue(ci.Arg<SiteStreamEvent>()));
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeSite(
|
||||
new SiteStreamRequest { CorrelationId = "e2e-site" }, writer, context));
|
||||
|
||||
// Wait until the server has actually subscribed to the site-wide hub, otherwise a
|
||||
// publish can race ahead of the materialized subscription and be missed.
|
||||
await WaitForConditionAsync(() => manager.SubscriptionCount == 1);
|
||||
|
||||
var raise = new DateTimeOffset(2026, 4, 2, 8, 30, 0, TimeSpan.Zero);
|
||||
var ts = raise.AddSeconds(5);
|
||||
|
||||
// A fully-enriched NATIVE alarm on instance A (the payload whose fidelity we trace).
|
||||
var native = new AlarmStateChanged(InstanceA, "Tank01.LevelAlarm", AlarmState.Active, 725, ts)
|
||||
{
|
||||
Level = AlarmLevel.HighHigh,
|
||||
Message = "Tank 01 level critically high",
|
||||
Kind = AlarmKind.NativeOpcUa,
|
||||
Condition = new AlarmConditionState(
|
||||
Active: true, Acknowledged: true, Confirmed: false,
|
||||
Shelve: AlarmShelveState.OneShotShelved, Suppressed: false, Severity: 725),
|
||||
SourceReference = "Tank01.Level.HiHi",
|
||||
AlarmTypeName = "AnalogLimitAlarm.HiHi",
|
||||
Category = "Process",
|
||||
OperatorUser = "op.jane",
|
||||
OperatorComment = "ack — investigating",
|
||||
OriginalRaiseTime = raise,
|
||||
CurrentValue = "98.4",
|
||||
LimitValue = "95.0",
|
||||
NativeSourceCanonicalName = "Tank01.LevelAlarm",
|
||||
IsConfiguredPlaceholder = false
|
||||
};
|
||||
|
||||
// A COMPUTED alarm on a DIFFERENT instance — must arrive over the SAME site-wide
|
||||
// stream (proves the per-instance filter is gone).
|
||||
var computedOtherInstance = new AlarmStateChanged(InstanceB, "OverSpeed", AlarmState.Active, 400, ts);
|
||||
|
||||
// Noise that MUST be dropped somewhere in the pipe:
|
||||
var attribute = new AttributeValueChanged(InstanceA, "Modules.Flow", "GPM", 12.3, "Good", ts);
|
||||
var placeholder = new AlarmStateChanged(InstanceA, "Motor1.MotorAlarms", AlarmState.Normal, 0, ts)
|
||||
{
|
||||
Kind = AlarmKind.NativeOpcUa,
|
||||
IsConfiguredPlaceholder = true
|
||||
};
|
||||
|
||||
// Interleave so a leak of the dropped rows would change the observed count/order.
|
||||
manager.PublishAttributeValueChanged(attribute); // filtered at the manager (alarm-only)
|
||||
manager.PublishAlarmStateChanged(native); // → 1 proto
|
||||
manager.PublishAlarmStateChanged(placeholder); // dropped by StreamRelayActor
|
||||
manager.PublishAlarmStateChanged(computedOtherInstance); // → 1 proto
|
||||
|
||||
await WaitForConditionAsync(() => written.Count >= 2);
|
||||
// Give any leaked (attribute/placeholder) event a chance to show up before asserting.
|
||||
await Task.Delay(150);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var protos = written.ToArray();
|
||||
Assert.Equal(2, protos.Length);
|
||||
Assert.All(protos, p =>
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AlarmChanged, p.EventCase)); // no attribute leaked
|
||||
Assert.DoesNotContain(protos, p => p.AlarmChanged.IsConfiguredPlaceholder); // no placeholder leaked
|
||||
|
||||
// Wire round-trip (serialize→deserialize) then the REAL client mapping back to domain.
|
||||
var mapped = protos
|
||||
.Select(p => SiteStreamGrpcClient.ConvertToAlarmEvent(RoundTripOverWire(p)))
|
||||
.ToList();
|
||||
Assert.All(mapped, m => Assert.NotNull(m));
|
||||
|
||||
// Both instances present — one site-wide stream, no per-instance filter.
|
||||
var backA = mapped.Single(m => m!.InstanceUniqueName == InstanceA)!;
|
||||
var backB = mapped.Single(m => m!.InstanceUniqueName == InstanceB)!;
|
||||
Assert.Equal("OverSpeed", backB.AlarmName);
|
||||
|
||||
// AlarmKey identity survived end to end.
|
||||
Assert.Equal("Tank01.LevelAlarm", backA.AlarmName);
|
||||
Assert.Equal("Tank01.Level.HiHi", backA.SourceReference);
|
||||
|
||||
// Native enrichment survived every boundary (manager→relay→proto→wire→client).
|
||||
Assert.Equal(AlarmKind.NativeOpcUa, backA.Kind);
|
||||
Assert.Equal(AlarmState.Active, backA.State);
|
||||
Assert.Equal(725, backA.Priority);
|
||||
Assert.Equal(AlarmLevel.HighHigh, backA.Level);
|
||||
Assert.Equal("Tank 01 level critically high", backA.Message);
|
||||
Assert.True(backA.Condition.Active);
|
||||
Assert.True(backA.Condition.Acknowledged);
|
||||
Assert.Equal(AlarmShelveState.OneShotShelved, backA.Condition.Shelve);
|
||||
Assert.Equal(725, backA.Condition.Severity);
|
||||
Assert.Equal("AnalogLimitAlarm.HiHi", backA.AlarmTypeName);
|
||||
Assert.Equal("Process", backA.Category);
|
||||
Assert.Equal("op.jane", backA.OperatorUser);
|
||||
Assert.Equal("ack — investigating", backA.OperatorComment);
|
||||
Assert.Equal(raise, backA.OriginalRaiseTime);
|
||||
Assert.Equal("98.4", backA.CurrentValue);
|
||||
Assert.Equal("95.0", backA.LimitValue);
|
||||
Assert.Equal("Tank01.LevelAlarm", backA.NativeSourceCanonicalName);
|
||||
Assert.Equal(ts, backA.Timestamp);
|
||||
Assert.False(backA.IsConfiguredPlaceholder);
|
||||
}
|
||||
|
||||
// ── Snapshot-seed vs live-delta identity parity, through the real pipe ────────
|
||||
|
||||
[Fact]
|
||||
public async Task RealPipeMapped_SeedRow_And_LiveDelta_ForSameNativeAlarm_CollapseOntoOneCacheRow()
|
||||
{
|
||||
// Produce TWO proto events for the SAME native alarm identity through the real
|
||||
// manager→relay→proto pipe: an older one (used as the snapshot-seed row) and a
|
||||
// newer one (fed as the live delta). Because both are mapped by the SAME real
|
||||
// ConvertToAlarmEvent, their AlarmKeys are identical by construction — so the real
|
||||
// aggregator must collapse them onto ONE row (no seed/live ghost duplicate).
|
||||
var manager = new SiteStreamManager(
|
||||
new SiteRuntimeOptions { StreamBufferSize = 256 },
|
||||
NullLogger<SiteStreamManager>.Instance);
|
||||
manager.Initialize(Sys);
|
||||
var server = new SiteStreamGrpcServer(manager, NullLogger<SiteStreamGrpcServer>.Instance);
|
||||
server.SetReady(Sys);
|
||||
|
||||
var written = new ConcurrentQueue<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci => written.Enqueue(ci.Arg<SiteStreamEvent>()));
|
||||
using var cts = new CancellationTokenSource();
|
||||
var streamTask = Task.Run(() => server.SubscribeSite(
|
||||
new SiteStreamRequest { CorrelationId = "e2e-parity" }, writer, CreateMockContext(cts.Token)));
|
||||
await WaitForConditionAsync(() => manager.SubscriptionCount == 1);
|
||||
|
||||
AlarmStateChanged Native(int priority, DateTimeOffset t) =>
|
||||
new(InstanceA, "Tank01.LevelAlarm", AlarmState.Active, priority, t)
|
||||
{
|
||||
Kind = AlarmKind.NativeOpcUa,
|
||||
SourceReference = "Tank01.Level.HiHi",
|
||||
NativeSourceCanonicalName = "Tank01.LevelAlarm"
|
||||
};
|
||||
|
||||
var t0 = new DateTimeOffset(2026, 4, 2, 9, 0, 0, TimeSpan.Zero);
|
||||
manager.PublishAlarmStateChanged(Native(300, t0)); // → seed row (older)
|
||||
manager.PublishAlarmStateChanged(Native(800, t0.AddSeconds(5))); // → live delta (newer)
|
||||
await WaitForConditionAsync(() => written.Count >= 2);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var protos = written.ToArray();
|
||||
var seedRow = SiteStreamGrpcClient.ConvertToAlarmEvent(RoundTripOverWire(protos[0]))!;
|
||||
var liveRow = SiteStreamGrpcClient.ConvertToAlarmEvent(RoundTripOverWire(protos[1]))!;
|
||||
|
||||
// Drive a REAL aggregator: seed returns the older mapped row; then Tell it the newer
|
||||
// mapped delta on the live path.
|
||||
var factory = new NoopSiteStreamClientFactory();
|
||||
var sink = new PublishSink();
|
||||
var aggregator = Sys.ActorOf(Props.Create(() => new SiteAlarmAggregatorActor(
|
||||
"site-alpha", "e2e-parity", _ => Task.FromResult<IReadOnlyList<AlarmStateChanged>>(new[] { seedRow }),
|
||||
sink.Publish, factory, "http://a:5100", "http://b:5100", TimeSpan.FromMinutes(10))));
|
||||
|
||||
await WaitForConditionAsync(() => sink.Latest is { Count: 1 }); // seed applied
|
||||
aggregator.Tell(liveRow);
|
||||
await WaitForConditionAsync(() =>
|
||||
sink.Latest is { Count: 1 } l && l[0].Priority == 800); // live delta collapsed in place
|
||||
|
||||
var final = sink.Latest!;
|
||||
Assert.Single(final); // exactly ONE row for the (instance, name, sourceRef) identity
|
||||
Assert.Equal("Tank01.Level.HiHi", final[0].SourceReference);
|
||||
Assert.Equal(AlarmKind.NativeOpcUa, final[0].Kind);
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Serialize→deserialize a proto event to mimic the gRPC HTTP/2 hop, proving the
|
||||
/// enriched <c>AlarmStateUpdate</c> is fully wire-representable (no field lost in codec).
|
||||
/// </summary>
|
||||
private static SiteStreamEvent RoundTripOverWire(SiteStreamEvent evt) =>
|
||||
SiteStreamEvent.Parser.ParseFrom(evt.ToByteArray());
|
||||
|
||||
private static ServerCallContext CreateMockContext(CancellationToken cancellationToken)
|
||||
{
|
||||
var context = Substitute.For<ServerCallContext>();
|
||||
context.CancellationToken.Returns(cancellationToken);
|
||||
return context;
|
||||
}
|
||||
|
||||
private static async Task WaitForConditionAsync(Func<bool> condition, int timeoutMs = 5000)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
||||
while (!condition() && DateTime.UtcNow < deadline)
|
||||
await Task.Delay(25);
|
||||
Assert.True(condition(), $"Condition not met within {timeoutMs}ms");
|
||||
}
|
||||
|
||||
private sealed class PublishSink
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
private IReadOnlyList<AlarmStateChanged>? _latest;
|
||||
public void Publish(IReadOnlyList<AlarmStateChanged> snapshot)
|
||||
{
|
||||
lock (_lock) { _latest = snapshot; }
|
||||
}
|
||||
public IReadOnlyList<AlarmStateChanged>? Latest
|
||||
{
|
||||
get { lock (_lock) { return _latest; } }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Aggregator transport double: its live stream simply stays open until cancelled.</summary>
|
||||
private sealed class NoopSiteStreamClient : SiteStreamGrpcClient
|
||||
{
|
||||
public override Task SubscribeSiteAsync(
|
||||
string correlationId, Action<AlarmStateChanged> onAlarmEvent, Action<Exception> onError, CancellationToken ct)
|
||||
{
|
||||
var tcs = new TaskCompletionSource();
|
||||
ct.Register(() => tcs.TrySetResult());
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
public override void Unsubscribe(string correlationId) { }
|
||||
}
|
||||
|
||||
private sealed class NoopSiteStreamClientFactory : SiteStreamGrpcClientFactory
|
||||
{
|
||||
private readonly NoopSiteStreamClient _client = new();
|
||||
public NoopSiteStreamClientFactory() : base(NullLoggerFactory.Instance) { }
|
||||
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint) => _client;
|
||||
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint) => _client;
|
||||
}
|
||||
}
|
||||
@@ -103,6 +103,50 @@ public class SiteStreamManagerTests : TestKit, IDisposable
|
||||
probe2.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SubscribeSiteAlarms_ForwardsAlarmsForAllInstances_ButNotAttributes()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
_streamManager.SubscribeSiteAlarms(probe.Ref);
|
||||
|
||||
// Alarm events from two different instances — both should arrive.
|
||||
_streamManager.PublishAlarmStateChanged(new AlarmStateChanged(
|
||||
"Pump1", "HighTemp", AlarmState.Active, 1, DateTimeOffset.UtcNow));
|
||||
_streamManager.PublishAlarmStateChanged(new AlarmStateChanged(
|
||||
"Pump2", "LowFlow", AlarmState.Active, 2, DateTimeOffset.UtcNow));
|
||||
|
||||
// Attribute events must be filtered out entirely.
|
||||
_streamManager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
"Pump1", "Temperature", "Temperature", "100", "Good", DateTimeOffset.UtcNow));
|
||||
_streamManager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
"Pump3", "Flow", "Flow", "42", "Good", DateTimeOffset.UtcNow));
|
||||
|
||||
// Collect the two alarms (order across instances is not guaranteed).
|
||||
var received = new[]
|
||||
{
|
||||
probe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(3)),
|
||||
probe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(3)),
|
||||
};
|
||||
|
||||
var instances = received.Select(a => a.InstanceUniqueName).OrderBy(n => n).ToArray();
|
||||
Assert.Equal(new[] { "Pump1", "Pump2" }, instances);
|
||||
|
||||
// No attribute events (nor any further alarm) should be delivered.
|
||||
probe.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SubscribeSiteAlarms_SubscriptionRemovableViaUnsubscribe()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var id = _streamManager.SubscribeSiteAlarms(probe.Ref);
|
||||
|
||||
Assert.NotNull(id);
|
||||
Assert.Equal(1, _streamManager.SubscriptionCount);
|
||||
Assert.True(_streamManager.Unsubscribe(id));
|
||||
Assert.Equal(0, _streamManager.SubscriptionCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemoveSubscriber_RemovesAllSubscriptionsForActor()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user