Joseph Doherty
9a87d7a328
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.
Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.
Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.
Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.
Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.
Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.
Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
674 lines
34 KiB
C#
674 lines
34 KiB
C#
using System.Text.Json;
|
||
using Akka.Actor;
|
||
using Akka.TestKit.Xunit2;
|
||
using Microsoft.Extensions.Logging.Abstractions;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
|
||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
|
||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||
|
||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||
|
||
/// <summary>
|
||
/// Task 15: NativeAlarmActor mirrors a source's native alarms — subscribe on start,
|
||
/// emit enriched AlarmStateChanged, snapshot atomic swap, retention, persistence.
|
||
/// </summary>
|
||
public class NativeAlarmActorTests : TestKit, IDisposable
|
||
{
|
||
private readonly TestLocalDb _localDb;
|
||
private readonly SiteStorageService _storage;
|
||
private readonly SiteRuntimeOptions _options = new();
|
||
|
||
public NativeAlarmActorTests()
|
||
{
|
||
// SiteStorageService takes an ILocalDb now; LocalDb has no in-memory mode, so the
|
||
// fixture is a real temp file (deleted in Dispose, after the TestKit shutdown).
|
||
_localDb = TestLocalDb.CreateTemp("naa");
|
||
_storage = new SiteStorageService(_localDb.Db, NullLogger<SiteStorageService>.Instance);
|
||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||
}
|
||
|
||
private static ResolvedNativeAlarmSource Source() => new()
|
||
{
|
||
CanonicalName = "Pressure",
|
||
ConnectionName = "Opc",
|
||
SourceReference = "ns=2;s=T01"
|
||
};
|
||
|
||
private static NativeAlarmTransition Transition(
|
||
string sourceRef, AlarmTransitionKind kind, AlarmConditionState condition, DateTimeOffset? time = null) =>
|
||
new(sourceRef, "T01", "AnalogLimit.Hi", kind, condition,
|
||
"Process", "hi", "hi", "", "", null, time ?? DateTimeOffset.UtcNow, "92", "90");
|
||
|
||
private IActorRef Spawn(IActorRef instanceActor, IActorRef dclManager, IServiceProvider? serviceProvider = null) =>
|
||
Spawn(instanceActor, dclManager, _options, serviceProvider);
|
||
|
||
private IActorRef Spawn(IActorRef instanceActor, IActorRef dclManager, SiteRuntimeOptions options,
|
||
IServiceProvider? serviceProvider = null) =>
|
||
ActorOf(Props.Create(() => new NativeAlarmActor(
|
||
Source(), "inst", instanceActor, dclManager, _storage, options,
|
||
NullLogger<NativeAlarmActor>.Instance, AlarmKind.NativeOpcUa, serviceProvider)));
|
||
|
||
[Fact]
|
||
public void Stop_ReleasesAlarmSubscriptionWithOwnerIdentity()
|
||
{
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(CreateTestProbe().Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
Sys.Stop(actor);
|
||
var request = dcl.ExpectMsg<UnsubscribeAlarmsRequest>();
|
||
Assert.Equal(Source().SourceReference, request.SourceReference);
|
||
Assert.Equal(actor, dcl.LastSender);
|
||
}
|
||
|
||
[Fact]
|
||
public void DuplicateSnapshotEnd_DoesNotClearCommittedAlarm()
|
||
{
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
var epoch = Guid.NewGuid();
|
||
NativeAlarmTransition Framed(AlarmTransitionKind kind)
|
||
{
|
||
var transition = Transition("T01.Hi", kind,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800));
|
||
var json = JsonSerializer.SerializeToNode(transition)!;
|
||
json["SnapshotId"] = JsonSerializer.SerializeToNode(epoch);
|
||
json["SnapshotSourceReference"] = Source().SourceReference;
|
||
return json.Deserialize<NativeAlarmTransition>()!;
|
||
}
|
||
// Append-only enum value for the new start marker; before the fix it is not handled.
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Framed((AlarmTransitionKind)7)));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Framed(AlarmTransitionKind.Snapshot)));
|
||
var end = new NativeAlarmTransitionUpdate("Opc", Framed(AlarmTransitionKind.SnapshotComplete));
|
||
actor.Tell(end);
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
// Before the fix there is also an unwanted Active event for the unrecognized start.
|
||
// Fish out all initial active events before applying the duplicate marker.
|
||
actor.Tell(new SubscribeAlarmsResponse("barrier", "inst", true, null, DateTimeOffset.UtcNow));
|
||
instance.ReceiveWhile(TimeSpan.FromMilliseconds(100), msg => msg);
|
||
actor.Tell(end);
|
||
instance.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||
}
|
||
|
||
[Fact]
|
||
public void SubscribeOnStart_SendsRequestForSourceBinding()
|
||
{
|
||
var dcl = CreateTestProbe();
|
||
Spawn(CreateTestProbe().Ref, dcl.Ref);
|
||
|
||
var req = dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
Assert.Equal("inst", req.InstanceUniqueName);
|
||
Assert.Equal("Opc", req.ConnectionName);
|
||
Assert.Equal("ns=2;s=T01", req.SourceReference);
|
||
}
|
||
|
||
[Fact]
|
||
public void Raise_EmitsEnrichedAlarmStateChanged()
|
||
{
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
|
||
var emitted = instance.ExpectMsg<AlarmStateChanged>();
|
||
Assert.Equal(AlarmKind.NativeOpcUa, emitted.Kind);
|
||
Assert.Equal("T01.Hi", emitted.SourceReference);
|
||
Assert.Equal(AlarmState.Active, emitted.State);
|
||
Assert.Equal(800, emitted.Condition.Severity);
|
||
}
|
||
|
||
[Fact]
|
||
public void Raise_StampsNativeSourceCanonicalName_FromConfiguredBinding()
|
||
{
|
||
// DV-1: every emitted condition carries the canonical name of the source
|
||
// BINDING it belongs to, so the DebugView can nest live native conditions
|
||
// under their configured binding node. It must equal the binding's
|
||
// CanonicalName used to construct the actor (Source().CanonicalName).
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
|
||
var emitted = instance.ExpectMsg<AlarmStateChanged>();
|
||
Assert.Equal(Source().CanonicalName, emitted.NativeSourceCanonicalName);
|
||
Assert.Equal("Pressure", emitted.NativeSourceCanonicalName);
|
||
}
|
||
|
||
[Fact]
|
||
public void SnapshotComplete_WithMissingPriorAlarm_EmitsReturnToNormal()
|
||
{
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
// A fresh snapshot that no longer contains T01.Hi means it cleared at the source.
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.SnapshotComplete,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
|
||
var cleared = instance.ExpectMsg<AlarmStateChanged>();
|
||
Assert.Equal("T01.Hi", cleared.SourceReference);
|
||
Assert.Equal(AlarmState.Normal, cleared.State);
|
||
}
|
||
|
||
[Fact]
|
||
public void OlderTransition_IsIgnored()
|
||
{
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var t0 = DateTimeOffset.UtcNow;
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), t0)));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
// An out-of-order (older) transition must not overwrite newer state.
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Clear,
|
||
new AlarmConditionState(false, true, null, AlarmShelveState.Unshelved, false, 0), t0.AddSeconds(-30))));
|
||
|
||
instance.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||
}
|
||
|
||
// ── PLAN-03 Task 20 (P4): coalesced batched persistence ─────────────────
|
||
|
||
private IActorRef SpawnWithFlush(IActorRef instanceActor, IActorRef dclManager, TimeSpan flushInterval) =>
|
||
ActorOf(Props.Create(() => new NativeAlarmActor(
|
||
Source(), "inst", instanceActor, dclManager, _storage, _options,
|
||
NullLogger<NativeAlarmActor>.Instance, AlarmKind.NativeOpcUa, null, flushInterval)));
|
||
|
||
private static void DeliverLiveTransition(IActorRef actor, string sourceRef, int severity, DateTimeOffset time) =>
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
sourceRef, AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, severity), time)));
|
||
|
||
[Fact]
|
||
public async Task AlarmStorm_CoalescesPersistence_LatestPerSourceRefWins()
|
||
{
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = SpawnWithFlush(instance.Ref, dcl.Ref, TimeSpan.FromMilliseconds(100));
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
// 50 live transitions across 5 source refs (strictly increasing time per ref so none
|
||
// is dropped as stale) — the coalesced flush must collapse them to one row per ref.
|
||
var baseTime = DateTimeOffset.UtcNow;
|
||
for (var i = 0; i < 50; i++)
|
||
DeliverLiveTransition(actor, $"ref-{i % 5}", severity: i, baseTime.AddMilliseconds(i));
|
||
|
||
await Task.Delay(500); // > flush interval
|
||
|
||
var rows = await _storage.GetNativeAlarmsAsync("inst", Source().CanonicalName);
|
||
Assert.Equal(5, rows.Count);
|
||
// The LAST write for ref-4 carries severity 49 (i = 49, 49 % 5 == 4).
|
||
Assert.Contains(rows, r => r.ConditionJson.Contains("49"));
|
||
}
|
||
|
||
// ── PLAN-03 Task 21 (UA4): rehydration restores display metadata ────────
|
||
|
||
[Fact]
|
||
public async Task Rehydration_RestoresDisplayMetadata()
|
||
{
|
||
var instance1 = CreateTestProbe();
|
||
var dcl1 = CreateTestProbe();
|
||
var actor1 = SpawnWithFlush(instance1.Ref, dcl1.Ref, TimeSpan.FromMilliseconds(100));
|
||
dcl1.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
// A live raise carrying full display metadata.
|
||
actor1.Tell(new NativeAlarmTransitionUpdate("Opc", new NativeAlarmTransition(
|
||
"ref-1", "T01", "HighLevelAlarm", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800),
|
||
"Process", "desc", "Tank overflow", "", "", null, DateTimeOffset.UtcNow, "92", "90")));
|
||
instance1.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
// Wait for the coalesced flush to land the metadata in SQLite.
|
||
await AwaitAssertAsync(async () =>
|
||
{
|
||
var rows = await _storage.GetNativeAlarmsAsync("inst", Source().CanonicalName);
|
||
Assert.Contains(rows, r => r.MetadataJson != null && r.MetadataJson.Contains("HighLevelAlarm"));
|
||
}, TimeSpan.FromSeconds(3));
|
||
|
||
// A fresh actor on the SAME storage rehydrates in PreStart and re-emits the condition
|
||
// with the restored metadata instead of empty strings.
|
||
var instance2 = CreateTestProbe();
|
||
var dcl2 = CreateTestProbe();
|
||
SpawnWithFlush(instance2.Ref, dcl2.Ref, TimeSpan.FromMilliseconds(100));
|
||
|
||
var emitted = instance2.FishForMessage<AlarmStateChanged>(m => m.SourceReference == "ref-1", TimeSpan.FromSeconds(5));
|
||
Assert.Equal("HighLevelAlarm", emitted.AlarmTypeName);
|
||
Assert.Equal("Process", emitted.Category);
|
||
Assert.Equal("Tank overflow", emitted.Message);
|
||
}
|
||
|
||
// ── M1.5: site event log `alarm` category ──────────────────────────────
|
||
|
||
[Fact]
|
||
public void Raise_EmitsAlarmSiteEvent()
|
||
{
|
||
var siteLog = new FakeSiteEventLogger();
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref, new SingleServiceProvider(siteLog));
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
AwaitAssert(() =>
|
||
{
|
||
var rows = siteLog.OfType("alarm");
|
||
Assert.Single(rows);
|
||
var row = rows[0];
|
||
Assert.Equal("Error", row.Severity); // severity 800 → Error
|
||
Assert.Equal("inst", row.InstanceId);
|
||
Assert.Equal("NativeAlarmActor:Pressure", row.Source);
|
||
}, TimeSpan.FromSeconds(2));
|
||
}
|
||
|
||
[Fact]
|
||
public void Clear_EmitsInfoAlarmSiteEvent()
|
||
{
|
||
var siteLog = new FakeSiteEventLogger();
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref, new SingleServiceProvider(siteLog));
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var t0 = DateTimeOffset.UtcNow;
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), t0)));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
// Clear (inactive but not yet acked → stays mirrored, return-to-normal emit).
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Clear,
|
||
new AlarmConditionState(false, false, null, AlarmShelveState.Unshelved, false, 0), t0.AddSeconds(5))));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Normal);
|
||
|
||
AwaitAssert(() =>
|
||
{
|
||
var rows = siteLog.OfType("alarm");
|
||
Assert.Equal(2, rows.Count); // raise + clear
|
||
Assert.Equal("Error", rows[0].Severity);
|
||
Assert.Equal("Info", rows[1].Severity); // return-to-normal → Info
|
||
}, TimeSpan.FromSeconds(2));
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Rehydration_DoesNotEmitSiteEvent()
|
||
{
|
||
// Pre-populate SQLite with an active condition so the actor rehydrates
|
||
// it on PreStart. Rehydration replays last-known state — it is NOT a
|
||
// live transition, so it must surface upward (for the DebugView) but
|
||
// must NOT re-log an `alarm` operational event.
|
||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||
await _storage.UpsertNativeAlarmAsync(
|
||
"inst", "Pressure", "T01.Hi",
|
||
JsonSerializer.Serialize(condition), DateTimeOffset.UtcNow);
|
||
|
||
var siteLog = new FakeSiteEventLogger();
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
Spawn(instance.Ref, dcl.Ref, new SingleServiceProvider(siteLog));
|
||
|
||
// The rehydrated condition is surfaced upward...
|
||
var emitted = instance.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(2));
|
||
Assert.Equal("T01.Hi", emitted.SourceReference);
|
||
Assert.Equal(AlarmState.Active, emitted.State);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
// ...but no `alarm` operational event is logged for it.
|
||
AwaitAssert(
|
||
() => Assert.Empty(siteLog.OfType("alarm")),
|
||
TimeSpan.FromSeconds(1));
|
||
}
|
||
|
||
[Fact]
|
||
public void SnapshotSwap_ExistingActiveCondition_DoesNotReEmit()
|
||
{
|
||
var siteLog = new FakeSiteEventLogger();
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref, new SingleServiceProvider(siteLog));
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
// Live raise — the one and only `alarm` event we expect.
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
AwaitAssert(() => Assert.Single(siteLog.OfType("alarm")), TimeSpan.FromSeconds(2));
|
||
|
||
// A reconnect snapshot that RE-INCLUDES the same still-active condition is
|
||
// a re-sync, not a live transition. It must NOT re-log a second `alarm`
|
||
// event (regression for the spurious-reconnect-event bug).
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Snapshot,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.SnapshotComplete,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800))));
|
||
|
||
// The snapshot still surfaces the condition upward (DebugView re-sync)...
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.SourceReference == "T01.Hi" && m.State == AlarmState.Active);
|
||
|
||
// ...but the `alarm` event count stays at exactly 1 — no re-emit.
|
||
Thread.Sleep(200); // give any spurious fire-and-forget log time to land
|
||
Assert.Single(siteLog.OfType("alarm"));
|
||
}
|
||
|
||
[Fact]
|
||
public void Acknowledge_EmitsInfoAlarmSiteEventMentioningAcknowledged()
|
||
{
|
||
var siteLog = new FakeSiteEventLogger();
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref, new SingleServiceProvider(siteLog));
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var t0 = DateTimeOffset.UtcNow;
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), t0)));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
// Operator acknowledges the still-active condition. The Acknowledge
|
||
// branch of LogAlarmEvent logs Info and mentions "acknowledged".
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Acknowledge,
|
||
new AlarmConditionState(true, true, null, AlarmShelveState.Unshelved, false, 800), t0.AddSeconds(5))));
|
||
instance.ExpectMsg<AlarmStateChanged>();
|
||
|
||
AwaitAssert(() =>
|
||
{
|
||
var rows = siteLog.OfType("alarm");
|
||
Assert.Equal(2, rows.Count); // raise + acknowledge
|
||
var ack = rows[1];
|
||
Assert.Equal("Info", ack.Severity);
|
||
Assert.Contains("acknowledged", ack.Message, StringComparison.OrdinalIgnoreCase);
|
||
}, TimeSpan.FromSeconds(2));
|
||
}
|
||
|
||
// ── SiteRuntime-028: cap eviction emits a return-to-normal for an active drop ──
|
||
|
||
[Fact]
|
||
public void EnforceCap_EvictsActiveOldestCondition_EmitsReturnToNormalAndDropSignal()
|
||
{
|
||
// SiteRuntime-028: a cap eviction that drops a still-Active condition without a
|
||
// return-to-normal leaves the Instance Actor (and central's stream / the
|
||
// operator Alarm Summary) showing a phantom Active alarm forever. With cap=1,
|
||
// raising a second condition evicts the oldest (still Active) one — which must
|
||
// produce a Normal AlarmStateChanged for the evicted SourceReference, plus the
|
||
// SiteRuntime-027 NativeAlarmDropped so the parent evicts its stale key.
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var options = new SiteRuntimeOptions { MirroredAlarmCapPerSource = 1 };
|
||
var actor = Spawn(instance.Ref, dcl.Ref, options);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var t0 = DateTimeOffset.UtcNow;
|
||
// Oldest active condition (will be evicted when the cap is exceeded).
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"OLD.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), t0)));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.SourceReference == "OLD.Hi" && m.State == AlarmState.Active);
|
||
|
||
// Newer active condition pushes the set to 2 > cap(1) → OLD.Hi is evicted.
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"NEW.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), t0.AddSeconds(5))));
|
||
|
||
// The new condition is emitted active...
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.SourceReference == "NEW.Hi" && m.State == AlarmState.Active);
|
||
// ...and the evicted oldest condition must be cleared (return-to-normal), not
|
||
// left phantom-active.
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.SourceReference == "OLD.Hi" && m.State == AlarmState.Normal);
|
||
// ...and the parent is told to evict the stale _latestAlarmEvents key.
|
||
instance.ExpectMsg<NativeAlarmDropped>(m => m.SourceReference == "OLD.Hi");
|
||
}
|
||
|
||
// ── SiteRuntime-027: terminal retention drop signals the parent to evict its key ──
|
||
|
||
[Fact]
|
||
public void RetentionDrop_ResolvedCondition_EmitsReturnToNormalThenDropSignal()
|
||
{
|
||
// SiteRuntime-027: when a native condition resolves (inactive AND acknowledged)
|
||
// it drops out of the mirror. The Instance Actor must be told (NativeAlarmDropped)
|
||
// so its _latestAlarmEvents map does not retain a stale (Normal) entry forever —
|
||
// otherwise a source that mints a fresh SourceReference per occurrence leaks one
|
||
// entry per condition the instance has ever seen.
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var t0 = DateTimeOffset.UtcNow;
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), t0)));
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.State == AlarmState.Active);
|
||
|
||
// Resolved: inactive AND acknowledged → return-to-normal emitted, then dropped.
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Clear,
|
||
new AlarmConditionState(false, true, null, AlarmShelveState.Unshelved, false, 0), t0.AddSeconds(5))));
|
||
|
||
instance.ExpectMsg<AlarmStateChanged>(m => m.SourceReference == "T01.Hi" && m.State == AlarmState.Normal);
|
||
instance.ExpectMsg<NativeAlarmDropped>(m => m.SourceReference == "T01.Hi");
|
||
}
|
||
|
||
void IDisposable.Dispose()
|
||
{
|
||
// Shut the actor system down FIRST: in-flight alarm actors still hold the
|
||
// ILocalDb, and their coalesced flush would hit a disposed database otherwise.
|
||
Shutdown();
|
||
// Then dispose — the master connection anchors the WAL, so the sidecars cannot
|
||
// be removed while it is open.
|
||
var path = _localDb.Path;
|
||
_localDb.Dispose();
|
||
TestLocalDb.DeleteFiles(path);
|
||
}
|
||
|
||
// ── MES alarm-status API §6.4: AckTime through the mirror ──────────────
|
||
|
||
[Fact]
|
||
public void Emit_CarriesTheAdaptersAckTimeVerbatim()
|
||
{
|
||
// The DCL adapter already decided whether an ack instant applies; the mirror must
|
||
// neither invent one for an unacked raise nor drop the one on an ack transition.
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(instance.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var raisedAt = DateTimeOffset.UtcNow;
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition(
|
||
"T01.Hi", AlarmTransitionKind.Raise,
|
||
new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800), raisedAt)));
|
||
Assert.Null(instance.ExpectMsg<AlarmStateChanged>().AckTime);
|
||
|
||
var ackedAt = raisedAt.AddMinutes(2);
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", new NativeAlarmTransition(
|
||
"T01.Hi", "T01", "AnalogLimit.Hi", AlarmTransitionKind.Acknowledge,
|
||
new AlarmConditionState(true, true, null, AlarmShelveState.Unshelved, false, 800),
|
||
"Process", "hi", "hi", "op1", "ack", null, ackedAt, "92", "90", AckTime: ackedAt)));
|
||
|
||
var acked = instance.ExpectMsg<AlarmStateChanged>();
|
||
Assert.Equal(ackedAt, acked.AckTime);
|
||
Assert.True(acked.Condition.Acknowledged);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Rehydration_RestoresAckTime_SoItSurvivesFailover()
|
||
{
|
||
// A failover must not make an acknowledged alarm look freshly unacknowledged: the
|
||
// ack instant rides metadata_json (see NativeAlarmMetadata) and comes back on the
|
||
// rehydration emit of a fresh actor over the SAME site database.
|
||
var instance1 = CreateTestProbe();
|
||
var dcl1 = CreateTestProbe();
|
||
var actor1 = SpawnWithFlush(instance1.Ref, dcl1.Ref, TimeSpan.FromMilliseconds(100));
|
||
dcl1.ExpectMsg<SubscribeAlarmsRequest>();
|
||
|
||
var ackedAt = new DateTimeOffset(2026, 8, 1, 12, 0, 0, TimeSpan.Zero);
|
||
actor1.Tell(new NativeAlarmTransitionUpdate("Opc", new NativeAlarmTransition(
|
||
"ref-ack", "T01", "HighLevelAlarm", AlarmTransitionKind.Acknowledge,
|
||
new AlarmConditionState(true, true, null, AlarmShelveState.Unshelved, false, 800),
|
||
"Process", "desc", "Tank overflow", "op1", "ack", null, DateTimeOffset.UtcNow, "92", "90",
|
||
AckTime: ackedAt)));
|
||
instance1.ExpectMsg<AlarmStateChanged>(m => m.AckTime == ackedAt);
|
||
|
||
await AwaitAssertAsync(async () =>
|
||
{
|
||
var rows = await _storage.GetNativeAlarmsAsync("inst", Source().CanonicalName);
|
||
Assert.Contains(rows, r => r.MetadataJson != null && r.MetadataJson.Contains("AckTime"));
|
||
}, TimeSpan.FromSeconds(3));
|
||
|
||
var instance2 = CreateTestProbe();
|
||
var dcl2 = CreateTestProbe();
|
||
SpawnWithFlush(instance2.Ref, dcl2.Ref, TimeSpan.FromMilliseconds(100));
|
||
|
||
var emitted = instance2.FishForMessage<AlarmStateChanged>(
|
||
m => m.SourceReference == "ref-ack", TimeSpan.FromSeconds(5));
|
||
Assert.Equal(ackedAt, emitted.AckTime);
|
||
}
|
||
|
||
[Fact]
|
||
public async Task Rehydration_OfAPreAckTimeRow_LeavesAckTimeNull()
|
||
{
|
||
// Rows written before this change carry metadata_json without an AckTime property.
|
||
// System.Text.Json leaves the missing property at null — "ack time unknown" — rather
|
||
// than failing the rehydration and discarding the condition.
|
||
var legacyMetadata = JsonSerializer.Serialize(new
|
||
{
|
||
AlarmTypeName = "HighLevelAlarm",
|
||
Category = "Process",
|
||
Message = "Tank overflow",
|
||
CurrentValue = "92",
|
||
LimitValue = "90"
|
||
});
|
||
await _storage.UpsertNativeAlarmAsync(
|
||
"inst", Source().CanonicalName, "ref-legacy",
|
||
JsonSerializer.Serialize(
|
||
new AlarmConditionState(true, true, null, AlarmShelveState.Unshelved, false, 800)),
|
||
DateTimeOffset.UtcNow, legacyMetadata);
|
||
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
Spawn(instance.Ref, dcl.Ref);
|
||
|
||
var emitted = instance.FishForMessage<AlarmStateChanged>(
|
||
m => m.SourceReference == "ref-legacy", TimeSpan.FromSeconds(5));
|
||
Assert.Equal("HighLevelAlarm", emitted.AlarmTypeName); // metadata still restored…
|
||
Assert.Null(emitted.AckTime); // …with no ack time invented
|
||
}
|
||
|
||
[Fact]
|
||
public void LostSubscribeResponse_ResendsSubscribe()
|
||
{
|
||
// S4: a LOST subscribe response (no reply at all) must be re-sent, not only
|
||
// a failure reply. The response-timeout arm fires at 2× the retry interval.
|
||
var options = new SiteRuntimeOptions { NativeAlarmRetryIntervalMs = 200 };
|
||
var dcl = CreateTestProbe();
|
||
Spawn(CreateTestProbe().Ref, dcl.Ref, options);
|
||
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
// Swallow the response entirely — the actor must still re-subscribe.
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>(TimeSpan.FromSeconds(5));
|
||
}
|
||
[Theory]
|
||
[InlineData(false)]
|
||
[InlineData(true)]
|
||
public async Task SR12_LateRehydrationCannotResurrectResolvedOrSnapshotRemovedAlarm(bool snapshot)
|
||
{
|
||
var instance = CreateTestProbe();
|
||
var dcl = CreateTestProbe();
|
||
var read = new TaskCompletionSource<IReadOnlyList<NativeAlarmRow>>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||
var actor = ActorOf(Props.Create(() => new NativeAlarmActor(Source(), "inst", instance.Ref, dcl.Ref, _storage, _options,
|
||
NullLogger.Instance, AlarmKind.NativeOpcUa, null, null, read.Task)));
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||
await _storage.UpsertNativeAlarmAsync("saved", "Pressure", "T01.Hi", JsonSerializer.Serialize(condition), DateTimeOffset.UtcNow);
|
||
var rows = await _storage.GetNativeAlarmsAsync("saved", "Pressure");
|
||
if (snapshot)
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("", AlarmTransitionKind.SnapshotComplete, condition)));
|
||
else
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("T01.Hi", AlarmTransitionKind.Clear, condition with { Active = false, Acknowledged = true })));
|
||
read.SetResult(rows);
|
||
var messages = instance.ReceiveWhile<object>(o => o, TimeSpan.FromMilliseconds(300));
|
||
Assert.DoesNotContain(messages.OfType<AlarmStateChanged>(), e => e.State == AlarmState.Active);
|
||
}
|
||
|
||
[Fact]
|
||
public void SR3_StaleSuccessCannotCancelCurrentSubscribeRetry()
|
||
{
|
||
var dcl = CreateTestProbe();
|
||
var actor = Spawn(CreateTestProbe().Ref, dcl.Ref, new SiteRuntimeOptions { NativeAlarmRetryIntervalMs = 100 });
|
||
var first = dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
var second = dcl.ExpectMsg<SubscribeAlarmsRequest>(TimeSpan.FromSeconds(2));
|
||
Assert.NotEqual(first.CorrelationId, second.CorrelationId);
|
||
actor.Tell(new SubscribeAlarmsResponse(first.CorrelationId, "inst", true, null, DateTimeOffset.UtcNow));
|
||
var third = dcl.ExpectMsg<SubscribeAlarmsRequest>(TimeSpan.FromSeconds(3));
|
||
actor.Tell(new SubscribeAlarmsResponse(third.CorrelationId, "inst", true, null, DateTimeOffset.UtcNow));
|
||
dcl.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||
}
|
||
|
||
[Fact]
|
||
public void DC5_ConnectionWideRestartDiscardsInterruptedMxSnapshot()
|
||
{
|
||
var parent = CreateTestProbe(); var dcl = CreateTestProbe();
|
||
var actor = Spawn(parent.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("Old.Hi", AlarmTransitionKind.Snapshot, condition)));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("", AlarmTransitionKind.SnapshotStart, condition)));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", Transition("", AlarmTransitionKind.SnapshotComplete, condition)));
|
||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||
}
|
||
|
||
[Fact]
|
||
public void DC5_LiveClearDuringSnapshotCannotBeResurrectedBySnapshotEnd()
|
||
{
|
||
var parent = CreateTestProbe(); var dcl = CreateTestProbe();
|
||
var actor = Spawn(parent.Ref, dcl.Ref);
|
||
dcl.ExpectMsg<SubscribeAlarmsRequest>();
|
||
var condition = new AlarmConditionState(true, false, null, AlarmShelveState.Unshelved, false, 800);
|
||
var snapshot = Transition("A.Hi", AlarmTransitionKind.Snapshot, condition) with { SnapshotId = Guid.NewGuid(), SnapshotSourceReference = Source().SourceReference };
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot with { Kind = AlarmTransitionKind.SnapshotStart }));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot with { Kind = AlarmTransitionKind.Clear, Condition = condition with { Active = false, Acknowledged = true }, TransitionTime = snapshot.TransitionTime.AddSeconds(1) }));
|
||
actor.Tell(new NativeAlarmTransitionUpdate("Opc", snapshot with { Kind = AlarmTransitionKind.SnapshotComplete }));
|
||
var messages = parent.ReceiveWhile<object>(o => o, TimeSpan.FromMilliseconds(300));
|
||
Assert.DoesNotContain(messages.OfType<AlarmStateChanged>(), m => m.State == AlarmState.Active);
|
||
}
|
||
|
||
}
|