fix(alarms): re-assert non-normal scripted-alarm conditions after a (re)load (#487)
A deploy that triggers a FULL address-space rebuild clears `_alarmConditions` and re-materialises every condition node fresh — inactive, acked, confirmed, unshelved. The engine reloads its persisted state and re-derives Active from the predicate, so it ends up correct; but there is no transition to report, so `LoadAsync` yields `EmissionKind.None` and `OnEngineEmission` drops it. Nothing writes the node. Live-reproduced on the docker-dev rig against the pre-fix image: an alarm that had fired and cleared but was still UNACKNOWLEDGED came back from a rebuild reporting Acknowledged with Retain=false — so it disappeared from ConditionRefresh entirely, while the engine still held Unacknowledged. An operator's outstanding alarm silently vanishes from the alarm list on deploy. `ScriptedAlarmHostActor.ReassertConditionNodes` closes it. After each load it reads the new `ScriptedAlarmEngine.GetProjections()` and sends one `AlarmStateUpdate` per alarm NOT in the Part 9 no-event position. Load-bearing properties: - Node-only. It sends `AlarmStateUpdate` and nothing else — never the `alerts` topic, never the telemetry hub. `alerts` is the historization path, so a row per alarm per deploy would append a duplicate historian/AVEVA record every time anyone deploys. This is why it reads a projection rather than re-emitting: `ScriptedAlarmProjection` carries no `EmissionKind`, so there is nothing for the emission machinery — or a future refactor — to mistake for a transition. The issue's sketch said `GetState(alarmId)` + `LoadedAlarmIds`; that alone would have clobbered the node's severity and message, so the projection also carries the definition's severity, the resolved message template, and the last-observed worst-input quality. - Only non-normal alarms. One in the no-event position already matches what the materialise built, so a never-fired alarm stays byte-identical to a deploy without this pass (including its Message, which the materialise seeds with the display name), and an all-normal deploy writes nothing at all. - Timestamp is the condition's own `LastTransitionUtc`, not the deploy instant. - No duplicate Part 9 events: `WriteAlarmCondition`'s delta-gate compares against the node's CURRENT state, so a re-assert onto a freshly materialised node is a genuine delta (fires once — correct, the rebuild reset what clients see) while one onto a surgically-preserved node is suppressed. Tests: 6 new host tests + 6 new engine tests. Falsifiability checked by disabling the call — 4 of the 6 host tests go red; the other 2 are absence guards against an over-broad fix and pass either way by design. Live gate (docker-dev, central-2, pre-fix image vs. rebuilt image): - pre-fix: condition absent from ConditionRefresh after a rebuild; - post-fix: present, Unacknowledged, Retain=True, stamped with its own LastTransitionUtc rather than the restart instant; - /alerts stayed empty across two re-asserts, with the panel proven live by a real ACTIVATED transition immediately afterwards — no duplicate history.
This commit is contained in:
Joseph Doherty
+164
-4
@@ -86,20 +86,22 @@ public sealed class ScriptedAlarmHostActorTests : RuntimeActorTestBase
|
||||
Retain: false,
|
||||
Enabled: true);
|
||||
|
||||
private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream)
|
||||
private static ScriptedAlarmEngine BuildEngine(
|
||||
DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null)
|
||||
{
|
||||
var logger = new LoggerConfiguration().CreateLogger();
|
||||
return new ScriptedAlarmEngine(upstream, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
|
||||
return new ScriptedAlarmEngine(
|
||||
upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
|
||||
}
|
||||
|
||||
/// <summary>The local node id used by the redundancy-gating tests.</summary>
|
||||
private static readonly NodeId LocalNode = new("node-A");
|
||||
|
||||
private (IActorRef Host, DependencyMuxTagUpstreamSource Upstream) Spawn(
|
||||
TestProbe publish, TestProbe mux, NodeId? localNode = null)
|
||||
TestProbe publish, TestProbe mux, NodeId? localNode = null, IAlarmStateStore? store = null)
|
||||
{
|
||||
var upstream = new DependencyMuxTagUpstreamSource();
|
||||
var engine = BuildEngine(upstream);
|
||||
var engine = BuildEngine(upstream, store);
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine, localNode));
|
||||
return (host, upstream);
|
||||
}
|
||||
@@ -794,4 +796,162 @@ public sealed class ScriptedAlarmHostActorTests : RuntimeActorTestBase
|
||||
evt.AlarmId.ShouldBe("alm-1");
|
||||
evt.TransitionKind.ShouldBe("Activated");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// #487 — post-(re)load condition-node re-assert.
|
||||
//
|
||||
// A deploy that triggers a FULL address-space rebuild re-materialises every condition node fresh
|
||||
// (inactive/acked/confirmed/unshelved). The engine reloads its persisted state and re-derives Active
|
||||
// from the predicate, so it ends up CORRECT — but a still-active alarm has no transition to report,
|
||||
// yielding EmissionKind.None, which OnEngineEmission drops. Without the re-assert the node reads
|
||||
// NORMAL to every OPC UA client until the alarm's next real transition — which, for an alarm with
|
||||
// static dependencies, may never come.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
/// <summary>A persisted condition state as it survives a restart / redeploy, seeded into the store the
|
||||
/// host's engine loads from.</summary>
|
||||
private static AlarmConditionState Persisted(
|
||||
string alarmId = "alm-1",
|
||||
AlarmEnabledState enabled = AlarmEnabledState.Enabled,
|
||||
AlarmActiveState active = AlarmActiveState.Active,
|
||||
AlarmAckedState acked = AlarmAckedState.Unacknowledged,
|
||||
AlarmConfirmedState confirmed = AlarmConfirmedState.Confirmed,
|
||||
ShelvingKind shelving = ShelvingKind.Unshelved,
|
||||
DateTime? lastTransitionUtc = null) =>
|
||||
new(alarmId,
|
||||
enabled,
|
||||
active,
|
||||
acked,
|
||||
confirmed,
|
||||
shelving == ShelvingKind.Unshelved ? ShelvingState.Unshelved : new ShelvingState(shelving, null),
|
||||
lastTransitionUtc ?? DateTime.UtcNow,
|
||||
LastActiveUtc: active == AlarmActiveState.Active ? lastTransitionUtc ?? DateTime.UtcNow : null,
|
||||
LastClearedUtc: null,
|
||||
LastAckUtc: null, LastAckUser: null, LastAckComment: null,
|
||||
LastConfirmUtc: null, LastConfirmUser: null, LastConfirmComment: null,
|
||||
Comments: []);
|
||||
|
||||
/// <summary>Seed <paramref name="state"/> into a fresh store and hand it back for
|
||||
/// <see cref="Spawn"/>. The engine reads it during LoadAsync, exactly as it would after a restart.</summary>
|
||||
private static IAlarmStateStore StoreWith(AlarmConditionState state)
|
||||
{
|
||||
var store = new InMemoryAlarmStateStore();
|
||||
store.SaveAsync(state, CancellationToken.None).GetAwaiter().GetResult();
|
||||
return store;
|
||||
}
|
||||
|
||||
/// <summary>#487, the regression: an alarm whose persisted state is ACTIVE emits nothing on reload
|
||||
/// (no transition to report), but the host must still write its state onto the freshly materialised
|
||||
/// condition node — otherwise an active alarm silently reads normal after a full-rebuild deploy.</summary>
|
||||
[Fact]
|
||||
public void Reload_reasserts_an_active_alarm_onto_its_condition_node()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var (host, _) = Spawn(publish, mux, store: StoreWith(Persisted()));
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan(severity: 800) }));
|
||||
|
||||
var state = publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
|
||||
state.AlarmNodeId.ShouldBe("alm-1");
|
||||
state.Realm.ShouldBe(AddressSpaceRealm.Uns);
|
||||
state.State.Active.ShouldBeTrue("the persisted state was Active — the rebuilt node must say so");
|
||||
state.State.Acknowledged.ShouldBeFalse();
|
||||
state.State.Enabled.ShouldBeTrue();
|
||||
// The projection carries the definition's severity + resolved message, so a re-asserted node is
|
||||
// indistinguishable from one the alarm's own transition would have written.
|
||||
state.State.Severity.ShouldBe((ushort)1000); // 800 → Critical bucket → 1000
|
||||
state.State.Message.ShouldBe("condition");
|
||||
}
|
||||
|
||||
/// <summary>The re-assert is node-ONLY: it must never reach the <c>alerts</c> topic. That topic is the
|
||||
/// historization path, so an alerts row per alarm per deploy would append a duplicate historian /
|
||||
/// AVEVA record every time anyone deploys.</summary>
|
||||
[Fact]
|
||||
public void Reassert_never_publishes_to_the_alerts_topic()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var alerts = CreateTestProbe();
|
||||
SubscribeToAlerts(alerts);
|
||||
|
||||
var (host, _) = Spawn(publish, mux, store: StoreWith(Persisted()));
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
|
||||
// Wait for the re-assert to actually happen FIRST — otherwise the absence window below could
|
||||
// elapse before the load even completed and would prove nothing.
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
|
||||
|
||||
// Absence assertion: the elapsed time IS the assertion, so this stays deliberately short.
|
||||
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>An alarm sitting in the Part 9 no-event position already matches what the materialise
|
||||
/// built, so it is NOT re-asserted — a deploy where nothing is in alarm writes no condition nodes at
|
||||
/// all, and a never-fired alarm keeps the display name the materialise seeded as its Message.</summary>
|
||||
[Fact]
|
||||
public void A_normal_alarm_is_not_reasserted()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var (host, _) = Spawn(publish, mux); // empty store ⇒ Fresh ⇒ no-event position
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
|
||||
// Sequence the absence: the re-assert (if any) happens right after the mux registration, so
|
||||
// waiting for that first makes the window below cover the moment the write would have landed.
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Not just Active: every field a rebuild would silently lose is restored. A shelved (but
|
||||
/// inactive) alarm is re-asserted too — a rebuilt node would otherwise come back unshelved, so an
|
||||
/// operator's suppression would evaporate on the next deploy.</summary>
|
||||
[Fact]
|
||||
public void Reload_reasserts_a_shelved_but_inactive_alarm()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var (host, _) = Spawn(publish, mux, store: StoreWith(Persisted(
|
||||
active: AlarmActiveState.Inactive,
|
||||
acked: AlarmAckedState.Acknowledged,
|
||||
shelving: ShelvingKind.OneShot)));
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
|
||||
var state = publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
|
||||
state.State.Active.ShouldBeFalse();
|
||||
state.State.Shelving.ShouldBe(AlarmShelvingKind.OneShot);
|
||||
}
|
||||
|
||||
/// <summary>The re-asserted timestamp is when the state actually came about, NOT the deploy instant —
|
||||
/// a client reading Time/ReceiveTime after a rebuild should still see when the alarm went active.</summary>
|
||||
[Fact]
|
||||
public void Reassert_carries_the_conditions_own_transition_timestamp()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var wentActiveAt = DateTime.UtcNow.AddHours(-3);
|
||||
var (host, _) = Spawn(publish, mux, store: StoreWith(Persisted(lastTransitionUtc: wentActiveAt)));
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout)
|
||||
.TimestampUtc.ShouldBe(wentActiveAt);
|
||||
}
|
||||
|
||||
/// <summary>A disabled PLAN exposes no condition node at all (MaterialiseScriptedAlarms skips it) and
|
||||
/// is not loaded into the engine — so it must not be re-asserted either, or the write would target a
|
||||
/// node that does not exist.</summary>
|
||||
[Fact]
|
||||
public void A_disabled_plan_is_not_reasserted()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var (host, _) = Spawn(publish, mux, store: StoreWith(Persisted()));
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan(enabled: false) }));
|
||||
|
||||
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user