From f54922f94c5b9e947d1ba09450a9427e184bc1cd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 13:29:15 -0500 Subject: [PATCH] fix(alarms): revert a native ack that reaches no driver; refuse native acks while the host is Stale (lmxopcua#765) Residuals of #760: 1. An ack routed to a disabled / not-yet-spawned driver, or to a stubbed child (driver does not run on this node, or its factory rejected the config), was dropped and left the condition Acknowledged. DriverHostActor now reverts it through the same NativeAlarmProjector.TryRevertAcknowledge path as a refused ack. 2. A native ack arriving while the host is Stale was dropped before the projector recorded it, so it could never be reverted. Stale now marks the whole NativeAlarmAckPreflightView unavailable (SetHostUnavailable), so the node manager refuses every native ack synchronously; BecomeSteady clears it. 3. (keyed sessions without a Galaxy identity) is not addressed. Tests: DriverHostActorNativeAlarmTests' client-ack test and the FOCAS respawn theory now start the host's own driver (AckingAlarmDriver; a factory building a real FocasDriver per spawn over per-instance fake CNCs, with a real config-change respawn) - both fail against the fix in their old shape. New: ack to a disabled driver reverts, ack to a stubbed driver reverts, a Stale host refuses through the pre-check view - all three fail with the fix mutated out. Runtime.Tests 943/0/3. Docs: AlarmTracking.md, .claude/rules/alarms.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- .claude/rules/alarms.md | 9 +- docs/AlarmTracking.md | 8 +- .../Drivers/DriverHostActor.cs | 24 +++- .../Drivers/NativeAlarmAckPreflightView.cs | 12 ++ .../Drivers/AckingAlarmDriver.cs | 99 +++++++++++++ ...iverHostActorNativeAlarmAckRoutingTests.cs | 80 ++++++++++- .../DriverHostActorNativeAlarmTests.cs | 31 ++-- .../Drivers/FocasNativeAlarmEndToEndTests.cs | 136 ++++++++++-------- 8 files changed, 312 insertions(+), 87 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs diff --git a/.claude/rules/alarms.md b/.claude/rules/alarms.md index e82bda20..d3dbec29 100644 --- a/.claude/rules/alarms.md +++ b/.claude/rules/alarms.md @@ -36,9 +36,12 @@ driver can decide without I/O (`IAlarmAcknowledgePreflight` — Galaxy: anonymou account) fails the Part 9 call `BadUserAccessDenied` via `NativeAlarmAckPreflight`, read lock-free from the `NativeAlarmAckPreflightView` snapshot `DriverHostActor` publishes; never do I/O (e.g. a Secrets lookup) in a pre-check — it runs under the SDK `Lock`. (2) anything else (keyed session without an identity, MXAccess -non-zero `hresult`, driver still connecting/reconnecting; a disabled/unspawned driver is NOT reverted) comes back as `DriverInstanceActor.AlarmAckFailed` and -`DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in -`NativeAlarmProjector` (any driver transition since then wins). +non-zero `hresult`, driver still connecting/reconnecting) comes back as `DriverInstanceActor.AlarmAckFailed` — or, for a +disabled/unspawned/stubbed driver, is decided in the host itself (#765) — and `DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in +`NativeAlarmProjector` (any driver transition since then wins). A host that came up **Stale** cannot route any +native ack, so it marks the whole `NativeAlarmAckPreflightView` unavailable and every native ack is refused +synchronously until `BecomeSteady` (#765). A test that routes an ack and expects it to STAND must start the owning +driver (`AckingAlarmDriver` in Runtime.Tests). Behaviours that were wrong or missing and are now fixed — keep them: the SDK's `OnTimedUnshelve` is a **no-op** (the engine's `AutoUnshelve` owns timed-shelve expiry, and `UnshelveTime` reports the real diff --git a/docs/AlarmTracking.md b/docs/AlarmTracking.md index b7d540cb..5d629ac0 100644 --- a/docs/AlarmTracking.md +++ b/docs/AlarmTracking.md @@ -464,12 +464,16 @@ from the scripted `AlarmCommandRouter`: driver refuses or fails the ack (or is still connecting / reconnecting), `DriverInstanceActor` replies `AlarmAckFailed` and `DriverHostActor` re-projects the condition **unacknowledged** (a fresh - condition event). Only an ack still pending in `NativeAlarmProjector` is + condition event). An ack whose owning driver is not running (disabled, + not yet spawned) or is a stub on this node is reverted the same way by + the host itself (#765). Only an ack still pending in `NativeAlarmProjector` is reverted: a driver transition projected since then is authoritative. A refusal the driver can decide without I/O (Galaxy: an anonymous session and no fallback ack account) is instead returned synchronously as `BadUserAccessDenied` by the node manager's `NativeAlarmAckPreflight` - before anything is applied. Galaxy value writes have a separate correlated + before anything is applied — as is every native ack while the driver host + is Stale (configuration database unreachable at boot), when no ack can be + routed or reverted (#765). Galaxy value writes have a separate correlated write-outcome path. An awaited ack RPC does not prove the ack reached AVEVA event history: the H6 live verification remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md). diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 962d134a..49abd04d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -1528,6 +1528,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // Cancelled BEFORE the unstash so a re-stash by the very next apply starts a fresh window. Timers.Cancel(ApplyWedgeTimerKey); _deferredDispatches.Clear(); + // Leaving Stale (or never in it): native acks are routable again (lmxopcua#765). + _ackPreflightView?.SetHostUnavailable(null); Become(Steady); Stash.UnstashAll(); } @@ -2325,12 +2327,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // deploy rather than resetting to its acked default. _nativeAlarmProjector.MarkAcknowledged(msg.ConditionNodeId); - if (!_children.TryGetValue(target.DriverInstanceId, out var entry)) + if (!_children.TryGetValue(target.DriverInstanceId, out var entry) || entry.Stubbed) { - _log.Debug("DriverHost {Node}: driver {Driver} not running for alarm ack of {Cond} — dropped", - _localNode, target.DriverInstanceId, msg.ConditionNodeId); - // Deliberately NOT reverted (lmxopcua#760 covers what a driver reports): a disabled or not-yet-spawned - // driver produces no transitions either, so the local ack stands, as before. + // A disabled / not-yet-spawned driver, or a stub (the driver does not run on this node, or its factory + // rejected the config), can deliver the ack nowhere. The node manager already applied it, so revert it + // like any other ack that never reached the alarm system (lmxopcua#765) — otherwise the node keeps + // claiming an acknowledgement the upstream system never saw. + var reason = entry is null ? "the owning driver is not running" : "the owning driver is stubbed on this node"; + _log.Debug("DriverHost {Node}: driver {Driver} cannot take the alarm ack of {Cond} ({Reason}) — dropped", + _localNode, target.DriverInstanceId, msg.ConditionNodeId, reason); + RevertNativeAlarmAck(msg.ConditionNodeId, reason); return; } @@ -2568,6 +2574,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS private void Stale() { + // lmxopcua#765: a native ack routed here while Stale is dropped before the projector records it, so it + // could not be reverted and the node would show Acknowledged for an ack no driver saw. Refuse it + // synchronously at the node manager instead (cleared by BecomeSteady, the only way out of Stale). + _ackPreflightView?.SetHostUnavailable("the driver host is stale (configuration database unreachable)"); Receive(_ => { _log.Warning("DriverHost {Node}: ignoring DispatchDeployment while Stale (DB unreachable)", _localNode); @@ -2602,8 +2612,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(_ => Sender.Tell(new NodeWriteResult(false, "driver host stale (config DB unreachable)"))); // An inbound native-condition ack can't be serviced while Stale either (the alarm inverse map is - // empty until an apply runs). The ack is fire-and-forget (no reply), so just drop it with a log — - // the local OPC UA condition state already committed on the Part 9 ack. + // empty until an apply runs). The node manager's pre-check refuses it synchronously while Stale (see the + // top of this method); one that raced that refusal is dropped here with a log. Receive(msg => _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Node2} while Stale (config DB unreachable)", _localNode, msg.ConditionNodeId)); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs index be54db74..3ea88c53 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs @@ -19,6 +19,8 @@ public sealed class NativeAlarmAckPreflightView private volatile ImmutableDictionary _byConditionNodeId = ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal); + private volatile string? _hostUnavailableReason; + /// /// Returns why the driver owning would refuse an acknowledge from a /// session authenticated with , or null to proceed (unknown condition, a @@ -30,6 +32,8 @@ public sealed class NativeAlarmAckPreflightView public string? Check(string conditionNodeId, string? apiKeyId) { if (string.IsNullOrEmpty(conditionNodeId)) return null; + // A host that cannot route any native ack (Stale — lmxopcua#765) refuses every one, whatever its driver. + if (_hostUnavailableReason is { } unavailable) return unavailable; if (!_byConditionNodeId.TryGetValue(conditionNodeId, out var preflight)) return null; try { @@ -43,6 +47,14 @@ public sealed class NativeAlarmAckPreflightView } } + /// + /// Marks the whole driver host unable to route a native acknowledge (lmxopcua#765): while set, + /// refuses every condition with . Null clears it. Called by + /// only — set on entering Stale, cleared on returning to Steady. + /// + /// Why no native ack can be routed, or null when acks are routable. + internal void SetHostUnavailable(string? reason) => _hostUnavailableReason = reason; + /// Replaces the snapshot. Called by only. /// Condition NodeId → the owning driver's pre-check. internal void Publish(IEnumerable> byConditionNodeId) => diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs new file mode 100644 index 00000000..0db54ed7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs @@ -0,0 +1,99 @@ +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; + +/// +/// A minimal + that accepts every acknowledge and raises +/// nothing — the host's own running driver for tests that route a client acknowledge but drive the alarm +/// transitions themselves. Since lmxopcua#765 an ack to a driver that is not running (or is stubbed) is +/// reverted, so such a test must start the owning driver for the ack to stand. +/// +internal sealed class AckingAlarmDriver : IDriver, IAlarmSource +{ + private int _acks; + private bool _alarmsSubscribed; + + /// + public string DriverInstanceId { get; private set; } = string.Empty; + + /// + public string DriverType { get; private set; } = string.Empty; + + /// How many acknowledge requests the driver accepted. + public int Acks => Volatile.Read(ref _acks); + + /// True once the driver child reached Connected and opened its alarm subscription. + public bool AlarmsSubscribed => Volatile.Read(ref _alarmsSubscribed); + + /// Sets the identity once the factory is asked to create it. + /// The driver instance id. + /// The driver type. + public void Bind(string id, string type) + { + DriverInstanceId = id; + DriverType = type; + } + + /// + public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null); + + /// + public Task SubscribeAlarmsAsync( + IReadOnlyList sourceNodeIds, CancellationToken cancellationToken) + { + Volatile.Write(ref _alarmsSubscribed, true); + return Task.FromResult(new Handle()); + } + + /// + public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) => + Task.CompletedTask; + + /// + public Task AcknowledgeAsync( + IReadOnlyList acknowledgements, CancellationToken cancellationToken) + { + Interlocked.Add(ref _acks, acknowledgements.Count); + return Task.CompletedTask; + } + + /// + public event EventHandler? OnAlarmEvent + { + add { } + remove { } + } + + private sealed class Handle : IAlarmSubscriptionHandle + { + public string DiagnosticId => "acking-alarm-sub"; + } +} + +/// Factory handing the host one shared for a single driver type. +/// The driver type this factory creates. +internal sealed class AckingAlarmDriverFactory(string supportedType) : IDriverFactory +{ + /// The driver every spawn of receives. + public AckingAlarmDriver Driver { get; } = new(); + + /// + public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) + { + if (!string.Equals(driverType, supportedType, StringComparison.Ordinal)) return null; + Driver.Bind(driverInstanceId, driverType); + return Driver; + } + + /// + public IReadOnlyCollection SupportedTypes => [supportedType]; +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs index 23c7db40..57769ecc 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs @@ -247,6 +247,77 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest view.Check("Plant/gw/dev1/does-not-exist", apiKeyId: null).ShouldBeNull(); } + /// + /// lmxopcua#765 — an ack routed to a driver that is not running (disabled here) reaches no alarm system, so + /// the host reverts the condition the node manager optimistically acknowledged, as for a refused ack. + /// + [Fact] + public void An_ack_to_a_driver_that_is_not_running_reverts_the_condition() + { + var db = NewInMemoryDbFactory(); + var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway"); + var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi", enabled: false); + var publish = this.CreateCleanOutcomePublishProbe(); + var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref, awaitConnected: false); + + RaiseAndExpectUnacked(actor, publish); + actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice")); + + var reverted = NextAlarmUpdate(publish); + reverted.AlarmNodeId.ShouldBe(AlarmRawPath); + reverted.State.Active.ShouldBeTrue(); + reverted.State.Acknowledged.ShouldBeFalse(); + recorder.Acks.ShouldBeEmpty("no driver was running to take the ack"); + } + + /// + /// lmxopcua#765 — likewise a STUBBED child (here: no factory for the driver type, so the host falls back to + /// a stub that has no alarm backend) silently swallows the ack; the host reverts it rather than routing it. + /// + [Fact] + public void An_ack_to_a_stubbed_driver_reverts_the_condition() + { + var db = NewInMemoryDbFactory(); + var noFactoryForType = new RecordingAlarmDriverFactory("SomeOtherType"); + var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi"); + var publish = this.CreateCleanOutcomePublishProbe(); + var actor = SpawnHostAndApply(db, deploymentId, noFactoryForType, publish.Ref, awaitConnected: false); + + RaiseAndExpectUnacked(actor, publish); + actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice")); + + NextAlarmUpdate(publish).State.Acknowledged.ShouldBeFalse(); + } + + /// + /// lmxopcua#765 — a host that came up Stale (config DB unreachable) drops every native ack before the + /// projector records it, so it could not be reverted. The pre-check view refuses every native ack while the + /// host is Stale, so the node manager answers synchronously instead of applying an ack nothing will route. + /// + [Fact] + public void A_stale_host_refuses_every_native_ack_through_the_preflight_view() + { + var view = new NativeAlarmAckPreflightView(); + view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldBeNull("positive control: nothing refuses before the host starts"); + + var host = Sys.ActorOf(DriverHostActor.Props( + new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref, + localRoles: new HashSet { "driver" }, + ackPreflightView: view)); + // Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale. + var asker = CreateTestProbe(); + host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref); + asker.ExpectMsg(Timeout).Reason!.ShouldContain("stale"); + + view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldNotBeNull().ShouldContain("stale"); + view.Check("any/other/condition", apiKeyId: null).ShouldNotBeNull(); + } + + private sealed class ThrowingDbFactory : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable"); + } + private static AlarmEventArgs Evt(AlarmTransitionKind kind) => new( new StubAlarmHandleForTests(), SourceNodeId: "Temp", ConditionId: AlarmRawPath, AlarmType: "OffNormalAlarm", Message: "temperature high", Severity: AlarmSeverity.High, SourceTimestampUtc: DateTime.UtcNow, Kind: kind); @@ -272,7 +343,8 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest /// completed before the test routes an ack. private IActorRef SpawnHostAndApply( IDbContextFactory db, DeploymentId deploymentId, IDriverFactory factory, - IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null) + IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null, + bool awaitConnected = true) { var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( @@ -291,7 +363,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest // the control itself was dropped and the wait read [] — 4 of 20 runs under thread-pool starvation, // and on Gitea (#606). SubscribeAlarmsAsync is only ever called from the Connected state (a // pre-connect request is swallowed and re-issued on the Connected entry), so wait for it. - if (factory is RecordingAlarmDriverFactory recording) + if (awaitConnected && factory is RecordingAlarmDriverFactory recording) AwaitCondition(() => recording.AlarmsSubscribed, Timeout); return actor; @@ -305,7 +377,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest /// DriverType + Enabled flag spawn a REAL child. /// private static DeploymentId SeedV3AlarmDeployment( - IDbContextFactory db, RevisionHash rev, string Driver, string Tag) + IDbContextFactory db, RevisionHash rev, string Driver, string Tag, bool enabled = true) { var artifact = JsonSerializer.SerializeToUtf8Bytes(new { @@ -319,7 +391,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest RawFolderId = "rf-plant", Name = "gw", DriverType = "GalaxyMxGateway", // not Windows-only ⇒ a real child is spawned (not stubbed) - Enabled = true, + Enabled = enabled, DriverConfig = "{}", ClusterId = "c1", }, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs index 1f244b62..32149a65 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs @@ -479,20 +479,29 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase /// after one defaulted Acked to true and happened to be right for a client-acked alarm; now that the /// projector survives the deploy, a projector that never heard the ack projects the Clear as UNacked and /// un-acknowledges the alarm the operator acknowledged. + /// + /// lmxopcua#765 — the host's own driver is started here: an ack routed to a driver that is not running + /// is now reverted (it reached no alarm system), so only a delivered ack may stand. + /// /// [Fact] public void A_clear_after_a_client_acknowledge_and_a_redeploy_stays_acknowledged() { var db = NewInMemoryDbFactory(); - var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi"); - var (actor, publish, coordinator) = SpawnHostAndApplyWithCoordinator(db, deploymentId); + var factory = new AckingAlarmDriverFactory("Modbus"); + var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi", enabled: true); + var (actor, publish, coordinator) = SpawnHostAndApplyWithCoordinator(db, deploymentId, factory); + // An ack reaching a still-Connecting child is dropped (and now reverted); wait for Connected. + AwaitCondition(() => factory.Driver.AlarmsSubscribed, TimeSpan.FromSeconds(5)); actor.Tell(Transition(AlarmTransitionKind.Raise)); - publish.ExpectMsg(TimeSpan.FromSeconds(5)).State.Acknowledged.ShouldBeFalse(); + publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)) + .State.Acknowledged.ShouldBeFalse(); - // The client's Acknowledge on the node, routed toward the driver (no transition comes back for it). + // The client's Acknowledge on the node, routed to the driver (no transition comes back for it). actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "seen", "op")); - Redeploy(db, actor, publish, coordinator); + AwaitCondition(() => factory.Driver.Acks == 1, TimeSpan.FromSeconds(5)); + Redeploy(db, actor, publish, coordinator, enabled: true); actor.Tell(Transition(AlarmTransitionKind.Clear)); var cleared = publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); @@ -507,10 +516,11 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase /// A second deployment carrying the SAME alarm (a new revision of an identical artifact). private void Redeploy( - IDbContextFactory db, IActorRef actor, TestProbe publish, TestProbe coordinator) + IDbContextFactory db, IActorRef actor, TestProbe publish, TestProbe coordinator, + bool enabled = false) { var revB = RevisionHash.Parse(new string('b', 64)); - var second = SeedV3AlarmDeployment(db, revB, Driver: "drv-1", Tag: "temp_hi"); + var second = SeedV3AlarmDeployment(db, revB, Driver: "drv-1", Tag: "temp_hi", enabled: enabled); actor.Tell(new DispatchDeployment(second, revB, CorrelationId.NewId())); coordinator.FishForMessage(a => a.DeploymentId == second, TimeSpan.FromSeconds(5)) .Outcome.ShouldBe(ApplyAckOutcome.Applied); @@ -553,7 +563,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase } private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator( - IDbContextFactory db, DeploymentId deploymentId) + IDbContextFactory db, DeploymentId deploymentId, IDriverFactory? driverFactory = null) { var coordinator = CreateTestProbe(); var publish = this.CreateCleanOutcomePublishProbe(); @@ -562,6 +572,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, + driverFactory: driverFactory, localRoles: new HashSet { "driver" }, dependencyMux: mux.Ref, opcUaPublishActor: publish.Ref, @@ -588,7 +599,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase /// private static DeploymentId SeedV3AlarmDeployment( IDbContextFactory db, RevisionHash rev, - string Driver, string Tag, bool? historizeToAveva = null) + string Driver, string Tag, bool? historizeToAveva = null, bool enabled = false) { // historizeToAveva absent (null) ⇒ omit the key so the absent ⇒ historize default path is exercised; // a concrete true/false writes the bool into the alarm object so the native path threads it. @@ -601,7 +612,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } }, DriverInstances = new[] { - new { DriverInstanceId = Driver, RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = false }, + new { DriverInstanceId = Driver, RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }, }, Devices = new[] { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs index bd3406c7..97887ac2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs @@ -103,83 +103,95 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase /// servo alarm, so the condition's displayed set changed) is a new statement even though the fresh driver /// can only say Raise. It must un-acknowledge and write its alerts row. /// + /// + /// lmxopcua#765 — the host spawns, subscribes and respawns its OWN FOCAS driver here (a factory over the fake + /// CNC), so the operator's ack is delivered to a running driver. An ack to a driver that is not running is + /// reverted, so the old shape — the host's driver disabled, the test driving a detached instance — could no + /// longer keep the ack. The respawn is a real one: the second deployment changes the driver's config. + /// [Theory] [InlineData(false)] [InlineData(true)] - public async Task A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn) + public void A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn) { var db = NewInMemoryDbFactory(); - var blob = BuildArtifact(); + var blob = BuildArtifact(enabled: true); var alerts = CreateTestProbe(); DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref); alerts.ExpectMsg(TimeSpan.FromSeconds(5)); - var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev)); - var rawTags = RawTagsFor(blob, DriverId); - var cnc = new AlarmOnlyFocasClient(); - cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF")); - - var first = await StartDriverAsync(rawTags, cnc, host); - try - { - publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)) - .State.Acknowledged.ShouldBeFalse(); - alerts.ExpectMsg(TimeSpan.FromSeconds(5)); - } - finally - { - await first.ShutdownAsync(CancellationToken.None); - } - - // The operator acknowledges, then a deploy changes the driver's config => stop + respawn. - host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op")); - var revB = RevisionHash.Parse(new string('e', 64)); - host.Tell(new DispatchDeployment(Seed(db, blob, revB), revB, CorrelationId.NewId())); - coordinator.FishForMessage(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5)); - publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); - + // One fake CNC per driver instance, so what the fresh instance reads is fixed before it can poll. + var firstCnc = new AlarmOnlyFocasClient(); + firstCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF")); + var respawnCnc = new AlarmOnlyFocasClient(); if (newAlarmDuringRespawn) - cnc.SetAlarms( + respawnCnc.SetAlarms( new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"), new FocasActiveAlarm(410, FocasAlarmType.Servo, 2, "SERVO EXCESS ERROR")); + else + respawnCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF")); + var factory = new HostFocasDriverFactory(firstCnc, respawnCnc); + var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev), factory); - var respawned = await StartDriverAsync(rawTags, cnc, host); - try + publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)) + .State.Acknowledged.ShouldBeFalse(); + alerts.ExpectMsg(TimeSpan.FromSeconds(5)); + + // The operator acknowledges (the running driver takes it), then a deploy changes the driver's config => + // the host stops it and spawns a fresh instance. + host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op")); + var revB = RevisionHash.Parse(new string('e', 64)); + var changedConfig = BuildArtifact(enabled: true, driverConfig: "{\"respawn\":1}"); + host.Tell(new DispatchDeployment(Seed(db, changedConfig, revB), revB, CorrelationId.NewId())); + coordinator.FishForMessage(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5)); + + var restated = publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); + factory.Created.ShouldBe(2, "the config change must have respawned the host's driver"); + restated.AlarmNodeId.ShouldBe(ExpectedRawPath); + restated.State.Active.ShouldBeTrue(); + if (newAlarmDuringRespawn) { - var restated = publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); - restated.AlarmNodeId.ShouldBe(ExpectedRawPath); - restated.State.Active.ShouldBeTrue(); - if (newAlarmDuringRespawn) - { - restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack"); - alerts.ExpectMsg(TimeSpan.FromSeconds(5)); - } - else - { - restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge"); - alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); - } + restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack"); + alerts.ExpectMsg(TimeSpan.FromSeconds(5)); } - finally + else { - await respawned.ShutdownAsync(CancellationToken.None); + restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge"); + alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); } } - private static async Task StartDriverAsync(List rawTags, IFocasClient cnc, IActorRef host) + /// + /// The host's driver factory for the respawn test: every spawn gets a fresh over the + /// next fake CNC (the last one repeats), fed the raw tags from the merged DriverConfig the host hands it — as + /// production does. + /// + private sealed class HostFocasDriverFactory(params IFocasClient[] cncs) : IDriverFactory { - var driver = new FocasDriver(new FocasDriverOptions + private int _created; + + /// How many driver instances the host has created. + public int Created => Volatile.Read(ref _created); + + /// + public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) { - Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")], - RawTags = rawTags, - Probe = new FocasProbeOptions { Enabled = false }, - AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) }, - }, DriverId, new SingleClientFactory(cnc)); - driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e)); - await driver.InitializeAsync("{}", CancellationToken.None); -#pragma warning disable OTOPCUA0001 // the routing is under test, not resilience wrapping - await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None); -#pragma warning restore OTOPCUA0001 - return driver; + if (driverType != "FOCAS") return null; + var cnc = cncs[Math.Min(Interlocked.Increment(ref _created), cncs.Length) - 1]; + using var doc = JsonDocument.Parse(driverConfigJson); + var rawTags = JsonSerializer.Deserialize>( + doc.RootElement.GetProperty("RawTags").GetRawText(), + new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; + return new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")], + RawTags = rawTags, + Probe = new FocasProbeOptions { Enabled = false }, + AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) }, + }, driverInstanceId, new SingleClientFactory(cnc)); + } + + /// + public IReadOnlyCollection SupportedTypes => ["FOCAS"]; } /// The seeded artifact's FOCAS driver raw tags, decoded through the production artifact reader. @@ -193,13 +205,14 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; } - private static byte[] BuildArtifact() => JsonSerializer.SerializeToUtf8Bytes(new + private static byte[] BuildArtifact(bool enabled = false, string driverConfig = "{}") => JsonSerializer.SerializeToUtf8Bytes(new { RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } }, DriverInstances = new[] { - // Enabled=false: the host must not spawn a real wire driver — the test drives its own instance. - new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = "{}", ClusterId = "c1", Enabled = false }, + // Enabled=false (default): the host spawns no driver — the test drives its own instance. Enabled=true + // spawns the host's own driver through the test's factory (never a real wire driver). + new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = driverConfig, ClusterId = "c1", Enabled = enabled }, }, Devices = new[] { @@ -244,12 +257,13 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase } private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply( - IDbContextFactory db, DeploymentId deploymentId) + IDbContextFactory db, DeploymentId deploymentId, IDriverFactory? driverFactory = null) { var coordinator = CreateTestProbe(); var publish = this.CreateCleanOutcomePublishProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, + driverFactory: driverFactory, localRoles: new HashSet { "driver" }, dependencyMux: CreateTestProbe().Ref, opcUaPublishActor: publish.Ref,