diff --git a/.claude/rules/alarms.md b/.claude/rules/alarms.md index e82bda20..d3dbec29 100644 --- a/.claude/rules/alarms.md +++ b/.claude/rules/alarms.md @@ -36,9 +36,12 @@ driver can decide without I/O (`IAlarmAcknowledgePreflight` — Galaxy: anonymou account) fails the Part 9 call `BadUserAccessDenied` via `NativeAlarmAckPreflight`, read lock-free from the `NativeAlarmAckPreflightView` snapshot `DriverHostActor` publishes; never do I/O (e.g. a Secrets lookup) in a pre-check — it runs under the SDK `Lock`. (2) anything else (keyed session without an identity, MXAccess -non-zero `hresult`, driver still connecting/reconnecting; a disabled/unspawned driver is NOT reverted) comes back as `DriverInstanceActor.AlarmAckFailed` and -`DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in -`NativeAlarmProjector` (any driver transition since then wins). +non-zero `hresult`, driver still connecting/reconnecting) comes back as `DriverInstanceActor.AlarmAckFailed` — or, for a +disabled/unspawned/stubbed driver, is decided in the host itself (#765) — and `DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in +`NativeAlarmProjector` (any driver transition since then wins). A host that came up **Stale** cannot route any +native ack, so it marks the whole `NativeAlarmAckPreflightView` unavailable and every native ack is refused +synchronously until `BecomeSteady` (#765). A test that routes an ack and expects it to STAND must start the owning +driver (`AckingAlarmDriver` in Runtime.Tests). Behaviours that were wrong or missing and are now fixed — keep them: the SDK's `OnTimedUnshelve` is a **no-op** (the engine's `AutoUnshelve` owns timed-shelve expiry, and `UnshelveTime` reports the real diff --git a/docs/AlarmTracking.md b/docs/AlarmTracking.md index b7d540cb..5d629ac0 100644 --- a/docs/AlarmTracking.md +++ b/docs/AlarmTracking.md @@ -464,12 +464,16 @@ from the scripted `AlarmCommandRouter`: driver refuses or fails the ack (or is still connecting / reconnecting), `DriverInstanceActor` replies `AlarmAckFailed` and `DriverHostActor` re-projects the condition **unacknowledged** (a fresh - condition event). Only an ack still pending in `NativeAlarmProjector` is + condition event). An ack whose owning driver is not running (disabled, + not yet spawned) or is a stub on this node is reverted the same way by + the host itself (#765). Only an ack still pending in `NativeAlarmProjector` is reverted: a driver transition projected since then is authoritative. A refusal the driver can decide without I/O (Galaxy: an anonymous session and no fallback ack account) is instead returned synchronously as `BadUserAccessDenied` by the node manager's `NativeAlarmAckPreflight` - before anything is applied. Galaxy value writes have a separate correlated + before anything is applied — as is every native ack while the driver host + is Stale (configuration database unreachable at boot), when no ack can be + routed or reverted (#765). Galaxy value writes have a separate correlated write-outcome path. An awaited ack RPC does not prove the ack reached AVEVA event history: the H6 live verification remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md). diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs index 962d134a..49abd04d 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/DriverHostActor.cs @@ -1528,6 +1528,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // Cancelled BEFORE the unstash so a re-stash by the very next apply starts a fresh window. Timers.Cancel(ApplyWedgeTimerKey); _deferredDispatches.Clear(); + // Leaving Stale (or never in it): native acks are routable again (lmxopcua#765). + _ackPreflightView?.SetHostUnavailable(null); Become(Steady); Stash.UnstashAll(); } @@ -2325,12 +2327,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS // deploy rather than resetting to its acked default. _nativeAlarmProjector.MarkAcknowledged(msg.ConditionNodeId); - if (!_children.TryGetValue(target.DriverInstanceId, out var entry)) + if (!_children.TryGetValue(target.DriverInstanceId, out var entry) || entry.Stubbed) { - _log.Debug("DriverHost {Node}: driver {Driver} not running for alarm ack of {Cond} — dropped", - _localNode, target.DriverInstanceId, msg.ConditionNodeId); - // Deliberately NOT reverted (lmxopcua#760 covers what a driver reports): a disabled or not-yet-spawned - // driver produces no transitions either, so the local ack stands, as before. + // A disabled / not-yet-spawned driver, or a stub (the driver does not run on this node, or its factory + // rejected the config), can deliver the ack nowhere. The node manager already applied it, so revert it + // like any other ack that never reached the alarm system (lmxopcua#765) — otherwise the node keeps + // claiming an acknowledgement the upstream system never saw. + var reason = entry is null ? "the owning driver is not running" : "the owning driver is stubbed on this node"; + _log.Debug("DriverHost {Node}: driver {Driver} cannot take the alarm ack of {Cond} ({Reason}) — dropped", + _localNode, target.DriverInstanceId, msg.ConditionNodeId, reason); + RevertNativeAlarmAck(msg.ConditionNodeId, reason); return; } @@ -2568,6 +2574,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS private void Stale() { + // lmxopcua#765: a native ack routed here while Stale is dropped before the projector records it, so it + // could not be reverted and the node would show Acknowledged for an ack no driver saw. Refuse it + // synchronously at the node manager instead (cleared by BecomeSteady, the only way out of Stale). + _ackPreflightView?.SetHostUnavailable("the driver host is stale (configuration database unreachable)"); Receive(_ => { _log.Warning("DriverHost {Node}: ignoring DispatchDeployment while Stale (DB unreachable)", _localNode); @@ -2602,8 +2612,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS Receive(_ => Sender.Tell(new NodeWriteResult(false, "driver host stale (config DB unreachable)"))); // An inbound native-condition ack can't be serviced while Stale either (the alarm inverse map is - // empty until an apply runs). The ack is fire-and-forget (no reply), so just drop it with a log — - // the local OPC UA condition state already committed on the Part 9 ack. + // empty until an apply runs). The node manager's pre-check refuses it synchronously while Stale (see the + // top of this method); one that raced that refusal is dropped here with a log. Receive(msg => _log.Debug("DriverHost {Node}: dropping native-alarm ack for {Node2} while Stale (config DB unreachable)", _localNode, msg.ConditionNodeId)); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs index be54db74..3ea88c53 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/NativeAlarmAckPreflightView.cs @@ -19,6 +19,8 @@ public sealed class NativeAlarmAckPreflightView private volatile ImmutableDictionary _byConditionNodeId = ImmutableDictionary.Empty.WithComparers(StringComparer.Ordinal); + private volatile string? _hostUnavailableReason; + /// /// Returns why the driver owning would refuse an acknowledge from a /// session authenticated with , or null to proceed (unknown condition, a @@ -30,6 +32,8 @@ public sealed class NativeAlarmAckPreflightView public string? Check(string conditionNodeId, string? apiKeyId) { if (string.IsNullOrEmpty(conditionNodeId)) return null; + // A host that cannot route any native ack (Stale — lmxopcua#765) refuses every one, whatever its driver. + if (_hostUnavailableReason is { } unavailable) return unavailable; if (!_byConditionNodeId.TryGetValue(conditionNodeId, out var preflight)) return null; try { @@ -43,6 +47,14 @@ public sealed class NativeAlarmAckPreflightView } } + /// + /// Marks the whole driver host unable to route a native acknowledge (lmxopcua#765): while set, + /// refuses every condition with . Null clears it. Called by + /// only — set on entering Stale, cleared on returning to Steady. + /// + /// Why no native ack can be routed, or null when acks are routable. + internal void SetHostUnavailable(string? reason) => _hostUnavailableReason = reason; + /// Replaces the snapshot. Called by only. /// Condition NodeId → the owning driver's pre-check. internal void Publish(IEnumerable> byConditionNodeId) => diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs new file mode 100644 index 00000000..0db54ed7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/AckingAlarmDriver.cs @@ -0,0 +1,99 @@ +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; + +/// +/// A minimal + that accepts every acknowledge and raises +/// nothing — the host's own running driver for tests that route a client acknowledge but drive the alarm +/// transitions themselves. Since lmxopcua#765 an ack to a driver that is not running (or is stubbed) is +/// reverted, so such a test must start the owning driver for the ack to stand. +/// +internal sealed class AckingAlarmDriver : IDriver, IAlarmSource +{ + private int _acks; + private bool _alarmsSubscribed; + + /// + public string DriverInstanceId { get; private set; } = string.Empty; + + /// + public string DriverType { get; private set; } = string.Empty; + + /// How many acknowledge requests the driver accepted. + public int Acks => Volatile.Read(ref _acks); + + /// True once the driver child reached Connected and opened its alarm subscription. + public bool AlarmsSubscribed => Volatile.Read(ref _alarmsSubscribed); + + /// Sets the identity once the factory is asked to create it. + /// The driver instance id. + /// The driver type. + public void Bind(string id, string type) + { + DriverInstanceId = id; + DriverType = type; + } + + /// + public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null); + + /// + public Task SubscribeAlarmsAsync( + IReadOnlyList sourceNodeIds, CancellationToken cancellationToken) + { + Volatile.Write(ref _alarmsSubscribed, true); + return Task.FromResult(new Handle()); + } + + /// + public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) => + Task.CompletedTask; + + /// + public Task AcknowledgeAsync( + IReadOnlyList acknowledgements, CancellationToken cancellationToken) + { + Interlocked.Add(ref _acks, acknowledgements.Count); + return Task.CompletedTask; + } + + /// + public event EventHandler? OnAlarmEvent + { + add { } + remove { } + } + + private sealed class Handle : IAlarmSubscriptionHandle + { + public string DiagnosticId => "acking-alarm-sub"; + } +} + +/// Factory handing the host one shared for a single driver type. +/// The driver type this factory creates. +internal sealed class AckingAlarmDriverFactory(string supportedType) : IDriverFactory +{ + /// The driver every spawn of receives. + public AckingAlarmDriver Driver { get; } = new(); + + /// + public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) + { + if (!string.Equals(driverType, supportedType, StringComparison.Ordinal)) return null; + Driver.Bind(driverInstanceId, driverType); + return Driver; + } + + /// + public IReadOnlyCollection SupportedTypes => [supportedType]; +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs index 23c7db40..57769ecc 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs @@ -247,6 +247,77 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest view.Check("Plant/gw/dev1/does-not-exist", apiKeyId: null).ShouldBeNull(); } + /// + /// lmxopcua#765 — an ack routed to a driver that is not running (disabled here) reaches no alarm system, so + /// the host reverts the condition the node manager optimistically acknowledged, as for a refused ack. + /// + [Fact] + public void An_ack_to_a_driver_that_is_not_running_reverts_the_condition() + { + var db = NewInMemoryDbFactory(); + var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway"); + var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi", enabled: false); + var publish = this.CreateCleanOutcomePublishProbe(); + var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref, awaitConnected: false); + + RaiseAndExpectUnacked(actor, publish); + actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice")); + + var reverted = NextAlarmUpdate(publish); + reverted.AlarmNodeId.ShouldBe(AlarmRawPath); + reverted.State.Active.ShouldBeTrue(); + reverted.State.Acknowledged.ShouldBeFalse(); + recorder.Acks.ShouldBeEmpty("no driver was running to take the ack"); + } + + /// + /// lmxopcua#765 — likewise a STUBBED child (here: no factory for the driver type, so the host falls back to + /// a stub that has no alarm backend) silently swallows the ack; the host reverts it rather than routing it. + /// + [Fact] + public void An_ack_to_a_stubbed_driver_reverts_the_condition() + { + var db = NewInMemoryDbFactory(); + var noFactoryForType = new RecordingAlarmDriverFactory("SomeOtherType"); + var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi"); + var publish = this.CreateCleanOutcomePublishProbe(); + var actor = SpawnHostAndApply(db, deploymentId, noFactoryForType, publish.Ref, awaitConnected: false); + + RaiseAndExpectUnacked(actor, publish); + actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice")); + + NextAlarmUpdate(publish).State.Acknowledged.ShouldBeFalse(); + } + + /// + /// lmxopcua#765 — a host that came up Stale (config DB unreachable) drops every native ack before the + /// projector records it, so it could not be reverted. The pre-check view refuses every native ack while the + /// host is Stale, so the node manager answers synchronously instead of applying an ack nothing will route. + /// + [Fact] + public void A_stale_host_refuses_every_native_ack_through_the_preflight_view() + { + var view = new NativeAlarmAckPreflightView(); + view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldBeNull("positive control: nothing refuses before the host starts"); + + var host = Sys.ActorOf(DriverHostActor.Props( + new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref, + localRoles: new HashSet { "driver" }, + ackPreflightView: view)); + // Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale. + var asker = CreateTestProbe(); + host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref); + asker.ExpectMsg(Timeout).Reason!.ShouldContain("stale"); + + view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldNotBeNull().ShouldContain("stale"); + view.Check("any/other/condition", apiKeyId: null).ShouldNotBeNull(); + } + + private sealed class ThrowingDbFactory : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable"); + } + private static AlarmEventArgs Evt(AlarmTransitionKind kind) => new( new StubAlarmHandleForTests(), SourceNodeId: "Temp", ConditionId: AlarmRawPath, AlarmType: "OffNormalAlarm", Message: "temperature high", Severity: AlarmSeverity.High, SourceTimestampUtc: DateTime.UtcNow, Kind: kind); @@ -272,7 +343,8 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest /// completed before the test routes an ack. private IActorRef SpawnHostAndApply( IDbContextFactory db, DeploymentId deploymentId, IDriverFactory factory, - IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null) + IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null, + bool awaitConnected = true) { var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( @@ -291,7 +363,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest // the control itself was dropped and the wait read [] — 4 of 20 runs under thread-pool starvation, // and on Gitea (#606). SubscribeAlarmsAsync is only ever called from the Connected state (a // pre-connect request is swallowed and re-issued on the Connected entry), so wait for it. - if (factory is RecordingAlarmDriverFactory recording) + if (awaitConnected && factory is RecordingAlarmDriverFactory recording) AwaitCondition(() => recording.AlarmsSubscribed, Timeout); return actor; @@ -305,7 +377,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest /// DriverType + Enabled flag spawn a REAL child. /// private static DeploymentId SeedV3AlarmDeployment( - IDbContextFactory db, RevisionHash rev, string Driver, string Tag) + IDbContextFactory db, RevisionHash rev, string Driver, string Tag, bool enabled = true) { var artifact = JsonSerializer.SerializeToUtf8Bytes(new { @@ -319,7 +391,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest RawFolderId = "rf-plant", Name = "gw", DriverType = "GalaxyMxGateway", // not Windows-only ⇒ a real child is spawned (not stubbed) - Enabled = true, + Enabled = enabled, DriverConfig = "{}", ClusterId = "c1", }, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs index 1f244b62..32149a65 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmTests.cs @@ -479,20 +479,29 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase /// after one defaulted Acked to true and happened to be right for a client-acked alarm; now that the /// projector survives the deploy, a projector that never heard the ack projects the Clear as UNacked and /// un-acknowledges the alarm the operator acknowledged. + /// + /// lmxopcua#765 — the host's own driver is started here: an ack routed to a driver that is not running + /// is now reverted (it reached no alarm system), so only a delivered ack may stand. + /// /// [Fact] public void A_clear_after_a_client_acknowledge_and_a_redeploy_stays_acknowledged() { var db = NewInMemoryDbFactory(); - var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi"); - var (actor, publish, coordinator) = SpawnHostAndApplyWithCoordinator(db, deploymentId); + var factory = new AckingAlarmDriverFactory("Modbus"); + var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi", enabled: true); + var (actor, publish, coordinator) = SpawnHostAndApplyWithCoordinator(db, deploymentId, factory); + // An ack reaching a still-Connecting child is dropped (and now reverted); wait for Connected. + AwaitCondition(() => factory.Driver.AlarmsSubscribed, TimeSpan.FromSeconds(5)); actor.Tell(Transition(AlarmTransitionKind.Raise)); - publish.ExpectMsg(TimeSpan.FromSeconds(5)).State.Acknowledged.ShouldBeFalse(); + publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)) + .State.Acknowledged.ShouldBeFalse(); - // The client's Acknowledge on the node, routed toward the driver (no transition comes back for it). + // The client's Acknowledge on the node, routed to the driver (no transition comes back for it). actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "seen", "op")); - Redeploy(db, actor, publish, coordinator); + AwaitCondition(() => factory.Driver.Acks == 1, TimeSpan.FromSeconds(5)); + Redeploy(db, actor, publish, coordinator, enabled: true); actor.Tell(Transition(AlarmTransitionKind.Clear)); var cleared = publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); @@ -507,10 +516,11 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase /// A second deployment carrying the SAME alarm (a new revision of an identical artifact). private void Redeploy( - IDbContextFactory db, IActorRef actor, TestProbe publish, TestProbe coordinator) + IDbContextFactory db, IActorRef actor, TestProbe publish, TestProbe coordinator, + bool enabled = false) { var revB = RevisionHash.Parse(new string('b', 64)); - var second = SeedV3AlarmDeployment(db, revB, Driver: "drv-1", Tag: "temp_hi"); + var second = SeedV3AlarmDeployment(db, revB, Driver: "drv-1", Tag: "temp_hi", enabled: enabled); actor.Tell(new DispatchDeployment(second, revB, CorrelationId.NewId())); coordinator.FishForMessage(a => a.DeploymentId == second, TimeSpan.FromSeconds(5)) .Outcome.ShouldBe(ApplyAckOutcome.Applied); @@ -553,7 +563,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase } private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator( - IDbContextFactory db, DeploymentId deploymentId) + IDbContextFactory db, DeploymentId deploymentId, IDriverFactory? driverFactory = null) { var coordinator = CreateTestProbe(); var publish = this.CreateCleanOutcomePublishProbe(); @@ -562,6 +572,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, + driverFactory: driverFactory, localRoles: new HashSet { "driver" }, dependencyMux: mux.Ref, opcUaPublishActor: publish.Ref, @@ -588,7 +599,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase /// private static DeploymentId SeedV3AlarmDeployment( IDbContextFactory db, RevisionHash rev, - string Driver, string Tag, bool? historizeToAveva = null) + string Driver, string Tag, bool? historizeToAveva = null, bool enabled = false) { // historizeToAveva absent (null) ⇒ omit the key so the absent ⇒ historize default path is exercised; // a concrete true/false writes the bool into the alarm object so the native path threads it. @@ -601,7 +612,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } }, DriverInstances = new[] { - new { DriverInstanceId = Driver, RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = false }, + new { DriverInstanceId = Driver, RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }, }, Devices = new[] { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs index bd3406c7..97887ac2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs @@ -103,83 +103,95 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase /// servo alarm, so the condition's displayed set changed) is a new statement even though the fresh driver /// can only say Raise. It must un-acknowledge and write its alerts row. /// + /// + /// lmxopcua#765 — the host spawns, subscribes and respawns its OWN FOCAS driver here (a factory over the fake + /// CNC), so the operator's ack is delivered to a running driver. An ack to a driver that is not running is + /// reverted, so the old shape — the host's driver disabled, the test driving a detached instance — could no + /// longer keep the ack. The respawn is a real one: the second deployment changes the driver's config. + /// [Theory] [InlineData(false)] [InlineData(true)] - public async Task A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn) + public void A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn) { var db = NewInMemoryDbFactory(); - var blob = BuildArtifact(); + var blob = BuildArtifact(enabled: true); var alerts = CreateTestProbe(); DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref); alerts.ExpectMsg(TimeSpan.FromSeconds(5)); - var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev)); - var rawTags = RawTagsFor(blob, DriverId); - var cnc = new AlarmOnlyFocasClient(); - cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF")); - - var first = await StartDriverAsync(rawTags, cnc, host); - try - { - publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)) - .State.Acknowledged.ShouldBeFalse(); - alerts.ExpectMsg(TimeSpan.FromSeconds(5)); - } - finally - { - await first.ShutdownAsync(CancellationToken.None); - } - - // The operator acknowledges, then a deploy changes the driver's config => stop + respawn. - host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op")); - var revB = RevisionHash.Parse(new string('e', 64)); - host.Tell(new DispatchDeployment(Seed(db, blob, revB), revB, CorrelationId.NewId())); - coordinator.FishForMessage(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5)); - publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); - + // One fake CNC per driver instance, so what the fresh instance reads is fixed before it can poll. + var firstCnc = new AlarmOnlyFocasClient(); + firstCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF")); + var respawnCnc = new AlarmOnlyFocasClient(); if (newAlarmDuringRespawn) - cnc.SetAlarms( + respawnCnc.SetAlarms( new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"), new FocasActiveAlarm(410, FocasAlarmType.Servo, 2, "SERVO EXCESS ERROR")); + else + respawnCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF")); + var factory = new HostFocasDriverFactory(firstCnc, respawnCnc); + var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev), factory); - var respawned = await StartDriverAsync(rawTags, cnc, host); - try + publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)) + .State.Acknowledged.ShouldBeFalse(); + alerts.ExpectMsg(TimeSpan.FromSeconds(5)); + + // The operator acknowledges (the running driver takes it), then a deploy changes the driver's config => + // the host stops it and spawns a fresh instance. + host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op")); + var revB = RevisionHash.Parse(new string('e', 64)); + var changedConfig = BuildArtifact(enabled: true, driverConfig: "{\"respawn\":1}"); + host.Tell(new DispatchDeployment(Seed(db, changedConfig, revB), revB, CorrelationId.NewId())); + coordinator.FishForMessage(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5)); + + var restated = publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); + factory.Created.ShouldBe(2, "the config change must have respawned the host's driver"); + restated.AlarmNodeId.ShouldBe(ExpectedRawPath); + restated.State.Active.ShouldBeTrue(); + if (newAlarmDuringRespawn) { - var restated = publish.FishForMessage(_ => true, TimeSpan.FromSeconds(5)); - restated.AlarmNodeId.ShouldBe(ExpectedRawPath); - restated.State.Active.ShouldBeTrue(); - if (newAlarmDuringRespawn) - { - restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack"); - alerts.ExpectMsg(TimeSpan.FromSeconds(5)); - } - else - { - restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge"); - alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); - } + restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack"); + alerts.ExpectMsg(TimeSpan.FromSeconds(5)); } - finally + else { - await respawned.ShutdownAsync(CancellationToken.None); + restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge"); + alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); } } - private static async Task StartDriverAsync(List rawTags, IFocasClient cnc, IActorRef host) + /// + /// The host's driver factory for the respawn test: every spawn gets a fresh over the + /// next fake CNC (the last one repeats), fed the raw tags from the merged DriverConfig the host hands it — as + /// production does. + /// + private sealed class HostFocasDriverFactory(params IFocasClient[] cncs) : IDriverFactory { - var driver = new FocasDriver(new FocasDriverOptions + private int _created; + + /// How many driver instances the host has created. + public int Created => Volatile.Read(ref _created); + + /// + public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) { - Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")], - RawTags = rawTags, - Probe = new FocasProbeOptions { Enabled = false }, - AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) }, - }, DriverId, new SingleClientFactory(cnc)); - driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e)); - await driver.InitializeAsync("{}", CancellationToken.None); -#pragma warning disable OTOPCUA0001 // the routing is under test, not resilience wrapping - await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None); -#pragma warning restore OTOPCUA0001 - return driver; + if (driverType != "FOCAS") return null; + var cnc = cncs[Math.Min(Interlocked.Increment(ref _created), cncs.Length) - 1]; + using var doc = JsonDocument.Parse(driverConfigJson); + var rawTags = JsonSerializer.Deserialize>( + doc.RootElement.GetProperty("RawTags").GetRawText(), + new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; + return new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")], + RawTags = rawTags, + Probe = new FocasProbeOptions { Enabled = false }, + AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) }, + }, driverInstanceId, new SingleClientFactory(cnc)); + } + + /// + public IReadOnlyCollection SupportedTypes => ["FOCAS"]; } /// The seeded artifact's FOCAS driver raw tags, decoded through the production artifact reader. @@ -193,13 +205,14 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; } - private static byte[] BuildArtifact() => JsonSerializer.SerializeToUtf8Bytes(new + private static byte[] BuildArtifact(bool enabled = false, string driverConfig = "{}") => JsonSerializer.SerializeToUtf8Bytes(new { RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } }, DriverInstances = new[] { - // Enabled=false: the host must not spawn a real wire driver — the test drives its own instance. - new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = "{}", ClusterId = "c1", Enabled = false }, + // Enabled=false (default): the host spawns no driver — the test drives its own instance. Enabled=true + // spawns the host's own driver through the test's factory (never a real wire driver). + new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = driverConfig, ClusterId = "c1", Enabled = enabled }, }, Devices = new[] { @@ -244,12 +257,13 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase } private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply( - IDbContextFactory db, DeploymentId deploymentId) + IDbContextFactory db, DeploymentId deploymentId, IDriverFactory? driverFactory = null) { var coordinator = CreateTestProbe(); var publish = this.CreateCleanOutcomePublishProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, + driverFactory: driverFactory, localRoles: new HashSet { "driver" }, dependencyMux: CreateTestProbe().Ref, opcUaPublishActor: publish.Ref,