fix(alarms): revert a native ack that reaches no driver; refuse native acks while the host is Stale (lmxopcua#765)
Residuals of #760: 1. An ack routed to a disabled / not-yet-spawned driver, or to a stubbed child (driver does not run on this node, or its factory rejected the config), was dropped and left the condition Acknowledged. DriverHostActor now reverts it through the same NativeAlarmProjector.TryRevertAcknowledge path as a refused ack. 2. A native ack arriving while the host is Stale was dropped before the projector recorded it, so it could never be reverted. Stale now marks the whole NativeAlarmAckPreflightView unavailable (SetHostUnavailable), so the node manager refuses every native ack synchronously; BecomeSteady clears it. 3. (keyed sessions without a Galaxy identity) is not addressed. Tests: DriverHostActorNativeAlarmTests' client-ack test and the FOCAS respawn theory now start the host's own driver (AckingAlarmDriver; a factory building a real FocasDriver per spawn over per-instance fake CNCs, with a real config-change respawn) - both fail against the fix in their old shape. New: ack to a disabled driver reverts, ack to a stubbed driver reverts, a Stale host refuses through the pre-check view - all three fail with the fix mutated out. Runtime.Tests 943/0/3. Docs: AlarmTracking.md, .claude/rules/alarms.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -36,9 +36,12 @@ driver can decide without I/O (`IAlarmAcknowledgePreflight` — Galaxy: anonymou
|
||||
account) fails the Part 9 call `BadUserAccessDenied` via `NativeAlarmAckPreflight`, read lock-free from the
|
||||
`NativeAlarmAckPreflightView` snapshot `DriverHostActor` publishes; never do I/O (e.g. a Secrets lookup) in a
|
||||
pre-check — it runs under the SDK `Lock`. (2) anything else (keyed session without an identity, MXAccess
|
||||
non-zero `hresult`, driver still connecting/reconnecting; a disabled/unspawned driver is NOT reverted) comes back as `DriverInstanceActor.AlarmAckFailed` and
|
||||
`DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in
|
||||
`NativeAlarmProjector` (any driver transition since then wins).
|
||||
non-zero `hresult`, driver still connecting/reconnecting) comes back as `DriverInstanceActor.AlarmAckFailed` — or, for a
|
||||
disabled/unspawned/stubbed driver, is decided in the host itself (#765) — and `DriverHostActor` re-projects the condition unacknowledged — only while the ack is still pending in
|
||||
`NativeAlarmProjector` (any driver transition since then wins). A host that came up **Stale** cannot route any
|
||||
native ack, so it marks the whole `NativeAlarmAckPreflightView` unavailable and every native ack is refused
|
||||
synchronously until `BecomeSteady` (#765). A test that routes an ack and expects it to STAND must start the owning
|
||||
driver (`AckingAlarmDriver` in Runtime.Tests).
|
||||
|
||||
Behaviours that were wrong or missing and are now fixed — keep them: the SDK's `OnTimedUnshelve` is a
|
||||
**no-op** (the engine's `AutoUnshelve` owns timed-shelve expiry, and `UnshelveTime` reports the real
|
||||
|
||||
@@ -464,12 +464,16 @@ from the scripted `AlarmCommandRouter`:
|
||||
driver refuses or fails the ack (or is still connecting / reconnecting),
|
||||
`DriverInstanceActor` replies `AlarmAckFailed` and
|
||||
`DriverHostActor` re-projects the condition **unacknowledged** (a fresh
|
||||
condition event). Only an ack still pending in `NativeAlarmProjector` is
|
||||
condition event). An ack whose owning driver is not running (disabled,
|
||||
not yet spawned) or is a stub on this node is reverted the same way by
|
||||
the host itself (#765). Only an ack still pending in `NativeAlarmProjector` is
|
||||
reverted: a driver transition projected since then is authoritative. A
|
||||
refusal the driver can decide without I/O (Galaxy: an anonymous session
|
||||
and no fallback ack account) is instead returned synchronously as
|
||||
`BadUserAccessDenied` by the node manager's `NativeAlarmAckPreflight`
|
||||
before anything is applied. Galaxy value writes have a separate correlated
|
||||
before anything is applied — as is every native ack while the driver host
|
||||
is Stale (configuration database unreachable at boot), when no ack can be
|
||||
routed or reverted (#765). Galaxy value writes have a separate correlated
|
||||
write-outcome path. An awaited
|
||||
ack RPC does not prove the ack reached AVEVA event history: the H6 live
|
||||
verification remains [operator-gated](plans/2026-06-15-stillpending-phase-3-opcua-standards-design.md).
|
||||
|
||||
@@ -1528,6 +1528,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// Cancelled BEFORE the unstash so a re-stash by the very next apply starts a fresh window.
|
||||
Timers.Cancel(ApplyWedgeTimerKey);
|
||||
_deferredDispatches.Clear();
|
||||
// Leaving Stale (or never in it): native acks are routable again (lmxopcua#765).
|
||||
_ackPreflightView?.SetHostUnavailable(null);
|
||||
Become(Steady);
|
||||
Stash.UnstashAll();
|
||||
}
|
||||
@@ -2325,12 +2327,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// deploy rather than resetting to its acked default.
|
||||
_nativeAlarmProjector.MarkAcknowledged(msg.ConditionNodeId);
|
||||
|
||||
if (!_children.TryGetValue(target.DriverInstanceId, out var entry))
|
||||
if (!_children.TryGetValue(target.DriverInstanceId, out var entry) || entry.Stubbed)
|
||||
{
|
||||
_log.Debug("DriverHost {Node}: driver {Driver} not running for alarm ack of {Cond} — dropped",
|
||||
_localNode, target.DriverInstanceId, msg.ConditionNodeId);
|
||||
// Deliberately NOT reverted (lmxopcua#760 covers what a driver reports): a disabled or not-yet-spawned
|
||||
// driver produces no transitions either, so the local ack stands, as before.
|
||||
// A disabled / not-yet-spawned driver, or a stub (the driver does not run on this node, or its factory
|
||||
// rejected the config), can deliver the ack nowhere. The node manager already applied it, so revert it
|
||||
// like any other ack that never reached the alarm system (lmxopcua#765) — otherwise the node keeps
|
||||
// claiming an acknowledgement the upstream system never saw.
|
||||
var reason = entry is null ? "the owning driver is not running" : "the owning driver is stubbed on this node";
|
||||
_log.Debug("DriverHost {Node}: driver {Driver} cannot take the alarm ack of {Cond} ({Reason}) — dropped",
|
||||
_localNode, target.DriverInstanceId, msg.ConditionNodeId, reason);
|
||||
RevertNativeAlarmAck(msg.ConditionNodeId, reason);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2568,6 +2574,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
|
||||
private void Stale()
|
||||
{
|
||||
// lmxopcua#765: a native ack routed here while Stale is dropped before the projector records it, so it
|
||||
// could not be reverted and the node would show Acknowledged for an ack no driver saw. Refuse it
|
||||
// synchronously at the node manager instead (cleared by BecomeSteady, the only way out of Stale).
|
||||
_ackPreflightView?.SetHostUnavailable("the driver host is stale (configuration database unreachable)");
|
||||
Receive<DispatchDeployment>(_ =>
|
||||
{
|
||||
_log.Warning("DriverHost {Node}: ignoring DispatchDeployment while Stale (DB unreachable)", _localNode);
|
||||
@@ -2602,8 +2612,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Receive<RouteNodeWrite>(_ =>
|
||||
Sender.Tell(new NodeWriteResult(false, "driver host stale (config DB unreachable)")));
|
||||
// An inbound native-condition ack can't be serviced while Stale either (the alarm inverse map is
|
||||
// empty until an apply runs). The ack is fire-and-forget (no reply), so just drop it with a log —
|
||||
// the local OPC UA condition state already committed on the Part 9 ack.
|
||||
// empty until an apply runs). The node manager's pre-check refuses it synchronously while Stale (see the
|
||||
// top of this method); one that raced that refusal is dropped here with a log.
|
||||
Receive<RouteNativeAlarmAck>(msg =>
|
||||
_log.Debug("DriverHost {Node}: dropping native-alarm ack for {Node2} while Stale (config DB unreachable)",
|
||||
_localNode, msg.ConditionNodeId));
|
||||
|
||||
@@ -19,6 +19,8 @@ public sealed class NativeAlarmAckPreflightView
|
||||
private volatile ImmutableDictionary<string, IAlarmAcknowledgePreflight> _byConditionNodeId =
|
||||
ImmutableDictionary<string, IAlarmAcknowledgePreflight>.Empty.WithComparers(StringComparer.Ordinal);
|
||||
|
||||
private volatile string? _hostUnavailableReason;
|
||||
|
||||
/// <summary>
|
||||
/// Returns why the driver owning <paramref name="conditionNodeId"/> would refuse an acknowledge from a
|
||||
/// session authenticated with <paramref name="apiKeyId"/>, or null to proceed (unknown condition, a
|
||||
@@ -30,6 +32,8 @@ public sealed class NativeAlarmAckPreflightView
|
||||
public string? Check(string conditionNodeId, string? apiKeyId)
|
||||
{
|
||||
if (string.IsNullOrEmpty(conditionNodeId)) return null;
|
||||
// A host that cannot route any native ack (Stale — lmxopcua#765) refuses every one, whatever its driver.
|
||||
if (_hostUnavailableReason is { } unavailable) return unavailable;
|
||||
if (!_byConditionNodeId.TryGetValue(conditionNodeId, out var preflight)) return null;
|
||||
try
|
||||
{
|
||||
@@ -43,6 +47,14 @@ public sealed class NativeAlarmAckPreflightView
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the whole driver host unable to route a native acknowledge (lmxopcua#765): while set,
|
||||
/// <see cref="Check"/> refuses every condition with <paramref name="reason"/>. Null clears it. Called by
|
||||
/// <see cref="DriverHostActor"/> only — set on entering <c>Stale</c>, cleared on returning to <c>Steady</c>.
|
||||
/// </summary>
|
||||
/// <param name="reason">Why no native ack can be routed, or null when acks are routable.</param>
|
||||
internal void SetHostUnavailable(string? reason) => _hostUnavailableReason = reason;
|
||||
|
||||
/// <summary>Replaces the snapshot. Called by <see cref="DriverHostActor"/> only.</summary>
|
||||
/// <param name="byConditionNodeId">Condition NodeId → the owning driver's pre-check.</param>
|
||||
internal void Publish(IEnumerable<KeyValuePair<string, IAlarmAcknowledgePreflight>> byConditionNodeId) =>
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// A minimal <see cref="IDriver"/> + <see cref="IAlarmSource"/> that accepts every acknowledge and raises
|
||||
/// nothing — the host's own running driver for tests that route a client acknowledge but drive the alarm
|
||||
/// transitions themselves. Since lmxopcua#765 an ack to a driver that is not running (or is stubbed) is
|
||||
/// reverted, so such a test must start the owning driver for the ack to stand.
|
||||
/// </summary>
|
||||
internal sealed class AckingAlarmDriver : IDriver, IAlarmSource
|
||||
{
|
||||
private int _acks;
|
||||
private bool _alarmsSubscribed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string DriverInstanceId { get; private set; } = string.Empty;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string DriverType { get; private set; } = string.Empty;
|
||||
|
||||
/// <summary>How many acknowledge requests the driver accepted.</summary>
|
||||
public int Acks => Volatile.Read(ref _acks);
|
||||
|
||||
/// <summary>True once the driver child reached Connected and opened its alarm subscription.</summary>
|
||||
public bool AlarmsSubscribed => Volatile.Read(ref _alarmsSubscribed);
|
||||
|
||||
/// <summary>Sets the identity once the factory is asked to create it.</summary>
|
||||
/// <param name="id">The driver instance id.</param>
|
||||
/// <param name="type">The driver type.</param>
|
||||
public void Bind(string id, string type)
|
||||
{
|
||||
DriverInstanceId = id;
|
||||
DriverType = type;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
|
||||
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken)
|
||||
{
|
||||
Volatile.Write(ref _alarmsSubscribed, true);
|
||||
return Task.FromResult<IAlarmSubscriptionHandle>(new Handle());
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) =>
|
||||
Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task AcknowledgeAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Add(ref _acks, acknowledgements.Count);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public event EventHandler<AlarmEventArgs>? OnAlarmEvent
|
||||
{
|
||||
add { }
|
||||
remove { }
|
||||
}
|
||||
|
||||
private sealed class Handle : IAlarmSubscriptionHandle
|
||||
{
|
||||
public string DiagnosticId => "acking-alarm-sub";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Factory handing the host one shared <see cref="AckingAlarmDriver"/> for a single driver type.</summary>
|
||||
/// <param name="supportedType">The driver type this factory creates.</param>
|
||||
internal sealed class AckingAlarmDriverFactory(string supportedType) : IDriverFactory
|
||||
{
|
||||
/// <summary>The driver every spawn of <paramref name="supportedType"/> receives.</summary>
|
||||
public AckingAlarmDriver Driver { get; } = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
||||
{
|
||||
if (!string.Equals(driverType, supportedType, StringComparison.Ordinal)) return null;
|
||||
Driver.Bind(driverInstanceId, driverType);
|
||||
return Driver;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> SupportedTypes => [supportedType];
|
||||
}
|
||||
+76
-4
@@ -247,6 +247,77 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
view.Check("Plant/gw/dev1/does-not-exist", apiKeyId: null).ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#765 — an ack routed to a driver that is not running (disabled here) reaches no alarm system, so
|
||||
/// the host reverts the condition the node manager optimistically acknowledged, as for a refused ack.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_ack_to_a_driver_that_is_not_running_reverts_the_condition()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi", enabled: false);
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref, awaitConnected: false);
|
||||
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
|
||||
var reverted = NextAlarmUpdate(publish);
|
||||
reverted.AlarmNodeId.ShouldBe(AlarmRawPath);
|
||||
reverted.State.Active.ShouldBeTrue();
|
||||
reverted.State.Acknowledged.ShouldBeFalse();
|
||||
recorder.Acks.ShouldBeEmpty("no driver was running to take the ack");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#765 — likewise a STUBBED child (here: no factory for the driver type, so the host falls back to
|
||||
/// a stub that has no alarm backend) silently swallows the ack; the host reverts it rather than routing it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_ack_to_a_stubbed_driver_reverts_the_condition()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var noFactoryForType = new RecordingAlarmDriverFactory("SomeOtherType");
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
var actor = SpawnHostAndApply(db, deploymentId, noFactoryForType, publish.Ref, awaitConnected: false);
|
||||
|
||||
RaiseAndExpectUnacked(actor, publish);
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
|
||||
|
||||
NextAlarmUpdate(publish).State.Acknowledged.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#765 — a host that came up Stale (config DB unreachable) drops every native ack before the
|
||||
/// projector records it, so it could not be reverted. The pre-check view refuses every native ack while the
|
||||
/// host is Stale, so the node manager answers synchronously instead of applying an ack nothing will route.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_stale_host_refuses_every_native_ack_through_the_preflight_view()
|
||||
{
|
||||
var view = new NativeAlarmAckPreflightView();
|
||||
view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldBeNull("positive control: nothing refuses before the host starts");
|
||||
|
||||
var host = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
ackPreflightView: view));
|
||||
// Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale.
|
||||
var asker = CreateTestProbe();
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout).Reason!.ShouldContain("stale");
|
||||
|
||||
view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldNotBeNull().ShouldContain("stale");
|
||||
view.Check("any/other/condition", apiKeyId: null).ShouldNotBeNull();
|
||||
}
|
||||
|
||||
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable");
|
||||
}
|
||||
|
||||
private static AlarmEventArgs Evt(AlarmTransitionKind kind) => new(
|
||||
new StubAlarmHandleForTests(), SourceNodeId: "Temp", ConditionId: AlarmRawPath, AlarmType: "OffNormalAlarm",
|
||||
Message: "temperature high", Severity: AlarmSeverity.High, SourceTimestampUtc: DateTime.UtcNow, Kind: kind);
|
||||
@@ -272,7 +343,8 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
/// completed before the test routes an ack.</summary>
|
||||
private IActorRef SpawnHostAndApply(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory,
|
||||
IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null)
|
||||
IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null,
|
||||
bool awaitConnected = true)
|
||||
{
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
@@ -291,7 +363,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
// the control itself was dropped and the wait read [] — 4 of 20 runs under thread-pool starvation,
|
||||
// and on Gitea (#606). SubscribeAlarmsAsync is only ever called from the Connected state (a
|
||||
// pre-connect request is swallowed and re-issued on the Connected entry), so wait for it.
|
||||
if (factory is RecordingAlarmDriverFactory recording)
|
||||
if (awaitConnected && factory is RecordingAlarmDriverFactory recording)
|
||||
AwaitCondition(() => recording.AlarmsSubscribed, Timeout);
|
||||
|
||||
return actor;
|
||||
@@ -305,7 +377,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
/// <c>DriverType</c> + Enabled flag spawn a REAL <see cref="DriverInstanceActor"/> child.
|
||||
/// </summary>
|
||||
private static DeploymentId SeedV3AlarmDeployment(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, string Driver, string Tag)
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, string Driver, string Tag, bool enabled = true)
|
||||
{
|
||||
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
@@ -319,7 +391,7 @@ public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTest
|
||||
RawFolderId = "rf-plant",
|
||||
Name = "gw",
|
||||
DriverType = "GalaxyMxGateway", // not Windows-only ⇒ a real child is spawned (not stubbed)
|
||||
Enabled = true,
|
||||
Enabled = enabled,
|
||||
DriverConfig = "{}",
|
||||
ClusterId = "c1",
|
||||
},
|
||||
|
||||
+21
-10
@@ -479,20 +479,29 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
|
||||
/// after one defaulted Acked to true and happened to be right for a client-acked alarm; now that the
|
||||
/// projector survives the deploy, a projector that never heard the ack projects the Clear as UNacked and
|
||||
/// un-acknowledges the alarm the operator acknowledged.
|
||||
/// <para>
|
||||
/// lmxopcua#765 — the host's own driver is started here: an ack routed to a driver that is not running
|
||||
/// is now reverted (it reached no alarm system), so only a delivered ack may stand.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_clear_after_a_client_acknowledge_and_a_redeploy_stays_acknowledged()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
|
||||
var (actor, publish, coordinator) = SpawnHostAndApplyWithCoordinator(db, deploymentId);
|
||||
var factory = new AckingAlarmDriverFactory("Modbus");
|
||||
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi", enabled: true);
|
||||
var (actor, publish, coordinator) = SpawnHostAndApplyWithCoordinator(db, deploymentId, factory);
|
||||
// An ack reaching a still-Connecting child is dropped (and now reverted); wait for Connected.
|
||||
AwaitCondition(() => factory.Driver.AlarmsSubscribed, TimeSpan.FromSeconds(5));
|
||||
|
||||
actor.Tell(Transition(AlarmTransitionKind.Raise));
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(TimeSpan.FromSeconds(5)).State.Acknowledged.ShouldBeFalse();
|
||||
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5))
|
||||
.State.Acknowledged.ShouldBeFalse();
|
||||
|
||||
// The client's Acknowledge on the node, routed toward the driver (no transition comes back for it).
|
||||
// The client's Acknowledge on the node, routed to the driver (no transition comes back for it).
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "seen", "op"));
|
||||
Redeploy(db, actor, publish, coordinator);
|
||||
AwaitCondition(() => factory.Driver.Acks == 1, TimeSpan.FromSeconds(5));
|
||||
Redeploy(db, actor, publish, coordinator, enabled: true);
|
||||
|
||||
actor.Tell(Transition(AlarmTransitionKind.Clear));
|
||||
var cleared = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
|
||||
@@ -507,10 +516,11 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
|
||||
|
||||
/// <summary>A second deployment carrying the SAME alarm (a new revision of an identical artifact).</summary>
|
||||
private void Redeploy(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, IActorRef actor, TestProbe publish, TestProbe coordinator)
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, IActorRef actor, TestProbe publish, TestProbe coordinator,
|
||||
bool enabled = false)
|
||||
{
|
||||
var revB = RevisionHash.Parse(new string('b', 64));
|
||||
var second = SeedV3AlarmDeployment(db, revB, Driver: "drv-1", Tag: "temp_hi");
|
||||
var second = SeedV3AlarmDeployment(db, revB, Driver: "drv-1", Tag: "temp_hi", enabled: enabled);
|
||||
actor.Tell(new DispatchDeployment(second, revB, CorrelationId.NewId()));
|
||||
coordinator.FishForMessage<ApplyAck>(a => a.DeploymentId == second, TimeSpan.FromSeconds(5))
|
||||
.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
@@ -553,7 +563,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
|
||||
}
|
||||
|
||||
private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId)
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory? driverFactory = null)
|
||||
{
|
||||
var coordinator = CreateTestProbe();
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
@@ -562,6 +572,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
driverFactory: driverFactory,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
dependencyMux: mux.Ref,
|
||||
opcUaPublishActor: publish.Ref,
|
||||
@@ -588,7 +599,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
|
||||
/// </summary>
|
||||
private static DeploymentId SeedV3AlarmDeployment(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev,
|
||||
string Driver, string Tag, bool? historizeToAveva = null)
|
||||
string Driver, string Tag, bool? historizeToAveva = null, bool enabled = false)
|
||||
{
|
||||
// historizeToAveva absent (null) ⇒ omit the key so the absent ⇒ historize default path is exercised;
|
||||
// a concrete true/false writes the bool into the alarm object so the native path threads it.
|
||||
@@ -601,7 +612,7 @@ public sealed class DriverHostActorNativeAlarmTests : RuntimeActorTestBase
|
||||
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
||||
DriverInstances = new[]
|
||||
{
|
||||
new { DriverInstanceId = Driver, RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = false },
|
||||
new { DriverInstanceId = Driver, RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled },
|
||||
},
|
||||
Devices = new[]
|
||||
{
|
||||
|
||||
+75
-61
@@ -103,83 +103,95 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase
|
||||
/// servo alarm, so the condition's displayed set changed) is a new statement even though the fresh driver
|
||||
/// can only say Raise. It must un-acknowledge and write its alerts row.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// lmxopcua#765 — the host spawns, subscribes and respawns its OWN FOCAS driver here (a factory over the fake
|
||||
/// CNC), so the operator's ack is delivered to a running driver. An ack to a driver that is not running is
|
||||
/// reverted, so the old shape — the host's driver disabled, the test driving a detached instance — could no
|
||||
/// longer keep the ack. The respawn is a real one: the second deployment changes the driver's config.
|
||||
/// </para>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn)
|
||||
public void A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn)
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var blob = BuildArtifact();
|
||||
var blob = BuildArtifact(enabled: true);
|
||||
var alerts = CreateTestProbe();
|
||||
DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref);
|
||||
alerts.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
|
||||
var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev));
|
||||
var rawTags = RawTagsFor(blob, DriverId);
|
||||
var cnc = new AlarmOnlyFocasClient();
|
||||
cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
|
||||
|
||||
var first = await StartDriverAsync(rawTags, cnc, host);
|
||||
try
|
||||
{
|
||||
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5))
|
||||
.State.Acknowledged.ShouldBeFalse();
|
||||
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
finally
|
||||
{
|
||||
await first.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
// The operator acknowledges, then a deploy changes the driver's config => stop + respawn.
|
||||
host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op"));
|
||||
var revB = RevisionHash.Parse(new string('e', 64));
|
||||
host.Tell(new DispatchDeployment(Seed(db, blob, revB), revB, CorrelationId.NewId()));
|
||||
coordinator.FishForMessage<ApplyAck>(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5));
|
||||
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, TimeSpan.FromSeconds(5));
|
||||
|
||||
// One fake CNC per driver instance, so what the fresh instance reads is fixed before it can poll.
|
||||
var firstCnc = new AlarmOnlyFocasClient();
|
||||
firstCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
|
||||
var respawnCnc = new AlarmOnlyFocasClient();
|
||||
if (newAlarmDuringRespawn)
|
||||
cnc.SetAlarms(
|
||||
respawnCnc.SetAlarms(
|
||||
new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"),
|
||||
new FocasActiveAlarm(410, FocasAlarmType.Servo, 2, "SERVO EXCESS ERROR"));
|
||||
else
|
||||
respawnCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
|
||||
var factory = new HostFocasDriverFactory(firstCnc, respawnCnc);
|
||||
var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev), factory);
|
||||
|
||||
var respawned = await StartDriverAsync(rawTags, cnc, host);
|
||||
try
|
||||
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5))
|
||||
.State.Acknowledged.ShouldBeFalse();
|
||||
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
|
||||
|
||||
// The operator acknowledges (the running driver takes it), then a deploy changes the driver's config =>
|
||||
// the host stops it and spawns a fresh instance.
|
||||
host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op"));
|
||||
var revB = RevisionHash.Parse(new string('e', 64));
|
||||
var changedConfig = BuildArtifact(enabled: true, driverConfig: "{\"respawn\":1}");
|
||||
host.Tell(new DispatchDeployment(Seed(db, changedConfig, revB), revB, CorrelationId.NewId()));
|
||||
coordinator.FishForMessage<ApplyAck>(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5));
|
||||
|
||||
var restated = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
|
||||
factory.Created.ShouldBe(2, "the config change must have respawned the host's driver");
|
||||
restated.AlarmNodeId.ShouldBe(ExpectedRawPath);
|
||||
restated.State.Active.ShouldBeTrue();
|
||||
if (newAlarmDuringRespawn)
|
||||
{
|
||||
var restated = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
|
||||
restated.AlarmNodeId.ShouldBe(ExpectedRawPath);
|
||||
restated.State.Active.ShouldBeTrue();
|
||||
if (newAlarmDuringRespawn)
|
||||
{
|
||||
restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack");
|
||||
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
else
|
||||
{
|
||||
restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge");
|
||||
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack");
|
||||
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
finally
|
||||
else
|
||||
{
|
||||
await respawned.ShutdownAsync(CancellationToken.None);
|
||||
restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge");
|
||||
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<FocasDriver> StartDriverAsync(List<RawTagEntry> rawTags, IFocasClient cnc, IActorRef host)
|
||||
/// <summary>
|
||||
/// The host's driver factory for the respawn test: every spawn gets a fresh <see cref="FocasDriver"/> over the
|
||||
/// next fake CNC (the last one repeats), fed the raw tags from the merged DriverConfig the host hands it — as
|
||||
/// production does.
|
||||
/// </summary>
|
||||
private sealed class HostFocasDriverFactory(params IFocasClient[] cncs) : IDriverFactory
|
||||
{
|
||||
var driver = new FocasDriver(new FocasDriverOptions
|
||||
private int _created;
|
||||
|
||||
/// <summary>How many driver instances the host has created.</summary>
|
||||
public int Created => Volatile.Read(ref _created);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
||||
{
|
||||
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
|
||||
RawTags = rawTags,
|
||||
Probe = new FocasProbeOptions { Enabled = false },
|
||||
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
|
||||
}, DriverId, new SingleClientFactory(cnc));
|
||||
driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e));
|
||||
await driver.InitializeAsync("{}", CancellationToken.None);
|
||||
#pragma warning disable OTOPCUA0001 // the routing is under test, not resilience wrapping
|
||||
await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None);
|
||||
#pragma warning restore OTOPCUA0001
|
||||
return driver;
|
||||
if (driverType != "FOCAS") return null;
|
||||
var cnc = cncs[Math.Min(Interlocked.Increment(ref _created), cncs.Length) - 1];
|
||||
using var doc = JsonDocument.Parse(driverConfigJson);
|
||||
var rawTags = JsonSerializer.Deserialize<List<RawTagEntry>>(
|
||||
doc.RootElement.GetProperty("RawTags").GetRawText(),
|
||||
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
|
||||
return new FocasDriver(new FocasDriverOptions
|
||||
{
|
||||
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
|
||||
RawTags = rawTags,
|
||||
Probe = new FocasProbeOptions { Enabled = false },
|
||||
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
|
||||
}, driverInstanceId, new SingleClientFactory(cnc));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> SupportedTypes => ["FOCAS"];
|
||||
}
|
||||
|
||||
/// <summary>The seeded artifact's FOCAS driver raw tags, decoded through the production artifact reader.</summary>
|
||||
@@ -193,13 +205,14 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase
|
||||
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
|
||||
}
|
||||
|
||||
private static byte[] BuildArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
|
||||
private static byte[] BuildArtifact(bool enabled = false, string driverConfig = "{}") => JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
||||
DriverInstances = new[]
|
||||
{
|
||||
// Enabled=false: the host must not spawn a real wire driver — the test drives its own instance.
|
||||
new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = "{}", ClusterId = "c1", Enabled = false },
|
||||
// Enabled=false (default): the host spawns no driver — the test drives its own instance. Enabled=true
|
||||
// spawns the host's own driver through the test's factory (never a real wire driver).
|
||||
new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = driverConfig, ClusterId = "c1", Enabled = enabled },
|
||||
},
|
||||
Devices = new[]
|
||||
{
|
||||
@@ -244,12 +257,13 @@ public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase
|
||||
}
|
||||
|
||||
private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId)
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory? driverFactory = null)
|
||||
{
|
||||
var coordinator = CreateTestProbe();
|
||||
var publish = this.CreateCleanOutcomePublishProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
driverFactory: driverFactory,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
dependencyMux: CreateTestProbe().Ref,
|
||||
opcUaPublishActor: publish.Ref,
|
||||
|
||||
Reference in New Issue
Block a user