Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs
T
Joseph Doherty f54922f94c fix(alarms): revert a native ack that reaches no driver; refuse native acks while the host is Stale (lmxopcua#765)
Residuals of #760:

1. An ack routed to a disabled / not-yet-spawned driver, or to a stubbed child
   (driver does not run on this node, or its factory rejected the config), was
   dropped and left the condition Acknowledged. DriverHostActor now reverts it
   through the same NativeAlarmProjector.TryRevertAcknowledge path as a refused
   ack.

2. A native ack arriving while the host is Stale was dropped before the projector
   recorded it, so it could never be reverted. Stale now marks the whole
   NativeAlarmAckPreflightView unavailable (SetHostUnavailable), so the node
   manager refuses every native ack synchronously; BecomeSteady clears it.

3. (keyed sessions without a Galaxy identity) is not addressed.

Tests: DriverHostActorNativeAlarmTests' client-ack test and the FOCAS respawn
theory now start the host's own driver (AckingAlarmDriver; a factory building a
real FocasDriver per spawn over per-instance fake CNCs, with a real config-change
respawn) - both fail against the fix in their old shape. New: ack to a disabled
driver reverts, ack to a stubbed driver reverts, a Stale host refuses through the
pre-check view - all three fail with the fix mutated out. Runtime.Tests 943/0/3.
Docs: AlarmTracking.md, .claude/rules/alarms.md.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 14:03:59 -05:00

331 lines
17 KiB
C#

using System.Text.Json;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.TestKit;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Gitea #647 — a FOCAS CNC alarm reaches its Part 9 condition END TO END: the REAL <see cref="FocasDriver"/>
/// (over a fake CNC client), fed the raw tags exactly as the deployment artifact delivers them, raises into the
/// REAL <see cref="DriverHostActor"/>, whose <c>ForwardNativeAlarm</c> + <c>NativeAlarmProjector</c> must turn it
/// into an Active and then an inactive condition at the tag's RawPath.
/// </summary>
/// <remarks>
/// The lesson this exists for: OpcUaClient and AbCip native alarms were "routed" — events raised, unit tests
/// green — but never reached a condition, because nothing checked the event against the key the host actually
/// routes on (ConditionId == RawPath) or the field its projector actually reads (Kind). A test that stops at the
/// driver's event cannot see either.
/// </remarks>
public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("focas-alarm-e2e");
private static readonly RevisionHash Rev = RevisionHash.Parse(new string('f', 64));
private const string Host = "focas://10.0.0.5:8193";
private const string DriverId = "drv-focas";
private const string ExpectedRawPath = "Plant/Cnc/m1/servo_alarms";
[Fact]
public async Task A_CNC_servo_alarm_activates_then_clears_the_condition_at_the_RawPath()
{
var db = NewInMemoryDbFactory();
var blob = BuildArtifact();
var deploymentId = Seed(db, blob, Rev);
var (host, publish, _) = SpawnHostAndApply(db, deploymentId);
// The raw tags exactly as the driver receives them: the artifact's merged DriverConfig RawTags array.
var rawTags = RawTagsFor(blob, DriverId);
rawTags.Select(t => t.RawPath).ShouldBe([ExpectedRawPath]);
var cnc = new AlarmOnlyFocasClient();
var driver = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
RawTags = rawTags,
Probe = new FocasProbeOptions { Enabled = false },
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
}, DriverId, new SingleClientFactory(cnc));
driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e));
await driver.InitializeAsync("{}", CancellationToken.None);
// The host's alarm subscription set is the alarm-bearing raw tags' RawPaths. Called directly, as the
// DriverInstanceActor would through its invoker: the routing is under test, not resilience wrapping.
#pragma warning disable OTOPCUA0001
await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None);
#pragma warning restore OTOPCUA0001
try
{
cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
var active = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
active.AlarmNodeId.ShouldBe(ExpectedRawPath);
active.Realm.ShouldBe(AddressSpaceRealm.Raw);
active.State.Active.ShouldBeTrue("the Raise must project to an ACTIVE condition, not 'keep prior state'");
active.State.Acknowledged.ShouldBeFalse();
active.State.Severity.ShouldBe((ushort)800, "the authored alarm.severity is the condition's severity");
active.State.Message.ShouldStartWith("401 SERVO V-READY OFF");
cnc.SetAlarms();
var cleared = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
cleared.AlarmNodeId.ShouldBe(ExpectedRawPath);
cleared.State.Active.ShouldBeFalse();
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
/// <summary>
/// Gitea #647 review (MAJOR-1) — a deploy that changes ANY of the driver's config respawns it, and the fresh
/// instance has no record of what the server was told. Its first poll re-reads the still-active alarm and
/// sends Raise; the host must treat that as a re-statement: the operator's acknowledgement survives and no
/// second "Activated" alerts row is written. Driven through the real host with a FRESH driver instance.
/// </summary>
/// <para>
/// Re-review R1 — the other half: a DIFFERENT alarm that appeared while the driver was down (here a second
/// servo alarm, so the condition's displayed set changed) is a new statement even though the fresh driver
/// can only say Raise. It must un-acknowledge and write its alerts row.
/// </para>
/// <para>
/// lmxopcua#765 — the host spawns, subscribes and respawns its OWN FOCAS driver here (a factory over the fake
/// CNC), so the operator's ack is delivered to a running driver. An ack to a driver that is not running is
/// reverted, so the old shape — the host's driver disabled, the test driving a detached instance — could no
/// longer keep the ack. The respawn is a real one: the second deployment changes the driver's config.
/// </para>
[Theory]
[InlineData(false)]
[InlineData(true)]
public void A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack(bool newAlarmDuringRespawn)
{
var db = NewInMemoryDbFactory();
var blob = BuildArtifact(enabled: true);
var alerts = CreateTestProbe();
DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref);
alerts.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
// One fake CNC per driver instance, so what the fresh instance reads is fixed before it can poll.
var firstCnc = new AlarmOnlyFocasClient();
firstCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
var respawnCnc = new AlarmOnlyFocasClient();
if (newAlarmDuringRespawn)
respawnCnc.SetAlarms(
new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"),
new FocasActiveAlarm(410, FocasAlarmType.Servo, 2, "SERVO EXCESS ERROR"));
else
respawnCnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
var factory = new HostFocasDriverFactory(firstCnc, respawnCnc);
var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev), factory);
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5))
.State.Acknowledged.ShouldBeFalse();
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
// The operator acknowledges (the running driver takes it), then a deploy changes the driver's config =>
// the host stops it and spawns a fresh instance.
host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op"));
var revB = RevisionHash.Parse(new string('e', 64));
var changedConfig = BuildArtifact(enabled: true, driverConfig: "{\"respawn\":1}");
host.Tell(new DispatchDeployment(Seed(db, changedConfig, revB), revB, CorrelationId.NewId()));
coordinator.FishForMessage<ApplyAck>(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5));
var restated = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
factory.Created.ShouldBe(2, "the config change must have respawned the host's driver");
restated.AlarmNodeId.ShouldBe(ExpectedRawPath);
restated.State.Active.ShouldBeTrue();
if (newAlarmDuringRespawn)
{
restated.State.Acknowledged.ShouldBeFalse("an alarm the operator has not seen must not inherit the old ack");
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
}
else
{
restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge");
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
}
/// <summary>
/// The host's driver factory for the respawn test: every spawn gets a fresh <see cref="FocasDriver"/> over the
/// next fake CNC (the last one repeats), fed the raw tags from the merged DriverConfig the host hands it — as
/// production does.
/// </summary>
private sealed class HostFocasDriverFactory(params IFocasClient[] cncs) : IDriverFactory
{
private int _created;
/// <summary>How many driver instances the host has created.</summary>
public int Created => Volatile.Read(ref _created);
/// <inheritdoc />
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
{
if (driverType != "FOCAS") return null;
var cnc = cncs[Math.Min(Interlocked.Increment(ref _created), cncs.Length) - 1];
using var doc = JsonDocument.Parse(driverConfigJson);
var rawTags = JsonSerializer.Deserialize<List<RawTagEntry>>(
doc.RootElement.GetProperty("RawTags").GetRawText(),
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
return new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
RawTags = rawTags,
Probe = new FocasProbeOptions { Enabled = false },
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
}, driverInstanceId, new SingleClientFactory(cnc));
}
/// <inheritdoc />
public IReadOnlyCollection<string> SupportedTypes => ["FOCAS"];
}
/// <summary>The seeded artifact's FOCAS driver raw tags, decoded through the production artifact reader.</summary>
private static List<RawTagEntry> RawTagsFor(byte[] blob, string driverInstanceId)
{
using var artifact = DeploymentArtifact.Open(blob);
var spec = artifact.ParseDriverInstances().Single(s => s.DriverInstanceId == driverInstanceId);
using var doc = JsonDocument.Parse(spec.DriverConfig);
return JsonSerializer.Deserialize<List<RawTagEntry>>(
doc.RootElement.GetProperty("RawTags").GetRawText(),
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
}
private static byte[] BuildArtifact(bool enabled = false, string driverConfig = "{}") => JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
// Enabled=false (default): the host spawns no driver — the test drives its own instance. Enabled=true
// spawns the host's own driver through the test's factory (never a real wire driver).
new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = driverConfig, ClusterId = "c1", Enabled = enabled },
},
Devices = new[]
{
new { DeviceId = "dev-m1", DriverInstanceId = DriverId, Name = "m1", DeviceConfig = JsonSerializer.Serialize(new { HostAddress = Host }) },
},
TagGroups = Array.Empty<object>(),
Tags = new[]
{
new
{
TagId = "tag-servo",
DeviceId = "dev-m1",
TagGroupId = (string?)null,
Name = "servo_alarms",
DataType = "Int32",
AccessLevel = 0,
TagConfig = JsonSerializer.Serialize(new
{
address = "ALARM:Servo",
dataType = "Int32",
alarm = new { alarmType = "AlarmCondition", severity = 800 },
}),
},
},
});
private static DeploymentId Seed(IDbContextFactory<OtOpcUaConfigDbContext> db, byte[] blob, RevisionHash rev)
{
var id = DeploymentId.NewId();
using var ctx = db.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = rev.Value,
Status = DeploymentStatus.Sealed,
CreatedBy = "test",
SealedAtUtc = DateTime.UtcNow,
ArtifactBlob = blob,
});
ctx.SaveChanges();
return id;
}
private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply(
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory? driverFactory = null)
{
var coordinator = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: driverFactory,
localRoles: new HashSet<string> { "driver" },
dependencyMux: CreateTestProbe().Ref,
opcUaPublishActor: publish.Ref,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: CreateTestProbe().Ref));
actor.Tell(new DispatchDeployment(deploymentId, Rev, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(TimeSpan.FromSeconds(5));
return (actor, publish, coordinator);
}
private sealed class SingleClientFactory(IFocasClient client) : IFocasClientFactory
{
public IFocasClient Create() => client;
public void EnsureUsable() { }
}
/// <summary>A CNC that only answers the alarm list — everything else this test never reaches.</summary>
private sealed class AlarmOnlyFocasClient : IFocasClient
{
private readonly Lock _gate = new();
private FocasActiveAlarm[] _alarms = [];
public bool IsConnected { get; private set; }
public void SetAlarms(params FocasActiveAlarm[] alarms)
{
lock (_gate) _alarms = alarms;
}
public Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken cancellationToken)
{
IsConnected = true;
return Task.CompletedTask;
}
public Task<IReadOnlyList<FocasActiveAlarm>> ReadAlarmsAsync(CancellationToken cancellationToken)
{
lock (_gate) return Task.FromResult<IReadOnlyList<FocasActiveAlarm>>([.. _alarms]);
}
public Task<bool> ProbeAsync(CancellationToken cancellationToken) => Task.FromResult(true);
public void Dispose() => IsConnected = false;
public Task<(object? value, uint status)> ReadAsync(FocasAddress address, FocasDataType type, CancellationToken cancellationToken) => throw Unused();
public Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken cancellationToken) => throw Unused();
public Task<(IReadOnlyList<long>? slots, uint status)> ReadPmcSlotsAsync(string letter, FocasDataType slotType, int start, int end, CancellationToken cancellationToken) => throw Unused();
public Task<FocasSysInfo> GetSysInfoAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<FocasAxisName>> GetAxisNamesAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<FocasSpindleName>> GetSpindleNamesAsync(CancellationToken cancellationToken) => throw Unused();
public Task<FocasDynamicSnapshot> ReadDynamicAsync(int axisIndex, CancellationToken cancellationToken) => throw Unused();
public Task<FocasProgramInfo> GetProgramInfoAsync(CancellationToken cancellationToken) => throw Unused();
public Task<FocasStatus> GetStatusAsync(CancellationToken cancellationToken) => throw Unused();
public Task<FocasTimer> GetTimerAsync(FocasTimerKind kind, CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<FocasServoLoad>> GetServoLoadsAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<double?>> GetSpindleLoadsAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<double?>> GetSpindleMotorSpeedsAsync(CancellationToken cancellationToken) => throw Unused();
private static NotSupportedException Unused() => new("not reached by the alarm end-to-end test");
}
}