Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/FocasNativeAlarmEndToEndTests.cs
T
Joseph Doherty 89961af6e6 fix(alarms): #647 review — Raise on an active condition is a re-statement; FOCAS outage quality + health
MAJOR-1: a deploy that changes a driver's config respawns it, and the fresh instance's empty told
record re-sent Raise for every still-active alarm, un-acking acknowledged conditions and writing a
duplicate "Activated" alerts row. NativeAlarmProjector now treats Raise on an already-active
condition as a re-statement (keeps Acked; ForwardNativeAlarm skips the alerts row); only Retrigger
(or inactive->active) un-acks. Sources whose upstream ack state is authoritative send Retrigger for
an unacked re-statement: OpcUaClient on first sight (no prior told), Galaxy for an unacked active
snapshot entry. Runtime E2E drives a FRESH FOCAS driver after a redeploy and asserts the ack survives.

MAJOR-2: a failed FOCAS alarm read annotates each subscribed condition BadNoCommunication once per
outage (#626 quality-only path) with one Warning (MINOR-6), lifted to Good on recovery; failed alarm
reads degrade driver health with LastSuccessfulRead frozen (successful ones advance it), so an
alarm-only driver trips the #527 comms-loss watchdog.

MINOR-3 CLAUDE.md #630 bullet; MINOR-4 split line; MINOR-5 a non-empty cnc_rdalmmsg2 payload that
fits neither stride throws; MINOR-7 value-tag + parsed-address tables are built and swapped, never
cleared in place under the retained poll engine. Docs corrected (FOCAS.md, AlarmTracking.md, CLAUDE.md).

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-08 20:18:03 -04:00

297 lines
15 KiB
C#

using System.Text.Json;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.TestKit;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Gitea #647 — a FOCAS CNC alarm reaches its Part 9 condition END TO END: the REAL <see cref="FocasDriver"/>
/// (over a fake CNC client), fed the raw tags exactly as the deployment artifact delivers them, raises into the
/// REAL <see cref="DriverHostActor"/>, whose <c>ForwardNativeAlarm</c> + <c>NativeAlarmProjector</c> must turn it
/// into an Active and then an inactive condition at the tag's RawPath.
/// </summary>
/// <remarks>
/// The lesson this exists for: OpcUaClient and AbCip native alarms were "routed" — events raised, unit tests
/// green — but never reached a condition, because nothing checked the event against the key the host actually
/// routes on (ConditionId == RawPath) or the field its projector actually reads (Kind). A test that stops at the
/// driver's event cannot see either.
/// </remarks>
public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("focas-alarm-e2e");
private static readonly RevisionHash Rev = RevisionHash.Parse(new string('f', 64));
private const string Host = "focas://10.0.0.5:8193";
private const string DriverId = "drv-focas";
private const string ExpectedRawPath = "Plant/Cnc/m1/servo_alarms";
[Fact]
public async Task A_CNC_servo_alarm_activates_then_clears_the_condition_at_the_RawPath()
{
var db = NewInMemoryDbFactory();
var blob = BuildArtifact();
var deploymentId = Seed(db, blob, Rev);
var (host, publish, _) = SpawnHostAndApply(db, deploymentId);
// The raw tags exactly as the driver receives them: the artifact's merged DriverConfig RawTags array.
var rawTags = RawTagsFor(blob, DriverId);
rawTags.Select(t => t.RawPath).ShouldBe([ExpectedRawPath]);
var cnc = new AlarmOnlyFocasClient();
var driver = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
RawTags = rawTags,
Probe = new FocasProbeOptions { Enabled = false },
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
}, DriverId, new SingleClientFactory(cnc));
driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e));
await driver.InitializeAsync("{}", CancellationToken.None);
// The host's alarm subscription set is the alarm-bearing raw tags' RawPaths. Called directly, as the
// DriverInstanceActor would through its invoker: the routing is under test, not resilience wrapping.
#pragma warning disable OTOPCUA0001
await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None);
#pragma warning restore OTOPCUA0001
try
{
cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
var active = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
active.AlarmNodeId.ShouldBe(ExpectedRawPath);
active.Realm.ShouldBe(AddressSpaceRealm.Raw);
active.State.Active.ShouldBeTrue("the Raise must project to an ACTIVE condition, not 'keep prior state'");
active.State.Acknowledged.ShouldBeFalse();
active.State.Severity.ShouldBe((ushort)800, "the authored alarm.severity is the condition's severity");
active.State.Message.ShouldStartWith("401 SERVO V-READY OFF");
cnc.SetAlarms();
var cleared = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
cleared.AlarmNodeId.ShouldBe(ExpectedRawPath);
cleared.State.Active.ShouldBeFalse();
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
/// <summary>
/// Gitea #647 review (MAJOR-1) — a deploy that changes ANY of the driver's config respawns it, and the fresh
/// instance has no record of what the server was told. Its first poll re-reads the still-active alarm and
/// sends Raise; the host must treat that as a re-statement: the operator's acknowledgement survives and no
/// second "Activated" alerts row is written. Driven through the real host with a FRESH driver instance.
/// </summary>
[Fact]
public async Task A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack()
{
var db = NewInMemoryDbFactory();
var blob = BuildArtifact();
var alerts = CreateTestProbe();
DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref);
alerts.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev));
var rawTags = RawTagsFor(blob, DriverId);
var cnc = new AlarmOnlyFocasClient();
cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
var first = await StartDriverAsync(rawTags, cnc, host);
try
{
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5))
.State.Acknowledged.ShouldBeFalse();
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
}
finally
{
await first.ShutdownAsync(CancellationToken.None);
}
// The operator acknowledges, then a deploy changes the driver's config => stop + respawn.
host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op"));
var revB = RevisionHash.Parse(new string('e', 64));
host.Tell(new DispatchDeployment(Seed(db, blob, revB), revB, CorrelationId.NewId()));
coordinator.FishForMessage<ApplyAck>(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5));
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, TimeSpan.FromSeconds(5));
var respawned = await StartDriverAsync(rawTags, cnc, host);
try
{
var restated = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
restated.AlarmNodeId.ShouldBe(ExpectedRawPath);
restated.State.Active.ShouldBeTrue();
restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge");
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
finally
{
await respawned.ShutdownAsync(CancellationToken.None);
}
}
private static async Task<FocasDriver> StartDriverAsync(List<RawTagEntry> rawTags, IFocasClient cnc, IActorRef host)
{
var driver = new FocasDriver(new FocasDriverOptions
{
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
RawTags = rawTags,
Probe = new FocasProbeOptions { Enabled = false },
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
}, DriverId, new SingleClientFactory(cnc));
driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e));
await driver.InitializeAsync("{}", CancellationToken.None);
#pragma warning disable OTOPCUA0001 // the routing is under test, not resilience wrapping
await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None);
#pragma warning restore OTOPCUA0001
return driver;
}
/// <summary>The seeded artifact's FOCAS driver raw tags, decoded through the production artifact reader.</summary>
private static List<RawTagEntry> RawTagsFor(byte[] blob, string driverInstanceId)
{
using var artifact = DeploymentArtifact.Open(blob);
var spec = artifact.ParseDriverInstances().Single(s => s.DriverInstanceId == driverInstanceId);
using var doc = JsonDocument.Parse(spec.DriverConfig);
return JsonSerializer.Deserialize<List<RawTagEntry>>(
doc.RootElement.GetProperty("RawTags").GetRawText(),
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
}
private static byte[] BuildArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
// Enabled=false: the host must not spawn a real wire driver — the test drives its own instance.
new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = "{}", ClusterId = "c1", Enabled = false },
},
Devices = new[]
{
new { DeviceId = "dev-m1", DriverInstanceId = DriverId, Name = "m1", DeviceConfig = JsonSerializer.Serialize(new { HostAddress = Host }) },
},
TagGroups = Array.Empty<object>(),
Tags = new[]
{
new
{
TagId = "tag-servo",
DeviceId = "dev-m1",
TagGroupId = (string?)null,
Name = "servo_alarms",
DataType = "Int32",
AccessLevel = 0,
TagConfig = JsonSerializer.Serialize(new
{
address = "ALARM:Servo",
dataType = "Int32",
alarm = new { alarmType = "AlarmCondition", severity = 800 },
}),
},
},
});
private static DeploymentId Seed(IDbContextFactory<OtOpcUaConfigDbContext> db, byte[] blob, RevisionHash rev)
{
var id = DeploymentId.NewId();
using var ctx = db.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = rev.Value,
Status = DeploymentStatus.Sealed,
CreatedBy = "test",
SealedAtUtc = DateTime.UtcNow,
ArtifactBlob = blob,
});
ctx.SaveChanges();
return id;
}
private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply(
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId)
{
var coordinator = CreateTestProbe();
var publish = this.CreateCleanOutcomePublishProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
localRoles: new HashSet<string> { "driver" },
dependencyMux: CreateTestProbe().Ref,
opcUaPublishActor: publish.Ref,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: CreateTestProbe().Ref));
actor.Tell(new DispatchDeployment(deploymentId, Rev, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(TimeSpan.FromSeconds(5));
return (actor, publish, coordinator);
}
private sealed class SingleClientFactory(IFocasClient client) : IFocasClientFactory
{
public IFocasClient Create() => client;
public void EnsureUsable() { }
}
/// <summary>A CNC that only answers the alarm list — everything else this test never reaches.</summary>
private sealed class AlarmOnlyFocasClient : IFocasClient
{
private readonly Lock _gate = new();
private FocasActiveAlarm[] _alarms = [];
public bool IsConnected { get; private set; }
public void SetAlarms(params FocasActiveAlarm[] alarms)
{
lock (_gate) _alarms = alarms;
}
public Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken cancellationToken)
{
IsConnected = true;
return Task.CompletedTask;
}
public Task<IReadOnlyList<FocasActiveAlarm>> ReadAlarmsAsync(CancellationToken cancellationToken)
{
lock (_gate) return Task.FromResult<IReadOnlyList<FocasActiveAlarm>>([.. _alarms]);
}
public Task<bool> ProbeAsync(CancellationToken cancellationToken) => Task.FromResult(true);
public void Dispose() => IsConnected = false;
public Task<(object? value, uint status)> ReadAsync(FocasAddress address, FocasDataType type, CancellationToken cancellationToken) => throw Unused();
public Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken cancellationToken) => throw Unused();
public Task<(IReadOnlyList<long>? slots, uint status)> ReadPmcSlotsAsync(string letter, FocasDataType slotType, int start, int end, CancellationToken cancellationToken) => throw Unused();
public Task<FocasSysInfo> GetSysInfoAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<FocasAxisName>> GetAxisNamesAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<FocasSpindleName>> GetSpindleNamesAsync(CancellationToken cancellationToken) => throw Unused();
public Task<FocasDynamicSnapshot> ReadDynamicAsync(int axisIndex, CancellationToken cancellationToken) => throw Unused();
public Task<FocasProgramInfo> GetProgramInfoAsync(CancellationToken cancellationToken) => throw Unused();
public Task<FocasStatus> GetStatusAsync(CancellationToken cancellationToken) => throw Unused();
public Task<FocasTimer> GetTimerAsync(FocasTimerKind kind, CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<FocasServoLoad>> GetServoLoadsAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<double?>> GetSpindleLoadsAsync(CancellationToken cancellationToken) => throw Unused();
public Task<IReadOnlyList<double?>> GetSpindleMotorSpeedsAsync(CancellationToken cancellationToken) => throw Unused();
private static NotSupportedException Unused() => new("not reached by the alarm end-to-end test");
}
}