89961af6e6
MAJOR-1: a deploy that changes a driver's config respawns it, and the fresh instance's empty told record re-sent Raise for every still-active alarm, un-acking acknowledged conditions and writing a duplicate "Activated" alerts row. NativeAlarmProjector now treats Raise on an already-active condition as a re-statement (keeps Acked; ForwardNativeAlarm skips the alerts row); only Retrigger (or inactive->active) un-acks. Sources whose upstream ack state is authoritative send Retrigger for an unacked re-statement: OpcUaClient on first sight (no prior told), Galaxy for an unacked active snapshot entry. Runtime E2E drives a FRESH FOCAS driver after a redeploy and asserts the ack survives. MAJOR-2: a failed FOCAS alarm read annotates each subscribed condition BadNoCommunication once per outage (#626 quality-only path) with one Warning (MINOR-6), lifted to Good on recovery; failed alarm reads degrade driver health with LastSuccessfulRead frozen (successful ones advance it), so an alarm-only driver trips the #527 comms-loss watchdog. MINOR-3 CLAUDE.md #630 bullet; MINOR-4 split line; MINOR-5 a non-empty cnc_rdalmmsg2 payload that fits neither stride throws; MINOR-7 value-tag + parsed-address tables are built and swapped, never cleared in place under the retained poll engine. Docs corrected (FOCAS.md, AlarmTracking.md, CLAUDE.md). Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
297 lines
15 KiB
C#
297 lines
15 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Akka.Cluster.Tools.PublishSubscribe;
|
|
using Akka.TestKit;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Gitea #647 — a FOCAS CNC alarm reaches its Part 9 condition END TO END: the REAL <see cref="FocasDriver"/>
|
|
/// (over a fake CNC client), fed the raw tags exactly as the deployment artifact delivers them, raises into the
|
|
/// REAL <see cref="DriverHostActor"/>, whose <c>ForwardNativeAlarm</c> + <c>NativeAlarmProjector</c> must turn it
|
|
/// into an Active and then an inactive condition at the tag's RawPath.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The lesson this exists for: OpcUaClient and AbCip native alarms were "routed" — events raised, unit tests
|
|
/// green — but never reached a condition, because nothing checked the event against the key the host actually
|
|
/// routes on (ConditionId == RawPath) or the field its projector actually reads (Kind). A test that stops at the
|
|
/// driver's event cannot see either.
|
|
/// </remarks>
|
|
public sealed class FocasNativeAlarmEndToEndTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("focas-alarm-e2e");
|
|
private static readonly RevisionHash Rev = RevisionHash.Parse(new string('f', 64));
|
|
private const string Host = "focas://10.0.0.5:8193";
|
|
private const string DriverId = "drv-focas";
|
|
private const string ExpectedRawPath = "Plant/Cnc/m1/servo_alarms";
|
|
|
|
[Fact]
|
|
public async Task A_CNC_servo_alarm_activates_then_clears_the_condition_at_the_RawPath()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var blob = BuildArtifact();
|
|
var deploymentId = Seed(db, blob, Rev);
|
|
var (host, publish, _) = SpawnHostAndApply(db, deploymentId);
|
|
|
|
// The raw tags exactly as the driver receives them: the artifact's merged DriverConfig RawTags array.
|
|
var rawTags = RawTagsFor(blob, DriverId);
|
|
rawTags.Select(t => t.RawPath).ShouldBe([ExpectedRawPath]);
|
|
|
|
var cnc = new AlarmOnlyFocasClient();
|
|
var driver = new FocasDriver(new FocasDriverOptions
|
|
{
|
|
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
|
|
RawTags = rawTags,
|
|
Probe = new FocasProbeOptions { Enabled = false },
|
|
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
|
|
}, DriverId, new SingleClientFactory(cnc));
|
|
driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e));
|
|
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
// The host's alarm subscription set is the alarm-bearing raw tags' RawPaths. Called directly, as the
|
|
// DriverInstanceActor would through its invoker: the routing is under test, not resilience wrapping.
|
|
#pragma warning disable OTOPCUA0001
|
|
await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None);
|
|
#pragma warning restore OTOPCUA0001
|
|
try
|
|
{
|
|
cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
|
|
var active = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
|
|
active.AlarmNodeId.ShouldBe(ExpectedRawPath);
|
|
active.Realm.ShouldBe(AddressSpaceRealm.Raw);
|
|
active.State.Active.ShouldBeTrue("the Raise must project to an ACTIVE condition, not 'keep prior state'");
|
|
active.State.Acknowledged.ShouldBeFalse();
|
|
active.State.Severity.ShouldBe((ushort)800, "the authored alarm.severity is the condition's severity");
|
|
active.State.Message.ShouldStartWith("401 SERVO V-READY OFF");
|
|
|
|
cnc.SetAlarms();
|
|
var cleared = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
|
|
cleared.AlarmNodeId.ShouldBe(ExpectedRawPath);
|
|
cleared.State.Active.ShouldBeFalse();
|
|
}
|
|
finally
|
|
{
|
|
await driver.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gitea #647 review (MAJOR-1) — a deploy that changes ANY of the driver's config respawns it, and the fresh
|
|
/// instance has no record of what the server was told. Its first poll re-reads the still-active alarm and
|
|
/// sends Raise; the host must treat that as a re-statement: the operator's acknowledgement survives and no
|
|
/// second "Activated" alerts row is written. Driven through the real host with a FRESH driver instance.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_respawned_driver_re_reading_an_acknowledged_active_alarm_keeps_the_ack()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var blob = BuildArtifact();
|
|
var alerts = CreateTestProbe();
|
|
DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref);
|
|
alerts.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
|
|
var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev));
|
|
var rawTags = RawTagsFor(blob, DriverId);
|
|
var cnc = new AlarmOnlyFocasClient();
|
|
cnc.SetAlarms(new FocasActiveAlarm(401, FocasAlarmType.Servo, 1, "SERVO V-READY OFF"));
|
|
|
|
var first = await StartDriverAsync(rawTags, cnc, host);
|
|
try
|
|
{
|
|
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5))
|
|
.State.Acknowledged.ShouldBeFalse();
|
|
alerts.ExpectMsg<AlarmTransitionEvent>(TimeSpan.FromSeconds(5));
|
|
}
|
|
finally
|
|
{
|
|
await first.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
|
|
// The operator acknowledges, then a deploy changes the driver's config => stop + respawn.
|
|
host.Tell(new DriverHostActor.RouteNativeAlarmAck(ExpectedRawPath, "seen", "op"));
|
|
var revB = RevisionHash.Parse(new string('e', 64));
|
|
host.Tell(new DispatchDeployment(Seed(db, blob, revB), revB, CorrelationId.NewId()));
|
|
coordinator.FishForMessage<ApplyAck>(a => a.Outcome == ApplyAckOutcome.Applied, TimeSpan.FromSeconds(5));
|
|
publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, TimeSpan.FromSeconds(5));
|
|
|
|
var respawned = await StartDriverAsync(rawTags, cnc, host);
|
|
try
|
|
{
|
|
var restated = publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, TimeSpan.FromSeconds(5));
|
|
restated.AlarmNodeId.ShouldBe(ExpectedRawPath);
|
|
restated.State.Active.ShouldBeTrue();
|
|
restated.State.Acknowledged.ShouldBeTrue("a respawned driver's re-statement must not un-acknowledge");
|
|
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
|
}
|
|
finally
|
|
{
|
|
await respawned.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
private static async Task<FocasDriver> StartDriverAsync(List<RawTagEntry> rawTags, IFocasClient cnc, IActorRef host)
|
|
{
|
|
var driver = new FocasDriver(new FocasDriverOptions
|
|
{
|
|
Devices = [new FocasDeviceOptions(Host, DeviceName: "m1")],
|
|
RawTags = rawTags,
|
|
Probe = new FocasProbeOptions { Enabled = false },
|
|
AlarmProjection = new FocasAlarmProjectionOptions { PollInterval = TimeSpan.FromMilliseconds(20) },
|
|
}, DriverId, new SingleClientFactory(cnc));
|
|
driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(DriverId, e));
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
#pragma warning disable OTOPCUA0001 // the routing is under test, not resilience wrapping
|
|
await driver.SubscribeAlarmsAsync([.. rawTags.Select(t => t.RawPath)], CancellationToken.None);
|
|
#pragma warning restore OTOPCUA0001
|
|
return driver;
|
|
}
|
|
|
|
/// <summary>The seeded artifact's FOCAS driver raw tags, decoded through the production artifact reader.</summary>
|
|
private static List<RawTagEntry> RawTagsFor(byte[] blob, string driverInstanceId)
|
|
{
|
|
using var artifact = DeploymentArtifact.Open(blob);
|
|
var spec = artifact.ParseDriverInstances().Single(s => s.DriverInstanceId == driverInstanceId);
|
|
using var doc = JsonDocument.Parse(spec.DriverConfig);
|
|
return JsonSerializer.Deserialize<List<RawTagEntry>>(
|
|
doc.RootElement.GetProperty("RawTags").GetRawText(),
|
|
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
|
|
}
|
|
|
|
private static byte[] BuildArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
// Enabled=false: the host must not spawn a real wire driver — the test drives its own instance.
|
|
new { DriverInstanceId = DriverId, RawFolderId = "rf-plant", Name = "Cnc", DriverType = "FOCAS", DriverConfig = "{}", ClusterId = "c1", Enabled = false },
|
|
},
|
|
Devices = new[]
|
|
{
|
|
new { DeviceId = "dev-m1", DriverInstanceId = DriverId, Name = "m1", DeviceConfig = JsonSerializer.Serialize(new { HostAddress = Host }) },
|
|
},
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = new[]
|
|
{
|
|
new
|
|
{
|
|
TagId = "tag-servo",
|
|
DeviceId = "dev-m1",
|
|
TagGroupId = (string?)null,
|
|
Name = "servo_alarms",
|
|
DataType = "Int32",
|
|
AccessLevel = 0,
|
|
TagConfig = JsonSerializer.Serialize(new
|
|
{
|
|
address = "ALARM:Servo",
|
|
dataType = "Int32",
|
|
alarm = new { alarmType = "AlarmCondition", severity = 800 },
|
|
}),
|
|
},
|
|
},
|
|
});
|
|
|
|
private static DeploymentId Seed(IDbContextFactory<OtOpcUaConfigDbContext> db, byte[] blob, RevisionHash rev)
|
|
{
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = blob,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId)
|
|
{
|
|
var coordinator = CreateTestProbe();
|
|
var publish = this.CreateCleanOutcomePublishProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
dependencyMux: CreateTestProbe().Ref,
|
|
opcUaPublishActor: publish.Ref,
|
|
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
|
|
virtualTagHostOverride: CreateTestProbe().Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, Rev, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(TimeSpan.FromSeconds(5));
|
|
return (actor, publish, coordinator);
|
|
}
|
|
|
|
private sealed class SingleClientFactory(IFocasClient client) : IFocasClientFactory
|
|
{
|
|
public IFocasClient Create() => client;
|
|
|
|
public void EnsureUsable() { }
|
|
}
|
|
|
|
/// <summary>A CNC that only answers the alarm list — everything else this test never reaches.</summary>
|
|
private sealed class AlarmOnlyFocasClient : IFocasClient
|
|
{
|
|
private readonly Lock _gate = new();
|
|
private FocasActiveAlarm[] _alarms = [];
|
|
|
|
public bool IsConnected { get; private set; }
|
|
|
|
public void SetAlarms(params FocasActiveAlarm[] alarms)
|
|
{
|
|
lock (_gate) _alarms = alarms;
|
|
}
|
|
|
|
public Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken cancellationToken)
|
|
{
|
|
IsConnected = true;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
public Task<IReadOnlyList<FocasActiveAlarm>> ReadAlarmsAsync(CancellationToken cancellationToken)
|
|
{
|
|
lock (_gate) return Task.FromResult<IReadOnlyList<FocasActiveAlarm>>([.. _alarms]);
|
|
}
|
|
|
|
public Task<bool> ProbeAsync(CancellationToken cancellationToken) => Task.FromResult(true);
|
|
|
|
public void Dispose() => IsConnected = false;
|
|
|
|
public Task<(object? value, uint status)> ReadAsync(FocasAddress address, FocasDataType type, CancellationToken cancellationToken) => throw Unused();
|
|
public Task<uint> WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken cancellationToken) => throw Unused();
|
|
public Task<(IReadOnlyList<long>? slots, uint status)> ReadPmcSlotsAsync(string letter, FocasDataType slotType, int start, int end, CancellationToken cancellationToken) => throw Unused();
|
|
public Task<FocasSysInfo> GetSysInfoAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<IReadOnlyList<FocasAxisName>> GetAxisNamesAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<IReadOnlyList<FocasSpindleName>> GetSpindleNamesAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<FocasDynamicSnapshot> ReadDynamicAsync(int axisIndex, CancellationToken cancellationToken) => throw Unused();
|
|
public Task<FocasProgramInfo> GetProgramInfoAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<FocasStatus> GetStatusAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<FocasTimer> GetTimerAsync(FocasTimerKind kind, CancellationToken cancellationToken) => throw Unused();
|
|
public Task<IReadOnlyList<FocasServoLoad>> GetServoLoadsAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<IReadOnlyList<double?>> GetSpindleLoadsAsync(CancellationToken cancellationToken) => throw Unused();
|
|
public Task<IReadOnlyList<double?>> GetSpindleMotorSpeedsAsync(CancellationToken cancellationToken) => throw Unused();
|
|
|
|
private static NotSupportedException Unused() => new("not reached by the alarm end-to-end test");
|
|
}
|
|
}
|