Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/OpcUaPublishActorApplyFailureTests.cs
T
Joseph Doherty 2d20aeb779 fix(opcua,drivers): carry driver status sub-codes to clients (#519) + health-write ordering (#517)
#519 — a driver's OPC UA status code now reaches a client with its sub-code intact.
DriverInstanceActor projected the driver's uint onto a 3-state Good/Uncertain/Bad enum
and OtOpcUaNodeManager re-expanded Bad to a bare 0x80000000, so BadNoCommunication,
BadTypeMismatch and BadNotSupported were indistinguishable on the wire and every
driver's status constants (incl. the 16 #497 had just corrected) were diagnostic in
logs only. Resolved by option (b): the raw uint rides AttributeValuePublished ->
AttributeValueUpdate -> IOpcUaAddressSpaceSink.WriteValue, OpcUaQuality is deleted, and
severity classification is a helper (OpcUaStatus.IsGood/IsUncertain/IsBad).

The surface was wider than the issue costed (41 files, not "the sink and 3
implementations"): OpcUaQuality had non-driver producers with no uint to carry.
Each got its natural code — BadInternalError for a failed virtual-tag script,
BadNoCommunication for driver comms loss — and ScriptedAlarmHostActor's pair of
mutually-inverse uint<->enum mappers is deleted, having round-tripped a code the
engine already held. AddressSpaceApplier's removal writes deliberately keep the
generic Bad: not losing a code the driver had is this issue; inventing one for the
deploy path is a separate call.

Behaviour change: WriteAlarmQuality fires its Part 9 event on a status-code change
rather than a severity-bucket change. While the annotation was 3-state those were the
same predicate; now a move between two Bad reasons changes what a subscriber reads.

New SDK-free named constants live on Commons.OpcUa.OpcUaStatus, and
StatusCodeParityTests is widened to cover them — that guard is reflection-driven, so
it can only see a named constant, never an inline literal.

#517 — the symptom is real but the issue named the wrong path and its fix was unsafe.
ShutdownAsync cannot race: DriverInstanceActor calls it from PostStop, after the
mailbox drains. (Core/Hosting/DriverHost, the one class with a concurrent GetHealth,
has no production call site.) InitializeAsync can: it runs on a fire-and-forget
Task.Run, and MTConnectDriver writes Healthy then starts loops that can throw into a
catch which disposes the client before writing Faulted.

Moving the write earlier, as suggested, inverts the terminal state — teardown drains
in-flight loops that write health as they unwind, so a late loop could leave Healthy at
rest with every connection closed. The fix is a bracket (conservative before, terminal
after), applied to MTConnect's init-failure path only; shutdown keeps its single
trailing write, and the rule plus the reachability table are documented on
IDriver.ShutdownAsync.

CalculationDriver.GetHealth returned a hard-coded Healthy that consulted nothing —
Healthy before initialize, after shutdown and after dispose. Now derived from _running.
Its test, named Health_is_always_Connected, asserted the defect and is replaced.

Every new guard was falsified by breaking the production code and watching it go red.
The two sub-code tests each assert two different Bad codes, since a single-code
assertion could pass against a hard-coded constant under the old projection.

Pre-existing failures, verified unchanged on a stashed tree and not chased: 4 in
Driver.AbLegacy.IntegrationTests (slc500/micrologix/plc5 fixtures not up) and 3 in
Host.IntegrationTests.

Closes #517
Closes #519

Claude-Session: https://claude.ai/code/session_015p7wGqy3YpZNCpDzTpGMKo
2026-07-31 05:25:27 -04:00

194 lines
10 KiB
C#

using System.Diagnostics.Metrics;
using System.Text.Json;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
/// <summary>
/// archreview 01/S-1: when the applier reports a degraded apply (a rebuild that threw, or per-node
/// materialise failures), <see cref="OpcUaPublishActor.HandleRebuild"/> must surface it — increment the
/// <c>otopcua.opcua.apply.failed</c> meter and log at Error — instead of the optimistic Info line. The
/// happy path must stay Info-only (zero increments).
/// </summary>
public sealed class OpcUaPublishActorApplyFailureTests : RuntimeActorTestBase
{
/// <summary>A rebuild whose sink throws increments otopcua.opcua.apply.failed (kind=rebuild). R2-07 — a
/// pure-add no longer rebuilds, so the degraded-rebuild path is exercised via a rebuild-forcing change:
/// a first PureAdd deploy establishes the applied composition, then a rename (ChangedEquipment ⇒ Rebuild)
/// drives the throwing SafeRebuild.</summary>
[Fact]
public void Rebuild_when_sink_throws_increments_apply_failed_meter()
{
using var recorder = new MeterRecorder("otopcua.opcua.apply.failed");
var db = NewInMemoryDbFactory();
var sink = new ThrowOnRebuildSink();
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
var dep1 = SeedEquipmentDeployment(db, ("eq-1", "eq-1"));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(sink: sink, dbFactory: db, applier: applier));
// First deploy: PureAdd — no rebuild, so the throwing sink is not hit yet.
actor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), new DeploymentId(dep1)));
Thread.Sleep(200);
recorder.Total.ShouldBe(0);
// Second deploy: eq-1 RENAMED ⇒ ChangedEquipment ⇒ Rebuild ⇒ the sink's RebuildAddressSpace throws.
var dep2 = SeedEquipmentDeployment(db, ("eq-1", "eq-1-renamed"));
actor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId(), new DeploymentId(dep2)));
// A PRESENCE budget, so it is an upper bound before giving up rather than a wait: the helper polls
// and returns the instant the meter fires, so a generous value costs nothing on the happy path and
// can never make a genuinely failing assertion pass. Two seconds covered an idle machine but not a
// fully parallel assembly run, where this deploy→rebuild→throw chain (two DB-backed applies) has to
// share 14 cores with 40-odd other Akka test classes; it failed there roughly 1 run in 30.
AwaitAssert(() =>
{
recorder.Total.ShouldBeGreaterThanOrEqualTo(1);
recorder.WithTag("kind", "rebuild").ShouldBeGreaterThanOrEqualTo(1);
}, duration: TimeSpan.FromSeconds(15));
}
/// <summary>A clean rebuild does NOT increment the apply-failed meter (Info-only happy path).</summary>
[Fact]
public void Rebuild_happy_path_does_not_increment_apply_failed_meter()
{
using var recorder = new MeterRecorder("otopcua.opcua.apply.failed");
var db = NewInMemoryDbFactory();
var sink = new NoopSink();
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
SeedEquipmentDeployment(db, ("eq-1", "eq-1"));
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(sink: sink, dbFactory: db, applier: applier));
actor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId()));
// Give the actor time to process, then confirm the failed meter never fired.
Thread.Sleep(400);
recorder.Total.ShouldBe(0);
}
private void AwaitAssert(Action assertion, TimeSpan duration)
{
var deadline = DateTime.UtcNow + duration;
Exception? last = null;
while (DateTime.UtcNow < deadline)
{
try { assertion(); return; }
catch (Exception ex) { last = ex; Thread.Sleep(25); }
}
if (last is not null) throw last;
}
private static Guid SeedEquipmentDeployment(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params (string Id, string Name)[] equipment)
{
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
{
Equipment = equipment.Select(e => new
{
EquipmentId = e.Id,
MachineCode = e.Id.ToUpperInvariant(),
UnsLineId = "line-1",
Name = e.Name,
}).ToArray(),
DriverInstances = Array.Empty<object>(),
ScriptedAlarms = Array.Empty<object>(),
});
var id = Guid.NewGuid();
using var ctx = dbFactory.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id,
RevisionHash = new string('a', 64),
Status = DeploymentStatus.Sealed,
CreatedBy = "test",
SealedAtUtc = DateTime.UtcNow,
ArtifactBlob = artifact,
});
ctx.SaveChanges();
return id;
}
/// <summary>A sink whose RebuildAddressSpace throws (drives the applier's SafeRebuild catch → RebuildFailed).</summary>
private sealed class ThrowOnRebuildSink : IOpcUaAddressSpaceSink
{
public void WriteValue(string nodeId, object? value, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void WriteAlarmQuality(string alarmNodeId, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm) { }
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, AddressSpaceRealm realm, bool isNative = false) { }
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, AddressSpaceRealm realm, string? historianTagname = null, bool isArray = false, uint? arrayLength = null) { }
public void RebuildAddressSpace() => throw new InvalidOperationException("simulated rebuild fault");
public void RaiseNodesAddedModelChange(string affectedNodeId, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void WireAlarmNotifiers(string alarmNodeId, AddressSpaceRealm alarmRealm, IReadOnlyList<string> notifierFolderNodeIds, AddressSpaceRealm notifierFolderRealm) { }
public void AddReference(string sourceNodeId, AddressSpaceRealm sourceRealm, string targetNodeId, AddressSpaceRealm targetRealm, string referenceType = "Organizes") { }
}
/// <summary>A no-op sink — a clean apply (no failures).</summary>
private sealed class NoopSink : IOpcUaAddressSpaceSink
{
public void WriteValue(string nodeId, object? value, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void WriteAlarmQuality(string alarmNodeId, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm) { }
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, AddressSpaceRealm realm, bool isNative = false) { }
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, AddressSpaceRealm realm, string? historianTagname = null, bool isArray = false, uint? arrayLength = null) { }
public void RebuildAddressSpace() { }
public void RaiseNodesAddedModelChange(string affectedNodeId, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
public void WireAlarmNotifiers(string alarmNodeId, AddressSpaceRealm alarmRealm, IReadOnlyList<string> notifierFolderNodeIds, AddressSpaceRealm notifierFolderRealm) { }
public void AddReference(string sourceNodeId, AddressSpaceRealm sourceRealm, string targetNodeId, AddressSpaceRealm targetRealm, string referenceType = "Organizes") { }
}
/// <summary>Listens to a single instrument by name on the central meter and tallies value + tags.</summary>
private sealed class MeterRecorder : IDisposable
{
private readonly string _name;
private readonly MeterListener _listener;
private long _total;
private readonly List<KeyValuePair<string, object?>[]> _tagSets = new();
private readonly object _gate = new();
public MeterRecorder(string instrumentName)
{
_name = instrumentName;
_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == _name)
listener.EnableMeasurementEvents(instrument);
}
};
_listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
{
lock (_gate)
{
_total += value;
_tagSets.Add(tags.ToArray());
}
});
_listener.Start();
}
public long Total { get { lock (_gate) return _total; } }
public int WithTag(string key, string value)
{
lock (_gate) return _tagSets.Count(set => set.Any(t => t.Key == key && Equals(t.Value, value)));
}
public void Dispose() => _listener.Dispose();
}
}