fix(twincat): native/probe evidence advances LastSuccessfulRead; probe failures degrade (H12)

In native-notification mode LastSuccessfulRead never moved past the init
stamp, so one Degraded write on a connection older than 90 s latched the
comms-loss watchdog into a full reconnect (every notification on every device
deleted and re-registered). Conversely an unreachable AMS router was swallowed
by the probe and never left Healthy.

- a native push and a successful probe now stamp LastSuccessfulRead (throttled
  to one health record per 250 ms) and restore Healthy unless a device is still
  probe-unreachable;
- 3 consecutive failed probe ticks write Degraded with the stamp preserved;
- a NotSupportedException write is BadNotSupported with no health write (#611);
- a reconnect replay the PLC refuses republishes the refusal's status without
  degrading (it used to report BadCommunicationError + Degraded on every
  reconnect for a correctly-isolated type-mismatch tag).

Refs #739
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 14:03:28 -04:00
parent 17f4bf6d33
commit ecdd6dbecb
2 changed files with 252 additions and 15 deletions
@@ -0,0 +1,161 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests;
/// <summary>
/// The driver health contract (#739 H12): data proving the device answered — a native notification or
/// a successful probe — advances <c>LastSuccessfulRead</c>; sustained probe/connect failure degrades with
/// it preserved; a refusal (a write the client cannot express, a replay the PLC refuses) is no comms fault.
/// </summary>
[Trait("Category", "Unit")]
public sealed class TwinCATHealthContractTests
{
private const string Host = "ads://5.23.91.23.1.1:851";
private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native = true) =>
new(new TwinCATDriverOptions
{
Devices = [new TwinCATDeviceOptions(Host)],
RawTags = TwinCATRawTags.Entries([new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)]),
Probe = probe,
UseNativeNotifications = native,
}, "drv-1", factory);
private static TwinCATProbeOptions FastProbe => new()
{
Enabled = true,
Interval = TimeSpan.FromMilliseconds(30),
Timeout = TimeSpan.FromMilliseconds(100),
};
/// <summary>A native push advances <c>LastSuccessfulRead</c> and restores Healthy after a comms-fault write.</summary>
[Fact]
public async Task A_native_notification_advances_LastSuccessfulRead_and_restores_Healthy()
{
var fake = new FakeTwinCATClient();
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false });
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead;
initStamp.ShouldNotBeNull();
// A write that fails on the wire degrades (comms fault) with the stamp preserved.
fake.ThrowOnWrite = true;
_ = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
drv.GetHealth().LastSuccessfulRead.ShouldBe(initStamp);
await Task.Delay(300, TestContext.Current.CancellationToken);
fake.FireNotification("MAIN.Speed", 7);
var after = drv.GetHealth();
after.State.ShouldBe(DriverState.Healthy);
after.LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp!.Value);
}
/// <summary>A successful probe advances <c>LastSuccessfulRead</c> in a driver that serves only notifications.</summary>
[Fact]
public async Task A_successful_probe_advances_LastSuccessfulRead()
{
var factory = new FakeTwinCATClientFactory();
var drv = NewDriver(factory, FastProbe);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead!.Value;
await WaitForAsync(() => drv.GetHealth().LastSuccessfulRead > initStamp.AddMilliseconds(250), TimeSpan.FromSeconds(3));
drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp.AddMilliseconds(250));
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>
/// An unreachable AMS router (every probe connect throws) degrades the driver after
/// <see cref="TwinCATDriver.ProbeFailuresBeforeDegraded"/> consecutive failures, preserving the stamp —
/// before, the probe caught it and health stayed Healthy forever.
/// </summary>
[Fact]
public async Task Consecutive_probe_failures_degrade_preserving_LastSuccessfulRead()
{
var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { ThrowOnConnect = true } };
var drv = NewDriver(factory, FastProbe);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var initStamp = drv.GetHealth().LastSuccessfulRead;
await WaitForAsync(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(3));
var health = drv.GetHealth();
health.State.ShouldBe(DriverState.Degraded);
health.LastSuccessfulRead.ShouldBe(initStamp);
factory.Clients.Count.ShouldBeGreaterThanOrEqualTo(TwinCATDriver.ProbeFailuresBeforeDegraded);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
}
/// <summary>
/// A write the client refuses with <see cref="NotSupportedException"/> is <c>BadNotSupported</c> with no
/// health write (#611: a refusal is not a comms fault). Positive control: a wire failure does degrade.
/// </summary>
[Fact]
public async Task A_NotSupported_write_leaves_health_alone()
{
var fake = new FakeTwinCATClient { ThrowOnWrite = true, Exception = new NotSupportedException("no") };
var factory = new FakeTwinCATClientFactory { Customise = () => fake };
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false);
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var result = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNotSupported);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
fake.Exception = new InvalidOperationException("wire");
result = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
}
/// <summary>
/// A reconnect replay the PLC REFUSES (an isolated type-mismatch tag re-attempted on every reconnect)
/// republishes the refusal's status and does not degrade. Positive control: a replay that fails on the
/// wire degrades (covered for the same flow in <c>TwinCATReconnectReplayTests</c>).
/// </summary>
[Fact]
public async Task A_refused_replay_republishes_the_refusal_status_without_degrading()
{
var build = 0;
var factory = new FakeTwinCATClientFactory
{
Customise = () =>
{
var c = new FakeTwinCATClient();
if (++build >= 2)
c.AddNotificationErrors["MAIN.Speed"] = new TwinCATTypeMismatchException("MAIN.Speed", "now a STRING");
return c;
},
};
var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false });
await drv.InitializeAsync("{}", TestContext.Current.CancellationToken);
var events = new ConcurrentQueue<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => events.Enqueue(e);
_ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken);
factory.Clients[0].SimulateWireDrop();
// A write reconnects (and replays) without itself touching health on success.
_ = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken);
factory.Clients.Count.ShouldBe(2);
events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadTypeMismatch);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
private static async Task WaitForAsync(Func<bool> condition, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (!condition() && DateTime.UtcNow < deadline)
await Task.Delay(20);
}
}