Merge feat/676-opcuaclient-followups: endpoint host+port rewrite, TCP-stall wire test, ReconnectPeriod removed, IDriverConnectionLossSource fast recovery, trailing re-read, replay summary (#676)

This commit is contained in:
Joseph Doherty
2026-10-11 11:45:26 -04:00
22 changed files with 1876 additions and 117 deletions
@@ -137,4 +137,17 @@ public sealed class OpcUaClientDriverBrowserTests
sut.LastUsedUtc.ShouldBeGreaterThan(before,
"AttributesAsync must refresh LastUsedUtc to satisfy the IBrowseSession contract");
}
/// <summary>
/// #676: a config still carrying the deleted <c>reconnectPeriod</c> key must deserialise — the browser's own
/// validation (here: Certificate auth refused) is what fires, not a JSON error.
/// </summary>
[Fact]
public async Task OpenAsync_with_a_legacy_reconnectPeriod_key_still_parses()
{
var json = """{"EndpointUrl":"opc.tcp://127.0.0.1:1","reconnectPeriod":"00:00:05","AuthType":"Certificate"}""";
var ex = await Should.ThrowAsync<InvalidOperationException>(
() => _sut.OpenAsync(json, TestContext.Current.CancellationToken));
ex.Message.ShouldContain("Certificate");
}
}
@@ -0,0 +1,662 @@
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Moq;
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests;
/// <summary>
/// Gitea #676 — the driver half of the fast recovery path (<see cref="IDriverConnectionLossSource"/>), the
/// per-recovery replay-drop summary, and the trailing reconcile read after an in-place recovery.
/// </summary>
[Trait("Category", "Unit")]
public sealed class OpcUaClientConnectionLossSignalTests
{
private const uint BadNoCommunication = 0x80310000u;
private const string SpeedRef = "Plant/dev/Speed";
private const string CounterRef = "Plant/dev/Counter";
private sealed class ManualClock(DateTimeOffset start) : TimeProvider
{
public DateTimeOffset Now { get; set; } = start;
public override DateTimeOffset GetUtcNow() => Now;
public void Advance(TimeSpan by) => Now += by;
}
private sealed class ListLogger : ILogger<OpcUaClientDriver>
{
public List<(LogLevel Level, string Message)> Entries { get; } = [];
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(LogLevel logLevel) => true;
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
Func<TState, Exception?, string> formatter)
{
lock (Entries) Entries.Add((logLevel, formatter(state, exception)));
}
public string[] Info(string contains) => At(LogLevel.Information, contains);
public string[] At(LogLevel level, string contains)
{
lock (Entries)
return [.. Entries.Where(e => e.Level == level && e.Message.Contains(contains)).Select(e => e.Message)];
}
}
private static RawTagEntry Tag(string rawPath, string nodeId) =>
new(rawPath, JsonSerializer.Serialize(new { nodeId }), WriteIdempotent: true);
private static readonly IReadOnlyList<RawTagEntry> Tags =
[
Tag(SpeedRef, "ns=0;s=Speed"),
Tag(CounterRef, "ns=0;s=Counter"),
];
private static Mock<ISession> NewSession()
{
var mock = new Mock<ISession>(MockBehavior.Loose);
mock.SetupGet(s => s.MessageContext).Returns(new ServiceMessageContext(SdkTelemetry.Context));
mock.SetupGet(s => s.NamespaceUris).Returns(new NamespaceTable());
return mock;
}
private static (OpcUaClientDriver Driver, Mock<ISession> Session, ManualClock Clock, ListLogger Log) NewConnectedDriver()
{
var log = new ListLogger();
var drv = new OpcUaClientDriver(new OpcUaClientDriverOptions { RawTags = Tags }, "opcua-676", log);
var session = NewSession();
var clock = new ManualClock(new DateTimeOffset(2026, 10, 11, 12, 0, 0, TimeSpan.Zero));
drv.Clock = clock;
drv.SetSessionForTest(session.Object);
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
return (drv, session, clock, log);
}
private static DataValue Good(object value, DateTime ts) => new(new Variant(value), StatusCodes.Good, ts, ts);
/// <summary>Every read returns <paramref name="values"/> (one per requested node, in order) at <paramref name="at"/>.</summary>
private static void SetupRead(Mock<ISession> session, Func<DateTime> at, params object[] values) =>
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((RequestHeader _, double _, TimestampsToReturn _, ReadValueIdCollection nodes, CancellationToken _) =>
new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = [.. nodes.Select((_, i) => Good(values[i], at()))],
DiagnosticInfos = [],
});
// ── IDriverConnectionLossSource ──────────────────────────────────────────────────────────────────────────────
public static TheoryData<uint> DefinitiveStatuses =>
[
StatusCodes.BadSessionIdInvalid,
StatusCodes.BadSessionClosed,
StatusCodes.BadSessionNotActivated,
StatusCodes.BadServerHalted,
];
public static TheoryData<uint> AmbiguousStatuses =>
[
StatusCodes.BadNoCommunication,
StatusCodes.BadConnectionClosed,
StatusCodes.BadNotConnected,
StatusCodes.BadRequestTimeout,
StatusCodes.BadTimeout,
StatusCodes.BadSecureChannelClosed,
];
[Theory]
[MemberData(nameof(DefinitiveStatuses))]
public void A_definitive_keep_alive_raises_ConnectionLost_once_after_publishing_the_outage(uint status)
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var order = new List<string>();
drv.OnDataChange += (_, e) => order.Add($"bad:{e.Snapshot.StatusCode:X8}");
var raised = new List<DriverConnectionLossEventArgs>();
drv.ConnectionLost += (_, e) => { raised.Add(e); order.Add("lost"); };
drv.RaiseKeepAliveForTest(status);
drv.RaiseKeepAliveForTest(status);
drv.RaiseKeepAliveForTest(StatusCodes.BadSessionIdInvalid);
raised.ShouldHaveSingleItem("once per session, however many definitive keep-alives follow")
.Reason.ShouldContain(StatusCodes.GetBrowseName(status));
order.ShouldBe([$"bad:{BadNoCommunication:X8}", "lost"], "subscribers see the Bad before the host tears down");
}
[Theory]
[MemberData(nameof(AmbiguousStatuses))]
public void An_ambiguous_keep_alive_does_not_raise_ConnectionLost(uint status)
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var raised = 0;
drv.ConnectionLost += (_, _) => raised++;
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
for (var i = 0; i < 5; i++) drv.RaiseKeepAliveForTest(status);
// Positive control: the keep-alives WERE processed as an outage — the outage just is not definitive.
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
raised.ShouldBe(0, "an ambiguous loss may yet resume in place — it belongs to the #527 watchdog");
}
[Fact]
public void An_ambiguous_outage_that_turns_definitive_raises_once()
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
var raised = 0;
drv.ConnectionLost += (_, _) => raised++;
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
raised.ShouldBe(0);
drv.RaiseKeepAliveForTest(StatusCodes.BadServerHalted);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.BadServerHalted);
raised.ShouldBe(1);
}
[Fact]
public async Task The_signal_is_re_armed_by_a_re_initialise()
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
var raised = 0;
drv.ConnectionLost += (_, _) => raised++;
drv.RaiseKeepAliveForTest(StatusCodes.BadSessionIdInvalid);
raised.ShouldBe(1);
// The host's response: ShutdownAsync + InitializeAsync, i.e. a NEW session.
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
var fresh = NewSession();
drv.SetSessionForTest(fresh.Object);
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
drv.RaiseKeepAliveForTest(StatusCodes.BadSessionIdInvalid);
raised.ShouldBe(2, "the next session's definitive loss must be reported too");
}
[Fact]
public void A_definitive_keep_alive_from_a_superseded_session_is_ignored()
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
var raised = 0;
drv.ConnectionLost += (_, _) => raised++;
drv.RaiseKeepAliveForTest(StatusCodes.BadSessionIdInvalid, NewSession().Object);
raised.ShouldBe(0);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
[Fact]
public void A_subscription_the_server_dropped_raises_ConnectionLost_once()
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var raised = new List<DriverConnectionLossEventArgs>();
drv.ConnectionLost += (_, e) => raised.Add(e);
drv.RaisePublishErrorForTest(StatusCodes.BadSubscriptionIdInvalid, subscriptionId: 4242); // not ours
raised.ShouldBeEmpty();
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
raised.ShouldHaveSingleItem().Reason.ShouldContain("subscription lost");
}
[Fact]
public void A_faulting_ConnectionLost_subscriber_does_not_break_the_keep_alive_path()
{
var (drv, _, _, _) = NewConnectedDriver();
using var _ = drv;
drv.ConnectionLost += (_, _) => throw new InvalidOperationException("subscriber bug");
Should.NotThrow(() => drv.RaiseKeepAliveForTest(StatusCodes.BadSessionClosed));
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
}
/// <summary>
/// Review of 3b88056f: only a SERVER-reported subscription loss is definitive. The publishing-stall heuristic
/// (SDK <c>PublishingStopped</c> persisting past a grace) is an inference from silence — it still degrades health
/// and freezes the stamp, but recovery stays with the #527 watchdog.
/// </summary>
[Fact]
public void The_publishing_stall_heuristic_does_not_raise_ConnectionLost()
{
var (drv, _, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
drv.IsPublishingStopped = _ => true;
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
var raised = 0;
drv.ConnectionLost += (_, _) => raised++;
drv.RaiseKeepAliveForTest(StatusCodes.Good);
clock.Advance(TimeSpan.FromSeconds(11));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
// Positive control: the heuristic DID fire.
drv.GetHealth().LastError.ShouldNotBeNull().ShouldContain("subscription lost");
raised.ShouldBe(0);
}
// ── Replay-drop summary ──────────────────────────────────────────────────────────────────────────────────────
[Fact]
public async Task The_replay_window_logs_one_Information_summary_of_its_drops_when_it_closes()
{
var (drv, session, clock, log) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.FromDays(1); // keep the trailing read out of this test
var handle = drv.AddSubscriptionForTest([SpeedRef, CounterRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1, t0.AddSeconds(5)));
drv.DeliverNotificationForTest(handle, CounterRef, Good(2, t0.AddSeconds(5)));
SetupRead(session, () => t0.AddSeconds(5), 1, 2);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
// Three late replays across two refs, one fresh notification that is accepted.
drv.DeliverNotificationForTest(handle, SpeedRef, Good(0, t0));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1, t0.AddSeconds(5)));
drv.DeliverNotificationForTest(handle, CounterRef, Good(0, t0));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(3, t0.AddSeconds(6)));
log.Info("replay window closed").ShouldBeEmpty("the summary is logged once, when the window closes");
clock.Advance(TimeSpan.FromSeconds(31));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(4, t0.AddSeconds(40))); // first notification after expiry
drv.DeliverNotificationForTest(handle, SpeedRef, Good(5, t0.AddSeconds(41)));
log.Info("replay window closed").ShouldHaveSingleItem()
.ShouldContain("dropped 3 late notifications across 2 refs after recovery");
}
[Fact]
public async Task A_window_with_no_drops_closes_at_Debug_not_Information()
{
var (drv, session, clock, log) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.FromDays(1);
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1, t0));
SetupRead(session, () => t0, 1);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
clock.Advance(TimeSpan.FromSeconds(31));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(2, t0.AddSeconds(40)));
log.Info("replay window closed").ShouldBeEmpty("nothing was dropped — not worth an Information line per recovery");
log.At(LogLevel.Debug, "replay window closed").ShouldHaveSingleItem().ShouldContain("dropped 0");
}
[Fact]
public async Task A_re_arm_inside_an_open_window_reports_the_earlier_drops_once_and_starts_counting_afresh()
{
var (drv, session, clock, log) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.FromDays(1);
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1, t0));
SetupRead(session, () => t0, 1);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(0, t0.AddSeconds(-1))); // one drop in window 1
clock.Advance(TimeSpan.FromSeconds(5)); // window 1 still open …
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication); // … a second outage re-arms it
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
clock.Advance(TimeSpan.FromSeconds(31));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(3, t0.AddSeconds(80)));
log.Info("replay window closed").ShouldBe(
["OpcUaClientDriver 'opcua-676': post-recovery replay window closed — dropped 1 late notifications across 1 refs after recovery."],
"window 1's drop is reported once, when the re-arm supersedes it — not carried into window 2's summary");
log.At(LogLevel.Debug, "replay window closed").ShouldHaveSingleItem().ShouldContain("dropped 0");
}
// ── Trailing reconcile read ──────────────────────────────────────────────────────────────────────────────────
[Fact]
public async Task The_trailing_re_read_publishes_a_change_the_subscription_lost_and_nothing_it_already_has()
{
var (drv, session, clock, log) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef, CounterRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, t0));
drv.DeliverNotificationForTest(handle, CounterRef, Good(7, t0));
// The recovery read sees both unchanged; by the trailing read Speed has moved on (its notification was lost in
// the stalled publish pipeline) while Counter has not.
var reads = 0;
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(() =>
{
var trailing = Interlocked.Increment(ref reads) > 1;
return new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = trailing
? [Good(40f, t0.AddSeconds(9)), Good(7, t0)]
: [Good(10f, t0), Good(7, t0)],
DiagnosticInfos = [],
};
});
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => { lock (seen) seen.Add(e); };
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
await drv.ReconcileTaskForTest;
reads.ShouldBe(2, "the recovery read plus exactly one trailing read");
DataChangeEventArgs[] afterRecovery;
lock (seen) afterRecovery = [.. seen.Where(e => e.Snapshot.StatusCode == 0u)];
// Recovery read republishes both Good (they were published Bad); the trailing read adds ONLY the lost change.
afterRecovery.Select(e => (e.FullReference, e.Snapshot.Value))
.ShouldBe([(SpeedRef, (object?)10f), (CounterRef, (object?)7), (SpeedRef, (object?)40f)]);
log.Info("trailing re-read").ShouldHaveSingleItem().ShouldContain("found 1 of 2 refs");
}
[Fact]
public async Task The_trailing_re_read_never_publishes_an_older_value_than_a_notification_already_accepted()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
var gate = new TaskCompletionSource();
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, t0));
var reads = 0;
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.Returns(async () =>
{
if (Interlocked.Increment(ref reads) > 1) await gate.Task; // hold the trailing read
return new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = [Good(reads > 1 ? 20f : 10f, t0.AddSeconds(reads > 1 ? 5 : 0))],
DiagnosticInfos = [],
};
});
var seen = new List<(uint Status, object? Value)>();
drv.OnDataChange += (_, e) => { lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value)); };
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
// While the trailing read is in flight, a NEWER value arrives through the subscription.
drv.DeliverNotificationForTest(handle, SpeedRef, Good(30f, t0.AddSeconds(8)));
gate.SetResult();
await drv.ReconcileTaskForTest;
reads.ShouldBe(2, "the trailing read must actually have run, or the absence below proves nothing");
lock (seen)
seen.ShouldBe([(BadNoCommunication, (object?)10f), (0u, 10f), (0u, 30f)],
"the trailing read's 20 @ t0+5 is older than the 30 @ t0+8 already accepted, so it is not published");
}
[Fact]
public async Task The_trailing_re_read_does_nothing_if_the_session_was_replaced()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.FromMilliseconds(300);
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, t0));
SetupRead(session, () => t0.AddSeconds(5), 99f);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
// A re-initialise lands between the recovery read and the trailing one.
drv.SetSessionForTest(NewSession().Object);
await drv.ReconcileTaskForTest;
session.Verify(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()), Times.Once,
"only the recovery read — the trailing read must not run against a session that is no longer current");
}
/// <summary>Wires the mock session so the first read (the recovery read) returns <paramref name="recovery"/> and
/// every later one (the trailing read) returns <paramref name="trailing"/>.</summary>
private static Func<int> SetupRecoveryThenTrailing(Mock<ISession> session, DataValue recovery, DataValue trailing)
{
var reads = 0;
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(() => new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = [Interlocked.Increment(ref reads) > 1 ? trailing : recovery],
DiagnosticInfos = [],
});
return () => Volatile.Read(ref reads);
}
private static async Task<List<(uint Status, object? Value)>> RecoverAndReconcileAsync(
OpcUaClientDriver drv, string fullRef = SpeedRef)
{
var seen = new List<(uint Status, object? Value)>();
drv.OnDataChange += (_, e) => { if (e.FullReference == fullRef) lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value)); };
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
await drv.ReconcileTaskForTest;
return seen;
}
/// <summary>Review of 3b88056f: a Good→Bad change lost in the stalled pipeline left the node stale-GOOD; the
/// trailing read repairs any (value, status) difference, not just Good ones.</summary>
[Fact]
public async Task The_trailing_re_read_repairs_a_lost_change_to_Bad()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, t0));
var reads = SetupRecoveryThenTrailing(session, Good(10f, t0),
new DataValue(new Variant(10f), StatusCodes.BadSensorFailure, t0.AddSeconds(9), t0.AddSeconds(9)));
var seen = await RecoverAndReconcileAsync(drv);
reads().ShouldBe(2);
seen.ShouldBe([(BadNoCommunication, (object?)10f), (0u, 10f), ((uint)StatusCodes.BadSensorFailure, 10f)]);
}
/// <summary>A server that sends no source timestamp: the re-read is the server's CURRENT state, read after the
/// stalled pipeline drained, so a differing value is newer than anything delivered before — it is published.</summary>
[Fact]
public async Task The_trailing_re_read_repairs_a_server_that_sends_no_source_timestamp()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
DataValue NoSource(object v, DateTime server) =>
new(new Variant(v), StatusCodes.Good, DateTime.MinValue, server);
drv.DeliverNotificationForTest(handle, SpeedRef, NoSource(10f, t0));
var reads = SetupRecoveryThenTrailing(session, NoSource(10f, t0.AddSeconds(1)), NoSource(40f, t0.AddSeconds(9)));
var seen = await RecoverAndReconcileAsync(drv);
reads().ShouldBe(2);
seen.ShouldBe([(BadNoCommunication, (object?)10f), (0u, 10f), (0u, 40f)]);
}
/// <summary>
/// Equal source timestamp, different status: the re-read is the server's current state, not a replay, so it is
/// published — which also repairs the one case the replay guard drops by design (a status-only change that kept
/// its source timestamp inside the window). An IDENTICAL (value, status) is not republished.
/// </summary>
[Fact]
public async Task The_trailing_re_read_publishes_a_status_only_change_at_an_equal_source_time_but_not_an_identical_value()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef, CounterRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, t0));
drv.DeliverNotificationForTest(handle, CounterRef, Good(7, t0));
var reads = 0;
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(() => new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = Interlocked.Increment(ref reads) > 1
? [new DataValue(new Variant(10f), StatusCodes.UncertainLastUsableValue, t0, t0), Good(7, t0)]
: [Good(10f, t0), Good(7, t0)],
DiagnosticInfos = [],
});
var seen = new List<(string Ref, uint Status)>();
drv.OnDataChange += (_, e) => { lock (seen) seen.Add((e.FullReference, e.Snapshot.StatusCode)); };
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
await drv.ReconcileTaskForTest;
reads.ShouldBe(2);
seen.Where(e => e.Status != BadNoCommunication).ShouldBe([
(SpeedRef, 0u), (CounterRef, 0u), // recovery read
(SpeedRef, (uint)StatusCodes.UncertainLastUsableValue), // trailing read: status-only change at t0
]);
}
/// <summary>
/// Review of 3b88056f: the trailing read's compare and its publish are one critical section with the live
/// notification path. Here a NEWER notification arrives exactly between the trailing read's decision and its
/// publish; it must wait, so the older re-read can never land after it and regress the node.
/// </summary>
[Fact]
public async Task A_notification_racing_the_trailing_publish_is_serialised_after_it()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, t0));
SetupRecoveryThenTrailing(session, Good(10f, t0), Good(40f, t0.AddSeconds(5)));
Task? racer = null;
drv.BeforeReconcilePublishForTest = _ =>
{
racer = Task.Run(() => drv.DeliverNotificationForTest(handle, SpeedRef, Good(50f, t0.AddSeconds(8))));
// Bounded: with the critical section the racer cannot finish until this publish is done.
racer.Wait(TimeSpan.FromMilliseconds(300));
};
var seen = await RecoverAndReconcileAsync(drv);
await racer!.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
lock (seen) seen.Select(e => e.Value).ShouldBe([10f, 10f, 40f, 50f], "the newer 50 must be the last value delivered");
}
/// <summary>Review of 3b88056f: the background timers are bound to the session — a shutdown cancels a pending
/// trailing read instead of leaving it to fire (and read) against whatever session exists later.</summary>
[Fact]
public async Task Shutdown_cancels_a_pending_trailing_read()
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.FromHours(1);
var handle = drv.AddSubscriptionForTest([SpeedRef]);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10f, clock.Now.UtcDateTime));
SetupRead(session, () => clock.Now.UtcDateTime, 10f);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
drv.ReconcileTaskForTest.IsCompleted.ShouldBeFalse("pending for an hour");
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
await drv.ReconcileTaskForTest.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
}
/// <summary>
/// Re-review of 30a9d5f0: with no source timestamps to compare, "newer" cannot be judged from the values. The
/// trailing read samples V1; while it is in flight a live notification delivers V2; the read then returns V1 —
/// different from the last delivered value and with no timestamp to prove it older. It must NOT be published: a
/// ref delivered to after the read was issued is skipped (per-ref delivery sequence), or the node regresses to V1.
/// </summary>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task A_notification_delivered_while_the_trailing_read_is_in_flight_is_not_overwritten(bool readHasSourceTime)
{
var (drv, session, clock, _) = NewConnectedDriver();
using var _ = drv;
drv.PostRecoveryReconcileDelay = () => TimeSpan.Zero;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
DataValue NoSource(object v, DateTime server) => new(new Variant(v), StatusCodes.Good, DateTime.MinValue, server);
drv.DeliverNotificationForTest(handle, SpeedRef, NoSource(10f, t0));
var reads = 0;
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(() =>
{
DataValue result;
if (Interlocked.Increment(ref reads) == 1)
result = NoSource(10f, t0.AddSeconds(1)); // recovery read: unchanged
else
{
// The trailing read has sampled V1 at the server; before its response is processed, the live
// subscription delivers the newer V2.
drv.DeliverNotificationForTest(handle, SpeedRef, NoSource(20f, t0.AddSeconds(9)));
result = readHasSourceTime
? Good(15f, t0.AddSeconds(8)) // read carries a source time, V2 did not
: NoSource(15f, t0.AddSeconds(8));
}
return new ReadResponse { ResponseHeader = new ResponseHeader(), Results = [result], DiagnosticInfos = [] };
});
var seen = await RecoverAndReconcileAsync(drv);
reads.ShouldBe(2, "the trailing read must have run");
lock (seen) seen.Select(e => e.Value).ShouldBe([10f, 10f, 20f], "the read's stale 15 must not overwrite the newer 20");
}
}
@@ -75,4 +75,23 @@ public class OpcUaClientDriverFactoryTests
opts.RawTags[0].WriteIdempotent.ShouldBeFalse();
opts.RawTags[1].WriteIdempotent.ShouldBeTrue();
}
/// <summary>
/// #676 deleted the dead <c>ReconnectPeriod</c> option (unused since #670 removed the SDK reconnect handler).
/// DriverConfig rows stored before that still carry it and must keep parsing — and the keys around it must
/// still bind, so the unknown member is skipped rather than aborting the read.
/// </summary>
[Fact]
public void A_stored_config_still_carrying_reconnectPeriod_parses()
{
const string legacy =
"""{"endpointUrl":"opc.tcp://host:4840","reconnectPeriod":"00:00:05","keepAliveInterval":"00:00:07"}""";
var opts = OpcUaClientDriverFactoryExtensions.ParseOptions("drv-legacy", legacy);
opts.EndpointUrl.ShouldBe("opc.tcp://host:4840");
opts.KeepAliveInterval.ShouldBe(TimeSpan.FromSeconds(7));
var driver = OpcUaClientDriverFactoryExtensions.CreateInstance("drv-legacy", legacy, NullLoggerFactory.Instance);
driver.DriverInstanceId.ShouldBe("drv-legacy");
}
}
@@ -134,4 +134,27 @@ public sealed class OpcUaClientDriverProbeTests
await acceptTask.WaitAsync(TimeSpan.FromSeconds(5), testCt);
}
}
/// <summary>
/// #676: a stored config still carrying the deleted <c>reconnectPeriod</c> key must reach the connect phase, not
/// fail as invalid JSON — the probe parses with the same skip-unknown-members options as the factory.
/// </summary>
[Fact]
public async Task A_config_still_carrying_reconnectPeriod_is_probed_not_rejected()
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
var result = await _probe.ProbeAsync(
$$"""{"EndpointUrl":"opc.tcp://127.0.0.1:{{port}}","reconnectPeriod":"00:00:05"}""",
TimeSpan.FromSeconds(5),
TestContext.Current.CancellationToken);
result.Ok.ShouldBeFalse();
var message = result.Message.ShouldNotBeNull();
message.ShouldNotContain("invalid", Case.Insensitive);
message.ShouldContain("Connect failed", Case.Insensitive);
}
}
@@ -31,7 +31,6 @@ public sealed class OpcUaClientDriverScaffoldTests
var opts = new OpcUaClientDriverOptions();
opts.SessionTimeout.ShouldBe(TimeSpan.FromSeconds(120));
opts.KeepAliveInterval.ShouldBe(TimeSpan.FromSeconds(5));
opts.ReconnectPeriod.ShouldBe(TimeSpan.FromSeconds(5));
}
/// <summary>Verifies that driver reports type and ID before connecting.</summary>
@@ -0,0 +1,69 @@
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests;
/// <summary>
/// Gitea #676 — the driver connects to the endpoint URL the server ADVERTISES in GetEndpoints. A server behind NAT,
/// a port map or a proxy advertises an address the client cannot reach (or one that bypasses the proxy), so the
/// driver must rewrite host AND port to the configured URL when the scheme matches — what the SDK's own
/// <c>CoreClientUtils.SelectEndpoint</c> does.
/// </summary>
public sealed class OpcUaClientEndpointRewriteTests
{
[Fact]
public void Host_and_port_of_the_advertised_url_are_replaced_by_the_configured_ones()
{
OpcUaClientDriver.RewriteEndpointUrl("opc.tcp://internal-host:4840/OtOpcUa", "opc.tcp://10.1.2.3:54840/OtOpcUa")
.ShouldBe("opc.tcp://10.1.2.3:54840/OtOpcUa");
}
[Fact]
public void The_advertised_path_is_kept()
{
// The path is the server's to choose (several endpoints may share a host:port); only the address is ours.
OpcUaClientDriver.RewriteEndpointUrl("opc.tcp://plc-a:4840/Server/Ua", "opc.tcp://gateway:6000/")
.ShouldBe("opc.tcp://gateway:6000/Server/Ua");
}
[Fact]
public void Only_the_port_differs_still_rewrites()
{
// The in-process stall proxy shape: same host, the configured port is the proxy's.
OpcUaClientDriver.RewriteEndpointUrl("opc.tcp://127.0.0.1:4840/OtOpcUa", "opc.tcp://127.0.0.1:61000/OtOpcUa")
.ShouldBe("opc.tcp://127.0.0.1:61000/OtOpcUa");
}
[Fact]
public void A_configured_url_without_a_port_keeps_the_advertised_port()
{
// Review of 3b88056f: Uri.Port is -1 for a port-less opc.tcp URL; copying it would drop the advertised port and
// the SDK would fall back to 4840.
OpcUaClientDriver.RewriteEndpointUrl("opc.tcp://internal-host:62541/Server", "opc.tcp://gateway/Server")
.ShouldBe("opc.tcp://gateway:62541/Server");
}
[Fact]
public void An_IPv6_configured_host_is_bracketed()
{
OpcUaClientDriver.RewriteEndpointUrl("opc.tcp://internal-host:4840/OtOpcUa", "opc.tcp://[fd00::1]:54840/OtOpcUa")
.ShouldBe("opc.tcp://[fd00::1]:54840/OtOpcUa");
}
[Fact]
public void A_different_scheme_is_left_alone()
{
// An https endpoint advertised next to an opc.tcp one is a different transport, not a NAT artefact.
OpcUaClientDriver.RewriteEndpointUrl("https://srv:443/ua", "opc.tcp://srv:4840/ua")
.ShouldBe("https://srv:443/ua");
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData("not a url")]
public void An_unusable_advertised_url_falls_back_to_the_configured_one(string? advertised)
{
OpcUaClientDriver.RewriteEndpointUrl(advertised, "opc.tcp://srv:4840/ua").ShouldBe("opc.tcp://srv:4840/ua");
}
}
@@ -4,29 +4,15 @@ using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests;
/// <summary>
/// Scaffold tests for the reconnect options. Since Gitea #670 the driver has no in-driver reconnect
/// handler (session loss is reported and the host's comms-loss watchdog re-initialises the driver);
/// <c>ReconnectPeriod</c> is retained for config compatibility only. Outage behaviour is covered by
/// <see cref="OpcUaClientSessionLossTests"/> and, over the wire, <c>OpcUaClientDriverReconnectWireTests</c>.
/// Reconnect scaffolding. Since Gitea #670 the driver has no in-driver reconnect handler (session loss is reported
/// and the host re-initialises the driver — at once on a definitive loss, #676, else via the comms-loss watchdog),
/// and #676 deleted the dead <c>ReconnectPeriod</c> option; <see cref="OpcUaClientDriverFactoryTests"/> pins that
/// stored configs still carrying it parse. Outage behaviour is covered by <see cref="OpcUaClientSessionLossTests"/>,
/// <see cref="OpcUaClientConnectionLossSignalTests"/> and, over the wire, <c>OpcUaClientDriverReconnectWireTests</c>.
/// </summary>
[Trait("Category", "Unit")]
public sealed class OpcUaClientReconnectTests
{
/// <summary>Verifies that the default reconnect period matches the driver specification of 5 seconds.</summary>
[Fact]
public void Default_ReconnectPeriod_matches_driver_specs_5_seconds()
{
new OpcUaClientDriverOptions().ReconnectPeriod.ShouldBe(TimeSpan.FromSeconds(5));
}
/// <summary>Verifies that reconnect period can be configured for aggressive or relaxed retries.</summary>
[Fact]
public void Options_ReconnectPeriod_is_configurable_for_aggressive_or_relaxed_retry()
{
var opts = new OpcUaClientDriverOptions { ReconnectPeriod = TimeSpan.FromMilliseconds(500) };
opts.ReconnectPeriod.ShouldBe(TimeSpan.FromMilliseconds(500));
}
/// <summary>Verifies that the driver starts in the Unknown state before initialization.</summary>
[Fact]
public void Driver_starts_unknown_pre_init()
@@ -34,7 +34,6 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
Timeout = TimeSpan.FromSeconds(20),
SessionTimeout = TimeSpan.FromSeconds(180),
KeepAliveInterval = TimeSpan.FromSeconds(10),
ReconnectPeriod = TimeSpan.FromSeconds(15),
AutoAcceptCertificates = true,
BrowseRoot = "i=85",
MaxDiscoveredNodes = 5000,
@@ -58,7 +57,6 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
back.Timeout.ShouldBe(TimeSpan.FromSeconds(20));
back.SessionTimeout.ShouldBe(TimeSpan.FromSeconds(180));
back.KeepAliveInterval.ShouldBe(TimeSpan.FromSeconds(10));
back.ReconnectPeriod.ShouldBe(TimeSpan.FromSeconds(15));
back.AutoAcceptCertificates.ShouldBeTrue();
back.BrowseRoot.ShouldBe("i=85");
back.MaxDiscoveredNodes.ShouldBe(5000);
@@ -105,7 +103,6 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
Timeout = TimeSpan.FromSeconds(12),
SessionTimeout = TimeSpan.FromSeconds(150),
KeepAliveInterval = TimeSpan.FromSeconds(7),
ReconnectPeriod = TimeSpan.FromSeconds(8),
AutoAcceptCertificates = true,
BrowseRoot = "i=85",
MaxDiscoveredNodes = 3000,
@@ -133,7 +130,6 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
result.Timeout.ShouldBe(TimeSpan.FromSeconds(12));
result.SessionTimeout.ShouldBe(TimeSpan.FromSeconds(150));
result.KeepAliveInterval.ShouldBe(TimeSpan.FromSeconds(7));
result.ReconnectPeriod.ShouldBe(TimeSpan.FromSeconds(8));
result.AutoAcceptCertificates.ShouldBeTrue();
result.BrowseRoot.ShouldBe("i=85");
result.MaxDiscoveredNodes.ShouldBe(3000);
@@ -13,6 +13,8 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// comms-loss watchdog trips on). Recovery is the watchdog's sequence, replayed here as <c>DriverInstanceActor</c>
/// runs it: ShutdownAsync, InitializeAsync, re-subscribe — after which the value comes back Good. There is no
/// in-driver reconnect any more (#670: the SDK reconnect handler kept everything Good and Healthy forever).
/// Since #676 a gracefully stopped server is a DEFINITIVE loss (<c>BadServerHalted</c>), so the driver also raises
/// <c>ConnectionLost</c> and the host runs that sequence at once instead of after the watchdog's silence window.
/// </summary>
[Trait("Category", "Integration")]
public sealed class OpcUaClientDriverReconnectWireTests
@@ -39,6 +41,8 @@ public sealed class OpcUaClientDriverReconnectWireTests
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-reconnect");
var lost = new List<string>();
drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); };
await drv.InitializeAsync("{}", ct);
var seen = new List<(uint Status, object? Value)>();
@@ -67,12 +71,20 @@ public sealed class OpcUaClientDriverReconnectWireTests
frozen.ShouldNotBeNull();
await Task.Delay(TimeSpan.FromSeconds(3), ct); // three keep-alive intervals
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
// #676: a server that HALTED (graceful stop) is a definitive loss — measured here, the SDK reports
// BadServerHalted on every keep-alive and never re-opens the channel, so the session can never recover in
// place. The driver says so at once, so the host re-initialises it instead of waiting out the ~90 s watchdog.
await WaitAsync(() => { lock (lost) return lost.Count > 0; },
"a halted upstream must raise ConnectionLost", TimeSpan.FromSeconds(10));
lock (lost) lost.Single().ShouldContain("BadServerHalted");
});
// The restarted server knows nothing of the old session, so the driver stays down on its own …
await Task.Delay(TimeSpan.FromSeconds(3), ct);
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "nothing in the driver may paper over a dead session");
lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive");
lock (lost) lost.Count.ShouldBe(1, "once per session — the restarted server's continued refusals raise nothing more");
// … until the host's comms-loss watchdog re-initialises it (DriverInstanceActor: Reconnecting → ShutdownAsync +
// InitializeAsync → Connected → re-subscribe). The new server is empty: redeploy the nodes first, holding the
@@ -100,6 +112,7 @@ public sealed class OpcUaClientDriverReconnectWireTests
var back = await drv.ReadAsync([OpcUaClientDriverWireTests.SpeedRef], ct);
back[0].StatusCode.ShouldBe(0u);
Convert.ToSingle(back[0].Value).ShouldBe(88.0f);
lock (lost) lost.Count.ShouldBe(1, "the healthy re-initialised session reports no loss");
}
/// <summary>
@@ -117,6 +130,8 @@ public sealed class OpcUaClientDriverReconnectWireTests
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 5.0f);
await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-sublost");
var lost = new List<string>();
drv.ConnectionLost += (_, e) => { lock (lost) lost.Add(e.Reason); };
await drv.InitializeAsync("{}", ct);
var seen = new List<(uint Status, object? Value)>();
drv.OnDataChange += (_, e) =>
@@ -140,6 +155,8 @@ public sealed class OpcUaClientDriverReconnectWireTests
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "… and must not revive a driver whose data plane is dead");
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running, "the session itself is fine");
// #676: a subscription the server dropped never comes back on this session — a definitive loss.
lock (lost) lost.ShouldHaveSingleItem().ShouldContain("subscription lost");
// The watchdog's re-initialise recreates the subscription and the value comes back Good.
await drv.ShutdownAsync(ct);
@@ -0,0 +1,159 @@
using Opc.Ua;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Gitea #676 item 1 — the IN-PLACE blip (#670): the upstream stops answering but the TCP connection and the
/// session survive, keep-alives fail, then traffic resumes before the session times out. Until now only the live
/// gate (<c>docker pause</c> of opc-plc) covered it. <see cref="TcpStallProxy"/> reproduces it on loopback: the driver
/// connects THROUGH the proxy (which works only because of the #676 endpoint rewrite — the server advertises its own
/// port, which would bypass the proxy), and pausing the proxy stalls every byte without closing anything.
/// </summary>
/// <remarks>
/// Writing this test found a real defect: a change the server published right after the stall ended rode a Publish
/// response that had been in flight across the stall; the SDK timed that request out, dropped the response and never
/// republished it, so the node stayed Good on the PRE-change value indefinitely. The driver's trailing reconcile read
/// (<c>ReconcileAfterRecoveryAsync</c>) is the fix; the "40.0 arrives" leg below goes red without it.
/// </remarks>
[Trait("Category", "Integration")]
public sealed class OpcUaClientDriverStallWireTests
{
private const uint BadNoCommunication = 0x80310000u;
private static async Task WaitAsync(Func<bool> condition, string because, TimeSpan? timeout = null)
{
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(30));
while (DateTime.UtcNow < deadline)
{
if (condition()) return;
await Task.Delay(25);
}
condition().ShouldBeTrue(because);
}
private sealed record Seen(uint Status, object? Value, DateTime? Source);
[Fact]
public async Task A_stalled_connection_goes_Bad_once_and_recovers_in_place_on_the_same_session_without_regressing()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-stall", ct);
await using var proxy = TcpStallProxy.Start(new Uri(rig.EndpointUrl).Port);
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
// A 3 s operation timeout keeps the stall short (a keep-alive is declared failed when its request times out); a
// 120 s session timeout and a 3000-cycle (300 s at 100 ms) subscription lifetime keep both far outside the
// stall, so the test exercises the in-place recovery and nothing else.
await using var drv = OpcUaClientDriverWireTests.NewDriver(
rig, "wire-stall", endpointUrl: proxy.Through(rig.EndpointUrl), timeout: TimeSpan.FromSeconds(3),
sessionTimeout: TimeSpan.FromSeconds(120), subscriptionLifetimeCount: 3000);
var connectionLost = 0;
drv.ConnectionLost += (_, _) => Interlocked.Increment(ref connectionLost);
await drv.InitializeAsync("{}", ct);
proxy.AcceptedConnections.ShouldBeGreaterThan(0, "the session must run THROUGH the proxy, or pausing it proves nothing");
var seen = new List<Seen>();
drv.OnDataChange += (_, e) =>
{
if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
lock (seen) seen.Add(new Seen(e.Snapshot.StatusCode, e.Snapshot.Value, e.Snapshot.SourceTimestampUtc));
};
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 10.0f }); }, "initial value");
var session = drv.Session.ShouldNotBeNull();
var connectionsBefore = proxy.AcceptedConnections;
// Bad keep-alive TICKS: the SDK reports a failed tick as two events at the same instant (the request's status and
// BadNoCommunication), so events closer than half a keep-alive interval are one tick.
var badKeepAlives = 0;
var lastBadTick = DateTime.MinValue;
session.KeepAlive += (_, e) =>
{
if (!ServiceResult.IsBad(e.Status)) return;
lock (seen)
{
if (DateTime.UtcNow - lastBadTick < TimeSpan.FromMilliseconds(500)) return;
lastBadTick = DateTime.UtcNow;
badKeepAlives++;
}
};
// The SDK's late backlog flush: a data-carrying NotificationMessage whose sequence number is BELOW one already
// seen (keep-alive messages carry the next number). It is the positive control for the no-regression check.
uint maxSequence = 0;
var lateMessages = 0;
session.Notification += (_, e) =>
{
var msg = e.NotificationMessage;
if (msg is null) return;
lock (seen)
{
if (msg.SequenceNumber < maxSequence && msg.NotificationData.Count > 0) lateMessages++;
maxSequence = Math.Max(maxSequence, msg.SequenceNumber);
}
};
// ── Stall ────────────────────────────────────────────────────────────────────────────────────────────────
proxy.Pause();
await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
"a stalled upstream must publish BadNoCommunication for the subscribed ref");
lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f, "the Bad retains the value");
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
var frozen = drv.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
// The server's value moves on while the client cannot see it: away and back to the pre-outage value, so the
// recovered value is IDENTICAL to the one published Bad — the (value, status) dedup trap.
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 20.0f);
await Task.Delay(300, ct); // > one publishing interval, so the server queues 20.0 as its own notification
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
// Stays Bad and frozen across one further failed keep-alive tick (counted, not slept for).
int badAtDetection;
lock (seen) badAtDetection = badKeepAlives;
await WaitAsync(() => { lock (seen) return badKeepAlives > badAtDetection; }, "a further keep-alive must fail while stalled");
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "the read stamp is the #527 watchdog's silence measure");
lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive");
// ── Resume well inside the 30 s session timeout ──────────────────────────────────────────────────────────
proxy.Resume();
await WaitAsync(() => drv.GetHealth().State == DriverState.Healthy, "a resumed stall must recover in place");
int badIndex;
lock (seen) badIndex = seen.FindIndex(s => s.Status == BadNoCommunication);
await WaitAsync(() => { lock (seen) return seen.Skip(badIndex + 1).Any(s => s is { Status: 0u, Value: 10.0f }); },
"the identical value must be republished Good after the outage");
drv.Session.ShouldBeSameAs(session, "an in-place blip must not cost the session");
proxy.AcceptedConnections.ShouldBe(connectionsBefore, "and not the TCP connection either");
// A change published straight after recovery — while the Publish requests held across the stall are still
// being answered or timed out — must still reach the node.
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 40.0f);
await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 40.0f }); },
"a change published right after recovery must reach the node (via the subscription or the trailing re-read)",
TimeSpan.FromSeconds(60));
await drv.ReconcileTaskForTest;
// The SDK flushes the notifications held during the stall seconds after recovery (the #670 F2 shape). Wait for
// that flush before judging ordering, or the check below would pass on a run where nothing late arrived.
await WaitAsync(() => { lock (seen) return lateMessages > 0; },
"the SDK's late backlog flush did not arrive — the no-regression check below would be vacuous",
TimeSpan.FromSeconds(60));
lock (seen)
{
var good = seen.Where(s => s.Status == 0u && s.Source is not null).ToList();
for (var i = 1; i < good.Count; i++)
good[i].Source!.Value.ShouldBeGreaterThanOrEqualTo(good[i - 1].Source!.Value,
$"published Good value #{i} ({good[i].Value}) regressed in source time — a late replay reached the " +
$"node. Sequence: [{string.Join(", ", good.Select(s => $"{s.Value}@{s.Source:HH:mm:ss.fff}"))}]");
seen.Last().Value.ShouldBe(40.0f, "nothing older may land after the latest value");
seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1);
}
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.Session.ShouldBeSameAs(session);
Volatile.Read(ref connectionLost).ShouldBe(0,
"a stall is AMBIGUOUS (the session survived it) — it must go through the #527 watchdog, never the fast path");
}
}
@@ -41,10 +41,11 @@ public sealed class OpcUaClientDriverWireTests
internal static OpcUaClientDriver NewDriver(
ServerEventRig rig, string id, string? browseRoot = null, IReadOnlyList<RawTagEntry>? extraTags = null,
TimeSpan? reconnectPeriod = null) =>
string? endpointUrl = null, TimeSpan? timeout = null, TimeSpan? sessionTimeout = null,
uint? subscriptionLifetimeCount = null) =>
new(new OpcUaClientDriverOptions
{
EndpointUrl = rig.EndpointUrl,
EndpointUrl = endpointUrl ?? rig.EndpointUrl,
RawTags =
[
Tag(SpeedRef, ServerEventRig.RawNsu(SpeedId)),
@@ -58,11 +59,11 @@ public sealed class OpcUaClientDriverWireTests
Username = "apikey",
Password = ServerEventRig.KeyPassword,
AutoAcceptCertificates = true,
Timeout = TimeSpan.FromSeconds(10),
SessionTimeout = TimeSpan.FromSeconds(30),
Timeout = timeout ?? TimeSpan.FromSeconds(10),
SessionTimeout = sessionTimeout ?? TimeSpan.FromSeconds(30),
SubscriptionLifetimeCount = subscriptionLifetimeCount ?? 1000,
KeepAliveInterval = TimeSpan.FromSeconds(1),
BrowseRoot = browseRoot,
ReconnectPeriod = reconnectPeriod ?? TimeSpan.FromSeconds(5),
}, id);
internal static async Task<ServerEventRig> StartRigAsync(string name, CancellationToken ct)
@@ -0,0 +1,126 @@
using System.Net;
using System.Net.Sockets;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Gitea #676 — an in-test TCP forwarder that can STALL a connection without closing it: the shape of a
/// <c>docker pause</c>, a hung switch port or a congested WAN link, where the socket stays open and simply stops
/// carrying bytes. Each accepted connection gets a fresh upstream connection and two copy pumps; every pump awaits a
/// shared gate before each write, so <see cref="Pause"/> holds all traffic in both directions (bytes already read
/// are held, the rest back up in the kernel buffers) and <see cref="Resume"/> lets it all through in order. Nothing
/// is ever closed by a pause — the outage is invisible at the TCP level, exactly as in production.
/// </summary>
internal sealed class TcpStallProxy : IAsyncDisposable
{
private readonly TcpListener _listener = new(IPAddress.Loopback, 0);
private readonly int _targetPort;
private readonly CancellationTokenSource _cts = new();
private readonly List<TcpClient> _sockets = [];
private readonly Lock _lock = new();
private TaskCompletionSource _gate = Open();
private Task _acceptLoop = Task.CompletedTask;
private TcpStallProxy(int targetPort) => _targetPort = targetPort;
/// <summary>The loopback port clients connect to.</summary>
public int Port => ((IPEndPoint)_listener.LocalEndpoint).Port;
/// <summary>Number of client connections accepted so far — a new one means the client re-connected.</summary>
public int AcceptedConnections { get; private set; }
/// <summary>Starts forwarding <c>127.0.0.1:{Port}</c> to <c>127.0.0.1:{targetPort}</c>.</summary>
public static TcpStallProxy Start(int targetPort)
{
var proxy = new TcpStallProxy(targetPort);
proxy._listener.Start();
proxy._acceptLoop = proxy.AcceptLoopAsync();
return proxy;
}
/// <summary>Rewrites the port of an <c>opc.tcp</c> URL to this proxy's.</summary>
public string Through(string endpointUrl) => new UriBuilder(endpointUrl) { Port = Port }.Uri.ToString();
/// <summary>Stops all traffic in both directions; sockets stay open.</summary>
public void Pause()
{
lock (_lock)
if (_gate.Task.IsCompleted) _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
}
/// <summary>Releases everything held since <see cref="Pause"/>, in order.</summary>
public void Resume()
{
lock (_lock) _gate.TrySetResult();
}
private static TaskCompletionSource Open()
{
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
tcs.SetResult();
return tcs;
}
private Task CurrentGate()
{
lock (_lock) return _gate.Task;
}
private async Task AcceptLoopAsync()
{
while (!_cts.IsCancellationRequested)
{
TcpClient client;
try { client = await _listener.AcceptTcpClientAsync(_cts.Token); }
catch { return; }
var upstream = new TcpClient();
try { await upstream.ConnectAsync(IPAddress.Loopback, _targetPort, _cts.Token); }
catch
{
client.Dispose();
upstream.Dispose();
continue;
}
lock (_lock)
{
AcceptedConnections++;
_sockets.Add(client);
_sockets.Add(upstream);
}
_ = PumpAsync(client.GetStream(), upstream.GetStream());
_ = PumpAsync(upstream.GetStream(), client.GetStream());
}
}
private async Task PumpAsync(NetworkStream from, NetworkStream to)
{
var buffer = new byte[64 * 1024];
try
{
while (true)
{
var n = await from.ReadAsync(buffer, _cts.Token);
if (n == 0) break;
await CurrentGate().WaitAsync(_cts.Token);
await to.WriteAsync(buffer.AsMemory(0, n), _cts.Token);
}
}
catch { /* either side gone or the proxy disposed */ }
// Half-close propagation is not needed: a real close (server stopped) tears the pair down on the next read.
try { to.Socket.Shutdown(SocketShutdown.Send); } catch { }
}
public async ValueTask DisposeAsync()
{
Resume();
await _cts.CancelAsync();
_listener.Stop();
try { await _acceptLoop; } catch { }
lock (_lock)
{
foreach (var s in _sockets) s.Dispose();
_sockets.Clear();
}
_cts.Dispose();
}
}
@@ -0,0 +1,175 @@
using Akka.Actor;
using Akka.Event;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Gitea #676 — a driver that KNOWS its connection is gone (<see cref="IDriverConnectionLossSource"/>) gets
/// re-initialised at once instead of after the #527 watchdog's silence window (~90 s in production). The watchdog is
/// configured at an hour here, so anything that reconnects within seconds can only be the new signal.
/// </summary>
public sealed class DriverInstanceActorConnectionLossSignalTests : RuntimeActorTestBase
{
private static readonly TimeSpan FastPoll = TimeSpan.FromMilliseconds(50);
/// <summary>A permanently Healthy (fresh-read) stub — the watchdog can never fire for it — that can raise
/// <see cref="IDriverConnectionLossSource.ConnectionLost"/> on demand and counts its attached handlers.</summary>
private sealed class LossSourceStubDriver : StubDriver, IDriverConnectionLossSource
{
private EventHandler<DriverConnectionLossEventArgs>? _handlers;
private int _handlerCount;
private int _healthReads;
public int HandlerCount => Volatile.Read(ref _handlerCount);
/// <summary>Health-poll observations — the positive control for an absence assertion.</summary>
public int HealthReadCount => Volatile.Read(ref _healthReads);
public override DriverHealth GetHealth()
{
Interlocked.Increment(ref _healthReads);
return base.GetHealth();
}
public event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost
{
add { _handlers += value; Interlocked.Increment(ref _handlerCount); }
remove { _handlers -= value; Interlocked.Decrement(ref _handlerCount); }
}
/// <summary>Raised from a pool thread, as a real driver's keep-alive callback would.</summary>
public Task RaiseAsync(string reason) =>
Task.Run(() => _handlers?.Invoke(this, new DriverConnectionLossEventArgs(reason)));
}
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(LossSourceStubDriver driver)
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver,
reconnectInterval: TimeSpan.FromHours(1),
healthPollInterval: FastPoll,
commsLossTimeout: TimeSpan.FromHours(1)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(TimeSpan.FromSeconds(3)).Connected.ShouldBeTrue();
return (actor, parent);
}
[Fact]
public async Task A_connection_loss_from_a_connected_driver_reconnects_at_once_not_after_the_watchdog()
{
var driver = new LossSourceStubDriver();
var (_, parent) = Connect(driver);
var started = DateTime.UtcNow;
await driver.RaiseAsync("keep-alive reported BadServerHalted");
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(TimeSpan.FromSeconds(3)).Connected.ShouldBeFalse();
(DateTime.UtcNow - started).ShouldBeLessThan(TimeSpan.FromSeconds(3),
"the watchdog is configured at an hour, so only the connection-loss signal can have done this");
}
/// <summary>
/// The falsification control: the same driver and actor with the event suppressed stays Connected — so the
/// Reconnecting above is the signal's doing, not something the setup would have done anyway.
/// </summary>
[Fact]
public void Without_a_raise_the_same_driver_stays_connected()
{
var driver = new LossSourceStubDriver();
var (_, parent) = Connect(driver);
var ticksAtConnect = driver.HealthReadCount;
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(600));
(driver.HealthReadCount - ticksAtConnect).ShouldBeGreaterThanOrEqualTo(5,
"health-poll ticks did not run — the absence assertion above would be vacuous");
driver.HandlerCount.ShouldBe(1, "attached, so the absence above is not merely an unwired event");
driver.InitializeCount.ShouldBe(1, "and nothing re-initialised the driver");
}
/// <summary>
/// A raise while NOT Connected is ignored, not deferred: here it lands while the initial connect is still in
/// flight. Acting on it later would tear down the connection that very attempt is building.
/// </summary>
[Fact]
public async Task A_raise_while_connecting_is_ignored()
{
var driver = new LossSourceStubDriver();
var release = new TaskCompletionSource();
var initStarted = new TaskCompletionSource();
driver.InitBehavior = async _ =>
{
initStarted.TrySetResult();
await release.Task;
};
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver,
reconnectInterval: TimeSpan.FromHours(1),
healthPollInterval: FastPoll,
commsLossTimeout: TimeSpan.FromHours(1)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
await initStarted.Task.WaitAsync(TimeSpan.FromSeconds(3));
// The raise is enqueued before the connect can complete (RaiseAsync returns after the Tell), so the mailbox
// delivers it while the actor is still Connecting.
await driver.RaiseAsync("a loss reported mid-connect");
release.SetResult();
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(TimeSpan.FromSeconds(3)).Connected.ShouldBeTrue();
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(600));
driver.InitializeCount.ShouldBe(1);
}
/// <summary>Review of 3b88056f: a watchdog DisconnectObserved still queued when the connection-loss signal has
/// already moved the actor to Reconnecting is a no-op there, not an Unhandled message.</summary>
[Fact]
public async Task A_stray_DisconnectObserved_while_reconnecting_is_not_unhandled()
{
var driver = new LossSourceStubDriver();
var (actor, parent) = Connect(driver);
var unhandled = CreateTestProbe();
Sys.EventStream.Subscribe(unhandled.Ref, typeof(UnhandledMessage));
await driver.RaiseAsync("keep-alive reported BadServerHalted");
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(TimeSpan.FromSeconds(3)).Connected.ShouldBeFalse();
actor.Tell(new DriverInstanceActor.DisconnectObserved("late watchdog trip"));
unhandled.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
[Fact]
public void The_handler_is_attached_once_and_detached_when_the_actor_stops()
{
var driver = new LossSourceStubDriver();
var (actor, parent) = Connect(driver);
driver.HandlerCount.ShouldBe(1);
parent.Watch(actor);
Sys.Stop(actor);
// PostStop announces comms loss (03/S15) before the Terminated arrives.
parent.FishForMessage(m => m is Terminated t && t.ActorRef.Equals(actor), TimeSpan.FromSeconds(3));
driver.HandlerCount.ShouldBe(0, "the IDriver outlives the actor on a respawn — a leaked handler holds a dead Self");
}
/// <summary>A driver that does not implement the interface is simply never subscribed — the signal is opt-in.</summary>
[Fact]
public void A_driver_without_the_capability_connects_as_before()
{
var driver = new StubDriver();
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromHours(1), healthPollInterval: FastPoll,
commsLossTimeout: TimeSpan.FromHours(1)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(TimeSpan.FromSeconds(3)).Connected.ShouldBeTrue();
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
}