feat(opcuaclient): #676 follow-ups — endpoint rewrite, stall wire test, fast definitive-loss recovery
- Endpoint rewrite: the driver now connects to the configured host AND port (scheme match, advertised path kept) instead of the server-advertised URL — fixes NAT/port-map upstreams and lets an in-test proxy carry the session (SDK SelectEndpoint rule). - TcpStallProxy + OpcUaClientDriverStallWireTests: the in-place blip (#670) is now wire-tested on loopback — one Bad, frozen read stamp, same session, identical value republished Good, a post-recovery change arrives, no source-time regression after the SDK's late backlog flush. - Defect found by that test, fixed: a change published right after a stall rode a Publish response the SDK timed out and never republished, leaving the node Good on the pre-change value indefinitely. New trailing reconcile read (Timeout + KeepAliveInterval after recovery) publishes any Good value newer than the last accepted one, through the replay guard's mark. - IDriverConnectionLossSource (Core.Abstractions): opt-in definitive-loss signal. DriverInstanceActor attaches in PreStart / detaches in PostStop and turns a raise into a disconnect only while Connected. OpcUaClient raises once per session on keep-alive BadSessionIdInvalid/BadSessionClosed/BadSessionNotActivated/ BadServerHalted (measured: the SDK never re-opens a halted channel) or a dropped subscription; ambiguous codes stay with the #527 watchdog. - Replay guard: one Information summary per recovery when the window closes ("dropped N late notifications across M refs"), with a timer so quiet subscriptions log it too. - ReconnectPeriod deleted (options, AdminUI form, docs, tests); stored configs that carry reconnectPeriod still parse in the factory, probe and browser. - Docs: OpcUaClient.md, DriverLifecycle.md, driver-specs.md, CLAUDE.md.
This commit is contained in:
Joseph Doherty
+144
@@ -0,0 +1,144 @@
|
||||
using Opc.Ua;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #676 item 1 — the IN-PLACE blip (#670): the upstream stops answering but the TCP connection and the
|
||||
/// session survive, keep-alives fail, then traffic resumes before the session times out. Until now only the live
|
||||
/// gate (<c>docker pause</c> of opc-plc) covered it. <see cref="TcpStallProxy"/> reproduces it on loopback: the driver
|
||||
/// connects THROUGH the proxy (which works only because of the #676 endpoint rewrite — the server advertises its own
|
||||
/// port, which would bypass the proxy), and pausing the proxy stalls every byte without closing anything.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Writing this test found a real defect: a change the server published right after the stall ended rode a Publish
|
||||
/// response that had been in flight across the stall; the SDK timed that request out, dropped the response and never
|
||||
/// republished it, so the node stayed Good on the PRE-change value indefinitely. The driver's trailing reconcile read
|
||||
/// (<c>ReconcileAfterRecoveryAsync</c>) is the fix; the "40.0 arrives" leg below goes red without it.
|
||||
/// </remarks>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class OpcUaClientDriverStallWireTests
|
||||
{
|
||||
private const uint BadNoCommunication = 0x80310000u;
|
||||
|
||||
private static async Task WaitAsync(Func<bool> condition, string because, TimeSpan? timeout = null)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(30));
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (condition()) return;
|
||||
await Task.Delay(25);
|
||||
}
|
||||
condition().ShouldBeTrue(because);
|
||||
}
|
||||
|
||||
private sealed record Seen(uint Status, object? Value, DateTime? Source);
|
||||
|
||||
[Fact]
|
||||
public async Task A_stalled_connection_goes_Bad_once_and_recovers_in_place_on_the_same_session_without_regressing()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-stall", ct);
|
||||
await using var proxy = TcpStallProxy.Start(new Uri(rig.EndpointUrl).Port);
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
|
||||
|
||||
// A 3 s operation timeout keeps the stall short: a keep-alive is declared failed when its request times out.
|
||||
await using var drv = OpcUaClientDriverWireTests.NewDriver(
|
||||
rig, "wire-stall", endpointUrl: proxy.Through(rig.EndpointUrl), timeout: TimeSpan.FromSeconds(3));
|
||||
var connectionLost = 0;
|
||||
drv.ConnectionLost += (_, _) => Interlocked.Increment(ref connectionLost);
|
||||
await drv.InitializeAsync("{}", ct);
|
||||
proxy.AcceptedConnections.ShouldBeGreaterThan(0, "the session must run THROUGH the proxy, or pausing it proves nothing");
|
||||
|
||||
var seen = new List<Seen>();
|
||||
drv.OnDataChange += (_, e) =>
|
||||
{
|
||||
if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
|
||||
lock (seen) seen.Add(new Seen(e.Snapshot.StatusCode, e.Snapshot.Value, e.Snapshot.SourceTimestampUtc));
|
||||
};
|
||||
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 10.0f }); }, "initial value");
|
||||
|
||||
var session = drv.Session.ShouldNotBeNull();
|
||||
var connectionsBefore = proxy.AcceptedConnections;
|
||||
var badKeepAlives = 0;
|
||||
session.KeepAlive += (_, e) =>
|
||||
{
|
||||
if (ServiceResult.IsBad(e.Status)) Interlocked.Increment(ref badKeepAlives);
|
||||
};
|
||||
// The SDK's late backlog flush: a data-carrying NotificationMessage whose sequence number is BELOW one already
|
||||
// seen (keep-alive messages carry the next number). It is the positive control for the no-regression check.
|
||||
uint maxSequence = 0;
|
||||
var lateMessages = 0;
|
||||
session.Notification += (_, e) =>
|
||||
{
|
||||
var msg = e.NotificationMessage;
|
||||
if (msg is null) return;
|
||||
lock (seen)
|
||||
{
|
||||
if (msg.SequenceNumber < maxSequence && msg.NotificationData.Count > 0) lateMessages++;
|
||||
maxSequence = Math.Max(maxSequence, msg.SequenceNumber);
|
||||
}
|
||||
};
|
||||
|
||||
// ── Stall ────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
proxy.Pause();
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
|
||||
"a stalled upstream must publish BadNoCommunication for the subscribed ref");
|
||||
lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f, "the Bad retains the value");
|
||||
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
|
||||
var frozen = drv.GetHealth().LastSuccessfulRead.ShouldNotBeNull();
|
||||
|
||||
// The server's value moves on while the client cannot see it: away and back to the pre-outage value, so the
|
||||
// recovered value is IDENTICAL to the one published Bad — the (value, status) dedup trap.
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 20.0f);
|
||||
await Task.Delay(300, ct); // > one publishing interval, so the server queues 20.0 as its own notification
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
|
||||
|
||||
// Stays Bad and frozen across a further failed keep-alive (counted, not slept for).
|
||||
var badAtDetection = Volatile.Read(ref badKeepAlives);
|
||||
await WaitAsync(() => Volatile.Read(ref badKeepAlives) >= badAtDetection + 2, "further keep-alives must fail while stalled");
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "the read stamp is the #527 watchdog's silence measure");
|
||||
lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive");
|
||||
|
||||
// ── Resume well inside the 30 s session timeout ──────────────────────────────────────────────────────────
|
||||
proxy.Resume();
|
||||
await WaitAsync(() => drv.GetHealth().State == DriverState.Healthy, "a resumed stall must recover in place");
|
||||
int badIndex;
|
||||
lock (seen) badIndex = seen.FindIndex(s => s.Status == BadNoCommunication);
|
||||
await WaitAsync(() => { lock (seen) return seen.Skip(badIndex + 1).Any(s => s is { Status: 0u, Value: 10.0f }); },
|
||||
"the identical value must be republished Good after the outage");
|
||||
|
||||
drv.Session.ShouldBeSameAs(session, "an in-place blip must not cost the session");
|
||||
proxy.AcceptedConnections.ShouldBe(connectionsBefore, "and not the TCP connection either");
|
||||
|
||||
// A change published straight after recovery — while the Publish requests held across the stall are still
|
||||
// being answered or timed out — must still reach the node.
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 40.0f);
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s is { Status: 0u, Value: 40.0f }); },
|
||||
"a change published right after recovery must reach the node (via the subscription or the trailing re-read)");
|
||||
await drv.ReconcileTaskForTest;
|
||||
|
||||
// The SDK flushes the notifications held during the stall seconds after recovery (the #670 F2 shape). Wait for
|
||||
// that flush before judging ordering, or the check below would pass on a run where nothing late arrived.
|
||||
await WaitAsync(() => { lock (seen) return lateMessages > 0; },
|
||||
"the SDK's late backlog flush did not arrive — the no-regression check below would be vacuous");
|
||||
|
||||
lock (seen)
|
||||
{
|
||||
var good = seen.Where(s => s.Status == 0u && s.Source is not null).ToList();
|
||||
for (var i = 1; i < good.Count; i++)
|
||||
good[i].Source!.Value.ShouldBeGreaterThanOrEqualTo(good[i - 1].Source!.Value,
|
||||
$"published Good value #{i} ({good[i].Value}) regressed in source time — a late replay reached the " +
|
||||
$"node. Sequence: [{string.Join(", ", good.Select(s => $"{s.Value}@{s.Source:HH:mm:ss.fff}"))}]");
|
||||
seen.Last().Value.ShouldBe(40.0f, "nothing older may land after the latest value");
|
||||
seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1);
|
||||
}
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
drv.Session.ShouldBeSameAs(session);
|
||||
Volatile.Read(ref connectionLost).ShouldBe(0,
|
||||
"a stall is AMBIGUOUS (the session survived it) — it must go through the #527 watchdog, never the fast path");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user