dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(opcuaclient): #670 review — subscription loss with a live session, stale recovery latch

Browse Source 
M1: a subscription that dies while the session lives (outage longer than
the subscription lifetime but shorter than the session timeout) used to
end Healthy with fresh keep-alive stamps and frozen values. PublishError
BadNoSubscription / BadSubscriptionIdInvalid naming a tracked
subscription, or a tracked subscription whose SDK PublishingStopped
persists one more keep-alive window + 5 s across good keep-alives, now
declares it lost: Degraded, outage published once, and nothing advances
LastSuccessfulRead until the session is released, so the #527 watchdog
re-initialises and re-subscribes. Both detectors wire-tested.

M2: in-place recovery is bound to the session whose keep-alive recovered;
one overtaken by a re-initialise no longer re-arms the new session's
outage latch or degrades its health (ReadCoreAsync expectedSession).

L1 CAS health updates; L2 re-check before each recovery publish; L3 skip
CloseSession on a lost session; L4 SDK log forwarding rate-limited
(20/min); L5 per-driver SDK LoggerFactory disposed with the driver.
Wire test asserts the read stamp stays frozen while the server is down.
This commit is contained in:
Joseph Doherty
2026-10-11 08:41:27 -04:00
parent 2a8f9af5e4
commit b028322568
6 changed files with 484 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests/OpcUaClientSessionLossTests.cs
+176
View File
@@ -292,4 +292,180 @@ public sealed class OpcUaClientSessionLossTests
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
/// <summary>
/// Review M2: a recovery overtaken by a re-initialise (watchdog Shutdown + Initialize) must not leak its
/// outcome onto the NEW session — previously its failed read re-armed the outage latch, so the new session's
/// first real outage was swallowed.
/// </summary>
[Fact]
public async Task A_recovery_overtaken_by_a_reinit_does_not_latch_the_new_session()
{
var ct = TestContext.Current.CancellationToken;
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1.0f, clock.Now.UtcDateTime));
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
await drv.Gate.WaitAsync(ct); // park the recovery read on the gate
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.ShutdownAsync(ct); // the watchdog's re-initialise overtakes it …
var fresh = NewSession(); // … and a new session is up (its reads would fail)
drv.SetSessionForTest(fresh.Object);
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
drv.AddSubscriptionForTest([SpeedRef]);
drv.Gate.Release();
await drv.RecoveryTaskForTest;
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the stale recovery must not degrade the new session");
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
seen.ShouldHaveSingleItem("the new session's first outage must be published, not swallowed by a stale latch")
.Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
/// <summary>
/// Review M1: the subscription dies while the session lives (an outage longer than the subscription lifetime
/// but shorter than the session timeout). Keep-alives stay Good, so without detection the driver ends Healthy
/// with fresh stamps and frozen values forever. It must go Degraded, publish the loss once, and STOP advancing
/// the read stamp — so the watchdog's reinit recreates the subscription.
/// </summary>
[Fact]
public void A_subscription_that_stops_publishing_while_the_session_lives_freezes_the_stamp_and_goes_Bad()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var otherHandle = drv.AddSubscriptionForTest([CounterRef]);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(3.0f, clock.Now.UtcDateTime));
var stalled = false;
drv.IsPublishingStopped = sub => stalled && sub.DisplayName == "opcua-sub-1";
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
stalled = true;
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good); // first observation — within grace
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
clock.Advance(TimeSpan.FromSeconds(10));
drv.RaiseKeepAliveForTest(StatusCodes.Good); // grace elapsed
var h = drv.GetHealth();
h.State.ShouldBe(DriverState.Degraded);
h.LastError.ShouldNotBeNull().ShouldContain("subscription lost");
var frozen = h.LastSuccessfulRead;
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(2, "every subscribed ref published Bad once");
seen.Single(e => e.FullReference == SpeedRef).Snapshot.Value.ShouldBe(3.0f);
// Nothing may revive the stamp: not good keep-alives, not a bad→good keep-alive cycle, not notifications
// from a still-healthy sibling subscription.
for (var i = 0; i < 30; i++)
{
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
drv.DeliverNotificationForTest(otherHandle, CounterRef, Good(i, clock.Now.UtcDateTime));
}
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
h = drv.GetHealth();
h.State.ShouldBe(DriverState.Degraded);
h.LastSuccessfulRead.ShouldBe(frozen, "the #527 watchdog needs a frozen stamp to re-initialise the driver");
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(2, "the loss is published once per connection");
}
/// <summary>Control for M1: a subscription that is briefly silent (e.g. right after an outage) and then resumes
/// within its grace is NOT declared dead.</summary>
[Fact]
public void A_subscription_silent_for_less_than_its_grace_is_not_declared_lost()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var stalled = true;
drv.IsPublishingStopped = _ => stalled;
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
clock.Advance(TimeSpan.FromSeconds(9));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
stalled = false;
for (var i = 0; i < 10; i++)
{
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
}
stalled = true; // a NEW stall restarts the grace rather than inheriting the old one
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime);
}
[Fact]
public void A_BadNoSubscription_publish_error_with_tracked_subscriptions_declares_the_subscription_lost()
{
var (drv, _, _) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.RaisePublishErrorForTest(StatusCodes.BadSubscriptionIdInvalid, subscriptionId: 4242); // not ours
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
[Fact]
public void A_BadNoSubscription_with_nothing_tracked_is_not_a_loss()
{
var (drv, _, _) = NewConnectedDriver();
using var _ = drv;
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
[Fact]
public async Task Shutdown_clears_a_subscription_loss()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
drv.SetSessionForTest(NewSession().Object);
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime, "the re-subscribed connection starts clean");
}
[Fact]
public void Sdk_log_forwarding_is_rate_limited()
{
var target = new CountingLogger();
var limited = new SdkTelemetry.RateLimitedLogger(target, max: 3, window: TimeSpan.FromHours(1));
for (var i = 0; i < 50; i++)
Microsoft.Extensions.Logging.LoggerExtensions.LogWarning(limited, "sdk warning {I}", i);
target.Count.ShouldBe(3);
}
private sealed class CountingLogger : Microsoft.Extensions.Logging.ILogger
{
public int Count;
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
TState state, Exception? exception, Func<TState, Exception?, string> formatter) => Count++;
}
}
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/OpcUaClientDriverReconnectWireTests.cs
+56
View File
@@ -61,6 +61,12 @@ public sealed class OpcUaClientDriverReconnectWireTests
lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f);
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
drv.GetHostStatuses().Single().State.ShouldNotBe(HostState.Running);
// The read stamp must stay frozen for the whole outage — it is the watchdog's silence measure.
var frozen = drv.GetHealth().LastSuccessfulRead;
frozen.ShouldNotBeNull();
await Task.Delay(TimeSpan.FromSeconds(3), ct); // three keep-alive intervals
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
});
// The restarted server knows nothing of the old session, so the driver stays down on its own …
@@ -95,4 +101,54 @@ public sealed class OpcUaClientDriverReconnectWireTests
back[0].StatusCode.ShouldBe(0u);
Convert.ToSingle(back[0].Value).ShouldBe(88.0f);
}
/// <summary>
/// #670 review M1 — the subscription dies while the SESSION lives (in production: an outage longer than the
/// subscription lifetime but shorter than the session timeout; here: the server deletes it). Keep-alives stay
/// Good throughout, so a driver that only watched them would end Healthy with fresh read stamps and a frozen
/// value forever. It must go Bad, stay non-Healthy with the stamp frozen (so the #527 watchdog fires), and the
/// watchdog's re-initialise must bring the value back.
/// </summary>
[Fact]
public async Task A_subscription_lost_while_the_session_lives_is_reported_and_recovered_by_reinit()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-sublost", ct);
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 5.0f);
await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-sublost");
await drv.InitializeAsync("{}", ct);
var seen = new List<(uint Status, object? Value)>();
drv.OnDataChange += (_, e) =>
{
if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value));
};
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 5.0f)); }, "initial value");
// Delete the subscription on the server behind the SDK's back; the session itself stays healthy.
var session = drv.Session.ShouldNotBeNull();
var subId = session.Subscriptions.Single(s => s.Created).Id;
await session.DeleteSubscriptionsAsync(null, [subId], ct);
await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
"a subscription the server dropped must be reported as comms loss", TimeSpan.FromSeconds(30));
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
var frozen = drv.GetHealth().LastSuccessfulRead;
await Task.Delay(TimeSpan.FromSeconds(3), ct); // good keep-alives keep arriving …
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "… and must not revive a driver whose data plane is dead");
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running, "the session itself is fine");
// The watchdog's re-initialise recreates the subscription and the value comes back Good.
await drv.ShutdownAsync(ct);
await drv.InitializeAsync("{}", ct);
int before;
lock (seen) before = seen.Count;
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 5.0f)); },
"the re-created subscription must deliver the value Good");
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
}