dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(opcuaclient): #670 review — subscription loss with a live session, stale recovery latch

Browse Source 
M1: a subscription that dies while the session lives (outage longer than
the subscription lifetime but shorter than the session timeout) used to
end Healthy with fresh keep-alive stamps and frozen values. PublishError
BadNoSubscription / BadSubscriptionIdInvalid naming a tracked
subscription, or a tracked subscription whose SDK PublishingStopped
persists one more keep-alive window + 5 s across good keep-alives, now
declares it lost: Degraded, outage published once, and nothing advances
LastSuccessfulRead until the session is released, so the #527 watchdog
re-initialises and re-subscribes. Both detectors wire-tested.

M2: in-place recovery is bound to the session whose keep-alive recovered;
one overtaken by a re-initialise no longer re-arms the new session's
outage latch or degrades its health (ReadCoreAsync expectedSession).

L1 CAS health updates; L2 re-check before each recovery publish; L3 skip
CloseSession on a lost session; L4 SDK log forwarding rate-limited
(20/min); L5 per-driver SDK LoggerFactory disposed with the driver.
Wire test asserts the read stamp stays frozen while the server is down.
This commit is contained in:
Joseph Doherty
2026-10-11 08:41:27 -04:00
parent 2a8f9af5e4
commit b028322568
6 changed files with 484 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests/OpcUaClientSessionLossTests.cs
+176
View File
@@ -292,4 +292,180 @@ public sealed class OpcUaClientSessionLossTests
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
/// <summary>
/// Review M2: a recovery overtaken by a re-initialise (watchdog Shutdown + Initialize) must not leak its
/// outcome onto the NEW session — previously its failed read re-armed the outage latch, so the new session's
/// first real outage was swallowed.
/// </summary>
[Fact]
public async Task A_recovery_overtaken_by_a_reinit_does_not_latch_the_new_session()
{
var ct = TestContext.Current.CancellationToken;
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1.0f, clock.Now.UtcDateTime));
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
await drv.Gate.WaitAsync(ct); // park the recovery read on the gate
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.ShutdownAsync(ct); // the watchdog's re-initialise overtakes it …
var fresh = NewSession(); // … and a new session is up (its reads would fail)
drv.SetSessionForTest(fresh.Object);
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
drv.AddSubscriptionForTest([SpeedRef]);
drv.Gate.Release();
await drv.RecoveryTaskForTest;
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the stale recovery must not degrade the new session");
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
seen.ShouldHaveSingleItem("the new session's first outage must be published, not swallowed by a stale latch")
.Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
/// <summary>
/// Review M1: the subscription dies while the session lives (an outage longer than the subscription lifetime
/// but shorter than the session timeout). Keep-alives stay Good, so without detection the driver ends Healthy
/// with fresh stamps and frozen values forever. It must go Degraded, publish the loss once, and STOP advancing
/// the read stamp — so the watchdog's reinit recreates the subscription.
/// </summary>
[Fact]
public void A_subscription_that_stops_publishing_while_the_session_lives_freezes_the_stamp_and_goes_Bad()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var otherHandle = drv.AddSubscriptionForTest([CounterRef]);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(3.0f, clock.Now.UtcDateTime));
var stalled = false;
drv.IsPublishingStopped = sub => stalled && sub.DisplayName == "opcua-sub-1";
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
stalled = true;
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good); // first observation — within grace
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
clock.Advance(TimeSpan.FromSeconds(10));
drv.RaiseKeepAliveForTest(StatusCodes.Good); // grace elapsed
var h = drv.GetHealth();
h.State.ShouldBe(DriverState.Degraded);
h.LastError.ShouldNotBeNull().ShouldContain("subscription lost");
var frozen = h.LastSuccessfulRead;
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(2, "every subscribed ref published Bad once");
seen.Single(e => e.FullReference == SpeedRef).Snapshot.Value.ShouldBe(3.0f);
// Nothing may revive the stamp: not good keep-alives, not a bad→good keep-alive cycle, not notifications
// from a still-healthy sibling subscription.
for (var i = 0; i < 30; i++)
{
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
drv.DeliverNotificationForTest(otherHandle, CounterRef, Good(i, clock.Now.UtcDateTime));
}
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
h = drv.GetHealth();
h.State.ShouldBe(DriverState.Degraded);
h.LastSuccessfulRead.ShouldBe(frozen, "the #527 watchdog needs a frozen stamp to re-initialise the driver");
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(2, "the loss is published once per connection");
}
/// <summary>Control for M1: a subscription that is briefly silent (e.g. right after an outage) and then resumes
/// within its grace is NOT declared dead.</summary>
[Fact]
public void A_subscription_silent_for_less_than_its_grace_is_not_declared_lost()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var stalled = true;
drv.IsPublishingStopped = _ => stalled;
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
clock.Advance(TimeSpan.FromSeconds(9));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
stalled = false;
for (var i = 0; i < 10; i++)
{
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
}
stalled = true; // a NEW stall restarts the grace rather than inheriting the old one
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime);
}
[Fact]
public void A_BadNoSubscription_publish_error_with_tracked_subscriptions_declares_the_subscription_lost()
{
var (drv, _, _) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.RaisePublishErrorForTest(StatusCodes.BadSubscriptionIdInvalid, subscriptionId: 4242); // not ours
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
[Fact]
public void A_BadNoSubscription_with_nothing_tracked_is_not_a_loss()
{
var (drv, _, _) = NewConnectedDriver();
using var _ = drv;
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
}
[Fact]
public async Task Shutdown_clears_a_subscription_loss()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
drv.AddSubscriptionForTest([SpeedRef]);
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
drv.SetSessionForTest(NewSession().Object);
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
clock.Advance(TimeSpan.FromSeconds(5));
drv.RaiseKeepAliveForTest(StatusCodes.Good);
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime, "the re-subscribed connection starts clean");
}
[Fact]
public void Sdk_log_forwarding_is_rate_limited()
{
var target = new CountingLogger();
var limited = new SdkTelemetry.RateLimitedLogger(target, max: 3, window: TimeSpan.FromHours(1));
for (var i = 0; i < 50; i++)
Microsoft.Extensions.Logging.LoggerExtensions.LogWarning(limited, "sdk warning {I}", i);
target.Count.ShouldBe(3);
}
private sealed class CountingLogger : Microsoft.Extensions.Logging.ILogger
{
public int Count;
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
TState state, Exception? exception, Func<TState, Exception?, string> formatter) => Count++;
}
}