Merge fix/670-opcuaclient-session-loss: OpcUaClient upstream session loss goes Bad and recovers (#670)
This commit is contained in:
Joseph Doherty
+2
-6
@@ -57,9 +57,7 @@ public sealed class OpcUaClientDiagnosticsTests
|
||||
d.Record(OpcUaClientDiagnostics.BadStatusValue);
|
||||
d.Record(OpcUaClientDiagnostics.PublishError);
|
||||
d.Record(OpcUaClientDiagnostics.KeepAliveMiss);
|
||||
d.Record(OpcUaClientDiagnostics.ReconnectAttempt);
|
||||
d.Record(OpcUaClientDiagnostics.ReconnectSuccess);
|
||||
d.Record(OpcUaClientDiagnostics.ReconnectExhausted);
|
||||
d.Record(OpcUaClientDiagnostics.KeepAliveRecovered);
|
||||
|
||||
cap[OpcUaClientDiagnostics.PublishResponse].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.Notification].ShouldBe(3);
|
||||
@@ -67,9 +65,7 @@ public sealed class OpcUaClientDiagnosticsTests
|
||||
cap[OpcUaClientDiagnostics.BadStatusValue].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.PublishError].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.KeepAliveMiss].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.ReconnectAttempt].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.ReconnectSuccess].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.ReconnectExhausted].ShouldBe(1);
|
||||
cap[OpcUaClientDiagnostics.KeepAliveRecovered].ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
+6
-8
@@ -4,9 +4,10 @@ using Xunit;
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Scaffold tests for <see cref="SessionReconnectHandler"/> wiring. Wire-level
|
||||
/// disconnect-reconnect-resume coverage against a live upstream server lands with the
|
||||
/// in-process fixture — too much machinery for a unit-test-only lane.
|
||||
/// Scaffold tests for the reconnect options. Since Gitea #670 the driver has no in-driver reconnect
|
||||
/// handler (session loss is reported and the host's comms-loss watchdog re-initialises the driver);
|
||||
/// <c>ReconnectPeriod</c> is retained for config compatibility only. Outage behaviour is covered by
|
||||
/// <see cref="OpcUaClientSessionLossTests"/> and, over the wire, <c>OpcUaClientDriverReconnectWireTests</c>.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class OpcUaClientReconnectTests
|
||||
@@ -26,13 +27,10 @@ public sealed class OpcUaClientReconnectTests
|
||||
opts.ReconnectPeriod.ShouldBe(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the driver starts with no reconnect handler active before initialization.</summary>
|
||||
/// <summary>Verifies that the driver starts in the Unknown state before initialization.</summary>
|
||||
[Fact]
|
||||
public void Driver_starts_with_no_reconnect_handler_active_pre_init()
|
||||
public void Driver_starts_unknown_pre_init()
|
||||
{
|
||||
// The reconnect handler is lazy — spun up only when a bad keep-alive fires. Pre-init
|
||||
// there's no session to reconnect, so the field must be null (indirectly verified by
|
||||
// the lifecycle-shape test suite catching any accidental construction).
|
||||
using var drv = new OpcUaClientDriver(new OpcUaClientDriverOptions(), "opcua-reconnect");
|
||||
drv.GetHealth().State.ShouldBe(Core.Abstractions.DriverState.Unknown);
|
||||
}
|
||||
|
||||
+471
@@ -0,0 +1,471 @@
|
||||
using System.Text.Json;
|
||||
using Moq;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #670 — upstream session loss must reach the driver's health and its subscribers. Before the fix a bad
|
||||
/// keep-alive only flipped the host status: the subscribed values stayed Good forever, health stayed Healthy (so the
|
||||
/// #527 comms-loss watchdog could never trip — its first conjunct is "non-Healthy"), and <c>LastSuccessfulRead</c>
|
||||
/// only moved on init / <c>ReadAsync</c>, which a subscription-only driver never calls.
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class OpcUaClientSessionLossTests
|
||||
{
|
||||
private const uint BadNoCommunication = 0x80310000u;
|
||||
private const string SpeedRef = "Plant/dev/Speed";
|
||||
private const string CounterRef = "Plant/dev/Counter";
|
||||
|
||||
private sealed class ManualClock(DateTimeOffset start) : TimeProvider
|
||||
{
|
||||
public DateTimeOffset Now { get; set; } = start;
|
||||
public override DateTimeOffset GetUtcNow() => Now;
|
||||
public void Advance(TimeSpan by) => Now += by;
|
||||
}
|
||||
|
||||
private static RawTagEntry Tag(string rawPath, string nodeId) =>
|
||||
new(rawPath, JsonSerializer.Serialize(new { nodeId }), WriteIdempotent: true);
|
||||
|
||||
private static readonly IReadOnlyList<RawTagEntry> Tags =
|
||||
[
|
||||
Tag(SpeedRef, "ns=0;s=Speed"),
|
||||
Tag(CounterRef, "ns=0;s=Counter"),
|
||||
];
|
||||
|
||||
private static Mock<ISession> NewSession()
|
||||
{
|
||||
var mock = new Mock<ISession>(MockBehavior.Loose);
|
||||
mock.SetupGet(s => s.MessageContext)
|
||||
.Returns(new ServiceMessageContext(SdkTelemetry.Context));
|
||||
mock.SetupGet(s => s.NamespaceUris).Returns(new NamespaceTable());
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static (OpcUaClientDriver Driver, Mock<ISession> Session, ManualClock Clock) NewConnectedDriver()
|
||||
{
|
||||
var drv = new OpcUaClientDriver(new OpcUaClientDriverOptions { RawTags = Tags }, "opcua-670");
|
||||
var session = NewSession();
|
||||
var clock = new ManualClock(new DateTimeOffset(2026, 10, 11, 12, 0, 0, TimeSpan.Zero));
|
||||
drv.Clock = clock;
|
||||
drv.SetSessionForTest(session.Object);
|
||||
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
|
||||
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
|
||||
return (drv, session, clock);
|
||||
}
|
||||
|
||||
private static DataValue Good(object value, DateTime ts) =>
|
||||
new(new Variant(value), StatusCodes.Good, ts, ts);
|
||||
|
||||
private static void SetupRead(Mock<ISession> session, params object[] values) =>
|
||||
session.Setup(s => s.ReadAsync(
|
||||
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
|
||||
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((RequestHeader _, double _, TimestampsToReturn _, ReadValueIdCollection nodes, CancellationToken _) =>
|
||||
new ReadResponse
|
||||
{
|
||||
ResponseHeader = new ResponseHeader(),
|
||||
Results = [.. nodes.Select((_, i) => new DataValue(new Variant(values[i]), StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow))],
|
||||
DiagnosticInfos = [],
|
||||
});
|
||||
|
||||
private static async Task WaitForAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (condition()) return;
|
||||
await Task.Delay(10);
|
||||
}
|
||||
condition().ShouldBeTrue(because);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_bad_keep_alive_degrades_health_and_preserves_the_last_successful_read()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var before = drv.GetHealth().LastSuccessfulRead;
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
var h = drv.GetHealth();
|
||||
h.State.ShouldNotBe(DriverState.Healthy, "a lost session must be visible to the #527 watchdog's first conjunct");
|
||||
h.State.ShouldBe(DriverState.Degraded);
|
||||
h.LastSuccessfulRead.ShouldBe(before, "the watchdog measures silence from this stamp, so loss must not move it");
|
||||
h.LastError.ShouldNotBeNull().ShouldContain("keep-alive");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_bad_keep_alive_publishes_BadNoCommunication_once_per_ref_retaining_the_value()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var handle = drv.AddSubscriptionForTest([SpeedRef, CounterRef]);
|
||||
var t0 = clock.Now.UtcDateTime;
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(42.5f, t0));
|
||||
drv.DeliverNotificationForTest(handle, CounterRef, Good(7, t0));
|
||||
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => seen.Add(e);
|
||||
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
seen.Count.ShouldBe(2, "exactly one Bad per subscribed ref");
|
||||
var speed = seen.Single(e => e.FullReference == SpeedRef);
|
||||
speed.Snapshot.StatusCode.ShouldBe(BadNoCommunication);
|
||||
speed.Snapshot.Value.ShouldBe(42.5f, "the value is retained — only the quality changes (as the host does)");
|
||||
speed.SubscriptionHandle.ShouldBe(handle);
|
||||
seen.Single(e => e.FullReference == CounterRef).Snapshot.Value.ShouldBe(7);
|
||||
|
||||
// The SDK re-raises a bad keep-alive every interval while the channel stays down: no further publishes.
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadConnectionClosed);
|
||||
seen.Count.ShouldBe(2, "a second bad keep-alive in the same outage must publish nothing");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Good_keep_alives_and_notifications_advance_the_last_successful_read()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var handle = drv.AddSubscriptionForTest([SpeedRef]);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(30));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime);
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(30));
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1.0f, clock.Now.UtcDateTime));
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Falsification control for the R2 half of the fix: a long-running, idle, subscription-only driver whose
|
||||
/// server is perfectly alive must never look dead to the #527 watchdog (non-Healthy AND silent ≥ 90 s).
|
||||
/// Without good keep-alives advancing <c>LastSuccessfulRead</c>, fixing R1 alone would trip it on the first
|
||||
/// bad tick of any connection older than 90 s.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void An_idle_healthy_subscription_driver_never_looks_dead_to_the_watchdog()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.AddSubscriptionForTest([SpeedRef]); // static value — no notifications ever arrive
|
||||
|
||||
for (var i = 0; i < 60; i++) // 5 minutes of 5 s keep-alives
|
||||
{
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
var h = drv.GetHealth();
|
||||
h.State.ShouldBe(DriverState.Healthy);
|
||||
(clock.Now.UtcDateTime - h.LastSuccessfulRead!.Value).ShouldBeLessThan(TimeSpan.FromSeconds(90));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Recovery_on_the_same_session_republishes_the_identical_value_Good()
|
||||
{
|
||||
var (drv, session, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var handle = drv.AddSubscriptionForTest([SpeedRef]);
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(42.5f, clock.Now.UtcDateTime));
|
||||
SetupRead(session, 42.5f); // the server still holds the SAME value — it will not resend it
|
||||
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => { lock (seen) seen.Add(e); };
|
||||
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
clock.Advance(TimeSpan.FromSeconds(20));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime);
|
||||
await WaitForAsync(() => { lock (seen) return seen.Count >= 2; }, "the recovery read must publish");
|
||||
|
||||
lock (seen)
|
||||
{
|
||||
seen.Count.ShouldBe(2);
|
||||
seen[0].Snapshot.StatusCode.ShouldBe(BadNoCommunication);
|
||||
seen[1].FullReference.ShouldBe(SpeedRef);
|
||||
seen[1].SubscriptionHandle.ShouldBe(handle);
|
||||
seen[1].Snapshot.StatusCode.ShouldBe(0u);
|
||||
seen[1].Snapshot.Value.ShouldBe(42.5f, "the IDENTICAL value must come back Good — a (value,status) dedup lets it through");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_notification_during_recovery_wins_over_the_recovery_read()
|
||||
{
|
||||
var (drv, session, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var handle = drv.AddSubscriptionForTest([SpeedRef]);
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1.0f, clock.Now.UtcDateTime));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
// A fresh notification arrives before the keep-alive recovers: the read must not overwrite it.
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(2.0f, clock.Now.UtcDateTime));
|
||||
SetupRead(session, 1.0f);
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => { lock (seen) seen.Add(e); };
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
|
||||
await Task.Delay(200, TestContext.Current.CancellationToken);
|
||||
lock (seen) seen.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_bad_keep_alive_annotates_subscribed_alarm_conditions_quality_only_once()
|
||||
{
|
||||
var (drv, _, _) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.AddAlarmSubscriptionForTest([SpeedRef]);
|
||||
var seen = new List<AlarmEventArgs>();
|
||||
drv.OnAlarmEvent += (_, e) => seen.Add(e);
|
||||
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
var e = seen.ShouldHaveSingleItem();
|
||||
e.ConditionId.ShouldBe(SpeedRef);
|
||||
e.IsQualityOnly.ShouldBeTrue("an outage says nothing about the alarm state — quality only, no transition");
|
||||
e.QualityStatusCode.ShouldBe(BadNoCommunication);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Recovery_restores_alarm_condition_quality_Good()
|
||||
{
|
||||
var (drv, session, _) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
SetupRead(session);
|
||||
drv.AddAlarmSubscriptionForTest([SpeedRef]);
|
||||
var seen = new List<AlarmEventArgs>();
|
||||
drv.OnAlarmEvent += (_, e) => { lock (seen) seen.Add(e); };
|
||||
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
|
||||
await WaitForAsync(() => { lock (seen) return seen.Count >= 2; }, "recovery must re-annotate Good");
|
||||
lock (seen)
|
||||
{
|
||||
seen[1].IsQualityOnly.ShouldBeTrue();
|
||||
seen[1].QualityStatusCode.ShouldBe(0u);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_keep_alive_from_a_superseded_session_is_ignored()
|
||||
{
|
||||
var (drv, _, _) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var stale = NewSession();
|
||||
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication, stale.Object);
|
||||
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Shutdown_clears_the_outage_so_a_fresh_session_reports_its_own_loss()
|
||||
{
|
||||
var (drv, _, _) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.AddSubscriptionForTest([SpeedRef]);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
// As after a watchdog reinit: a new session + subscription, and a new outage must be reported again.
|
||||
var fresh = NewSession();
|
||||
drv.SetSessionForTest(fresh.Object);
|
||||
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
|
||||
drv.AddSubscriptionForTest([SpeedRef]);
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => seen.Add(e);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review M2: a recovery overtaken by a re-initialise (watchdog Shutdown + Initialize) must not leak its
|
||||
/// outcome onto the NEW session — previously its failed read re-armed the outage latch, so the new session's
|
||||
/// first real outage was swallowed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_recovery_overtaken_by_a_reinit_does_not_latch_the_new_session()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var handle = drv.AddSubscriptionForTest([SpeedRef]);
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1.0f, clock.Now.UtcDateTime));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
|
||||
await drv.Gate.WaitAsync(ct); // park the recovery read on the gate
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
await drv.ShutdownAsync(ct); // the watchdog's re-initialise overtakes it …
|
||||
var fresh = NewSession(); // … and a new session is up (its reads would fail)
|
||||
drv.SetSessionForTest(fresh.Object);
|
||||
drv.SetNamespaceMapForTest(NamespaceMap.FromTable(new NamespaceTable(), Tags));
|
||||
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
|
||||
drv.AddSubscriptionForTest([SpeedRef]);
|
||||
drv.Gate.Release();
|
||||
await drv.RecoveryTaskForTest;
|
||||
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the stale recovery must not degrade the new session");
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => seen.Add(e);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
seen.ShouldHaveSingleItem("the new session's first outage must be published, not swallowed by a stale latch")
|
||||
.Snapshot.StatusCode.ShouldBe(BadNoCommunication);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review M1: the subscription dies while the session lives (an outage longer than the subscription lifetime
|
||||
/// but shorter than the session timeout). Keep-alives stay Good, so without detection the driver ends Healthy
|
||||
/// with fresh stamps and frozen values forever. It must go Degraded, publish the loss once, and STOP advancing
|
||||
/// the read stamp — so the watchdog's reinit recreates the subscription.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_subscription_that_stops_publishing_while_the_session_lives_freezes_the_stamp_and_goes_Bad()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
var handle = drv.AddSubscriptionForTest([SpeedRef]);
|
||||
var otherHandle = drv.AddSubscriptionForTest([CounterRef]);
|
||||
drv.DeliverNotificationForTest(handle, SpeedRef, Good(3.0f, clock.Now.UtcDateTime));
|
||||
var stalled = false;
|
||||
drv.IsPublishingStopped = sub => stalled && sub.DisplayName == "opcua-sub-1";
|
||||
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => seen.Add(e);
|
||||
|
||||
stalled = true;
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good); // first observation — within grace
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
clock.Advance(TimeSpan.FromSeconds(10));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good); // grace elapsed
|
||||
|
||||
var h = drv.GetHealth();
|
||||
h.State.ShouldBe(DriverState.Degraded);
|
||||
h.LastError.ShouldNotBeNull().ShouldContain("subscription lost");
|
||||
var frozen = h.LastSuccessfulRead;
|
||||
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(2, "every subscribed ref published Bad once");
|
||||
seen.Single(e => e.FullReference == SpeedRef).Snapshot.Value.ShouldBe(3.0f);
|
||||
|
||||
// Nothing may revive the stamp: not good keep-alives, not a bad→good keep-alive cycle, not notifications
|
||||
// from a still-healthy sibling subscription.
|
||||
for (var i = 0; i < 30; i++)
|
||||
{
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
drv.DeliverNotificationForTest(otherHandle, CounterRef, Good(i, clock.Now.UtcDateTime));
|
||||
}
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
|
||||
h = drv.GetHealth();
|
||||
h.State.ShouldBe(DriverState.Degraded);
|
||||
h.LastSuccessfulRead.ShouldBe(frozen, "the #527 watchdog needs a frozen stamp to re-initialise the driver");
|
||||
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(2, "the loss is published once per connection");
|
||||
}
|
||||
|
||||
/// <summary>Control for M1: a subscription that is briefly silent (e.g. right after an outage) and then resumes
|
||||
/// within its grace is NOT declared dead.</summary>
|
||||
[Fact]
|
||||
public void A_subscription_silent_for_less_than_its_grace_is_not_declared_lost()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.AddSubscriptionForTest([SpeedRef]);
|
||||
var stalled = true;
|
||||
drv.IsPublishingStopped = _ => stalled;
|
||||
drv.PublishingStallGrace = _ => TimeSpan.FromSeconds(10);
|
||||
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
clock.Advance(TimeSpan.FromSeconds(9));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
stalled = false;
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
}
|
||||
stalled = true; // a NEW stall restarts the grace rather than inheriting the old one
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_BadNoSubscription_publish_error_with_tracked_subscriptions_declares_the_subscription_lost()
|
||||
{
|
||||
var (drv, _, _) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.AddSubscriptionForTest([SpeedRef]);
|
||||
var seen = new List<DataChangeEventArgs>();
|
||||
drv.OnDataChange += (_, e) => seen.Add(e);
|
||||
|
||||
drv.RaisePublishErrorForTest(StatusCodes.BadSubscriptionIdInvalid, subscriptionId: 4242); // not ours
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
|
||||
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Degraded);
|
||||
seen.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_BadNoSubscription_with_nothing_tracked_is_not_a_loss()
|
||||
{
|
||||
var (drv, _, _) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Shutdown_clears_a_subscription_loss()
|
||||
{
|
||||
var (drv, _, clock) = NewConnectedDriver();
|
||||
using var _ = drv;
|
||||
drv.AddSubscriptionForTest([SpeedRef]);
|
||||
drv.RaisePublishErrorForTest(StatusCodes.BadNoSubscription);
|
||||
|
||||
await drv.ShutdownAsync(TestContext.Current.CancellationToken);
|
||||
drv.SetSessionForTest(NewSession().Object);
|
||||
drv.SetHealthForTest(new DriverHealth(DriverState.Healthy, clock.Now.UtcDateTime, null));
|
||||
clock.Advance(TimeSpan.FromSeconds(5));
|
||||
drv.RaiseKeepAliveForTest(StatusCodes.Good);
|
||||
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(clock.Now.UtcDateTime, "the re-subscribed connection starts clean");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Sdk_log_forwarding_is_rate_limited()
|
||||
{
|
||||
var target = new CountingLogger();
|
||||
var limited = new SdkTelemetry.RateLimitedLogger(target, max: 3, window: TimeSpan.FromHours(1));
|
||||
for (var i = 0; i < 50; i++)
|
||||
Microsoft.Extensions.Logging.LoggerExtensions.LogWarning(limited, "sdk warning {I}", i);
|
||||
target.Count.ShouldBe(3);
|
||||
}
|
||||
|
||||
private sealed class CountingLogger : Microsoft.Extensions.Logging.ILogger
|
||||
{
|
||||
public int Count;
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true;
|
||||
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
|
||||
TState state, Exception? exception, Func<TState, Exception?, string> formatter) => Count++;
|
||||
}
|
||||
}
|
||||
+101
-27
@@ -6,14 +6,19 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Reconnect + resume leg of the OpcUaClient driver's wire coverage (Gitea #622): the in-process server is
|
||||
/// stopped and a fresh one is booted on the SAME port / application URI / PKI store — a process restart as the
|
||||
/// driver sees it, with an empty address space the test then re-creates. The driver must detect the loss,
|
||||
/// rebuild a session, and its existing subscription must deliver again with no caller action.
|
||||
/// Outage + recovery leg of the OpcUaClient driver's wire coverage (Gitea #622, reworked for #670): the in-process
|
||||
/// server is stopped and a fresh one is booted on the SAME port / application URI / PKI store — a process restart
|
||||
/// as the driver sees it, with an empty address space the test then re-creates. The driver must REPORT the loss
|
||||
/// (subscribed value BadNoCommunication with the value retained, health non-Healthy — what the host's #527
|
||||
/// comms-loss watchdog trips on). Recovery is the watchdog's sequence, replayed here as <c>DriverInstanceActor</c>
|
||||
/// runs it: ShutdownAsync, InitializeAsync, re-subscribe — after which the value comes back Good. There is no
|
||||
/// in-driver reconnect any more (#670: the SDK reconnect handler kept everything Good and Healthy forever).
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class OpcUaClientDriverReconnectWireTests
|
||||
{
|
||||
private const uint BadNoCommunication = 0x80310000u;
|
||||
|
||||
private static async Task WaitAsync(Func<bool> condition, string because, TimeSpan? timeout = null)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(60));
|
||||
@@ -26,55 +31,124 @@ public sealed class OpcUaClientDriverReconnectWireTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task After_a_server_restart_the_driver_reconnects_and_the_subscription_resumes_delivering()
|
||||
public async Task A_server_restart_goes_Bad_then_the_watchdog_reinit_brings_the_value_back_Good()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-reconnect", ct);
|
||||
await using var _rig = rig;
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
|
||||
|
||||
await using var drv = OpcUaClientDriverWireTests.NewDriver(
|
||||
rig, "wire-reconnect", reconnectPeriod: TimeSpan.FromSeconds(1));
|
||||
await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-reconnect");
|
||||
await drv.InitializeAsync("{}", ct);
|
||||
|
||||
var seen = new List<float>();
|
||||
var seen = new List<(uint Status, object? Value)>();
|
||||
drv.OnDataChange += (_, e) =>
|
||||
{
|
||||
if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef && e.Snapshot.StatusCode == 0u)
|
||||
lock (seen) seen.Add(Convert.ToSingle(e.Snapshot.Value));
|
||||
if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
|
||||
lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value));
|
||||
};
|
||||
var states = new List<HostState>();
|
||||
drv.OnHostStatusChanged += (_, e) => { lock (states) states.Add(e.NewState); };
|
||||
|
||||
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
|
||||
await WaitAsync(() => { lock (seen) return seen.Contains(10.0f); }, "initial value must arrive before the outage");
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 10.0f)); },
|
||||
"initial value must arrive before the outage");
|
||||
|
||||
// Outage: the loss must be DETECTED (host leaves Running) while the server is genuinely down.
|
||||
// Outage: the loss must be REPORTED while the server is genuinely down — Bad on the value with the value
|
||||
// retained, and a health state the watchdog can act on.
|
||||
await rig.RestartAsync(ct, whileDown: async () =>
|
||||
{
|
||||
await WaitAsync(
|
||||
() => { lock (states) return states.Any(s => s != HostState.Running); },
|
||||
"the driver must notice the server is gone", TimeSpan.FromSeconds(30));
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
|
||||
"a lost session must publish BadNoCommunication for the subscribed ref", TimeSpan.FromSeconds(30));
|
||||
lock (seen) seen.Single(s => s.Status == BadNoCommunication).Value.ShouldBe(10.0f);
|
||||
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
|
||||
drv.GetHostStatuses().Single().State.ShouldNotBe(HostState.Running);
|
||||
|
||||
// The read stamp must stay frozen for the whole outage — it is the watchdog's silence measure.
|
||||
var frozen = drv.GetHealth().LastSuccessfulRead;
|
||||
frozen.ShouldNotBeNull();
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), ct); // three keep-alive intervals
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
|
||||
});
|
||||
|
||||
// The new server is empty: redeploy the nodes and publish a value no earlier delivery could have carried.
|
||||
// The restarted server knows nothing of the old session, so the driver stays down on its own …
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), ct);
|
||||
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "nothing in the driver may paper over a dead session");
|
||||
lock (seen) seen.Count(s => s.Status == BadNoCommunication).ShouldBe(1, "one Bad per outage, not per keep-alive");
|
||||
|
||||
// … until the host's comms-loss watchdog re-initialises it (DriverInstanceActor: Reconnecting → ShutdownAsync +
|
||||
// InitializeAsync → Connected → re-subscribe). The new server is empty: redeploy the nodes first, holding the
|
||||
// SAME value as before the outage.
|
||||
rig.EnsureRawVariable("Speed", "Float", writable: true);
|
||||
rig.EnsureRawVariable("Counter", "Int32", writable: false);
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 77.0f);
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 10.0f);
|
||||
|
||||
await WaitAsync(() => drv.GetHostStatuses().Single().State == HostState.Running,
|
||||
"the driver must reconnect after the server is back");
|
||||
await WaitAsync(() => { lock (seen) return seen.Contains(77.0f); },
|
||||
"the pre-outage subscription must resume and deliver the post-restart value");
|
||||
await drv.ShutdownAsync(ct);
|
||||
await drv.InitializeAsync("{}", ct);
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
int before;
|
||||
lock (seen) before = seen.Count;
|
||||
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
|
||||
|
||||
// The IDENTICAL value comes back Good — a (value, status) dedup downstream lets it through.
|
||||
await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 10.0f)); },
|
||||
"the re-subscribed ref must come back Good with its value");
|
||||
|
||||
// Ongoing delivery, not just a one-off initial value.
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 88.0f);
|
||||
await WaitAsync(() => { lock (seen) return seen.Contains(88.0f); },
|
||||
"a later change must also reach the resumed subscription");
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 88.0f)); },
|
||||
"a later change must also reach the new subscription");
|
||||
drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running);
|
||||
|
||||
// And the data plane works for plain reads on the new session.
|
||||
var back = await drv.ReadAsync([OpcUaClientDriverWireTests.SpeedRef], ct);
|
||||
back[0].StatusCode.ShouldBe(0u);
|
||||
Convert.ToSingle(back[0].Value).ShouldBe(88.0f);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #670 review M1 — the subscription dies while the SESSION lives (in production: an outage longer than the
|
||||
/// subscription lifetime but shorter than the session timeout; here: the server deletes it). Keep-alives stay
|
||||
/// Good throughout, so a driver that only watched them would end Healthy with fresh read stamps and a frozen
|
||||
/// value forever. It must go Bad, stay non-Healthy with the stamp frozen (so the #527 watchdog fires), and the
|
||||
/// watchdog's re-initialise must bring the value back.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_subscription_lost_while_the_session_lives_is_reported_and_recovered_by_reinit()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await OpcUaClientDriverWireTests.StartRigAsync("drv-sublost", ct);
|
||||
OpcUaClientDriverWireTests.Publish(rig, OpcUaClientDriverWireTests.SpeedId, 5.0f);
|
||||
|
||||
await using var drv = OpcUaClientDriverWireTests.NewDriver(rig, "wire-sublost");
|
||||
await drv.InitializeAsync("{}", ct);
|
||||
var seen = new List<(uint Status, object? Value)>();
|
||||
drv.OnDataChange += (_, e) =>
|
||||
{
|
||||
if (e.FullReference == OpcUaClientDriverWireTests.SpeedRef)
|
||||
lock (seen) seen.Add((e.Snapshot.StatusCode, e.Snapshot.Value));
|
||||
};
|
||||
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s is (0u, 5.0f)); }, "initial value");
|
||||
|
||||
// Delete the subscription on the server behind the SDK's back; the session itself stays healthy.
|
||||
var session = drv.Session.ShouldNotBeNull();
|
||||
var subId = session.Subscriptions.Single(s => s.Created).Id;
|
||||
await session.DeleteSubscriptionsAsync(null, [subId], ct);
|
||||
|
||||
await WaitAsync(() => { lock (seen) return seen.Any(s => s.Status == BadNoCommunication); },
|
||||
"a subscription the server dropped must be reported as comms loss", TimeSpan.FromSeconds(30));
|
||||
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy);
|
||||
var frozen = drv.GetHealth().LastSuccessfulRead;
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), ct); // good keep-alives keep arriving …
|
||||
drv.GetHealth().State.ShouldNotBe(DriverState.Healthy, "… and must not revive a driver whose data plane is dead");
|
||||
drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen);
|
||||
drv.GetHostStatuses().Single().State.ShouldBe(HostState.Running, "the session itself is fine");
|
||||
|
||||
// The watchdog's re-initialise recreates the subscription and the value comes back Good.
|
||||
await drv.ShutdownAsync(ct);
|
||||
await drv.InitializeAsync("{}", ct);
|
||||
int before;
|
||||
lock (seen) before = seen.Count;
|
||||
await drv.SubscribeAsync([OpcUaClientDriverWireTests.SpeedRef], TimeSpan.FromMilliseconds(100), ct);
|
||||
await WaitAsync(() => { lock (seen) return seen.Skip(before).Any(s => s is (0u, 5.0f)); },
|
||||
"the re-created subscription must deliver the value Good");
|
||||
drv.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user