Merge fix/670-followups: outage Bad stamped at detection; post-recovery replay guard (#670 live-gate F1/F2)

This commit is contained in:
Joseph Doherty
2026-10-11 09:16:03 -04:00
5 changed files with 277 additions and 5 deletions
@@ -468,4 +468,173 @@ public sealed class OpcUaClientSessionLossTests
public void Log<TState>(Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId,
TState state, Exception? exception, Func<TState, Exception?, string> formatter) => Count++;
}
/// <summary>
/// #670 follow-up F1 (live gate): the outage Bad carried the last Good sample's SOURCE timestamp, so the
/// recorder (which historizes <c>SourceTimestampUtc ?? ServerTimestampUtc</c>) wrote it at the Good row's key —
/// the historian's insert-if-absent (TagId, SourceTimeUtc) key dropped it and no outage marker was ever stored.
/// The Bad is a fresh observation: it is stamped with the detection time, like the host's own comms-loss fan.
/// </summary>
[Fact]
public void The_outage_Bad_is_stamped_with_the_detection_time_not_the_last_good_source_time()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var goodAt = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(42.5f, goodAt));
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
clock.Advance(TimeSpan.FromSeconds(37));
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
var bad = seen.ShouldHaveSingleItem().Snapshot;
bad.StatusCode.ShouldBe(BadNoCommunication);
bad.Value.ShouldBe(42.5f, "the value is still retained");
bad.SourceTimestampUtc.ShouldBe(clock.Now.UtcDateTime, "a Bad at the Good row's source time collides with it in history");
bad.ServerTimestampUtc.ShouldBe(clock.Now.UtcDateTime);
}
private static void SetupReadAt(Mock<ISession> session, object value, DateTime sourceTime) =>
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = [new DataValue(new Variant(value), StatusCodes.Good, sourceTime, sourceTime)],
DiagnosticInfos = [],
});
/// <summary>Drives a same-session outage + recovery (bad → good keep-alive) and waits for the recovery read.</summary>
private static async Task RecoverInPlaceAsync(OpcUaClientDriver drv)
{
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
}
/// <summary>
/// #670 follow-up F2 (live gate): after a same-session recovery the SDK delivered one notification late with
/// an OLDER source time, and the node regressed to it. Inside the post-recovery replay window an older
/// notification is dropped; an equal one (a status-only change) passes and disarms the ref, after which the
/// ref is back to accepting anything.
/// </summary>
[Fact]
public async Task After_a_recovery_a_late_older_notification_is_dropped_until_the_ref_is_back_in_order()
{
var (drv, session, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(668, t0.AddSeconds(2)));
SetupReadAt(session, 668, t0.AddSeconds(2));
await RecoverInPlaceAsync(drv);
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(667, t0.AddSeconds(1))); // late replay
drv.DeliverNotificationForTest(handle, SpeedRef,
new DataValue(new Variant(668), StatusCodes.UncertainLastUsableValue, t0.AddSeconds(2), t0.AddSeconds(2))); // same ts, status-only
drv.DeliverNotificationForTest(handle, SpeedRef, Good(669, t0.AddSeconds(3)));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(665, t0)); // disarmed: accepted
seen.Select(e => e.Snapshot.Value).ShouldBe(new object[] { 668, 669, 665 });
seen[0].Snapshot.StatusCode.ShouldBe((uint)StatusCodes.UncertainLastUsableValue);
}
/// <summary>
/// Coordinator review of F2: with no recovery in play, an upstream clock that steps BACKWARDS (NTP correction,
/// a PLC with a reset clock) must not freeze the node — every notification is published, whatever its time.
/// </summary>
[Fact]
public void In_steady_state_a_clock_step_backwards_still_publishes()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1, t0.AddHours(1)));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(2, t0)); // clock stepped back an hour
drv.DeliverNotificationForTest(handle, SpeedRef, Good(3, t0.AddSeconds(1)));
seen.Select(e => e.Snapshot.Value).ShouldBe(new object[] { 1, 2, 3 });
}
/// <summary>The replay guard is bounded in time: once its window has passed, an older notification is accepted
/// even for a ref that never saw an in-order one.</summary>
[Fact]
public async Task The_replay_guard_expires()
{
var (drv, session, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(10, t0.AddSeconds(5)));
SetupReadAt(session, 10, t0.AddSeconds(5));
await RecoverInPlaceAsync(drv);
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
clock.Advance(TimeSpan.FromSeconds(29));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(9, t0));
seen.ShouldBeEmpty("still inside the 30 s window");
clock.Advance(TimeSpan.FromSeconds(2));
drv.DeliverNotificationForTest(handle, SpeedRef, Good(8, t0));
seen.ShouldHaveSingleItem().Snapshot.Value.ShouldBe(8);
}
[Fact]
public void The_outage_Bad_does_not_poison_the_ordering_check()
{
var (drv, _, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(1, t0));
clock.Advance(TimeSpan.FromSeconds(60));
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication); // stamped t0+60 s
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
// An upstream sample produced during the outage (server-side) and delivered on recovery: older than the Bad's
// detection stamp, newer than the last Good — it is real data and must pass.
drv.DeliverNotificationForTest(handle, SpeedRef, Good(2, t0.AddSeconds(30)));
seen.ShouldHaveSingleItem().Snapshot.Value.ShouldBe(2);
}
[Fact]
public async Task A_late_notification_older_than_the_recovery_read_is_dropped()
{
var (drv, session, clock) = NewConnectedDriver();
using var _ = drv;
var handle = drv.AddSubscriptionForTest([SpeedRef]);
var t0 = clock.Now.UtcDateTime;
drv.DeliverNotificationForTest(handle, SpeedRef, Good(666, t0));
var readAt = t0.AddSeconds(10);
session.Setup(s => s.ReadAsync(
It.IsAny<RequestHeader>(), It.IsAny<double>(), It.IsAny<TimestampsToReturn>(),
It.IsAny<ReadValueIdCollection>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new ReadResponse
{
ResponseHeader = new ResponseHeader(),
Results = [new DataValue(new Variant(670), StatusCodes.Good, readAt, readAt)],
DiagnosticInfos = [],
});
drv.RaiseKeepAliveForTest(StatusCodes.BadNoCommunication);
drv.RaiseKeepAliveForTest(StatusCodes.Good);
await drv.RecoveryTaskForTest;
var seen = new List<DataChangeEventArgs>();
drv.OnDataChange += (_, e) => seen.Add(e);
drv.DeliverNotificationForTest(handle, SpeedRef, Good(667, t0.AddSeconds(5))); // replayed, older than the read
seen.ShouldBeEmpty();
}
}