Files
lmxopcua/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs
T
Joseph Doherty dac7e1885d feat(resilience): route IDriver.InitializeAsync through the capability invoker (R2-10 T10 closed)
The connect path was the one capability call the resilience pipeline never
saw, so a driver that could not connect contributed zero breaker input and
the /hosts resilience panel read 'All targets healthy' against a dead
endpoint. InitializeAsync now routes through the invoker as
DriverCapability.Initialize, keyed on the driver-instance id.

- No pipeline retry: the actor's RetryConnect timer is the retry loop
  (parser forces retryCount overrides to 0, same funnel as write-shaped).
- Breaker threshold 2, not the Read-sized 5: the breaker samples 30 s and
  the reconnect timer fires every 10 s, so at most three attempts fit one
  window — threshold 5 could never open. Pinned by its own test.
- An open breaker fast-fails into the same InitializeFailed path as a real
  connect failure; the 15 s break window costs at most one extra tick.
  The best-effort teardown ShutdownAsync stays unwrapped.
- Companion fix: DriverCapabilityInvokerFactory.Create now invalidates the
  instance's tracker snapshots with its pipelines — a respawn during an
  outage discarded the only breaker whose OnClosed could clear the OPEN
  flag, stranding a false 'Breaker OPEN' chip forever.

Live gate PASSED on docker-dev (image symbol-verified per container):
rtu-gate against dead 10.100.0.35:5021 opened its Initialize breaker on
both centrals with the red 'BREAKER OPEN' chip on /hosts; a socat
5021→5020 forwarder recovered it (both breakers closed within 1 s, panel
back to 'All targets healthy'); healthy MAIN-modbus recorded zero breaker
events. Full suite 50/50 green. Evidence:
docs/plans/2026-08-07-adminui-followups-and-live-gates.md §7.
2026-08-08 05:45:08 -04:00

176 lines
6.4 KiB
C#

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
[Trait("Category", "Unit")]
public sealed class DriverResilienceStatusTrackerTests
{
private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
/// <summary>Verifies that TryGet returns null before any write operations.</summary>
[Fact]
public void TryGet_Returns_Null_Before_AnyWrite()
{
var tracker = new DriverResilienceStatusTracker();
tracker.TryGet("drv", "host").ShouldBeNull();
}
/// <summary>Verifies that RecordFailure accumulates consecutive failures.</summary>
[Fact]
public void RecordFailure_Accumulates_ConsecutiveFailures()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host", Now);
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
tracker.RecordFailure("drv", "host", Now.AddSeconds(2));
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(3);
}
/// <summary>Verifies that RecordSuccess resets consecutive failures to zero.</summary>
[Fact]
public void RecordSuccess_Resets_ConsecutiveFailures()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host", Now);
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
tracker.RecordSuccess("drv", "host", Now.AddSeconds(2));
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(0);
}
/// <summary>Verifies that RecordBreakerOpen populates the LastBreakerOpenUtc timestamp.</summary>
[Fact]
public void RecordBreakerOpen_Populates_LastBreakerOpenUtc()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("drv", "host", Now);
tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now);
}
/// <summary>Verifies that RecordBreakerClose stamps the close timestamp.</summary>
[Fact]
public void RecordBreakerClose_StampsCloseTime()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("drv", "host", Now);
tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20));
tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20));
}
/// <summary>Verifies that IsBreakerOpen tracks open → close → reopen transitions.</summary>
[Fact]
public void IsBreakerOpen_TracksOpenCloseReopen()
{
var tracker = new DriverResilienceStatusTracker();
// A snapshot with no breaker event is never "open".
tracker.RecordFailure("drv", "host", Now);
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();
// Opened.
tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1));
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
// Closed later → no longer open.
tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2));
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();
// Re-opened after the last close → open again.
tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3));
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
}
/// <summary>
/// Invalidate drops every entry for one instance and leaves other instances alone. Without
/// this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale
/// "Breaker OPEN" snapshot: the respawn discards the pipelines whose <c>OnClosed</c> is the
/// only thing that stamps the close, so nothing can ever clear the flag again.
/// </summary>
[Fact]
public void Invalidate_DropsTheInstance_LeavesOthers()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("drv", "host-a", Now);
tracker.RecordBreakerOpen("drv", "host-b", Now);
tracker.RecordBreakerOpen("other", "host-a", Now);
tracker.Invalidate("drv");
tracker.TryGet("drv", "host-a").ShouldBeNull();
tracker.TryGet("drv", "host-b").ShouldBeNull();
tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue();
}
/// <summary>Verifies that RecordRecycle populates the LastRecycleUtc timestamp.</summary>
[Fact]
public void RecordRecycle_Populates_LastRecycleUtc()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordRecycle("drv", "host", Now);
tracker.TryGet("drv", "host")!.LastRecycleUtc.ShouldBe(Now);
}
/// <summary>Verifies that RecordFootprint captures baseline and current memory usage.</summary>
[Fact]
public void RecordFootprint_CapturesBaselineAndCurrent()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFootprint("drv", "host", baselineBytes: 100_000_000, currentBytes: 150_000_000, Now);
var snap = tracker.TryGet("drv", "host")!;
snap.BaselineFootprintBytes.ShouldBe(100_000_000);
snap.CurrentFootprintBytes.ShouldBe(150_000_000);
}
/// <summary>Verifies that different hosts are tracked independently.</summary>
[Fact]
public void DifferentHosts_AreIndependent()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host-a", Now);
tracker.RecordFailure("drv", "host-b", Now);
tracker.RecordSuccess("drv", "host-a", Now.AddSeconds(1));
tracker.TryGet("drv", "host-a")!.ConsecutiveFailures.ShouldBe(0);
tracker.TryGet("drv", "host-b")!.ConsecutiveFailures.ShouldBe(1);
}
/// <summary>Verifies that Snapshot returns all tracked driver-host pairs.</summary>
[Fact]
public void Snapshot_ReturnsAll_TrackedPairs()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv-1", "host-a", Now);
tracker.RecordFailure("drv-1", "host-b", Now);
tracker.RecordFailure("drv-2", "host-a", Now);
var snapshot = tracker.Snapshot();
snapshot.Count.ShouldBe(3);
}
/// <summary>Verifies that concurrent writes do not lose failure records.</summary>
[Fact]
public void ConcurrentWrites_DoNotLose_Failures()
{
var tracker = new DriverResilienceStatusTracker();
Parallel.For(0, 500, _ => tracker.RecordFailure("drv", "host", Now));
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(500);
}
}