dac7e1885d
The connect path was the one capability call the resilience pipeline never saw, so a driver that could not connect contributed zero breaker input and the /hosts resilience panel read 'All targets healthy' against a dead endpoint. InitializeAsync now routes through the invoker as DriverCapability.Initialize, keyed on the driver-instance id. - No pipeline retry: the actor's RetryConnect timer is the retry loop (parser forces retryCount overrides to 0, same funnel as write-shaped). - Breaker threshold 2, not the Read-sized 5: the breaker samples 30 s and the reconnect timer fires every 10 s, so at most three attempts fit one window — threshold 5 could never open. Pinned by its own test. - An open breaker fast-fails into the same InitializeFailed path as a real connect failure; the 15 s break window costs at most one extra tick. The best-effort teardown ShutdownAsync stays unwrapped. - Companion fix: DriverCapabilityInvokerFactory.Create now invalidates the instance's tracker snapshots with its pipelines — a respawn during an outage discarded the only breaker whose OnClosed could clear the OPEN flag, stranding a false 'Breaker OPEN' chip forever. Live gate PASSED on docker-dev (image symbol-verified per container): rtu-gate against dead 10.100.0.35:5021 opened its Initialize breaker on both centrals with the red 'BREAKER OPEN' chip on /hosts; a socat 5021→5020 forwarder recovered it (both breakers closed within 1 s, panel back to 'All targets healthy'); healthy MAIN-modbus recorded zero breaker events. Full suite 50/50 green. Evidence: docs/plans/2026-08-07-adminui-followups-and-live-gates.md §7.
176 lines
6.4 KiB
C#
176 lines
6.4 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
|
|
|
|
[Trait("Category", "Unit")]
|
|
public sealed class DriverResilienceStatusTrackerTests
|
|
{
|
|
private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
|
|
|
|
/// <summary>Verifies that TryGet returns null before any write operations.</summary>
|
|
[Fact]
|
|
public void TryGet_Returns_Null_Before_AnyWrite()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
tracker.TryGet("drv", "host").ShouldBeNull();
|
|
}
|
|
|
|
/// <summary>Verifies that RecordFailure accumulates consecutive failures.</summary>
|
|
[Fact]
|
|
public void RecordFailure_Accumulates_ConsecutiveFailures()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
tracker.RecordFailure("drv", "host", Now);
|
|
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
|
|
tracker.RecordFailure("drv", "host", Now.AddSeconds(2));
|
|
|
|
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(3);
|
|
}
|
|
|
|
/// <summary>Verifies that RecordSuccess resets consecutive failures to zero.</summary>
|
|
[Fact]
|
|
public void RecordSuccess_Resets_ConsecutiveFailures()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
tracker.RecordFailure("drv", "host", Now);
|
|
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
|
|
|
|
tracker.RecordSuccess("drv", "host", Now.AddSeconds(2));
|
|
|
|
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>Verifies that RecordBreakerOpen populates the LastBreakerOpenUtc timestamp.</summary>
|
|
[Fact]
|
|
public void RecordBreakerOpen_Populates_LastBreakerOpenUtc()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
tracker.RecordBreakerOpen("drv", "host", Now);
|
|
|
|
tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now);
|
|
}
|
|
|
|
/// <summary>Verifies that RecordBreakerClose stamps the close timestamp.</summary>
|
|
[Fact]
|
|
public void RecordBreakerClose_StampsCloseTime()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
tracker.RecordBreakerOpen("drv", "host", Now);
|
|
|
|
tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20));
|
|
|
|
tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20));
|
|
}
|
|
|
|
/// <summary>Verifies that IsBreakerOpen tracks open → close → reopen transitions.</summary>
|
|
[Fact]
|
|
public void IsBreakerOpen_TracksOpenCloseReopen()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
// A snapshot with no breaker event is never "open".
|
|
tracker.RecordFailure("drv", "host", Now);
|
|
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();
|
|
|
|
// Opened.
|
|
tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1));
|
|
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
|
|
|
|
// Closed later → no longer open.
|
|
tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2));
|
|
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();
|
|
|
|
// Re-opened after the last close → open again.
|
|
tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3));
|
|
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Invalidate drops every entry for one instance and leaves other instances alone. Without
|
|
/// this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale
|
|
/// "Breaker OPEN" snapshot: the respawn discards the pipelines whose <c>OnClosed</c> is the
|
|
/// only thing that stamps the close, so nothing can ever clear the flag again.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Invalidate_DropsTheInstance_LeavesOthers()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
tracker.RecordBreakerOpen("drv", "host-a", Now);
|
|
tracker.RecordBreakerOpen("drv", "host-b", Now);
|
|
tracker.RecordBreakerOpen("other", "host-a", Now);
|
|
|
|
tracker.Invalidate("drv");
|
|
|
|
tracker.TryGet("drv", "host-a").ShouldBeNull();
|
|
tracker.TryGet("drv", "host-b").ShouldBeNull();
|
|
tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>Verifies that RecordRecycle populates the LastRecycleUtc timestamp.</summary>
|
|
[Fact]
|
|
public void RecordRecycle_Populates_LastRecycleUtc()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
tracker.RecordRecycle("drv", "host", Now);
|
|
|
|
tracker.TryGet("drv", "host")!.LastRecycleUtc.ShouldBe(Now);
|
|
}
|
|
|
|
/// <summary>Verifies that RecordFootprint captures baseline and current memory usage.</summary>
|
|
[Fact]
|
|
public void RecordFootprint_CapturesBaselineAndCurrent()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
tracker.RecordFootprint("drv", "host", baselineBytes: 100_000_000, currentBytes: 150_000_000, Now);
|
|
|
|
var snap = tracker.TryGet("drv", "host")!;
|
|
snap.BaselineFootprintBytes.ShouldBe(100_000_000);
|
|
snap.CurrentFootprintBytes.ShouldBe(150_000_000);
|
|
}
|
|
|
|
/// <summary>Verifies that different hosts are tracked independently.</summary>
|
|
[Fact]
|
|
public void DifferentHosts_AreIndependent()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
|
|
tracker.RecordFailure("drv", "host-a", Now);
|
|
tracker.RecordFailure("drv", "host-b", Now);
|
|
tracker.RecordSuccess("drv", "host-a", Now.AddSeconds(1));
|
|
|
|
tracker.TryGet("drv", "host-a")!.ConsecutiveFailures.ShouldBe(0);
|
|
tracker.TryGet("drv", "host-b")!.ConsecutiveFailures.ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>Verifies that Snapshot returns all tracked driver-host pairs.</summary>
|
|
[Fact]
|
|
public void Snapshot_ReturnsAll_TrackedPairs()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
tracker.RecordFailure("drv-1", "host-a", Now);
|
|
tracker.RecordFailure("drv-1", "host-b", Now);
|
|
tracker.RecordFailure("drv-2", "host-a", Now);
|
|
|
|
var snapshot = tracker.Snapshot();
|
|
|
|
snapshot.Count.ShouldBe(3);
|
|
}
|
|
|
|
/// <summary>Verifies that concurrent writes do not lose failure records.</summary>
|
|
[Fact]
|
|
public void ConcurrentWrites_DoNotLose_Failures()
|
|
{
|
|
var tracker = new DriverResilienceStatusTracker();
|
|
Parallel.For(0, 500, _ => tracker.RecordFailure("drv", "host", Now));
|
|
|
|
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(500);
|
|
}
|
|
}
|