using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Resilience;
namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;
[Trait("Category", "Unit")]
public sealed class DriverResilienceStatusTrackerTests
{
private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);
/// Verifies that TryGet returns null before any write operations.
[Fact]
public void TryGet_Returns_Null_Before_AnyWrite()
{
var tracker = new DriverResilienceStatusTracker();
tracker.TryGet("drv", "host").ShouldBeNull();
}
/// Verifies that RecordFailure accumulates consecutive failures.
[Fact]
public void RecordFailure_Accumulates_ConsecutiveFailures()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host", Now);
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
tracker.RecordFailure("drv", "host", Now.AddSeconds(2));
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(3);
}
/// Verifies that RecordSuccess resets consecutive failures to zero.
[Fact]
public void RecordSuccess_Resets_ConsecutiveFailures()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host", Now);
tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
tracker.RecordSuccess("drv", "host", Now.AddSeconds(2));
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(0);
}
/// Verifies that RecordBreakerOpen populates the LastBreakerOpenUtc timestamp.
[Fact]
public void RecordBreakerOpen_Populates_LastBreakerOpenUtc()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("drv", "host", Now);
tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now);
}
/// Verifies that RecordBreakerClose stamps the close timestamp.
[Fact]
public void RecordBreakerClose_StampsCloseTime()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("drv", "host", Now);
tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20));
tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20));
}
/// Verifies that IsBreakerOpen tracks open → close → reopen transitions.
[Fact]
public void IsBreakerOpen_TracksOpenCloseReopen()
{
var tracker = new DriverResilienceStatusTracker();
// A snapshot with no breaker event is never "open".
tracker.RecordFailure("drv", "host", Now);
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();
// Opened.
tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1));
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
// Closed later → no longer open.
tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2));
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();
// Re-opened after the last close → open again.
tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3));
tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
}
///
/// Invalidate drops every entry for one instance and leaves other instances alone. Without
/// this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale
/// "Breaker OPEN" snapshot: the respawn discards the pipelines whose OnClosed is the
/// only thing that stamps the close, so nothing can ever clear the flag again.
///
[Fact]
public void Invalidate_DropsTheInstance_LeavesOthers()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("drv", "host-a", Now);
tracker.RecordBreakerOpen("drv", "host-b", Now);
tracker.RecordBreakerOpen("other", "host-a", Now);
tracker.Invalidate("drv");
tracker.TryGet("drv", "host-a").ShouldBeNull();
tracker.TryGet("drv", "host-b").ShouldBeNull();
tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue();
}
/// Verifies that RecordRecycle populates the LastRecycleUtc timestamp.
[Fact]
public void RecordRecycle_Populates_LastRecycleUtc()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordRecycle("drv", "host", Now);
tracker.TryGet("drv", "host")!.LastRecycleUtc.ShouldBe(Now);
}
/// Verifies that RecordFootprint captures baseline and current memory usage.
[Fact]
public void RecordFootprint_CapturesBaselineAndCurrent()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFootprint("drv", "host", baselineBytes: 100_000_000, currentBytes: 150_000_000, Now);
var snap = tracker.TryGet("drv", "host")!;
snap.BaselineFootprintBytes.ShouldBe(100_000_000);
snap.CurrentFootprintBytes.ShouldBe(150_000_000);
}
/// Verifies that different hosts are tracked independently.
[Fact]
public void DifferentHosts_AreIndependent()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host-a", Now);
tracker.RecordFailure("drv", "host-b", Now);
tracker.RecordSuccess("drv", "host-a", Now.AddSeconds(1));
tracker.TryGet("drv", "host-a")!.ConsecutiveFailures.ShouldBe(0);
tracker.TryGet("drv", "host-b")!.ConsecutiveFailures.ShouldBe(1);
}
/// Verifies that Snapshot returns all tracked driver-host pairs.
[Fact]
public void Snapshot_ReturnsAll_TrackedPairs()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv-1", "host-a", Now);
tracker.RecordFailure("drv-1", "host-b", Now);
tracker.RecordFailure("drv-2", "host-a", Now);
var snapshot = tracker.Snapshot();
snapshot.Count.ShouldBe(3);
}
/// Verifies that concurrent writes do not lose failure records.
[Fact]
public void ConcurrentWrites_DoNotLose_Failures()
{
var tracker = new DriverResilienceStatusTracker();
Parallel.For(0, 500, _ => tracker.RecordFailure("drv", "host", Now));
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(500);
}
///
/// #609 — Retire (a removed instance) drops the instance's snapshots AND ignores every write
/// that arrives after it. A removed driver's child stops asynchronously, so a capability call it still
/// had in flight finishes through its invoker after the host released it; each of those writes used to
/// re-create the entry, and the resilience publisher re-emitted the removed driver every tick, forever.
/// The neighbour is the positive control: the same writes against a live instance still land.
///
[Fact]
public void Retire_drops_the_instance_and_ignores_every_late_write_for_it()
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordBreakerOpen("gone", "host-a", Now);
tracker.RecordFailure("gone", "host-b", Now);
tracker.Retire("gone").ShouldBe(2);
foreach (var id in new[] { "gone", "live" })
{
tracker.RecordCallStart(id, "host-a");
tracker.RecordSuccess(id, "host-a", Now);
tracker.RecordCallComplete(id, "host-a");
tracker.RecordFailure(id, "host-a", Now);
tracker.RecordBreakerOpen(id, "host-a", Now);
tracker.RecordBreakerClose(id, "host-a", Now);
tracker.RecordRecycle(id, "host-a", Now);
tracker.RecordFootprint(id, "host-a", 1, 2, Now);
}
tracker.Snapshot().Select(e => e.DriverInstanceId).ShouldBe(new[] { "live" });
tracker.TryGet("gone", "host-a").ShouldBeNull();
tracker.TryGet("gone", "host-b").ShouldBeNull();
}
///
/// #609 — a retired id is not retired forever: the same instance id re-added in a later deployment is
/// Reinstated (by the invoker factory's Create) and records again.
///
[Fact]
public void Reinstate_lets_a_retired_instance_record_again()
{
var tracker = new DriverResilienceStatusTracker();
tracker.Retire("drv");
tracker.RecordFailure("drv", "host", Now);
tracker.TryGet("drv", "host").ShouldBeNull(); // control: still retired
tracker.Reinstate("drv");
tracker.RecordFailure("drv", "host", Now);
tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(1);
}
///
/// #609 — a write racing Retire never survives it. Writers hammer the instance while it is
/// retired; whatever interleaving occurs, nothing is left once both sides finish.
///
[Fact]
public async Task A_write_racing_Retire_never_resurrects_the_instance()
{
for (var round = 0; round < 50; round++)
{
var tracker = new DriverResilienceStatusTracker();
tracker.RecordFailure("drv", "host", Now);
using var go = new ManualResetEventSlim();
var writers = Enumerable.Range(0, 4).Select(_ => Task.Run(() =>
{
go.Wait();
for (var i = 0; i < 200; i++) tracker.RecordFailure("drv", "host", Now);
}, TestContext.Current.CancellationToken)).ToArray();
go.Set();
tracker.Retire("drv");
await Task.WhenAll(writers);
tracker.TryGet("drv", "host").ShouldBeNull($"round {round}");
}
}
}