using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Resilience; namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience; [Trait("Category", "Unit")] public sealed class DriverResilienceStatusTrackerTests { private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc); /// Verifies that TryGet returns null before any write operations. [Fact] public void TryGet_Returns_Null_Before_AnyWrite() { var tracker = new DriverResilienceStatusTracker(); tracker.TryGet("drv", "host").ShouldBeNull(); } /// Verifies that RecordFailure accumulates consecutive failures. [Fact] public void RecordFailure_Accumulates_ConsecutiveFailures() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordFailure("drv", "host", Now); tracker.RecordFailure("drv", "host", Now.AddSeconds(1)); tracker.RecordFailure("drv", "host", Now.AddSeconds(2)); tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(3); } /// Verifies that RecordSuccess resets consecutive failures to zero. [Fact] public void RecordSuccess_Resets_ConsecutiveFailures() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordFailure("drv", "host", Now); tracker.RecordFailure("drv", "host", Now.AddSeconds(1)); tracker.RecordSuccess("drv", "host", Now.AddSeconds(2)); tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(0); } /// Verifies that RecordBreakerOpen populates the LastBreakerOpenUtc timestamp. [Fact] public void RecordBreakerOpen_Populates_LastBreakerOpenUtc() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordBreakerOpen("drv", "host", Now); tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now); } /// Verifies that RecordBreakerClose stamps the close timestamp. [Fact] public void RecordBreakerClose_StampsCloseTime() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordBreakerOpen("drv", "host", Now); tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20)); tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20)); } /// Verifies that IsBreakerOpen tracks open → close → reopen transitions. [Fact] public void IsBreakerOpen_TracksOpenCloseReopen() { var tracker = new DriverResilienceStatusTracker(); // A snapshot with no breaker event is never "open". tracker.RecordFailure("drv", "host", Now); tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse(); // Opened. tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1)); tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); // Closed later → no longer open. tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2)); tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse(); // Re-opened after the last close → open again. tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3)); tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); } /// /// Invalidate drops every entry for one instance and leaves other instances alone. Without /// this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale /// "Breaker OPEN" snapshot: the respawn discards the pipelines whose OnClosed is the /// only thing that stamps the close, so nothing can ever clear the flag again. /// [Fact] public void Invalidate_DropsTheInstance_LeavesOthers() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordBreakerOpen("drv", "host-a", Now); tracker.RecordBreakerOpen("drv", "host-b", Now); tracker.RecordBreakerOpen("other", "host-a", Now); tracker.Invalidate("drv"); tracker.TryGet("drv", "host-a").ShouldBeNull(); tracker.TryGet("drv", "host-b").ShouldBeNull(); tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue(); } /// Verifies that RecordRecycle populates the LastRecycleUtc timestamp. [Fact] public void RecordRecycle_Populates_LastRecycleUtc() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordRecycle("drv", "host", Now); tracker.TryGet("drv", "host")!.LastRecycleUtc.ShouldBe(Now); } /// Verifies that RecordFootprint captures baseline and current memory usage. [Fact] public void RecordFootprint_CapturesBaselineAndCurrent() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordFootprint("drv", "host", baselineBytes: 100_000_000, currentBytes: 150_000_000, Now); var snap = tracker.TryGet("drv", "host")!; snap.BaselineFootprintBytes.ShouldBe(100_000_000); snap.CurrentFootprintBytes.ShouldBe(150_000_000); } /// Verifies that different hosts are tracked independently. [Fact] public void DifferentHosts_AreIndependent() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordFailure("drv", "host-a", Now); tracker.RecordFailure("drv", "host-b", Now); tracker.RecordSuccess("drv", "host-a", Now.AddSeconds(1)); tracker.TryGet("drv", "host-a")!.ConsecutiveFailures.ShouldBe(0); tracker.TryGet("drv", "host-b")!.ConsecutiveFailures.ShouldBe(1); } /// Verifies that Snapshot returns all tracked driver-host pairs. [Fact] public void Snapshot_ReturnsAll_TrackedPairs() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordFailure("drv-1", "host-a", Now); tracker.RecordFailure("drv-1", "host-b", Now); tracker.RecordFailure("drv-2", "host-a", Now); var snapshot = tracker.Snapshot(); snapshot.Count.ShouldBe(3); } /// Verifies that concurrent writes do not lose failure records. [Fact] public void ConcurrentWrites_DoNotLose_Failures() { var tracker = new DriverResilienceStatusTracker(); Parallel.For(0, 500, _ => tracker.RecordFailure("drv", "host", Now)); tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(500); } /// /// #609 — Retire (a removed instance) drops the instance's snapshots AND ignores every write /// that arrives after it. A removed driver's child stops asynchronously, so a capability call it still /// had in flight finishes through its invoker after the host released it; each of those writes used to /// re-create the entry, and the resilience publisher re-emitted the removed driver every tick, forever. /// The neighbour is the positive control: the same writes against a live instance still land. /// [Fact] public void Retire_drops_the_instance_and_ignores_every_late_write_for_it() { var tracker = new DriverResilienceStatusTracker(); tracker.RecordBreakerOpen("gone", "host-a", Now); tracker.RecordFailure("gone", "host-b", Now); tracker.Retire("gone").ShouldBe(2); foreach (var id in new[] { "gone", "live" }) { tracker.RecordCallStart(id, "host-a"); tracker.RecordSuccess(id, "host-a", Now); tracker.RecordCallComplete(id, "host-a"); tracker.RecordFailure(id, "host-a", Now); tracker.RecordBreakerOpen(id, "host-a", Now); tracker.RecordBreakerClose(id, "host-a", Now); tracker.RecordRecycle(id, "host-a", Now); tracker.RecordFootprint(id, "host-a", 1, 2, Now); } tracker.Snapshot().Select(e => e.DriverInstanceId).ShouldBe(new[] { "live" }); tracker.TryGet("gone", "host-a").ShouldBeNull(); tracker.TryGet("gone", "host-b").ShouldBeNull(); } /// /// #609 — a retired id is not retired forever: the same instance id re-added in a later deployment is /// Reinstated (by the invoker factory's Create) and records again. /// [Fact] public void Reinstate_lets_a_retired_instance_record_again() { var tracker = new DriverResilienceStatusTracker(); tracker.Retire("drv"); tracker.RecordFailure("drv", "host", Now); tracker.TryGet("drv", "host").ShouldBeNull(); // control: still retired tracker.Reinstate("drv"); tracker.RecordFailure("drv", "host", Now); tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(1); } /// /// #609 — a write racing Retire never survives it. Writers hammer the instance while it is /// retired; whatever interleaving occurs, nothing is left once both sides finish. /// [Fact] public async Task A_write_racing_Retire_never_resurrects_the_instance() { for (var round = 0; round < 50; round++) { var tracker = new DriverResilienceStatusTracker(); tracker.RecordFailure("drv", "host", Now); using var go = new ManualResetEventSlim(); var writers = Enumerable.Range(0, 4).Select(_ => Task.Run(() => { go.Wait(); for (var i = 0; i < 200; i++) tracker.RecordFailure("drv", "host", Now); }, TestContext.Current.CancellationToken)).ToArray(); go.Set(); tracker.Retire("drv"); await Task.WhenAll(writers); tracker.TryGet("drv", "host").ShouldBeNull($"round {round}"); } } }