lmxopcua/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Resilience;

namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;

[Trait("Category", "Unit")]
public sealed class DriverResilienceStatusTrackerTests
{
    private static readonly DateTime Now = new(2026, 4, 19, 12, 0, 0, DateTimeKind.Utc);

    /// <summary>Verifies that TryGet returns null before any write operations.</summary>
    [Fact]
    public void TryGet_Returns_Null_Before_AnyWrite()
    {
        var tracker = new DriverResilienceStatusTracker();

        tracker.TryGet("drv", "host").ShouldBeNull();
    }

    /// <summary>Verifies that RecordFailure accumulates consecutive failures.</summary>
    [Fact]
    public void RecordFailure_Accumulates_ConsecutiveFailures()
    {
        var tracker = new DriverResilienceStatusTracker();

        tracker.RecordFailure("drv", "host", Now);
        tracker.RecordFailure("drv", "host", Now.AddSeconds(1));
        tracker.RecordFailure("drv", "host", Now.AddSeconds(2));

        tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(3);
    }

    /// <summary>Verifies that RecordSuccess resets consecutive failures to zero.</summary>
    [Fact]
    public void RecordSuccess_Resets_ConsecutiveFailures()
    {
        var tracker = new DriverResilienceStatusTracker();
        tracker.RecordFailure("drv", "host", Now);
        tracker.RecordFailure("drv", "host", Now.AddSeconds(1));

        tracker.RecordSuccess("drv", "host", Now.AddSeconds(2));

        tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(0);
    }

    /// <summary>Verifies that RecordBreakerOpen populates the LastBreakerOpenUtc timestamp.</summary>
    [Fact]
    public void RecordBreakerOpen_Populates_LastBreakerOpenUtc()
    {
        var tracker = new DriverResilienceStatusTracker();

        tracker.RecordBreakerOpen("drv", "host", Now);

        tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now);
    }

    /// <summary>Verifies that RecordBreakerClose stamps the close timestamp.</summary>
    [Fact]
    public void RecordBreakerClose_StampsCloseTime()
    {
        var tracker = new DriverResilienceStatusTracker();
        tracker.RecordBreakerOpen("drv", "host", Now);

        tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20));

        tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20));
    }

    /// <summary>Verifies that IsBreakerOpen tracks open → close → reopen transitions.</summary>
    [Fact]
    public void IsBreakerOpen_TracksOpenCloseReopen()
    {
        var tracker = new DriverResilienceStatusTracker();

        // A snapshot with no breaker event is never "open".
        tracker.RecordFailure("drv", "host", Now);
        tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();

        // Opened.
        tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1));
        tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();

        // Closed later → no longer open.
        tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2));
        tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse();

        // Re-opened after the last close → open again.
        tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3));
        tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue();
    }

    /// <summary>
    ///     Invalidate drops every entry for one instance and leaves other instances alone. Without
    ///     this, a driver-child respawn (deploy) while a breaker is OPEN strands a permanently-stale
    ///     "Breaker OPEN" snapshot: the respawn discards the pipelines whose <c>OnClosed</c> is the
    ///     only thing that stamps the close, so nothing can ever clear the flag again.
    /// </summary>
    [Fact]
    public void Invalidate_DropsTheInstance_LeavesOthers()
    {
        var tracker = new DriverResilienceStatusTracker();
        tracker.RecordBreakerOpen("drv", "host-a", Now);
        tracker.RecordBreakerOpen("drv", "host-b", Now);
        tracker.RecordBreakerOpen("other", "host-a", Now);

        tracker.Invalidate("drv");

        tracker.TryGet("drv", "host-a").ShouldBeNull();
        tracker.TryGet("drv", "host-b").ShouldBeNull();
        tracker.TryGet("other", "host-a")!.IsBreakerOpen.ShouldBeTrue();
    }

    /// <summary>Verifies that RecordRecycle populates the LastRecycleUtc timestamp.</summary>
    [Fact]
    public void RecordRecycle_Populates_LastRecycleUtc()
    {
        var tracker = new DriverResilienceStatusTracker();

        tracker.RecordRecycle("drv", "host", Now);

        tracker.TryGet("drv", "host")!.LastRecycleUtc.ShouldBe(Now);
    }

    /// <summary>Verifies that RecordFootprint captures baseline and current memory usage.</summary>
    [Fact]
    public void RecordFootprint_CapturesBaselineAndCurrent()
    {
        var tracker = new DriverResilienceStatusTracker();

        tracker.RecordFootprint("drv", "host", baselineBytes: 100_000_000, currentBytes: 150_000_000, Now);

        var snap = tracker.TryGet("drv", "host")!;
        snap.BaselineFootprintBytes.ShouldBe(100_000_000);
        snap.CurrentFootprintBytes.ShouldBe(150_000_000);
    }

    /// <summary>Verifies that different hosts are tracked independently.</summary>
    [Fact]
    public void DifferentHosts_AreIndependent()
    {
        var tracker = new DriverResilienceStatusTracker();

        tracker.RecordFailure("drv", "host-a", Now);
        tracker.RecordFailure("drv", "host-b", Now);
        tracker.RecordSuccess("drv", "host-a", Now.AddSeconds(1));

        tracker.TryGet("drv", "host-a")!.ConsecutiveFailures.ShouldBe(0);
        tracker.TryGet("drv", "host-b")!.ConsecutiveFailures.ShouldBe(1);
    }

    /// <summary>Verifies that Snapshot returns all tracked driver-host pairs.</summary>
    [Fact]
    public void Snapshot_ReturnsAll_TrackedPairs()
    {
        var tracker = new DriverResilienceStatusTracker();
        tracker.RecordFailure("drv-1", "host-a", Now);
        tracker.RecordFailure("drv-1", "host-b", Now);
        tracker.RecordFailure("drv-2", "host-a", Now);

        var snapshot = tracker.Snapshot();

        snapshot.Count.ShouldBe(3);
    }

    /// <summary>Verifies that concurrent writes do not lose failure records.</summary>
    [Fact]
    public void ConcurrentWrites_DoNotLose_Failures()
    {
        var tracker = new DriverResilienceStatusTracker();
        Parallel.For(0, 500, _ => tracker.RecordFailure("drv", "host", Now));

        tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(500);
    }

    /// <summary>
    ///     #609 — <c>Retire</c> (a removed instance) drops the instance's snapshots AND ignores every write
    ///     that arrives after it. A removed driver's child stops asynchronously, so a capability call it still
    ///     had in flight finishes through its invoker after the host released it; each of those writes used to
    ///     re-create the entry, and the resilience publisher re-emitted the removed driver every tick, forever.
    ///     The neighbour is the positive control: the same writes against a live instance still land.
    /// </summary>
    [Fact]
    public void Retire_drops_the_instance_and_ignores_every_late_write_for_it()
    {
        var tracker = new DriverResilienceStatusTracker();
        tracker.RecordBreakerOpen("gone", "host-a", Now);
        tracker.RecordFailure("gone", "host-b", Now);

        tracker.Retire("gone").ShouldBe(2);

        foreach (var id in new[] { "gone", "live" })
        {
            tracker.RecordCallStart(id, "host-a");
            tracker.RecordSuccess(id, "host-a", Now);
            tracker.RecordCallComplete(id, "host-a");
            tracker.RecordFailure(id, "host-a", Now);
            tracker.RecordBreakerOpen(id, "host-a", Now);
            tracker.RecordBreakerClose(id, "host-a", Now);
            tracker.RecordRecycle(id, "host-a", Now);
            tracker.RecordFootprint(id, "host-a", 1, 2, Now);
        }

        tracker.Snapshot().Select(e => e.DriverInstanceId).ShouldBe(new[] { "live" });
        tracker.TryGet("gone", "host-a").ShouldBeNull();
        tracker.TryGet("gone", "host-b").ShouldBeNull();
    }

    /// <summary>
    ///     #609 — a retired id is not retired forever: the same instance id re-added in a later deployment is
    ///     <c>Reinstate</c>d (by the invoker factory's <c>Create</c>) and records again.
    /// </summary>
    [Fact]
    public void Reinstate_lets_a_retired_instance_record_again()
    {
        var tracker = new DriverResilienceStatusTracker();
        tracker.Retire("drv");
        tracker.RecordFailure("drv", "host", Now);
        tracker.TryGet("drv", "host").ShouldBeNull(); // control: still retired

        tracker.Reinstate("drv");
        tracker.RecordFailure("drv", "host", Now);

        tracker.TryGet("drv", "host")!.ConsecutiveFailures.ShouldBe(1);
    }

    /// <summary>
    ///     #609 — a write racing <c>Retire</c> never survives it. Writers hammer the instance while it is
    ///     retired; whatever interleaving occurs, nothing is left once both sides finish.
    /// </summary>
    [Fact]
    public async Task A_write_racing_Retire_never_resurrects_the_instance()
    {
        for (var round = 0; round < 50; round++)
        {
            var tracker = new DriverResilienceStatusTracker();
            tracker.RecordFailure("drv", "host", Now);
            using var go = new ManualResetEventSlim();
            var writers = Enumerable.Range(0, 4).Select(_ => Task.Run(() =>
            {
                go.Wait();
                for (var i = 0; i < 200; i++) tracker.RecordFailure("drv", "host", Now);
            }, TestContext.Current.CancellationToken)).ToArray();

            go.Set();
            tracker.Retire("drv");
            await Task.WhenAll(writers);

            tracker.TryGet("drv", "host").ShouldBeNull($"round {round}");
        }
    }
}