lmxopcua/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs

using Polly.CircuitBreaker;
using Polly.Timeout;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Resilience;

namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience;

[Trait("Category", "Unit")]
public sealed class DriverResiliencePipelineBuilderTests
{
    private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A };

    [Fact]
    public async Task Read_Retries_Transient_Failures()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
        var attempts = 0;

        await pipeline.ExecuteAsync(async _ =>
        {
            attempts++;
            if (attempts < 3) throw new InvalidOperationException("transient");
            await Task.Yield();
        });

        attempts.ShouldBe(3);
    }

    [Fact]
    public async Task Write_DoesNotRetry_OnFailure()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);
        var attempts = 0;

        var ex = await Should.ThrowAsync<InvalidOperationException>(async () =>
        {
            await pipeline.ExecuteAsync(async _ =>
            {
                attempts++;
                await Task.Yield();
                throw new InvalidOperationException("boom");
            });
        });

        attempts.ShouldBe(1);
        ex.Message.ShouldBe("boom");
    }

    [Fact]
    public async Task AlarmAcknowledge_DoesNotRetry_OnFailure()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.AlarmAcknowledge, TierAOptions);
        var attempts = 0;

        await Should.ThrowAsync<InvalidOperationException>(async () =>
        {
            await pipeline.ExecuteAsync(async _ =>
            {
                attempts++;
                await Task.Yield();
                throw new InvalidOperationException("boom");
            });
        });

        attempts.ShouldBe(1);
    }

    [Fact]
    public void Pipeline_IsIsolated_PerHost()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var driverId = "drv-test";

        var hostA = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
        var hostB = builder.GetOrCreate(driverId, "host-b", DriverCapability.Read, TierAOptions);

        hostA.ShouldNotBeSameAs(hostB);
        builder.CachedPipelineCount.ShouldBe(2);
    }

    [Fact]
    public void Pipeline_IsReused_ForSameTriple()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var driverId = "drv-test";

        var first = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
        var second = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);

        first.ShouldBeSameAs(second);
        builder.CachedPipelineCount.ShouldBe(1);
    }

    [Fact]
    public void Pipeline_IsIsolated_PerCapability()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var driverId = "drv-test";

        var read = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions);
        var write = builder.GetOrCreate(driverId, "host-a", DriverCapability.Write, TierAOptions);

        read.ShouldNotBeSameAs(write);
    }

    [Fact]
    public async Task DeadHost_DoesNotOpenBreaker_ForSiblingHost()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var driverId = "drv-test";

        var deadHost = builder.GetOrCreate(driverId, "dead-plc", DriverCapability.Read, TierAOptions);
        var liveHost = builder.GetOrCreate(driverId, "live-plc", DriverCapability.Read, TierAOptions);

        var threshold = TierAOptions.Resolve(DriverCapability.Read).BreakerFailureThreshold;
        for (var i = 0; i < threshold + 5; i++)
        {
            await Should.ThrowAsync<Exception>(async () =>
                await deadHost.ExecuteAsync(async _ =>
                {
                    await Task.Yield();
                    throw new InvalidOperationException("dead plc");
                }));
        }

        var liveAttempts = 0;
        await liveHost.ExecuteAsync(async _ =>
        {
            liveAttempts++;
            await Task.Yield();
        });

        liveAttempts.ShouldBe(1, "healthy sibling host must not be affected by dead peer");
    }

    [Fact]
    public async Task CircuitBreaker_Opens_AfterFailureThreshold_OnTierA()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions);

        var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
        for (var i = 0; i < threshold; i++)
        {
            await Should.ThrowAsync<InvalidOperationException>(async () =>
                await pipeline.ExecuteAsync(async _ =>
                {
                    await Task.Yield();
                    throw new InvalidOperationException("boom");
                }));
        }

        await Should.ThrowAsync<BrokenCircuitException>(async () =>
            await pipeline.ExecuteAsync(async _ =>
            {
                await Task.Yield();
            }));
    }

    [Fact]
    public async Task Timeout_Cancels_SlowOperation()
    {
        var tierAWithShortTimeout = new DriverResilienceOptions
        {
            Tier = DriverTier.A,
            CapabilityPolicies = new Dictionary<DriverCapability, CapabilityPolicy>
            {
                [DriverCapability.Read] = new(TimeoutSeconds: 1, RetryCount: 0, BreakerFailureThreshold: 5),
            },
        };
        var builder = new DriverResiliencePipelineBuilder();
        var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, tierAWithShortTimeout);

        await Should.ThrowAsync<TimeoutRejectedException>(async () =>
            await pipeline.ExecuteAsync(async ct =>
            {
                await Task.Delay(TimeSpan.FromSeconds(5), ct);
            }));
    }

    [Fact]
    public void Invalidate_Removes_OnlyMatchingInstance()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var keepId = "drv-keep";
        var dropId = "drv-drop";

        builder.GetOrCreate(keepId, "h", DriverCapability.Read,  TierAOptions);
        builder.GetOrCreate(keepId, "h", DriverCapability.Write, TierAOptions);
        builder.GetOrCreate(dropId, "h", DriverCapability.Read,  TierAOptions);

        var removed = builder.Invalidate(dropId);

        removed.ShouldBe(1);
        builder.CachedPipelineCount.ShouldBe(2);
    }

    [Fact]
    public async Task Cancellation_IsNot_Retried()
    {
        var builder = new DriverResiliencePipelineBuilder();
        var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions);
        var attempts = 0;
        using var cts = new CancellationTokenSource();
        cts.Cancel();

        await Should.ThrowAsync<OperationCanceledException>(async () =>
            await pipeline.ExecuteAsync(async ct =>
            {
                attempts++;
                ct.ThrowIfCancellationRequested();
                await Task.Yield();
            }, cts.Token));

        attempts.ShouldBeLessThanOrEqualTo(1);
    }

    [Fact]
    public async Task Tracker_RecordsFailure_OnEveryRetry()
    {
        var tracker = new DriverResilienceStatusTracker();
        var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
        var pipeline = builder.GetOrCreate("drv-trk", "host-x", DriverCapability.Read, TierAOptions);

        await Should.ThrowAsync<InvalidOperationException>(async () =>
            await pipeline.ExecuteAsync(async _ =>
            {
                await Task.Yield();
                throw new InvalidOperationException("always fails");
            }));

        var snap = tracker.TryGet("drv-trk", "host-x");
        snap.ShouldNotBeNull();
        var retryCount = TierAOptions.Resolve(DriverCapability.Read).RetryCount;
        snap!.ConsecutiveFailures.ShouldBe(retryCount);
    }

    [Fact]
    public async Task Tracker_StampsBreakerOpen_WhenBreakerTrips()
    {
        var tracker = new DriverResilienceStatusTracker();
        var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
        var pipeline = builder.GetOrCreate("drv-trk", "host-b", DriverCapability.Write, TierAOptions);

        var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold;
        for (var i = 0; i < threshold; i++)
        {
            await Should.ThrowAsync<InvalidOperationException>(async () =>
                await pipeline.ExecuteAsync(async _ =>
                {
                    await Task.Yield();
                    throw new InvalidOperationException("boom");
                }));
        }

        var snap = tracker.TryGet("drv-trk", "host-b");
        snap.ShouldNotBeNull();
        snap!.LastBreakerOpenUtc.ShouldNotBeNull();
    }

    [Fact]
    public async Task Tracker_IsolatesCounters_PerHost()
    {
        var tracker = new DriverResilienceStatusTracker();
        var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker);
        var dead = builder.GetOrCreate("drv-trk", "dead", DriverCapability.Read, TierAOptions);
        var live = builder.GetOrCreate("drv-trk", "live", DriverCapability.Read, TierAOptions);

        await Should.ThrowAsync<InvalidOperationException>(async () =>
            await dead.ExecuteAsync(async _ =>
            {
                await Task.Yield();
                throw new InvalidOperationException("dead");
            }));
        await live.ExecuteAsync(async _ => await Task.Yield());

        tracker.TryGet("drv-trk", "dead")!.ConsecutiveFailures.ShouldBeGreaterThan(0);
        tracker.TryGet("drv-trk", "live").ShouldBeNull();
    }
}