using Polly.CircuitBreaker; using Polly.Timeout; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Resilience; namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience; [Trait("Category", "Unit")] public sealed class DriverResiliencePipelineBuilderTests { private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A }; [Fact] public async Task Read_Retries_Transient_Failures() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions); var attempts = 0; await pipeline.ExecuteAsync(async _ => { attempts++; if (attempts < 3) throw new InvalidOperationException("transient"); await Task.Yield(); }); attempts.ShouldBe(3); } [Fact] public async Task Write_DoesNotRetry_OnFailure() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions); var attempts = 0; var ex = await Should.ThrowAsync(async () => { await pipeline.ExecuteAsync(async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); }); }); attempts.ShouldBe(1); ex.Message.ShouldBe("boom"); } [Fact] public async Task AlarmAcknowledge_DoesNotRetry_OnFailure() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.AlarmAcknowledge, TierAOptions); var attempts = 0; await Should.ThrowAsync(async () => { await pipeline.ExecuteAsync(async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); }); }); attempts.ShouldBe(1); } [Fact] public void Pipeline_IsIsolated_PerHost() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var hostA = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); var hostB = builder.GetOrCreate(driverId, "host-b", DriverCapability.Read, TierAOptions); hostA.ShouldNotBeSameAs(hostB); builder.CachedPipelineCount.ShouldBe(2); } [Fact] public void Pipeline_IsReused_ForSameTriple() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var first = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); var second = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); first.ShouldBeSameAs(second); builder.CachedPipelineCount.ShouldBe(1); } [Fact] public void Pipeline_IsIsolated_PerCapability() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var read = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); var write = builder.GetOrCreate(driverId, "host-a", DriverCapability.Write, TierAOptions); read.ShouldNotBeSameAs(write); } [Fact] public async Task DeadHost_DoesNotOpenBreaker_ForSiblingHost() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var deadHost = builder.GetOrCreate(driverId, "dead-plc", DriverCapability.Read, TierAOptions); var liveHost = builder.GetOrCreate(driverId, "live-plc", DriverCapability.Read, TierAOptions); var threshold = TierAOptions.Resolve(DriverCapability.Read).BreakerFailureThreshold; for (var i = 0; i < threshold + 5; i++) { await Should.ThrowAsync(async () => await deadHost.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("dead plc"); })); } var liveAttempts = 0; await liveHost.ExecuteAsync(async _ => { liveAttempts++; await Task.Yield(); }); liveAttempts.ShouldBe(1, "healthy sibling host must not be affected by dead peer"); } [Fact] public async Task CircuitBreaker_Opens_AfterFailureThreshold_OnTierA() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions); var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold; for (var i = 0; i < threshold; i++) { await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("boom"); })); } await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); })); } [Fact] public async Task Timeout_Cancels_SlowOperation() { var tierAWithShortTimeout = new DriverResilienceOptions { Tier = DriverTier.A, CapabilityPolicies = new Dictionary { [DriverCapability.Read] = new(TimeoutSeconds: 1, RetryCount: 0, BreakerFailureThreshold: 5), }, }; var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, tierAWithShortTimeout); await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async ct => { await Task.Delay(TimeSpan.FromSeconds(5), ct); })); } [Fact] public void Invalidate_Removes_OnlyMatchingInstance() { var builder = new DriverResiliencePipelineBuilder(); var keepId = "drv-keep"; var dropId = "drv-drop"; builder.GetOrCreate(keepId, "h", DriverCapability.Read, TierAOptions); builder.GetOrCreate(keepId, "h", DriverCapability.Write, TierAOptions); builder.GetOrCreate(dropId, "h", DriverCapability.Read, TierAOptions); var removed = builder.Invalidate(dropId); removed.ShouldBe(1); builder.CachedPipelineCount.ShouldBe(2); } [Fact] public async Task Cancellation_IsNot_Retried() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions); var attempts = 0; using var cts = new CancellationTokenSource(); cts.Cancel(); await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async ct => { attempts++; ct.ThrowIfCancellationRequested(); await Task.Yield(); }, cts.Token)); attempts.ShouldBeLessThanOrEqualTo(1); } }