using Microsoft.Extensions.Logging; using Polly.CircuitBreaker; using Polly.Timeout; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Resilience; namespace ZB.MOM.WW.OtOpcUa.Core.Tests.Resilience; [Trait("Category", "Unit")] public sealed class DriverResiliencePipelineBuilderTests { private static readonly DriverResilienceOptions TierAOptions = new() { Tier = DriverTier.A }; /// Verifies that read operations retry transient failures. [Fact] public async Task Read_Retries_Transient_Failures() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions); var attempts = 0; await pipeline.ExecuteAsync(async _ => { attempts++; if (attempts < 3) throw new InvalidOperationException("transient"); await Task.Yield(); }); attempts.ShouldBe(3); } /// Verifies that write operations do not retry on failure. [Fact] public async Task Write_DoesNotRetry_OnFailure() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions); var attempts = 0; var ex = await Should.ThrowAsync(async () => { await pipeline.ExecuteAsync(async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); }); }); attempts.ShouldBe(1); ex.Message.ShouldBe("boom"); } /// Verifies that alarm acknowledge operations do not retry on failure. [Fact] public async Task AlarmAcknowledge_DoesNotRetry_OnFailure() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.AlarmAcknowledge, TierAOptions); var attempts = 0; await Should.ThrowAsync(async () => { await pipeline.ExecuteAsync(async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); }); }); attempts.ShouldBe(1); } /// Verifies that pipelines are isolated per host. [Fact] public void Pipeline_IsIsolated_PerHost() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var hostA = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); var hostB = builder.GetOrCreate(driverId, "host-b", DriverCapability.Read, TierAOptions); hostA.ShouldNotBeSameAs(hostB); builder.CachedPipelineCount.ShouldBe(2); } /// Verifies that pipelines are reused for the same driver, host, and capability triple. [Fact] public void Pipeline_IsReused_ForSameTriple() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var first = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); var second = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); first.ShouldBeSameAs(second); builder.CachedPipelineCount.ShouldBe(1); } /// Verifies that pipelines are isolated per capability. [Fact] public void Pipeline_IsIsolated_PerCapability() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var read = builder.GetOrCreate(driverId, "host-a", DriverCapability.Read, TierAOptions); var write = builder.GetOrCreate(driverId, "host-a", DriverCapability.Write, TierAOptions); read.ShouldNotBeSameAs(write); } /// Verifies that a dead host does not open the breaker for a sibling host. [Fact] public async Task DeadHost_DoesNotOpenBreaker_ForSiblingHost() { var builder = new DriverResiliencePipelineBuilder(); var driverId = "drv-test"; var deadHost = builder.GetOrCreate(driverId, "dead-plc", DriverCapability.Read, TierAOptions); var liveHost = builder.GetOrCreate(driverId, "live-plc", DriverCapability.Read, TierAOptions); var threshold = TierAOptions.Resolve(DriverCapability.Read).BreakerFailureThreshold; for (var i = 0; i < threshold + 5; i++) { await Should.ThrowAsync(async () => await deadHost.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("dead plc"); })); } var liveAttempts = 0; await liveHost.ExecuteAsync(async _ => { liveAttempts++; await Task.Yield(); }); liveAttempts.ShouldBe(1, "healthy sibling host must not be affected by dead peer"); } /// Verifies that the circuit breaker opens after the failure threshold on tier A. [Fact] public async Task CircuitBreaker_Opens_AfterFailureThreshold_OnTierA() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Write, TierAOptions); var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold; for (var i = 0; i < threshold; i++) { await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("boom"); })); } await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); })); } /// Verifies that timeout cancels slow operations. [Fact] public async Task Timeout_Cancels_SlowOperation() { var tierAWithShortTimeout = new DriverResilienceOptions { Tier = DriverTier.A, CapabilityPolicies = new Dictionary { [DriverCapability.Read] = new(TimeoutSeconds: 1, RetryCount: 0, BreakerFailureThreshold: 5), }, }; var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, tierAWithShortTimeout); await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async ct => { await Task.Delay(TimeSpan.FromSeconds(5), ct); })); } /// Verifies that invalidate removes only the matching instance. [Fact] public void Invalidate_Removes_OnlyMatchingInstance() { var builder = new DriverResiliencePipelineBuilder(); var keepId = "drv-keep"; var dropId = "drv-drop"; builder.GetOrCreate(keepId, "h", DriverCapability.Read, TierAOptions); builder.GetOrCreate(keepId, "h", DriverCapability.Write, TierAOptions); builder.GetOrCreate(dropId, "h", DriverCapability.Read, TierAOptions); var removed = builder.Invalidate(dropId); removed.ShouldBe(1); builder.CachedPipelineCount.ShouldBe(2); } /// Verifies that cancellation is not retried. [Fact] public async Task Cancellation_IsNot_Retried() { var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("drv-test", "host-1", DriverCapability.Read, TierAOptions); var attempts = 0; using var cts = new CancellationTokenSource(); cts.Cancel(); await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async ct => { attempts++; ct.ThrowIfCancellationRequested(); await Task.Yield(); }, cts.Token)); attempts.ShouldBeLessThanOrEqualTo(1); } /// Verifies that the tracker records failure on every retry. [Fact] public async Task Tracker_RecordsFailure_OnEveryRetry() { var tracker = new DriverResilienceStatusTracker(); var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker); var pipeline = builder.GetOrCreate("drv-trk", "host-x", DriverCapability.Read, TierAOptions); await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("always fails"); })); var snap = tracker.TryGet("drv-trk", "host-x"); snap.ShouldNotBeNull(); var retryCount = TierAOptions.Resolve(DriverCapability.Read).RetryCount; snap!.ConsecutiveFailures.ShouldBe(retryCount); } /// Verifies that the tracker stamps the breaker open when it trips. [Fact] public async Task Tracker_StampsBreakerOpen_WhenBreakerTrips() { var tracker = new DriverResilienceStatusTracker(); var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker); var pipeline = builder.GetOrCreate("drv-trk", "host-b", DriverCapability.Write, TierAOptions); var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold; for (var i = 0; i < threshold; i++) { await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("boom"); })); } var snap = tracker.TryGet("drv-trk", "host-b"); snap.ShouldNotBeNull(); snap!.LastBreakerOpenUtc.ShouldNotBeNull(); } /// /// R2-10 truthfulness fix: after the breaker opens and later recovers (a successful probe /// past BreakDuration), the pipeline builder's OnClosed hook must record the /// breaker close on the tracker — so /// reads false again and is stamped. /// Uses a controllable so the 15 s break duration elapses deterministically. /// [Fact] public async Task Tracker_RecordsBreakerClose_OnRecovery() { var clock = new ManualTimeProvider(new DateTime(2026, 7, 13, 0, 0, 0, DateTimeKind.Utc)); var tracker = new DriverResilienceStatusTracker(); var builder = new DriverResiliencePipelineBuilder(timeProvider: clock, statusTracker: tracker); var pipeline = builder.GetOrCreate("drv-trk", "host-c", DriverCapability.Write, TierAOptions); // Drive the breaker open with the tier's throughput of failures. var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold; for (var i = 0; i < threshold; i++) { await Should.ThrowAsync(async () => await pipeline.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("boom"); })); } tracker.TryGet("drv-trk", "host-c")!.IsBreakerOpen.ShouldBeTrue("breaker must be open after the failure threshold"); // Elapse the break duration (15 s) so the breaker half-opens, then let a probe succeed → OnClosed. clock.Advance(TimeSpan.FromSeconds(16)); await pipeline.ExecuteAsync(async _ => await Task.Yield()); var snap = tracker.TryGet("drv-trk", "host-c")!; snap.LastBreakerClosedUtc.ShouldNotBeNull("OnClosed must stamp the close time on recovery"); snap.IsBreakerOpen.ShouldBeFalse("the breaker must read closed after a successful recovery probe"); } /// Verifies that the tracker isolates counters per host. [Fact] public async Task Tracker_IsolatesCounters_PerHost() { var tracker = new DriverResilienceStatusTracker(); var builder = new DriverResiliencePipelineBuilder(statusTracker: tracker); var dead = builder.GetOrCreate("drv-trk", "dead", DriverCapability.Read, TierAOptions); var live = builder.GetOrCreate("drv-trk", "live", DriverCapability.Read, TierAOptions); await Should.ThrowAsync(async () => await dead.ExecuteAsync(async _ => { await Task.Yield(); throw new InvalidOperationException("dead"); })); await live.ExecuteAsync(async _ => await Task.Yield()); tracker.TryGet("drv-trk", "dead")!.ConsecutiveFailures.ShouldBeGreaterThan(0); tracker.TryGet("drv-trk", "live").ShouldBeNull(); } /// /// Retry events are logged when a logger is supplied — this is the operator-facing observability /// surface the arch-review #10 live-verify relies on (the Admin /hosts reader isn't built). /// Without it the resilience pipeline runs silently and a live retry/breaker transition is invisible. /// [Fact] public async Task Retry_Is_Logged_When_Logger_Supplied() { var logger = new CapturingLogger(); var builder = new DriverResiliencePipelineBuilder(logger: logger); var pipeline = builder.GetOrCreate("drv-log", "host-9", DriverCapability.Read, TierAOptions); var attempts = 0; await pipeline.ExecuteAsync(async _ => { attempts++; if (attempts < 3) throw new InvalidOperationException("transient"); await Task.Yield(); }); attempts.ShouldBe(3); logger.Entries.ShouldContain(e => e.Level == LogLevel.Warning && e.Message.Contains("resilience retry") && e.Message.Contains("host-9")); } /// /// 01/S-6 builder invariant: building a pipeline from a hostile, directly-constructed /// (bypassing the parser) must NEVER throw Polly's /// ValidationException. Sweeps the full matrix of {int.MinValue, -1, 0, 1, int.MaxValue} /// across timeout/retry/breaker and asserts every built pipeline executes a trivial call. /// [Fact] public async Task Build_NeverThrows_ForHostileDirectOptions() { int[] hostile = { int.MinValue, -1, 0, 1, int.MaxValue }; var host = 0; foreach (var t in hostile) foreach (var r in hostile) foreach (var b in hostile) { var options = new DriverResilienceOptions { Tier = DriverTier.A, CapabilityPolicies = new Dictionary { [DriverCapability.Read] = new(TimeoutSeconds: t, RetryCount: r, BreakerFailureThreshold: b), }, }; var builder = new DriverResiliencePipelineBuilder(); var pipeline = builder.GetOrCreate("hostile", $"h{host++}", DriverCapability.Read, options); var result = await pipeline.ExecuteAsync(_ => ValueTask.FromResult(1), CancellationToken.None); result.ShouldBe(1, $"pipeline built from hostile (t={t}, r={r}, b={b}) must execute, not throw"); } } /// /// 01/S-7 = 03/S13: a stale-epoch re-cache must not be served to a new generation. Deterministic /// replay of the respawn race — the old child's late GetOrCreate re-caches under generation 1 /// AFTER the invalidate; the new invoker (generation 2) with new options must build + serve a pipeline /// honoring the NEW options, not the poisoned old-generation entry. Generations make the interleaving /// expressible sequentially — no timing dependence. /// [Fact] public async Task StaleGeneration_ReCache_IsNotServed_ToNewGeneration() { var builder = new DriverResiliencePipelineBuilder(); var optsA = new DriverResilienceOptions { Tier = DriverTier.A, CapabilityPolicies = new Dictionary { [DriverCapability.Read] = new(TimeoutSeconds: 5, RetryCount: 3, BreakerFailureThreshold: 0), }, }; var optsB = new DriverResilienceOptions { Tier = DriverTier.A, CapabilityPolicies = new Dictionary { [DriverCapability.Read] = new(TimeoutSeconds: 5, RetryCount: 0, BreakerFailureThreshold: 0), }, }; // gen 1 warms the cache with the retrying optsA. builder.GetOrCreate("i", "h", DriverCapability.Read, optsA, optionsGeneration: 1); builder.Invalidate("i"); // The lingering old child's late call re-caches optsA under its OWN generation 1. builder.GetOrCreate("i", "h", DriverCapability.Read, optsA, optionsGeneration: 1); // The new invoker (gen 2, optsB) must get a pipeline honoring optsB (no retry), not the poisoned one. var newPipeline = builder.GetOrCreate("i", "h", DriverCapability.Read, optsB, optionsGeneration: 2); var attempts = 0; await Should.ThrowAsync(async () => await newPipeline.ExecuteAsync(async _ => { attempts++; await Task.Yield(); throw new InvalidOperationException("boom"); })); attempts.ShouldBe(1, "the new generation must honor optsB (retryCount 0), not the stale optsA re-cache"); } /// /// A controllable whose clock advances only on . /// Drives both wall-clock () and monotonic () /// reads off the same tick counter so Polly's circuit-breaker break-duration timing is deterministic. /// private sealed class ManualTimeProvider : TimeProvider { private long _ticks; public ManualTimeProvider(DateTime start) => _ticks = start.Ticks; public void Advance(TimeSpan by) => _ticks += by.Ticks; public override DateTimeOffset GetUtcNow() => new(_ticks, TimeSpan.Zero); public override long GetTimestamp() => _ticks; public override long TimestampFrequency => TimeSpan.TicksPerSecond; } /// Minimal in-memory capturing (level, formatted-message) pairs so the /// resilience logging contract can be asserted without a real logging provider. private sealed class CapturingLogger : ILogger { public List<(LogLevel Level, string Message)> Entries { get; } = new(); public IDisposable BeginScope(TState state) where TState : notnull => NullScope.Instance; public bool IsEnabled(LogLevel logLevel) => true; public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => Entries.Add((logLevel, formatter(state, exception))); private sealed class NullScope : IDisposable { public static readonly NullScope Instance = new(); public void Dispose() { } } } }