diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index f6048cd0..a3879c72 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -4,7 +4,7 @@ "tasks": [ { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "completed", "blockedBy": [] }, { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "completed", "blockedBy": ["T1"] }, - { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "pending", "blockedBy": ["T1"] }, + { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "pending", "blockedBy": [] }, { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "pending", "blockedBy": ["T1", "T4"] }, { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "pending", "blockedBy": ["T5"] }, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs index a985acb7..f68c6c9f 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs @@ -41,7 +41,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker /// pipeline-invalidate can take effect without restarting the invoker. /// /// Driver type name for structured-log enrichment (e.g. "Modbus"). - /// Optional resilience-status tracker. When wired, every capability call records start/complete so Admin /hosts can surface as the in-flight-call-depth proxy. + /// Optional resilience-status tracker. When wired, every capability call records start/complete so the AdminUI resilience panel can surface as the in-flight-call-depth proxy, and each successful call resets so the counter reflects only an active retry storm. /// /// Options epoch for this invoker (01/S-7). Threaded into every pipeline-cache lookup so a lingering /// old child's late re-cache can never poison a newer invoker's pipeline. Defaults to 0 so existing @@ -81,7 +81,11 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker { using (LogContextEnricher.Push(_driverInstanceId, _driverType, capability, LogContextEnricher.NewCorrelationId())) { - return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + var result = await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + // Success resets ConsecutiveFailures so the counter is a true "consecutive" gauge + // (nonzero only during an active retry storm), not stuck at the last blip's count. + _statusTracker?.RecordSuccess(_driverInstanceId, hostName, DateTime.UtcNow); + return result; } } finally @@ -106,6 +110,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker using (LogContextEnricher.Push(_driverInstanceId, _driverType, capability, LogContextEnricher.NewCorrelationId())) { await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + _statusTracker?.RecordSuccess(_driverInstanceId, hostName, DateTime.UtcNow); } } finally @@ -139,7 +144,9 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker { using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId())) { - return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + var result = await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + _statusTracker?.RecordSuccess(_driverInstanceId, hostName, DateTime.UtcNow); + return result; } } finally diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs index 68a4d00f..bfb0703c 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs @@ -235,6 +235,63 @@ public sealed class CapabilityInvokerTests builder.CachedPipelineCount.ShouldBe(1, "two writes to the same host share one cached no-retry pipeline"); } + /// + /// R2-10: a successful capability call must reset ConsecutiveFailures so the counter + /// is a true "consecutive" gauge — nonzero only during an active retry storm, not stuck at the + /// last blip's count forever. Previously only the breaker's OnClosed reset it, so a + /// transient 2-retry blip that never opened the breaker read "2 failures" indefinitely. + /// + [Fact] + public async Task ExecuteAsync_OnSuccess_ResetsConsecutiveFailures() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow); + tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow); + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(2, "seeded failures"); + + var invoker = new CapabilityInvoker( + new DriverResiliencePipelineBuilder(), + "drv-test", + () => new DriverResilienceOptions { Tier = DriverTier.A }, + statusTracker: tracker); + + await invoker.ExecuteAsync( + DriverCapability.Read, + "host-1", + _ => ValueTask.FromResult(1), + CancellationToken.None); + + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(0, + "a successful capability call must clear the consecutive-failure counter"); + } + + /// + /// R2-10 companion: a successful non-idempotent write must also reset the consecutive-failure + /// counter on the caller's host (the write arm has its own success path). + /// + [Fact] + public async Task ExecuteWriteAsync_NonIdempotent_OnSuccess_ResetsConsecutiveFailures() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow); + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(1, "seeded failure"); + + var invoker = new CapabilityInvoker( + new DriverResiliencePipelineBuilder(), + "drv-test", + () => new DriverResilienceOptions { Tier = DriverTier.A }, + statusTracker: tracker); + + await invoker.ExecuteWriteAsync( + "host-1", + isIdempotent: false, + _ => ValueTask.FromResult(0), + CancellationToken.None); + + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(0, + "a successful non-idempotent write must clear the consecutive-failure counter"); + } + /// /// Core-009 regression — companion consistency assertion: the non-idempotent branch must /// not observe two different option snapshots even if the accessor's returned value changes