From 205d398055e5dba59aa472d2168a113627eb734c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:35:41 -0400 Subject: [PATCH] fix(r2-10): breaker-close marker + IsBreakerOpen on ResilienceStatusSnapshot --- ...esilience-observability-plan.md.tasks.json | 2 +- .../DriverResilienceStatusTracker.cs | 48 ++++++++++++++++--- .../DriverResilienceStatusTrackerTests.cs | 35 ++++++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 7498bc68..0bba16d4 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -2,7 +2,7 @@ "planPath": "archreview/plans/R2-10-resilience-observability-plan.md", "lastUpdated": "2026-07-12", "tasks": [ - { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "pending", "blockedBy": [] }, + { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "completed", "blockedBy": [] }, { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "pending", "blockedBy": ["T1"] }, { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "pending", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "pending", "blockedBy": [] }, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs index 9dcf3327..8db62bbd 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs @@ -4,15 +4,19 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience; /// /// Process-singleton tracker of live resilience counters per -/// (DriverInstanceId, HostName). Populated by the CapabilityInvoker and the -/// MemoryTracking layer; consumed by a HostedService that periodically persists a -/// snapshot to the DriverInstanceResilienceStatus table for Admin /hosts. +/// (DriverInstanceId, HostName). Populated by the CapabilityInvoker (call +/// start/complete + success reset), the DriverResiliencePipelineBuilder's Polly +/// telemetry (retry failure, breaker open/close), and the MemoryTracking layer +/// (footprint). Read by DriverResilienceStatusPublisherService (Host, driver +/// nodes) which periodically publishes each snapshot to the +/// driver-resilience-status DistributedPubSub topic for the AdminUI +/// resilience panel. /// /// -/// Per Phase 6.1 Stream E. No DB dependency here — the tracker is pure in-memory so -/// tests can exercise it without EF Core or SQL Server. The HostedService that writes -/// snapshots lives in the Server project (Stream E.2); the actual SignalR push + Blazor -/// page refresh (E.3) lands in a follow-up visual-review PR. +/// Pure in-memory (no EF Core / SQL Server dependency) so tests can exercise it directly. +/// The operator-facing reader is the DPS publisher named above, mirroring the driver-health +/// flow — not the DB-table/HostedService shape originally sketched for Phase 6.1 Stream E +/// (R2-10 built the DPS reader instead). /// public sealed class DriverResilienceStatusTracker { @@ -62,6 +66,23 @@ public sealed class DriverResilienceStatusTracker (_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow }); } + /// + /// Record a circuit-breaker close (recovery) event — the paired counterpart to + /// . Stamps + /// so can report whether the breaker + /// is currently open. Called from the pipeline builder's OnClosed hook. + /// + /// The driver instance identifier. + /// The host name. + /// The UTC timestamp of the breaker close event. + public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow) + { + var key = new StatusKey(driverInstanceId, hostName); + _status.AddOrUpdate(key, + _ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }, + (_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }); + } + /// Record a process recycle event (Tier C only). /// The driver instance identifier. /// The host name. @@ -147,8 +168,21 @@ public sealed record ResilienceStatusSnapshot public int ConsecutiveFailures { get; init; } /// Gets the UTC timestamp of the last circuit-breaker open event, if any. public DateTime? LastBreakerOpenUtc { get; init; } + /// Gets the UTC timestamp of the last circuit-breaker close (recovery) event, if any. + public DateTime? LastBreakerClosedUtc { get; init; } /// Gets the UTC timestamp of the last recycle event, if any. public DateTime? LastRecycleUtc { get; init; } + + /// + /// Gets whether the circuit breaker is currently open for this (instance, host): + /// an open event has been recorded and no later close event has superseded it. + /// Note: Polly half-opens after BreakDuration without firing OnClosed + /// (which fires only on a successful probe), so an idle driver can read "open" until + /// the next call self-corrects it. + /// + public bool IsBreakerOpen => + LastBreakerOpenUtc is { } open && + (LastBreakerClosedUtc is not { } closed || open > closed); /// Gets the baseline process footprint in bytes. public long BaselineFootprintBytes { get; init; } /// Gets the current process footprint in bytes. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs index b97dc51d..b38de154 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs @@ -55,6 +55,41 @@ public sealed class DriverResilienceStatusTrackerTests tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now); } + /// Verifies that RecordBreakerClose stamps the close timestamp. + [Fact] + public void RecordBreakerClose_StampsCloseTime() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordBreakerOpen("drv", "host", Now); + + tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20)); + + tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20)); + } + + /// Verifies that IsBreakerOpen tracks open → close → reopen transitions. + [Fact] + public void IsBreakerOpen_TracksOpenCloseReopen() + { + var tracker = new DriverResilienceStatusTracker(); + + // A snapshot with no breaker event is never "open". + tracker.RecordFailure("drv", "host", Now); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse(); + + // Opened. + tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1)); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); + + // Closed later → no longer open. + tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2)); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse(); + + // Re-opened after the last close → open again. + tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3)); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); + } + /// Verifies that RecordRecycle populates the LastRecycleUtc timestamp. [Fact] public void RecordRecycle_Populates_LastRecycleUtc()