fix(r2-10): breaker-close marker + IsBreakerOpen on ResilienceStatusSnapshot

This commit is contained in:
Joseph Doherty
2026-07-13 10:35:41 -04:00
parent 46fedda3e4
commit 205d398055
3 changed files with 77 additions and 8 deletions
@@ -4,15 +4,19 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience;
/// <summary>
/// Process-singleton tracker of live resilience counters per
/// <c>(DriverInstanceId, HostName)</c>. Populated by the CapabilityInvoker and the
/// MemoryTracking layer; consumed by a HostedService that periodically persists a
/// snapshot to the <c>DriverInstanceResilienceStatus</c> table for Admin <c>/hosts</c>.
/// <c>(DriverInstanceId, HostName)</c>. Populated by the CapabilityInvoker (call
/// start/complete + success reset), the DriverResiliencePipelineBuilder's Polly
/// telemetry (retry failure, breaker open/close), and the MemoryTracking layer
/// (footprint). Read by <c>DriverResilienceStatusPublisherService</c> (Host, driver
/// nodes) which periodically publishes each snapshot to the
/// <c>driver-resilience-status</c> DistributedPubSub topic for the AdminUI
/// resilience panel.
/// </summary>
/// <remarks>
/// Per Phase 6.1 Stream E. No DB dependency here — the tracker is pure in-memory so
/// tests can exercise it without EF Core or SQL Server. The HostedService that writes
/// snapshots lives in the Server project (Stream E.2); the actual SignalR push + Blazor
/// page refresh (E.3) lands in a follow-up visual-review PR.
/// Pure in-memory (no EF Core / SQL Server dependency) so tests can exercise it directly.
/// The operator-facing reader is the DPS publisher named above, mirroring the driver-health
/// flow — not the DB-table/HostedService shape originally sketched for Phase 6.1 Stream E
/// (R2-10 built the DPS reader instead).
/// </remarks>
public sealed class DriverResilienceStatusTracker
{
@@ -62,6 +66,23 @@ public sealed class DriverResilienceStatusTracker
(_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow });
}
/// <summary>
/// Record a circuit-breaker close (recovery) event — the paired counterpart to
/// <see cref="RecordBreakerOpen"/>. Stamps <see cref="ResilienceStatusSnapshot.LastBreakerClosedUtc"/>
/// so <see cref="ResilienceStatusSnapshot.IsBreakerOpen"/> can report whether the breaker
/// is currently open. Called from the pipeline builder's <c>OnClosed</c> hook.
/// </summary>
/// <param name="driverInstanceId">The driver instance identifier.</param>
/// <param name="hostName">The host name.</param>
/// <param name="utcNow">The UTC timestamp of the breaker close event.</param>
public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow)
{
var key = new StatusKey(driverInstanceId, hostName);
_status.AddOrUpdate(key,
_ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow },
(_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow });
}
/// <summary>Record a process recycle event (Tier C only).</summary>
/// <param name="driverInstanceId">The driver instance identifier.</param>
/// <param name="hostName">The host name.</param>
@@ -147,8 +168,21 @@ public sealed record ResilienceStatusSnapshot
public int ConsecutiveFailures { get; init; }
/// <summary>Gets the UTC timestamp of the last circuit-breaker open event, if any.</summary>
public DateTime? LastBreakerOpenUtc { get; init; }
/// <summary>Gets the UTC timestamp of the last circuit-breaker close (recovery) event, if any.</summary>
public DateTime? LastBreakerClosedUtc { get; init; }
/// <summary>Gets the UTC timestamp of the last recycle event, if any.</summary>
public DateTime? LastRecycleUtc { get; init; }
/// <summary>
/// Gets whether the circuit breaker is currently open for this (instance, host):
/// an open event has been recorded and no later close event has superseded it.
/// Note: Polly half-opens after <c>BreakDuration</c> without firing <c>OnClosed</c>
/// (which fires only on a successful probe), so an idle driver can read "open" until
/// the next call self-corrects it.
/// </summary>
public bool IsBreakerOpen =>
LastBreakerOpenUtc is { } open &&
(LastBreakerClosedUtc is not { } closed || open > closed);
/// <summary>Gets the baseline process footprint in bytes.</summary>
public long BaselineFootprintBytes { get; init; }
/// <summary>Gets the current process footprint in bytes.</summary>