From 205d398055e5dba59aa472d2168a113627eb734c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:35:41 -0400 Subject: [PATCH 01/10] fix(r2-10): breaker-close marker + IsBreakerOpen on ResilienceStatusSnapshot --- ...esilience-observability-plan.md.tasks.json | 2 +- .../DriverResilienceStatusTracker.cs | 48 ++++++++++++++++--- .../DriverResilienceStatusTrackerTests.cs | 35 ++++++++++++++ 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 7498bc68..0bba16d4 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -2,7 +2,7 @@ "planPath": "archreview/plans/R2-10-resilience-observability-plan.md", "lastUpdated": "2026-07-12", "tasks": [ - { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "pending", "blockedBy": [] }, + { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "completed", "blockedBy": [] }, { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "pending", "blockedBy": ["T1"] }, { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "pending", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "pending", "blockedBy": [] }, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs index 9dcf3327..8db62bbd 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs @@ -4,15 +4,19 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Resilience; /// /// Process-singleton tracker of live resilience counters per -/// (DriverInstanceId, HostName). Populated by the CapabilityInvoker and the -/// MemoryTracking layer; consumed by a HostedService that periodically persists a -/// snapshot to the DriverInstanceResilienceStatus table for Admin /hosts. +/// (DriverInstanceId, HostName). Populated by the CapabilityInvoker (call +/// start/complete + success reset), the DriverResiliencePipelineBuilder's Polly +/// telemetry (retry failure, breaker open/close), and the MemoryTracking layer +/// (footprint). Read by DriverResilienceStatusPublisherService (Host, driver +/// nodes) which periodically publishes each snapshot to the +/// driver-resilience-status DistributedPubSub topic for the AdminUI +/// resilience panel. /// /// -/// Per Phase 6.1 Stream E. No DB dependency here — the tracker is pure in-memory so -/// tests can exercise it without EF Core or SQL Server. The HostedService that writes -/// snapshots lives in the Server project (Stream E.2); the actual SignalR push + Blazor -/// page refresh (E.3) lands in a follow-up visual-review PR. +/// Pure in-memory (no EF Core / SQL Server dependency) so tests can exercise it directly. +/// The operator-facing reader is the DPS publisher named above, mirroring the driver-health +/// flow — not the DB-table/HostedService shape originally sketched for Phase 6.1 Stream E +/// (R2-10 built the DPS reader instead). /// public sealed class DriverResilienceStatusTracker { @@ -62,6 +66,23 @@ public sealed class DriverResilienceStatusTracker (_, existing) => existing with { LastBreakerOpenUtc = utcNow, LastSampledUtc = utcNow }); } + /// + /// Record a circuit-breaker close (recovery) event — the paired counterpart to + /// . Stamps + /// so can report whether the breaker + /// is currently open. Called from the pipeline builder's OnClosed hook. + /// + /// The driver instance identifier. + /// The host name. + /// The UTC timestamp of the breaker close event. + public void RecordBreakerClose(string driverInstanceId, string hostName, DateTime utcNow) + { + var key = new StatusKey(driverInstanceId, hostName); + _status.AddOrUpdate(key, + _ => new ResilienceStatusSnapshot { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }, + (_, existing) => existing with { LastBreakerClosedUtc = utcNow, LastSampledUtc = utcNow }); + } + /// Record a process recycle event (Tier C only). /// The driver instance identifier. /// The host name. @@ -147,8 +168,21 @@ public sealed record ResilienceStatusSnapshot public int ConsecutiveFailures { get; init; } /// Gets the UTC timestamp of the last circuit-breaker open event, if any. public DateTime? LastBreakerOpenUtc { get; init; } + /// Gets the UTC timestamp of the last circuit-breaker close (recovery) event, if any. + public DateTime? LastBreakerClosedUtc { get; init; } /// Gets the UTC timestamp of the last recycle event, if any. public DateTime? LastRecycleUtc { get; init; } + + /// + /// Gets whether the circuit breaker is currently open for this (instance, host): + /// an open event has been recorded and no later close event has superseded it. + /// Note: Polly half-opens after BreakDuration without firing OnClosed + /// (which fires only on a successful probe), so an idle driver can read "open" until + /// the next call self-corrects it. + /// + public bool IsBreakerOpen => + LastBreakerOpenUtc is { } open && + (LastBreakerClosedUtc is not { } closed || open > closed); /// Gets the baseline process footprint in bytes. public long BaselineFootprintBytes { get; init; } /// Gets the current process footprint in bytes. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs index b97dc51d..b38de154 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResilienceStatusTrackerTests.cs @@ -55,6 +55,41 @@ public sealed class DriverResilienceStatusTrackerTests tracker.TryGet("drv", "host")!.LastBreakerOpenUtc.ShouldBe(Now); } + /// Verifies that RecordBreakerClose stamps the close timestamp. + [Fact] + public void RecordBreakerClose_StampsCloseTime() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordBreakerOpen("drv", "host", Now); + + tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(20)); + + tracker.TryGet("drv", "host")!.LastBreakerClosedUtc.ShouldBe(Now.AddSeconds(20)); + } + + /// Verifies that IsBreakerOpen tracks open → close → reopen transitions. + [Fact] + public void IsBreakerOpen_TracksOpenCloseReopen() + { + var tracker = new DriverResilienceStatusTracker(); + + // A snapshot with no breaker event is never "open". + tracker.RecordFailure("drv", "host", Now); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse(); + + // Opened. + tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(1)); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); + + // Closed later → no longer open. + tracker.RecordBreakerClose("drv", "host", Now.AddSeconds(2)); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeFalse(); + + // Re-opened after the last close → open again. + tracker.RecordBreakerOpen("drv", "host", Now.AddSeconds(3)); + tracker.TryGet("drv", "host")!.IsBreakerOpen.ShouldBeTrue(); + } + /// Verifies that RecordRecycle populates the LastRecycleUtc timestamp. [Fact] public void RecordRecycle_Populates_LastRecycleUtc() From 2cce61fc04d293faf8afdbb8aa183a9892018458 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:37:12 -0400 Subject: [PATCH 02/10] fix(r2-10): OnClosed records breaker-close on the status tracker --- ...esilience-observability-plan.md.tasks.json | 2 +- .../DriverResiliencePipelineBuilder.cs | 19 ++++--- .../DriverResiliencePipelineBuilderTests.cs | 53 +++++++++++++++++++ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 0bba16d4..f6048cd0 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -3,7 +3,7 @@ "lastUpdated": "2026-07-12", "tasks": [ { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "completed", "blockedBy": [] }, - { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "pending", "blockedBy": ["T1"] }, + { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "pending", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "pending", "blockedBy": [] }, { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "pending", "blockedBy": ["T1", "T4"] }, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResiliencePipelineBuilder.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResiliencePipelineBuilder.cs index 2a6b3952..fe5b6cff 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResiliencePipelineBuilder.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResiliencePipelineBuilder.cs @@ -38,14 +38,16 @@ public sealed class DriverResiliencePipelineBuilder /// Clock source for pipeline timeouts + breaker sampling. Defaults to system. /// When non-null, every built pipeline wires Polly telemetry into /// the tracker — retries increment ConsecutiveFailures, breaker-open stamps - /// LastBreakerOpenUtc, breaker-close resets failures. Feeds Admin /hosts + + /// LastBreakerOpenUtc, breaker-close resets failures and stamps + /// LastBreakerClosedUtc (so IsBreakerOpen reads false again). Feeds the AdminUI + /// resilience panel via the driver-resilience-status DPS publisher + /// the in-flight-call-depth column. Absent tracker means no telemetry (unit tests + /// deployments that don't care about resilience observability). /// When non-null, retry / circuit-breaker-open / breaker-close events are - /// logged (instance + host + capability + attempt/exception). This is the operator-facing - /// observability surface for the resilience pipeline until the Admin /hosts reader - /// (Phase 6.1 Stream E.2/E.3) ships — without it the pipeline runs silently. Absent logger = - /// no resilience logging (unit tests). + /// logged (instance + host + capability + attempt/exception). This is the structured-log + /// drill-down for the resilience pipeline, complementing the AdminUI resilience panel fed by + /// the tracker — without it the pipeline runs silently. Absent logger = no resilience logging + /// (unit tests). public DriverResiliencePipelineBuilder( TimeProvider? timeProvider = null, DriverResilienceStatusTracker? statusTracker = null, @@ -186,8 +188,11 @@ public sealed class DriverResiliencePipelineBuilder breakerOptions.OnClosed = args => { // Closing the breaker means the target recovered — reset the consecutive- - // failure counter so Admin UI stops flashing red for this host. - tracker?.RecordSuccess(driverInstanceId, hostName, timeProvider.GetUtcNow().UtcDateTime); + // failure counter AND stamp the close time so IsBreakerOpen reads false again + // (the resilience panel stops showing "Breaker OPEN" for this host). + var closedUtc = timeProvider.GetUtcNow().UtcDateTime; + tracker?.RecordSuccess(driverInstanceId, hostName, closedUtc); + tracker?.RecordBreakerClose(driverInstanceId, hostName, closedUtc); logger?.LogInformation( "Driver resilience circuit-breaker closed (recovered): instance={DriverInstanceId} host={HostName} capability={Capability}", driverInstanceId, hostName, capability); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs index b9bebed5..44642c28 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/DriverResiliencePipelineBuilderTests.cs @@ -277,6 +277,44 @@ public sealed class DriverResiliencePipelineBuilderTests snap!.LastBreakerOpenUtc.ShouldNotBeNull(); } + /// + /// R2-10 truthfulness fix: after the breaker opens and later recovers (a successful probe + /// past BreakDuration), the pipeline builder's OnClosed hook must record the + /// breaker close on the tracker — so + /// reads false again and is stamped. + /// Uses a controllable so the 15 s break duration elapses deterministically. + /// + [Fact] + public async Task Tracker_RecordsBreakerClose_OnRecovery() + { + var clock = new ManualTimeProvider(new DateTime(2026, 7, 13, 0, 0, 0, DateTimeKind.Utc)); + var tracker = new DriverResilienceStatusTracker(); + var builder = new DriverResiliencePipelineBuilder(timeProvider: clock, statusTracker: tracker); + var pipeline = builder.GetOrCreate("drv-trk", "host-c", DriverCapability.Write, TierAOptions); + + // Drive the breaker open with the tier's throughput of failures. + var threshold = TierAOptions.Resolve(DriverCapability.Write).BreakerFailureThreshold; + for (var i = 0; i < threshold; i++) + { + await Should.ThrowAsync(async () => + await pipeline.ExecuteAsync(async _ => + { + await Task.Yield(); + throw new InvalidOperationException("boom"); + })); + } + + tracker.TryGet("drv-trk", "host-c")!.IsBreakerOpen.ShouldBeTrue("breaker must be open after the failure threshold"); + + // Elapse the break duration (15 s) so the breaker half-opens, then let a probe succeed → OnClosed. + clock.Advance(TimeSpan.FromSeconds(16)); + await pipeline.ExecuteAsync(async _ => await Task.Yield()); + + var snap = tracker.TryGet("drv-trk", "host-c")!; + snap.LastBreakerClosedUtc.ShouldNotBeNull("OnClosed must stamp the close time on recovery"); + snap.IsBreakerOpen.ShouldBeFalse("the breaker must read closed after a successful recovery probe"); + } + /// Verifies that the tracker isolates counters per host. [Fact] public async Task Tracker_IsolatesCounters_PerHost() @@ -404,6 +442,21 @@ public sealed class DriverResiliencePipelineBuilderTests attempts.ShouldBe(1, "the new generation must honor optsB (retryCount 0), not the stale optsA re-cache"); } + /// + /// A controllable whose clock advances only on . + /// Drives both wall-clock () and monotonic () + /// reads off the same tick counter so Polly's circuit-breaker break-duration timing is deterministic. + /// + private sealed class ManualTimeProvider : TimeProvider + { + private long _ticks; + public ManualTimeProvider(DateTime start) => _ticks = start.Ticks; + public void Advance(TimeSpan by) => _ticks += by.Ticks; + public override DateTimeOffset GetUtcNow() => new(_ticks, TimeSpan.Zero); + public override long GetTimestamp() => _ticks; + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + } + /// Minimal in-memory capturing (level, formatted-message) pairs so the /// resilience logging contract can be asserted without a real logging provider. private sealed class CapturingLogger : ILogger From 0ba60db1e4d4c719679e379ace46fa2cf6c821af Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:38:51 -0400 Subject: [PATCH 03/10] fix(r2-10): successful capability calls reset ConsecutiveFailures --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Resilience/CapabilityInvoker.cs | 13 ++++- .../Resilience/CapabilityInvokerTests.cs | 57 +++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index f6048cd0..a3879c72 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -4,7 +4,7 @@ "tasks": [ { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "completed", "blockedBy": [] }, { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "completed", "blockedBy": ["T1"] }, - { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "pending", "blockedBy": ["T1"] }, + { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "pending", "blockedBy": [] }, { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "pending", "blockedBy": ["T1", "T4"] }, { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "pending", "blockedBy": ["T5"] }, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs index a985acb7..f68c6c9f 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/CapabilityInvoker.cs @@ -41,7 +41,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker /// pipeline-invalidate can take effect without restarting the invoker. /// /// Driver type name for structured-log enrichment (e.g. "Modbus"). - /// Optional resilience-status tracker. When wired, every capability call records start/complete so Admin /hosts can surface as the in-flight-call-depth proxy. + /// Optional resilience-status tracker. When wired, every capability call records start/complete so the AdminUI resilience panel can surface as the in-flight-call-depth proxy, and each successful call resets so the counter reflects only an active retry storm. /// /// Options epoch for this invoker (01/S-7). Threaded into every pipeline-cache lookup so a lingering /// old child's late re-cache can never poison a newer invoker's pipeline. Defaults to 0 so existing @@ -81,7 +81,11 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker { using (LogContextEnricher.Push(_driverInstanceId, _driverType, capability, LogContextEnricher.NewCorrelationId())) { - return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + var result = await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + // Success resets ConsecutiveFailures so the counter is a true "consecutive" gauge + // (nonzero only during an active retry storm), not stuck at the last blip's count. + _statusTracker?.RecordSuccess(_driverInstanceId, hostName, DateTime.UtcNow); + return result; } } finally @@ -106,6 +110,7 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker using (LogContextEnricher.Push(_driverInstanceId, _driverType, capability, LogContextEnricher.NewCorrelationId())) { await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + _statusTracker?.RecordSuccess(_driverInstanceId, hostName, DateTime.UtcNow); } } finally @@ -139,7 +144,9 @@ public sealed class CapabilityInvoker : IDriverCapabilityInvoker { using (LogContextEnricher.Push(_driverInstanceId, _driverType, DriverCapability.Write, LogContextEnricher.NewCorrelationId())) { - return await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + var result = await pipeline.ExecuteAsync(callSite, cancellationToken).ConfigureAwait(false); + _statusTracker?.RecordSuccess(_driverInstanceId, hostName, DateTime.UtcNow); + return result; } } finally diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs index 68a4d00f..bfb0703c 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Tests/Resilience/CapabilityInvokerTests.cs @@ -235,6 +235,63 @@ public sealed class CapabilityInvokerTests builder.CachedPipelineCount.ShouldBe(1, "two writes to the same host share one cached no-retry pipeline"); } + /// + /// R2-10: a successful capability call must reset ConsecutiveFailures so the counter + /// is a true "consecutive" gauge — nonzero only during an active retry storm, not stuck at the + /// last blip's count forever. Previously only the breaker's OnClosed reset it, so a + /// transient 2-retry blip that never opened the breaker read "2 failures" indefinitely. + /// + [Fact] + public async Task ExecuteAsync_OnSuccess_ResetsConsecutiveFailures() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow); + tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow); + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(2, "seeded failures"); + + var invoker = new CapabilityInvoker( + new DriverResiliencePipelineBuilder(), + "drv-test", + () => new DriverResilienceOptions { Tier = DriverTier.A }, + statusTracker: tracker); + + await invoker.ExecuteAsync( + DriverCapability.Read, + "host-1", + _ => ValueTask.FromResult(1), + CancellationToken.None); + + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(0, + "a successful capability call must clear the consecutive-failure counter"); + } + + /// + /// R2-10 companion: a successful non-idempotent write must also reset the consecutive-failure + /// counter on the caller's host (the write arm has its own success path). + /// + [Fact] + public async Task ExecuteWriteAsync_NonIdempotent_OnSuccess_ResetsConsecutiveFailures() + { + var tracker = new DriverResilienceStatusTracker(); + tracker.RecordFailure("drv-test", "host-1", DateTime.UtcNow); + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(1, "seeded failure"); + + var invoker = new CapabilityInvoker( + new DriverResiliencePipelineBuilder(), + "drv-test", + () => new DriverResilienceOptions { Tier = DriverTier.A }, + statusTracker: tracker); + + await invoker.ExecuteWriteAsync( + "host-1", + isIdempotent: false, + _ => ValueTask.FromResult(0), + CancellationToken.None); + + tracker.TryGet("drv-test", "host-1")!.ConsecutiveFailures.ShouldBe(0, + "a successful non-idempotent write must clear the consecutive-failure counter"); + } + /// /// Core-009 regression — companion consistency assertion: the non-idempotent branch must /// not observe two different option snapshots even if the accessor's returned value changes From 2cb3f962b969fd92c35a567cac21d66378bfd727 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:39:18 -0400 Subject: [PATCH 04/10] fix(r2-10): DriverResilienceStatusChanged DPS contract --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Drivers/DriverResilienceStatusChanged.cs | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Drivers/DriverResilienceStatusChanged.cs diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index a3879c72..39fc2774 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -5,7 +5,7 @@ { "id": "T1", "subject": "Tracker: LastBreakerClosedUtc + IsBreakerOpen + RecordBreakerClose (Core, TDD in DriverResilienceStatusTrackerTests)", "status": "completed", "blockedBy": [] }, { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "completed", "blockedBy": ["T1"] }, - { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "pending", "blockedBy": [] }, + { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "completed", "blockedBy": [] }, { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "pending", "blockedBy": ["T1", "T4"] }, { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "pending", "blockedBy": ["T5"] }, { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "pending", "blockedBy": ["T4"] }, diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Drivers/DriverResilienceStatusChanged.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Drivers/DriverResilienceStatusChanged.cs new file mode 100644 index 00000000..b167dfe1 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Commons/Messages/Drivers/DriverResilienceStatusChanged.cs @@ -0,0 +1,35 @@ +namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; + +/// +/// Snapshot of one driver instance's live resilience counters for a single target host, +/// published periodically to the driver-resilience-status DistributedPubSub topic by +/// DriverResilienceStatusPublisherService (Host, driver-role nodes) and consumed by the +/// AdminUI DriverResilienceStatusBridge → IDriverResilienceStatusStore → +/// resilience section of the driver status panel. Mirrors the +/// flow one-for-one; a separate topic keeps the two feeds (event-driven health vs. periodic +/// resilience) decoupled. +/// +/// Globally-unique driver instance ID (the panel filters on this alone). +/// Target host the counters aggregate for (per-device breaker isolation key). +/// Whether the circuit breaker is currently open for this (instance, host). +/// Consecutive pipeline failures — nonzero only during an active retry storm. +/// In-flight capability-call depth against this (instance, host). +/// UTC of the most recent breaker-open event; null if never. +/// UTC the tracker counters were last updated. +/// UTC this snapshot was published (drives staleness detection in the panel). +public sealed record DriverResilienceStatusChanged( + string DriverInstanceId, + string HostName, + bool BreakerOpen, + int ConsecutiveFailures, + int CurrentInFlight, + DateTime? LastBreakerOpenUtc, + DateTime LastSampledUtc, + DateTime PublishedUtc) +{ + /// + /// DPS topic name. Both the Host publisher and the AdminUI bridge reference this single + /// constant on the contract so a rename can't silently desynchronise publisher and subscriber. + /// + public const string TopicName = "driver-resilience-status"; +} From 37f180c5a35f78f2dc49406a5729bc2d194d7267 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:40:54 -0400 Subject: [PATCH 05/10] fix(r2-10): periodic DPS publisher for tracker snapshots --- ...esilience-observability-plan.md.tasks.json | 2 +- .../DriverResilienceStatusPublisherService.cs | 93 +++++++++++++++++++ ...erResilienceStatusPublisherServiceTests.cs | 56 +++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DriverResilienceStatusPublisherServiceTests.cs diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 39fc2774..882d5331 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -6,7 +6,7 @@ { "id": "T2", "subject": "Builder: OnClosed records breaker-close on the tracker (TDD in DriverResiliencePipelineBuilderTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "completed", "blockedBy": [] }, - { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "pending", "blockedBy": ["T1", "T4"] }, + { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "completed", "blockedBy": ["T1", "T4"] }, { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "pending", "blockedBy": ["T5"] }, { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "pending", "blockedBy": ["T4"] }, { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "pending", "blockedBy": ["T7"] }, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs new file mode 100644 index 00000000..76d1265d --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverResilienceStatusPublisherService.cs @@ -0,0 +1,93 @@ +using Akka.Actor; +using Akka.Cluster.Tools.PublishSubscribe; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; +using ZB.MOM.WW.OtOpcUa.Core.Resilience; + +namespace ZB.MOM.WW.OtOpcUa.Host.Drivers; + +/// +/// The operator-facing reader for . Runs on driver-role +/// nodes (registered next to the tracker it reads) and every publishes the +/// full tracker snapshot — one per (instance, host) +/// — to the driver-resilience-status DistributedPubSub topic. The AdminUI +/// DriverResilienceStatusBridge subscribes and feeds the resilience section of the driver +/// status panel. Mirrors the driver-health flow; publishing the full set every tick self-heals +/// late-subscribing admin nodes and gives the panel staleness detection for free. +/// +public sealed class DriverResilienceStatusPublisherService : BackgroundService +{ + private static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(5); + + private readonly DriverResilienceStatusTracker _tracker; + private readonly Func _actorSystemAccessor; + private readonly ILogger _logger; + private readonly TimeSpan _interval; + + /// Initializes a new instance of . + /// The process-singleton resilience-status tracker to read each tick. + /// Lazy accessor for the Akka whose DPS mediator publishes the snapshots. + /// Logger for publish diagnostics. + /// Publish cadence; defaults to 5 s when null. + public DriverResilienceStatusPublisherService( + DriverResilienceStatusTracker tracker, + Func actorSystemAccessor, + ILogger logger, + TimeSpan? interval = null) + { + _tracker = tracker; + _actorSystemAccessor = actorSystemAccessor; + _logger = logger; + _interval = interval ?? DefaultInterval; + } + + /// + /// Map the tracker's current snapshot to one wire message per (instance, host) pair. + /// Pure + unit-testable; the timer/publish shell in calls this each tick. + /// + /// The tracker to read. + /// The publish timestamp stamped on every emitted message. + /// One per tracked pair (empty when the tracker is empty). + public static IReadOnlyList BuildMessages( + DriverResilienceStatusTracker tracker, DateTime publishedUtc) => + tracker.Snapshot() + .Select(entry => new DriverResilienceStatusChanged( + entry.DriverInstanceId, + entry.HostName, + entry.Snapshot.IsBreakerOpen, + entry.Snapshot.ConsecutiveFailures, + entry.Snapshot.CurrentInFlight, + entry.Snapshot.LastBreakerOpenUtc, + entry.Snapshot.LastSampledUtc, + publishedUtc)) + .ToList(); + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(_interval); + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) + { + try + { + var messages = BuildMessages(_tracker, DateTime.UtcNow); + if (messages.Count == 0) + continue; + + var mediator = DistributedPubSub.Get(_actorSystemAccessor()).Mediator; + foreach (var message in messages) + mediator.Tell(new Publish(DriverResilienceStatusChanged.TopicName, message)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + // A transient DPS / actor-system hiccup must never kill the publisher — the next tick retries. + _logger.LogWarning(ex, "DriverResilienceStatusPublisherService: publish tick failed; retrying next interval"); + } + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DriverResilienceStatusPublisherServiceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DriverResilienceStatusPublisherServiceTests.cs new file mode 100644 index 00000000..246e14eb --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/DriverResilienceStatusPublisherServiceTests.cs @@ -0,0 +1,56 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Resilience; +using ZB.MOM.WW.OtOpcUa.Host.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// Pure-DI (no fixture) coverage of the publisher's snapshot→message mapping — the only unit-testable +/// seam of . The DPS Publish shell is +/// verified by the R2-10 live gate; this asserts the mapping the shell publishes. +/// +public sealed class DriverResilienceStatusPublisherServiceTests +{ + private static readonly DateTime Published = new(2026, 7, 13, 9, 0, 0, DateTimeKind.Utc); + private static readonly DateTime Sampled = new(2026, 7, 13, 8, 59, 0, DateTimeKind.Utc); + + [Fact] + public void BuildMessages_EmptyTracker_ReturnsEmpty() + { + var tracker = new DriverResilienceStatusTracker(); + + var messages = DriverResilienceStatusPublisherService.BuildMessages(tracker, Published); + + messages.ShouldBeEmpty(); + } + + [Fact] + public void BuildMessages_MapsTrackerSnapshot_OneMessagePerInstanceHost() + { + var tracker = new DriverResilienceStatusTracker(); + // (drv-1, host-a): an open breaker with a standing failure count. + tracker.RecordFailure("drv-1", "host-a", Sampled); + tracker.RecordFailure("drv-1", "host-a", Sampled); + tracker.RecordBreakerOpen("drv-1", "host-a", Sampled); + // (drv-1, host-b): a healthy host with in-flight calls. + tracker.RecordCallStart("drv-1", "host-b"); + + var messages = DriverResilienceStatusPublisherService.BuildMessages(tracker, Published); + + messages.Count.ShouldBe(2); + + var open = messages.Single(m => m.HostName == "host-a"); + open.DriverInstanceId.ShouldBe("drv-1"); + open.BreakerOpen.ShouldBeTrue(); + open.ConsecutiveFailures.ShouldBe(2); + open.LastBreakerOpenUtc.ShouldBe(Sampled); + open.PublishedUtc.ShouldBe(Published); + + var healthy = messages.Single(m => m.HostName == "host-b"); + healthy.BreakerOpen.ShouldBeFalse(); + healthy.ConsecutiveFailures.ShouldBe(0); + healthy.CurrentInFlight.ShouldBe(1); + healthy.PublishedUtc.ShouldBe(Published); + } +} From 12b2ce078b927df0db3dd4af228edd626cd57ed2 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:42:18 -0400 Subject: [PATCH 06/10] fix(r2-10): register the tracker's production reader on driver nodes + wiring guard --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Drivers/DriverFactoryBootstrap.cs | 7 +++++ ...ResilienceStatusReaderRegistrationTests.cs | 30 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ResilienceStatusReaderRegistrationTests.cs diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 882d5331..7c6e7eb1 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -7,7 +7,7 @@ { "id": "T3", "subject": "Invoker: successful Execute* resets ConsecutiveFailures (TDD in CapabilityInvokerTests)", "status": "completed", "blockedBy": ["T1"] }, { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "completed", "blockedBy": [] }, { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "completed", "blockedBy": ["T1", "T4"] }, - { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "pending", "blockedBy": ["T5"] }, + { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "completed", "blockedBy": ["T5"] }, { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "pending", "blockedBy": ["T4"] }, { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "pending", "blockedBy": ["T7"] }, { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "pending", "blockedBy": ["T7"] }, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverFactoryBootstrap.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverFactoryBootstrap.cs index bf6a5f37..1d4fd573 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverFactoryBootstrap.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Drivers/DriverFactoryBootstrap.cs @@ -70,6 +70,13 @@ public static class DriverFactoryBootstrap logger: sp.GetService()?.CreateLogger("ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory")); }); + // The tracker's operator-facing reader: a periodic DPS publisher that mirrors each snapshot to the + // AdminUI resilience panel (R2-10 — closes the "tracker fed by everything, read by nothing" gap). + // Lives on driver nodes exactly where the tracker exists. The lazy Func is idempotent + // with the one Program.cs registers for the alarm-command router. + services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + services.AddHostedService(); + // Driver nodes also carry the probe set so a fused admin,driver node has it; the admin-only // case is covered by Program.cs calling AddOtOpcUaDriverProbes() in the hasAdmin block. services.AddOtOpcUaDriverProbes(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ResilienceStatusReaderRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ResilienceStatusReaderRegistrationTests.cs new file mode 100644 index 00000000..19b4ae72 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/ResilienceStatusReaderRegistrationTests.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Host.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// Guards that the resilience-status tracker now HAS a production reader — the exact +/// "built-but-never-wired" class of bug the whole arch-review targets (mirrors +/// ). AddOtOpcUaDriverFactories (the +/// driver-node bootstrap) must register as an +/// . A refactor that drops the registration fails this test, not production. +/// +public sealed class ResilienceStatusReaderRegistrationTests +{ + [Fact] + public void AddOtOpcUaDriverFactories_registers_the_resilience_status_reader_hosted_service() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOtOpcUaDriverFactories(); + + services.ShouldContain( + d => d.ServiceType == typeof(IHostedService) + && d.ImplementationType == typeof(DriverResilienceStatusPublisherService), + "the resilience-status tracker must have a production reader — the periodic DPS publisher hosted service on driver nodes"); + } +} From 99ce5b401e80635e2f6c206a0ae90076ecd59e1b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:43:47 -0400 Subject: [PATCH 07/10] fix(r2-10): in-process resilience status store --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Hubs/HubServiceCollectionExtensions.cs | 1 + .../Hubs/IDriverResilienceStatusStore.cs | 36 ++++++++ .../InMemoryDriverResilienceStatusStore.cs | 33 +++++++ .../DriverResilienceStatusStoreTests.cs | 87 +++++++++++++++++++ 5 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/IDriverResilienceStatusStore.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/InMemoryDriverResilienceStatusStore.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/DriverResilienceStatusStoreTests.cs diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 7c6e7eb1..1053d3e8 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -8,7 +8,7 @@ { "id": "T4", "subject": "Commons: DriverResilienceStatusChanged record + driver-resilience-status TopicName", "status": "completed", "blockedBy": [] }, { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "completed", "blockedBy": ["T1", "T4"] }, { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "completed", "blockedBy": ["T5"] }, - { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "pending", "blockedBy": ["T4"] }, + { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "completed", "blockedBy": ["T4"] }, { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "pending", "blockedBy": ["T7"] }, { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "pending", "blockedBy": ["T7"] }, { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "pending", "blockedBy": ["T6", "T8", "T9"] }, diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs index 6867d222..b77d7048 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs @@ -28,6 +28,7 @@ public static class HubServiceCollectionExtensions public static IServiceCollection AddOtOpcUaDriverStatusServices(this IServiceCollection services) { services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(typeof(IInProcessBroadcaster<>), typeof(InProcessBroadcaster<>)); return services; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/IDriverResilienceStatusStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/IDriverResilienceStatusStore.cs new file mode 100644 index 00000000..0e503824 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/IDriverResilienceStatusStore.cs @@ -0,0 +1,36 @@ +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Hubs; + +/// +/// Singleton last-snapshot-per-(instance, host) cache of driver resilience status +/// (breaker open / consecutive failures / in-flight depth). Populated by +/// DriverResilienceStatusBridge as it forwards driver-resilience-status DPS +/// messages; read in-process by the driver status panel's resilience section via +/// + . Mirrors +/// — the panel reads this singleton directly instead of +/// opening a self-targeted SignalR connection (which a server-side Blazor component cannot reach +/// behind a reverse proxy). +/// +public interface IDriverResilienceStatusStore +{ + /// Stores or replaces the last-known resilience snapshot for a (instance, host) pair. + /// The resilience status snapshot to store. + void Upsert(DriverResilienceStatusChanged snapshot); + + /// Returns every stored host snapshot for one driver instance (empty if none). + /// The driver instance's stable logical ID. + /// The per-host resilience snapshots for that instance. + IReadOnlyList GetForInstance(string driverInstanceId); + + /// Returns a point-in-time snapshot of every tracked (instance, host) pair. + /// A read-only collection of the last-known resilience snapshot for each pair. + IReadOnlyCollection GetAll(); + + /// + /// Raised after every with the just-stored snapshot. Lets the Blazor Server + /// driver status panel receive live updates by reading this singleton directly. Handlers run on + /// the caller's thread (the bridge actor), so subscribers must marshal to their own sync context. + /// + event Action? SnapshotChanged; +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/InMemoryDriverResilienceStatusStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/InMemoryDriverResilienceStatusStore.cs new file mode 100644 index 00000000..0097b644 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/InMemoryDriverResilienceStatusStore.cs @@ -0,0 +1,33 @@ +using System.Collections.Concurrent; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Hubs; + +/// +/// Thread-safe in-memory implementation of . +/// Keyed by (DriverInstanceId, HostName); last write wins. +/// +public sealed class InMemoryDriverResilienceStatusStore : IDriverResilienceStatusStore +{ + private readonly ConcurrentDictionary<(string Instance, string Host), DriverResilienceStatusChanged> _byPair = new(); + + /// + public event Action? SnapshotChanged; + + /// + public void Upsert(DriverResilienceStatusChanged snapshot) + { + _byPair[(snapshot.DriverInstanceId, snapshot.HostName)] = snapshot; + // Capture-then-invoke so a concurrent unsubscribe can't null the delegate mid-raise. + SnapshotChanged?.Invoke(snapshot); + } + + /// + public IReadOnlyList GetForInstance(string driverInstanceId) => + _byPair.Values + .Where(s => string.Equals(s.DriverInstanceId, driverInstanceId, StringComparison.Ordinal)) + .ToList(); + + /// + public IReadOnlyCollection GetAll() => _byPair.Values.ToArray(); +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/DriverResilienceStatusStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/DriverResilienceStatusStoreTests.cs new file mode 100644 index 00000000..4dd5ab91 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/DriverResilienceStatusStoreTests.cs @@ -0,0 +1,87 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests; + +/// +/// Covers the in-process push contract the Blazor Server driver status panel's resilience section +/// relies on: fires on every +/// , keyed per (instance, host) with +/// last-write-wins, and GetForInstance returns only the queried instance's hosts. Mirrors +/// ; the panel reads this store directly (no +/// self-targeted SignalR connection). +/// +public sealed class DriverResilienceStatusStoreTests +{ + private static DriverResilienceStatusChanged Msg( + string instance, string host, bool breakerOpen = false, int failures = 0) => + new(instance, host, breakerOpen, failures, 0, null, + new DateTime(2026, 7, 13, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2026, 7, 13, 0, 0, 0, DateTimeKind.Utc)); + + [Fact] + public void Upsert_raises_SnapshotChanged_with_the_stored_snapshot() + { + var store = new InMemoryDriverResilienceStatusStore(); + var received = new List(); + store.SnapshotChanged += received.Add; + + var msg = Msg("drv-1", "host-a", breakerOpen: true); + store.Upsert(msg); + + received.Count.ShouldBe(1); + received[0].ShouldBeSameAs(msg); + } + + [Fact] + public void Upsert_is_last_write_wins_per_instance_host() + { + var store = new InMemoryDriverResilienceStatusStore(); + store.Upsert(Msg("drv-1", "host-a", failures: 3)); + store.Upsert(Msg("drv-1", "host-a", failures: 0)); + + var hosts = store.GetForInstance("drv-1"); + hosts.Count.ShouldBe(1); + hosts[0].ConsecutiveFailures.ShouldBe(0); + } + + [Fact] + public void GetForInstance_returns_only_the_queried_instances_hosts() + { + var store = new InMemoryDriverResilienceStatusStore(); + store.Upsert(Msg("drv-1", "host-a")); + store.Upsert(Msg("drv-1", "host-b")); + store.Upsert(Msg("drv-2", "host-a")); + + var hosts = store.GetForInstance("drv-1"); + + hosts.Count.ShouldBe(2); + hosts.ShouldAllBe(h => h.DriverInstanceId == "drv-1"); + } + + [Fact] + public void GetForInstance_returns_empty_for_unknown_instance() + { + var store = new InMemoryDriverResilienceStatusStore(); + store.Upsert(Msg("drv-1", "host-a")); + + store.GetForInstance("nope").ShouldBeEmpty(); + } + + [Fact] + public void Unsubscribed_handler_stops_receiving_after_removal() + { + var store = new InMemoryDriverResilienceStatusStore(); + var count = 0; + void Handler(DriverResilienceStatusChanged _) => count++; + + store.SnapshotChanged += Handler; + store.Upsert(Msg("drv-1", "host-a")); + store.SnapshotChanged -= Handler; + store.Upsert(Msg("drv-1", "host-a")); + + count.ShouldBe(1); + } +} From 38c059dae7687ab28e95337ff30d1924cbe5a3a8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:44:44 -0400 Subject: [PATCH 08/10] fix(r2-10): driver-resilience-status DPS bridge --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Hubs/DriverResilienceStatusBridge.cs | 39 +++++++++++++++++++ .../Hubs/HubServiceCollectionExtensions.cs | 8 ++++ 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/DriverResilienceStatusBridge.cs diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 1053d3e8..a63899c7 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -9,7 +9,7 @@ { "id": "T5", "subject": "Host: DriverResilienceStatusPublisherService (BuildMessages mapping + 5s DPS timer shell; TDD mapping in Host.IntegrationTests)", "status": "completed", "blockedBy": ["T1", "T4"] }, { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "completed", "blockedBy": ["T5"] }, { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "completed", "blockedBy": ["T4"] }, - { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "pending", "blockedBy": ["T7"] }, + { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "completed", "blockedBy": ["T7"] }, { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "pending", "blockedBy": ["T7"] }, { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "pending", "blockedBy": ["T6", "T8", "T9"] }, { "id": "T11", "subject": "Docs + review ledger: mark 03/U10 + 04/U-5 remediated; update STATUS.md + FOLLOWUP-10", "status": "pending", "blockedBy": ["T10"] } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/DriverResilienceStatusBridge.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/DriverResilienceStatusBridge.cs new file mode 100644 index 00000000..8b488327 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/DriverResilienceStatusBridge.cs @@ -0,0 +1,39 @@ +using Akka.Actor; +using Akka.Cluster.Tools.PublishSubscribe; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Hubs; + +/// +/// Akka actor that subscribes to the driver-resilience-status DistributedPubSub topic and +/// upserts every snapshot into the in-process +/// . The driver status panel's resilience section reads +/// that store directly (per the house ban on server-side self-targeted SignalR connections), so — +/// unlike — there is no SignalR hub here; resilience has no +/// browser-JS consumer. Spawned per admin-role node by WithOtOpcUaSignalRBridges. +/// +public sealed class DriverResilienceStatusBridge : ReceiveActor +{ + public const string TopicName = DriverResilienceStatusChanged.TopicName; + + private readonly IDriverResilienceStatusStore _store; + + /// Creates actor props for a . + /// Resilience status store updated on every forwarded snapshot. + /// An Akka.NET instance for spawning the bridge actor. + public static Props Props(IDriverResilienceStatusStore store) => + Akka.Actor.Props.Create(() => new DriverResilienceStatusBridge(store)); + + /// Initializes a new instance of . + /// Resilience status store updated on every forwarded snapshot. + public DriverResilienceStatusBridge(IDriverResilienceStatusStore store) + { + _store = store; + Receive(msg => _store.Upsert(msg)); + Receive(_ => { /* DPS confirmation */ }); + } + + /// + protected override void PreStart() => + DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(TopicName, Self)); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs index b77d7048..f3675564 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Hubs/HubServiceCollectionExtensions.cs @@ -11,6 +11,7 @@ public static class HubServiceCollectionExtensions public const string AlertSignalRBridgeName = "alert-signalr-bridge"; public const string ScriptLogSignalRBridgeName = "script-log-signalr-bridge"; public const string DriverStatusSignalRBridgeName = "driver-status-signalr-bridge"; + public const string DriverResilienceStatusBridgeName = "driver-resilience-status-bridge"; /// /// Registers the in-process live-push services the AdminUI's Blazor Server panels read @@ -71,6 +72,12 @@ public static class HubServiceCollectionExtensions var driverStatusStore = resolver.GetService(); var driverStatusBridge = system.ActorOf(DriverStatusSignalRBridge.Props(driverStatusHub, driverStatusStore), DriverStatusSignalRBridgeName); registry.Register(driverStatusBridge); + + // Resilience-status bridge: DPS topic -> in-process store (no SignalR hub — the panel reads + // the store directly, and resilience has no browser-JS consumer). + var resilienceStore = resolver.GetService(); + var resilienceBridge = system.ActorOf(DriverResilienceStatusBridge.Props(resilienceStore), DriverResilienceStatusBridgeName); + registry.Register(resilienceBridge); }); return builder; } @@ -81,3 +88,4 @@ public sealed class FleetStatusSignalRBridgeKey { } public sealed class AlertSignalRBridgeKey { } public sealed class ScriptLogSignalRBridgeKey { } public sealed class DriverStatusSignalRBridgeKey { } +public sealed class DriverResilienceStatusBridgeKey { } From cc5ecb58687914fb09e37f53f67a82dfc7be95e3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:46:47 -0400 Subject: [PATCH 09/10] fix(r2-10): breaker/retry chips on the driver status panel --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Shared/Drivers/DriverStatusPanel.razor | 81 ++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index a63899c7..4f00eb90 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -10,7 +10,7 @@ { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "completed", "blockedBy": ["T5"] }, { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "completed", "blockedBy": ["T4"] }, { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "completed", "blockedBy": ["T7"] }, - { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "pending", "blockedBy": ["T7"] }, + { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "completed", "blockedBy": ["T7"] }, { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "pending", "blockedBy": ["T6", "T8", "T9"] }, { "id": "T11", "subject": "Docs + review ledger: mark 03/U10 + 04/U-5 remediated; update STATUS.md + FOLLOWUP-10", "status": "pending", "blockedBy": ["T10"] } ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor index e069aab0..74327f20 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor @@ -12,6 +12,7 @@ @inject IAuthorizationService AuthorizationService @inject IAdminOperationsClient AdminOps @inject IDriverStatusSnapshotStore StatusStore +@inject IDriverResilienceStatusStore ResilienceStore
@@ -73,6 +74,58 @@ } } + @* --- Resilience (retry storm / circuit-breaker) — fed by the driver-resilience-status DPS store --- *@ + @if (Enabled) + { + @if (_resilience.Count == 0) + { +

No resilience activity recorded yet.

+ } + else + { + var active = _resilience.Values + .Where(h => h.BreakerOpen || h.ConsecutiveFailures > 0 || h.CurrentInFlight > 0 || IsStale(h)) + .OrderBy(h => h.HostName, StringComparer.Ordinal) + .ToList(); + +
+
Resilience
+ @if (active.Count == 0) + { +

All targets healthy.

+ } + else + { + @foreach (var host in active) + { + var stale = IsStale(host); +
+ @host.HostName + @if (host.BreakerOpen) + { + + Breaker OPEN@(host.LastBreakerOpenUtc is { } opened ? $" · opened {HumanizeAge(opened)} ago" : "") + + } + @if (host.ConsecutiveFailures > 0) + { + @host.ConsecutiveFailures consecutive failure@(host.ConsecutiveFailures == 1 ? "" : "s") + } + @if (host.CurrentInFlight > 0) + { + in-flight @host.CurrentInFlight + } + @if (stale) + { + stale + } +
+ } + } +
+ } + } + @* --- Reconnect / Restart action buttons (DriverOperator-gated) --- *@ @if (_canOperate && Enabled) { @@ -142,6 +195,10 @@ private DriverHealthChanged? _snapshot; private DateTime _lastUpdateUtc = DateTime.MinValue; private bool _stale; + + // Per-host resilience snapshots for THIS instance, keyed by HostName. Fed by the + // driver-resilience-status DPS store (read in-process, same house pattern as the health store). + private readonly Dictionary _resilience = new(StringComparer.Ordinal); private bool _connecting; private string? _error; private System.Threading.Timer? _timer; @@ -195,6 +252,11 @@ _snapshot = snap; _lastUpdateUtc = DateTime.UtcNow; } + + // Prime + subscribe the resilience store (same in-process read pattern as the health store). + ResilienceStore.SnapshotChanged += OnResilienceChanged; + foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId)) + _resilience[r.HostName] = r; } catch (Exception ex) { @@ -219,6 +281,22 @@ InvokeAsync(StateHasChanged); } + // Invoked by the resilience store (on the bridge actor's thread) for every (instance, host); + // ignore snapshots for other instances and marshal onto the render sync context. + private void OnResilienceChanged(DriverResilienceStatusChanged msg) + { + if (!string.Equals(msg.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal)) + return; + + _resilience[msg.HostName] = msg; + InvokeAsync(StateHasChanged); + } + + // A host row is stale once its last publish is older than 15 s (3× the 5 s publish interval) — + // covers driver-node death, where entries freeze. Re-evaluated by the existing 5 s render timer. + private static bool IsStale(DriverResilienceStatusChanged h) => + (DateTime.UtcNow - h.PublishedUtc).TotalSeconds > 15; + private async Task ReconnectAsync() { _busyReconnect = true; @@ -300,8 +378,9 @@ public async ValueTask DisposeAsync() { - // Unsubscribe first so the singleton store can't invoke a handler on a disposed component. + // Unsubscribe first so the singleton stores can't invoke a handler on a disposed component. StatusStore.SnapshotChanged -= OnSnapshotChanged; + ResilienceStore.SnapshotChanged -= OnResilienceChanged; // Drain BOTH timers so an in-flight callback can't invoke StateHasChanged on a component // that's already gone. System.Threading.Timer's async dispose awaits any in-flight // callback (.NET 6+). From 1e3552dbd5e67a7565588791ed2a0181595b2bbe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:50:02 -0400 Subject: [PATCH 10/10] docs(archreview): R2-10 resilience observability shipped --- archreview/03-server-runtime.md | 5 ++--- archreview/04-adminui.md | 4 +++- .../plans/FOLLOWUP-10-resilience-dispatch-gap.md | 10 +++++++++- .../plans/R2-10-resilience-observability-plan.md | 11 +++++++++++ .../R2-10-resilience-observability-plan.md.tasks.json | 4 ++-- archreview/plans/STATUS.md | 1 + 6 files changed, 28 insertions(+), 7 deletions(-) diff --git a/archreview/03-server-runtime.md b/archreview/03-server-runtime.md index b6c37ceb..66b212fe 100644 --- a/archreview/03-server-runtime.md +++ b/archreview/03-server-runtime.md @@ -175,9 +175,8 @@ Closed since 9cad9ed0: **hard-kill failover** (`HardKillFailoverTests`, in-proce **U9 — LOW: `EnsureVariable` silently ignores changed historize-intent on an existing node** (`OtOpcUaNodeManager.cs:1345-1349`) — invariant still lives in two files' comments rather than an assert. -**U10 — LOW (NEW): The `DriverResilienceStatusTracker` has no reader — pipeline observability is log-only.** -The tracker is registered, fed by every retry/breaker event, and consumed by nothing (Admin `/hosts` Stream E.2/E.3 never shipped). The interim mitigation (retry/breaker-open/breaker-close logging in `DriverResiliencePipelineBuilder.cs:124-170`, added with #10) is real and correctly gated, but breaker state — now a production behavior that can reject calls (`BrokenCircuitException`) — is invisible to operators except in logs. Known residual from FOLLOWUP-10; recorded here so the review tracks it. -*Recommendation:* ship the `/hosts` resilience column, or at minimum a metric per breaker-open. +**U10 — LOW (NEW): The `DriverResilienceStatusTracker` has no reader — pipeline observability is log-only.** — ✅ **REMEDIATED** (R2-10, branch `r2/10-resilience-observability`). The tracker now has a production reader: a periodic (5 s) full-snapshot publisher (`DriverResilienceStatusPublisherService`, driver nodes, registered next to the tracker with a wiring guard `ResilienceStatusReaderRegistrationTests`) → `driver-resilience-status` DPS topic → per-admin-node `DriverResilienceStatusBridge` → `IDriverResilienceStatusStore` → a resilience section on `DriverStatusPanel.razor` (breaker-OPEN / consecutive-failures / in-flight / staleness chips, read in-process per the self-HubConnection ban). Two tracker truthfulness gaps fixed along the way: `RecordBreakerClose` + `IsBreakerOpen` (breaker-open state is now derivable) and `RecordSuccess` wired onto the invoker success paths (`ConsecutiveFailures` is now a true consecutive counter). Mirrors the driver-health flow one-for-one. *(Original finding:)* The tracker was registered, fed by every retry/breaker event, and consumed by nothing (Admin `/hosts` Stream E.2/E.3 never shipped); the interim mitigation was retry/breaker logging in `DriverResiliencePipelineBuilder.cs`. +*Live gate (T10) pending:* docker-dev breaker-open observation on both centrals + staleness dim. --- diff --git a/archreview/04-adminui.md b/archreview/04-adminui.md index 9b790c74..cbc275b3 100644 --- a/archreview/04-adminui.md +++ b/archreview/04-adminui.md @@ -240,7 +240,9 @@ There is **no bUnit** — 77 razor components have no render/binding coverage, w ~31 `aria-*`/`role=` attributes across 77 components, concentrated in the modals (`role="dialog"`, `btn-close` aria-labels). Gaps: text-glyph expander buttons (`▼`/`▶` in `UnsTree.razor:49`) with no `aria-expanded`/label; modals have no focus trap or Escape handling; live tails (`Alerts`) have no `aria-live` region; tables lack captions/scope. Form labeling is decent (most inputs have `for`/`id` pairs). Acceptable for an internal ops tool; worth a targeted pass on the tree and the modals if operator diversity matters. -### U-5 (NEW) — Resilience runtime status has no AdminUI surface; overrides are authored blind +### U-5 (NEW) — Resilience runtime status has no AdminUI surface; overrides are authored blind — ✅ REMEDIATED (R2-10) + +**Remediated** on branch `r2/10-resilience-observability`: `DriverResilienceStatusTracker` now flows to the AdminUI through exactly the recommended idiom — a per-node DPS bridge (`DriverResilienceStatusBridge`) → in-process store (`IDriverResilienceStatusStore`, registered beside `IDriverStatusSnapshotStore` in `AddOtOpcUaDriverStatusServices`) → a **resilience section on `DriverStatusPanel.razor`** (embedded on all 8 driver pages, right where the `ResilienceConfig` overrides are authored — closing the feedback loop). It renders breaker-OPEN / consecutive-failures / in-flight / staleness chips, read in-process (self-HubConnection ban honored). The driver-node feed is a periodic 5 s full-snapshot publisher (`DriverResilienceStatusPublisherService`) with a wiring guard. Live gate (T10) — docker-dev breaker-open on both centrals + staleness — pending. *(Original finding below.)* The remediation merges made per-driver resilience live in production: #10 (`bacea1a4`) wired `CapabilityInvoker` into every `DriverInstanceActor` dispatch site, and #13 (`75403caa`) applies the AdminUI-authored `ResilienceConfig` overrides from the deploy artifact. But `DriverResilienceStatusTracker` (`src/Core/ZB.MOM.WW.OtOpcUa.Core/Resilience/DriverResilienceStatusTracker.cs`) still has **no reader** — no AdminUI component, hub, or endpoint consumes it (`grep -rn Resilience src/Server/ZB.MOM.WW.OtOpcUa.AdminUI` matches only the authoring form + driver pages). The planned `/hosts` Stream E.2/E.3 status panels never shipped; the interim observability surface is retry/breaker **log lines** added during #10. `archreview/plans/STATUS.md` explicitly records this as "still open as an Admin-UI follow-on." diff --git a/archreview/plans/FOLLOWUP-10-resilience-dispatch-gap.md b/archreview/plans/FOLLOWUP-10-resilience-dispatch-gap.md index febcac70..998438c9 100644 --- a/archreview/plans/FOLLOWUP-10-resilience-dispatch-gap.md +++ b/archreview/plans/FOLLOWUP-10-resilience-dispatch-gap.md @@ -39,11 +39,19 @@ the concrete invoker in would reverse that architectural decision. ResilienceConfig change. Per-instance overrides now reach the invoker. See [`FOLLOWUP-13`](FOLLOWUP-13-resilience-config-artifact.md). -**Observability finding + fix.** The resilience-status tracker has **no reader** (Admin `/hosts`, Phase 6.1 Stream +**Observability finding + fix.** The resilience-status tracker had **no reader** (Admin `/hosts`, Phase 6.1 Stream E.2/E.3, was never built) and the pipeline builder logged nothing — so the pipeline ran **invisibly**. Added retry / breaker-open / breaker-close **structured logging** to `DriverResiliencePipelineBuilder` (optional ILogger, wired in the Host DI). This is the operator-facing observability surface AND the prerequisite for any live-verify. +> **UPDATE (R2-10, branch `r2/10-resilience-observability`):** the tracker now HAS a proper reader. A periodic +> 5 s DPS publisher (`DriverResilienceStatusPublisherService`, driver nodes) mirrors each snapshot → +> `driver-resilience-status` topic → per-admin `DriverResilienceStatusBridge` → `IDriverResilienceStatusStore` → +> breaker/retry/in-flight/staleness chips on `DriverStatusPanel.razor` (mirrors the driver-health flow). Tracker +> truthfulness fixed too: `IsBreakerOpen`/`RecordBreakerClose` + `RecordSuccess` on the invoker success paths. The +> log-line remains the drill-down; the panel is the primary surface. **The live behavioral gate below (observing the +> `circuit-breaker OPENED` line on a dead endpoint) is now folded into R2-10's T10 live-`/run` — still pending.** + **Verification done (deterministic + local):** - Negative control: unwrapping any site fails the Runtime build with OTOPCUA0001. - Recording-invoker Runtime test: write routes via `ExecuteWriteAsync` with the `IPerCallHostResolver` host; diff --git a/archreview/plans/R2-10-resilience-observability-plan.md b/archreview/plans/R2-10-resilience-observability-plan.md index 9d552d58..a24c8daa 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md +++ b/archreview/plans/R2-10-resilience-observability-plan.md @@ -191,3 +191,14 @@ Subscription lifecycle copies the existing pattern exactly: inject the store, su 1. Mark U10/U-5 remediated with the branch/commit; update STATUS.md; annotate FOLLOWUP-10 that the reader shipped and (if observed in T10) the live behavioral log-line gate closed. Commit: `docs(archreview): R2-10 resilience observability shipped`. **Full-suite gate before merge:** `dotnet test ZB.MOM.WW.OtOpcUa.slnx` green (integration suites need their fixtures per CLAUDE.md; the new tests themselves are fixture-free). + +--- + +## Execution deviations (R2-10) + +Executed on branch `r2/10-resilience-observability` (off master `46fedda3`). T1–T9 + T11 completed; T10 deferred-live. + +- **T2 breaker-close test uses a controllable `TimeProvider` (`ManualTimeProvider`), not wall-clock waits.** Driving the real Polly breaker to `OnClosed` requires elapsing the 15 s `BreakDuration` then a successful half-open probe. Rather than a 15 s `Task.Delay`, the test injects a `TimeProvider` that advances both `GetUtcNow` and `GetTimestamp` off one tick counter (`Advance(16 s)`), making the close deterministic and instant. Same self-contained-fake style the repo already uses (`TriePermissionEvaluatorTests`); no new package. This is the one deviation from the plan's "extend the existing breaker test" wording — the existing test only reaches breaker-open, and open→close needs time control. +- **T9 panel renders healthy hosts quietly** rather than one always-visible row per host. Per the plan's "healthy is quiet" intent, the resilience section shows: `No resilience activity recorded yet.` when the store has no entries for the instance; `All targets healthy.` when entries exist but none has an active signal; and a chip row **only** for hosts with breaker-OPEN / consecutive-failures>0 / in-flight>0 / stale. Keeps the panel silent under normal operation (the tracker populates an entry for every host that has ever had a successful call, so an always-on per-host row would be noisy). +- **T10 (live-`/run` gate) deferred-live.** Code-complete + offline-verified (unit + full-solution build). The docker-dev breaker-open observation on both centrals + recovery-clear + staleness-dim is a heavy serial pass left for the consolidated live session (also the first live sighting of the #10 `circuit-breaker OPENED` log line, closing FOLLOWUP-10's behavioral gate). Marked `deferred-live` in `.tasks.json`. +- **T6/T8 registration:** the publisher's `Func` is registered via `TryAddSingleton` (idempotent with the one `Program.cs` registers for the alarm-command router, per plan). The resilience bridge intentionally has **no SignalR hub** (the panel reads the store in-process; resilience has no browser-JS consumer) — a deliberate simplification the plan calls for. diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index 4f00eb90..a8a1e80f 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -11,7 +11,7 @@ { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "completed", "blockedBy": ["T4"] }, { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "completed", "blockedBy": ["T7"] }, { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "completed", "blockedBy": ["T7"] }, - { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "pending", "blockedBy": ["T6", "T8", "T9"] }, - { "id": "T11", "subject": "Docs + review ledger: mark 03/U10 + 04/U-5 remediated; update STATUS.md + FOLLOWUP-10", "status": "pending", "blockedBy": ["T10"] } + { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "deferred-live", "blockedBy": ["T6", "T8", "T9"] }, + { "id": "T11", "subject": "Docs + review ledger: mark 03/U10 + 04/U-5 remediated; update STATUS.md + FOLLOWUP-10", "status": "completed", "blockedBy": ["T10"] } ] } diff --git a/archreview/plans/STATUS.md b/archreview/plans/STATUS.md index 59eaad4e..b97ac4d5 100644 --- a/archreview/plans/STATUS.md +++ b/archreview/plans/STATUS.md @@ -174,6 +174,7 @@ what the unit leg cannot; (c) migrate the 3 xunit v2 holdouts once Akka ships an | **R2-01** | 05/STAB-14 (High regression, rank 1), 05/STAB-15 (rank 3), 05/STAB-8 seam | `r2/01-s7-fault-hardening` (off `1676c8f4`) | `fcdcf0e2`…`bdb7ed41` (13 task commits) | **Code-complete, offline-verified.** All prod changes in `S7Driver.cs`. STAB-14: connect-timeout surfaces as `TimeoutException` (not an escaping OCE) at the `EnsureConnectedAsync`/`InitializeAsync` source + `when(token.IsCancellationRequested)` filters on the read/write ensure-wrappers and all three poll-loop OCE catches — subscription poll loops survive an unreachable-host outage and recover (T2 regression RED→GREEN). STAB-15: `IsS7ConnectionFatal` broadened to the ISO-on-TCP framing surface (TPKT/TPDU/WrongNumberOfBytes/`PlcException{WrongNumberReceivedBytes}`, NOT `InvalidDataException`); mid-PDU cancel marks handle dead; probe classifies+marks-dead under the gate. STAB-8 (S7 part): `EnsureConnectedAsync` documented as the R2-09 connect-throttle seam (note-only). Driver.S7.Tests 246/246. **Live outage gate deferred (T11)** — `S7_1500ConnectTimeoutOutageTests` (env `S7_TIMEOUT_OUTAGE_START_CMD`/`STOP_CMD`, SYN-blackhole) skips clean offline; the connect-*timeout* class the `docker restart` bounce can't produce. (Pre-existing unrelated CLI docs-scan test fails at base — out of scope.) | | **R2-03** | 02/S13 (rank 5), 02/S12, 02/P7 | `r2/03-vt-failure-quality` (off `1676c8f4`) | `b9775485`…`6bb91e8d` (12 task commits) | **Code-complete, offline-verified.** S13: VT script failure/timeout degrades the node to `OpcUaQuality.Bad` (carrying last-known value) instead of freezing on last-Good forever — additive `EvaluationResult.Quality`, bridge historizes `0x80020000u` BadInternalError, published once per Good→Bad transition gated on all deps arrived (inputs-ready, no cold-start flash), recovery force-publishes next Good. S12: apply-boundary `ClearCompiledScripts` racing an in-flight eval into ObjectDisposedException retried once in `RoslynVirtualTagEvaluator` (re-fetch→recompile into current generation); stress guard reproduces pre-fix, deterministically green post-fix. P7: apply-boundary clear gated on a changed deployed expression set (`HashSet.SetEquals`, ordinal) — identical redeploy / rename-with-same-expressions no longer forces full VT recompile. Runtime.Tests 372/372, RoslynVirtualTagEvaluatorTests 20/20. **Deferred (T11/T12):** full cross-suite sweep + docker-dev live-`/run` (fail→Bad→recover + P7 no-recompile spot-check). | | **R2-02** | 01/S-6 (High, rank 2), 01/U-6, 01/S-8=03/S12 (rank 4), 04/C-7, 01/S-7=03/S13 | `r2/02-resilience-config-hardening` (off `1676c8f4`) | `4be13429`…`dd82e740` (17 task commits) | **Code-complete, offline-verified.** **S-6:** parser clamps timeout/retry/breaker overrides (timeout≤0→tier default, breaker==1→2, caps) with diagnostics + belt-and-braces `Build` clamp so pipeline construction can never throw Polly `ValidationException` (new `ResiliencePolicyRanges` in Core.Abstractions); brick-repro + `Build_NeverThrows_ForHostileDirectOptions` + factory-seam wiring proof; AdminUI `Validate()` range warnings. **S-8:** parser forces Write/AlarmAck retry→0 (spec invariant) **and** dispatch flips to `isIdempotent:false` (RED-first on the Runtime wiring test); non-idempotent arm caches the no-retry snapshot once/invoker (01/P-4) + tracker in-flight accounting; `WriteIdempotentAttribute` false claim corrected; retry cells disabled. **U-6:** bulkhead knob deleted end-to-end (migration-free; stored keys ignored) + `Options_properties_are_exactly_the_pipeline_wired_set` inertness guard. **C-7:** `ResilienceFormModel` JsonObject-bag round-trip preserves unknown keys; malformed JSON → `ParseFailed` lock + verbatim `ToJson` + discard button. **S-7:** per-`Create` `Interlocked` options-generation in the pipeline-cache key kills the respawn stale-pipeline race. Core resilience 121/121, AdminUI form 10/10, Analyzers 32/32, Runtime 364/364; 0 production OTOPCUA0001. **Deferred (T15/T18):** docker-dev `:9200` `/run` (warning banner, input lockout, unknown-key survival) + whole-solution sweep. | +| **R2-10** | 03/U10=04/U-5 (Medium/Low, action #10) | `r2/10-resilience-observability` (off `46fedda3`) | `205d3980`…`` (11 task commits) | **Code-complete, offline-verified.** The resilience status tracker now has a production reader, mirroring the driver-health flow. Core truthfulness: `RecordBreakerClose` + derived `IsBreakerOpen` (breaker-open state derivable), `OnClosed` records the close, and `RecordSuccess` wired onto the invoker success paths (`ConsecutiveFailures` is a true consecutive counter, not stuck-forever). Transport: `DriverResilienceStatusChanged` Commons contract → periodic 5 s full-snapshot publisher `DriverResilienceStatusPublisherService` (driver nodes, registered in `AddOtOpcUaDriverFactories` with wiring guard `ResilienceStatusReaderRegistrationTests` — closes the built-but-never-wired class) → `driver-resilience-status` DPS topic → per-admin-node `DriverResilienceStatusBridge` → `IDriverResilienceStatusStore` → resilience chips (breaker-OPEN / consecutive-failures / in-flight / staleness) on `DriverStatusPanel.razor`, read in-process (self-HubConnection ban honored). Core.Tests resilience 31/31 (incl. deterministic breaker-close via a controllable TimeProvider), Host.IntegrationTests publisher+guard 6/6, AdminUI.Tests store 5/5; full-solution build clean. **Deferred (T10):** docker-dev live-`/run` — S7 dead-endpoint breaker-open observed on BOTH centrals + recovery-clear + staleness-dim (also the first live sighting of the #10 `circuit-breaker OPENED` log line, closing FOLLOWUP-10's behavioral gate). | **Report anchor drifts to fold into the next report refresh** (from the R2-02 verification pass): (1) `01-core-composition.md` U-6 anchor `DriverResilienceOptions.cs:23-25` → the two bulkhead properties sat at