From cc5ecb58687914fb09e37f53f67a82dfc7be95e3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:46:47 -0400 Subject: [PATCH] fix(r2-10): breaker/retry chips on the driver status panel --- ...esilience-observability-plan.md.tasks.json | 2 +- .../Shared/Drivers/DriverStatusPanel.razor | 81 ++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json index a63899c7..4f00eb90 100644 --- a/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json +++ b/archreview/plans/R2-10-resilience-observability-plan.md.tasks.json @@ -10,7 +10,7 @@ { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "completed", "blockedBy": ["T5"] }, { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "completed", "blockedBy": ["T4"] }, { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "completed", "blockedBy": ["T7"] }, - { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "pending", "blockedBy": ["T7"] }, + { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "completed", "blockedBy": ["T7"] }, { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "pending", "blockedBy": ["T6", "T8", "T9"] }, { "id": "T11", "subject": "Docs + review ledger: mark 03/U10 + 04/U-5 remediated; update STATUS.md + FOLLOWUP-10", "status": "pending", "blockedBy": ["T10"] } ] diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor index e069aab0..74327f20 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor @@ -12,6 +12,7 @@ @inject IAuthorizationService AuthorizationService @inject IAdminOperationsClient AdminOps @inject IDriverStatusSnapshotStore StatusStore +@inject IDriverResilienceStatusStore ResilienceStore
@@ -73,6 +74,58 @@ } } + @* --- Resilience (retry storm / circuit-breaker) — fed by the driver-resilience-status DPS store --- *@ + @if (Enabled) + { + @if (_resilience.Count == 0) + { +

No resilience activity recorded yet.

+ } + else + { + var active = _resilience.Values + .Where(h => h.BreakerOpen || h.ConsecutiveFailures > 0 || h.CurrentInFlight > 0 || IsStale(h)) + .OrderBy(h => h.HostName, StringComparer.Ordinal) + .ToList(); + +
+
Resilience
+ @if (active.Count == 0) + { +

All targets healthy.

+ } + else + { + @foreach (var host in active) + { + var stale = IsStale(host); +
+ @host.HostName + @if (host.BreakerOpen) + { + + Breaker OPEN@(host.LastBreakerOpenUtc is { } opened ? $" · opened {HumanizeAge(opened)} ago" : "") + + } + @if (host.ConsecutiveFailures > 0) + { + @host.ConsecutiveFailures consecutive failure@(host.ConsecutiveFailures == 1 ? "" : "s") + } + @if (host.CurrentInFlight > 0) + { + in-flight @host.CurrentInFlight + } + @if (stale) + { + stale + } +
+ } + } +
+ } + } + @* --- Reconnect / Restart action buttons (DriverOperator-gated) --- *@ @if (_canOperate && Enabled) { @@ -142,6 +195,10 @@ private DriverHealthChanged? _snapshot; private DateTime _lastUpdateUtc = DateTime.MinValue; private bool _stale; + + // Per-host resilience snapshots for THIS instance, keyed by HostName. Fed by the + // driver-resilience-status DPS store (read in-process, same house pattern as the health store). + private readonly Dictionary _resilience = new(StringComparer.Ordinal); private bool _connecting; private string? _error; private System.Threading.Timer? _timer; @@ -195,6 +252,11 @@ _snapshot = snap; _lastUpdateUtc = DateTime.UtcNow; } + + // Prime + subscribe the resilience store (same in-process read pattern as the health store). + ResilienceStore.SnapshotChanged += OnResilienceChanged; + foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId)) + _resilience[r.HostName] = r; } catch (Exception ex) { @@ -219,6 +281,22 @@ InvokeAsync(StateHasChanged); } + // Invoked by the resilience store (on the bridge actor's thread) for every (instance, host); + // ignore snapshots for other instances and marshal onto the render sync context. + private void OnResilienceChanged(DriverResilienceStatusChanged msg) + { + if (!string.Equals(msg.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal)) + return; + + _resilience[msg.HostName] = msg; + InvokeAsync(StateHasChanged); + } + + // A host row is stale once its last publish is older than 15 s (3× the 5 s publish interval) — + // covers driver-node death, where entries freeze. Re-evaluated by the existing 5 s render timer. + private static bool IsStale(DriverResilienceStatusChanged h) => + (DateTime.UtcNow - h.PublishedUtc).TotalSeconds > 15; + private async Task ReconnectAsync() { _busyReconnect = true; @@ -300,8 +378,9 @@ public async ValueTask DisposeAsync() { - // Unsubscribe first so the singleton store can't invoke a handler on a disposed component. + // Unsubscribe first so the singleton stores can't invoke a handler on a disposed component. StatusStore.SnapshotChanged -= OnSnapshotChanged; + ResilienceStore.SnapshotChanged -= OnResilienceChanged; // Drain BOTH timers so an in-flight callback can't invoke StateHasChanged on a component // that's already gone. System.Threading.Timer's async dispose awaits any in-flight // callback (.NET 6+).