fix(r2-10): breaker/retry chips on the driver status panel

archreview/plans/R2-10-resilience-observability-plan.md.tasks.json

@@ -10,7 +10,7 @@
     { "id": "T6", "subject": "Host DI: register the publisher in AddOtOpcUaDriverFactories + ResilienceStatusReaderRegistrationTests wiring guard (tracker HAS a production reader)", "status": "completed", "blockedBy": ["T5"] },
     { "id": "T7", "subject": "AdminUI: IDriverResilienceStatusStore + InMemory impl + AddOtOpcUaDriverStatusServices registration (TDD mirroring DriverStatusSnapshotStoreTests)", "status": "completed", "blockedBy": ["T4"] },
     { "id": "T8", "subject": "AdminUI: DriverResilienceStatusBridge DPS actor + spawn in WithOtOpcUaSignalRBridges", "status": "completed", "blockedBy": ["T7"] },
-    { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "pending", "blockedBy": ["T7"] },
+    { "id": "T9", "subject": "AdminUI: resilience chip section in DriverStatusPanel.razor (in-process store read; no bUnit — verified by T10)", "status": "completed", "blockedBy": ["T7"] },
     { "id": "T10", "subject": "Live-/run gate on docker-dev: rebuild BOTH centrals, S7 dead-endpoint breaker-open, both-replica chips, recovery clear, staleness dim", "status": "pending", "blockedBy": ["T6", "T8", "T9"] },
     { "id": "T11", "subject": "Docs + review ledger: mark 03/U10 + 04/U-5 remediated; update STATUS.md + FOLLOWUP-10", "status": "pending", "blockedBy": ["T10"] }
   ]

src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/DriverStatusPanel.razor

@@ -12,6 +12,7 @@
 @inject IAuthorizationService AuthorizationService
 @inject IAdminOperationsClient AdminOps
 @inject IDriverStatusSnapshotStore StatusStore
+@inject IDriverResilienceStatusStore ResilienceStore
 
 <section class="panel rise mt-3" style="animation-delay:.04s; @(_stale ? "opacity:0.5;" : "")">
     <div class="panel-head d-flex align-items-center gap-2">
@@ -73,6 +74,58 @@
             }
         }
 
+        @* --- Resilience (retry storm / circuit-breaker) — fed by the driver-resilience-status DPS store --- *@
+        @if (Enabled)
+        {
+            @if (_resilience.Count == 0)
+            {
+                <p class="mb-0 mt-3" style="color:var(--ink-faint); font-size:0.85rem">No resilience activity recorded yet.</p>
+            }
+            else
+            {
+                var active = _resilience.Values
+                    .Where(h => h.BreakerOpen || h.ConsecutiveFailures > 0 || h.CurrentInFlight > 0 || IsStale(h))
+                    .OrderBy(h => h.HostName, StringComparer.Ordinal)
+                    .ToList();
+
+                <div class="mt-3">
+                    <div style="color:var(--ink-soft); font-size:0.8rem; font-weight:600; text-transform:uppercase; letter-spacing:.03em">Resilience</div>
+                    @if (active.Count == 0)
+                    {
+                        <p class="mb-0 mt-1" style="color:var(--ink-faint); font-size:0.85rem">All targets healthy.</p>
+                    }
+                    else
+                    {
+                        @foreach (var host in active)
+                        {
+                            var stale = IsStale(host);
+                            <div class="d-flex flex-wrap gap-2 align-items-baseline mt-1" style="@(stale ? "opacity:0.5;" : "")">
+                                <span style="color:var(--ink-soft); font-size:0.85rem"><strong>@host.HostName</strong></span>
+                                @if (host.BreakerOpen)
+                                {
+                                    <span class="chip chip-bad">
+                                        Breaker OPEN@(host.LastBreakerOpenUtc is { } opened ? $" · opened {HumanizeAge(opened)} ago" : "")
+                                    </span>
+                                }
+                                @if (host.ConsecutiveFailures > 0)
+                                {
+                                    <span class="chip chip-warn">@host.ConsecutiveFailures consecutive failure@(host.ConsecutiveFailures == 1 ? "" : "s")</span>
+                                }
+                                @if (host.CurrentInFlight > 0)
+                                {
+                                    <span class="chip chip-idle">in-flight @host.CurrentInFlight</span>
+                                }
+                                @if (stale)
+                                {
+                                    <span class="chip chip-idle">stale</span>
+                                }
+                            </div>
+                        }
+                    }
+                </div>
+            }
+        }
+
         @* --- Reconnect / Restart action buttons (DriverOperator-gated) --- *@
         @if (_canOperate && Enabled)
         {
@@ -142,6 +195,10 @@
     private DriverHealthChanged? _snapshot;
     private DateTime _lastUpdateUtc = DateTime.MinValue;
     private bool _stale;
+
+    // Per-host resilience snapshots for THIS instance, keyed by HostName. Fed by the
+    // driver-resilience-status DPS store (read in-process, same house pattern as the health store).
+    private readonly Dictionary<string, DriverResilienceStatusChanged> _resilience = new(StringComparer.Ordinal);
     private bool _connecting;
     private string? _error;
     private System.Threading.Timer? _timer;
@@ -195,6 +252,11 @@
                 _snapshot = snap;
                 _lastUpdateUtc = DateTime.UtcNow;
             }
+
+            // Prime + subscribe the resilience store (same in-process read pattern as the health store).
+            ResilienceStore.SnapshotChanged += OnResilienceChanged;
+            foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId))
+                _resilience[r.HostName] = r;
         }
         catch (Exception ex)
         {
@@ -219,6 +281,22 @@
         InvokeAsync(StateHasChanged);
     }
 
+    // Invoked by the resilience store (on the bridge actor's thread) for every (instance, host);
+    // ignore snapshots for other instances and marshal onto the render sync context.
+    private void OnResilienceChanged(DriverResilienceStatusChanged msg)
+    {
+        if (!string.Equals(msg.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal))
+            return;
+
+        _resilience[msg.HostName] = msg;
+        InvokeAsync(StateHasChanged);
+    }
+
+    // A host row is stale once its last publish is older than 15 s (3× the 5 s publish interval) —
+    // covers driver-node death, where entries freeze. Re-evaluated by the existing 5 s render timer.
+    private static bool IsStale(DriverResilienceStatusChanged h) =>
+        (DateTime.UtcNow - h.PublishedUtc).TotalSeconds > 15;
+
     private async Task ReconnectAsync()
     {
         _busyReconnect = true;
@@ -300,8 +378,9 @@
 
     public async ValueTask DisposeAsync()
     {
-        // Unsubscribe first so the singleton store can't invoke a handler on a disposed component.
+        // Unsubscribe first so the singleton stores can't invoke a handler on a disposed component.
         StatusStore.SnapshotChanged -= OnSnapshotChanged;
+        ResilienceStore.SnapshotChanged -= OnResilienceChanged;
         // Drain BOTH timers so an in-flight callback can't invoke StateHasChanged on a component
         // that's already gone. System.Threading.Timer's async dispose awaits any in-flight
         // callback (.NET 6+).