fix(r2-10): breaker/retry chips on the driver status panel

This commit is contained in:
Joseph Doherty
2026-07-13 10:46:47 -04:00
parent 38c059dae7
commit cc5ecb5868
2 changed files with 81 additions and 2 deletions
@@ -12,6 +12,7 @@
@inject IAuthorizationService AuthorizationService
@inject IAdminOperationsClient AdminOps
@inject IDriverStatusSnapshotStore StatusStore
@inject IDriverResilienceStatusStore ResilienceStore
<section class="panel rise mt-3" style="animation-delay:.04s; @(_stale ? "opacity:0.5;" : "")">
<div class="panel-head d-flex align-items-center gap-2">
@@ -73,6 +74,58 @@
}
}
@* --- Resilience (retry storm / circuit-breaker) — fed by the driver-resilience-status DPS store --- *@
@if (Enabled)
{
@if (_resilience.Count == 0)
{
<p class="mb-0 mt-3" style="color:var(--ink-faint); font-size:0.85rem">No resilience activity recorded yet.</p>
}
else
{
var active = _resilience.Values
.Where(h => h.BreakerOpen || h.ConsecutiveFailures > 0 || h.CurrentInFlight > 0 || IsStale(h))
.OrderBy(h => h.HostName, StringComparer.Ordinal)
.ToList();
<div class="mt-3">
<div style="color:var(--ink-soft); font-size:0.8rem; font-weight:600; text-transform:uppercase; letter-spacing:.03em">Resilience</div>
@if (active.Count == 0)
{
<p class="mb-0 mt-1" style="color:var(--ink-faint); font-size:0.85rem">All targets healthy.</p>
}
else
{
@foreach (var host in active)
{
var stale = IsStale(host);
<div class="d-flex flex-wrap gap-2 align-items-baseline mt-1" style="@(stale ? "opacity:0.5;" : "")">
<span style="color:var(--ink-soft); font-size:0.85rem"><strong>@host.HostName</strong></span>
@if (host.BreakerOpen)
{
<span class="chip chip-bad">
Breaker OPEN@(host.LastBreakerOpenUtc is { } opened ? $" · opened {HumanizeAge(opened)} ago" : "")
</span>
}
@if (host.ConsecutiveFailures > 0)
{
<span class="chip chip-warn">@host.ConsecutiveFailures consecutive failure@(host.ConsecutiveFailures == 1 ? "" : "s")</span>
}
@if (host.CurrentInFlight > 0)
{
<span class="chip chip-idle">in-flight @host.CurrentInFlight</span>
}
@if (stale)
{
<span class="chip chip-idle">stale</span>
}
</div>
}
}
</div>
}
}
@* --- Reconnect / Restart action buttons (DriverOperator-gated) --- *@
@if (_canOperate && Enabled)
{
@@ -142,6 +195,10 @@
private DriverHealthChanged? _snapshot;
private DateTime _lastUpdateUtc = DateTime.MinValue;
private bool _stale;
// Per-host resilience snapshots for THIS instance, keyed by HostName. Fed by the
// driver-resilience-status DPS store (read in-process, same house pattern as the health store).
private readonly Dictionary<string, DriverResilienceStatusChanged> _resilience = new(StringComparer.Ordinal);
private bool _connecting;
private string? _error;
private System.Threading.Timer? _timer;
@@ -195,6 +252,11 @@
_snapshot = snap;
_lastUpdateUtc = DateTime.UtcNow;
}
// Prime + subscribe the resilience store (same in-process read pattern as the health store).
ResilienceStore.SnapshotChanged += OnResilienceChanged;
foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId))
_resilience[r.HostName] = r;
}
catch (Exception ex)
{
@@ -219,6 +281,22 @@
InvokeAsync(StateHasChanged);
}
// Invoked by the resilience store (on the bridge actor's thread) for every (instance, host);
// ignore snapshots for other instances and marshal onto the render sync context.
private void OnResilienceChanged(DriverResilienceStatusChanged msg)
{
if (!string.Equals(msg.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal))
return;
_resilience[msg.HostName] = msg;
InvokeAsync(StateHasChanged);
}
// A host row is stale once its last publish is older than 15 s (3× the 5 s publish interval) —
// covers driver-node death, where entries freeze. Re-evaluated by the existing 5 s render timer.
private static bool IsStale(DriverResilienceStatusChanged h) =>
(DateTime.UtcNow - h.PublishedUtc).TotalSeconds > 15;
private async Task ReconnectAsync()
{
_busyReconnect = true;
@@ -300,8 +378,9 @@
public async ValueTask DisposeAsync()
{
// Unsubscribe first so the singleton store can't invoke a handler on a disposed component.
// Unsubscribe first so the singleton stores can't invoke a handler on a disposed component.
StatusStore.SnapshotChanged -= OnSnapshotChanged;
ResilienceStore.SnapshotChanged -= OnResilienceChanged;
// Drain BOTH timers so an in-flight callback can't invoke StateHasChanged on a component
// that's already gone. System.Threading.Timer's async dispose awaits any in-flight
// callback (.NET 6+).