fix(r2-10): breaker/retry chips on the driver status panel
This commit is contained in:
+80
-1
@@ -12,6 +12,7 @@
|
||||
@inject IAuthorizationService AuthorizationService
|
||||
@inject IAdminOperationsClient AdminOps
|
||||
@inject IDriverStatusSnapshotStore StatusStore
|
||||
@inject IDriverResilienceStatusStore ResilienceStore
|
||||
|
||||
<section class="panel rise mt-3" style="animation-delay:.04s; @(_stale ? "opacity:0.5;" : "")">
|
||||
<div class="panel-head d-flex align-items-center gap-2">
|
||||
@@ -73,6 +74,58 @@
|
||||
}
|
||||
}
|
||||
|
||||
@* --- Resilience (retry storm / circuit-breaker) — fed by the driver-resilience-status DPS store --- *@
|
||||
@if (Enabled)
|
||||
{
|
||||
@if (_resilience.Count == 0)
|
||||
{
|
||||
<p class="mb-0 mt-3" style="color:var(--ink-faint); font-size:0.85rem">No resilience activity recorded yet.</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
var active = _resilience.Values
|
||||
.Where(h => h.BreakerOpen || h.ConsecutiveFailures > 0 || h.CurrentInFlight > 0 || IsStale(h))
|
||||
.OrderBy(h => h.HostName, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
<div class="mt-3">
|
||||
<div style="color:var(--ink-soft); font-size:0.8rem; font-weight:600; text-transform:uppercase; letter-spacing:.03em">Resilience</div>
|
||||
@if (active.Count == 0)
|
||||
{
|
||||
<p class="mb-0 mt-1" style="color:var(--ink-faint); font-size:0.85rem">All targets healthy.</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
@foreach (var host in active)
|
||||
{
|
||||
var stale = IsStale(host);
|
||||
<div class="d-flex flex-wrap gap-2 align-items-baseline mt-1" style="@(stale ? "opacity:0.5;" : "")">
|
||||
<span style="color:var(--ink-soft); font-size:0.85rem"><strong>@host.HostName</strong></span>
|
||||
@if (host.BreakerOpen)
|
||||
{
|
||||
<span class="chip chip-bad">
|
||||
Breaker OPEN@(host.LastBreakerOpenUtc is { } opened ? $" · opened {HumanizeAge(opened)} ago" : "")
|
||||
</span>
|
||||
}
|
||||
@if (host.ConsecutiveFailures > 0)
|
||||
{
|
||||
<span class="chip chip-warn">@host.ConsecutiveFailures consecutive failure@(host.ConsecutiveFailures == 1 ? "" : "s")</span>
|
||||
}
|
||||
@if (host.CurrentInFlight > 0)
|
||||
{
|
||||
<span class="chip chip-idle">in-flight @host.CurrentInFlight</span>
|
||||
}
|
||||
@if (stale)
|
||||
{
|
||||
<span class="chip chip-idle">stale</span>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
}
|
||||
</div>
|
||||
}
|
||||
}
|
||||
|
||||
@* --- Reconnect / Restart action buttons (DriverOperator-gated) --- *@
|
||||
@if (_canOperate && Enabled)
|
||||
{
|
||||
@@ -142,6 +195,10 @@
|
||||
private DriverHealthChanged? _snapshot;
|
||||
private DateTime _lastUpdateUtc = DateTime.MinValue;
|
||||
private bool _stale;
|
||||
|
||||
// Per-host resilience snapshots for THIS instance, keyed by HostName. Fed by the
|
||||
// driver-resilience-status DPS store (read in-process, same house pattern as the health store).
|
||||
private readonly Dictionary<string, DriverResilienceStatusChanged> _resilience = new(StringComparer.Ordinal);
|
||||
private bool _connecting;
|
||||
private string? _error;
|
||||
private System.Threading.Timer? _timer;
|
||||
@@ -195,6 +252,11 @@
|
||||
_snapshot = snap;
|
||||
_lastUpdateUtc = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
// Prime + subscribe the resilience store (same in-process read pattern as the health store).
|
||||
ResilienceStore.SnapshotChanged += OnResilienceChanged;
|
||||
foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId))
|
||||
_resilience[r.HostName] = r;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -219,6 +281,22 @@
|
||||
InvokeAsync(StateHasChanged);
|
||||
}
|
||||
|
||||
// Invoked by the resilience store (on the bridge actor's thread) for every (instance, host);
|
||||
// ignore snapshots for other instances and marshal onto the render sync context.
|
||||
private void OnResilienceChanged(DriverResilienceStatusChanged msg)
|
||||
{
|
||||
if (!string.Equals(msg.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal))
|
||||
return;
|
||||
|
||||
_resilience[msg.HostName] = msg;
|
||||
InvokeAsync(StateHasChanged);
|
||||
}
|
||||
|
||||
// A host row is stale once its last publish is older than 15 s (3× the 5 s publish interval) —
|
||||
// covers driver-node death, where entries freeze. Re-evaluated by the existing 5 s render timer.
|
||||
private static bool IsStale(DriverResilienceStatusChanged h) =>
|
||||
(DateTime.UtcNow - h.PublishedUtc).TotalSeconds > 15;
|
||||
|
||||
private async Task ReconnectAsync()
|
||||
{
|
||||
_busyReconnect = true;
|
||||
@@ -300,8 +378,9 @@
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
// Unsubscribe first so the singleton store can't invoke a handler on a disposed component.
|
||||
// Unsubscribe first so the singleton stores can't invoke a handler on a disposed component.
|
||||
StatusStore.SnapshotChanged -= OnSnapshotChanged;
|
||||
ResilienceStore.SnapshotChanged -= OnResilienceChanged;
|
||||
// Drain BOTH timers so an in-flight callback can't invoke StateHasChanged on a component
|
||||
// that's already gone. System.Threading.Timer's async dispose awaits any in-flight
|
||||
// callback (.NET 6+).
|
||||
|
||||
Reference in New Issue
Block a user