Guard driver operations and inline restart confirmation

This commit is contained in:
Joseph Doherty
2026-11-14 00:47:11 -05:00
parent 8f0bf834f0
commit 3ae92f4d6e
21 changed files with 4141 additions and 149 deletions
@@ -16,8 +16,8 @@
@inject IDriverStatusSnapshotStore StatusStore
@inject IDriverResilienceStatusStore ResilienceStore
@{ var nodes = _byNode?.Entries ?? []; var now = DateTime.UtcNow; }
<section class="panel rise mt-3" style="animation-delay:.04s">
@{ var nodes = _byNode?.Entries ?? []; var now = DateTime.UtcNow; var presentation = _generation; var confirmation = _confirmationEpoch; var busyAction = BusyChanged; }
<section class="panel rise mt-3" style="animation-delay:.04s" aria-busy="@(Busy ? "true" : "false")">
<div class="panel-head d-flex align-items-center gap-2 flex-wrap">
<span>Driver status</span>
@if (nodes.Count > 0)
@@ -148,12 +148,12 @@
@* --- Reconnect / Restart action buttons (DriverOperator-gated) --- *@
@if (_canOperate && Enabled)
{
<div class="d-flex gap-2 align-items-center mt-3">
<div class="d-flex flex-wrap gap-2 align-items-center mt-3">
<div class="btn-group btn-group-sm" role="group">
<button type="button"
class="btn btn-outline-secondary"
disabled="@_busyReconnect"
@onclick="ReconnectAsync"
disabled="@Busy"
@onclick="() => ReconnectAsync(presentation, busyAction)"
title="Re-establish driver transport without restarting the actor">
@if (_busyReconnect)
{
@@ -168,8 +168,8 @@
<button type="button"
class="btn btn-outline-danger"
disabled="@_busyRestart"
@onclick="() => _showRestartConfirm = true"
disabled="@Busy"
@ref="_restartButton" @onclick="() => BeginRestart(presentation)"
title="Stop and respawn the driver actor — briefly interrupts active subscriptions">
@if (_busyRestart)
{
@@ -185,7 +185,7 @@
@if (_opResultMessage is not null)
{
<span class="chip @(_opResultOk ? "chip-ok" : "chip-bad")" style="font-size:0.8rem">@_opResultMessage</span>
<span role="status" class="chip @(_opResultOk ? "chip-ok" : "chip-bad")" style="font-size:0.8rem">@_opResultMessage</span>
@if (!string.Equals(_opResultMessage, _opResultMessageFull, StringComparison.Ordinal))
{
<DiagnosticDetails Summary="Operation details" Text="@_opResultMessageFull" />
@@ -196,14 +196,12 @@
@* Inline confirm dialog for Restart (no JS confirm — keeps SignalR event loop unblocked) *@
@if (_showRestartConfirm)
{
<div class="mt-2 p-2 border rounded" style="background:var(--surface-raised,#fff); border-color:var(--bad,#dc3545)!important; max-width:420px; font-size:0.9rem">
<p class="mb-2" style="color:var(--ink)">
Restart driver <code>@DriverInstanceId</code>?<br />
<span style="color:var(--ink-soft)">This briefly interrupts active subscriptions and clears in-memory state.</span>
</p>
<div @key="(presentation, confirmation)" role="group" aria-labelledby='@FieldId("restart-title")' aria-describedby='@FieldId("restart-impact")' class="mt-2 p-2 border rounded" style="background:var(--card); border-color:var(--bad-ink)!important; max-width:420px; font-size:0.9rem">
<h3 id='@FieldId("restart-title")' class="h6 mb-2">Restart driver <code>@DriverInstanceId</code>?</h3>
<p id='@FieldId("restart-impact")' class="mb-2" style="color:var(--ink-soft)">This briefly interrupts active subscriptions and clears in-memory state.</p>
<div class="btn-group btn-group-sm" role="group">
<button type="button" class="btn btn-danger" @onclick="RestartConfirmedAsync">Confirm restart</button>
<button type="button" class="btn btn-outline-secondary" @onclick="() => _showRestartConfirm = false">Cancel</button>
<button type="button" class="btn btn-danger" disabled="@Busy" @onclick="() => RestartConfirmedAsync(presentation, confirmation, busyAction)">Confirm restart</button>
<button type="button" class="btn btn-outline-secondary" data-initial-focus="true" @ref="_confirmationCancel" disabled="@Busy" @onclick="() => CancelRestart(presentation, confirmation)">Cancel</button>
</div>
</div>
}
@@ -216,8 +214,10 @@
/// <summary>Cluster identifier forwarded in Reconnect/Restart messages for audit.</summary>
[Parameter] public string ClusterId { get; set; } = "";
[Parameter] public bool Enabled { get; set; } = true;
/// <summary>Reports a current single-flight operation to an owning controls surface.</summary>
[Parameter] public EventCallback<bool> BusyChanged { get; set; }
// One entry per reporting node (lmxopcua#851). Created in OnInitializedAsync once DriverInstanceId is bound.
// One entry per reporting node (lmxopcua#851), recreated only when the bound driver context changes.
// Staleness is per node: a node that stops publishing dims on its own while its peer stays live.
private DriverStatusByNode? _byNode;
private static readonly TimeSpan StaleAfter = TimeSpan.FromSeconds(30);
@@ -233,8 +233,8 @@
private bool _canOperate;
// Action state
private bool _busyReconnect;
private bool _busyRestart;
private bool _busyReconnect => Busy && _operation!.Restart == false;
private bool _busyRestart => Busy && _operation!.Restart;
private bool _showRestartConfirm;
// Keep the outcome chip compact; complete metadata is available in a native disclosure.
private string? _opResultMessage;
@@ -242,181 +242,226 @@
private bool _opResultOk;
private System.Threading.Timer? _opResultClearTimer;
protected override async Task OnInitializedAsync()
private object? _identity;
private int _generation;
private volatile bool _disposed;
private bool _healthSubscribed, _resilienceSubscribed;
private Operation? _operation;
private int _resultSequence;
private int _confirmationEpoch;
private ElementReference _restartButton, _confirmationCancel;
private FocusRequest? _focusRequest;
private readonly string _fieldScope = "ot-driver-operation-" + Guid.NewGuid().ToString("N");
private string FieldId(string suffix) => _fieldScope + "-" + suffix;
private bool Current(int generation) => !_disposed && generation == _generation &&
Equals(_identity, (DriverInstanceId, ClusterId, Enabled));
private bool Busy => _operation is { } operation && Current(operation.Generation);
private bool OperationCurrent(Operation operation) => Current(operation.Generation) && ReferenceEquals(_operation, operation);
protected override async Task OnParametersSetAsync()
{
// Check DriverOperator authorization so buttons only render for permitted users.
// The username for audit logging is re-read at button-click time (not captured here)
// so token-refreshes mid-session land in audit entries accurately.
var auth = await AuthState.GetAuthenticationStateAsync();
var authResult = await AuthorizationService.AuthorizeAsync(auth.User, null, AdminUiPolicies.DriverOperator);
_canOperate = authResult.Succeeded;
var identity = (DriverInstanceId, ClusterId, Enabled);
if (_disposed || Equals(_identity, identity)) return;
var generation = ++_generation;
_identity = identity;
RetireSubscriptions();
_timer?.Dispose(); _timer = null;
_opResultClearTimer?.Dispose(); _opResultClearTimer = null;
++_resultSequence;
_operation = null;
_canOperate = false; _connecting = false; _error = null;
_byNode = null; _resilience.Clear();
_opResultMessage = null; _opResultMessageFull = null;
RetireConfirmation();
if (!Enabled)
return;
_connecting = true;
_byNode = new DriverStatusByNode(DriverInstanceId);
// Tick every 5 s to re-render the per-node stale dimming and humanized ages.
_timer = new System.Threading.Timer(_ => InvokeAsync(StateHasChanged),
null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
// Read live status straight from the in-process snapshot store rather than opening a
// self-targeted SignalR connection. This component runs server-side (Blazor
// InteractiveServer), so a HubConnection to the browser's public URL (e.g.
// http://localhost:9200 behind Traefik) would dial that port from *inside* the container —
// where only Kestrel's :9000 listens — and fail with "Connection refused". The store is fed
// on every admin node by DriverStatusSignalRBridge (a per-node DistributedPubSub
// subscriber), so the local singleton is always current regardless of which replica serves
// this circuit.
AuthorizationResult authResult;
try
{
// Subscribe BEFORE priming so a publish racing the read is not missed; a primed copy that lands after
// a newer racing live one is dropped by Upsert (it keeps the later PublishedUtc per node). Primed
// copies keep the store's own receive time (#857): stamping them "now" showed a node that stopped
// publishing before the dialog opened as live for the first StaleAfter.
StatusStore.SnapshotChanged += OnSnapshotChanged;
_byNode.Prime(StatusStore.GetAllWithReceiveTime(), DateTime.UtcNow, StaleAfter);
var auth = await AuthState.GetAuthenticationStateAsync();
if (!Current(generation)) return;
authResult = await AuthorizationService.AuthorizeAsync(auth.User, null, AdminUiPolicies.DriverOperator);
}
catch (Exception) when (!Current(generation)) { return; }
if (!Current(generation)) return;
_canOperate = authResult.Succeeded;
if (!Enabled) return;
// Prime + subscribe the resilience store (same in-process read pattern as the health store).
_connecting = true;
_byNode = new DriverStatusByNode(DriverInstanceId);
_timer = new System.Threading.Timer(state => { _ = RenderTickAsync(generation); },
null, TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(5));
try
{
// Subscribe before priming; preserve the store's actual local receive time per node.
StatusStore.SnapshotChanged += OnSnapshotChanged;
_healthSubscribed = true;
_byNode.Prime(StatusStore.GetAllWithReceiveTime(), DateTime.UtcNow, StaleAfter);
ResilienceStore.SnapshotChanged += OnResilienceChanged;
foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId))
_resilience[r.HostName] = r;
}
catch (Exception ex)
{
_error = ex.Message;
}
finally
{
_connecting = false;
_resilienceSubscribed = true;
foreach (var r in ResilienceStore.GetForInstance(DriverInstanceId)) _resilience[r.HostName] = r;
}
catch (Exception ex) { if (Current(generation)) _error = ex.Message; }
finally { if (Current(generation)) _connecting = false; }
}
protected override async Task OnAfterRenderAsync(bool firstRender)
{
var request = _focusRequest;
_focusRequest = null;
if (request is null || !Current(request.Generation) || request.Epoch != _confirmationEpoch ||
Busy || _showRestartConfirm != request.Cancel) return;
var target = request.Cancel ? _confirmationCancel : _restartButton;
try { await target.FocusAsync(); }
catch (JSDisconnectedException) { }
catch (JSException) { }
catch (InvalidOperationException) { } // Static SSR has no browser element runtime.
}
private void RetireSubscriptions()
{
if (_healthSubscribed) StatusStore.SnapshotChanged -= OnSnapshotChanged;
if (_resilienceSubscribed) ResilienceStore.SnapshotChanged -= OnResilienceChanged;
_healthSubscribed = _resilienceSubscribed = false;
}
private Task RenderTickAsync(int generation) => MarshalAsync(generation, StateHasChanged);
private async Task MarshalAsync(int generation, Action update)
{
if (!Current(generation)) return;
try { await InvokeAsync(() => { if (Current(generation)) update(); }); }
catch (Exception ex) when (ex is ObjectDisposedException or TaskCanceledException) { }
catch (InvalidOperationException) when (_disposed || !Current(generation)) { }
}
// Invoked by the snapshot store (on the bridge actor's thread) for every driver instance;
// ignore snapshots for other instances and marshal onto the render sync context.
private void OnSnapshotChanged(DriverHealthChanged snap)
{
if (_byNode is null || !_byNode.Upsert(snap, DateTime.UtcNow))
return;
InvokeAsync(StateHasChanged);
var generation = _generation;
var view = _byNode;
if (view is null || !Enabled || !Current(generation)) return;
_ = MarshalAsync(generation, () =>
{
if (ReferenceEquals(view, _byNode) && view.Upsert(snap, DateTime.UtcNow)) StateHasChanged();
});
}
// Invoked by the resilience store (on the bridge actor's thread) for every (instance, host);
// ignore snapshots for other instances and marshal onto the render sync context.
private void OnResilienceChanged(DriverResilienceStatusChanged msg)
{
if (!string.Equals(msg.DriverInstanceId, DriverInstanceId, StringComparison.Ordinal))
return;
_resilience[msg.HostName] = msg;
InvokeAsync(StateHasChanged);
var generation = _generation;
var driver = DriverInstanceId;
if (!Enabled || !Current(generation) || !string.Equals(msg.DriverInstanceId, driver, StringComparison.Ordinal)) return;
_ = MarshalAsync(generation, () => { _resilience[msg.HostName] = msg; StateHasChanged(); });
}
// A host row is stale once its last publish is older than 15 s (3× the 5 s publish interval) —
// covers driver-node death, where entries freeze. Re-evaluated by the existing 5 s render timer.
// Preserve the existing three-publish resilience age boundary independently of health receive time.
private static bool IsStale(DriverResilienceStatusChanged h) =>
(DateTime.UtcNow - h.PublishedUtc).TotalSeconds > 15;
private async Task ReconnectAsync()
private void RetireConfirmation()
{
_busyReconnect = true;
_opResultMessage = null;
_opResultMessageFull = null;
_showRestartConfirm = false; ++_confirmationEpoch; _focusRequest = null;
}
private void BeginRestart(int generation)
{
if (!Current(generation) || !Enabled || !_canOperate || Busy || _showRestartConfirm) return;
_showRestartConfirm = true;
var epoch = ++_confirmationEpoch;
_focusRequest = new(generation, epoch, true);
}
private void CancelRestart(int generation, int epoch)
{
if (!Current(generation) || epoch != _confirmationEpoch || !_showRestartConfirm || Busy) return;
RetireConfirmation();
_focusRequest = new(generation, _confirmationEpoch, false);
}
private Task ReconnectAsync(int generation, EventCallback<bool> busyAction) => RunOperationAsync(generation, false, busyAction);
private Task RestartConfirmedAsync(int generation, int epoch, EventCallback<bool> busyAction) =>
!Current(generation) || !_showRestartConfirm || epoch != _confirmationEpoch
? Task.CompletedTask : RunOperationAsync(generation, true, busyAction);
private async Task RunOperationAsync(int generation, bool restart, EventCallback<bool> busyAction)
{
if (!Current(generation) || !Enabled || !_canOperate || Busy) return;
var operation = new Operation(generation, DriverInstanceId, ClusterId, restart, Guid.NewGuid(), busyAction);
_operation = operation;
RetireConfirmation();
_opResultMessage = null; _opResultMessageFull = null;
_opResultClearTimer?.Dispose(); _opResultClearTimer = null; ++_resultSequence;
StateHasChanged();
try
{
var userName = await GetCurrentUserNameAsync();
await operation.BusyAction.InvokeAsync(true);
if (!OperationCurrent(operation) || !Enabled) return;
// Re-read claims at click time; current authorization is checked again before actor dispatch.
var auth = await AuthState.GetAuthenticationStateAsync();
if (!OperationCurrent(operation) || !Enabled) return;
var allowed = await AuthorizationService.AuthorizeAsync(auth.User, null, AdminUiPolicies.DriverOperator);
if (!OperationCurrent(operation) || !Enabled) return;
if (!allowed.Succeeded) { _canOperate = false; return; }
var userName = auth.User.Identity?.Name
?? auth.User.FindFirst(System.Security.Claims.ClaimTypes.NameIdentifier)?.Value
?? "unknown";
using var cts = new System.Threading.CancellationTokenSource(TimeSpan.FromSeconds(15));
var result = await AdminOps.AskAsync<ReconnectDriverResult>(
new ReconnectDriver(ClusterId, DriverInstanceId, userName, Guid.NewGuid()),
cts.Token);
ShowOpResult(result.Ok, result.Ok ? "Reconnect dispatched" : (result.Message ?? "Failed"));
}
catch (Exception ex)
{
ShowOpResult(false, ex.Message);
if (restart)
{
var result = await AdminOps.AskAsync<RestartDriverResult>(
new RestartDriver(operation.Cluster, operation.Driver, userName, operation.Correlation), cts.Token);
if (OperationCurrent(operation)) ShowOpResult(result.Ok, result.Ok ? "Restart dispatched" : (result.Message ?? "Failed"), generation);
}
else
{
var result = await AdminOps.AskAsync<ReconnectDriverResult>(
new ReconnectDriver(operation.Cluster, operation.Driver, userName, operation.Correlation), cts.Token);
if (OperationCurrent(operation)) ShowOpResult(result.Ok, result.Ok ? "Reconnect dispatched" : (result.Message ?? "Failed"), generation);
}
}
catch (Exception ex) { if (OperationCurrent(operation)) ShowOpResult(false, ex.Message, generation); }
finally
{
_busyReconnect = false;
StateHasChanged();
if (ReferenceEquals(_operation, operation))
{
try { if (Current(generation)) await operation.BusyAction.InvokeAsync(false); }
finally
{
if (ReferenceEquals(_operation, operation)) _operation = null;
if (Current(generation)) StateHasChanged();
}
}
}
}
private async Task RestartConfirmedAsync()
{
_showRestartConfirm = false;
_busyRestart = true;
_opResultMessage = null;
_opResultMessageFull = null;
StateHasChanged();
try
{
var userName = await GetCurrentUserNameAsync();
using var cts = new System.Threading.CancellationTokenSource(TimeSpan.FromSeconds(15));
var result = await AdminOps.AskAsync<RestartDriverResult>(
new RestartDriver(ClusterId, DriverInstanceId, userName, Guid.NewGuid()),
cts.Token);
ShowOpResult(result.Ok, result.Ok ? "Restart dispatched" : (result.Message ?? "Failed"));
}
catch (Exception ex)
{
ShowOpResult(false, ex.Message);
}
finally
{
_busyRestart = false;
StateHasChanged();
}
}
/// <summary>
/// Re-reads the AuthenticationState at call time so the username forwarded to the
/// audit log reflects the current claims-principal — survives token refresh / role
/// change during a long-lived Blazor circuit. Returns "unknown" if no Name claim is
/// present (auth requirements upstream should normally prevent this).
/// </summary>
private async Task<string> GetCurrentUserNameAsync()
{
var auth = await AuthState.GetAuthenticationStateAsync();
return auth.User.Identity?.Name
?? auth.User.FindFirst(System.Security.Claims.ClaimTypes.NameIdentifier)?.Value
?? "unknown";
}
private void ShowOpResult(bool ok, string message)
private void ShowOpResult(bool ok, string message, int generation)
{
if (!Current(generation)) return;
_opResultOk = ok;
// Single truncation choke point: callers pass the untruncated message, the chip renders the
// abbreviation and carries the original on its title attribute.
_opResultMessageFull = message;
_opResultMessage = DisplayText.Abbreviate(message, 60);
// Auto-clear the result chip after 8 s. System.Threading.Timer is used (not
// System.Timers.Timer) so DisposeAsync can drain any in-flight callback.
var resultSequence = ++_resultSequence;
_opResultClearTimer?.Dispose();
_opResultClearTimer = new System.Threading.Timer(_ =>
_opResultClearTimer = new System.Threading.Timer(state =>
{
_opResultMessage = null;
_opResultMessageFull = null;
InvokeAsync(StateHasChanged);
_ = MarshalAsync(generation, () =>
{
if (resultSequence != _resultSequence) return;
_opResultMessage = null; _opResultMessageFull = null; StateHasChanged();
});
}, null, TimeSpan.FromSeconds(8), Timeout.InfiniteTimeSpan);
}
public async ValueTask DisposeAsync()
{
// Unsubscribe first so the singleton stores can't invoke a handler on a disposed component.
StatusStore.SnapshotChanged -= OnSnapshotChanged;
ResilienceStore.SnapshotChanged -= OnResilienceChanged;
// Drain BOTH timers so an in-flight callback can't invoke StateHasChanged on a component
// that's already gone. System.Threading.Timer's async dispose awaits any in-flight
// callback (.NET 6+).
if (_timer is not null) await _timer.DisposeAsync();
if (_opResultClearTimer is not null) await _opResultClearTimer.DisposeAsync();
if (_disposed) return;
_disposed = true; ++_generation; ++_resultSequence;
_operation = null; RetireConfirmation(); RetireSubscriptions();
var tick = _timer; var outcome = _opResultClearTimer;
_timer = null; _opResultClearTimer = null;
if (tick is not null) await tick.DisposeAsync();
if (outcome is not null) await outcome.DisposeAsync();
}
private sealed record Operation(int Generation, string Driver, string Cluster, bool Restart,
Guid Correlation, EventCallback<bool> BusyAction);
private sealed record FocusRequest(int Generation, int Epoch, bool Cancel);
// Map DriverState string → chip CSS class using the 4 defined theme variants.
private static string ChipClass(string? state) => state switch
{