fix(runtime,galaxy): #869 review round 1 — confirmation off the actor scheduler, stop/async-fault tests

- Start the ack confirmation with Task.Run + ConfigureAwait(false) so it never
  depends on the ReceiveAsync handler's ActorTaskScheduler (the PostStop
  abandon-and-revert report must run after the actor stopped).
- DriverInstanceActorAckConfirmationTests: a confirmation that faults after the
  handler returned is reported; one pending at actor stop is reported; a
  confirmed ack reports nothing (positive control).
- Deterministic tracker matching test (Raise/Clear/Retrigger/other alarm do not
  confirm; case-insensitive match does).
- AdminUI min/max on the confirm-timeout field; Galaxy.md documents the
  reconnect, UNACK_RTN-as-Clear and RPC-budget limits.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:11:49 -05:00
parent 4e492e6550
commit 77872b1d1c
5 changed files with 204 additions and 7 deletions
@@ -111,7 +111,7 @@
</div>
<div class="col-md-3">
<label class="form-label">Ack confirm timeout (ms)</label>
<InputNumber @bind-Value="_form.Galaxy.AckConfirmTimeoutMs" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<InputNumber @bind-Value="_form.Galaxy.AckConfirmTimeoutMs" @bind-Value:after="EmitAsync" min="0" max="120000" class="form-control form-control-sm" />
<div class="form-text">When the gateway sends an ack but cannot confirm it, wait this long for Galaxy's acknowledged transition before reverting the condition to unacknowledged (#869). Default 10000; 0 = do not wait.</div>
</div>
</div>
@@ -1376,8 +1376,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
// "sent but unconfirmed" OK). The confirmation wait is seconds long, so it runs OFF this mailbox and
// OUTSIDE the resilience pipeline (a wait on an upstream event — it must neither be cut by the
// AlarmAcknowledge capability timeout nor count toward the breaker). Failure reverts like any other.
// Task.Run, not a bare call: this handler is a ReceiveAsync, so TaskScheduler.Current is the actor's
// scheduler; the confirmation must not depend on this actor's context at all — in particular not after
// PostStop, when it still has to report the abandoned ack (pinned by DriverInstanceActorAckConfirmationTests).
if (src is IAlarmAcknowledgeConfirmation confirmer)
_ = RunAckConfirmationAsync(confirmer, request, msg, replyTo, _ackConfirmCts.Token, _driverInstanceId, _log);
{
var stopToken = _ackConfirmCts.Token;
var driverInstanceId = _driverInstanceId;
var log = _log;
_ = Task.Run(() => RunAckConfirmationAsync(confirmer, request, msg, replyTo, stopToken, driverInstanceId, log));
}
}
catch (DriverOperationRefusedException ex)
{
@@ -1412,10 +1420,10 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
IAlarmAcknowledgeConfirmation confirmer, IReadOnlyList<AlarmAcknowledgeRequest> request, RouteAlarmAck msg,
IActorRef replyTo, CancellationToken stopToken, string driverInstanceId, ILoggingAdapter log)
{
await Task.Yield();
try
{
await confirmer.ConfirmAcknowledgedAsync(request, stopToken).WaitAsync(AckConfirmationCeiling, stopToken);
await confirmer.ConfirmAcknowledgedAsync(request, stopToken)
.WaitAsync(AckConfirmationCeiling, stopToken).ConfigureAwait(false);
}
catch (DriverOperationRefusedException ex)
{