fix(worker): unconditional fault observation for abandoned pipe I/O; exception-total transport dispose
This commit is contained in:
Joseph Doherty
@@ -167,18 +167,24 @@ public sealed class WorkerPipeSession
|
||||
|
||||
// Closing the transport is what actually ends a pipe read parked in the kernel: on net48
|
||||
// NamedPipeClientStream.ReadAsync ignores its CancellationToken, so the message loop's
|
||||
// cancellation can never reach one (WRK-31). This is deliberately the LAST teardown step,
|
||||
// because every frame this session will ever write has completed by the time control
|
||||
// reaches here: WorkerFrameWriter.WriteAsync signals only after the frame is written AND
|
||||
// flushed, and every exit path awaits its final write before unwinding — the shutdown ack
|
||||
// and shutdown-timeout fault inside the loop's dispatch, the event-drain and
|
||||
// oversized-event faults inside the drain task the loop awaits, the watchdog fault inside
|
||||
// the heartbeat task the loop awaits, and the handshake fault inside
|
||||
// CompleteStartupHandshakeAsync's catch. In-flight command replies are the one class of
|
||||
// write that can still be racing here, and they raced the identical disposal before this
|
||||
// change (WorkerPipeClient's `using` fired on the very next statement after RunAsync);
|
||||
// both ProcessCommandAsync's Ready-state gate and TryWriteFaultAsync's
|
||||
// ObjectDisposedException/IOException swallow already cover that race.
|
||||
// cancellation can never reach one (WRK-31). It is deliberately the LAST teardown step,
|
||||
// because in the ordinary case every frame this session will ever write has completed by
|
||||
// the time control reaches here: WorkerFrameWriter.WriteAsync signals only after the
|
||||
// frame is written AND flushed, and each exit path awaits its final write before
|
||||
// unwinding — the shutdown ack and shutdown-timeout fault inside the loop's dispatch, the
|
||||
// event-drain and oversized-event faults inside the drain task the loop awaits, the
|
||||
// watchdog fault inside the heartbeat task the loop awaits, and the handshake fault
|
||||
// inside CompleteStartupHandshakeAsync's catch.
|
||||
//
|
||||
// "Ordinary" is the honest word, not "always": the loop's wait on the heartbeat and
|
||||
// drain tasks is budgeted (BackgroundTaskStopTimeout), and a stream write is genuinely
|
||||
// uncancellable, so a write that overran the budget can still be in flight against the
|
||||
// stream being disposed here. In-flight command replies are in the same position, and
|
||||
// they raced the identical disposal before this change (WorkerPipeClient's `using` fired
|
||||
// on the very next statement after RunAsync). That is precisely why disposal below is
|
||||
// exception-tolerant and why every abandoned task gets a fault-observing continuation
|
||||
// from ObserveBackgroundTaskStopAsync — a write losing its stream mid-flight must be a
|
||||
// logged non-event, not a lost terminal exception or an unobserved task.
|
||||
//
|
||||
// Owning the disposal here — rather than leaving it to WorkerPipeClient's `using` — is
|
||||
// what makes the abandoned read observable: the fault it takes on disposal lands on a
|
||||
@@ -191,12 +197,21 @@ public sealed class WorkerPipeSession
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the transport this session owns, if it was handed one. Dispose-time failures are
|
||||
/// logged and swallowed: this runs inside <see cref="RunAsync"/>'s finally, where letting an
|
||||
/// <see cref="IOException"/> escape would replace the exception that actually ended the
|
||||
/// session (a shutdown timeout, a protocol violation, an event too large to frame) with a
|
||||
/// far less actionable one. Disposal is idempotent, so <c>WorkerPipeClient</c>'s outer
|
||||
/// <c>using</c> re-disposing the same stream immediately afterwards is a no-op.
|
||||
/// logged and swallowed: this runs inside <see cref="RunAsync"/>'s finally, where letting a
|
||||
/// failure escape would replace the exception that actually ended the session (a shutdown
|
||||
/// timeout, a protocol violation, an event too large to frame) with a far less actionable
|
||||
/// one. Disposal is idempotent, so <c>WorkerPipeClient</c>'s outer <c>using</c> re-disposing
|
||||
/// the same stream immediately afterwards is a no-op.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The catch is deliberately total rather than the <see cref="IOException"/> /
|
||||
/// <see cref="ObjectDisposedException"/> pair the fault-write paths use, matching
|
||||
/// <see cref="ObserveBackgroundTaskStopAsync"/>'s shape. Narrowing it to the expected types
|
||||
/// would let an unexpected one — a <c>Win32Exception</c> surfaced by the handle close, say —
|
||||
/// do the exact harm this guard exists to prevent. The rule is about the position in the
|
||||
/// code, not about which exceptions are plausible: nothing thrown while releasing a handle
|
||||
/// is more actionable than the session's terminal exception, so nothing may displace it.
|
||||
/// </remarks>
|
||||
private void DisposeTransportStream()
|
||||
{
|
||||
if (_transportStream is null)
|
||||
@@ -208,7 +223,7 @@ public sealed class WorkerPipeSession
|
||||
{
|
||||
_transportStream.Dispose();
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException || exception is ObjectDisposedException)
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger?.Error(
|
||||
"WorkerPipeSessionTransportDisposeFailed",
|
||||
@@ -226,16 +241,26 @@ public sealed class WorkerPipeSession
|
||||
/// event-drain loops, so every fault exit (event-drain fault, oversized event, heartbeat
|
||||
/// write failure) leaves a read outstanding on a task the loop never awaits again. Once
|
||||
/// <see cref="DisposeTransportStream"/> has closed the handle that read faults with
|
||||
/// <see cref="ObjectDisposedException"/> or <see cref="IOException"/>; awaiting it here is
|
||||
/// what keeps the fault observed, because the worker installs no
|
||||
/// <see cref="ObjectDisposedException"/>, <see cref="IOException"/>, or a zero-byte read
|
||||
/// mapped to <c>EndOfStream</c>, and that fault has to be observed — the worker installs no
|
||||
/// <c>TaskScheduler.UnobservedTaskException</c> handler.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// No second read can follow this one. The message loop only ever issues a read after
|
||||
/// awaiting the previous one, and it never re-enters after unwinding, so
|
||||
/// <see cref="WorkerFrameReader"/>'s single-consumer invariant — and with it the safety of
|
||||
/// its reused length-prefix buffer and its pooled payload buffer, which the abandoned read
|
||||
/// still owns until it faults — holds through teardown.
|
||||
/// <para>
|
||||
/// The fault is observed <em>unconditionally</em>; it is only <em>logged</em> within
|
||||
/// <see cref="BackgroundTaskStopTimeout"/>. Windows is under no obligation to deliver
|
||||
/// the abandoned read's completion inside that budget, so a bounded await alone would
|
||||
/// reopen the orphaning window it was added to close. <see cref="ObserveBackgroundTaskStopAsync"/>
|
||||
/// therefore hands the task a fault-observing continuation when it gives up waiting,
|
||||
/// which makes the budget a diagnostics decision rather than a correctness one.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// No second read can follow this one. The message loop only ever issues a read after
|
||||
/// awaiting the previous one, and it never re-enters after unwinding, so
|
||||
/// <see cref="WorkerFrameReader"/>'s single-consumer invariant — and with it the safety
|
||||
/// of its reused length-prefix buffer and its pooled payload buffer, which the abandoned
|
||||
/// read still owns until it faults — holds through teardown.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private async Task ObserveAbandonedPipeReadAsync()
|
||||
@@ -405,6 +430,9 @@ public sealed class WorkerPipeSession
|
||||
.CreateLinkedTokenSource(cancellationToken);
|
||||
Task heartbeatTask = RunHeartbeatLoopAsync(heartbeatCancellation.Token);
|
||||
Task eventDrainTask = RunEventDrainLoopAsync(heartbeatCancellation.Token);
|
||||
Debug.Assert(
|
||||
_pendingReadTask is null,
|
||||
"A frame read must never be issued while another is outstanding: WorkerFrameReader is single-consumer, and a second read would race the abandoned one for the reused prefix buffer and could return a pooled payload buffer twice.");
|
||||
Task<WorkerEnvelope> readTask = _reader.ReadAsync(loopCancellation.Token);
|
||||
_pendingReadTask = readTask;
|
||||
|
||||
@@ -425,6 +453,9 @@ public sealed class WorkerPipeSession
|
||||
return;
|
||||
}
|
||||
|
||||
Debug.Assert(
|
||||
_pendingReadTask is null,
|
||||
"The previous read must have been awaited before the next is issued: WorkerFrameReader is single-consumer.");
|
||||
readTask = _reader.ReadAsync(loopCancellation.Token);
|
||||
_pendingReadTask = readTask;
|
||||
}
|
||||
@@ -447,6 +478,32 @@ public sealed class WorkerPipeSession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits a bounded time for a background task to stop and records what happened.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The budget bounds the <em>logging</em>, never the observation. Every task this is
|
||||
/// asked to observe is uncancellable at the point that matters — a net48 pipe read
|
||||
/// ignores its token outright, and <c>WorkerFrameWriter.WriteFrameAsync</c> issues the
|
||||
/// stream write under <c>CancellationToken.None</c> so a frame is never left
|
||||
/// half-written on the wire — so any of them can outlive the budget and only then fault,
|
||||
/// typically against a transport <see cref="RunAsync"/> has since disposed. Overrunning
|
||||
/// the budget and returning is therefore not enough: the task would be left with nobody
|
||||
/// holding it, which is the exact orphaning this method exists to prevent.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// So the timeout path hands the task a fault-observing continuation before returning.
|
||||
/// The fault is then observed unconditionally, whenever it arrives; the budget only
|
||||
/// decides whether it also gets logged here or is swallowed silently by the
|
||||
/// continuation. That distinction matters because the worker installs no
|
||||
/// <c>TaskScheduler.UnobservedTaskException</c> handler, so an unheld faulted task
|
||||
/// surfaces only at finalization.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="task">Background task being stopped.</param>
|
||||
/// <param name="taskName">Name recorded in the diagnostic logs.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private async Task ObserveBackgroundTaskStopAsync(
|
||||
Task task,
|
||||
string taskName)
|
||||
@@ -456,6 +513,7 @@ public sealed class WorkerPipeSession
|
||||
.ConfigureAwait(false);
|
||||
if (completedTask != task)
|
||||
{
|
||||
ObserveFaultWhenever(task);
|
||||
_logger?.Error(
|
||||
"WorkerPipeSessionBackgroundTaskStopTimedOut",
|
||||
new Dictionary<string, object?>
|
||||
@@ -485,6 +543,24 @@ public sealed class WorkerPipeSession
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attaches a continuation that observes <paramref name="task"/>'s exception whenever the
|
||||
/// task eventually faults, so a task nobody is awaiting any more can never reach the
|
||||
/// finalizer with an unobserved exception. Mirrors the shape
|
||||
/// <c>WorkerFrameWriter.ObserveAbandonedFault</c> uses for frames a cancelled caller stops
|
||||
/// awaiting (NEXT-04). Faulting is the only outcome that runs the continuation, and the
|
||||
/// continuation is scheduled inline, so this costs nothing on the ordinary path.
|
||||
/// </summary>
|
||||
/// <param name="task">Task that may fault after its awaiter has walked away.</param>
|
||||
private static void ObserveFaultWhenever(Task task)
|
||||
{
|
||||
_ = task.ContinueWith(
|
||||
static faultedTask => _ = faultedTask.Exception,
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
private async Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
|
||||
Reference in New Issue
Block a user