Merge fix/c3-worker-residuals: WRK-33 observed command-task wrapper, WRK-32 pre-baseline pump + recency contract documented, WRK-34 Watchdog CommandMethod label

This commit is contained in:
Joseph Doherty
2026-08-20 09:21:42 -04:00
4 changed files with 91 additions and 9 deletions
@@ -1232,7 +1232,14 @@ public sealed class WorkerPipeSession
_activeCommandTasks.Add(commandTask);
}
_ = ObserveCommandTaskAsync(commandTask);
// WRK-33: every other discarded task in this file is wrapped with ObserveFaultWhenever
// (NEXT-04 discipline) so a fault reaching the finalizer unobserved is impossible. This
// wrapper task was the one exception — ObserveCommandTaskAsync only swallows
// OperationCanceledException, so any other exception escaping ProcessCommandAsync's own
// catch path (e.g. a future edit, or a failure past TryWriteFaultAsync's narrow filter)
// would fault the discarded task with no observer. Close the gap the same way the rest of
// the file does.
ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));
}
private async Task ObserveCommandTaskAsync(Task commandTask)
@@ -1381,11 +1388,18 @@ public sealed class WorkerPipeSession
// the StaHung fault just sent. Without this the heartbeat loop keeps
// advertising a non-faulted state that contradicts the fault.
_state = WorkerState.Faulted;
// WRK-34: CommandMethod is a method-name field, not a correlation-id field, so a
// consumer reading it (dashboard, gateway logs) shouldn't see a bare correlation id
// there. Use a stable label and carry the stuck command's correlation id (if any) in
// the diagnostic message instead.
string correlationSuffix = string.IsNullOrEmpty(snapshot.CurrentCommandCorrelationId)
? string.Empty
: $" Stuck command correlation id: {snapshot.CurrentCommandCorrelationId}.";
await TryWriteFaultAsync(
CreateFault(
WorkerFaultCategory.StaHung,
snapshot.CurrentCommandCorrelationId,
$"STA activity is stale by {staleFor}."),
"Watchdog",
$"STA activity is stale by {staleFor}.{correlationSuffix}"),
cancellationToken).ConfigureAwait(false);
}
@@ -445,7 +445,8 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
// also fire-and-forget in MXAccess, and its commit outcome only exists
// in the later OnWriteComplete callback.
MxAccessWriteCompletionCache completionCache = session.WriteCompletionCache;
ulong completionBaseline = completionCache.CurrentVersion(
ulong completionBaseline = CaptureCompletionBaseline(
completionCache,
writeCommand.ServerHandle,
writeCommand.ItemHandle);
@@ -485,7 +486,8 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
// Same pre-call baseline rule as ExecuteWriteSecured.
MxAccessWriteCompletionCache completionCache = session.WriteCompletionCache;
ulong completionBaseline = completionCache.CurrentVersion(
ulong completionBaseline = CaptureCompletionBaseline(
completionCache,
write2Command.ServerHandle,
write2Command.ItemHandle);
@@ -523,7 +525,8 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
// immediately after WriteSecured bumps the version past this snapshot,
// so a fast commit still correlates (no missed-callback window).
MxAccessWriteCompletionCache completionCache = session.WriteCompletionCache;
ulong completionBaseline = completionCache.CurrentVersion(
ulong completionBaseline = CaptureCompletionBaseline(
completionCache,
writeSecuredCommand.ServerHandle,
writeSecuredCommand.ItemHandle);
@@ -564,7 +567,8 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
// Same pre-call baseline rule as ExecuteWriteSecured.
MxAccessWriteCompletionCache completionCache = session.WriteCompletionCache;
ulong completionBaseline = completionCache.CurrentVersion(
ulong completionBaseline = CaptureCompletionBaseline(
completionCache,
writeSecured2Command.ServerHandle,
writeSecured2Command.ItemHandle);
@@ -994,7 +998,31 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
}
/// <summary>
/// Bounded pump-wait for the OnWriteComplete row matching a
/// Captures the write-completion baseline for <paramref name="serverHandle"/> /
/// <paramref name="itemHandle"/> immediately before a Write/Write2/WriteSecured/
/// WriteSecured2 COM call. Pumps once first (WRK-32) so an <c>OnWriteComplete</c>
/// already queued for this handle — most commonly the still-undelivered callback of an
/// earlier write to the same tag that this worker gave up waiting on — folds into the
/// baseline instead of being available to ride this write's reply.
/// </summary>
/// <remarks>
/// This narrows, but does not close, the correlation gap documented on
/// <see cref="AwaitWriteCompletion"/>: a completion that MXAccess queues after this pump
/// step but before the COM call returns is still indistinguishable from this write's own
/// completion, because <c>OnWriteComplete</c> carries no per-call token — only the
/// server/item handle pair.
/// </remarks>
private ulong CaptureCompletionBaseline(
MxAccessWriteCompletionCache completionCache,
int serverHandle,
int itemHandle)
{
pumpStep();
return completionCache.CurrentVersion(serverHandle, itemHandle);
}
/// <summary>
/// Bounded pump-wait for the OnWriteComplete row matching a Write/Write2/
/// WriteSecured/WriteSecured2 call, copied onto the reply when it
/// arrives in time. The executor holds the STA thread but pumps
/// Windows messages each poll (ReadBulk precedent) so the COM callback
@@ -1002,6 +1030,21 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor
/// statuses — the consumer's unconfirmed path, never a synthesized
/// failure. Protocol status/hresult stay acceptance-only either way.
/// </summary>
/// <remarks>
/// Correlation is recency-based per handle, not per call: MXAccess's
/// <c>OnWriteComplete</c> callback carries no correlation token, only the server/item
/// handle pair, so this method reports the first completion whose version exceeds
/// <paramref name="completionBaseline"/> as this write's outcome. If an earlier write to
/// the same handle times out unconfirmed and its callback is still undelivered, that
/// stale completion can dispatch during this write's pump-wait and ride this reply
/// instead — <see cref="CaptureCompletionBaseline"/>'s pre-call pump only folds in a
/// completion already queued before this write started; it cannot fold in one queued
/// during the COM call or the wait itself. The <c>Hresult</c>/<c>ProtocolStatus</c>
/// fields stay acceptance-only regardless, so the exposure is limited to the enrichment
/// <c>statuses</c> field, and it is inherent to the COM surface — a native client
/// correlating by handle has the identical ambiguity. Full closure needs an
/// MXAccess-side correlation token, which does not exist.
/// </remarks>
private void AwaitWriteCompletion(
MxCommandReply reply,
MxAccessWriteCompletionCache completionCache,