fix(alarms): background tasks register before they start; volatile _disposed

This commit is contained in:
Joseph Doherty
2026-08-17 04:04:45 -04:00
parent cda82c074e
commit 5ae430237f
2 changed files with 221 additions and 13 deletions
@@ -169,7 +169,13 @@ public sealed class ScriptedAlarmEngine : IDisposable
private readonly SemaphoreSlim _evalGate = new(1, 1);
private Timer? _shelvingTimer;
private bool _loaded;
private bool _disposed;
// volatile: written by Dispose and read unsynchronized from every background
// path (LoadAsync's guard, OnUpstreamChange, the drain loop, ReevaluateAsync /
// ShelvingCheckAsync's post-gate re-checks, TrackBackgroundTask's registration
// check). Without it those reads may be hoisted or cached by the JIT, so a
// pushing thread can keep queueing work against a disposed engine.
private volatile bool _disposed;
// Tracks fire-and-forget background work launched by OnUpstreamChange
// (DrainDirtyAsync) and RunShelvingCheck (ShelvingCheckAsync). Dispose drains
@@ -556,7 +562,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
// Started outside the lock: DrainDirtyAsync runs synchronously up to its
// first real await, and nothing it does should happen under _dirtyLock.
if (startDrain) TrackBackgroundTask(DrainDirtyAsync(CancellationToken.None));
if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None));
}
/// <summary>
@@ -663,7 +669,9 @@ public sealed class ScriptedAlarmEngine : IDisposable
if (!restart) _drainScheduled = false;
}
if (restart) TrackBackgroundTask(Task.Run(() => DrainDirtyAsync(ct)));
// Task.Run stays INSIDE the factory: the queued-not-called handover is what
// keeps the replacement off this unwinding stack (see the Stack bound above).
if (restart) TrackBackgroundTask(() => Task.Run(() => DrainDirtyAsync(ct)));
}
}
}
@@ -986,22 +994,80 @@ public sealed class ScriptedAlarmEngine : IDisposable
{
if (_disposed) return;
var ids = _alarms.Keys.ToArray();
TrackBackgroundTask(ShelvingCheckAsync(ids, CancellationToken.None));
TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None));
}
/// <summary>
/// Register a fire-and-forget task so <see cref="Dispose"/> can await it.
/// The task removes itself from the set on completion via a continuation.
/// Start a fire-and-forget task and register it so <see cref="Dispose"/> can await it.
/// </summary>
private void TrackBackgroundTask(Task task)
/// <remarks>
/// <para><b>The factory shape is the point: registration happens BEFORE the work starts.</b>
/// Taking a <see cref="Task"/> meant the caller evaluated the argument — which starts the
/// work — and only then called in to register it. <see cref="Dispose"/> snapshots
/// <c>_inFlight</c> under <c>_inFlightLock</c> and synchronously awaits the snapshot, so a
/// Dispose landing in that start→Add window drained an empty (or short) set and returned
/// while a genuinely running <c>SaveAsync</c> was still in flight — exactly the outliving
/// store the drain exists to prevent. The window is removed, not shrunk: a placeholder is
/// registered under the lock first, and only then is the factory invoked, so any task that
/// runs at all is already visible to every subsequent snapshot.</para>
/// <para>The factory is invoked OUTSIDE <c>_inFlightLock</c> — it runs engine code
/// synchronously up to its first real await (and, via the injected clock, can re-enter
/// <see cref="OnUpstreamChange"/> and this method), none of which belongs under that lock.</para>
/// <para><b>Post-disposal the call DECLINES: the factory is never invoked and nothing is
/// registered</b> (the method returns silently — no throw, since every caller is a
/// fire-and-forget path that already treats a disposing engine as "stop"). The drain-latch
/// handover is the only caller that would otherwise leave <c>_drainScheduled</c> stuck true;
/// that is harmless, because a disposed engine drops pushes in
/// <see cref="OnUpstreamChange"/> and so never consults the latch again.</para>
/// </remarks>
/// <param name="factory">Starts the background work. Invoked at most once, outside the lock.</param>
private void TrackBackgroundTask(Func<Task> factory)
{
lock (_inFlightLock) { _inFlight.Add(task); }
// Use ContinueWith with ExecuteSynchronously so the removal runs on the
// completing thread — avoids scheduler delay between completion and
// unregistration that would otherwise let Dispose see a stale set.
task.ContinueWith(t =>
// RunContinuationsAsynchronously: the placeholder is completed from the real
// task's ExecuteSynchronously continuation, i.e. on the completing thread —
// this keeps an awaiter of the placeholder from being resumed inline there.
var placeholder = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
lock (_inFlightLock)
{
lock (_inFlightLock) { _inFlight.Remove(t); }
if (_disposed) return;
_inFlight.Add(placeholder.Task);
}
Task work;
try
{
work = factory() ?? Task.CompletedTask;
}
catch (Exception ex)
{
// A factory that throws synchronously started nothing, so the placeholder must
// not be left pending — a Dispose that already snapshotted it would block on it
// forever. Fault it (and observe the fault, so an un-snapshotted placeholder is
// not reported as an unobserved task exception), then let the caller see the throw
// it would have seen when the argument was evaluated at the call site.
lock (_inFlightLock) { _inFlight.Remove(placeholder.Task); }
placeholder.TrySetException(ex);
_ = placeholder.Task.Exception;
throw;
}
// ExecuteSynchronously so the removal runs on the completing thread — avoids
// scheduler delay between completion and unregistration that would otherwise
// let Dispose see a stale set. The placeholder mirrors the real task's outcome
// so Dispose's WhenAll still observes (and logs) a background failure.
work.ContinueWith(t =>
{
lock (_inFlightLock) { _inFlight.Remove(placeholder.Task); }
if (t.IsFaulted)
{
placeholder.TrySetException(t.Exception!.InnerExceptions);
// Observe the mirror. A Dispose that snapshotted it still sees the fault
// through WhenAll (observing does not suppress propagation); one that did
// not would otherwise leave a second unobserved task exception behind.
_ = placeholder.Task.Exception;
}
else if (t.IsCanceled) placeholder.TrySetCanceled();
else placeholder.TrySetResult();
}, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
}
@@ -1012,6 +1078,20 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// </summary>
internal void RunShelvingCheckForTest() => RunShelvingCheck();
/// <summary>
/// Test hook — launches background work through the same registration path the
/// engine's own fire-and-forget callers use, so tests can pin the
/// register-before-start ordering directly.
/// </summary>
/// <param name="factory">Starts the background work.</param>
internal void TrackBackgroundTaskForTest(Func<Task> factory) => TrackBackgroundTask(factory);
/// <summary>Test hook — the number of background tasks currently registered for the Dispose drain.</summary>
internal int InFlightCountForTest
{
get { lock (_inFlightLock) { return _inFlight.Count; } }
}
private async Task ShelvingCheckAsync(IReadOnlyList<string> alarmIds, CancellationToken ct)
{
var pending = new List<ScriptedAlarmEvent>(0);