fix: release the alarm drain latch on escape; cancel before disposing the page poll timers

This commit is contained in:
Joseph Doherty
2026-08-16 05:40:27 -04:00
parent b9e9df4081
commit 445bb855e2
5 changed files with 306 additions and 22 deletions
@@ -569,32 +569,77 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// reads the set under, so the last drainer out and the next push in can never
/// both decide "someone else will handle it".
/// </summary>
/// <remarks>
/// <b>INVARIANT: <c>_drainScheduled</c> is released on EVERY exit path, the
/// exceptional one included.</b> The flag is the only thing that lets
/// <see cref="OnUpstreamChange"/> start a drainer, so an escape that leaves it
/// stuck <c>true</c> permanently and silently stops evaluation for every scripted
/// alarm in the process: pushes keep filling <c>_valueCache</c> and the dirty set,
/// no drainer is ever started again, no predicate ever runs again, and nothing is
/// logged. <see cref="ReevaluateAsync"/> catches everything today, so the throw
/// surface is currently empty — the <c>finally</c> below is what keeps the
/// invariant true when it stops being (an audit-mode Serilog sink rethrowing out
/// of that catch block's own <c>Error()</c> call is enough).
/// </remarks>
private async Task DrainDirtyAsync(CancellationToken ct)
{
while (true)
// Set inside _dirtyLock, together with the flag clear, so the finally can tell
// "I already released" from "I am unwinding" without ever double-releasing —
// a second clear could stomp a _drainScheduled = true a racing push had just
// set for the drainer it started, leaving two drainers running.
var released = false;
try
{
string[] batch;
lock (_dirtyLock)
while (true)
{
// Bail promptly on a disposing engine, but ALWAYS release
// _drainScheduled on the way out — leaving it stuck true would make a
// hypothetical post-dispose push believe a drainer is coming. (Such
// pushes are already dropped in OnUpstreamChange; this keeps the flag
// honest regardless.)
if (_disposed || _dirtyAlarmIds.Count == 0)
string[] batch;
lock (_dirtyLock)
{
// Bail promptly on a disposing engine, but ALWAYS release
// _drainScheduled on the way out — leaving it stuck true would make a
// hypothetical post-dispose push believe a drainer is coming. (Such
// pushes are already dropped in OnUpstreamChange; this keeps the flag
// honest regardless.)
if (_disposed || _dirtyAlarmIds.Count == 0)
{
_dirtyAlarmIds.Clear();
_drainScheduled = false;
released = true;
return;
}
batch = [.. _dirtyAlarmIds];
_dirtyAlarmIds.Clear();
_drainScheduled = false;
return;
}
batch = [.. _dirtyAlarmIds];
_dirtyAlarmIds.Clear();
// Reused as-is: the gate / evaluate / persist / emit-outside-the-gate shape
// is already correct for a batch. Only the spawn-per-push is gone.
await ReevaluateAsync(batch, ct).ConfigureAwait(false);
}
}
finally
{
// Exceptional exit only (see `released`). The batch the faulted pass was
// carrying dies with it — ReevaluateAsync's own catch is what normally makes
// that a logged non-event — but ids that queued BEHIND it are still in the
// set and would otherwise sit there until some later push happened to arrive,
// so hand the latch straight to a replacement drainer instead of releasing
// it. Handing over under the same lock keeps "exactly one drainer" true with
// no window a push could start a second one in. This cannot spin: every pass
// swaps the set out and clears it before evaluating, so a replacement finds
// work only if fresh pushes arrived, and a permanently-throwing evaluation
// costs one pass per push — the same rate as a healthy one.
if (!released)
{
bool restart;
lock (_dirtyLock)
{
restart = !_disposed && _dirtyAlarmIds.Count > 0;
if (!restart) _drainScheduled = false;
}
// Reused as-is: the gate / evaluate / persist / emit-outside-the-gate shape
// is already correct for a batch. Only the spawn-per-push is gone.
await ReevaluateAsync(batch, ct).ConfigureAwait(false);
if (restart) TrackBackgroundTask(DrainDirtyAsync(ct));
}
}
}