fix(sitestream): deathwatch resets live-cache liveness on aggregator termination (plan R2-07 T10)

This commit is contained in:
Joseph Doherty
2026-07-13 11:01:54 -04:00
parent 76ef97f729
commit 5b16429635
4 changed files with 97 additions and 2 deletions
@@ -60,6 +60,6 @@ public interface ISiteAlarmLiveCache
/// cache is authoritative.
/// </summary>
/// <param name="siteId">The numeric site id.</param>
/// <returns><c>true</c> once the site's aggregator has seeded and published at least once.</returns>
/// <returns><c>true</c> while a <b>living</b> aggregator has seeded and published; reverts to false if the aggregator terminates (R2 N6) — a frozen snapshot is never reported as live.</returns>
bool IsLive(int siteId);
}
@@ -239,6 +239,14 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
TimeSpan.FromSeconds(60))); // stability window — former StabilityWindow static default
entry.Actor = system.ActorOf(props, $"site-alarm-aggregator-{entry.SiteId}-{Guid.NewGuid():N}");
var aggregator = entry.Actor;
// Deathwatch (arch-review R2 N6): if the aggregator terminates for any reason,
// liveness must reset — a dead feed must never keep IsLive == true, or the page
// grafts a frozen snapshot over fresh poll data for up to 15 s.
system.ActorOf(Props.Create(() => new AggregatorTerminationWatcher(
aggregator!, () => OnAggregatorTerminated(entry.SiteId, aggregator!))));
entry.StartRetryTimer?.Dispose();
entry.StartRetryTimer = null;
_logger.LogInformation("Started live alarm aggregator for site {SiteId} ({SiteIdentifier})",
@@ -421,6 +429,57 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
}
}
// ── Deathwatch (R2 N6) ──────────────────────────────────────────────────────
/// <summary>Watches one aggregator and fires a callback exactly once on Terminated.</summary>
private sealed class AggregatorTerminationWatcher : ReceiveActor
{
public AggregatorTerminationWatcher(IActorRef watched, Action onTerminated)
{
Context.Watch(watched);
Receive<Terminated>(_ =>
{
onTerminated();
Context.Stop(Self);
});
}
}
/// <summary>
/// Terminated hook (R2 N6). A DELIBERATE stop (linger fired, entry already removed
/// from _sites) is a no-op — the ReferenceEquals guard also ignores a stale watcher
/// firing after a newer aggregator replaced the dead one. A crash-death with live
/// viewers resets liveness (IsLive → false, snapshot cleared, so the page's poll is
/// authoritative again) and arms the existing bounded start-retry so the feature
/// self-heals, mirroring StartAggregatorAsync's transient-failure path.
/// </summary>
private void OnAggregatorTerminated(int siteId, IActorRef deadActor)
{
lock (_lock)
{
if (!_sites.TryGetValue(siteId, out var entry) || !ReferenceEquals(entry.Actor, deadActor))
return;
entry.Actor = null;
entry.HasPublished = false;
entry.Current = Empty;
if (entry.Subscribers.Count > 0 && !entry.Starting)
{
entry.Starting = true;
entry.StartRetryTimer?.Dispose();
entry.StartRetryTimer = new Timer(
_ => { _ = StartAggregatorAsync(entry); },
state: null,
dueTime: _options.LiveAlarmCacheReconcileInterval,
period: Timeout.InfiniteTimeSpan);
}
}
_logger.LogWarning(
"Live alarm aggregator for site {SiteId} terminated unexpectedly; liveness reset " +
"(page falls back to polling) and a restart is armed while viewers remain", siteId);
}
// ── Nested types ────────────────────────────────────────────────────────────
/// <summary>Per-site bookkeeping. All mutable fields are guarded by the service's <c>_lock</c>.</summary>