fix(health): evict deleted sites from the aggregator on the periodic site refresh

This commit is contained in:
Joseph Doherty
2026-07-08 16:27:38 -04:00
parent 87c7255912
commit d962c77bb7
7 changed files with 99 additions and 3 deletions
@@ -499,7 +499,12 @@ public class CentralCommunicationActor : ReceiveActor
var frozen = contacts.ToDictionary(
kvp => kvp.Key,
kvp => (IReadOnlyList<string>)kvp.Value.AsReadOnly());
return new SiteAddressCacheLoaded(frozen);
// Carry the full configured-site identifier set (not just the
// address-bearing subset in `frozen`) so the aggregator prunes only
// genuinely-deleted sites and never an addressless-but-configured one.
var knownSiteIds = sites.Select(s => s.SiteIdentifier).ToList();
return new SiteAddressCacheLoaded(frozen, knownSiteIds);
}).PipeTo(self);
}
@@ -570,6 +575,12 @@ public class CentralCommunicationActor : ReceiveActor
}
_log.Info("Site ClusterClient cache refreshed with {0} site(s)", _siteClients.Count);
// Self-healing eviction: a site deleted from configuration would otherwise
// linger in the aggregator as a permanently-offline tile (and live KPI
// sample source) forever. Prune on every refresh so it disappears within
// one refresh interval without needing a dedicated deletion event.
_serviceProvider.GetService<ICentralHealthAggregator>()?.PruneUnknownSites(msg.KnownSiteIds);
}
// TrackMessageForCleanup removed — the dicts it fed
@@ -650,7 +661,9 @@ public record RefreshSiteAddresses;
/// discipline. The producer wraps the constructed buckets with
/// <c>List&lt;T&gt;.AsReadOnly()</c> before piping to Self.
/// </summary>
internal record SiteAddressCacheLoaded(IReadOnlyDictionary<string, IReadOnlyList<string>> SiteContacts);
internal record SiteAddressCacheLoaded(
IReadOnlyDictionary<string, IReadOnlyList<string>> SiteContacts,
IReadOnlyCollection<string> KnownSiteIds);
/// <summary>
/// Notification sent to debug view subscribers when the stream is terminated
@@ -196,6 +196,20 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
return state;
}
/// <inheritdoc />
public void PruneUnknownSites(IReadOnlyCollection<string> knownSiteIds)
{
var known = new HashSet<string>(knownSiteIds, StringComparer.Ordinal);
foreach (var siteId in _siteStates.Keys)
{
if (siteId == CentralHealthReportLoop.CentralSiteId || known.Contains(siteId))
continue;
if (_siteStates.TryRemove(siteId, out _))
_logger.LogInformation(
"Site {SiteId} evicted from health aggregator (no longer configured)", siteId);
}
}
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
@@ -39,4 +39,16 @@ public interface ICentralHealthAggregator
/// <param name="siteId">The string identifier of the site to look up.</param>
/// <returns>The <see cref="SiteHealthState"/> for the site, or null if not yet seen.</returns>
SiteHealthState? GetSiteState(string siteId);
/// <summary>
/// Removes tracked sites not in <paramref name="knownSiteIds"/> (i.e. deleted
/// from configuration). The synthetic central id is always retained.
/// Self-healing: called from the CentralCommunicationActor's periodic site
/// refresh, so a deleted site disappears within one refresh interval without
/// needing a dedicated deletion event. Without this the in-memory state only
/// ever grew, leaving a deleted site as a permanently-offline dashboard tile
/// (and a live KPI sample source) forever.
/// </summary>
/// <param name="knownSiteIds">The identifiers of all currently-configured sites.</param>
void PruneUnknownSites(IReadOnlyCollection<string> knownSiteIds);
}