fix(communication): replicate heartbeat marks to the peer central node (closes post-failover offline blind window)

This commit is contained in:
Joseph Doherty
2026-07-08 20:11:54 -04:00
parent 59d9cba621
commit 66cb0febad
2 changed files with 62 additions and 0 deletions
@@ -197,6 +197,7 @@ public class CentralCommunicationActor : ReceiveActor
Receive<HeartbeatMessage>(HandleHeartbeat);
Receive<SiteHealthReport>(HandleSiteHealthReport);
Receive<SiteHealthReportReplica>(r => ProcessLocally(r.Report));
Receive<SiteHeartbeatReplica>(r => MarkHeartbeatLocally(r.Heartbeat));
Receive<SubscribeAck>(_ => { /* DistributedPubSub subscribe confirmation */ });
// Route enveloped messages to sites
@@ -371,6 +372,31 @@ public class CentralCommunicationActor : ReceiveActor
}
private void HandleHeartbeat(HeartbeatMessage heartbeat)
{
MarkHeartbeatLocally(heartbeat);
// Fan the heartbeat out to the peer central node so BOTH aggregators mark
// it, regardless of which central node the site's ClusterClient delivered
// to. Without this, a heartbeat that only ever reaches one node leaves the
// other node's aggregator blind to that site's liveness after a failover
// (arch review 02, Low). MarkHeartbeat is idempotent (timestamp overwrite),
// so self-delivery via pub-sub is harmless.
try
{
DistributedPubSub.Get(Context.System).Mediator.Tell(
new Publish(HealthReportTopic, new SiteHeartbeatReplica(heartbeat)));
}
catch
{
// No-op in non-clustered hosts (TestKit).
}
}
/// <summary>
/// Marks a site heartbeat on the local aggregator without re-broadcasting.
/// Used for both site-originated heartbeats and peer-replicated ones.
/// </summary>
private void MarkHeartbeatLocally(HeartbeatMessage heartbeat)
{
var aggregator = _serviceProvider.GetService<ICentralHealthAggregator>();
aggregator?.MarkHeartbeat(heartbeat.SiteId, heartbeat.Timestamp);
@@ -665,6 +691,15 @@ internal record SiteAddressCacheLoaded(
IReadOnlyDictionary<string, IReadOnlyList<string>> SiteContacts,
IReadOnlyCollection<string> KnownSiteIds);
/// <summary>
/// Peer-replication envelope for a site heartbeat, fanned out over the same
/// DistributedPubSub topic as <see cref="SiteHealthReportReplica"/> so both
/// central aggregators mark heartbeats regardless of which node the site's
/// ClusterClient delivered to. Only ever travels central↔central (same assembly
/// version, rolled together), so the default reflective serializer is safe.
/// </summary>
public sealed record SiteHeartbeatReplica(HeartbeatMessage Heartbeat);
/// <summary>
/// Notification sent to debug view subscribers when the stream is terminated
/// due to site disconnection.