fix(communication): replicate heartbeat marks to the peer central node (closes post-failover offline blind window)
This commit is contained in:
@@ -197,6 +197,7 @@ public class CentralCommunicationActor : ReceiveActor
|
|||||||
Receive<HeartbeatMessage>(HandleHeartbeat);
|
Receive<HeartbeatMessage>(HandleHeartbeat);
|
||||||
Receive<SiteHealthReport>(HandleSiteHealthReport);
|
Receive<SiteHealthReport>(HandleSiteHealthReport);
|
||||||
Receive<SiteHealthReportReplica>(r => ProcessLocally(r.Report));
|
Receive<SiteHealthReportReplica>(r => ProcessLocally(r.Report));
|
||||||
|
Receive<SiteHeartbeatReplica>(r => MarkHeartbeatLocally(r.Heartbeat));
|
||||||
Receive<SubscribeAck>(_ => { /* DistributedPubSub subscribe confirmation */ });
|
Receive<SubscribeAck>(_ => { /* DistributedPubSub subscribe confirmation */ });
|
||||||
|
|
||||||
// Route enveloped messages to sites
|
// Route enveloped messages to sites
|
||||||
@@ -371,6 +372,31 @@ public class CentralCommunicationActor : ReceiveActor
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void HandleHeartbeat(HeartbeatMessage heartbeat)
|
private void HandleHeartbeat(HeartbeatMessage heartbeat)
|
||||||
|
{
|
||||||
|
MarkHeartbeatLocally(heartbeat);
|
||||||
|
|
||||||
|
// Fan the heartbeat out to the peer central node so BOTH aggregators mark
|
||||||
|
// it, regardless of which central node the site's ClusterClient delivered
|
||||||
|
// to. Without this, a heartbeat that only ever reaches one node leaves the
|
||||||
|
// other node's aggregator blind to that site's liveness after a failover
|
||||||
|
// (arch review 02, Low). MarkHeartbeat is idempotent (timestamp overwrite),
|
||||||
|
// so self-delivery via pub-sub is harmless.
|
||||||
|
try
|
||||||
|
{
|
||||||
|
DistributedPubSub.Get(Context.System).Mediator.Tell(
|
||||||
|
new Publish(HealthReportTopic, new SiteHeartbeatReplica(heartbeat)));
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// No-op in non-clustered hosts (TestKit).
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Marks a site heartbeat on the local aggregator without re-broadcasting.
|
||||||
|
/// Used for both site-originated heartbeats and peer-replicated ones.
|
||||||
|
/// </summary>
|
||||||
|
private void MarkHeartbeatLocally(HeartbeatMessage heartbeat)
|
||||||
{
|
{
|
||||||
var aggregator = _serviceProvider.GetService<ICentralHealthAggregator>();
|
var aggregator = _serviceProvider.GetService<ICentralHealthAggregator>();
|
||||||
aggregator?.MarkHeartbeat(heartbeat.SiteId, heartbeat.Timestamp);
|
aggregator?.MarkHeartbeat(heartbeat.SiteId, heartbeat.Timestamp);
|
||||||
@@ -665,6 +691,15 @@ internal record SiteAddressCacheLoaded(
|
|||||||
IReadOnlyDictionary<string, IReadOnlyList<string>> SiteContacts,
|
IReadOnlyDictionary<string, IReadOnlyList<string>> SiteContacts,
|
||||||
IReadOnlyCollection<string> KnownSiteIds);
|
IReadOnlyCollection<string> KnownSiteIds);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Peer-replication envelope for a site heartbeat, fanned out over the same
|
||||||
|
/// DistributedPubSub topic as <see cref="SiteHealthReportReplica"/> so both
|
||||||
|
/// central aggregators mark heartbeats regardless of which node the site's
|
||||||
|
/// ClusterClient delivered to. Only ever travels central↔central (same assembly
|
||||||
|
/// version, rolled together), so the default reflective serializer is safe.
|
||||||
|
/// </summary>
|
||||||
|
public sealed record SiteHeartbeatReplica(HeartbeatMessage Heartbeat);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Notification sent to debug view subscribers when the stream is terminated
|
/// Notification sent to debug view subscribers when the stream is terminated
|
||||||
/// due to site disconnection.
|
/// due to site disconnection.
|
||||||
|
|||||||
@@ -148,6 +148,33 @@ public class CentralCommunicationActorTests : TestKit
|
|||||||
AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site1", timestamp));
|
AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site1", timestamp));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void HeartbeatReplica_MarksLocalAggregator_WithoutRebroadcast()
|
||||||
|
{
|
||||||
|
// Task 16 (arch review 02): a heartbeat replicated from the peer central
|
||||||
|
// node marks the local aggregator (so post-failover both nodes know the
|
||||||
|
// site's liveness) but is NOT re-published (it arrives via pub-sub already).
|
||||||
|
var mockRepo = Substitute.For<ISiteRepository>();
|
||||||
|
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
||||||
|
.Returns(new List<Site>());
|
||||||
|
|
||||||
|
var aggregator = Substitute.For<ICentralHealthAggregator>();
|
||||||
|
|
||||||
|
var services = new ServiceCollection();
|
||||||
|
services.AddScoped(_ => mockRepo);
|
||||||
|
services.AddSingleton(aggregator);
|
||||||
|
var sp = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
var siteClientFactory = Substitute.For<ISiteClientFactory>();
|
||||||
|
var centralActor = Sys.ActorOf(
|
||||||
|
Props.Create(() => new CentralCommunicationActor(sp, siteClientFactory)));
|
||||||
|
|
||||||
|
var ts = DateTimeOffset.UtcNow;
|
||||||
|
centralActor.Tell(new SiteHeartbeatReplica(new HeartbeatMessage("site-1", "host-a", true, ts)));
|
||||||
|
|
||||||
|
AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site-1", ts));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void RefreshSiteAddresses_UpdatesCache()
|
public void RefreshSiteAddresses_UpdatesCache()
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user