fix(host): unify active-node on oldest-member semantics — purge/self-report/labels now track the singleton host

This commit is contained in:
Joseph Doherty
2026-07-08 15:48:07 -04:00
parent 6320be9954
commit da6d5ae12c
5 changed files with 70 additions and 21 deletions
@@ -13,9 +13,13 @@ public interface IClusterNodeProvider
IReadOnlyList<NodeStatus> GetClusterNodes();
/// <summary>
/// True when this node is currently the cluster leader (Primary) for the
/// provider's role scope. Used by the central report loop to decide which
/// node should generate the "central" health report.
/// True when this node is currently the oldest Up member for the provider's
/// role scope — the node the ClusterSingletonManager hosts singletons on.
/// This is the canonical product "active node" check (review 01): the site
/// event-log purge gate, the central self-report loop, and (via plan 02) the
/// S&amp;F delivery gate all consume it. It is deliberately NOT the cluster
/// leader — leadership is address-ordered and diverges from singleton
/// placement once the original first node restarts and rejoins.
/// </summary>
bool SelfIsPrimary { get; }
}
@@ -9,9 +9,12 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Health;
/// <see cref="IActiveNodeGate"/> backed by the running Akka.NET cluster.
///
/// The inbound API is "Central cluster only (active node)" — a standby central
/// node must not execute method scripts or <c>Route.To()</c> calls. This gate
/// mirrors the leadership check in <see cref="ActiveNodeHealthCheck"/> (the
/// node is the cluster leader, <see cref="MemberStatus.Up"/>), so
/// node must not execute method scripts or <c>Route.To()</c> calls. Active =
/// the oldest Up member (the singleton host), evaluated by
/// <see cref="ClusterActivityEvaluator.SelfIsOldest(Cluster, string?)"/> — NOT
/// the cluster leader, whose address-ordered definition diverges from singleton
/// placement after a restart (review 01 [High]). This is the same evaluator
/// backing <see cref="OldestNodeActiveHealthCheck"/>, so
/// <see cref="InboundApiEndpointFilter"/> can return HTTP 503 on a standby.
///
/// Registered only in the Central-role branch of <c>Program.cs</c>. The gate
@@ -42,12 +45,7 @@ public sealed class ActiveNodeGate : IActiveNodeGate
return false;
var cluster = Cluster.Get(system);
var self = cluster.SelfMember;
if (self.Status != MemberStatus.Up)
return false;
var leader = cluster.State.Leader;
return leader != null && leader == self.Address;
return ClusterActivityEvaluator.SelfIsOldest(cluster);
}
}
}
@@ -33,9 +33,7 @@ public class AkkaClusterNodeProvider : IClusterNodeProvider
var system = _akkaService.ActorSystem;
if (system == null) return false;
var cluster = Cluster.Get(system);
if (cluster.SelfMember.Status != MemberStatus.Up) return false;
var leader = cluster.State.Leader;
return leader != null && leader.Equals(cluster.SelfAddress);
return ClusterActivityEvaluator.SelfIsOldest(cluster, _siteRole);
}
}
@@ -46,8 +44,7 @@ public class AkkaClusterNodeProvider : IClusterNodeProvider
if (system == null) return [];
var cluster = Cluster.Get(system);
var selfAddress = cluster.SelfAddress;
var leader = cluster.State.Leader;
var oldest = ClusterActivityEvaluator.OldestUpMember(cluster, _siteRole);
var nodes = new List<NodeStatus>();
foreach (var member in cluster.State.Members)
@@ -57,8 +54,8 @@ public class AkkaClusterNodeProvider : IClusterNodeProvider
var hostname = member.Address.Host ?? member.Address.ToString();
var isOnline = member.Status == MemberStatus.Up;
var isLeader = member.Address.Equals(leader);
var role = isLeader ? "Primary" : "Standby";
var isPrimary = oldest != null && member.UniqueAddress.Equals(oldest.UniqueAddress);
var role = isPrimary ? "Primary" : "Standby";
nodes.Add(new NodeStatus(hostname, isOnline, role));
}
@@ -109,8 +109,8 @@ public static class SiteServiceRegistration
// The EventLogPurgeService runs on every
// site host node but consults this optional gate each tick and early-exits on
// the standby. Register it to delegate to IClusterNodeProvider.SelfIsPrimary
// (the canonical "this node is Up AND cluster leader" check) so purge runs ONLY
// on the active node — no duplicated cluster logic. Non-clustered test hosts that
// (the canonical "this node is the oldest Up member (singleton host)" check) so purge
// runs ONLY on the active node — no duplicated cluster logic. Non-clustered test hosts that
// never call SiteServiceRegistration leave it unregistered, so the purge defaults
// to always-run (the pre-fix behaviour, preserved).
services.AddSingleton<SiteEventLogActiveNodeCheck>(sp =>
@@ -1,5 +1,11 @@
using Akka.Actor;
using Akka.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
using ZB.MOM.WW.ScadaBridge.Host.Health;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
@@ -46,6 +52,50 @@ akka {{
Assert.False(ClusterActivityEvaluator.SelfIsOldest(cluster, "site-nonexistent"));
}
[Fact]
public async Task ActiveNodeGate_And_NodeProvider_AgreeWithEvaluator()
{
// Both product gates must be backed by the oldest-member evaluator.
// A single-node Up cluster: evaluator says true, so both must say true.
// (Pinning test — the leader==oldest divergence is proven in Task 8's
// two-node integration test; here we assert the wiring holds.)
var port = FreePort();
var self = $"akka.tcp://scadabridge@127.0.0.1:{port}";
var akkaService = new AkkaHostedService(
new ServiceCollection().BuildServiceProvider(),
Options.Create(new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = port }),
Options.Create(new ClusterOptions
{
SeedNodes = new List<string> { self },
SplitBrainResolverStrategy = "keep-oldest",
StableAfter = TimeSpan.FromSeconds(3),
HeartbeatInterval = TimeSpan.FromMilliseconds(500),
FailureDetectionThreshold = TimeSpan.FromSeconds(2),
MinNrOfMembers = 1,
DownIfAlone = true,
}),
Options.Create(new CommunicationOptions()),
NullLogger<AkkaHostedService>.Instance);
var system = akkaService.GetOrCreateActorSystem();
try
{
var cluster = Akka.Cluster.Cluster.Get(system);
var deadline = DateTime.UtcNow.AddSeconds(20);
while (cluster.SelfMember.Status != Akka.Cluster.MemberStatus.Up && DateTime.UtcNow < deadline)
await Task.Delay(100);
var gate = new ActiveNodeGate(akkaService);
var provider = new AkkaClusterNodeProvider(akkaService, "Central");
Assert.True(gate.IsActiveNode);
Assert.True(provider.SelfIsPrimary);
}
finally
{
await system.Terminate();
}
}
private static int FreePort()
{
var l = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);