using ZB.MOM.WW.Overview.Polling; namespace ZB.MOM.WW.Overview.Tests; /// /// Leader aggregation and the split-brain flag. The disagreement rule is the one output an operator /// would be woken by, so both directions are pinned: it must fire when two members genuinely claim /// different leaders, and it must NOT fire for the ordinary cases that superficially resemble that /// — a node that stopped answering, a node running a pre-0.2.0 Health package. /// public class LeaderResolverTests { private const string LeaderA = "akka.tcp://scadabridge@site-a-a:8082"; private const string LeaderB = "akka.tcp://scadabridge@site-a-b:8082"; [Fact] public void Resolve_AgreeingPair_NamesTheLeaderInstance() { var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA), Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderA), ]); var group = Assert.Single(groups); Assert.Equal("site-a", group.Name); Assert.Equal("site-a-a", group.LeaderInstance); Assert.Equal(LeaderA, group.LeaderAddress); Assert.False(group.Disagreement); } [Fact] public void Resolve_MembersClaimDifferentLeaders_FlagsDisagreement() { var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA), Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB), ]); var group = Assert.Single(groups); Assert.True(group.Disagreement); Assert.Equal([LeaderA, LeaderB], group.ReportedLeaders); } [Theory] [InlineData(InstanceStatus.Down)] [InlineData(InstanceStatus.Unreachable)] public void Resolve_FailedMemberDoesNotVote_SoAFailoverIsNotMisreadAsSplitBrain(InstanceStatus failed) { // Mid-failover the departed node's last body still names the old leader. Counting it would // raise a split-brain warning during the most normal event in a redundant pair's life. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: failed, leader: LeaderA), Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB), ]); var group = Assert.Single(groups); Assert.False(group.Disagreement); Assert.Equal("site-a-b", group.LeaderInstance); } [Fact] public void Resolve_DegradedMemberStillVotes() { // Degraded means answering with a full cluster view — its opinion on the leader is current. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: InstanceStatus.Degraded, leader: LeaderA), Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB), ]); Assert.True(Assert.Single(groups).Disagreement); } [Fact] public void Resolve_NoMemberPublishesLeaderData_YieldsNoLeaderAndNoWarning() { // The whole-fleet-on-Health-0.1.0 case: no data key anywhere. It must degrade to "leader // unknown", never to a false split-brain warning. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084"), Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"), ]); var group = Assert.Single(groups); Assert.Null(group.LeaderAddress); Assert.Null(group.LeaderInstance); Assert.False(group.Disagreement); Assert.Empty(group.ReportedLeaders); } [Fact] public void Resolve_OneMemberOnOldHealthPackage_UsesTheOneThatDoesReport() { // A partly-bumped fleet is the expected state during rollout, and one silent member is not // a disagreement. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA), Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"), ]); var group = Assert.Single(groups); Assert.Equal("site-a-a", group.LeaderInstance); Assert.False(group.Disagreement); } [Fact] public void Resolve_UnresolvableAddress_KeepsTheRawAddress() { // A leader on a host that is not in the registry (a node nobody added) still tells the // operator something useful — showing nothing would hide the discovery. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: "akka.tcp://scadabridge@ghost:8082"), ]); var group = Assert.Single(groups); Assert.Null(group.LeaderInstance); Assert.Equal("akka.tcp://scadabridge@ghost:8082", group.LeaderAddress); } [Fact] public void Resolve_MatchesOnHostOnly_BecauseAkkaAndHttpPortsDiffer() { // The registry knows the HTTP port (8084); the leader address carries the Akka remoting // port (8082). Matching on authority instead of host would never resolve anything. var name = LeaderResolver.ResolveToInstanceName( LeaderA, [Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084")]); Assert.Equal("site-a-a", name); } [Fact] public void Resolve_GroupWithoutActiveRole_IsNotAGroupAtAll() { // Instances grouped only for layout have no leader; a chip there would invent a concept. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("gw-1", "gateways", "http://gw-1:5120", hasActiveRole: false), Snapshots.Instance("gw-2", "gateways", "http://gw-2:5120", hasActiveRole: false), ]); Assert.Empty(groups); } [Fact] public void Resolve_UngroupedInstances_ProduceNoGroups() { Assert.Empty(LeaderResolver.Resolve([Snapshots.Instance("solo")])); } [Fact] public void Resolve_SeparateGroups_AreScopedIndependently() { // Two site pairs, each with its own leader: per-Cluster election is the family's actual // topology, so a leader from one group must never be attributed to another. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA), Snapshots.Instance("site-b-a", "site-b", "http://site-b-a:8084", leader: "akka.tcp://scadabridge@site-b-a:8082"), ]); Assert.Equal(2, groups.Count); Assert.Equal("site-a-a", groups[0].LeaderInstance); Assert.Equal("site-b-a", groups[1].LeaderInstance); Assert.All(groups, g => Assert.False(g.Disagreement)); } [Theory] [InlineData("akka.tcp://scadabridge@site-a-a:8082", "site-a-a")] [InlineData("akka.tcp://otopcua@10.100.0.35:4053", "10.100.0.35")] [InlineData("akka.tcp://sys@host", "host")] [InlineData("not-an-address", null)] [InlineData("", null)] [InlineData(null, null)] public void HostOf_ParsesAkkaAddresses(string? address, string? expected) { Assert.Equal(expected, LeaderResolver.HostOf(address)); } [Theory] [InlineData("akka-cluster")] // ScadaBridge [InlineData("akka")] // OtOpcUa [InlineData("cluster-view")] // any future app public void Leader_IsFoundWhateverTheCheckIsCalled(string checkName) { const string NodeALeader = "akka.tcp://otopcua@node-a:4053"; // The two apps register the SAME shared AkkaClusterHealthCheck under different names, so a // resolver keyed on one name renders no leader chip at all for the other — silently, since // a group with no votes is a legitimate state. That is exactly how this shipped past // review and unit tests and was only caught on a live rig. var groups = LeaderResolver.Resolve( [ Snapshots.Instance("node-a", "pair", "http://node-a:8080", leader: NodeALeader, clusterCheckName: checkName), Snapshots.Instance("node-b", "pair", "http://node-b:8080", leader: NodeALeader, clusterCheckName: checkName), ]); var group = Assert.Single(groups); Assert.Equal("node-a", group.LeaderInstance); Assert.False(group.Disagreement); } [Fact] public void Leader_IgnoresChecksThatPublishDataWithoutALeader() { // ScadaBridge's site nodes publish a localdb check whose data carries replication counters // and no leader. Scanning entries must pick the one that actually answers the question, // not merely the first one carrying a data object. var body = HealthBody.Ready( "Healthy", ("localdb", "Healthy", null), ("akka", "Healthy", "akka.tcp://otopcua@node-a:4053")); var report = System.Text.Json.JsonSerializer.Deserialize(body)!; var instance = Snapshots.Instance("node-a", "pair", "http://node-a:8080") with { Report = report }; Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance)); } [Fact] public void ActiveNode_WinsOverLeader_WhenTheyDisagree() { // The two are DIFFERENT NODES after any restart: leader is the lowest-addressed Up member, // the active node is the oldest. Observed live on the rebuilt OtOpcUa rig — leader central-1, // active central-2 — so ranking leader first would put the Active chip on the standby, which // is the same class of error as lmxopcua#494 rendered in the UI instead of in routing. var body = HealthBody.WithData( "Healthy", ("akka", "Healthy", new Dictionary { ["leader"] = "akka.tcp://otopcua@central-1:4053", }), ("cluster-primary", "Healthy", new Dictionary { ["activeNode"] = "akka.tcp://otopcua@central-2:4053", })); var report = System.Text.Json.JsonSerializer.Deserialize(body)!; var instance = Snapshots.Instance("central-1", "MAIN", "http://central-1:8080") with { Report = report }; Assert.Equal("akka.tcp://otopcua@central-2:4053", LeaderResolver.LeaderReportedBy(instance)); } [Fact] public void ActiveNode_ReportedByAStandby_NamesTheActiveNode_NotItself() { // Every member publishes who IS active, so a standby votes for the same address the active // node does. That is what lets the group resolve with one unanimous chip instead of a // manufactured disagreement, and it is why the standby's payload alone is enough. var body = HealthBody.WithData( "Unhealthy", ("cluster-primary", "Unhealthy", new Dictionary { ["activeNode"] = "akka.tcp://otopcua@site-a-1:4053", ["selfAddress"] = "akka.tcp://otopcua@site-a-2:4053", })); var report = System.Text.Json.JsonSerializer.Deserialize(body)!; var instance = Snapshots.Instance("site-a-2", "SITE-A", "http://site-a-2:8080") with { Report = report }; Assert.Equal("akka.tcp://otopcua@site-a-1:4053", LeaderResolver.LeaderReportedBy(instance)); } [Fact] public void Leader_IsStillUsed_WhenNoActiveNodeIsPublished() { // An instance on Health < 0.3.0, or one whose active tier was not probed, publishes only // `leader`. Showing it beats showing nothing, and it keeps the pre-0.3.0 behaviour intact. var body = HealthBody.WithData( "Healthy", ("akka-cluster", "Healthy", new Dictionary { ["leader"] = "akka.tcp://scadabridge@central-a:4053", })); var report = System.Text.Json.JsonSerializer.Deserialize(body)!; var instance = Snapshots.Instance("central-a", "central", "http://central-a:8080") with { Report = report }; Assert.Equal("akka.tcp://scadabridge@central-a:4053", LeaderResolver.LeaderReportedBy(instance)); } }