using ZB.MOM.WW.Overview.Polling;
namespace ZB.MOM.WW.Overview.Tests;
///
/// Leader aggregation and the split-brain flag. The disagreement rule is the one output an operator
/// would be woken by, so both directions are pinned: it must fire when two members genuinely claim
/// different leaders, and it must NOT fire for the ordinary cases that superficially resemble that
/// — a node that stopped answering, a node running a pre-0.2.0 Health package.
///
public class LeaderResolverTests
{
private const string LeaderA = "akka.tcp://scadabridge@site-a-a:8082";
private const string LeaderB = "akka.tcp://scadabridge@site-a-b:8082";
[Fact]
public void Resolve_AgreeingPair_NamesTheLeaderInstance()
{
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderA),
]);
var group = Assert.Single(groups);
Assert.Equal("site-a", group.Name);
Assert.Equal("site-a-a", group.LeaderInstance);
Assert.Equal(LeaderA, group.LeaderAddress);
Assert.False(group.Disagreement);
}
[Fact]
public void Resolve_MembersClaimDifferentLeaders_FlagsDisagreement()
{
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
]);
var group = Assert.Single(groups);
Assert.True(group.Disagreement);
Assert.Equal([LeaderA, LeaderB], group.ReportedLeaders);
}
[Theory]
[InlineData(InstanceStatus.Down)]
[InlineData(InstanceStatus.Unreachable)]
public void Resolve_FailedMemberDoesNotVote_SoAFailoverIsNotMisreadAsSplitBrain(InstanceStatus failed)
{
// Mid-failover the departed node's last body still names the old leader. Counting it would
// raise a split-brain warning during the most normal event in a redundant pair's life.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: failed, leader: LeaderA),
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
]);
var group = Assert.Single(groups);
Assert.False(group.Disagreement);
Assert.Equal("site-a-b", group.LeaderInstance);
}
[Fact]
public void Resolve_DegradedMemberStillVotes()
{
// Degraded means answering with a full cluster view — its opinion on the leader is current.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: InstanceStatus.Degraded, leader: LeaderA),
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
]);
Assert.True(Assert.Single(groups).Disagreement);
}
[Fact]
public void Resolve_NoMemberPublishesLeaderData_YieldsNoLeaderAndNoWarning()
{
// The whole-fleet-on-Health-0.1.0 case: no data key anywhere. It must degrade to "leader
// unknown", never to a false split-brain warning.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084"),
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
]);
var group = Assert.Single(groups);
Assert.Null(group.LeaderAddress);
Assert.Null(group.LeaderInstance);
Assert.False(group.Disagreement);
Assert.Empty(group.ReportedLeaders);
}
[Fact]
public void Resolve_OneMemberOnOldHealthPackage_UsesTheOneThatDoesReport()
{
// A partly-bumped fleet is the expected state during rollout, and one silent member is not
// a disagreement.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
]);
var group = Assert.Single(groups);
Assert.Equal("site-a-a", group.LeaderInstance);
Assert.False(group.Disagreement);
}
[Fact]
public void Resolve_UnresolvableAddress_KeepsTheRawAddress()
{
// A leader on a host that is not in the registry (a node nobody added) still tells the
// operator something useful — showing nothing would hide the discovery.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: "akka.tcp://scadabridge@ghost:8082"),
]);
var group = Assert.Single(groups);
Assert.Null(group.LeaderInstance);
Assert.Equal("akka.tcp://scadabridge@ghost:8082", group.LeaderAddress);
}
[Fact]
public void Resolve_MatchesOnHostOnly_BecauseAkkaAndHttpPortsDiffer()
{
// The registry knows the HTTP port (8084); the leader address carries the Akka remoting
// port (8082). Matching on authority instead of host would never resolve anything.
var name = LeaderResolver.ResolveToInstanceName(
LeaderA,
[Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084")]);
Assert.Equal("site-a-a", name);
}
[Fact]
public void Resolve_GroupWithoutActiveRole_IsNotAGroupAtAll()
{
// Instances grouped only for layout have no leader; a chip there would invent a concept.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("gw-1", "gateways", "http://gw-1:5120", hasActiveRole: false),
Snapshots.Instance("gw-2", "gateways", "http://gw-2:5120", hasActiveRole: false),
]);
Assert.Empty(groups);
}
[Fact]
public void Resolve_UngroupedInstances_ProduceNoGroups()
{
Assert.Empty(LeaderResolver.Resolve([Snapshots.Instance("solo")]));
}
[Fact]
public void Resolve_SeparateGroups_AreScopedIndependently()
{
// Two site pairs, each with its own leader: per-Cluster election is the family's actual
// topology, so a leader from one group must never be attributed to another.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
Snapshots.Instance("site-b-a", "site-b", "http://site-b-a:8084", leader: "akka.tcp://scadabridge@site-b-a:8082"),
]);
Assert.Equal(2, groups.Count);
Assert.Equal("site-a-a", groups[0].LeaderInstance);
Assert.Equal("site-b-a", groups[1].LeaderInstance);
Assert.All(groups, g => Assert.False(g.Disagreement));
}
[Theory]
[InlineData("akka.tcp://scadabridge@site-a-a:8082", "site-a-a")]
[InlineData("akka.tcp://otopcua@10.100.0.35:4053", "10.100.0.35")]
[InlineData("akka.tcp://sys@host", "host")]
[InlineData("not-an-address", null)]
[InlineData("", null)]
[InlineData(null, null)]
public void HostOf_ParsesAkkaAddresses(string? address, string? expected)
{
Assert.Equal(expected, LeaderResolver.HostOf(address));
}
[Theory]
[InlineData("akka-cluster")] // ScadaBridge
[InlineData("akka")] // OtOpcUa
[InlineData("cluster-view")] // any future app
public void Leader_IsFoundWhateverTheCheckIsCalled(string checkName)
{
const string NodeALeader = "akka.tcp://otopcua@node-a:4053";
// The two apps register the SAME shared AkkaClusterHealthCheck under different names, so a
// resolver keyed on one name renders no leader chip at all for the other — silently, since
// a group with no votes is a legitimate state. That is exactly how this shipped past
// review and unit tests and was only caught on a live rig.
var groups = LeaderResolver.Resolve(
[
Snapshots.Instance("node-a", "pair", "http://node-a:8080", leader: NodeALeader, clusterCheckName: checkName),
Snapshots.Instance("node-b", "pair", "http://node-b:8080", leader: NodeALeader, clusterCheckName: checkName),
]);
var group = Assert.Single(groups);
Assert.Equal("node-a", group.LeaderInstance);
Assert.False(group.Disagreement);
}
[Fact]
public void Leader_IgnoresChecksThatPublishDataWithoutALeader()
{
// ScadaBridge's site nodes publish a localdb check whose data carries replication counters
// and no leader. Scanning entries must pick the one that actually answers the question,
// not merely the first one carrying a data object.
var body = HealthBody.Ready(
"Healthy",
("localdb", "Healthy", null),
("akka", "Healthy", "akka.tcp://otopcua@node-a:4053"));
var report = System.Text.Json.JsonSerializer.Deserialize(body)!;
var instance = Snapshots.Instance("node-a", "pair", "http://node-a:8080") with { Report = report };
Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance));
}
[Fact]
public void ActiveNode_WinsOverLeader_WhenTheyDisagree()
{
// The two are DIFFERENT NODES after any restart: leader is the lowest-addressed Up member,
// the active node is the oldest. Observed live on the rebuilt OtOpcUa rig — leader central-1,
// active central-2 — so ranking leader first would put the Active chip on the standby, which
// is the same class of error as lmxopcua#494 rendered in the UI instead of in routing.
var body = HealthBody.WithData(
"Healthy",
("akka", "Healthy", new Dictionary
{
["leader"] = "akka.tcp://otopcua@central-1:4053",
}),
("cluster-primary", "Healthy", new Dictionary
{
["activeNode"] = "akka.tcp://otopcua@central-2:4053",
}));
var report = System.Text.Json.JsonSerializer.Deserialize(body)!;
var instance = Snapshots.Instance("central-1", "MAIN", "http://central-1:8080") with { Report = report };
Assert.Equal("akka.tcp://otopcua@central-2:4053", LeaderResolver.LeaderReportedBy(instance));
}
[Fact]
public void ActiveNode_ReportedByAStandby_NamesTheActiveNode_NotItself()
{
// Every member publishes who IS active, so a standby votes for the same address the active
// node does. That is what lets the group resolve with one unanimous chip instead of a
// manufactured disagreement, and it is why the standby's payload alone is enough.
var body = HealthBody.WithData(
"Unhealthy",
("cluster-primary", "Unhealthy", new Dictionary
{
["activeNode"] = "akka.tcp://otopcua@site-a-1:4053",
["selfAddress"] = "akka.tcp://otopcua@site-a-2:4053",
}));
var report = System.Text.Json.JsonSerializer.Deserialize(body)!;
var instance = Snapshots.Instance("site-a-2", "SITE-A", "http://site-a-2:8080") with { Report = report };
Assert.Equal("akka.tcp://otopcua@site-a-1:4053", LeaderResolver.LeaderReportedBy(instance));
}
[Fact]
public void Leader_IsStillUsed_WhenNoActiveNodeIsPublished()
{
// An instance on Health < 0.3.0, or one whose active tier was not probed, publishes only
// `leader`. Showing it beats showing nothing, and it keeps the pre-0.3.0 behaviour intact.
var body = HealthBody.WithData(
"Healthy",
("akka-cluster", "Healthy", new Dictionary
{
["leader"] = "akka.tcp://scadabridge@central-a:4053",
}));
var report = System.Text.Json.JsonSerializer.Deserialize(body)!;
var instance = Snapshots.Instance("central-a", "central", "http://central-a:8080") with { Report = report };
Assert.Equal("akka.tcp://scadabridge@central-a:4053", LeaderResolver.LeaderReportedBy(instance));
}
}