scadaproj/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/LeaderResolverTests.cs

using ZB.MOM.WW.Overview.Polling;

namespace ZB.MOM.WW.Overview.Tests;

/// <summary>
/// Leader aggregation and the split-brain flag. The disagreement rule is the one output an operator
/// would be woken by, so both directions are pinned: it must fire when two members genuinely claim
/// different leaders, and it must NOT fire for the ordinary cases that superficially resemble that
/// — a node that stopped answering, a node running a pre-0.2.0 Health package.
/// </summary>
public class LeaderResolverTests
{
    private const string LeaderA = "akka.tcp://scadabridge@site-a-a:8082";
    private const string LeaderB = "akka.tcp://scadabridge@site-a-b:8082";

    [Fact]
    public void Resolve_AgreeingPair_NamesTheLeaderInstance()
    {
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
            Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderA),
        ]);

        var group = Assert.Single(groups);
        Assert.Equal("site-a", group.Name);
        Assert.Equal("site-a-a", group.LeaderInstance);
        Assert.Equal(LeaderA, group.LeaderAddress);
        Assert.False(group.Disagreement);
    }

    [Fact]
    public void Resolve_MembersClaimDifferentLeaders_FlagsDisagreement()
    {
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
            Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
        ]);

        var group = Assert.Single(groups);
        Assert.True(group.Disagreement);
        Assert.Equal([LeaderA, LeaderB], group.ReportedLeaders);
    }

    [Theory]
    [InlineData(InstanceStatus.Down)]
    [InlineData(InstanceStatus.Unreachable)]
    public void Resolve_FailedMemberDoesNotVote_SoAFailoverIsNotMisreadAsSplitBrain(InstanceStatus failed)
    {
        // Mid-failover the departed node's last body still names the old leader. Counting it would
        // raise a split-brain warning during the most normal event in a redundant pair's life.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: failed, leader: LeaderA),
            Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
        ]);

        var group = Assert.Single(groups);
        Assert.False(group.Disagreement);
        Assert.Equal("site-a-b", group.LeaderInstance);
    }

    [Fact]
    public void Resolve_DegradedMemberStillVotes()
    {
        // Degraded means answering with a full cluster view — its opinion on the leader is current.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", status: InstanceStatus.Degraded, leader: LeaderA),
            Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084", leader: LeaderB),
        ]);

        Assert.True(Assert.Single(groups).Disagreement);
    }

    [Fact]
    public void Resolve_NoMemberPublishesLeaderData_YieldsNoLeaderAndNoWarning()
    {
        // The whole-fleet-on-Health-0.1.0 case: no data key anywhere. It must degrade to "leader
        // unknown", never to a false split-brain warning.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084"),
            Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
        ]);

        var group = Assert.Single(groups);
        Assert.Null(group.LeaderAddress);
        Assert.Null(group.LeaderInstance);
        Assert.False(group.Disagreement);
        Assert.Empty(group.ReportedLeaders);
    }

    [Fact]
    public void Resolve_OneMemberOnOldHealthPackage_UsesTheOneThatDoesReport()
    {
        // A partly-bumped fleet is the expected state during rollout, and one silent member is not
        // a disagreement.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
            Snapshots.Instance("site-a-b", "site-a", "http://site-a-b:8084"),
        ]);

        var group = Assert.Single(groups);
        Assert.Equal("site-a-a", group.LeaderInstance);
        Assert.False(group.Disagreement);
    }

    [Fact]
    public void Resolve_UnresolvableAddress_KeepsTheRawAddress()
    {
        // A leader on a host that is not in the registry (a node nobody added) still tells the
        // operator something useful — showing nothing would hide the discovery.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: "akka.tcp://scadabridge@ghost:8082"),
        ]);

        var group = Assert.Single(groups);
        Assert.Null(group.LeaderInstance);
        Assert.Equal("akka.tcp://scadabridge@ghost:8082", group.LeaderAddress);
    }

    [Fact]
    public void Resolve_MatchesOnHostOnly_BecauseAkkaAndHttpPortsDiffer()
    {
        // The registry knows the HTTP port (8084); the leader address carries the Akka remoting
        // port (8082). Matching on authority instead of host would never resolve anything.
        var name = LeaderResolver.ResolveToInstanceName(
            LeaderA,
            [Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084")]);

        Assert.Equal("site-a-a", name);
    }

    [Fact]
    public void Resolve_GroupWithoutActiveRole_IsNotAGroupAtAll()
    {
        // Instances grouped only for layout have no leader; a chip there would invent a concept.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("gw-1", "gateways", "http://gw-1:5120", hasActiveRole: false),
            Snapshots.Instance("gw-2", "gateways", "http://gw-2:5120", hasActiveRole: false),
        ]);

        Assert.Empty(groups);
    }

    [Fact]
    public void Resolve_UngroupedInstances_ProduceNoGroups()
    {
        Assert.Empty(LeaderResolver.Resolve([Snapshots.Instance("solo")]));
    }

    [Fact]
    public void Resolve_SeparateGroups_AreScopedIndependently()
    {
        // Two site pairs, each with its own leader: per-Cluster election is the family's actual
        // topology, so a leader from one group must never be attributed to another.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("site-a-a", "site-a", "http://site-a-a:8084", leader: LeaderA),
            Snapshots.Instance("site-b-a", "site-b", "http://site-b-a:8084", leader: "akka.tcp://scadabridge@site-b-a:8082"),
        ]);

        Assert.Equal(2, groups.Count);
        Assert.Equal("site-a-a", groups[0].LeaderInstance);
        Assert.Equal("site-b-a", groups[1].LeaderInstance);
        Assert.All(groups, g => Assert.False(g.Disagreement));
    }

    [Fact]
    public void Resolve_CoLocatedGroups_ResolveTheLeaderWithinTheirOwnGroup()
    {
        // Overview-002 (scadaproj#43): the QA layout — ScadaBridge central-a and site-a both on
        // vq03, central-b and site-b on vq04, all in ONE application. Resolving against every
        // instance named the site group's leader "central-a" (a node in another Akka cluster).
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("central-a", "central", "http://vq03:9000", leader: "akka.tcp://scadabridge@vq03:8081"),
            Snapshots.Instance("central-b", "central", "http://vq04:9000", leader: "akka.tcp://scadabridge@vq03:8081"),
            Snapshots.Instance("site-a", "site", "http://vq03:8084", leader: "akka.tcp://scadabridge@vq03:8082"),
            Snapshots.Instance("site-b", "site", "http://vq04:8084", leader: "akka.tcp://scadabridge@vq03:8082"),
        ]);

        Assert.Equal("central-a", groups.Single(g => g.Name == "central").LeaderInstance);
        Assert.Equal("site-a", groups.Single(g => g.Name == "site").LeaderInstance);
    }

    [Fact]
    public void Resolve_LeaderHostOnlyInAnotherGroup_IsNotAttributedAcrossGroups()
    {
        // A leader that matches no member of its own group keeps its raw address rather than
        // borrowing a name from a different cluster that happens to share the host.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("central-a", "central", "http://shared-host:9000", leader: "akka.tcp://scadabridge@shared-host:8081"),
            Snapshots.Instance("site-a", "site", "http://127.0.0.1:8084", leader: "akka.tcp://scadabridge@shared-host:8082"),
        ]);

        var site = groups.Single(g => g.Name == "site");
        Assert.Null(site.LeaderInstance);
        Assert.Equal("akka.tcp://scadabridge@shared-host:8082", site.LeaderAddress);
    }

    [Theory]
    [InlineData("akka.tcp://scadabridge@site-a-a:8082", "site-a-a")]
    [InlineData("akka.tcp://otopcua@10.100.0.35:4053", "10.100.0.35")]
    [InlineData("akka.tcp://sys@host", "host")]
    [InlineData("not-an-address", null)]
    [InlineData("", null)]
    [InlineData(null, null)]
    public void HostOf_ParsesAkkaAddresses(string? address, string? expected)
    {
        Assert.Equal(expected, LeaderResolver.HostOf(address));
    }

    [Theory]
    [InlineData("akka-cluster")]  // ScadaBridge
    [InlineData("akka")]          // OtOpcUa
    [InlineData("cluster-view")]  // any future app
    public void Leader_IsFoundWhateverTheCheckIsCalled(string checkName)
    {
        const string NodeALeader = "akka.tcp://otopcua@node-a:4053";

        // The two apps register the SAME shared AkkaClusterHealthCheck under different names, so a
        // resolver keyed on one name renders no leader chip at all for the other — silently, since
        // a group with no votes is a legitimate state. That is exactly how this shipped past
        // review and unit tests and was only caught on a live rig.
        var groups = LeaderResolver.Resolve(
        [
            Snapshots.Instance("node-a", "pair", "http://node-a:8080", leader: NodeALeader, clusterCheckName: checkName),
            Snapshots.Instance("node-b", "pair", "http://node-b:8080", leader: NodeALeader, clusterCheckName: checkName),
        ]);

        var group = Assert.Single(groups);
        Assert.Equal("node-a", group.LeaderInstance);
        Assert.False(group.Disagreement);
    }

    [Fact]
    public void Leader_IgnoresChecksThatPublishDataWithoutALeader()
    {
        // ScadaBridge's site nodes publish a localdb check whose data carries replication counters
        // and no leader. Scanning entries must pick the one that actually answers the question,
        // not merely the first one carrying a data object.
        var body = HealthBody.Ready(
            "Healthy",
            ("localdb", "Healthy", null),
            ("akka", "Healthy", "akka.tcp://otopcua@node-a:4053"));
        var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
        var instance = Snapshots.Instance("node-a", "pair", "http://node-a:8080") with { Report = report };

        Assert.Equal("akka.tcp://otopcua@node-a:4053", LeaderResolver.LeaderReportedBy(instance));
    }

    [Fact]
    public void ActiveNode_WinsOverLeader_WhenTheyDisagree()
    {
        // The two are DIFFERENT NODES after any restart: leader is the lowest-addressed Up member,
        // the active node is the oldest. Observed live on the rebuilt OtOpcUa rig — leader central-1,
        // active central-2 — so ranking leader first would put the Active chip on the standby, which
        // is the same class of error as lmxopcua#494 rendered in the UI instead of in routing.
        var body = HealthBody.WithData(
            "Healthy",
            ("akka", "Healthy", new Dictionary<string, object>
            {
                ["leader"] = "akka.tcp://otopcua@central-1:4053",
            }),
            ("cluster-primary", "Healthy", new Dictionary<string, object>
            {
                ["activeNode"] = "akka.tcp://otopcua@central-2:4053",
            }));
        var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
        var instance = Snapshots.Instance("central-1", "MAIN", "http://central-1:8080") with { Report = report };

        Assert.Equal("akka.tcp://otopcua@central-2:4053", LeaderResolver.LeaderReportedBy(instance));
    }

    [Fact]
    public void ActiveNode_ReportedByAStandby_NamesTheActiveNode_NotItself()
    {
        // Every member publishes who IS active, so a standby votes for the same address the active
        // node does. That is what lets the group resolve with one unanimous chip instead of a
        // manufactured disagreement, and it is why the standby's payload alone is enough.
        var body = HealthBody.WithData(
            "Unhealthy",
            ("cluster-primary", "Unhealthy", new Dictionary<string, object>
            {
                ["activeNode"] = "akka.tcp://otopcua@site-a-1:4053",
                ["selfAddress"] = "akka.tcp://otopcua@site-a-2:4053",
            }));
        var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
        var instance = Snapshots.Instance("site-a-2", "SITE-A", "http://site-a-2:8080") with { Report = report };

        Assert.Equal("akka.tcp://otopcua@site-a-1:4053", LeaderResolver.LeaderReportedBy(instance));
    }

    [Fact]
    public void Leader_IsStillUsed_WhenNoActiveNodeIsPublished()
    {
        // An instance on Health < 0.3.0, or one whose active tier was not probed, publishes only
        // `leader`. Showing it beats showing nothing, and it keeps the pre-0.3.0 behaviour intact.
        var body = HealthBody.WithData(
            "Healthy",
            ("akka-cluster", "Healthy", new Dictionary<string, object>
            {
                ["leader"] = "akka.tcp://scadabridge@central-a:4053",
            }));
        var report = System.Text.Json.JsonSerializer.Deserialize<ZbHealthReport>(body)!;
        var instance = Snapshots.Instance("central-a", "central", "http://central-a:8080") with { Report = report };

        Assert.Equal("akka.tcp://scadabridge@central-a:4053", LeaderResolver.LeaderReportedBy(instance));
    }
}