Joseph Doherty
50b55ee4b9
RedundancyStateActor derived the driver Primary from ClusterState.RoleLeader("driver").
Akka offers two different notions of a "first" member and they are not the same
one: role leader is the lowest-ADDRESSED Up member (host, then port), while
ClusterSingletonManager places singletons on the OLDEST — lowest up-number.
They agree on a freshly-formed cluster, which is why every existing test passed.
They diverge after any restart: the restarted node re-joins as the youngest while
keeping its address, so if it holds the lower address it becomes role leader while
the singletons stay put. The snapshot would then name a Primary that is not
hosting the work, and every Primary-gated surface follows it — inbound device
writes, native-alarm acks, the fleet-wide alerts emit, and the alarm-history
drain would all enable on the wrong node while the node actually running the
singletons stayed gated off.
BuildSnapshot now selects the oldest Up member carrying the driver role, matching
singleton placement. Leaving members are excluded: a node handing its singletons
over must not be named Primary.
NodeRedundancyState.IsRoleLeaderForDriver is renamed IsDriverPrimary, and
NodeHealthInputs.IsDriverRoleLeader likewise. Keeping the old names would have
left the wire contract asserting a derivation the code no longer uses — the same
drift that made this defect invisible.
Proven by a real two-node cluster rather than a mock. RedundancyPrimaryElectionTests
binds the first-joining node to the HIGHER port, so oldest and lowest-address name
different nodes, and includes a fixture assertion that the divergence actually
occurred — without it the real assertion could pass for the wrong reason. Positive
control: restoring the RoleLeader derivation turns exactly the two election tests
red while the fixture check stays green.
Runtime.Tests 440 passed, ControlPlane.Tests 82, Cluster.Tests 36.
Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
92 lines
3.6 KiB
C#
92 lines
3.6 KiB
C#
using Akka.Cluster;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Cluster.Redundancy;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
|
|
|
/// <summary>
|
|
/// Unit tests for <see cref="ServiceLevelCalculator"/> — the pure ServiceLevel tiering logic.
|
|
/// Moved from ControlPlane.Tests to Cluster.Tests because the type lives in Core.Cluster.
|
|
/// See code-reviews/Cluster/findings.md Cluster-005.
|
|
/// </summary>
|
|
public sealed class ServiceLevelCalculatorTests
|
|
{
|
|
/// <summary>Non-Up member statuses produce ServiceLevel 0 regardless of other inputs.</summary>
|
|
[Theory]
|
|
[InlineData(MemberStatus.Down)]
|
|
[InlineData(MemberStatus.Removed)]
|
|
[InlineData(MemberStatus.Exiting)]
|
|
[InlineData(MemberStatus.Leaving)]
|
|
public void NotUp_returns_zero(MemberStatus status)
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(status,
|
|
DbReachable: true, OpcUaProbeOk: true, Stale: false, IsDriverPrimary: true));
|
|
sl.ShouldBe((byte)0);
|
|
}
|
|
|
|
/// <summary>Up + DB reachable + probe ok + not stale + not leader = tier 240 (healthy follower).</summary>
|
|
[Fact]
|
|
public void Fully_healthy_non_leader_returns_240()
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Up,
|
|
DbReachable: true, OpcUaProbeOk: true, Stale: false, IsDriverPrimary: false));
|
|
sl.ShouldBe((byte)240);
|
|
}
|
|
|
|
/// <summary>Up + DB reachable + probe ok + not stale + is leader = tier 250 (healthy leader, +10 bonus).</summary>
|
|
[Fact]
|
|
public void Fully_healthy_role_leader_returns_250()
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Up,
|
|
DbReachable: true, OpcUaProbeOk: true, Stale: false, IsDriverPrimary: true));
|
|
sl.ShouldBe((byte)250);
|
|
}
|
|
|
|
/// <summary>DB reachable but data stale = tier 200 (stale).</summary>
|
|
[Fact]
|
|
public void Db_reachable_but_stale_returns_200()
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Up,
|
|
DbReachable: true, OpcUaProbeOk: true, Stale: true, IsDriverPrimary: false));
|
|
sl.ShouldBe((byte)200);
|
|
}
|
|
|
|
/// <summary>DB unreachable AND stale = tier 100 (critically degraded).</summary>
|
|
[Fact]
|
|
public void Db_unreachable_and_stale_returns_100()
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Up,
|
|
DbReachable: false, OpcUaProbeOk: false, Stale: true, IsDriverPrimary: false));
|
|
sl.ShouldBe((byte)100);
|
|
}
|
|
|
|
/// <summary>OPC UA probe failed while not stale falls through to the catch-all 0.</summary>
|
|
[Fact]
|
|
public void Opcua_probe_fail_when_not_stale_returns_zero()
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Up,
|
|
DbReachable: true, OpcUaProbeOk: false, Stale: false, IsDriverPrimary: false));
|
|
sl.ShouldBe((byte)0);
|
|
}
|
|
|
|
/// <summary>Joining is treated identically to Up for grading (node is joining the cluster).</summary>
|
|
[Fact]
|
|
public void Joining_member_is_treated_like_Up()
|
|
{
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Joining,
|
|
DbReachable: true, OpcUaProbeOk: true, Stale: false, IsDriverPrimary: false));
|
|
sl.ShouldBe((byte)240);
|
|
}
|
|
|
|
/// <summary>Leader bonus is clamped to 255 even if a hypothetical basis would overflow.</summary>
|
|
[Fact]
|
|
public void Result_is_clamped_to_255()
|
|
{
|
|
// basis 240 + 10 = 250, already within byte range; confirms Clamp is in path
|
|
var sl = ServiceLevelCalculator.Compute(new(MemberStatus.Up,
|
|
DbReachable: true, OpcUaProbeOk: true, Stale: false, IsDriverPrimary: true));
|
|
((int)sl).ShouldBeLessThanOrEqualTo(255);
|
|
}
|
|
}
|