Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests/ClusterOptionsValidatorTests.cs
T
Joseph Doherty 952db02aab fix(cluster): leader-auto-down restarts a node its peer has shut out; higher address restarts when both are (#132)
A node frozen past failure detection + StableAfter, or both sides of a partition
that heals after each downed the other, used to stay a second active node until an
operator restarted one: the peer quarantines the removed UID and Akka.Cluster
1.5.62 never acts on ThisActorSystemQuarantinedEvent.

ShutOutRestart (child of the LeaderAutoDown downing actor; opt-out via
ScadaBridge:Cluster:LeaderAutoDown:RestartWhenShutOut, default on for
leader-auto-down only) watches two signals:
- S1 ThisActorSystemQuarantinedEvent: the peer quarantined this node. If the peer
  is still a live member here, only this side can be shut out -> restart now.
- S2 AssociationErrorEvent 'has a UID that has been quarantined': the peer still
  runs the incarnation this node removed. A healed partition yields only S2, on
  both sides. Wait ShutOutGrace (60s) for a frozen peer to restart itself, then
  tiebreak.
Tiebreak: the higher address (Akka Member.AddressOrdering) restarts, the lower
stays and restarts only if still split one grace later. Removed peers are probed
(Identify, 5s -> 60s back-off) since nothing else contacts them after mutual
removal. Restart = Cluster.Down(self) -> existing self-down path.

The Host now exits 75 after an unplanned ActorSystem termination (and sets it on
the Windows service before it reports stopped): SCM recovery with failureflag 1
only restarts a service that stops with a non-zero code, and the watchdog used
to exit 0. Shipped default strategy stays auto-down.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-12 21:26:16 -05:00

359 lines
14 KiB
C#

using Microsoft.Extensions.Options;
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
/// <summary>
/// Tests that <see cref="ClusterOptionsValidator"/> rejects the
/// catastrophic misconfigurations the design doc warns against — a
/// <c>MinNrOfMembers</c> other than 1, an unsupported split-brain strategy,
/// empty seed nodes, and timings where the heartbeat is not below the
/// failure-detection threshold.
/// </summary>
public class ClusterOptionsValidatorTests
{
private static ClusterOptions ValidOptions() => new()
{
SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081", "akka.tcp://scadabridge@node2:8081" },
SplitBrainResolverStrategy = "keep-oldest",
StableAfter = TimeSpan.FromSeconds(15),
HeartbeatInterval = TimeSpan.FromSeconds(2),
FailureDetectionThreshold = TimeSpan.FromSeconds(10),
MinNrOfMembers = 1,
DownIfAlone = true
};
/// <summary>Verifies that the default options are valid.</summary>
[Fact]
public void DefaultOptions_AreValid()
{
var result = new ClusterOptionsValidator().Validate(null, ValidOptions());
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that a minimum member count other than one fails validation.</summary>
[Fact]
public void MinNrOfMembers_NotOne_FailsValidation()
{
var options = ValidOptions();
options.MinNrOfMembers = 2;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("MinNrOfMembers", result.FailureMessage);
}
/// <summary>Verifies that an unsupported split-brain strategy fails validation.</summary>
/// <param name="strategy">The unsupported strategy name.</param>
[Theory]
[InlineData("keep-majority")]
[InlineData("static-quorum")]
[InlineData("nonsense")]
public void UnsupportedSplitBrainStrategy_FailsValidation(string strategy)
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = strategy;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
}
/// <summary>Verifies that an empty seed node list fails validation.</summary>
[Fact]
public void EmptySeedNodes_FailsValidation()
{
var options = ValidOptions();
options.SeedNodes = new List<string>();
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
}
/// <summary>Verifies that a single seed node without acknowledgement fails validation.</summary>
[Fact]
public void SingleSeed_WithoutAcknowledgement_Fails()
{
var options = ValidOptions();
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("AllowSingleNodeCluster", result.FailureMessage);
}
/// <summary>Verifies that a single seed node passes when single-node clusters are explicitly allowed.</summary>
[Fact]
public void SingleSeed_WithAllowSingleNodeCluster_Passes()
{
// Review 01 [Low]: the shipped single-node artifact satisfied the 2-seed
// rule with a phantom seed the node dials forever. An explicit
// acknowledgement flag replaces the fiction.
var options = ValidOptions();
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
options.AllowSingleNodeCluster = true;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that an empty seed list fails even when single-node clusters are allowed.</summary>
[Fact]
public void EmptySeeds_FailsEvenWithFlag()
{
// Zero seeds is always invalid — the flag lowers the floor to 1, not 0.
var options = ValidOptions();
options.SeedNodes = new List<string>();
options.AllowSingleNodeCluster = true;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
}
/// <summary>Verifies that a single seed node fails validation.</summary>
[Fact]
public void SingleSeedNode_FailsValidation()
{
// The design doc says "both nodes are seed nodes" — a single-seed
// configuration defeats the no-startup-ordering-dependency guarantee and
// must be rejected by the contract owner's validator, not just by the
// Host's startup validator.
var options = ValidOptions();
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081" };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
Assert.Contains("2", result.FailureMessage);
}
/// <summary>Verifies that a heartbeat interval that is not below the failure threshold fails validation.</summary>
[Fact]
public void HeartbeatNotBelowFailureThreshold_FailsValidation()
{
var options = ValidOptions();
options.HeartbeatInterval = TimeSpan.FromSeconds(10);
options.FailureDetectionThreshold = TimeSpan.FromSeconds(10);
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("HeartbeatInterval", result.FailureMessage);
}
/// <summary>Verifies that a non-positive stable-after duration fails validation.</summary>
[Fact]
public void NonPositiveStableAfter_FailsValidation()
{
var options = ValidOptions();
options.StableAfter = TimeSpan.Zero;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("StableAfter", result.FailureMessage);
}
/// <summary>Verifies that disabling down-if-alone under the keep-oldest strategy fails validation.</summary>
[Fact]
public void DownIfAloneFalse_UnderKeepOldest_FailsValidation()
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = "keep-oldest";
options.DownIfAlone = false;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("DownIfAlone", result.FailureMessage);
}
/// <summary>Verifies that the auto-down strategy passes validation.</summary>
[Fact]
public void AutoDownStrategy_Passes()
{
// Decision 2026-07-21: 'auto-down' is the supported availability-first
// posture — either-node crash fails over; dual-active on a real partition
// is the accepted trade.
var options = ValidOptions();
options.SplitBrainResolverStrategy = "auto-down";
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that disabling down-if-alone under the auto-down strategy passes validation.</summary>
[Fact]
public void DownIfAloneFalse_UnderAutoDown_Passes()
{
// DownIfAlone is a keep-oldest knob; under auto-down it is inert and must
// not block startup.
var options = ValidOptions();
options.SplitBrainResolverStrategy = "auto-down";
options.DownIfAlone = false;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that validation accumulates every failure rather than stopping at the first.</summary>
[Fact]
public void Validate_AccumulatesAllFailures()
{
var options = new ClusterOptions
{
SeedNodes = new List<string>(),
SplitBrainResolverStrategy = "keep-majority",
MinNrOfMembers = 2,
StableAfter = TimeSpan.Zero,
HeartbeatInterval = TimeSpan.FromSeconds(20),
FailureDetectionThreshold = TimeSpan.FromSeconds(10)
};
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
Assert.Contains("MinNrOfMembers", result.FailureMessage);
Assert.Contains("StableAfter", result.FailureMessage);
Assert.Contains("HeartbeatInterval", result.FailureMessage);
}
// ---- BootstrapGuard timing validation ----
/// <summary>Verifies that a disabled bootstrap guard with zero timings passes validation.</summary>
[Fact]
public void BootstrapGuard_disabled_with_zero_timings_passes_validation()
{
// The knobs are inert when the guard is off, so a disabled guard must never block a boot —
// even if the timing values are nonsense (0 here).
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions
{
Enabled = false,
PartnerProbeSeconds = 0,
PartnerProbeIntervalMs = 0,
ProbeConnectTimeoutMs = 0,
};
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that an enabled bootstrap guard with default timings passes validation.</summary>
[Fact]
public void BootstrapGuard_enabled_with_default_timings_passes_validation()
{
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that an enabled bootstrap guard with non-positive probe seconds fails validation.</summary>
[Fact]
public void BootstrapGuard_enabled_with_nonpositive_probe_seconds_fails_validation()
{
// A zero PartnerProbeSeconds silently degrades the guard to "never wait, always conclude the
// partner is dead" — the higher node forms alone immediately and re-opens the split. Reject it.
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("PartnerProbeSeconds", result.FailureMessage);
}
/// <summary>Verifies that an enabled bootstrap guard with a non-positive interval or timeout fails validation.</summary>
[Fact]
public void BootstrapGuard_enabled_with_nonpositive_interval_or_timeout_fails_validation()
{
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions
{
Enabled = true,
PartnerProbeIntervalMs = 0,
ProbeConnectTimeoutMs = -1,
};
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("PartnerProbeIntervalMs", result.FailureMessage);
Assert.Contains("ProbeConnectTimeoutMs", result.FailureMessage);
}
/// <summary>Verifies that the leader-auto-down strategy (ScadaBridge#106) passes validation, case-insensitively.</summary>
/// <param name="strategy">The strategy spelling.</param>
[Theory]
[InlineData("leader-auto-down")]
[InlineData("Leader-Auto-Down")]
public void LeaderAutoDownStrategy_Passes(string strategy)
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = strategy;
options.DownIfAlone = false; // keep-oldest-only knob; inert here, must not block startup
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
/// <summary>Verifies that the rejection message lists leader-auto-down among the supported strategies.</summary>
[Fact]
public void UnsupportedStrategy_MessageNamesLeaderAutoDown()
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = "leader-autodown";
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("'leader-auto-down'", result.FailureMessage);
}
/// <summary>
/// ScadaBridge#132: a non-positive shut-out grace is refused under leader-auto-down with the
/// restart on (it would make both sides of a healed partition restart together), and ignored
/// when the restart is off or another strategy is selected.
/// </summary>
/// <param name="strategy">The strategy.</param>
/// <param name="restart">The shut-out restart switch.</param>
/// <param name="expectValid">Whether validation must pass.</param>
[Theory]
[InlineData("leader-auto-down", true, false)]
[InlineData("leader-auto-down", false, true)]
[InlineData("auto-down", true, true)]
public void LeaderAutoDown_ZeroShutOutGrace(string strategy, bool restart, bool expectValid)
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = strategy;
options.LeaderAutoDown = new LeaderAutoDownOptions { RestartWhenShutOut = restart, ShutOutGrace = TimeSpan.Zero };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.Equal(expectValid, result.Succeeded);
if (!expectValid)
{
Assert.Contains("ShutOutGrace", result.FailureMessage);
}
}
}