Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests/ClusterOptionsValidatorTests.cs
T
Joseph Doherty 248676ed16 feat(cluster): simultaneous-cold-start split-brain guard (#33)
Port OtOpcUa's bootstrap guard (lmxopcua d1dac87f) to ScadaBridge's
BuildHocon bootstrap. Both pair nodes are self-first seeds, so a true
simultaneous cold start (shared site power event) races FirstSeedNodeProcess
on both and forms two 1-node clusters that never merge.

Opt-in dark switch ScadaBridge:Cluster:BootstrapGuard:Enabled (default off,
guard-off behavior byte-identical). When on: BuildHocon emits an empty seed
list so Akka does not auto-join, and ClusterBootstrapCoordinator (IHostedService,
registered in both the Central and Site composition roots) picks the join order
from ClusterBootstrapGuard's pure decision core — the lower canonical host:port
is the founder (self-first, forms immediately); the higher node TCP-probes the
founder up to PartnerProbeSeconds and joins peer-first if reachable, else
self-first (cold-start-alone preserved). Decides before a single JoinSeedNodes,
never re-forms mid-handshake.

Review notes carried over: case-insensitive founder tie-break; fail-fast
validation of probe timings when enabled; higher-node-cold-start-alone covered
by a real-ActorSystem test. 21 unit + 5 real-cluster tests (incl. the headline
both-cold-start-together-form-one-cluster).
2026-07-24 08:55:48 -04:00

283 lines
9.7 KiB
C#

using Microsoft.Extensions.Options;
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
/// <summary>
/// CI-004: Tests that <see cref="ClusterOptionsValidator"/> rejects the
/// catastrophic misconfigurations the design doc warns against — a
/// <c>MinNrOfMembers</c> other than 1, an unsupported split-brain strategy,
/// empty seed nodes, and timings where the heartbeat is not below the
/// failure-detection threshold.
/// </summary>
public class ClusterOptionsValidatorTests
{
private static ClusterOptions ValidOptions() => new()
{
SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081", "akka.tcp://scadabridge@node2:8081" },
SplitBrainResolverStrategy = "keep-oldest",
StableAfter = TimeSpan.FromSeconds(15),
HeartbeatInterval = TimeSpan.FromSeconds(2),
FailureDetectionThreshold = TimeSpan.FromSeconds(10),
MinNrOfMembers = 1,
DownIfAlone = true
};
[Fact]
public void DefaultOptions_AreValid()
{
var result = new ClusterOptionsValidator().Validate(null, ValidOptions());
Assert.True(result.Succeeded, result.FailureMessage);
}
[Fact]
public void MinNrOfMembers_NotOne_FailsValidation()
{
var options = ValidOptions();
options.MinNrOfMembers = 2;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("MinNrOfMembers", result.FailureMessage);
}
[Theory]
[InlineData("keep-majority")]
[InlineData("static-quorum")]
[InlineData("nonsense")]
public void UnsupportedSplitBrainStrategy_FailsValidation(string strategy)
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = strategy;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
}
[Fact]
public void EmptySeedNodes_FailsValidation()
{
var options = ValidOptions();
options.SeedNodes = new List<string>();
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
}
[Fact]
public void SingleSeed_WithoutAcknowledgement_Fails()
{
var options = ValidOptions();
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("AllowSingleNodeCluster", result.FailureMessage);
}
[Fact]
public void SingleSeed_WithAllowSingleNodeCluster_Passes()
{
// Review 01 [Low]: the shipped single-node artifact satisfied the 2-seed
// rule with a phantom seed the node dials forever. An explicit
// acknowledgement flag replaces the fiction.
var options = ValidOptions();
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
options.AllowSingleNodeCluster = true;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
[Fact]
public void EmptySeeds_FailsEvenWithFlag()
{
// Zero seeds is always invalid — the flag lowers the floor to 1, not 0.
var options = ValidOptions();
options.SeedNodes = new List<string>();
options.AllowSingleNodeCluster = true;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
}
[Fact]
public void SingleSeedNode_FailsValidation()
{
// CI-012: design doc says "both nodes are seed nodes" — a single-seed
// configuration defeats the no-startup-ordering-dependency guarantee and
// must be rejected by the contract owner's validator, not just by the
// Host's startup validator.
var options = ValidOptions();
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081" };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
Assert.Contains("2", result.FailureMessage);
}
[Fact]
public void HeartbeatNotBelowFailureThreshold_FailsValidation()
{
var options = ValidOptions();
options.HeartbeatInterval = TimeSpan.FromSeconds(10);
options.FailureDetectionThreshold = TimeSpan.FromSeconds(10);
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("HeartbeatInterval", result.FailureMessage);
}
[Fact]
public void NonPositiveStableAfter_FailsValidation()
{
var options = ValidOptions();
options.StableAfter = TimeSpan.Zero;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("StableAfter", result.FailureMessage);
}
[Fact]
public void DownIfAloneFalse_UnderKeepOldest_FailsValidation()
{
var options = ValidOptions();
options.SplitBrainResolverStrategy = "keep-oldest";
options.DownIfAlone = false;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("DownIfAlone", result.FailureMessage);
}
[Fact]
public void AutoDownStrategy_Passes()
{
// Decision 2026-07-21: 'auto-down' is the supported availability-first
// posture — either-node crash fails over; dual-active on a real partition
// is the accepted trade.
var options = ValidOptions();
options.SplitBrainResolverStrategy = "auto-down";
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
[Fact]
public void DownIfAloneFalse_UnderAutoDown_Passes()
{
// DownIfAlone is a keep-oldest knob; under auto-down it is inert and must
// not block startup.
var options = ValidOptions();
options.SplitBrainResolverStrategy = "auto-down";
options.DownIfAlone = false;
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
[Fact]
public void Validate_AccumulatesAllFailures()
{
var options = new ClusterOptions
{
SeedNodes = new List<string>(),
SplitBrainResolverStrategy = "keep-majority",
MinNrOfMembers = 2,
StableAfter = TimeSpan.Zero,
HeartbeatInterval = TimeSpan.FromSeconds(20),
FailureDetectionThreshold = TimeSpan.FromSeconds(10)
};
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("SeedNodes", result.FailureMessage);
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
Assert.Contains("MinNrOfMembers", result.FailureMessage);
Assert.Contains("StableAfter", result.FailureMessage);
Assert.Contains("HeartbeatInterval", result.FailureMessage);
}
// ---- BootstrapGuard timing validation (Gitea #33) ----
[Fact]
public void BootstrapGuard_disabled_with_zero_timings_passes_validation()
{
// The knobs are inert when the guard is off, so a disabled guard must never block a boot —
// even if the timing values are nonsense (0 here).
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions
{
Enabled = false,
PartnerProbeSeconds = 0,
PartnerProbeIntervalMs = 0,
ProbeConnectTimeoutMs = 0,
};
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
[Fact]
public void BootstrapGuard_enabled_with_default_timings_passes_validation()
{
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Succeeded, result.FailureMessage);
}
[Fact]
public void BootstrapGuard_enabled_with_nonpositive_probe_seconds_fails_validation()
{
// A zero PartnerProbeSeconds silently degrades the guard to "never wait, always conclude the
// partner is dead" — the higher node forms alone immediately and re-opens the split. Reject it.
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 };
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("PartnerProbeSeconds", result.FailureMessage);
}
[Fact]
public void BootstrapGuard_enabled_with_nonpositive_interval_or_timeout_fails_validation()
{
var options = ValidOptions();
options.BootstrapGuard = new ClusterBootstrapGuardOptions
{
Enabled = true,
PartnerProbeIntervalMs = 0,
ProbeConnectTimeoutMs = -1,
};
var result = new ClusterOptionsValidator().Validate(null, options);
Assert.True(result.Failed);
Assert.Contains("PartnerProbeIntervalMs", result.FailureMessage);
Assert.Contains("ProbeConnectTimeoutMs", result.FailureMessage);
}
}