cf3bd52f93
Two-node keep-oldest could NEVER survive a crash of the oldest/active node:
Akka.NET 1.5.62 KeepOldest.OldestDecision only lets down-if-alone rescue a
side with >= 2 members, so the 1-vs-1 survivor takes DownReachable and downs
ITSELF — proven live on the rig ('SBR took decision ... including myself')
before this change. static-quorum(1) is worse (IsTooManyMembers -> DownAll);
keep-majority just re-keys the fatal crash to the lowest address.
SplitBrainResolverStrategy gains 'auto-down' (new default): BuildHocon emits
Akka's AutoDowning provider with auto-down-unreachable-after = StableAfter.
The leader among the REACHABLE members downs the unreachable peer, so the
survivor takes over singletons and /health/active in ~25s regardless of which
node died. Accepted trade (explicit owner decision): a real network partition
runs dual-active until an operator restarts one side. keep-oldest remains
supported; DownIfAlone validation is now scoped to it.
Live drill on the rebuilt rig: active-crash TAKEOVER in 28s (victim still
down; all 7 singletons Younger->Oldest), standby-crash removal 27s with 0
routing blips; victims rejoin as standby in 2s. New real-cluster tests pin
both directions (SbrFailoverTests.AutoDown_*); TwoNodeClusterFixture gains a
strategy knob. All 16 appsettings flipped (src, docker, docker-env2, and the
gitignored wonder-app-vd03 overlay on disk — owner must sync to the host).
Docs: decision record docs/plans/2026-07-21-auto-down-availability-decision.md,
Component-ClusterInfrastructure downing section rewritten, drill + README
reworked (active mode now asserts takeover), deferred-work SBR row resolved.
219 lines
7.3 KiB
C#
219 lines
7.3 KiB
C#
using Microsoft.Extensions.Options;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
|
|
|
|
/// <summary>
|
|
/// CI-004: Tests that <see cref="ClusterOptionsValidator"/> rejects the
|
|
/// catastrophic misconfigurations the design doc warns against — a
|
|
/// <c>MinNrOfMembers</c> other than 1, an unsupported split-brain strategy,
|
|
/// empty seed nodes, and timings where the heartbeat is not below the
|
|
/// failure-detection threshold.
|
|
/// </summary>
|
|
public class ClusterOptionsValidatorTests
|
|
{
|
|
private static ClusterOptions ValidOptions() => new()
|
|
{
|
|
SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081", "akka.tcp://scadabridge@node2:8081" },
|
|
SplitBrainResolverStrategy = "keep-oldest",
|
|
StableAfter = TimeSpan.FromSeconds(15),
|
|
HeartbeatInterval = TimeSpan.FromSeconds(2),
|
|
FailureDetectionThreshold = TimeSpan.FromSeconds(10),
|
|
MinNrOfMembers = 1,
|
|
DownIfAlone = true
|
|
};
|
|
|
|
[Fact]
|
|
public void DefaultOptions_AreValid()
|
|
{
|
|
var result = new ClusterOptionsValidator().Validate(null, ValidOptions());
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void MinNrOfMembers_NotOne_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.MinNrOfMembers = 2;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("MinNrOfMembers", result.FailureMessage);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("keep-majority")]
|
|
[InlineData("static-quorum")]
|
|
[InlineData("nonsense")]
|
|
public void UnsupportedSplitBrainStrategy_FailsValidation(string strategy)
|
|
{
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = strategy;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void EmptySeedNodes_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string>();
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void SingleSeed_WithoutAcknowledgement_Fails()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("AllowSingleNodeCluster", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void SingleSeed_WithAllowSingleNodeCluster_Passes()
|
|
{
|
|
// Review 01 [Low]: the shipped single-node artifact satisfied the 2-seed
|
|
// rule with a phantom seed the node dials forever. An explicit
|
|
// acknowledgement flag replaces the fiction.
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
|
|
options.AllowSingleNodeCluster = true;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void EmptySeeds_FailsEvenWithFlag()
|
|
{
|
|
// Zero seeds is always invalid — the flag lowers the floor to 1, not 0.
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string>();
|
|
options.AllowSingleNodeCluster = true;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void SingleSeedNode_FailsValidation()
|
|
{
|
|
// CI-012: design doc says "both nodes are seed nodes" — a single-seed
|
|
// configuration defeats the no-startup-ordering-dependency guarantee and
|
|
// must be rejected by the contract owner's validator, not just by the
|
|
// Host's startup validator.
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081" };
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
Assert.Contains("2", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void HeartbeatNotBelowFailureThreshold_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.HeartbeatInterval = TimeSpan.FromSeconds(10);
|
|
options.FailureDetectionThreshold = TimeSpan.FromSeconds(10);
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("HeartbeatInterval", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void NonPositiveStableAfter_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.StableAfter = TimeSpan.Zero;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("StableAfter", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void DownIfAloneFalse_UnderKeepOldest_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "keep-oldest";
|
|
options.DownIfAlone = false;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("DownIfAlone", result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void AutoDownStrategy_Passes()
|
|
{
|
|
// Decision 2026-07-21: 'auto-down' is the supported availability-first
|
|
// posture — either-node crash fails over; dual-active on a real partition
|
|
// is the accepted trade.
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "auto-down";
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void DownIfAloneFalse_UnderAutoDown_Passes()
|
|
{
|
|
// DownIfAlone is a keep-oldest knob; under auto-down it is inert and must
|
|
// not block startup.
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "auto-down";
|
|
options.DownIfAlone = false;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void Validate_AccumulatesAllFailures()
|
|
{
|
|
var options = new ClusterOptions
|
|
{
|
|
SeedNodes = new List<string>(),
|
|
SplitBrainResolverStrategy = "keep-majority",
|
|
MinNrOfMembers = 2,
|
|
StableAfter = TimeSpan.Zero,
|
|
HeartbeatInterval = TimeSpan.FromSeconds(20),
|
|
FailureDetectionThreshold = TimeSpan.FromSeconds(10)
|
|
};
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
|
|
Assert.Contains("MinNrOfMembers", result.FailureMessage);
|
|
Assert.Contains("StableAfter", result.FailureMessage);
|
|
Assert.Contains("HeartbeatInterval", result.FailureMessage);
|
|
}
|
|
}
|