ab561faa43
Adds LeaderAutoDowningProvider (Host/Actors), a line-for-line port of Akka.NET 1.5.62 AutoDowning/AutoDown/AutoDownBase read from the decompiled assembly: the leader among the reachable members downs a peer unreachable for StableAfter; a non-leader records it as pending and downs it on becoming leader; reachable-again or removed members are forgotten; Down/Exiting members are skipped. Its window is carried under scadabridge.cluster.leader-auto-down.unreachable-after, so no deprecated auto-down-unreachable-after key is emitted and Akka's boot-time "keep-majority will be used instead" warning goes away. DownRemovalMargin reads akka.cluster.down-removal-margin exactly as AutoDowning does (default off). New SplitBrainResolverStrategy value "leader-auto-down", wired through BuildHocon and ClusterOptionsValidator. The default stays "auto-down"; auto-down and keep-oldest HOCON is byte-identical to before. Placed in Host, not ClusterInfrastructure, per the Code Placement rule (ClusterInfrastructure stays Akka-free). Tests: HOCON + validator units; LeaderAutoDownTests ports Akka's AutoDownSpec cases (leader-only, pending-on-leader-change, losing leadership, reachable again, removed, Down/Exiting skipped, single Down per member); LeaderAutoDownFailoverTests clones SbrFailoverTests' oldest-crash and younger-crash cases on a real two-node cluster plus a test that Akka actually loaded the provider and its downing actor. Mutation-checked: a provider that never calls Cluster.Down fails both failover tests; never-downs, leader-check-dropped, reachable-ignored and pending-not-flushed mutations each fail the unit tests. Remaining gate before switching the default: the docker failover drill (DRILL_MODE=active and standby mode) on leader-auto-down. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
331 lines
13 KiB
C#
331 lines
13 KiB
C#
using Microsoft.Extensions.Options;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
|
|
|
|
/// <summary>
|
|
/// Tests that <see cref="ClusterOptionsValidator"/> rejects the
|
|
/// catastrophic misconfigurations the design doc warns against — a
|
|
/// <c>MinNrOfMembers</c> other than 1, an unsupported split-brain strategy,
|
|
/// empty seed nodes, and timings where the heartbeat is not below the
|
|
/// failure-detection threshold.
|
|
/// </summary>
|
|
public class ClusterOptionsValidatorTests
|
|
{
|
|
private static ClusterOptions ValidOptions() => new()
|
|
{
|
|
SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081", "akka.tcp://scadabridge@node2:8081" },
|
|
SplitBrainResolverStrategy = "keep-oldest",
|
|
StableAfter = TimeSpan.FromSeconds(15),
|
|
HeartbeatInterval = TimeSpan.FromSeconds(2),
|
|
FailureDetectionThreshold = TimeSpan.FromSeconds(10),
|
|
MinNrOfMembers = 1,
|
|
DownIfAlone = true
|
|
};
|
|
|
|
/// <summary>Verifies that the default options are valid.</summary>
|
|
[Fact]
|
|
public void DefaultOptions_AreValid()
|
|
{
|
|
var result = new ClusterOptionsValidator().Validate(null, ValidOptions());
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that a minimum member count other than one fails validation.</summary>
|
|
[Fact]
|
|
public void MinNrOfMembers_NotOne_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.MinNrOfMembers = 2;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("MinNrOfMembers", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that an unsupported split-brain strategy fails validation.</summary>
|
|
/// <param name="strategy">The unsupported strategy name.</param>
|
|
[Theory]
|
|
[InlineData("keep-majority")]
|
|
[InlineData("static-quorum")]
|
|
[InlineData("nonsense")]
|
|
public void UnsupportedSplitBrainStrategy_FailsValidation(string strategy)
|
|
{
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = strategy;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that an empty seed node list fails validation.</summary>
|
|
[Fact]
|
|
public void EmptySeedNodes_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string>();
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that a single seed node without acknowledgement fails validation.</summary>
|
|
[Fact]
|
|
public void SingleSeed_WithoutAcknowledgement_Fails()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("AllowSingleNodeCluster", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that a single seed node passes when single-node clusters are explicitly allowed.</summary>
|
|
[Fact]
|
|
public void SingleSeed_WithAllowSingleNodeCluster_Passes()
|
|
{
|
|
// Review 01 [Low]: the shipped single-node artifact satisfied the 2-seed
|
|
// rule with a phantom seed the node dials forever. An explicit
|
|
// acknowledgement flag replaces the fiction.
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@h:8081" };
|
|
options.AllowSingleNodeCluster = true;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that an empty seed list fails even when single-node clusters are allowed.</summary>
|
|
[Fact]
|
|
public void EmptySeeds_FailsEvenWithFlag()
|
|
{
|
|
// Zero seeds is always invalid — the flag lowers the floor to 1, not 0.
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string>();
|
|
options.AllowSingleNodeCluster = true;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that a single seed node fails validation.</summary>
|
|
[Fact]
|
|
public void SingleSeedNode_FailsValidation()
|
|
{
|
|
// The design doc says "both nodes are seed nodes" — a single-seed
|
|
// configuration defeats the no-startup-ordering-dependency guarantee and
|
|
// must be rejected by the contract owner's validator, not just by the
|
|
// Host's startup validator.
|
|
var options = ValidOptions();
|
|
options.SeedNodes = new List<string> { "akka.tcp://scadabridge@node1:8081" };
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
Assert.Contains("2", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that a heartbeat interval that is not below the failure threshold fails validation.</summary>
|
|
[Fact]
|
|
public void HeartbeatNotBelowFailureThreshold_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.HeartbeatInterval = TimeSpan.FromSeconds(10);
|
|
options.FailureDetectionThreshold = TimeSpan.FromSeconds(10);
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("HeartbeatInterval", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that a non-positive stable-after duration fails validation.</summary>
|
|
[Fact]
|
|
public void NonPositiveStableAfter_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.StableAfter = TimeSpan.Zero;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("StableAfter", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that disabling down-if-alone under the keep-oldest strategy fails validation.</summary>
|
|
[Fact]
|
|
public void DownIfAloneFalse_UnderKeepOldest_FailsValidation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "keep-oldest";
|
|
options.DownIfAlone = false;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("DownIfAlone", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that the auto-down strategy passes validation.</summary>
|
|
[Fact]
|
|
public void AutoDownStrategy_Passes()
|
|
{
|
|
// Decision 2026-07-21: 'auto-down' is the supported availability-first
|
|
// posture — either-node crash fails over; dual-active on a real partition
|
|
// is the accepted trade.
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "auto-down";
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that disabling down-if-alone under the auto-down strategy passes validation.</summary>
|
|
[Fact]
|
|
public void DownIfAloneFalse_UnderAutoDown_Passes()
|
|
{
|
|
// DownIfAlone is a keep-oldest knob; under auto-down it is inert and must
|
|
// not block startup.
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "auto-down";
|
|
options.DownIfAlone = false;
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that validation accumulates every failure rather than stopping at the first.</summary>
|
|
[Fact]
|
|
public void Validate_AccumulatesAllFailures()
|
|
{
|
|
var options = new ClusterOptions
|
|
{
|
|
SeedNodes = new List<string>(),
|
|
SplitBrainResolverStrategy = "keep-majority",
|
|
MinNrOfMembers = 2,
|
|
StableAfter = TimeSpan.Zero,
|
|
HeartbeatInterval = TimeSpan.FromSeconds(20),
|
|
FailureDetectionThreshold = TimeSpan.FromSeconds(10)
|
|
};
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("SeedNodes", result.FailureMessage);
|
|
Assert.Contains("SplitBrainResolverStrategy", result.FailureMessage);
|
|
Assert.Contains("MinNrOfMembers", result.FailureMessage);
|
|
Assert.Contains("StableAfter", result.FailureMessage);
|
|
Assert.Contains("HeartbeatInterval", result.FailureMessage);
|
|
}
|
|
|
|
// ---- BootstrapGuard timing validation ----
|
|
|
|
/// <summary>Verifies that a disabled bootstrap guard with zero timings passes validation.</summary>
|
|
[Fact]
|
|
public void BootstrapGuard_disabled_with_zero_timings_passes_validation()
|
|
{
|
|
// The knobs are inert when the guard is off, so a disabled guard must never block a boot —
|
|
// even if the timing values are nonsense (0 here).
|
|
var options = ValidOptions();
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions
|
|
{
|
|
Enabled = false,
|
|
PartnerProbeSeconds = 0,
|
|
PartnerProbeIntervalMs = 0,
|
|
ProbeConnectTimeoutMs = 0,
|
|
};
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that an enabled bootstrap guard with default timings passes validation.</summary>
|
|
[Fact]
|
|
public void BootstrapGuard_enabled_with_default_timings_passes_validation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true };
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that an enabled bootstrap guard with non-positive probe seconds fails validation.</summary>
|
|
[Fact]
|
|
public void BootstrapGuard_enabled_with_nonpositive_probe_seconds_fails_validation()
|
|
{
|
|
// A zero PartnerProbeSeconds silently degrades the guard to "never wait, always conclude the
|
|
// partner is dead" — the higher node forms alone immediately and re-opens the split. Reject it.
|
|
var options = ValidOptions();
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 };
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("PartnerProbeSeconds", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that an enabled bootstrap guard with a non-positive interval or timeout fails validation.</summary>
|
|
[Fact]
|
|
public void BootstrapGuard_enabled_with_nonpositive_interval_or_timeout_fails_validation()
|
|
{
|
|
var options = ValidOptions();
|
|
options.BootstrapGuard = new ClusterBootstrapGuardOptions
|
|
{
|
|
Enabled = true,
|
|
PartnerProbeIntervalMs = 0,
|
|
ProbeConnectTimeoutMs = -1,
|
|
};
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("PartnerProbeIntervalMs", result.FailureMessage);
|
|
Assert.Contains("ProbeConnectTimeoutMs", result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that the leader-auto-down strategy (ScadaBridge#106) passes validation, case-insensitively.</summary>
|
|
/// <param name="strategy">The strategy spelling.</param>
|
|
[Theory]
|
|
[InlineData("leader-auto-down")]
|
|
[InlineData("Leader-Auto-Down")]
|
|
public void LeaderAutoDownStrategy_Passes(string strategy)
|
|
{
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = strategy;
|
|
options.DownIfAlone = false; // keep-oldest-only knob; inert here, must not block startup
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Succeeded, result.FailureMessage);
|
|
}
|
|
|
|
/// <summary>Verifies that the rejection message lists leader-auto-down among the supported strategies.</summary>
|
|
[Fact]
|
|
public void UnsupportedStrategy_MessageNamesLeaderAutoDown()
|
|
{
|
|
var options = ValidOptions();
|
|
options.SplitBrainResolverStrategy = "leader-autodown";
|
|
|
|
var result = new ClusterOptionsValidator().Validate(null, options);
|
|
|
|
Assert.True(result.Failed);
|
|
Assert.Contains("'leader-auto-down'", result.FailureMessage);
|
|
}
|
|
}
|