248676ed16
Port OtOpcUa's bootstrap guard (lmxopcua d1dac87f) to ScadaBridge's BuildHocon bootstrap. Both pair nodes are self-first seeds, so a true simultaneous cold start (shared site power event) races FirstSeedNodeProcess on both and forms two 1-node clusters that never merge. Opt-in dark switch ScadaBridge:Cluster:BootstrapGuard:Enabled (default off, guard-off behavior byte-identical). When on: BuildHocon emits an empty seed list so Akka does not auto-join, and ClusterBootstrapCoordinator (IHostedService, registered in both the Central and Site composition roots) picks the join order from ClusterBootstrapGuard's pure decision core — the lower canonical host:port is the founder (self-first, forms immediately); the higher node TCP-probes the founder up to PartnerProbeSeconds and joins peer-first if reachable, else self-first (cold-start-alone preserved). Decides before a single JoinSeedNodes, never re-forms mid-handshake. Review notes carried over: case-insensitive founder tie-break; fail-fast validation of probe timings when enabled; higher-node-cold-start-alone covered by a real-ActorSystem test. 21 unit + 5 real-cluster tests (incl. the headline both-cold-start-together-form-one-cluster).
113 lines
6.4 KiB
C#
113 lines
6.4 KiB
C#
using ZB.MOM.WW.Configuration;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
|
|
|
/// <summary>
|
|
/// Validates <see cref="ClusterOptions"/> at startup. The values it
|
|
/// guards carry cluster-wide consequences — the design doc
|
|
/// (<c>Component-ClusterInfrastructure.md</c>) is emphatic that misconfiguring
|
|
/// them produces a total cluster shutdown or an indefinitely blocked singleton.
|
|
/// Registered with <c>ValidateOnStart()</c> so a bad <c>appsettings.json</c>
|
|
/// fails fast at boot rather than failing far from the cause.
|
|
/// </summary>
|
|
public sealed class ClusterOptionsValidator : OptionsValidatorBase<ClusterOptions>
|
|
{
|
|
/// <summary>
|
|
/// Downing strategies supported for ScadaBridge's two-node clusters.
|
|
/// <c>auto-down</c> (default) survives a crash of either node at the accepted cost
|
|
/// of dual-active during a real partition; <c>keep-oldest</c> is partition-safe but
|
|
/// cannot survive a crash of the oldest node. Quorum strategies are rejected:
|
|
/// <c>static-quorum</c> quorum-size 1 trips Akka's IsTooManyMembers guard (DownAll
|
|
/// on any unreachability in a 2-node cluster) and <c>keep-majority</c> keys the
|
|
/// fatal crash to the lowest-address node instead of the oldest.
|
|
/// </summary>
|
|
private static readonly HashSet<string> AllowedStrategies = new(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
"auto-down",
|
|
"keep-oldest"
|
|
};
|
|
|
|
/// <inheritdoc />
|
|
protected override void Validate(ValidationBuilder builder, ClusterOptions options)
|
|
{
|
|
ValidateBootstrapGuard(builder, options.BootstrapGuard);
|
|
|
|
// The design doc states "both nodes are seed nodes — each node lists
|
|
// both itself and its partner" so a properly-configured deployment lists
|
|
// two. Accepting a single-seed configuration silently defeats the
|
|
// "no startup ordering dependency" guarantee called out by
|
|
// Component-ClusterInfrastructure.md (Node Configuration).
|
|
var minSeeds = options.AllowSingleNodeCluster ? 1 : 2;
|
|
builder.RequireThat(options.SeedNodes is not null && options.SeedNodes.Count >= minSeeds,
|
|
options.AllowSingleNodeCluster
|
|
? "ClusterOptions.SeedNodes must contain at least 1 seed node."
|
|
: "ClusterOptions.SeedNodes must contain at least 2 seed nodes "
|
|
+ "(Component-ClusterInfrastructure.md → Node Configuration: both nodes are seed nodes); "
|
|
+ "for a deliberate single-node install set ClusterOptions.AllowSingleNodeCluster = true instead of listing a phantom seed.");
|
|
|
|
builder.RequireThat(
|
|
!string.IsNullOrWhiteSpace(options.SplitBrainResolverStrategy)
|
|
&& AllowedStrategies.Contains(options.SplitBrainResolverStrategy),
|
|
$"ClusterOptions.SplitBrainResolverStrategy must be 'auto-down' or 'keep-oldest' for a " +
|
|
$"two-node cluster; '{options.SplitBrainResolverStrategy}' would risk a total cluster " +
|
|
"shutdown on a partition or an unreachability event.");
|
|
|
|
builder.RequireThat(options.MinNrOfMembers == 1,
|
|
$"ClusterOptions.MinNrOfMembers must be 1 (was {options.MinNrOfMembers}); " +
|
|
"any other value blocks the cluster singleton after failover and halts all data collection.");
|
|
|
|
builder.RequireThat(options.StableAfter > TimeSpan.Zero,
|
|
"ClusterOptions.StableAfter must be a positive duration.");
|
|
|
|
builder.RequireThat(options.HeartbeatInterval > TimeSpan.Zero,
|
|
"ClusterOptions.HeartbeatInterval must be a positive duration.");
|
|
|
|
builder.RequireThat(options.FailureDetectionThreshold > TimeSpan.Zero,
|
|
"ClusterOptions.FailureDetectionThreshold must be a positive duration.");
|
|
|
|
builder.RequireThat(options.HeartbeatInterval < options.FailureDetectionThreshold,
|
|
$"ClusterOptions.HeartbeatInterval ({options.HeartbeatInterval}) must be well below " +
|
|
$"FailureDetectionThreshold ({options.FailureDetectionThreshold}); otherwise nodes are " +
|
|
"declared unreachable before a heartbeat can arrive.");
|
|
|
|
// DownIfAlone is a keep-oldest knob; under auto-down each side downs the
|
|
// unreachable peer regardless, so the flag is inert and any value is fine.
|
|
var isKeepOldest = string.Equals(
|
|
options.SplitBrainResolverStrategy, "keep-oldest", StringComparison.OrdinalIgnoreCase);
|
|
builder.RequireThat(!isKeepOldest || options.DownIfAlone,
|
|
"ClusterOptions.DownIfAlone must be true for the keep-oldest resolver "
|
|
+ "(Component-ClusterInfrastructure.md → Split-Brain Resolution); with it false the "
|
|
+ "oldest node can run as an isolated single-node cluster during a partition while the "
|
|
+ "younger node forms its own, producing two live clusters.");
|
|
}
|
|
|
|
/// <summary>
|
|
/// When the bootstrap guard is enabled (Gitea #33), its timing knobs must be positive. A
|
|
/// zero/negative <see cref="ClusterBootstrapGuardOptions.PartnerProbeSeconds"/> in particular
|
|
/// silently degrades the guard to "never wait, always conclude the partner is dead" — the
|
|
/// higher node would form alone immediately and re-open the very split the guard exists to
|
|
/// close. Fail fast at boot rather than producing that silent degradation. Nothing is checked
|
|
/// when the guard is off (the knobs are inert), so a disabled guard never blocks a boot.
|
|
/// </summary>
|
|
private static void ValidateBootstrapGuard(ValidationBuilder builder, ClusterBootstrapGuardOptions? guard)
|
|
{
|
|
if (guard is null || !guard.Enabled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
builder.RequireThat(guard.PartnerProbeSeconds > 0,
|
|
$"ClusterOptions.BootstrapGuard.PartnerProbeSeconds must be > 0 when the guard is enabled "
|
|
+ $"(was {guard.PartnerProbeSeconds}); a non-positive value makes the higher node conclude its "
|
|
+ "partner is dead without waiting and form alone, re-opening the split the guard prevents.");
|
|
|
|
builder.RequireThat(guard.PartnerProbeIntervalMs > 0,
|
|
$"ClusterOptions.BootstrapGuard.PartnerProbeIntervalMs must be > 0 when the guard is enabled "
|
|
+ $"(was {guard.PartnerProbeIntervalMs}).");
|
|
|
|
builder.RequireThat(guard.ProbeConnectTimeoutMs > 0,
|
|
$"ClusterOptions.BootstrapGuard.ProbeConnectTimeoutMs must be > 0 when the guard is enabled "
|
|
+ $"(was {guard.ProbeConnectTimeoutMs}).");
|
|
}
|
|
}
|