Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/HoconBuilderTests.cs
T
Joseph Doherty efade2eab0 feat(cluster)!: leader-auto-down is the shipped default downing strategy (#116)
SplitBrainResolverStrategy now defaults to leader-auto-down (the in-repo
LeaderAutoDowningProvider, #106, with the #132 shut-out restart on). auto-down and
keep-oldest stay supported. Drilled on the docker rig, docker-env2 and the QA pair
(central + site, kill and stop-and-hold) before the flip.

- ClusterOptions default + validator/HOCON comments updated.
- Shipped Host appsettings.Central/Site.json and every docker/ and docker-env2/
  node appsettings now say leader-auto-down. The leader-auto-down overlays are
  kept as no-ops (drill records reference them); new docker-compose.auto-down.yml
  overlays keep Akka AutoDowning available for comparison drills.
- Tests: default asserts leader-auto-down; new HOCON test for an unset strategy;
  TwoNodeClusterFixture defaults to the production posture.
- Docs: CLAUDE.md, Component-ClusterInfrastructure, components/Host + Cluster,
  README, failover procedures, Traefik, docker + env2 READMEs; register row 70 closed.

No deploy needed: DEV, QA and both rigs set the key explicitly.

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
2026-11-15 19:57:45 -05:00

391 lines
17 KiB
C#

using Akka.Configuration;
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <summary>
/// The Akka HOCON document must be assembled with every interpolated value
/// quoted/escaped, so a hostname, seed-node URI or split-brain strategy containing a
/// quote, backslash or whitespace cannot corrupt the document or be silently
/// misparsed.
/// </summary>
public class HoconBuilderTests
{
private static ClusterOptions DefaultCluster() => new()
{
SeedNodes = new List<string>
{
"akka.tcp://scadabridge@localhost:8081",
"akka.tcp://scadabridge@localhost:8082",
},
SplitBrainResolverStrategy = "keep-oldest",
MinNrOfMembers = 1,
StableAfter = TimeSpan.FromSeconds(15),
HeartbeatInterval = TimeSpan.FromSeconds(2),
FailureDetectionThreshold = TimeSpan.FromSeconds(10),
};
/// <summary>Verifies that the cluster leave phase timeout exceeds the drain budget.</summary>
[Fact]
public void BuildHocon_ClusterLeavePhaseTimeout_ExceedsDrainBudget()
{
// Review 01 [Medium]: the five singleton drain tasks GracefulStop(10s)
// inside cluster-leave, whose default phase timeout is 5s — Akka abandons
// the task at 5s and the drain guarantee is silently halved.
var node = new NodeOptions { Role = "Central", NodeHostname = "node1", RemotingPort = 8081 };
var hocon = AkkaHostedService.BuildHocon(
node, DefaultCluster(), new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
var phaseTimeout = config.GetTimeSpan("akka.coordinated-shutdown.phases.cluster-leave.timeout");
Assert.True(phaseTimeout >= TimeSpan.FromSeconds(15),
$"cluster-leave phase timeout ({phaseTimeout}) must exceed the 10s singleton GracefulStop budget");
}
/// <summary>Verifies that plain values produce a parseable document that preserves them.</summary>
[Fact]
public void BuildHocon_PlainValues_ParsesAndPreservesValues()
{
var node = new NodeOptions
{
Role = "Central",
NodeHostname = "central-node1",
RemotingPort = 8081,
};
var hocon = AkkaHostedService.BuildHocon(
node, DefaultCluster(), new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal("central-node1", config.GetString("akka.remote.dot-netty.tcp.hostname"));
Assert.Equal("keep-oldest", config.GetString("akka.cluster.split-brain-resolver.active-strategy"));
}
/// <summary>Verifies that a hostname containing a quote does not corrupt the HOCON document.</summary>
[Fact]
public void BuildHocon_HostnameWithQuote_DoesNotCorruptDocument()
{
// A hostname containing a double quote would, with raw interpolation, close
// the HOCON string literal early and corrupt every following key.
var node = new NodeOptions
{
Role = "Central",
NodeHostname = "evil\"host",
RemotingPort = 8081,
};
var hocon = AkkaHostedService.BuildHocon(
node, DefaultCluster(), new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
// Must still parse, and the keys after the hostname must remain intact.
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal("evil\"host", config.GetString("akka.remote.dot-netty.tcp.hostname"));
Assert.Equal(8081, config.GetInt("akka.remote.dot-netty.tcp.port"));
Assert.Equal(1, config.GetInt("akka.cluster.min-nr-of-members"));
}
/// <summary>Verifies that a strategy value containing whitespace remains quoted.</summary>
[Fact]
public void BuildHocon_StrategyWithWhitespace_RemainsQuoted()
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "keep oldest";
var node = new NodeOptions
{
Role = "Central",
NodeHostname = "node1",
RemotingPort = 8081,
};
var hocon = AkkaHostedService.BuildHocon(
node, cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal("keep oldest", config.GetString("akka.cluster.split-brain-resolver.active-strategy"));
}
private static NodeOptions DefaultNode() => new()
{
Role = "Central",
NodeHostname = "node1",
RemotingPort = 8081,
};
/// <summary>Verifies that setting DownIfAlone to true emits the on token.</summary>
[Fact]
public void BuildHocon_DownIfAloneTrue_EmitsOn()
{
// BuildHocon must consume ClusterOptions.DownIfAlone, not
// hard-code the down-if-alone token.
var cluster = DefaultCluster();
cluster.DownIfAlone = true;
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.True(config.GetBoolean(
"akka.cluster.split-brain-resolver.keep-oldest.down-if-alone"));
}
/// <summary>Verifies that setting DownIfAlone to false emits the off token.</summary>
[Fact]
public void BuildHocon_DownIfAloneFalse_EmitsOff()
{
// Setting DownIfAlone=false must actually flip the emitted token.
var cluster = DefaultCluster();
cluster.DownIfAlone = false;
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.False(config.GetBoolean(
"akka.cluster.split-brain-resolver.keep-oldest.down-if-alone"));
}
/// <summary>Verifies that sub-second timings are preserved without rounding.</summary>
[Fact]
public void BuildHocon_SubSecondTimings_PreservedWithoutRounding()
{
// Cluster timing values below one second must not be rounded to
// whole seconds. A 750ms heartbeat must survive as 750ms, not collapse to
// 1s — and a 500ms value must not collapse to a degenerate 0s.
var cluster = DefaultCluster();
cluster.HeartbeatInterval = TimeSpan.FromMilliseconds(750);
cluster.FailureDetectionThreshold = TimeSpan.FromMilliseconds(2600);
cluster.StableAfter = TimeSpan.FromMilliseconds(500);
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromMilliseconds(2500), TimeSpan.FromMilliseconds(7500));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal(
TimeSpan.FromMilliseconds(750),
config.GetTimeSpan("akka.cluster.failure-detector.heartbeat-interval"));
Assert.Equal(
TimeSpan.FromMilliseconds(2600),
config.GetTimeSpan("akka.cluster.failure-detector.acceptable-heartbeat-pause"));
Assert.Equal(
TimeSpan.FromMilliseconds(500),
config.GetTimeSpan("akka.cluster.split-brain-resolver.stable-after"));
Assert.Equal(
TimeSpan.FromMilliseconds(2500),
config.GetTimeSpan("akka.remote.transport-failure-detector.heartbeat-interval"));
Assert.Equal(
TimeSpan.FromMilliseconds(7500),
config.GetTimeSpan("akka.remote.transport-failure-detector.acceptable-heartbeat-pause"));
}
/// <summary>Verifies that the document enables the split-brain resolver downing provider.</summary>
[Fact]
public void BuildHocon_EnablesSplitBrainResolverDowningProvider()
{
// Review 01 [Critical]: without downing-provider-class the entire
// split-brain-resolver section is inert (Akka default = NoDowning) and
// singletons never migrate on a hard crash.
var node = new NodeOptions { Role = "Central", NodeHostname = "node1", RemotingPort = 8081 };
var hocon = AkkaHostedService.BuildHocon(
node, DefaultCluster(), new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal(
"Akka.Cluster.SBR.SplitBrainResolverProvider, Akka.Cluster",
config.GetString("akka.cluster.downing-provider-class"));
}
/// <summary>Verifies that the auto-down strategy emits the auto-downing provider.</summary>
[Fact]
public void BuildHocon_AutoDownStrategy_EmitsAutoDowningProvider()
{
// Decision 2026-07-21 (availability over partition-safety): 'auto-down' must
// swap the downing provider to Akka's AutoDowning so the survivor downs a
// crashed peer — including a crashed OLDEST, which two-node keep-oldest
// cannot survive — after StableAfter.
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "auto-down";
cluster.StableAfter = TimeSpan.FromSeconds(15);
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal(
"Akka.Cluster.AutoDowning, Akka.Cluster",
config.GetString("akka.cluster.downing-provider-class"));
Assert.Equal(
TimeSpan.FromSeconds(15),
config.GetTimeSpan("akka.cluster.auto-down-unreachable-after"));
// The SBR section must NOT be active alongside AutoDowning.
Assert.False(config.HasPath("akka.cluster.split-brain-resolver.active-strategy"));
}
/// <summary>Verifies that the auto-down strategy name is case-insensitive and the document stays intact.</summary>
[Fact]
public void BuildHocon_AutoDownStrategy_IsCaseInsensitive_AndDocumentStaysIntact()
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "Auto-Down";
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal(
"Akka.Cluster.AutoDowning, Akka.Cluster",
config.GetString("akka.cluster.downing-provider-class"));
// Keys after the downing block must remain intact (document not corrupted).
Assert.Equal(1, config.GetInt("akka.cluster.min-nr-of-members"));
Assert.True(config.GetBoolean("akka.cluster.run-coordinated-shutdown-when-down"));
}
/// <summary>
/// Verifies that the leader-auto-down strategy (ScadaBridge#106) installs the in-repo
/// provider, carries StableAfter under its own key, and emits neither Akka's deprecated
/// auto-down key nor an SBR section.
/// </summary>
[Fact]
public void BuildHocon_LeaderAutoDownStrategy_EmitsInRepoProvider_AndNoDeprecatedKey()
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "leader-auto-down";
cluster.StableAfter = TimeSpan.FromMilliseconds(15250);
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
var config = ConfigurationFactory.ParseString(hocon);
Assert.Equal(
"ZB.MOM.WW.ScadaBridge.Host.Actors.LeaderAutoDowningProvider, ZB.MOM.WW.ScadaBridge.Host",
config.GetString("akka.cluster.downing-provider-class"));
// The emitted name must resolve the way Akka resolves it (Type.GetType(name, true)).
Assert.Equal(typeof(LeaderAutoDowningProvider),
Type.GetType(config.GetString("akka.cluster.downing-provider-class"), throwOnError: true));
Assert.Equal(
TimeSpan.FromMilliseconds(15250),
config.GetTimeSpan(LeaderAutoDowningProvider.UnreachableAfterPath));
// No deprecated key (that is what triggers Akka's "keep-majority will be used" warning)
// and no SBR section alongside the provider.
Assert.False(config.HasPath("akka.cluster.auto-down-unreachable-after"));
Assert.False(config.HasPath("akka.cluster.split-brain-resolver.active-strategy"));
// Document stays intact after the downing block.
Assert.Equal(1, config.GetInt("akka.cluster.min-nr-of-members"));
Assert.True(config.GetBoolean("akka.cluster.run-coordinated-shutdown-when-down"));
}
/// <summary>Verifies that the leader-auto-down strategy name is case-insensitive.</summary>
[Fact]
public void BuildHocon_LeaderAutoDownStrategy_IsCaseInsensitive()
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "Leader-Auto-Down";
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
Assert.Equal(LeaderAutoDowningProvider.ProviderClassName,
config.GetString("akka.cluster.downing-provider-class"));
Assert.Equal(TimeSpan.FromSeconds(15), config.GetTimeSpan(LeaderAutoDowningProvider.UnreachableAfterPath));
}
/// <summary>
/// Verifies the leader-auto-down key is emitted ONLY for that strategy: the existing
/// auto-down and keep-oldest documents are unchanged by ScadaBridge#106.
/// </summary>
/// <param name="strategy">An existing strategy.</param>
[Theory]
[InlineData("auto-down")]
[InlineData("keep-oldest")]
public void BuildHocon_OtherStrategies_DoNotEmitLeaderAutoDownKey(string strategy)
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = strategy;
var hocon = AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
Assert.DoesNotContain("leader-auto-down", hocon);
Assert.DoesNotContain(nameof(LeaderAutoDowningProvider), hocon);
}
/// <summary>
/// ScadaBridge#132: leader-auto-down carries the shut-out restart switch (default ON) and its
/// grace into the provider's HOCON block, and the provider reads back exactly those values.
/// </summary>
/// <param name="enabled">The configured switch.</param>
[Theory]
[InlineData(true)]
[InlineData(false)]
public void BuildHocon_LeaderAutoDown_EmitsShutOutRestartSettings(bool enabled)
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "leader-auto-down";
cluster.LeaderAutoDown = new LeaderAutoDownOptions
{
RestartWhenShutOut = enabled,
ShutOutGrace = TimeSpan.FromMilliseconds(45500),
};
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
var settings = ShutOutRestartSettings.FromConfig(config);
Assert.Equal(enabled, settings.Enabled);
Assert.Equal(TimeSpan.FromMilliseconds(45500), settings.Grace);
}
/// <summary>ScadaBridge#132: the shut-out restart is ON by default when leader-auto-down is selected.</summary>
[Fact]
public void BuildHocon_LeaderAutoDown_ShutOutRestartOnByDefault()
{
var cluster = DefaultCluster();
cluster.SplitBrainResolverStrategy = "leader-auto-down";
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
Assert.True(config.GetBoolean(ShutOutRestartSettings.EnabledPath));
Assert.Equal(TimeSpan.FromSeconds(60), config.GetTimeSpan(ShutOutRestartSettings.GracePath));
}
/// <summary>
/// ScadaBridge#116: an environment that sets no strategy gets <c>leader-auto-down</c> with
/// <c>LeaderAutoDown</c> at its defaults — the in-repo provider, the StableAfter window under its
/// own key, the shut-out restart ON, and no deprecated <c>auto-down-unreachable-after</c> key.
/// </summary>
[Fact]
public void BuildHocon_UnsetStrategy_DefaultsToLeaderAutoDown()
{
var cluster = new ClusterOptions { SeedNodes = DefaultCluster().SeedNodes };
var config = ConfigurationFactory.ParseString(AkkaHostedService.BuildHocon(
DefaultNode(), cluster, new[] { "Central" },
TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)));
Assert.Equal(LeaderAutoDowningProvider.ProviderClassName,
config.GetString("akka.cluster.downing-provider-class"));
Assert.Equal(TimeSpan.FromSeconds(15), config.GetTimeSpan(LeaderAutoDowningProvider.UnreachableAfterPath));
Assert.True(config.GetBoolean(ShutOutRestartSettings.EnabledPath));
Assert.Equal(TimeSpan.FromSeconds(60), config.GetTimeSpan(ShutOutRestartSettings.GracePath));
Assert.False(config.HasPath("akka.cluster.auto-down-unreachable-after"));
Assert.False(config.HasPath("akka.cluster.split-brain-resolver.active-strategy"));
}
}