diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf index 154de1e8..b7e7a8a7 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Cluster/Resources/akka.conf @@ -15,6 +15,12 @@ akka { extensions = [ "Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools" + + # Per-cluster mesh Phase 2 — the central<->node command boundary. NOT auto-started: without + # this entry the receptionist materialises only if some code path happens to call + # ClusterClientReceptionist.Get(system) first, which makes "is the boundary listening?" + # depend on startup ordering rather than on configuration. + "Akka.Cluster.Tools.Client.ClusterClientReceptionistExtensionProvider, Akka.Cluster.Tools" ] actor { @@ -25,6 +31,20 @@ akka { dot-netty.tcp { hostname = "0.0.0.0" port = 4053 + + # FRAME SIZE. maximum-frame-size restates Akka's own default; what actually changes is + # log-frame-size-exceeding, which defaults to `off`. Over the limit the transport drops + # that ONE message without tearing down the association — heartbeats keep flowing, the + # node still reports healthy, and the only symptom is a command that never arrived. The + # sister project lost a deploy path to exactly this and could not attribute it; their + # postmortem also notes the default JSON serializer escapes an already-serialized + # payload a second time, so ~60-70 KB of real JSON is enough to blow 128 KB. + # + # OtOpcUa's deploy notify is payload-free by design (DispatchDeployment carries only + # DeploymentId + RevisionHash + CorrelationId), so this threshold should stay silent. + # It is a canary, and a quiet one is the point. + maximum-frame-size = 128000b + log-frame-size-exceeding = 32000b } transport-failure-detector { heartbeat-interval = 2s @@ -74,6 +94,31 @@ akka { down-removal-margin = 15s run-coordinated-shutdown-when-down = on + # CLUSTERCLIENT — the central<->node command boundary (per-cluster mesh Phase 2). + client { + # DELIBERATE, and NOT what the sister project runs: they never wrote this section, so + # they inherit buffer-size = 1000. Zero means "if the receptionist is unknown, drop the + # message now". Buffering would replay a DispatchDeployment on reconnect and apply a + # deployment ConfigPublishCoordinator had already sealed as TimedOut — the node would + # then be running a configuration the database says failed. Fail-fast keeps the DB the + # single record of truth, and reproduces today's DPS behaviour exactly: a node that is + # down misses the publish and self-heals from the DB on restart. + buffer-size = 0 + + # Retry forever rather than self-stopping the client. Restates Akka's default; stated + # explicitly because the behaviour is load-bearing — a central node down for an hour + # must not require a driver-node restart to become reachable again. + reconnect-timeout = off + + receptionist { + # Empty = every member hosts a receptionist. This is what makes the comm actors + # "registered per node, not as a singleton" work: contact rotation reaches whichever + # node answers. Restates Akka's default; do NOT scope this to a role, or the + # boundary becomes reachable through one node only and rotation silently fails. + role = "" + } + } + singleton { singleton-name = "singleton" } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs new file mode 100644 index 00000000..1960d0c3 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Cluster.Tests/MeshTransportHoconTests.cs @@ -0,0 +1,100 @@ +using Akka.Actor; +using Akka.Cluster.Tools.Client; +using Akka.Configuration; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests; + +/// +/// Asserts the ClusterClient / receptionist / frame-size settings that are actually in +/// force on a real built from the shipped base config. +/// +/// +/// Asserting the akka.conf text instead would pass while a competing HOCON fragment +/// silently overrode the value — the failure mode recorded at length in +/// ServiceCollectionExtensions.BuildDowningHocon and pinned the same way by +/// SplitBrainResolverActivationTests. In this codebase the effective value is the only +/// one worth asserting. +/// +public class MeshTransportHoconTests : IDisposable +{ + private readonly ActorSystem _sys; + + public MeshTransportHoconTests() + { + var config = ConfigurationFactory + .ParseString("akka.remote.dot-netty.tcp.port = 0\nakka.cluster.seed-nodes = []") + .WithFallback(HoconLoader.LoadBaseConfig()); + _sys = ActorSystem.Create("mesh-hocon-probe", config); + } + + [Fact] + public void Cluster_client_buffering_is_disabled() + { + // Asserted through ClusterClientSettings — the object the client is actually built from — + // NOT through Config.GetInt. GetInt returns 0 for a MISSING key, so a config assertion here + // passes identically whether the setting is explicitly 0 or absent with Akka's default of + // 1000 in force. It was green before this feature existed; that is the whole defect class. + // + // buffer-size = 0 means "drop now if the receptionist is unknown". Buffering would replay a + // DispatchDeployment on reconnect and apply a deployment the coordinator already sealed as + // TimedOut, leaving the node running a configuration the database says failed. This is a + // behaviour decision, not a tuning knob. + ClusterClientSettings.Create(_sys).BufferSize.ShouldBe(0); + } + + [Fact] + public void Cluster_client_retries_forever_rather_than_self_stopping() + { + // reconnect-timeout = off surfaces as a null TimeSpan on the settings object. A central node + // down for an hour must not require a driver-node restart to become reachable again. + ClusterClientSettings.Create(_sys).ReconnectTimeout.ShouldBeNull(); + } + + [Fact] + public void Receptionist_serves_every_member_regardless_of_role() + { + // An empty role is what makes "registered per node, not as a singleton" work: every member + // hosts a receptionist, so ClusterClient contact rotation reaches whichever node answers. + // Scoping this to a role would make the boundary reachable through one node only. + ClusterReceptionistSettings.Create(_sys).Role.ShouldBeNull(); + } + + [Fact] + public void Oversized_frames_are_logged_rather_than_dropped_silently() + { + // The sister project's most expensive incident. A message over the frame limit is dropped + // WITHOUT tearing down the association: heartbeats keep flowing, the node still reports + // healthy, and the only symptom is a command that never arrived. log-frame-size-exceeding + // converts that silence into a log line naming the offending size. + _sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.maximum-frame-size").ShouldBe(128000); + _sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.log-frame-size-exceeding").ShouldBe(32000); + } + + [Fact] + public void Receptionist_extension_is_registered_at_startup() + { + // Not auto-started. Without this entry the receptionist materialises only if some code path + // happens to call ClusterClientReceptionist.Get(system) first, which makes "is the boundary + // listening?" depend on startup ordering rather than on configuration. + _sys.Settings.Config.GetStringList("akka.extensions") + .ShouldContain(s => s.Contains("ClusterClientReceptionistExtensionProvider")); + } + + [Fact] + public void Distributed_pub_sub_extension_survives_alongside_the_receptionist() + { + // The dark switch keeps both transports live for one phase, and DPS additionally carries + // traffic Phase 2 does not touch at all — node-local alarm-command publishes and the + // node<->node redundancy-state probe leg. Losing it here would break far more than deploys. + _sys.Settings.Config.GetStringList("akka.extensions") + .ShouldContain(s => s.Contains("DistributedPubSubExtensionProvider")); + } + + public void Dispose() + { + _sys.Terminate().Wait(TimeSpan.FromSeconds(5)); + GC.SuppressFinalize(this); + } +}