feat(mesh): receptionist extension, zero ClusterClient buffering, frame-size logging
Phase 2 Task 1. Three changes to the embedded base config, plus a test that reads every one of them back off a RUNNING ActorSystem rather than off the HOCON text -- the settled idiom here (SplitBrainResolverActivationTests), because several fragments compete and the losing one still reads correctly in the file. - Registers the ClusterClientReceptionist extension. It is not auto-started, so without this the boundary only listens if some code path happens to call ClusterClientReceptionist.Get(system) first: "is the boundary up?" would depend on startup ordering rather than on configuration. - buffer-size = 0. This is the one genuine behaviour change and it is NOT what the sister project runs -- they never wrote this section and inherit 1000. Buffering would replay a DispatchDeployment on reconnect and apply a deployment the coordinator already sealed as TimedOut, leaving the node on a configuration the database says failed. - log-frame-size-exceeding = 32000b. An oversized frame is dropped WITHOUT tearing down the association: heartbeats keep flowing, the node reports healthy, and the only symptom is a command that never arrived. Our deploy notify is payload-free, so this canary should stay quiet. reconnect-timeout and receptionist.role restate Akka defaults; they are stated explicitly and pinned because the behaviour is load-bearing, and the comments say which is which. One test was VACUOUS on first write: Config.GetInt returns 0 for a missing key, so the buffering assertion passed before the feature existed -- it could not distinguish "explicitly 0" from "absent, default 1000 in force". Rewritten to assert through ClusterClientSettings/ClusterReceptionistSettings, the objects the client is actually built from. Sabotage-verified after the fix: setting buffer-size back to 1000 reddens exactly that one test. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
This commit is contained in:
@@ -15,6 +15,12 @@ akka {
|
||||
|
||||
extensions = [
|
||||
"Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools"
|
||||
|
||||
# Per-cluster mesh Phase 2 — the central<->node command boundary. NOT auto-started: without
|
||||
# this entry the receptionist materialises only if some code path happens to call
|
||||
# ClusterClientReceptionist.Get(system) first, which makes "is the boundary listening?"
|
||||
# depend on startup ordering rather than on configuration.
|
||||
"Akka.Cluster.Tools.Client.ClusterClientReceptionistExtensionProvider, Akka.Cluster.Tools"
|
||||
]
|
||||
|
||||
actor {
|
||||
@@ -25,6 +31,20 @@ akka {
|
||||
dot-netty.tcp {
|
||||
hostname = "0.0.0.0"
|
||||
port = 4053
|
||||
|
||||
# FRAME SIZE. maximum-frame-size restates Akka's own default; what actually changes is
|
||||
# log-frame-size-exceeding, which defaults to `off`. Over the limit the transport drops
|
||||
# that ONE message without tearing down the association — heartbeats keep flowing, the
|
||||
# node still reports healthy, and the only symptom is a command that never arrived. The
|
||||
# sister project lost a deploy path to exactly this and could not attribute it; their
|
||||
# postmortem also notes the default JSON serializer escapes an already-serialized
|
||||
# payload a second time, so ~60-70 KB of real JSON is enough to blow 128 KB.
|
||||
#
|
||||
# OtOpcUa's deploy notify is payload-free by design (DispatchDeployment carries only
|
||||
# DeploymentId + RevisionHash + CorrelationId), so this threshold should stay silent.
|
||||
# It is a canary, and a quiet one is the point.
|
||||
maximum-frame-size = 128000b
|
||||
log-frame-size-exceeding = 32000b
|
||||
}
|
||||
transport-failure-detector {
|
||||
heartbeat-interval = 2s
|
||||
@@ -74,6 +94,31 @@ akka {
|
||||
down-removal-margin = 15s
|
||||
run-coordinated-shutdown-when-down = on
|
||||
|
||||
# CLUSTERCLIENT — the central<->node command boundary (per-cluster mesh Phase 2).
|
||||
client {
|
||||
# DELIBERATE, and NOT what the sister project runs: they never wrote this section, so
|
||||
# they inherit buffer-size = 1000. Zero means "if the receptionist is unknown, drop the
|
||||
# message now". Buffering would replay a DispatchDeployment on reconnect and apply a
|
||||
# deployment ConfigPublishCoordinator had already sealed as TimedOut — the node would
|
||||
# then be running a configuration the database says failed. Fail-fast keeps the DB the
|
||||
# single record of truth, and reproduces today's DPS behaviour exactly: a node that is
|
||||
# down misses the publish and self-heals from the DB on restart.
|
||||
buffer-size = 0
|
||||
|
||||
# Retry forever rather than self-stopping the client. Restates Akka's default; stated
|
||||
# explicitly because the behaviour is load-bearing — a central node down for an hour
|
||||
# must not require a driver-node restart to become reachable again.
|
||||
reconnect-timeout = off
|
||||
|
||||
receptionist {
|
||||
# Empty = every member hosts a receptionist. This is what makes the comm actors
|
||||
# "registered per node, not as a singleton" work: contact rotation reaches whichever
|
||||
# node answers. Restates Akka's default; do NOT scope this to a role, or the
|
||||
# boundary becomes reachable through one node only and rotation silently fails.
|
||||
role = ""
|
||||
}
|
||||
}
|
||||
|
||||
singleton {
|
||||
singleton-name = "singleton"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.Client;
|
||||
using Akka.Configuration;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Cluster.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Asserts the ClusterClient / receptionist / frame-size settings that are actually <b>in
|
||||
/// force</b> on a real <see cref="ActorSystem"/> built from the shipped base config.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Asserting the <c>akka.conf</c> text instead would pass while a competing HOCON fragment
|
||||
/// silently overrode the value — the failure mode recorded at length in
|
||||
/// <c>ServiceCollectionExtensions.BuildDowningHocon</c> and pinned the same way by
|
||||
/// <c>SplitBrainResolverActivationTests</c>. In this codebase the effective value is the only
|
||||
/// one worth asserting.
|
||||
/// </remarks>
|
||||
public class MeshTransportHoconTests : IDisposable
|
||||
{
|
||||
private readonly ActorSystem _sys;
|
||||
|
||||
public MeshTransportHoconTests()
|
||||
{
|
||||
var config = ConfigurationFactory
|
||||
.ParseString("akka.remote.dot-netty.tcp.port = 0\nakka.cluster.seed-nodes = []")
|
||||
.WithFallback(HoconLoader.LoadBaseConfig());
|
||||
_sys = ActorSystem.Create("mesh-hocon-probe", config);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cluster_client_buffering_is_disabled()
|
||||
{
|
||||
// Asserted through ClusterClientSettings — the object the client is actually built from —
|
||||
// NOT through Config.GetInt. GetInt returns 0 for a MISSING key, so a config assertion here
|
||||
// passes identically whether the setting is explicitly 0 or absent with Akka's default of
|
||||
// 1000 in force. It was green before this feature existed; that is the whole defect class.
|
||||
//
|
||||
// buffer-size = 0 means "drop now if the receptionist is unknown". Buffering would replay a
|
||||
// DispatchDeployment on reconnect and apply a deployment the coordinator already sealed as
|
||||
// TimedOut, leaving the node running a configuration the database says failed. This is a
|
||||
// behaviour decision, not a tuning knob.
|
||||
ClusterClientSettings.Create(_sys).BufferSize.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cluster_client_retries_forever_rather_than_self_stopping()
|
||||
{
|
||||
// reconnect-timeout = off surfaces as a null TimeSpan on the settings object. A central node
|
||||
// down for an hour must not require a driver-node restart to become reachable again.
|
||||
ClusterClientSettings.Create(_sys).ReconnectTimeout.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Receptionist_serves_every_member_regardless_of_role()
|
||||
{
|
||||
// An empty role is what makes "registered per node, not as a singleton" work: every member
|
||||
// hosts a receptionist, so ClusterClient contact rotation reaches whichever node answers.
|
||||
// Scoping this to a role would make the boundary reachable through one node only.
|
||||
ClusterReceptionistSettings.Create(_sys).Role.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Oversized_frames_are_logged_rather_than_dropped_silently()
|
||||
{
|
||||
// The sister project's most expensive incident. A message over the frame limit is dropped
|
||||
// WITHOUT tearing down the association: heartbeats keep flowing, the node still reports
|
||||
// healthy, and the only symptom is a command that never arrived. log-frame-size-exceeding
|
||||
// converts that silence into a log line naming the offending size.
|
||||
_sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.maximum-frame-size").ShouldBe(128000);
|
||||
_sys.Settings.Config.GetByteSize("akka.remote.dot-netty.tcp.log-frame-size-exceeding").ShouldBe(32000);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Receptionist_extension_is_registered_at_startup()
|
||||
{
|
||||
// Not auto-started. Without this entry the receptionist materialises only if some code path
|
||||
// happens to call ClusterClientReceptionist.Get(system) first, which makes "is the boundary
|
||||
// listening?" depend on startup ordering rather than on configuration.
|
||||
_sys.Settings.Config.GetStringList("akka.extensions")
|
||||
.ShouldContain(s => s.Contains("ClusterClientReceptionistExtensionProvider"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Distributed_pub_sub_extension_survives_alongside_the_receptionist()
|
||||
{
|
||||
// The dark switch keeps both transports live for one phase, and DPS additionally carries
|
||||
// traffic Phase 2 does not touch at all — node-local alarm-command publishes and the
|
||||
// node<->node redundancy-state probe leg. Losing it here would break far more than deploys.
|
||||
_sys.Settings.Config.GetStringList("akka.extensions")
|
||||
.ShouldContain(s => s.Contains("DistributedPubSubExtensionProvider"));
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_sys.Terminate().Wait(TimeSpan.FromSeconds(5));
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user