fix(redundancy): a failed OPC UA endpoint drops ServiceLevel to 0 and steps the Primary down (lmxopcua#762)
v2-ci / unit-tests (push) Has been cancelled

IOpcUaEndpointState (review-remediation lane C3) recorded whether the SDK started, but only the
health check read it: a fused node whose endpoint failed (port 4840 taken, ...) still advertised
250/240 and stayed driver Primary, so the Primary-gated data plane ran on a node no client could
reach while the reachable peer refused writes as Secondary.

- Move IOpcUaEndpointState/OpcUaEndpointState/OpcUaEndpointStatus from Host to Commons.OpcUa so
  the Runtime can consume it.
- ServiceLevelCalculator: new NodeHealthInputs.EndpointListening (default true). false -> 0,
  ahead of every other input including the +10 Primary bonus.
- OpcUaPublishActor: takes the endpoint state, feeds EndpointListening (Failed only; NotStarted /
  Stopped are boot/shutdown windows) on the DB-backed and DB-less paths, and the legacy role-only
  seam returns 0 for a failed endpoint too.
- New OpcUaEndpointStepDownActor (one per driver node): a Primary (ClusterActiveNode.SelfIsActive,
  the election's own primitive) with a Failed endpoint gracefully Leaves its mesh - the manual
  failover mechanism - so the peer becomes the oldest Up driver member, hosts the singletons and is
  Primary. The election itself is not bent. Guarded on a peer having recently reported its own
  endpoint listening (EndpointReport on the new opcua-endpoint-state DPS topic), so a pair with
  both endpoints failed does not bounce the role and restart nodes forever, and a lone node never
  shuts itself down.
- Runtime now references ZB.MOM.WW.Health.Akka (ClusterActiveNode).
- Docs: Redundancy.md tier table + new "Endpoint start failure: Primary step-down" section;
  cluster-topology rule; HealthEndpoints / OpcUaEndpointHealthCheck comments (F-C1 landed).

Tests: ServiceLevelCalculatorTests (+5), OpcUaPublishActorEndpointStateTests (7: DB-less,
DB-backed, legacy seam, NotStarted/Stopped do not demote, a later failure demotes on the next
tick), OpcUaEndpointStepDownActorTests (3, real two-node mesh: holds while the peer is not
listening then leaves and the peer becomes Primary; failed Secondary and healthy Primary never
leave). Each was mutation-checked: removing the calculator/legacy guard, the Leave, the
peer-listening guard or the Primary guard fails the corresponding tests.
This commit is contained in:
Joseph Doherty
2026-11-09 12:15:59 -05:00
parent 8ef5adc63a
commit acfb4fbaca
13 changed files with 668 additions and 30 deletions
@@ -241,6 +241,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// <c>DbReachable</c> is fed constant <c>true</c> and only snapshot staleness can demote. See
/// <see cref="RecomputeServiceLevel"/>.</summary>
private readonly bool _dbLess;
/// <summary>This node's OPC UA endpoint lifecycle fact (lmxopcua#762). Null = no fact (harness) = listening.</summary>
private readonly IOpcUaEndpointState? _endpointState;
private readonly TimeSpan _staleWindow;
private readonly TimeSpan _probeFreshnessWindow;
private readonly TimeSpan _healthTickInterval;
@@ -306,6 +308,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
/// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
/// <param name="dataPlaneAuthority">The optional sink that receives the API keys and ACL grants carried by each applied artifact.</param>
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762). A
/// <see cref="OpcUaEndpointStatus.Failed"/> endpoint computes ServiceLevel 0. Null (harnesses, a node with no
/// hosted SDK) is treated as listening.</param>
/// <returns>The configured <see cref="Props"/> for creating this actor, pinned to the
/// <see cref="DispatcherId"/> dispatcher.</returns>
public static Props Props(
@@ -320,7 +325,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
TimeSpan? healthTickInterval = null,
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null) =>
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -334,7 +340,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
healthTickInterval,
dbLess,
dataPlaneAuthority,
utcNow)).WithDispatcher(DispatcherId);
utcNow,
endpointState)).WithDispatcher(DispatcherId);
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
@@ -359,6 +366,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
/// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
/// <param name="dataPlaneAuthority">The optional sink that receives the API keys and ACL grants carried by each applied artifact.</param>
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762). A
/// <see cref="OpcUaEndpointStatus.Failed"/> endpoint computes ServiceLevel 0. Null (harnesses, a node with no
/// hosted SDK) is treated as listening.</param>
/// <returns>The configured <see cref="Props"/> for creating this actor in tests, without dispatcher
/// pinning or the DPS subscribe.</returns>
public static Props PropsForTests(
@@ -374,7 +384,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
TimeSpan? healthTickInterval = null,
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null) =>
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null) =>
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
sink ?? NullOpcUaAddressSpaceSink.Instance,
serviceLevel ?? NullServiceLevelPublisher.Instance,
@@ -388,7 +399,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
healthTickInterval,
dbLess,
dataPlaneAuthority,
utcNow));
utcNow,
endpointState));
/// <summary>Test-only Props exposing the deferred-write bound and the plan-step seams.</summary>
internal static Props PropsForTests(
@@ -402,7 +414,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
// seam-carrying constructor is internal so the test seams stay off the public API (review M3).
Akka.Actor.Props.CreateBy(new TestProducer(() => new OpcUaPublishActor(
sink, NullServiceLevelPublisher.Instance, false, localNode, null, applier, null, null, null, null,
false, dataPlaneAuthority, null, deferredWriteCapacity, hooks)));
false, dataPlaneAuthority, null, deferredWriteCapacity, hooks, endpointState: null)));
/// <summary>Produces the actor through its internal (seam-carrying) constructor, for <see cref="PropsForTests(IOpcUaAddressSpaceSink, AddressSpaceApplier?, TestHooks?, int, NodeId?, IDataPlaneAuthoritySink?)"/>.</summary>
private sealed class TestProducer(Func<OpcUaPublishActor> create) : IIndirectActorProducer
@@ -434,6 +446,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
/// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
/// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
/// <param name="dataPlaneAuthority">The optional sink that receives the API keys and ACL grants carried by each applied artifact.</param>
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762); null is treated as listening.</param>
public OpcUaPublishActor(
IOpcUaAddressSpaceSink sink,
IServiceLevelPublisher serviceLevel,
@@ -447,15 +460,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
TimeSpan? healthTickInterval = null,
bool dbLess = false,
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
Func<DateTime>? utcNow = null)
Func<DateTime>? utcNow = null,
IOpcUaEndpointState? endpointState = null)
: this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow,
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null)
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null,
endpointState)
{
}
/// <summary>The full constructor: also the deferred-write bound and the test seams (internal — tests only).</summary>
/// <param name="deferredWriteCapacity">The bound on distinct nodes whose writes are held for replay while a rebuild plan is in flight.</param>
/// <param name="hooks">Test seams for the off-thread plan step; null in production.</param>
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762); null is treated as listening.</param>
internal OpcUaPublishActor(
IOpcUaAddressSpaceSink sink,
IServiceLevelPublisher serviceLevel,
@@ -471,7 +487,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
IDataPlaneAuthoritySink? dataPlaneAuthority,
Func<DateTime>? utcNow,
int deferredWriteCapacity,
TestHooks? hooks)
TestHooks? hooks,
IOpcUaEndpointState? endpointState)
{
_deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity));
_hooks = hooks;
@@ -486,6 +503,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
_applier = applier;
_dbHealthProbe = dbHealthProbe;
_dbLess = dbLess;
_endpointState = endpointState;
_staleWindow = staleWindow ?? TimeSpan.FromSeconds(30);
_probeFreshnessWindow = probeFreshnessWindow ?? TimeSpan.FromSeconds(30);
_healthTickInterval = healthTickInterval ?? TimeSpan.FromSeconds(5);
@@ -1536,6 +1554,15 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
return verdict.Ok;
}
/// <summary>The OPC UA endpoint input for the calculator (lmxopcua#762): <c>false</c> only when this node's own
/// SDK start is recorded as <see cref="OpcUaEndpointStatus.Failed"/>. <see cref="OpcUaEndpointStatus.NotStarted"/>
/// and <see cref="OpcUaEndpointStatus.Stopped"/> are boot and shutdown windows, not faults, and a node with no
/// endpoint fact at all (a harness) is treated as listening. Read on every recompute — the periodic health tick
/// and every redundancy snapshot (heartbeated every 10 s) — so a change is picked up without a notification.
/// It is a LOCAL fact on purpose: the peer TCP probe behind <see cref="OpcUaProbeOk"/> cannot see a failed start
/// when another process holds the port, because its connect then succeeds against that process.</summary>
private bool EndpointListening() => _endpointState?.Status != OpcUaEndpointStatus.Failed;
/// <summary>
/// Computes the local OPC UA ServiceLevel and routes it through <see cref="ServiceLevelChanged"/>
/// (the dedup/publish/metric handler). The full <see cref="ServiceLevelCalculator"/> path is
@@ -1575,7 +1602,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
DbReachable: true,
OpcUaProbeOk: OpcUaProbeOk(),
Stale: (utcNow - entry.AsOfUtc) > _staleWindow,
IsDriverPrimary: entry.IsDriverPrimary);
IsDriverPrimary: entry.IsDriverPrimary,
EndpointListening: EndpointListening());
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(dbLessInputs)));
return;
}
@@ -1585,7 +1613,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
// is the bootstrap value until the first DbHealthStatus lands.
if (_dbHealthProbe is null || _lastDbHealth is null)
{
Self.Tell(new ServiceLevelChanged(LegacyRoleOnly(entry)));
// The role-only seam has no health inputs, but a failed endpoint still wins (lmxopcua#762): the
// legacy switch would otherwise hand a node that serves nothing 240 as Primary.
Self.Tell(new ServiceLevelChanged(EndpointListening() ? LegacyRoleOnly(entry) : (byte)0));
return;
}
@@ -1597,7 +1627,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
Stale: !_lastDbHealth.Reachable
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|| (now - entry.AsOfUtc) > _staleWindow,
IsDriverPrimary: entry.IsDriverPrimary);
IsDriverPrimary: entry.IsDriverPrimary,
EndpointListening: EndpointListening());
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
}