fix(redundancy): a failed OPC UA endpoint drops ServiceLevel to 0 and steps the Primary down (lmxopcua#762)
v2-ci / unit-tests (push) Has been cancelled
v2-ci / unit-tests (push) Has been cancelled
IOpcUaEndpointState (review-remediation lane C3) recorded whether the SDK started, but only the health check read it: a fused node whose endpoint failed (port 4840 taken, ...) still advertised 250/240 and stayed driver Primary, so the Primary-gated data plane ran on a node no client could reach while the reachable peer refused writes as Secondary. - Move IOpcUaEndpointState/OpcUaEndpointState/OpcUaEndpointStatus from Host to Commons.OpcUa so the Runtime can consume it. - ServiceLevelCalculator: new NodeHealthInputs.EndpointListening (default true). false -> 0, ahead of every other input including the +10 Primary bonus. - OpcUaPublishActor: takes the endpoint state, feeds EndpointListening (Failed only; NotStarted / Stopped are boot/shutdown windows) on the DB-backed and DB-less paths, and the legacy role-only seam returns 0 for a failed endpoint too. - New OpcUaEndpointStepDownActor (one per driver node): a Primary (ClusterActiveNode.SelfIsActive, the election's own primitive) with a Failed endpoint gracefully Leaves its mesh - the manual failover mechanism - so the peer becomes the oldest Up driver member, hosts the singletons and is Primary. The election itself is not bent. Guarded on a peer having recently reported its own endpoint listening (EndpointReport on the new opcua-endpoint-state DPS topic), so a pair with both endpoints failed does not bounce the role and restart nodes forever, and a lone node never shuts itself down. - Runtime now references ZB.MOM.WW.Health.Akka (ClusterActiveNode). - Docs: Redundancy.md tier table + new "Endpoint start failure: Primary step-down" section; cluster-topology rule; HealthEndpoints / OpcUaEndpointHealthCheck comments (F-C1 landed). Tests: ServiceLevelCalculatorTests (+5), OpcUaPublishActorEndpointStateTests (7: DB-less, DB-backed, legacy seam, NotStarted/Stopped do not demote, a later failure demotes on the next tick), OpcUaEndpointStepDownActorTests (3, real two-node mesh: holds while the peer is not listening then leaves and the peer becomes Primary; failed Secondary and healthy Primary never leave). Each was mutation-checked: removing the calculator/legacy guard, the Leave, the peer-listening guard or the Primary guard fails the corresponding tests.
This commit is contained in:
@@ -241,6 +241,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// <c>DbReachable</c> is fed constant <c>true</c> and only snapshot staleness can demote. See
|
||||
/// <see cref="RecomputeServiceLevel"/>.</summary>
|
||||
private readonly bool _dbLess;
|
||||
/// <summary>This node's OPC UA endpoint lifecycle fact (lmxopcua#762). Null = no fact (harness) = listening.</summary>
|
||||
private readonly IOpcUaEndpointState? _endpointState;
|
||||
private readonly TimeSpan _staleWindow;
|
||||
private readonly TimeSpan _probeFreshnessWindow;
|
||||
private readonly TimeSpan _healthTickInterval;
|
||||
@@ -306,6 +308,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
|
||||
/// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
|
||||
/// <param name="dataPlaneAuthority">The optional sink that receives the API keys and ACL grants carried by each applied artifact.</param>
|
||||
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762). A
|
||||
/// <see cref="OpcUaEndpointStatus.Failed"/> endpoint computes ServiceLevel 0. Null (harnesses, a node with no
|
||||
/// hosted SDK) is treated as listening.</param>
|
||||
/// <returns>The configured <see cref="Props"/> for creating this actor, pinned to the
|
||||
/// <see cref="DispatcherId"/> dispatcher.</returns>
|
||||
public static Props Props(
|
||||
@@ -320,7 +325,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
TimeSpan? healthTickInterval = null,
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null) =>
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
@@ -334,7 +340,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
healthTickInterval,
|
||||
dbLess,
|
||||
dataPlaneAuthority,
|
||||
utcNow)).WithDispatcher(DispatcherId);
|
||||
utcNow,
|
||||
endpointState)).WithDispatcher(DispatcherId);
|
||||
|
||||
/// <summary>Test-only Props that omits the pinned-dispatcher requirement and skips the
|
||||
/// DPS subscribe so unit tests can spin up the actor on a vanilla TestKit cluster.</summary>
|
||||
@@ -359,6 +366,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
|
||||
/// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
|
||||
/// <param name="dataPlaneAuthority">The optional sink that receives the API keys and ACL grants carried by each applied artifact.</param>
|
||||
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762). A
|
||||
/// <see cref="OpcUaEndpointStatus.Failed"/> endpoint computes ServiceLevel 0. Null (harnesses, a node with no
|
||||
/// hosted SDK) is treated as listening.</param>
|
||||
/// <returns>The configured <see cref="Props"/> for creating this actor in tests, without dispatcher
|
||||
/// pinning or the DPS subscribe.</returns>
|
||||
public static Props PropsForTests(
|
||||
@@ -374,7 +384,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
TimeSpan? healthTickInterval = null,
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null) =>
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null) =>
|
||||
Akka.Actor.Props.Create(() => new OpcUaPublishActor(
|
||||
sink ?? NullOpcUaAddressSpaceSink.Instance,
|
||||
serviceLevel ?? NullServiceLevelPublisher.Instance,
|
||||
@@ -388,7 +399,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
healthTickInterval,
|
||||
dbLess,
|
||||
dataPlaneAuthority,
|
||||
utcNow));
|
||||
utcNow,
|
||||
endpointState));
|
||||
|
||||
/// <summary>Test-only Props exposing the deferred-write bound and the plan-step seams.</summary>
|
||||
internal static Props PropsForTests(
|
||||
@@ -402,7 +414,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
// seam-carrying constructor is internal so the test seams stay off the public API (review M3).
|
||||
Akka.Actor.Props.CreateBy(new TestProducer(() => new OpcUaPublishActor(
|
||||
sink, NullServiceLevelPublisher.Instance, false, localNode, null, applier, null, null, null, null,
|
||||
false, dataPlaneAuthority, null, deferredWriteCapacity, hooks)));
|
||||
false, dataPlaneAuthority, null, deferredWriteCapacity, hooks, endpointState: null)));
|
||||
|
||||
/// <summary>Produces the actor through its internal (seam-carrying) constructor, for <see cref="PropsForTests(IOpcUaAddressSpaceSink, AddressSpaceApplier?, TestHooks?, int, NodeId?, IDataPlaneAuthoritySink?)"/>.</summary>
|
||||
private sealed class TestProducer(Func<OpcUaPublishActor> create) : IIndirectActorProducer
|
||||
@@ -434,6 +446,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
/// ServiceLevel DB-reachability input is retired (<c>DbReachable</c> treated constant-true, only
|
||||
/// staleness demotes). No <paramref name="dbHealthProbe"/> is wired on such a node.</param>
|
||||
/// <param name="dataPlaneAuthority">The optional sink that receives the API keys and ACL grants carried by each applied artifact.</param>
|
||||
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762); null is treated as listening.</param>
|
||||
public OpcUaPublishActor(
|
||||
IOpcUaAddressSpaceSink sink,
|
||||
IServiceLevelPublisher serviceLevel,
|
||||
@@ -447,15 +460,18 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
TimeSpan? healthTickInterval = null,
|
||||
bool dbLess = false,
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority = null,
|
||||
Func<DateTime>? utcNow = null)
|
||||
Func<DateTime>? utcNow = null,
|
||||
IOpcUaEndpointState? endpointState = null)
|
||||
: this(sink, serviceLevel, subscribeRedundancyTopic, localNode, dbFactory, applier, dbHealthProbe, staleWindow,
|
||||
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null)
|
||||
probeFreshnessWindow, healthTickInterval, dbLess, dataPlaneAuthority, utcNow, DefaultDeferredWriteCapacity, hooks: null,
|
||||
endpointState)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>The full constructor: also the deferred-write bound and the test seams (internal — tests only).</summary>
|
||||
/// <param name="deferredWriteCapacity">The bound on distinct nodes whose writes are held for replay while a rebuild plan is in flight.</param>
|
||||
/// <param name="hooks">Test seams for the off-thread plan step; null in production.</param>
|
||||
/// <param name="endpointState">This node's OPC UA endpoint lifecycle fact (lmxopcua#762); null is treated as listening.</param>
|
||||
internal OpcUaPublishActor(
|
||||
IOpcUaAddressSpaceSink sink,
|
||||
IServiceLevelPublisher serviceLevel,
|
||||
@@ -471,7 +487,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
IDataPlaneAuthoritySink? dataPlaneAuthority,
|
||||
Func<DateTime>? utcNow,
|
||||
int deferredWriteCapacity,
|
||||
TestHooks? hooks)
|
||||
TestHooks? hooks,
|
||||
IOpcUaEndpointState? endpointState)
|
||||
{
|
||||
_deferred = new DeferredWrites(Math.Max(1, deferredWriteCapacity));
|
||||
_hooks = hooks;
|
||||
@@ -486,6 +503,7 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
_applier = applier;
|
||||
_dbHealthProbe = dbHealthProbe;
|
||||
_dbLess = dbLess;
|
||||
_endpointState = endpointState;
|
||||
_staleWindow = staleWindow ?? TimeSpan.FromSeconds(30);
|
||||
_probeFreshnessWindow = probeFreshnessWindow ?? TimeSpan.FromSeconds(30);
|
||||
_healthTickInterval = healthTickInterval ?? TimeSpan.FromSeconds(5);
|
||||
@@ -1536,6 +1554,15 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
return verdict.Ok;
|
||||
}
|
||||
|
||||
/// <summary>The OPC UA endpoint input for the calculator (lmxopcua#762): <c>false</c> only when this node's own
|
||||
/// SDK start is recorded as <see cref="OpcUaEndpointStatus.Failed"/>. <see cref="OpcUaEndpointStatus.NotStarted"/>
|
||||
/// and <see cref="OpcUaEndpointStatus.Stopped"/> are boot and shutdown windows, not faults, and a node with no
|
||||
/// endpoint fact at all (a harness) is treated as listening. Read on every recompute — the periodic health tick
|
||||
/// and every redundancy snapshot (heartbeated every 10 s) — so a change is picked up without a notification.
|
||||
/// It is a LOCAL fact on purpose: the peer TCP probe behind <see cref="OpcUaProbeOk"/> cannot see a failed start
|
||||
/// when another process holds the port, because its connect then succeeds against that process.</summary>
|
||||
private bool EndpointListening() => _endpointState?.Status != OpcUaEndpointStatus.Failed;
|
||||
|
||||
/// <summary>
|
||||
/// Computes the local OPC UA ServiceLevel and routes it through <see cref="ServiceLevelChanged"/>
|
||||
/// (the dedup/publish/metric handler). The full <see cref="ServiceLevelCalculator"/> path is
|
||||
@@ -1575,7 +1602,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
DbReachable: true,
|
||||
OpcUaProbeOk: OpcUaProbeOk(),
|
||||
Stale: (utcNow - entry.AsOfUtc) > _staleWindow,
|
||||
IsDriverPrimary: entry.IsDriverPrimary);
|
||||
IsDriverPrimary: entry.IsDriverPrimary,
|
||||
EndpointListening: EndpointListening());
|
||||
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(dbLessInputs)));
|
||||
return;
|
||||
}
|
||||
@@ -1585,7 +1613,9 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
// is the bootstrap value until the first DbHealthStatus lands.
|
||||
if (_dbHealthProbe is null || _lastDbHealth is null)
|
||||
{
|
||||
Self.Tell(new ServiceLevelChanged(LegacyRoleOnly(entry)));
|
||||
// The role-only seam has no health inputs, but a failed endpoint still wins (lmxopcua#762): the
|
||||
// legacy switch would otherwise hand a node that serves nothing 240 as Primary.
|
||||
Self.Tell(new ServiceLevelChanged(EndpointListening() ? LegacyRoleOnly(entry) : (byte)0));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1597,7 +1627,8 @@ public sealed class OpcUaPublishActor : ReceiveActor, IWithTimers
|
||||
Stale: !_lastDbHealth.Reachable
|
||||
|| (now - _lastDbHealth.AsOfUtc) > _staleWindow
|
||||
|| (now - entry.AsOfUtc) > _staleWindow,
|
||||
IsDriverPrimary: entry.IsDriverPrimary);
|
||||
IsDriverPrimary: entry.IsDriverPrimary,
|
||||
EndpointListening: EndpointListening());
|
||||
|
||||
Self.Tell(new ServiceLevelChanged(ServiceLevelCalculator.Compute(inputs)));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user